diff options
Diffstat (limited to 'drivers/net/ethernet')
334 files changed, 21545 insertions, 8140 deletions
diff --git a/drivers/net/ethernet/Kconfig b/drivers/net/ethernet/Kconfig index aa7103e7f47f..bdc29d143160 100644 --- a/drivers/net/ethernet/Kconfig +++ b/drivers/net/ethernet/Kconfig @@ -23,7 +23,6 @@ source "drivers/net/ethernet/agere/Kconfig" source "drivers/net/ethernet/airoha/Kconfig" source "drivers/net/ethernet/alacritech/Kconfig" source "drivers/net/ethernet/allwinner/Kconfig" -source "drivers/net/ethernet/alteon/Kconfig" source "drivers/net/ethernet/altera/Kconfig" source "drivers/net/ethernet/amazon/Kconfig" source "drivers/net/ethernet/amd/Kconfig" diff --git a/drivers/net/ethernet/Makefile b/drivers/net/ethernet/Makefile index 6615a67a63d5..6bffb60ba644 100644 --- a/drivers/net/ethernet/Makefile +++ b/drivers/net/ethernet/Makefile @@ -13,7 +13,6 @@ obj-$(CONFIG_NET_VENDOR_AGERE) += agere/ obj-$(CONFIG_NET_VENDOR_AIROHA) += airoha/ obj-$(CONFIG_NET_VENDOR_ALACRITECH) += alacritech/ obj-$(CONFIG_NET_VENDOR_ALLWINNER) += allwinner/ -obj-$(CONFIG_NET_VENDOR_ALTEON) += alteon/ obj-$(CONFIG_ALTERA_TSE) += altera/ obj-$(CONFIG_NET_VENDOR_AMAZON) += amazon/ obj-$(CONFIG_NET_VENDOR_AMD) += amd/ diff --git a/drivers/net/ethernet/airoha/airoha_eth.c b/drivers/net/ethernet/airoha/airoha_eth.c index 91cb63a32d99..e1ab15f1ee7d 100644 --- a/drivers/net/ethernet/airoha/airoha_eth.c +++ b/drivers/net/ethernet/airoha/airoha_eth.c @@ -76,7 +76,7 @@ static void airoha_set_macaddr(struct airoha_gdm_port *port, const u8 *addr) struct airoha_eth *eth = port->qdma->eth; u32 val, reg; - reg = airhoa_is_lan_gdm_port(port) ? REG_FE_LAN_MAC_H + reg = airoha_is_lan_gdm_port(port) ? REG_FE_LAN_MAC_H : REG_FE_WAN_MAC_H; val = (addr[0] << 16) | (addr[1] << 8) | addr[2]; airoha_fe_wr(eth, reg, val); @@ -107,19 +107,7 @@ static int airoha_set_vip_for_gdm_port(struct airoha_gdm_port *port, struct airoha_eth *eth = port->qdma->eth; u32 vip_port; - switch (port->id) { - case AIROHA_GDM3_IDX: - /* FIXME: handle XSI_PCIE1_PORT */ - vip_port = XSI_PCIE0_VIP_PORT_MASK; - break; - case AIROHA_GDM4_IDX: - /* FIXME: handle XSI_USB_PORT */ - vip_port = XSI_ETH_VIP_PORT_MASK; - break; - default: - return 0; - } - + vip_port = eth->soc->ops.get_vip_port(port, port->nbq); if (enable) { airoha_fe_set(eth, REG_FE_VIP_PORT_EN, vip_port); airoha_fe_set(eth, REG_FE_IFC_PORT_EN, vip_port); @@ -293,16 +281,18 @@ static void airoha_fe_pse_ports_init(struct airoha_eth *eth) [FE_PSE_PORT_GDM4] = 2, [FE_PSE_PORT_CDM5] = 2, }; - u32 all_rsv; int q; - all_rsv = airoha_fe_get_pse_all_rsv(eth); if (airoha_ppe_is_enabled(eth, 1)) { + u32 all_rsv; + /* hw misses PPE2 oq rsv */ + all_rsv = airoha_fe_get_pse_all_rsv(eth); all_rsv += PSE_RSV_PAGES * pse_port_num_queues[FE_PSE_PORT_PPE2]; + airoha_fe_rmw(eth, REG_FE_PSE_BUF_SET, PSE_ALLRSV_MASK, + FIELD_PREP(PSE_ALLRSV_MASK, all_rsv)); } - airoha_fe_set(eth, REG_FE_PSE_BUF_SET, all_rsv); /* CMD1 */ for (q = 0; q < pse_port_num_queues[FE_PSE_PORT_CDM1]; q++) @@ -458,9 +448,8 @@ static int airoha_fe_init(struct airoha_eth *eth) FIELD_PREP(PSE_IQ_RES2_P5_MASK, 0x40) | FIELD_PREP(PSE_IQ_RES2_P4_MASK, 0x34)); - /* enable FE copy engine for MC/KA/DPI */ - airoha_fe_wr(eth, REG_FE_PCE_CFG, - PCE_DPI_EN_MASK | PCE_KA_EN_MASK | PCE_MC_EN_MASK); + /* enable FE copy engine for KA/DPI */ + airoha_fe_wr(eth, REG_FE_PCE_CFG, PCE_DPI_EN_MASK | PCE_KA_EN_MASK); /* set vip queue selection to ring 1 */ airoha_fe_rmw(eth, REG_CDM_FWD_CFG(1), CDM_VIP_QSEL_MASK, FIELD_PREP(CDM_VIP_QSEL_MASK, 0x4)); @@ -572,11 +561,12 @@ static int airoha_qdma_fill_rx_queue(struct airoha_queue *q) WRITE_ONCE(desc->msg1, 0); WRITE_ONCE(desc->msg2, 0); WRITE_ONCE(desc->msg3, 0); + } + if (nframes) airoha_qdma_rmw(qdma, REG_RX_CPU_IDX(qid), RX_RING_CPU_IDX_MASK, FIELD_PREP(RX_RING_CPU_IDX_MASK, q->head)); - } return nframes; } @@ -584,7 +574,7 @@ static int airoha_qdma_fill_rx_queue(struct airoha_queue *q) static int airoha_qdma_get_gdm_port(struct airoha_eth *eth, struct airoha_qdma_desc *desc) { - u32 port, sport, msg1 = le32_to_cpu(desc->msg1); + u32 port, sport, msg1 = le32_to_cpu(READ_ONCE(desc->msg1)); sport = FIELD_GET(QDMA_ETH_RXMSG_SPORT_MASK, msg1); switch (sport) { @@ -612,21 +602,24 @@ static int airoha_qdma_rx_process(struct airoha_queue *q, int budget) while (done < budget) { struct airoha_queue_entry *e = &q->entry[q->tail]; struct airoha_qdma_desc *desc = &q->desc[q->tail]; - u32 hash, reason, msg1 = le32_to_cpu(desc->msg1); - struct page *page = virt_to_head_page(e->buf); - u32 desc_ctrl = le32_to_cpu(desc->ctrl); + u32 hash, reason, msg1, desc_ctrl; struct airoha_gdm_port *port; int data_len, len, p; + struct page *page; + desc_ctrl = le32_to_cpu(READ_ONCE(desc->ctrl)); if (!(desc_ctrl & QDMA_DESC_DONE_MASK)) break; + dma_rmb(); + q->tail = (q->tail + 1) % q->ndesc; q->queued--; dma_sync_single_for_cpu(eth->dev, e->dma_addr, SKB_WITH_OVERHEAD(q->buf_size), dir); + page = virt_to_head_page(e->buf); len = FIELD_GET(QDMA_DESC_LEN_MASK, desc_ctrl); data_len = q->skb ? q->buf_size : SKB_WITH_OVERHEAD(q->buf_size); @@ -670,8 +663,8 @@ static int airoha_qdma_rx_process(struct airoha_queue *q, int budget) * DMA descriptor. Report DSA tag to the DSA stack * via skb dst info. */ - u32 sptag = FIELD_GET(QDMA_ETH_RXMSG_SPTAG, - le32_to_cpu(desc->msg0)); + u32 msg0 = le32_to_cpu(READ_ONCE(desc->msg0)); + u32 sptag = FIELD_GET(QDMA_ETH_RXMSG_SPTAG, msg0); if (sptag < ARRAY_SIZE(port->dsa_meta) && port->dsa_meta[sptag]) @@ -679,6 +672,7 @@ static int airoha_qdma_rx_process(struct airoha_queue *q, int budget) &port->dsa_meta[sptag]->dst); } + msg1 = le32_to_cpu(READ_ONCE(desc->msg1)); hash = FIELD_GET(AIROHA_RXD4_FOE_ENTRY, msg1); if (hash != AIROHA_RXD4_FOE_ENTRY) skb_set_hash(q->skb, jhash_1word(hash, 0), @@ -819,6 +813,11 @@ static void airoha_qdma_cleanup_rx_queue(struct airoha_queue *q) } q->head = q->tail; + /* Set RX_DMA_IDX to RX_CPU_IDX to notify the hw the QDMA RX ring is + * empty. + */ + airoha_qdma_rmw(qdma, REG_RX_CPU_IDX(qid), RX_RING_CPU_IDX_MASK, + FIELD_PREP(RX_RING_CPU_IDX_MASK, q->head)); airoha_qdma_rmw(qdma, REG_RX_DMA_IDX(qid), RX_RING_DMA_IDX_MASK, FIELD_PREP(RX_RING_DMA_IDX_MASK, q->tail)); } @@ -1383,6 +1382,33 @@ static int airoha_qdma_init(struct platform_device *pdev, return airoha_qdma_hw_init(qdma); } +static void airoha_qdma_cleanup(struct airoha_qdma *qdma) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(qdma->q_rx); i++) { + if (!qdma->q_rx[i].ndesc) + continue; + + netif_napi_del(&qdma->q_rx[i].napi); + airoha_qdma_cleanup_rx_queue(&qdma->q_rx[i]); + if (qdma->q_rx[i].page_pool) { + page_pool_destroy(qdma->q_rx[i].page_pool); + qdma->q_rx[i].page_pool = NULL; + } + } + + for (i = 0; i < ARRAY_SIZE(qdma->q_tx_irq); i++) + netif_napi_del(&qdma->q_tx_irq[i].napi); + + for (i = 0; i < ARRAY_SIZE(qdma->q_tx); i++) { + if (!qdma->q_tx[i].ndesc) + continue; + + airoha_qdma_cleanup_tx_queue(&qdma->q_tx[i]); + } +} + static int airoha_hw_init(struct platform_device *pdev, struct airoha_eth *eth) { @@ -1410,41 +1436,30 @@ static int airoha_hw_init(struct platform_device *pdev, for (i = 0; i < ARRAY_SIZE(eth->qdma); i++) { err = airoha_qdma_init(pdev, eth, ð->qdma[i]); if (err) - return err; + goto error; } err = airoha_ppe_init(eth); if (err) - return err; + goto error; set_bit(DEV_STATE_INITIALIZED, ð->state); return 0; +error: + for (i = 0; i < ARRAY_SIZE(eth->qdma); i++) + airoha_qdma_cleanup(ð->qdma[i]); + + return err; } -static void airoha_hw_cleanup(struct airoha_qdma *qdma) +static void airoha_hw_cleanup(struct airoha_eth *eth) { int i; - for (i = 0; i < ARRAY_SIZE(qdma->q_rx); i++) { - if (!qdma->q_rx[i].ndesc) - continue; - - netif_napi_del(&qdma->q_rx[i].napi); - airoha_qdma_cleanup_rx_queue(&qdma->q_rx[i]); - if (qdma->q_rx[i].page_pool) - page_pool_destroy(qdma->q_rx[i].page_pool); - } - - for (i = 0; i < ARRAY_SIZE(qdma->q_tx_irq); i++) - netif_napi_del(&qdma->q_tx_irq[i].napi); - - for (i = 0; i < ARRAY_SIZE(qdma->q_tx); i++) { - if (!qdma->q_tx[i].ndesc) - continue; - - airoha_qdma_cleanup_tx_queue(&qdma->q_tx[i]); - } + for (i = 0; i < ARRAY_SIZE(eth->qdma); i++) + airoha_qdma_cleanup(ð->qdma[i]); + airoha_ppe_deinit(eth); } static void airoha_qdma_start_napi(struct airoha_qdma *qdma) @@ -1626,6 +1641,7 @@ static int airoha_dev_open(struct net_device *dev) int err, len = ETH_HLEN + dev->mtu + ETH_FCS_LEN; struct airoha_gdm_port *port = netdev_priv(dev); struct airoha_qdma *qdma = port->qdma; + u32 pse_port = FE_PSE_PORT_PPE1; netif_tx_start_all_queues(dev); err = airoha_set_vip_for_gdm_port(port, true); @@ -1649,6 +1665,14 @@ static int airoha_dev_open(struct net_device *dev) GLOBAL_CFG_RX_DMA_EN_MASK); atomic_inc(&qdma->users); + if (port->id == AIROHA_GDM2_IDX && + airoha_ppe_is_enabled(qdma->eth, 1)) { + /* For PPE2 always use secondary cpu port. */ + pse_port = FE_PSE_PORT_PPE2; + } + airoha_set_gdm_port_fwd_cfg(qdma->eth, REG_GDM_FWD_CFG(port->id), + pse_port); + return 0; } @@ -1666,6 +1690,9 @@ static int airoha_dev_stop(struct net_device *dev) for (i = 0; i < ARRAY_SIZE(qdma->q_tx); i++) netdev_tx_reset_subqueue(dev, i); + airoha_set_gdm_port_fwd_cfg(qdma->eth, REG_GDM_FWD_CFG(port->id), + FE_PSE_PORT_DROP); + if (atomic_dec_and_test(&qdma->users)) { airoha_qdma_clear(qdma, REG_QDMA_GLOBAL_CFG, GLOBAL_CFG_TX_DMA_EN_MASK | @@ -1696,10 +1723,10 @@ static int airoha_dev_set_macaddr(struct net_device *dev, void *p) return 0; } -static int airhoha_set_gdm2_loopback(struct airoha_gdm_port *port) +static int airoha_set_gdm2_loopback(struct airoha_gdm_port *port) { struct airoha_eth *eth = port->qdma->eth; - u32 val, pse_port, chan, nbq; + u32 val, pse_port, chan; int src_port; /* Forward the traffic to the proper GDM port */ @@ -1729,9 +1756,7 @@ static int airhoha_set_gdm2_loopback(struct airoha_gdm_port *port) airoha_fe_clear(eth, REG_FE_VIP_PORT_EN, BIT(AIROHA_GDM2_IDX)); airoha_fe_clear(eth, REG_FE_IFC_PORT_EN, BIT(AIROHA_GDM2_IDX)); - /* XXX: handle XSI_USB_PORT and XSI_PCE1_PORT */ - nbq = port->id == AIROHA_GDM3_IDX && airoha_is_7581(eth) ? 4 : 0; - src_port = eth->soc->ops.get_src_port_id(port, nbq); + src_port = eth->soc->ops.get_src_port_id(port, port->nbq); if (src_port < 0) return src_port; @@ -1742,12 +1767,14 @@ static int airhoha_set_gdm2_loopback(struct airoha_gdm_port *port) airoha_fe_rmw(eth, REG_SP_DFT_CPORT(src_port >> fls(SP_CPORT_DFT_MASK)), SP_CPORT_MASK(val), - FE_PSE_PORT_CDM2 << __ffs(SP_CPORT_MASK(val))); + __field_prep(SP_CPORT_MASK(val), FE_PSE_PORT_CDM2)); - if (port->id != AIROHA_GDM3_IDX && airoha_is_7581(eth)) - airoha_fe_rmw(eth, REG_SRC_PORT_FC_MAP6, - FC_ID_OF_SRC_PORT24_MASK, - FIELD_PREP(FC_ID_OF_SRC_PORT24_MASK, 2)); + if (port->id == AIROHA_GDM4_IDX && airoha_is_7581(eth)) { + u32 mask = FC_ID_OF_SRC_PORT_MASK(port->nbq); + + airoha_fe_rmw(eth, REG_SRC_PORT_FC_MAP6, mask, + __field_prep(mask, AIROHA_GDM2_IDX)); + } return 0; } @@ -1755,11 +1782,12 @@ static int airhoha_set_gdm2_loopback(struct airoha_gdm_port *port) static int airoha_dev_init(struct net_device *dev) { struct airoha_gdm_port *port = netdev_priv(dev); - struct airoha_qdma *qdma = port->qdma; - struct airoha_eth *eth = qdma->eth; - u32 pse_port, fe_cpu_port; - u8 ppe_id; + struct airoha_eth *eth = port->eth; + int i; + /* QDMA0 is used for lan ports while QDMA1 is used for WAN ports */ + port->qdma = ð->qdma[!airoha_is_lan_gdm_port(port)]; + port->dev->irq = port->qdma->irq_banks[0].irq; airoha_set_macaddr(port, dev->dev_addr); switch (port->id) { @@ -1769,34 +1797,17 @@ static int airoha_dev_init(struct net_device *dev) if (!eth->ports[1]) { int err; - err = airhoha_set_gdm2_loopback(port); + err = airoha_set_gdm2_loopback(port); if (err) return err; } - fallthrough; - case AIROHA_GDM2_IDX: - if (airoha_ppe_is_enabled(eth, 1)) { - /* For PPE2 always use secondary cpu port. */ - fe_cpu_port = FE_PSE_PORT_CDM2; - pse_port = FE_PSE_PORT_PPE2; - break; - } - fallthrough; - default: { - u8 qdma_id = qdma - ð->qdma[0]; - - /* For PPE1 select cpu port according to the running QDMA. */ - fe_cpu_port = qdma_id ? FE_PSE_PORT_CDM2 : FE_PSE_PORT_CDM1; - pse_port = FE_PSE_PORT_PPE1; break; - } + default: + break; } - airoha_set_gdm_port_fwd_cfg(eth, REG_GDM_FWD_CFG(port->id), pse_port); - ppe_id = pse_port == FE_PSE_PORT_PPE2 ? 1 : 0; - airoha_fe_rmw(eth, REG_PPE_DFT_CPORT0(ppe_id), - DFT_CPORT_MASK(port->id), - fe_cpu_port << __ffs(DFT_CPORT_MASK(port->id))); + for (i = 0; i < eth->soc->num_ppe; i++) + airoha_ppe_set_cpu_port(port, i); return 0; } @@ -1899,7 +1910,7 @@ static u32 airoha_get_dsa_tag(struct sk_buff *skb, struct net_device *dev) #endif } -static int airoha_get_fe_port(struct airoha_gdm_port *port) +int airoha_get_fe_port(struct airoha_gdm_port *port) { struct airoha_qdma *qdma = port->qdma; struct airoha_eth *eth = qdma->eth; @@ -2123,10 +2134,11 @@ airoha_ethtool_get_rmon_stats(struct net_device *dev, } while (u64_stats_fetch_retry(&port->stats.syncp, start)); } -static int airoha_qdma_set_chan_tx_sched(struct airoha_gdm_port *port, +static int airoha_qdma_set_chan_tx_sched(struct net_device *dev, int channel, enum tx_sched_mode mode, const u16 *weights, u8 n_weights) { + struct airoha_gdm_port *port = netdev_priv(dev); int i; for (i = 0; i < AIROHA_NUM_TX_RING; i++) @@ -2153,22 +2165,20 @@ static int airoha_qdma_set_chan_tx_sched(struct airoha_gdm_port *port, airoha_qdma_rmw(port->qdma, REG_CHAN_QOS_MODE(channel >> 3), CHAN_QOS_MODE_MASK(channel), - mode << __ffs(CHAN_QOS_MODE_MASK(channel))); + __field_prep(CHAN_QOS_MODE_MASK(channel), mode)); return 0; } -static int airoha_qdma_set_tx_prio_sched(struct airoha_gdm_port *port, - int channel) +static int airoha_qdma_set_tx_prio_sched(struct net_device *dev, int channel) { static const u16 w[AIROHA_NUM_QOS_QUEUES] = {}; - return airoha_qdma_set_chan_tx_sched(port, channel, TC_SCH_SP, w, + return airoha_qdma_set_chan_tx_sched(dev, channel, TC_SCH_SP, w, ARRAY_SIZE(w)); } -static int airoha_qdma_set_tx_ets_sched(struct airoha_gdm_port *port, - int channel, +static int airoha_qdma_set_tx_ets_sched(struct net_device *dev, int channel, struct tc_ets_qopt_offload *opt) { struct tc_ets_qopt_offload_replace_params *p = &opt->replace_params; @@ -2209,20 +2219,21 @@ static int airoha_qdma_set_tx_ets_sched(struct airoha_gdm_port *port, else if (nstrict < AIROHA_NUM_QOS_QUEUES - 1) mode = nstrict + 1; - return airoha_qdma_set_chan_tx_sched(port, channel, mode, w, + return airoha_qdma_set_chan_tx_sched(dev, channel, mode, w, ARRAY_SIZE(w)); } -static int airoha_qdma_get_tx_ets_stats(struct airoha_gdm_port *port, - int channel, +static int airoha_qdma_get_tx_ets_stats(struct net_device *dev, int channel, struct tc_ets_qopt_offload *opt) { + struct airoha_gdm_port *port = netdev_priv(dev); u64 cpu_tx_packets = airoha_qdma_rr(port->qdma, REG_CNTR_VAL(channel << 1)); u64 fwd_tx_packets = airoha_qdma_rr(port->qdma, REG_CNTR_VAL((channel << 1) + 1)); u64 tx_packets = (cpu_tx_packets - port->cpu_tx_packets) + (fwd_tx_packets - port->fwd_tx_packets); + _bstats_update(opt->stats.bstats, 0, tx_packets); port->cpu_tx_packets = cpu_tx_packets; @@ -2231,7 +2242,7 @@ static int airoha_qdma_get_tx_ets_stats(struct airoha_gdm_port *port, return 0; } -static int airoha_tc_setup_qdisc_ets(struct airoha_gdm_port *port, +static int airoha_tc_setup_qdisc_ets(struct net_device *dev, struct tc_ets_qopt_offload *opt) { int channel; @@ -2244,12 +2255,12 @@ static int airoha_tc_setup_qdisc_ets(struct airoha_gdm_port *port, switch (opt->command) { case TC_ETS_REPLACE: - return airoha_qdma_set_tx_ets_sched(port, channel, opt); + return airoha_qdma_set_tx_ets_sched(dev, channel, opt); case TC_ETS_DESTROY: /* PRIO is default qdisc scheduler */ - return airoha_qdma_set_tx_prio_sched(port, channel); + return airoha_qdma_set_tx_prio_sched(dev, channel); case TC_ETS_STATS: - return airoha_qdma_get_tx_ets_stats(port, channel, opt); + return airoha_qdma_get_tx_ets_stats(dev, channel, opt); default: return -EOPNOTSUPP; } @@ -2477,10 +2488,11 @@ static int airoha_qdma_set_trtcm_token_bucket(struct airoha_qdma *qdma, mode, val); } -static int airoha_qdma_set_tx_rate_limit(struct airoha_gdm_port *port, +static int airoha_qdma_set_tx_rate_limit(struct net_device *dev, int channel, u32 rate, u32 bucket_size) { + struct airoha_gdm_port *port = netdev_priv(dev); int i, err; for (i = 0; i <= TRTCM_PEAK_MODE; i++) { @@ -2500,21 +2512,20 @@ static int airoha_qdma_set_tx_rate_limit(struct airoha_gdm_port *port, return 0; } -static int airoha_tc_htb_alloc_leaf_queue(struct airoha_gdm_port *port, +static int airoha_tc_htb_alloc_leaf_queue(struct net_device *dev, struct tc_htb_qopt_offload *opt) { u32 channel = TC_H_MIN(opt->classid) % AIROHA_NUM_QOS_CHANNELS; u32 rate = div_u64(opt->rate, 1000) << 3; /* kbps */ - struct net_device *dev = port->dev; - int num_tx_queues = dev->real_num_tx_queues; - int err; + int err, num_tx_queues = dev->real_num_tx_queues; + struct airoha_gdm_port *port = netdev_priv(dev); if (opt->parent_classid != TC_HTB_CLASSID_ROOT) { NL_SET_ERR_MSG_MOD(opt->extack, "invalid parent classid"); return -EINVAL; } - err = airoha_qdma_set_tx_rate_limit(port, channel, rate, opt->quantum); + err = airoha_qdma_set_tx_rate_limit(dev, channel, rate, opt->quantum); if (err) { NL_SET_ERR_MSG_MOD(opt->extack, "failed configuring htb offload"); @@ -2526,7 +2537,7 @@ static int airoha_tc_htb_alloc_leaf_queue(struct airoha_gdm_port *port, err = netif_set_real_num_tx_queues(dev, num_tx_queues + 1); if (err) { - airoha_qdma_set_tx_rate_limit(port, channel, 0, opt->quantum); + airoha_qdma_set_tx_rate_limit(dev, channel, 0, opt->quantum); NL_SET_ERR_MSG_MOD(opt->extack, "failed setting real_num_tx_queues"); return err; @@ -2672,7 +2683,7 @@ static int airoha_dev_setup_tc_block_cb(enum tc_setup_type type, } } -static int airoha_dev_setup_tc_block(struct airoha_gdm_port *port, +static int airoha_dev_setup_tc_block(struct net_device *dev, struct flow_block_offload *f) { flow_setup_cb_t *cb = airoha_dev_setup_tc_block_cb; @@ -2685,12 +2696,12 @@ static int airoha_dev_setup_tc_block(struct airoha_gdm_port *port, f->driver_block_list = &block_cb_list; switch (f->command) { case FLOW_BLOCK_BIND: - block_cb = flow_block_cb_lookup(f->block, cb, port->dev); + block_cb = flow_block_cb_lookup(f->block, cb, dev); if (block_cb) { flow_block_cb_incref(block_cb); return 0; } - block_cb = flow_block_cb_alloc(cb, port->dev, port->dev, NULL); + block_cb = flow_block_cb_alloc(cb, dev, dev, NULL); if (IS_ERR(block_cb)) return PTR_ERR(block_cb); @@ -2699,7 +2710,7 @@ static int airoha_dev_setup_tc_block(struct airoha_gdm_port *port, list_add_tail(&block_cb->driver_list, &block_cb_list); return 0; case FLOW_BLOCK_UNBIND: - block_cb = flow_block_cb_lookup(f->block, cb, port->dev); + block_cb = flow_block_cb_lookup(f->block, cb, dev); if (!block_cb) return -ENOENT; @@ -2713,44 +2724,47 @@ static int airoha_dev_setup_tc_block(struct airoha_gdm_port *port, } } -static void airoha_tc_remove_htb_queue(struct airoha_gdm_port *port, int queue) +static void airoha_tc_remove_htb_queue(struct net_device *dev, int queue) { - struct net_device *dev = port->dev; + struct airoha_gdm_port *port = netdev_priv(dev); netif_set_real_num_tx_queues(dev, dev->real_num_tx_queues - 1); - airoha_qdma_set_tx_rate_limit(port, queue + 1, 0, 0); + airoha_qdma_set_tx_rate_limit(dev, queue + 1, 0, 0); clear_bit(queue, port->qos_sq_bmap); } -static int airoha_tc_htb_delete_leaf_queue(struct airoha_gdm_port *port, +static int airoha_tc_htb_delete_leaf_queue(struct net_device *dev, struct tc_htb_qopt_offload *opt) { u32 channel = TC_H_MIN(opt->classid) % AIROHA_NUM_QOS_CHANNELS; + struct airoha_gdm_port *port = netdev_priv(dev); if (!test_bit(channel, port->qos_sq_bmap)) { NL_SET_ERR_MSG_MOD(opt->extack, "invalid queue id"); return -EINVAL; } - airoha_tc_remove_htb_queue(port, channel); + airoha_tc_remove_htb_queue(dev, channel); return 0; } -static int airoha_tc_htb_destroy(struct airoha_gdm_port *port) +static int airoha_tc_htb_destroy(struct net_device *dev) { + struct airoha_gdm_port *port = netdev_priv(dev); int q; for_each_set_bit(q, port->qos_sq_bmap, AIROHA_NUM_QOS_CHANNELS) - airoha_tc_remove_htb_queue(port, q); + airoha_tc_remove_htb_queue(dev, q); return 0; } -static int airoha_tc_get_htb_get_leaf_queue(struct airoha_gdm_port *port, +static int airoha_tc_get_htb_get_leaf_queue(struct net_device *dev, struct tc_htb_qopt_offload *opt) { u32 channel = TC_H_MIN(opt->classid) % AIROHA_NUM_QOS_CHANNELS; + struct airoha_gdm_port *port = netdev_priv(dev); if (!test_bit(channel, port->qos_sq_bmap)) { NL_SET_ERR_MSG_MOD(opt->extack, "invalid queue id"); @@ -2762,23 +2776,23 @@ static int airoha_tc_get_htb_get_leaf_queue(struct airoha_gdm_port *port, return 0; } -static int airoha_tc_setup_qdisc_htb(struct airoha_gdm_port *port, +static int airoha_tc_setup_qdisc_htb(struct net_device *dev, struct tc_htb_qopt_offload *opt) { switch (opt->command) { case TC_HTB_CREATE: break; case TC_HTB_DESTROY: - return airoha_tc_htb_destroy(port); + return airoha_tc_htb_destroy(dev); case TC_HTB_NODE_MODIFY: case TC_HTB_LEAF_ALLOC_QUEUE: - return airoha_tc_htb_alloc_leaf_queue(port, opt); + return airoha_tc_htb_alloc_leaf_queue(dev, opt); case TC_HTB_LEAF_DEL: case TC_HTB_LEAF_DEL_LAST: case TC_HTB_LEAF_DEL_LAST_FORCE: - return airoha_tc_htb_delete_leaf_queue(port, opt); + return airoha_tc_htb_delete_leaf_queue(dev, opt); case TC_HTB_LEAF_QUERY_QUEUE: - return airoha_tc_get_htb_get_leaf_queue(port, opt); + return airoha_tc_get_htb_get_leaf_queue(dev, opt); default: return -EOPNOTSUPP; } @@ -2789,16 +2803,14 @@ static int airoha_tc_setup_qdisc_htb(struct airoha_gdm_port *port, static int airoha_dev_tc_setup(struct net_device *dev, enum tc_setup_type type, void *type_data) { - struct airoha_gdm_port *port = netdev_priv(dev); - switch (type) { case TC_SETUP_QDISC_ETS: - return airoha_tc_setup_qdisc_ets(port, type_data); + return airoha_tc_setup_qdisc_ets(dev, type_data); case TC_SETUP_QDISC_HTB: - return airoha_tc_setup_qdisc_htb(port, type_data); + return airoha_tc_setup_qdisc_htb(dev, type_data); case TC_SETUP_BLOCK: case TC_SETUP_FT: - return airoha_dev_setup_tc_block(port, type_data); + return airoha_dev_setup_tc_block(dev, type_data); default: return -EOPNOTSUPP; } @@ -2869,11 +2881,10 @@ bool airoha_is_valid_gdm_port(struct airoha_eth *eth, } static int airoha_alloc_gdm_port(struct airoha_eth *eth, - struct device_node *np, int index) + struct device_node *np) { const __be32 *id_ptr = of_get_property(np, "reg", NULL); struct airoha_gdm_port *port; - struct airoha_qdma *qdma; struct net_device *dev; int err, p; u32 id; @@ -2904,7 +2915,6 @@ static int airoha_alloc_gdm_port(struct airoha_eth *eth, return -ENOMEM; } - qdma = ð->qdma[index % AIROHA_MAX_NUM_QDMA]; dev->netdev_ops = &airoha_netdev_ops; dev->ethtool_ops = &airoha_ethtool_ops; dev->max_mtu = AIROHA_MAX_MTU; @@ -2916,7 +2926,6 @@ static int airoha_alloc_gdm_port(struct airoha_eth *eth, dev->features |= dev->hw_features; dev->vlan_features = dev->hw_features; dev->dev.of_node = np; - dev->irq = qdma->irq_banks[0].irq; SET_NETDEV_DEV(dev, eth->dev); /* reserve hw queues for HTB offloading */ @@ -2937,9 +2946,11 @@ static int airoha_alloc_gdm_port(struct airoha_eth *eth, port = netdev_priv(dev); u64_stats_init(&port->stats.syncp); spin_lock_init(&port->stats.lock); - port->qdma = qdma; + port->eth = eth; port->dev = dev; port->id = id; + /* XXX: Read nbq from DTS */ + port->nbq = id == AIROHA_GDM3_IDX && airoha_is_7581(eth) ? 4 : 0; eth->ports[p] = port; return airoha_metadata_dst_alloc(port); @@ -3034,12 +3045,11 @@ static int airoha_probe(struct platform_device *pdev) err = airoha_hw_init(pdev, eth); if (err) - goto error_hw_cleanup; + goto error_netdev_free; for (i = 0; i < ARRAY_SIZE(eth->qdma); i++) airoha_qdma_start_napi(ð->qdma[i]); - i = 0; for_each_child_of_node(pdev->dev.of_node, np) { if (!of_device_is_compatible(np, "airoha,eth-mac")) continue; @@ -3047,7 +3057,7 @@ static int airoha_probe(struct platform_device *pdev) if (!of_device_is_available(np)) continue; - err = airoha_alloc_gdm_port(eth, np, i++); + err = airoha_alloc_gdm_port(eth, np); if (err) { of_node_put(np); goto error_napi_stop; @@ -3063,10 +3073,6 @@ static int airoha_probe(struct platform_device *pdev) error_napi_stop: for (i = 0; i < ARRAY_SIZE(eth->qdma); i++) airoha_qdma_stop_napi(ð->qdma[i]); - airoha_ppe_deinit(eth); -error_hw_cleanup: - for (i = 0; i < ARRAY_SIZE(eth->qdma); i++) - airoha_hw_cleanup(ð->qdma[i]); for (i = 0; i < ARRAY_SIZE(eth->ports); i++) { struct airoha_gdm_port *port = eth->ports[i]; @@ -3078,6 +3084,8 @@ error_hw_cleanup: unregister_netdev(port->dev); airoha_metadata_dst_free(port); } + airoha_hw_cleanup(eth); +error_netdev_free: free_netdev(eth->napi_dev); platform_set_drvdata(pdev, NULL); @@ -3089,10 +3097,8 @@ static void airoha_remove(struct platform_device *pdev) struct airoha_eth *eth = platform_get_drvdata(pdev); int i; - for (i = 0; i < ARRAY_SIZE(eth->qdma); i++) { + for (i = 0; i < ARRAY_SIZE(eth->qdma); i++) airoha_qdma_stop_napi(ð->qdma[i]); - airoha_hw_cleanup(ð->qdma[i]); - } for (i = 0; i < ARRAY_SIZE(eth->ports); i++) { struct airoha_gdm_port *port = eth->ports[i]; @@ -3103,9 +3109,9 @@ static void airoha_remove(struct platform_device *pdev) unregister_netdev(port->dev); airoha_metadata_dst_free(port); } - free_netdev(eth->napi_dev); + airoha_hw_cleanup(eth); - airoha_ppe_deinit(eth); + free_netdev(eth->napi_dev); platform_set_drvdata(pdev, NULL); } @@ -3141,6 +3147,28 @@ static int airoha_en7581_get_src_port_id(struct airoha_gdm_port *port, int nbq) return -EINVAL; } +static u32 airoha_en7581_get_vip_port(struct airoha_gdm_port *port, int nbq) +{ + switch (port->id) { + case AIROHA_GDM3_IDX: + if (nbq == 4) + return XSI_PCIE0_VIP_PORT_MASK; + if (nbq == 5) + return XSI_PCIE1_VIP_PORT_MASK; + break; + case AIROHA_GDM4_IDX: + if (!nbq) + return XSI_ETH_VIP_PORT_MASK; + if (nbq == 1) + return XSI_USB_VIP_PORT_MASK; + break; + default: + break; + } + + return 0; +} + static const char * const an7583_xsi_rsts_names[] = { "xsi-mac", "hsi0-mac", @@ -3170,6 +3198,26 @@ static int airoha_an7583_get_src_port_id(struct airoha_gdm_port *port, int nbq) return -EINVAL; } +static u32 airoha_an7583_get_vip_port(struct airoha_gdm_port *port, int nbq) +{ + switch (port->id) { + case AIROHA_GDM3_IDX: + if (!nbq) + return XSI_ETH_VIP_PORT_MASK; + break; + case AIROHA_GDM4_IDX: + if (!nbq) + return XSI_PCIE0_VIP_PORT_MASK; + if (nbq == 1) + return XSI_USB_VIP_PORT_MASK; + break; + default: + break; + } + + return 0; +} + static const struct airoha_eth_soc_data en7581_soc_data = { .version = 0x7581, .xsi_rsts_names = en7581_xsi_rsts_names, @@ -3177,6 +3225,7 @@ static const struct airoha_eth_soc_data en7581_soc_data = { .num_ppe = 2, .ops = { .get_src_port_id = airoha_en7581_get_src_port_id, + .get_vip_port = airoha_en7581_get_vip_port, }, }; @@ -3187,6 +3236,7 @@ static const struct airoha_eth_soc_data an7583_soc_data = { .num_ppe = 1, .ops = { .get_src_port_id = airoha_an7583_get_src_port_id, + .get_vip_port = airoha_an7583_get_vip_port, }, }; diff --git a/drivers/net/ethernet/airoha/airoha_eth.h b/drivers/net/ethernet/airoha/airoha_eth.h index a97903569335..95e557638617 100644 --- a/drivers/net/ethernet/airoha/airoha_eth.h +++ b/drivers/net/ethernet/airoha/airoha_eth.h @@ -534,8 +534,10 @@ struct airoha_qdma { struct airoha_gdm_port { struct airoha_qdma *qdma; + struct airoha_eth *eth; struct net_device *dev; int id; + int nbq; struct airoha_hw_stats stats; @@ -576,6 +578,7 @@ struct airoha_eth_soc_data { int num_ppe; struct { int (*get_src_port_id)(struct airoha_gdm_port *port, int nbq); + u32 (*get_vip_port)(struct airoha_gdm_port *port, int nbq); } ops; }; @@ -627,7 +630,7 @@ u32 airoha_rmw(void __iomem *base, u32 offset, u32 mask, u32 val); #define airoha_qdma_clear(qdma, offset, val) \ airoha_rmw((qdma)->regs, (offset), (val), 0) -static inline bool airhoa_is_lan_gdm_port(struct airoha_gdm_port *port) +static inline bool airoha_is_lan_gdm_port(struct airoha_gdm_port *port) { /* GDM1 port on EN7581 SoC is connected to the lan dsa switch. * GDM{2,3,4} can be used as wan port connected to an external @@ -646,9 +649,11 @@ static inline bool airoha_is_7583(struct airoha_eth *eth) return eth->soc->version == 0x7583; } +int airoha_get_fe_port(struct airoha_gdm_port *port); bool airoha_is_valid_gdm_port(struct airoha_eth *eth, struct airoha_gdm_port *port); +void airoha_ppe_set_cpu_port(struct airoha_gdm_port *port, u8 ppe_id); bool airoha_ppe_is_enabled(struct airoha_eth *eth, int index); void airoha_ppe_check_skb(struct airoha_ppe_dev *dev, struct sk_buff *skb, u16 hash, bool rx_wlan); diff --git a/drivers/net/ethernet/airoha/airoha_ppe.c b/drivers/net/ethernet/airoha/airoha_ppe.c index c2c32b6833df..03115c1c1063 100644 --- a/drivers/net/ethernet/airoha/airoha_ppe.c +++ b/drivers/net/ethernet/airoha/airoha_ppe.c @@ -85,6 +85,20 @@ static u32 airoha_ppe_get_timestamp(struct airoha_ppe *ppe) return FIELD_GET(AIROHA_FOE_IB1_BIND_TIMESTAMP, timestamp); } +void airoha_ppe_set_cpu_port(struct airoha_gdm_port *port, u8 ppe_id) +{ + struct airoha_qdma *qdma = port->qdma; + u8 fport = airoha_get_fe_port(port); + struct airoha_eth *eth = qdma->eth; + u8 qdma_id = qdma - ð->qdma[0]; + u32 fe_cpu_port; + + fe_cpu_port = qdma_id ? FE_PSE_PORT_CDM2 : FE_PSE_PORT_CDM1; + airoha_fe_rmw(eth, REG_PPE_DFT_CPORT(ppe_id, fport), + DFT_CPORT_MASK(fport), + __field_prep(DFT_CPORT_MASK(fport), fe_cpu_port)); +} + static void airoha_ppe_hw_init(struct airoha_ppe *ppe) { u32 sram_ppe_num_data_entries = PPE_SRAM_NUM_ENTRIES, sram_num_entries; @@ -111,13 +125,13 @@ static void airoha_ppe_hw_init(struct airoha_ppe *ppe) airoha_fe_rmw(eth, REG_PPE_BND_AGE0(i), PPE_BIND_AGE0_DELTA_NON_L4 | PPE_BIND_AGE0_DELTA_UDP, - FIELD_PREP(PPE_BIND_AGE0_DELTA_NON_L4, 1) | - FIELD_PREP(PPE_BIND_AGE0_DELTA_UDP, 12)); + FIELD_PREP(PPE_BIND_AGE0_DELTA_NON_L4, 60) | + FIELD_PREP(PPE_BIND_AGE0_DELTA_UDP, 60)); airoha_fe_rmw(eth, REG_PPE_BND_AGE1(i), PPE_BIND_AGE1_DELTA_TCP_FIN | PPE_BIND_AGE1_DELTA_TCP, FIELD_PREP(PPE_BIND_AGE1_DELTA_TCP_FIN, 1) | - FIELD_PREP(PPE_BIND_AGE1_DELTA_TCP, 7)); + FIELD_PREP(PPE_BIND_AGE1_DELTA_TCP, 60)); airoha_fe_rmw(eth, REG_PPE_TB_HASH_CFG(i), PPE_SRAM_TABLE_EN_MASK | @@ -145,9 +159,19 @@ static void airoha_ppe_hw_init(struct airoha_ppe *ppe) FIELD_PREP(PPE_DRAM_TB_NUM_ENTRY_MASK, dram_num_entries)); + airoha_fe_rmw(eth, REG_PPE_BIND_RATE(i), + PPE_BIND_RATE_L2B_BIND_MASK | + PPE_BIND_RATE_BIND_MASK, + FIELD_PREP(PPE_BIND_RATE_L2B_BIND_MASK, 0x1e) | + FIELD_PREP(PPE_BIND_RATE_BIND_MASK, 0x1e)); + airoha_fe_wr(eth, REG_PPE_HASH_SEED(i), PPE_HASH_SEED); + airoha_fe_clear(eth, REG_PPE_PPE_FLOW_CFG(i), + PPE_FLOW_CFG_IP6_6RD_MASK); + + for (p = 0; p < ARRAY_SIZE(eth->ports); p++) { + struct airoha_gdm_port *port = eth->ports[p]; - for (p = 0; p < ARRAY_SIZE(eth->ports); p++) airoha_fe_rmw(eth, REG_PPE_MTU(i, p), FP0_EGRESS_MTU_MASK | FP1_EGRESS_MTU_MASK, @@ -155,6 +179,11 @@ static void airoha_ppe_hw_init(struct airoha_ppe *ppe) AIROHA_MAX_MTU) | FIELD_PREP(FP1_EGRESS_MTU_MASK, AIROHA_MAX_MTU)); + if (!port) + continue; + + airoha_ppe_set_cpu_port(port, i); + } } } @@ -323,7 +352,7 @@ static int airoha_ppe_foe_entry_prepare(struct airoha_eth *eth, /* For downlink traffic consume SRAM memory for hw * forwarding descriptors queue. */ - if (airhoa_is_lan_gdm_port(port)) + if (airoha_is_lan_gdm_port(port)) val |= AIROHA_FOE_IB2_FAST_PATH; if (dsa_port >= 0) val |= FIELD_PREP(AIROHA_FOE_IB2_NBQ, diff --git a/drivers/net/ethernet/airoha/airoha_regs.h b/drivers/net/ethernet/airoha/airoha_regs.h index ed4e3407f4a0..436f3c8779c1 100644 --- a/drivers/net/ethernet/airoha/airoha_regs.h +++ b/drivers/net/ethernet/airoha/airoha_regs.h @@ -312,10 +312,9 @@ #define REG_PPE_HASH_SEED(_n) (((_n) ? PPE2_BASE : PPE1_BASE) + 0x244) #define PPE_HASH_SEED 0x12345678 -#define REG_PPE_DFT_CPORT0(_n) (((_n) ? PPE2_BASE : PPE1_BASE) + 0x248) -#define DFT_CPORT_MASK(_n) GENMASK(3 + ((_n) << 2), ((_n) << 2)) - -#define REG_PPE_DFT_CPORT1(_n) (((_n) ? PPE2_BASE : PPE1_BASE) + 0x24c) +#define REG_PPE_DFT_CPORT_BASE(_n) (((_n) ? PPE2_BASE : PPE1_BASE) + 0x248) +#define REG_PPE_DFT_CPORT(_m, _n) (REG_PPE_DFT_CPORT_BASE(_m) + (((_n) / 8) << 2)) +#define DFT_CPORT_MASK(_n) GENMASK(3 + (((_n) % 8) << 2), (((_n) % 8) << 2)) #define REG_PPE_TB_HASH_CFG(_n) (((_n) ? PPE2_BASE : PPE1_BASE) + 0x250) #define PPE_DRAM_HASH1_MODE_MASK GENMASK(31, 28) @@ -376,10 +375,7 @@ #define SP_CPORT_MASK(_n) GENMASK(3 + ((_n) << 2), ((_n) << 2)) #define REG_SRC_PORT_FC_MAP6 0x2298 -#define FC_ID_OF_SRC_PORT27_MASK GENMASK(28, 24) -#define FC_ID_OF_SRC_PORT26_MASK GENMASK(20, 16) -#define FC_ID_OF_SRC_PORT25_MASK GENMASK(12, 8) -#define FC_ID_OF_SRC_PORT24_MASK GENMASK(4, 0) +#define FC_ID_OF_SRC_PORT_MASK(_n) GENMASK(4 + ((_n) << 3), ((_n) << 3)) #define REG_CDM5_RX_OQ1_DROP_CNT 0x29d4 diff --git a/drivers/net/ethernet/alteon/Kconfig b/drivers/net/ethernet/alteon/Kconfig deleted file mode 100644 index cfe1f3159d61..000000000000 --- a/drivers/net/ethernet/alteon/Kconfig +++ /dev/null @@ -1,47 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -# -# Alteon network device configuration -# - -config NET_VENDOR_ALTEON - bool "Alteon devices" - default y - depends on PCI - help - If you have a network (Ethernet) card belonging to this class, say Y. - - Note that the answer to this question doesn't directly affect the - kernel: saying N will just cause the configurator to skip all - the questions about Alteon cards. If you say Y, you will be asked for - your specific card in the following questions. - -if NET_VENDOR_ALTEON - -config ACENIC - tristate "Alteon AceNIC/3Com 3C985/NetGear GA620 Gigabit support" - depends on PCI - help - Say Y here if you have an Alteon AceNIC, 3Com 3C985(B), NetGear - GA620, SGI Gigabit or Farallon PN9000-SX PCI Gigabit Ethernet - adapter. The driver allows for using the Jumbo Frame option (9000 - bytes/frame) however it requires that your switches can handle this - as well. To enable Jumbo Frames, add `mtu 9000' to your ifconfig - line. - - To compile this driver as a module, choose M here: the - module will be called acenic. - -config ACENIC_OMIT_TIGON_I - bool "Omit support for old Tigon I based AceNICs" - depends on ACENIC - help - Say Y here if you only have Tigon II based AceNICs and want to leave - out support for the older Tigon I based cards which are no longer - being sold (ie. the original Alteon AceNIC and 3Com 3C985 (non B - version)). This will reduce the size of the driver object by - app. 100KB. If you are not sure whether your card is a Tigon I or a - Tigon II, say N here. - - The safe and default value for this is N. - -endif # NET_VENDOR_ALTEON diff --git a/drivers/net/ethernet/alteon/Makefile b/drivers/net/ethernet/alteon/Makefile deleted file mode 100644 index be5225559b6d..000000000000 --- a/drivers/net/ethernet/alteon/Makefile +++ /dev/null @@ -1,6 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -# -# Makefile for the Alteon network device drivers. -# - -obj-$(CONFIG_ACENIC) += acenic.o diff --git a/drivers/net/ethernet/alteon/acenic.c b/drivers/net/ethernet/alteon/acenic.c deleted file mode 100644 index 455ee8930824..000000000000 --- a/drivers/net/ethernet/alteon/acenic.c +++ /dev/null @@ -1,3178 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * acenic.c: Linux driver for the Alteon AceNIC Gigabit Ethernet card - * and other Tigon based cards. - * - * Copyright 1998-2002 by Jes Sorensen, <jes@trained-monkey.org>. - * - * Thanks to Alteon and 3Com for providing hardware and documentation - * enabling me to write this driver. - * - * A mailing list for discussing the use of this driver has been - * setup, please subscribe to the lists if you have any questions - * about the driver. Send mail to linux-acenic-help@sunsite.auc.dk to - * see how to subscribe. - * - * Additional credits: - * Pete Wyckoff <wyckoff@ca.sandia.gov>: Initial Linux/Alpha and trace - * dump support. The trace dump support has not been - * integrated yet however. - * Troy Benjegerdes: Big Endian (PPC) patches. - * Nate Stahl: Better out of memory handling and stats support. - * Aman Singla: Nasty race between interrupt handler and tx code dealing - * with 'testing the tx_ret_csm and setting tx_full' - * David S. Miller <davem@redhat.com>: conversion to new PCI dma mapping - * infrastructure and Sparc support - * Pierrick Pinasseau (CERN): For lending me an Ultra 5 to test the - * driver under Linux/Sparc64 - * Matt Domsch <Matt_Domsch@dell.com>: Detect Alteon 1000baseT cards - * ETHTOOL_GDRVINFO support - * Chip Salzenberg <chip@valinux.com>: Fix race condition between tx - * handler and close() cleanup. - * Ken Aaker <kdaaker@rchland.vnet.ibm.com>: Correct check for whether - * memory mapped IO is enabled to - * make the driver work on RS/6000. - * Takayoshi Kouchi <kouchi@hpc.bs1.fc.nec.co.jp>: Identifying problem - * where the driver would disable - * bus master mode if it had to disable - * write and invalidate. - * Stephen Hack <stephen_hack@hp.com>: Fixed ace_set_mac_addr for little - * endian systems. - * Val Henson <vhenson@esscom.com>: Reset Jumbo skb producer and - * rx producer index when - * flushing the Jumbo ring. - * Hans Grobler <grobh@sun.ac.za>: Memory leak fixes in the - * driver init path. - * Grant Grundler <grundler@cup.hp.com>: PCI write posting fixes. - */ - -#include <linux/module.h> -#include <linux/moduleparam.h> -#include <linux/types.h> -#include <linux/errno.h> -#include <linux/ioport.h> -#include <linux/pci.h> -#include <linux/dma-mapping.h> -#include <linux/kernel.h> -#include <linux/netdevice.h> -#include <linux/etherdevice.h> -#include <linux/skbuff.h> -#include <linux/delay.h> -#include <linux/mm.h> -#include <linux/highmem.h> -#include <linux/sockios.h> -#include <linux/firmware.h> -#include <linux/slab.h> -#include <linux/prefetch.h> -#include <linux/if_vlan.h> - -#ifdef SIOCETHTOOL -#include <linux/ethtool.h> -#endif - -#include <net/sock.h> -#include <net/ip.h> - -#include <asm/io.h> -#include <asm/irq.h> -#include <asm/byteorder.h> -#include <linux/uaccess.h> - - -#define DRV_NAME "acenic" - -#undef INDEX_DEBUG - -#ifdef CONFIG_ACENIC_OMIT_TIGON_I -#define ACE_IS_TIGON_I(ap) 0 -#define ACE_TX_RING_ENTRIES(ap) MAX_TX_RING_ENTRIES -#else -#define ACE_IS_TIGON_I(ap) (ap->version == 1) -#define ACE_TX_RING_ENTRIES(ap) ap->tx_ring_entries -#endif - -#ifndef PCI_VENDOR_ID_ALTEON -#define PCI_VENDOR_ID_ALTEON 0x12ae -#endif -#ifndef PCI_DEVICE_ID_ALTEON_ACENIC_FIBRE -#define PCI_DEVICE_ID_ALTEON_ACENIC_FIBRE 0x0001 -#define PCI_DEVICE_ID_ALTEON_ACENIC_COPPER 0x0002 -#endif -#ifndef PCI_DEVICE_ID_3COM_3C985 -#define PCI_DEVICE_ID_3COM_3C985 0x0001 -#endif -#ifndef PCI_VENDOR_ID_NETGEAR -#define PCI_VENDOR_ID_NETGEAR 0x1385 -#define PCI_DEVICE_ID_NETGEAR_GA620 0x620a -#endif -#ifndef PCI_DEVICE_ID_NETGEAR_GA620T -#define PCI_DEVICE_ID_NETGEAR_GA620T 0x630a -#endif - - -/* - * Farallon used the DEC vendor ID by mistake and they seem not - * to care - stinky! - */ -#ifndef PCI_DEVICE_ID_FARALLON_PN9000SX -#define PCI_DEVICE_ID_FARALLON_PN9000SX 0x1a -#endif -#ifndef PCI_DEVICE_ID_FARALLON_PN9100T -#define PCI_DEVICE_ID_FARALLON_PN9100T 0xfa -#endif -#ifndef PCI_VENDOR_ID_SGI -#define PCI_VENDOR_ID_SGI 0x10a9 -#endif -#ifndef PCI_DEVICE_ID_SGI_ACENIC -#define PCI_DEVICE_ID_SGI_ACENIC 0x0009 -#endif - -static const struct pci_device_id acenic_pci_tbl[] = { - { PCI_VENDOR_ID_ALTEON, PCI_DEVICE_ID_ALTEON_ACENIC_FIBRE, - PCI_ANY_ID, PCI_ANY_ID, PCI_CLASS_NETWORK_ETHERNET << 8, 0xffff00, }, - { PCI_VENDOR_ID_ALTEON, PCI_DEVICE_ID_ALTEON_ACENIC_COPPER, - PCI_ANY_ID, PCI_ANY_ID, PCI_CLASS_NETWORK_ETHERNET << 8, 0xffff00, }, - { PCI_VENDOR_ID_3COM, PCI_DEVICE_ID_3COM_3C985, - PCI_ANY_ID, PCI_ANY_ID, PCI_CLASS_NETWORK_ETHERNET << 8, 0xffff00, }, - { PCI_VENDOR_ID_NETGEAR, PCI_DEVICE_ID_NETGEAR_GA620, - PCI_ANY_ID, PCI_ANY_ID, PCI_CLASS_NETWORK_ETHERNET << 8, 0xffff00, }, - { PCI_VENDOR_ID_NETGEAR, PCI_DEVICE_ID_NETGEAR_GA620T, - PCI_ANY_ID, PCI_ANY_ID, PCI_CLASS_NETWORK_ETHERNET << 8, 0xffff00, }, - /* - * Farallon used the DEC vendor ID on their cards incorrectly, - * then later Alteon's ID. - */ - { PCI_VENDOR_ID_DEC, PCI_DEVICE_ID_FARALLON_PN9000SX, - PCI_ANY_ID, PCI_ANY_ID, PCI_CLASS_NETWORK_ETHERNET << 8, 0xffff00, }, - { PCI_VENDOR_ID_ALTEON, PCI_DEVICE_ID_FARALLON_PN9100T, - PCI_ANY_ID, PCI_ANY_ID, PCI_CLASS_NETWORK_ETHERNET << 8, 0xffff00, }, - { PCI_VENDOR_ID_SGI, PCI_DEVICE_ID_SGI_ACENIC, - PCI_ANY_ID, PCI_ANY_ID, PCI_CLASS_NETWORK_ETHERNET << 8, 0xffff00, }, - { } -}; -MODULE_DEVICE_TABLE(pci, acenic_pci_tbl); - -#define ace_sync_irq(irq) synchronize_irq(irq) - -#ifndef offset_in_page -#define offset_in_page(ptr) ((unsigned long)(ptr) & ~PAGE_MASK) -#endif - -#define ACE_MAX_MOD_PARMS 8 -#define BOARD_IDX_STATIC 0 -#define BOARD_IDX_OVERFLOW -1 - -#include "acenic.h" - -/* - * These must be defined before the firmware is included. - */ -#define MAX_TEXT_LEN 96*1024 -#define MAX_RODATA_LEN 8*1024 -#define MAX_DATA_LEN 2*1024 - -#ifndef tigon2FwReleaseLocal -#define tigon2FwReleaseLocal 0 -#endif - -/* - * This driver currently supports Tigon I and Tigon II based cards - * including the Alteon AceNIC, the 3Com 3C985[B] and NetGear - * GA620. The driver should also work on the SGI, DEC and Farallon - * versions of the card, however I have not been able to test that - * myself. - * - * This card is really neat, it supports receive hardware checksumming - * and jumbo frames (up to 9000 bytes) and does a lot of work in the - * firmware. Also the programming interface is quite neat, except for - * the parts dealing with the i2c eeprom on the card ;-) - * - * Using jumbo frames: - * - * To enable jumbo frames, simply specify an mtu between 1500 and 9000 - * bytes to ifconfig. Jumbo frames can be enabled or disabled at any time - * by running `ifconfig eth<X> mtu <MTU>' with <X> being the Ethernet - * interface number and <MTU> being the MTU value. - * - * Module parameters: - * - * When compiled as a loadable module, the driver allows for a number - * of module parameters to be specified. The driver supports the - * following module parameters: - * - * trace=<val> - Firmware trace level. This requires special traced - * firmware to replace the firmware supplied with - * the driver - for debugging purposes only. - * - * link=<val> - Link state. Normally you want to use the default link - * parameters set by the driver. This can be used to - * override these in case your switch doesn't negotiate - * the link properly. Valid values are: - * 0x0001 - Force half duplex link. - * 0x0002 - Do not negotiate line speed with the other end. - * 0x0010 - 10Mbit/sec link. - * 0x0020 - 100Mbit/sec link. - * 0x0040 - 1000Mbit/sec link. - * 0x0100 - Do not negotiate flow control. - * 0x0200 - Enable RX flow control Y - * 0x0400 - Enable TX flow control Y (Tigon II NICs only). - * Default value is 0x0270, ie. enable link+flow - * control negotiation. Negotiating the highest - * possible link speed with RX flow control enabled. - * - * When disabling link speed negotiation, only one link - * speed is allowed to be specified! - * - * tx_coal_tick=<val> - number of coalescing clock ticks (us) allowed - * to wait for more packets to arive before - * interrupting the host, from the time the first - * packet arrives. - * - * rx_coal_tick=<val> - number of coalescing clock ticks (us) allowed - * to wait for more packets to arive in the transmit ring, - * before interrupting the host, after transmitting the - * first packet in the ring. - * - * max_tx_desc=<val> - maximum number of transmit descriptors - * (packets) transmitted before interrupting the host. - * - * max_rx_desc=<val> - maximum number of receive descriptors - * (packets) received before interrupting the host. - * - * tx_ratio=<val> - 7 bit value (0 - 63) specifying the split in 64th - * increments of the NIC's on board memory to be used for - * transmit and receive buffers. For the 1MB NIC app. 800KB - * is available, on the 1/2MB NIC app. 300KB is available. - * 68KB will always be available as a minimum for both - * directions. The default value is a 50/50 split. - * dis_pci_mem_inval=<val> - disable PCI memory write and invalidate - * operations, default (1) is to always disable this as - * that is what Alteon does on NT. I have not been able - * to measure any real performance differences with - * this on my systems. Set <val>=0 if you want to - * enable these operations. - * - * If you use more than one NIC, specify the parameters for the - * individual NICs with a comma, ie. trace=0,0x00001fff,0 you want to - * run tracing on NIC #2 but not on NIC #1 and #3. - * - * TODO: - * - * - Proper multicast support. - * - NIC dump support. - * - More tuning parameters. - * - * The mini ring is not used under Linux and I am not sure it makes sense - * to actually use it. - * - * New interrupt handler strategy: - * - * The old interrupt handler worked using the traditional method of - * replacing an skbuff with a new one when a packet arrives. However - * the rx rings do not need to contain a static number of buffer - * descriptors, thus it makes sense to move the memory allocation out - * of the main interrupt handler and do it in a bottom half handler - * and only allocate new buffers when the number of buffers in the - * ring is below a certain threshold. In order to avoid starving the - * NIC under heavy load it is however necessary to force allocation - * when hitting a minimum threshold. The strategy for alloction is as - * follows: - * - * RX_LOW_BUF_THRES - allocate buffers in the bottom half - * RX_PANIC_LOW_THRES - we are very low on buffers, allocate - * the buffers in the interrupt handler - * RX_RING_THRES - maximum number of buffers in the rx ring - * RX_MINI_THRES - maximum number of buffers in the mini ring - * RX_JUMBO_THRES - maximum number of buffers in the jumbo ring - * - * One advantagous side effect of this allocation approach is that the - * entire rx processing can be done without holding any spin lock - * since the rx rings and registers are totally independent of the tx - * ring and its registers. This of course includes the kmalloc's of - * new skb's. Thus start_xmit can run in parallel with rx processing - * and the memory allocation on SMP systems. - * - * Note that running the skb reallocation in a bottom half opens up - * another can of races which needs to be handled properly. In - * particular it can happen that the interrupt handler tries to run - * the reallocation while the bottom half is either running on another - * CPU or was interrupted on the same CPU. To get around this the - * driver uses bitops to prevent the reallocation routines from being - * reentered. - * - * TX handling can also be done without holding any spin lock, wheee - * this is fun! since tx_ret_csm is only written to by the interrupt - * handler. The case to be aware of is when shutting down the device - * and cleaning up where it is necessary to make sure that - * start_xmit() is not running while this is happening. Well DaveM - * informs me that this case is already protected against ... bye bye - * Mr. Spin Lock, it was nice to know you. - * - * TX interrupts are now partly disabled so the NIC will only generate - * TX interrupts for the number of coal ticks, not for the number of - * TX packets in the queue. This should reduce the number of TX only, - * ie. when no RX processing is done, interrupts seen. - */ - -/* - * Threshold values for RX buffer allocation - the low water marks for - * when to start refilling the rings are set to 75% of the ring - * sizes. It seems to make sense to refill the rings entirely from the - * intrrupt handler once it gets below the panic threshold, that way - * we don't risk that the refilling is moved to another CPU when the - * one running the interrupt handler just got the slab code hot in its - * cache. - */ -#define RX_RING_SIZE 72 -#define RX_MINI_SIZE 64 -#define RX_JUMBO_SIZE 48 - -#define RX_PANIC_STD_THRES 16 -#define RX_PANIC_STD_REFILL (3*RX_PANIC_STD_THRES)/2 -#define RX_LOW_STD_THRES (3*RX_RING_SIZE)/4 -#define RX_PANIC_MINI_THRES 12 -#define RX_PANIC_MINI_REFILL (3*RX_PANIC_MINI_THRES)/2 -#define RX_LOW_MINI_THRES (3*RX_MINI_SIZE)/4 -#define RX_PANIC_JUMBO_THRES 6 -#define RX_PANIC_JUMBO_REFILL (3*RX_PANIC_JUMBO_THRES)/2 -#define RX_LOW_JUMBO_THRES (3*RX_JUMBO_SIZE)/4 - - -/* - * Size of the mini ring entries, basically these just should be big - * enough to take TCP ACKs - */ -#define ACE_MINI_SIZE 100 - -#define ACE_MINI_BUFSIZE ACE_MINI_SIZE -#define ACE_STD_BUFSIZE (ACE_STD_MTU + ETH_HLEN + 4) -#define ACE_JUMBO_BUFSIZE (ACE_JUMBO_MTU + ETH_HLEN + 4) - -/* - * There seems to be a magic difference in the effect between 995 and 996 - * but little difference between 900 and 995 ... no idea why. - * - * There is now a default set of tuning parameters which is set, depending - * on whether or not the user enables Jumbo frames. It's assumed that if - * Jumbo frames are enabled, the user wants optimal tuning for that case. - */ -#define DEF_TX_COAL 400 /* 996 */ -#define DEF_TX_MAX_DESC 60 /* was 40 */ -#define DEF_RX_COAL 120 /* 1000 */ -#define DEF_RX_MAX_DESC 25 -#define DEF_TX_RATIO 21 /* 24 */ - -#define DEF_JUMBO_TX_COAL 20 -#define DEF_JUMBO_TX_MAX_DESC 60 -#define DEF_JUMBO_RX_COAL 30 -#define DEF_JUMBO_RX_MAX_DESC 6 -#define DEF_JUMBO_TX_RATIO 21 - -#if tigon2FwReleaseLocal < 20001118 -/* - * Standard firmware and early modifications duplicate - * IRQ load without this flag (coal timer is never reset). - * Note that with this flag tx_coal should be less than - * time to xmit full tx ring. - * 400usec is not so bad for tx ring size of 128. - */ -#define TX_COAL_INTS_ONLY 1 /* worth it */ -#else -/* - * With modified firmware, this is not necessary, but still useful. - */ -#define TX_COAL_INTS_ONLY 1 -#endif - -#define DEF_TRACE 0 -#define DEF_STAT (2 * TICKS_PER_SEC) - - -static int link_state[ACE_MAX_MOD_PARMS]; -static int trace[ACE_MAX_MOD_PARMS]; -static int tx_coal_tick[ACE_MAX_MOD_PARMS]; -static int rx_coal_tick[ACE_MAX_MOD_PARMS]; -static int max_tx_desc[ACE_MAX_MOD_PARMS]; -static int max_rx_desc[ACE_MAX_MOD_PARMS]; -static int tx_ratio[ACE_MAX_MOD_PARMS]; -static int dis_pci_mem_inval[ACE_MAX_MOD_PARMS] = {1, 1, 1, 1, 1, 1, 1, 1}; - -MODULE_AUTHOR("Jes Sorensen <jes@trained-monkey.org>"); -MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("AceNIC/3C985/GA620 Gigabit Ethernet driver"); -#ifndef CONFIG_ACENIC_OMIT_TIGON_I -MODULE_FIRMWARE("acenic/tg1.bin"); -#endif -MODULE_FIRMWARE("acenic/tg2.bin"); - -module_param_array_named(link, link_state, int, NULL, 0); -module_param_array(trace, int, NULL, 0); -module_param_array(tx_coal_tick, int, NULL, 0); -module_param_array(max_tx_desc, int, NULL, 0); -module_param_array(rx_coal_tick, int, NULL, 0); -module_param_array(max_rx_desc, int, NULL, 0); -module_param_array(tx_ratio, int, NULL, 0); -MODULE_PARM_DESC(link, "AceNIC/3C985/NetGear link state"); -MODULE_PARM_DESC(trace, "AceNIC/3C985/NetGear firmware trace level"); -MODULE_PARM_DESC(tx_coal_tick, "AceNIC/3C985/GA620 max clock ticks to wait from first tx descriptor arrives"); -MODULE_PARM_DESC(max_tx_desc, "AceNIC/3C985/GA620 max number of transmit descriptors to wait"); -MODULE_PARM_DESC(rx_coal_tick, "AceNIC/3C985/GA620 max clock ticks to wait from first rx descriptor arrives"); -MODULE_PARM_DESC(max_rx_desc, "AceNIC/3C985/GA620 max number of receive descriptors to wait"); -MODULE_PARM_DESC(tx_ratio, "AceNIC/3C985/GA620 ratio of NIC memory used for TX/RX descriptors (range 0-63)"); - - -static const char version[] = - "acenic.c: v0.92 08/05/2002 Jes Sorensen, linux-acenic@SunSITE.dk\n" - " http://home.cern.ch/~jes/gige/acenic.html\n"; - -static int ace_get_link_ksettings(struct net_device *, - struct ethtool_link_ksettings *); -static int ace_set_link_ksettings(struct net_device *, - const struct ethtool_link_ksettings *); -static void ace_get_drvinfo(struct net_device *, struct ethtool_drvinfo *); - -static const struct ethtool_ops ace_ethtool_ops = { - .get_drvinfo = ace_get_drvinfo, - .get_link_ksettings = ace_get_link_ksettings, - .set_link_ksettings = ace_set_link_ksettings, -}; - -static void ace_watchdog(struct net_device *dev, unsigned int txqueue); - -static const struct net_device_ops ace_netdev_ops = { - .ndo_open = ace_open, - .ndo_stop = ace_close, - .ndo_tx_timeout = ace_watchdog, - .ndo_get_stats = ace_get_stats, - .ndo_start_xmit = ace_start_xmit, - .ndo_set_rx_mode = ace_set_multicast_list, - .ndo_validate_addr = eth_validate_addr, - .ndo_set_mac_address = ace_set_mac_addr, - .ndo_change_mtu = ace_change_mtu, -}; - -static int acenic_probe_one(struct pci_dev *pdev, - const struct pci_device_id *id) -{ - struct net_device *dev; - struct ace_private *ap; - static int boards_found; - - dev = alloc_etherdev(sizeof(struct ace_private)); - if (dev == NULL) - return -ENOMEM; - - SET_NETDEV_DEV(dev, &pdev->dev); - - ap = netdev_priv(dev); - ap->ndev = dev; - ap->pdev = pdev; - ap->name = pci_name(pdev); - - dev->features |= NETIF_F_SG | NETIF_F_IP_CSUM; - dev->features |= NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX; - - dev->watchdog_timeo = 5*HZ; - dev->min_mtu = 0; - dev->max_mtu = ACE_JUMBO_MTU; - - dev->netdev_ops = &ace_netdev_ops; - dev->ethtool_ops = &ace_ethtool_ops; - - /* we only display this string ONCE */ - if (!boards_found) - printk(version); - - if (pci_enable_device(pdev)) - goto fail_free_netdev; - - /* - * Enable master mode before we start playing with the - * pci_command word since pci_set_master() will modify - * it. - */ - pci_set_master(pdev); - - pci_read_config_word(pdev, PCI_COMMAND, &ap->pci_command); - - /* OpenFirmware on Mac's does not set this - DOH.. */ - if (!(ap->pci_command & PCI_COMMAND_MEMORY)) { - printk(KERN_INFO "%s: Enabling PCI Memory Mapped " - "access - was not enabled by BIOS/Firmware\n", - ap->name); - ap->pci_command = ap->pci_command | PCI_COMMAND_MEMORY; - pci_write_config_word(ap->pdev, PCI_COMMAND, - ap->pci_command); - wmb(); - } - - pci_read_config_byte(pdev, PCI_LATENCY_TIMER, &ap->pci_latency); - if (ap->pci_latency <= 0x40) { - ap->pci_latency = 0x40; - pci_write_config_byte(pdev, PCI_LATENCY_TIMER, ap->pci_latency); - } - - /* - * Remap the regs into kernel space - this is abuse of - * dev->base_addr since it was means for I/O port - * addresses but who gives a damn. - */ - dev->base_addr = pci_resource_start(pdev, 0); - ap->regs = ioremap(dev->base_addr, 0x4000); - if (!ap->regs) { - printk(KERN_ERR "%s: Unable to map I/O register, " - "AceNIC %i will be disabled.\n", - ap->name, boards_found); - goto fail_free_netdev; - } - - switch(pdev->vendor) { - case PCI_VENDOR_ID_ALTEON: - if (pdev->device == PCI_DEVICE_ID_FARALLON_PN9100T) { - printk(KERN_INFO "%s: Farallon PN9100-T ", - ap->name); - } else { - printk(KERN_INFO "%s: Alteon AceNIC ", - ap->name); - } - break; - case PCI_VENDOR_ID_3COM: - printk(KERN_INFO "%s: 3Com 3C985 ", ap->name); - break; - case PCI_VENDOR_ID_NETGEAR: - printk(KERN_INFO "%s: NetGear GA620 ", ap->name); - break; - case PCI_VENDOR_ID_DEC: - if (pdev->device == PCI_DEVICE_ID_FARALLON_PN9000SX) { - printk(KERN_INFO "%s: Farallon PN9000-SX ", - ap->name); - break; - } - fallthrough; - case PCI_VENDOR_ID_SGI: - printk(KERN_INFO "%s: SGI AceNIC ", ap->name); - break; - default: - printk(KERN_INFO "%s: Unknown AceNIC ", ap->name); - break; - } - - printk("Gigabit Ethernet at 0x%08lx, ", dev->base_addr); - printk("irq %d\n", pdev->irq); - -#ifdef CONFIG_ACENIC_OMIT_TIGON_I - if ((readl(&ap->regs->HostCtrl) >> 28) == 4) { - printk(KERN_ERR "%s: Driver compiled without Tigon I" - " support - NIC disabled\n", dev->name); - goto fail_uninit; - } -#endif - - if (ace_allocate_descriptors(dev)) - goto fail_free_netdev; - -#ifdef MODULE - if (boards_found >= ACE_MAX_MOD_PARMS) - ap->board_idx = BOARD_IDX_OVERFLOW; - else - ap->board_idx = boards_found; -#else - ap->board_idx = BOARD_IDX_STATIC; -#endif - - if (ace_init(dev)) - goto fail_free_netdev; - - if (register_netdev(dev)) { - printk(KERN_ERR "acenic: device registration failed\n"); - goto fail_uninit; - } - ap->name = dev->name; - - dev->features |= NETIF_F_HIGHDMA; - - pci_set_drvdata(pdev, dev); - - boards_found++; - return 0; - - fail_uninit: - ace_init_cleanup(dev); - fail_free_netdev: - free_netdev(dev); - return -ENODEV; -} - -static void acenic_remove_one(struct pci_dev *pdev) -{ - struct net_device *dev = pci_get_drvdata(pdev); - struct ace_private *ap = netdev_priv(dev); - struct ace_regs __iomem *regs = ap->regs; - short i; - - unregister_netdev(dev); - - writel(readl(®s->CpuCtrl) | CPU_HALT, ®s->CpuCtrl); - if (ap->version >= 2) - writel(readl(®s->CpuBCtrl) | CPU_HALT, ®s->CpuBCtrl); - - /* - * This clears any pending interrupts - */ - writel(1, ®s->Mb0Lo); - readl(®s->CpuCtrl); /* flush */ - - /* - * Make sure no other CPUs are processing interrupts - * on the card before the buffers are being released. - * Otherwise one might experience some `interesting' - * effects. - * - * Then release the RX buffers - jumbo buffers were - * already released in ace_close(). - */ - ace_sync_irq(dev->irq); - - for (i = 0; i < RX_STD_RING_ENTRIES; i++) { - struct sk_buff *skb = ap->skb->rx_std_skbuff[i].skb; - - if (skb) { - struct ring_info *ringp; - dma_addr_t mapping; - - ringp = &ap->skb->rx_std_skbuff[i]; - mapping = dma_unmap_addr(ringp, mapping); - dma_unmap_page(&ap->pdev->dev, mapping, - ACE_STD_BUFSIZE, DMA_FROM_DEVICE); - - ap->rx_std_ring[i].size = 0; - ap->skb->rx_std_skbuff[i].skb = NULL; - dev_kfree_skb(skb); - } - } - - if (ap->version >= 2) { - for (i = 0; i < RX_MINI_RING_ENTRIES; i++) { - struct sk_buff *skb = ap->skb->rx_mini_skbuff[i].skb; - - if (skb) { - struct ring_info *ringp; - dma_addr_t mapping; - - ringp = &ap->skb->rx_mini_skbuff[i]; - mapping = dma_unmap_addr(ringp,mapping); - dma_unmap_page(&ap->pdev->dev, mapping, - ACE_MINI_BUFSIZE, - DMA_FROM_DEVICE); - - ap->rx_mini_ring[i].size = 0; - ap->skb->rx_mini_skbuff[i].skb = NULL; - dev_kfree_skb(skb); - } - } - } - - for (i = 0; i < RX_JUMBO_RING_ENTRIES; i++) { - struct sk_buff *skb = ap->skb->rx_jumbo_skbuff[i].skb; - if (skb) { - struct ring_info *ringp; - dma_addr_t mapping; - - ringp = &ap->skb->rx_jumbo_skbuff[i]; - mapping = dma_unmap_addr(ringp, mapping); - dma_unmap_page(&ap->pdev->dev, mapping, - ACE_JUMBO_BUFSIZE, DMA_FROM_DEVICE); - - ap->rx_jumbo_ring[i].size = 0; - ap->skb->rx_jumbo_skbuff[i].skb = NULL; - dev_kfree_skb(skb); - } - } - - ace_init_cleanup(dev); - free_netdev(dev); -} - -static struct pci_driver acenic_pci_driver = { - .name = "acenic", - .id_table = acenic_pci_tbl, - .probe = acenic_probe_one, - .remove = acenic_remove_one, -}; - -static void ace_free_descriptors(struct net_device *dev) -{ - struct ace_private *ap = netdev_priv(dev); - int size; - - if (ap->rx_std_ring != NULL) { - size = (sizeof(struct rx_desc) * - (RX_STD_RING_ENTRIES + - RX_JUMBO_RING_ENTRIES + - RX_MINI_RING_ENTRIES + - RX_RETURN_RING_ENTRIES)); - dma_free_coherent(&ap->pdev->dev, size, ap->rx_std_ring, - ap->rx_ring_base_dma); - ap->rx_std_ring = NULL; - ap->rx_jumbo_ring = NULL; - ap->rx_mini_ring = NULL; - ap->rx_return_ring = NULL; - } - if (ap->evt_ring != NULL) { - size = (sizeof(struct event) * EVT_RING_ENTRIES); - dma_free_coherent(&ap->pdev->dev, size, ap->evt_ring, - ap->evt_ring_dma); - ap->evt_ring = NULL; - } - if (ap->tx_ring != NULL && !ACE_IS_TIGON_I(ap)) { - size = (sizeof(struct tx_desc) * MAX_TX_RING_ENTRIES); - dma_free_coherent(&ap->pdev->dev, size, ap->tx_ring, - ap->tx_ring_dma); - } - ap->tx_ring = NULL; - - if (ap->evt_prd != NULL) { - dma_free_coherent(&ap->pdev->dev, sizeof(u32), - (void *)ap->evt_prd, ap->evt_prd_dma); - ap->evt_prd = NULL; - } - if (ap->rx_ret_prd != NULL) { - dma_free_coherent(&ap->pdev->dev, sizeof(u32), - (void *)ap->rx_ret_prd, ap->rx_ret_prd_dma); - ap->rx_ret_prd = NULL; - } - if (ap->tx_csm != NULL) { - dma_free_coherent(&ap->pdev->dev, sizeof(u32), - (void *)ap->tx_csm, ap->tx_csm_dma); - ap->tx_csm = NULL; - } -} - - -static int ace_allocate_descriptors(struct net_device *dev) -{ - struct ace_private *ap = netdev_priv(dev); - int size; - - size = (sizeof(struct rx_desc) * - (RX_STD_RING_ENTRIES + - RX_JUMBO_RING_ENTRIES + - RX_MINI_RING_ENTRIES + - RX_RETURN_RING_ENTRIES)); - - ap->rx_std_ring = dma_alloc_coherent(&ap->pdev->dev, size, - &ap->rx_ring_base_dma, GFP_KERNEL); - if (ap->rx_std_ring == NULL) - goto fail; - - ap->rx_jumbo_ring = ap->rx_std_ring + RX_STD_RING_ENTRIES; - ap->rx_mini_ring = ap->rx_jumbo_ring + RX_JUMBO_RING_ENTRIES; - ap->rx_return_ring = ap->rx_mini_ring + RX_MINI_RING_ENTRIES; - - size = (sizeof(struct event) * EVT_RING_ENTRIES); - - ap->evt_ring = dma_alloc_coherent(&ap->pdev->dev, size, - &ap->evt_ring_dma, GFP_KERNEL); - - if (ap->evt_ring == NULL) - goto fail; - - /* - * Only allocate a host TX ring for the Tigon II, the Tigon I - * has to use PCI registers for this ;-( - */ - if (!ACE_IS_TIGON_I(ap)) { - size = (sizeof(struct tx_desc) * MAX_TX_RING_ENTRIES); - - ap->tx_ring = dma_alloc_coherent(&ap->pdev->dev, size, - &ap->tx_ring_dma, GFP_KERNEL); - - if (ap->tx_ring == NULL) - goto fail; - } - - ap->evt_prd = dma_alloc_coherent(&ap->pdev->dev, sizeof(u32), - &ap->evt_prd_dma, GFP_KERNEL); - if (ap->evt_prd == NULL) - goto fail; - - ap->rx_ret_prd = dma_alloc_coherent(&ap->pdev->dev, sizeof(u32), - &ap->rx_ret_prd_dma, GFP_KERNEL); - if (ap->rx_ret_prd == NULL) - goto fail; - - ap->tx_csm = dma_alloc_coherent(&ap->pdev->dev, sizeof(u32), - &ap->tx_csm_dma, GFP_KERNEL); - if (ap->tx_csm == NULL) - goto fail; - - return 0; - -fail: - /* Clean up. */ - ace_init_cleanup(dev); - return 1; -} - - -/* - * Generic cleanup handling data allocated during init. Used when the - * module is unloaded or if an error occurs during initialization - */ -static void ace_init_cleanup(struct net_device *dev) -{ - struct ace_private *ap; - - ap = netdev_priv(dev); - - ace_free_descriptors(dev); - - if (ap->info) - dma_free_coherent(&ap->pdev->dev, sizeof(struct ace_info), - ap->info, ap->info_dma); - kfree(ap->skb); - kfree(ap->trace_buf); - - if (dev->irq) - free_irq(dev->irq, dev); - - iounmap(ap->regs); -} - - -/* - * Commands are considered to be slow. - */ -static inline void ace_issue_cmd(struct ace_regs __iomem *regs, struct cmd *cmd) -{ - u32 idx; - - idx = readl(®s->CmdPrd); - - writel(*(u32 *)(cmd), ®s->CmdRng[idx]); - idx = (idx + 1) % CMD_RING_ENTRIES; - - writel(idx, ®s->CmdPrd); -} - - -static int ace_init(struct net_device *dev) -{ - struct ace_private *ap; - struct ace_regs __iomem *regs; - struct ace_info *info = NULL; - struct pci_dev *pdev; - unsigned long myjif; - u64 tmp_ptr; - u32 tig_ver, mac1, mac2, tmp, pci_state; - int board_idx, ecode = 0; - short i; - unsigned char cache_size; - u8 addr[ETH_ALEN]; - - ap = netdev_priv(dev); - regs = ap->regs; - - board_idx = ap->board_idx; - - /* - * aman@sgi.com - its useful to do a NIC reset here to - * address the `Firmware not running' problem subsequent - * to any crashes involving the NIC - */ - writel(HW_RESET | (HW_RESET << 24), ®s->HostCtrl); - readl(®s->HostCtrl); /* PCI write posting */ - udelay(5); - - /* - * Don't access any other registers before this point! - */ -#ifdef __BIG_ENDIAN - /* - * This will most likely need BYTE_SWAP once we switch - * to using __raw_writel() - */ - writel((WORD_SWAP | CLR_INT | ((WORD_SWAP | CLR_INT) << 24)), - ®s->HostCtrl); -#else - writel((CLR_INT | WORD_SWAP | ((CLR_INT | WORD_SWAP) << 24)), - ®s->HostCtrl); -#endif - readl(®s->HostCtrl); /* PCI write posting */ - - /* - * Stop the NIC CPU and clear pending interrupts - */ - writel(readl(®s->CpuCtrl) | CPU_HALT, ®s->CpuCtrl); - readl(®s->CpuCtrl); /* PCI write posting */ - writel(0, ®s->Mb0Lo); - - tig_ver = readl(®s->HostCtrl) >> 28; - - switch(tig_ver){ -#ifndef CONFIG_ACENIC_OMIT_TIGON_I - case 4: - case 5: - printk(KERN_INFO " Tigon I (Rev. %i), Firmware: %i.%i.%i, ", - tig_ver, ap->firmware_major, ap->firmware_minor, - ap->firmware_fix); - writel(0, ®s->LocalCtrl); - ap->version = 1; - ap->tx_ring_entries = TIGON_I_TX_RING_ENTRIES; - break; -#endif - case 6: - printk(KERN_INFO " Tigon II (Rev. %i), Firmware: %i.%i.%i, ", - tig_ver, ap->firmware_major, ap->firmware_minor, - ap->firmware_fix); - writel(readl(®s->CpuBCtrl) | CPU_HALT, ®s->CpuBCtrl); - readl(®s->CpuBCtrl); /* PCI write posting */ - /* - * The SRAM bank size does _not_ indicate the amount - * of memory on the card, it controls the _bank_ size! - * Ie. a 1MB AceNIC will have two banks of 512KB. - */ - writel(SRAM_BANK_512K, ®s->LocalCtrl); - writel(SYNC_SRAM_TIMING, ®s->MiscCfg); - ap->version = 2; - ap->tx_ring_entries = MAX_TX_RING_ENTRIES; - break; - default: - printk(KERN_WARNING " Unsupported Tigon version detected " - "(%i)\n", tig_ver); - ecode = -ENODEV; - goto init_error; - } - - /* - * ModeStat _must_ be set after the SRAM settings as this change - * seems to corrupt the ModeStat and possible other registers. - * The SRAM settings survive resets and setting it to the same - * value a second time works as well. This is what caused the - * `Firmware not running' problem on the Tigon II. - */ -#ifdef __BIG_ENDIAN - writel(ACE_BYTE_SWAP_DMA | ACE_WARN | ACE_FATAL | ACE_BYTE_SWAP_BD | - ACE_WORD_SWAP_BD | ACE_NO_JUMBO_FRAG, ®s->ModeStat); -#else - writel(ACE_BYTE_SWAP_DMA | ACE_WARN | ACE_FATAL | - ACE_WORD_SWAP_BD | ACE_NO_JUMBO_FRAG, ®s->ModeStat); -#endif - readl(®s->ModeStat); /* PCI write posting */ - - mac1 = 0; - for(i = 0; i < 4; i++) { - int t; - - mac1 = mac1 << 8; - t = read_eeprom_byte(dev, 0x8c+i); - if (t < 0) { - ecode = -EIO; - goto init_error; - } else - mac1 |= (t & 0xff); - } - mac2 = 0; - for(i = 4; i < 8; i++) { - int t; - - mac2 = mac2 << 8; - t = read_eeprom_byte(dev, 0x8c+i); - if (t < 0) { - ecode = -EIO; - goto init_error; - } else - mac2 |= (t & 0xff); - } - - writel(mac1, ®s->MacAddrHi); - writel(mac2, ®s->MacAddrLo); - - addr[0] = (mac1 >> 8) & 0xff; - addr[1] = mac1 & 0xff; - addr[2] = (mac2 >> 24) & 0xff; - addr[3] = (mac2 >> 16) & 0xff; - addr[4] = (mac2 >> 8) & 0xff; - addr[5] = mac2 & 0xff; - eth_hw_addr_set(dev, addr); - - printk("MAC: %pM\n", dev->dev_addr); - - /* - * Looks like this is necessary to deal with on all architectures, - * even this %$#%$# N440BX Intel based thing doesn't get it right. - * Ie. having two NICs in the machine, one will have the cache - * line set at boot time, the other will not. - */ - pdev = ap->pdev; - pci_read_config_byte(pdev, PCI_CACHE_LINE_SIZE, &cache_size); - cache_size <<= 2; - if (cache_size != SMP_CACHE_BYTES) { - printk(KERN_INFO " PCI cache line size set incorrectly " - "(%i bytes) by BIOS/FW, ", cache_size); - if (cache_size > SMP_CACHE_BYTES) - printk("expecting %i\n", SMP_CACHE_BYTES); - else { - printk("correcting to %i\n", SMP_CACHE_BYTES); - pci_write_config_byte(pdev, PCI_CACHE_LINE_SIZE, - SMP_CACHE_BYTES >> 2); - } - } - - pci_state = readl(®s->PciState); - printk(KERN_INFO " PCI bus width: %i bits, speed: %iMHz, " - "latency: %i clks\n", - (pci_state & PCI_32BIT) ? 32 : 64, - (pci_state & PCI_66MHZ) ? 66 : 33, - ap->pci_latency); - - /* - * Set the max DMA transfer size. Seems that for most systems - * the performance is better when no MAX parameter is - * set. However for systems enabling PCI write and invalidate, - * DMA writes must be set to the L1 cache line size to get - * optimal performance. - * - * The default is now to turn the PCI write and invalidate off - * - that is what Alteon does for NT. - */ - tmp = READ_CMD_MEM | WRITE_CMD_MEM; - if (ap->version >= 2) { - tmp |= (MEM_READ_MULTIPLE | (pci_state & PCI_66MHZ)); - /* - * Tuning parameters only supported for 8 cards - */ - if (board_idx == BOARD_IDX_OVERFLOW || - dis_pci_mem_inval[board_idx]) { - if (ap->pci_command & PCI_COMMAND_INVALIDATE) { - ap->pci_command &= ~PCI_COMMAND_INVALIDATE; - pci_write_config_word(pdev, PCI_COMMAND, - ap->pci_command); - printk(KERN_INFO " Disabling PCI memory " - "write and invalidate\n"); - } - } else if (ap->pci_command & PCI_COMMAND_INVALIDATE) { - printk(KERN_INFO " PCI memory write & invalidate " - "enabled by BIOS, enabling counter measures\n"); - - switch(SMP_CACHE_BYTES) { - case 16: - tmp |= DMA_WRITE_MAX_16; - break; - case 32: - tmp |= DMA_WRITE_MAX_32; - break; - case 64: - tmp |= DMA_WRITE_MAX_64; - break; - case 128: - tmp |= DMA_WRITE_MAX_128; - break; - default: - printk(KERN_INFO " Cache line size %i not " - "supported, PCI write and invalidate " - "disabled\n", SMP_CACHE_BYTES); - ap->pci_command &= ~PCI_COMMAND_INVALIDATE; - pci_write_config_word(pdev, PCI_COMMAND, - ap->pci_command); - } - } - } - -#ifdef __sparc__ - /* - * On this platform, we know what the best dma settings - * are. We use 64-byte maximum bursts, because if we - * burst larger than the cache line size (or even cross - * a 64byte boundary in a single burst) the UltraSparc - * PCI controller will disconnect at 64-byte multiples. - * - * Read-multiple will be properly enabled above, and when - * set will give the PCI controller proper hints about - * prefetching. - */ - tmp &= ~DMA_READ_WRITE_MASK; - tmp |= DMA_READ_MAX_64; - tmp |= DMA_WRITE_MAX_64; -#endif -#ifdef __alpha__ - tmp &= ~DMA_READ_WRITE_MASK; - tmp |= DMA_READ_MAX_128; - /* - * All the docs say MUST NOT. Well, I did. - * Nothing terrible happens, if we load wrong size. - * Bit w&i still works better! - */ - tmp |= DMA_WRITE_MAX_128; -#endif - writel(tmp, ®s->PciState); - -#if 0 - /* - * The Host PCI bus controller driver has to set FBB. - * If all devices on that PCI bus support FBB, then the controller - * can enable FBB support in the Host PCI Bus controller (or on - * the PCI-PCI bridge if that applies). - * -ggg - */ - /* - * I have received reports from people having problems when this - * bit is enabled. - */ - if (!(ap->pci_command & PCI_COMMAND_FAST_BACK)) { - printk(KERN_INFO " Enabling PCI Fast Back to Back\n"); - ap->pci_command |= PCI_COMMAND_FAST_BACK; - pci_write_config_word(pdev, PCI_COMMAND, ap->pci_command); - } -#endif - - /* - * Configure DMA attributes. - */ - if (dma_set_mask(&pdev->dev, DMA_BIT_MASK(64))) { - ecode = -ENODEV; - goto init_error; - } - - /* - * Initialize the generic info block and the command+event rings - * and the control blocks for the transmit and receive rings - * as they need to be setup once and for all. - */ - if (!(info = dma_alloc_coherent(&ap->pdev->dev, sizeof(struct ace_info), - &ap->info_dma, GFP_KERNEL))) { - ecode = -EAGAIN; - goto init_error; - } - ap->info = info; - - /* - * Get the memory for the skb rings. - */ - if (!(ap->skb = kzalloc_obj(struct ace_skb))) { - ecode = -EAGAIN; - goto init_error; - } - - ecode = request_irq(pdev->irq, ace_interrupt, IRQF_SHARED, - DRV_NAME, dev); - if (ecode) { - printk(KERN_WARNING "%s: Requested IRQ %d is busy\n", - DRV_NAME, pdev->irq); - goto init_error; - } else - dev->irq = pdev->irq; - -#ifdef INDEX_DEBUG - spin_lock_init(&ap->debug_lock); - ap->last_tx = ACE_TX_RING_ENTRIES(ap) - 1; - ap->last_std_rx = 0; - ap->last_mini_rx = 0; -#endif - - ecode = ace_load_firmware(dev); - if (ecode) - goto init_error; - - ap->fw_running = 0; - - tmp_ptr = ap->info_dma; - writel(tmp_ptr >> 32, ®s->InfoPtrHi); - writel(tmp_ptr & 0xffffffff, ®s->InfoPtrLo); - - memset(ap->evt_ring, 0, EVT_RING_ENTRIES * sizeof(struct event)); - - set_aceaddr(&info->evt_ctrl.rngptr, ap->evt_ring_dma); - info->evt_ctrl.flags = 0; - - *(ap->evt_prd) = 0; - wmb(); - set_aceaddr(&info->evt_prd_ptr, ap->evt_prd_dma); - writel(0, ®s->EvtCsm); - - set_aceaddr(&info->cmd_ctrl.rngptr, 0x100); - info->cmd_ctrl.flags = 0; - info->cmd_ctrl.max_len = 0; - - for (i = 0; i < CMD_RING_ENTRIES; i++) - writel(0, ®s->CmdRng[i]); - - writel(0, ®s->CmdPrd); - writel(0, ®s->CmdCsm); - - tmp_ptr = ap->info_dma; - tmp_ptr += (unsigned long) &(((struct ace_info *)0)->s.stats); - set_aceaddr(&info->stats2_ptr, (dma_addr_t) tmp_ptr); - - set_aceaddr(&info->rx_std_ctrl.rngptr, ap->rx_ring_base_dma); - info->rx_std_ctrl.max_len = ACE_STD_BUFSIZE; - info->rx_std_ctrl.flags = - RCB_FLG_TCP_UDP_SUM | RCB_FLG_NO_PSEUDO_HDR | RCB_FLG_VLAN_ASSIST; - - memset(ap->rx_std_ring, 0, - RX_STD_RING_ENTRIES * sizeof(struct rx_desc)); - - for (i = 0; i < RX_STD_RING_ENTRIES; i++) - ap->rx_std_ring[i].flags = BD_FLG_TCP_UDP_SUM; - - ap->rx_std_skbprd = 0; - atomic_set(&ap->cur_rx_bufs, 0); - - set_aceaddr(&info->rx_jumbo_ctrl.rngptr, - (ap->rx_ring_base_dma + - (sizeof(struct rx_desc) * RX_STD_RING_ENTRIES))); - info->rx_jumbo_ctrl.max_len = 0; - info->rx_jumbo_ctrl.flags = - RCB_FLG_TCP_UDP_SUM | RCB_FLG_NO_PSEUDO_HDR | RCB_FLG_VLAN_ASSIST; - - memset(ap->rx_jumbo_ring, 0, - RX_JUMBO_RING_ENTRIES * sizeof(struct rx_desc)); - - for (i = 0; i < RX_JUMBO_RING_ENTRIES; i++) - ap->rx_jumbo_ring[i].flags = BD_FLG_TCP_UDP_SUM | BD_FLG_JUMBO; - - ap->rx_jumbo_skbprd = 0; - atomic_set(&ap->cur_jumbo_bufs, 0); - - memset(ap->rx_mini_ring, 0, - RX_MINI_RING_ENTRIES * sizeof(struct rx_desc)); - - if (ap->version >= 2) { - set_aceaddr(&info->rx_mini_ctrl.rngptr, - (ap->rx_ring_base_dma + - (sizeof(struct rx_desc) * - (RX_STD_RING_ENTRIES + - RX_JUMBO_RING_ENTRIES)))); - info->rx_mini_ctrl.max_len = ACE_MINI_SIZE; - info->rx_mini_ctrl.flags = - RCB_FLG_TCP_UDP_SUM|RCB_FLG_NO_PSEUDO_HDR|RCB_FLG_VLAN_ASSIST; - - for (i = 0; i < RX_MINI_RING_ENTRIES; i++) - ap->rx_mini_ring[i].flags = - BD_FLG_TCP_UDP_SUM | BD_FLG_MINI; - } else { - set_aceaddr(&info->rx_mini_ctrl.rngptr, 0); - info->rx_mini_ctrl.flags = RCB_FLG_RNG_DISABLE; - info->rx_mini_ctrl.max_len = 0; - } - - ap->rx_mini_skbprd = 0; - atomic_set(&ap->cur_mini_bufs, 0); - - set_aceaddr(&info->rx_return_ctrl.rngptr, - (ap->rx_ring_base_dma + - (sizeof(struct rx_desc) * - (RX_STD_RING_ENTRIES + - RX_JUMBO_RING_ENTRIES + - RX_MINI_RING_ENTRIES)))); - info->rx_return_ctrl.flags = 0; - info->rx_return_ctrl.max_len = RX_RETURN_RING_ENTRIES; - - memset(ap->rx_return_ring, 0, - RX_RETURN_RING_ENTRIES * sizeof(struct rx_desc)); - - set_aceaddr(&info->rx_ret_prd_ptr, ap->rx_ret_prd_dma); - *(ap->rx_ret_prd) = 0; - - writel(TX_RING_BASE, ®s->WinBase); - - if (ACE_IS_TIGON_I(ap)) { - ap->tx_ring = (__force struct tx_desc *) regs->Window; - for (i = 0; i < (TIGON_I_TX_RING_ENTRIES - * sizeof(struct tx_desc)) / sizeof(u32); i++) - writel(0, (__force void __iomem *)ap->tx_ring + i * 4); - - set_aceaddr(&info->tx_ctrl.rngptr, TX_RING_BASE); - } else { - memset(ap->tx_ring, 0, - MAX_TX_RING_ENTRIES * sizeof(struct tx_desc)); - - set_aceaddr(&info->tx_ctrl.rngptr, ap->tx_ring_dma); - } - - info->tx_ctrl.max_len = ACE_TX_RING_ENTRIES(ap); - tmp = RCB_FLG_TCP_UDP_SUM | RCB_FLG_NO_PSEUDO_HDR | RCB_FLG_VLAN_ASSIST; - - /* - * The Tigon I does not like having the TX ring in host memory ;-( - */ - if (!ACE_IS_TIGON_I(ap)) - tmp |= RCB_FLG_TX_HOST_RING; -#if TX_COAL_INTS_ONLY - tmp |= RCB_FLG_COAL_INT_ONLY; -#endif - info->tx_ctrl.flags = tmp; - - set_aceaddr(&info->tx_csm_ptr, ap->tx_csm_dma); - - /* - * Potential item for tuning parameter - */ -#if 0 /* NO */ - writel(DMA_THRESH_16W, ®s->DmaReadCfg); - writel(DMA_THRESH_16W, ®s->DmaWriteCfg); -#else - writel(DMA_THRESH_8W, ®s->DmaReadCfg); - writel(DMA_THRESH_8W, ®s->DmaWriteCfg); -#endif - - writel(0, ®s->MaskInt); - writel(1, ®s->IfIdx); -#if 0 - /* - * McKinley boxes do not like us fiddling with AssistState - * this early - */ - writel(1, ®s->AssistState); -#endif - - writel(DEF_STAT, ®s->TuneStatTicks); - writel(DEF_TRACE, ®s->TuneTrace); - - ace_set_rxtx_parms(dev, 0); - - if (board_idx == BOARD_IDX_OVERFLOW) { - printk(KERN_WARNING "%s: more than %i NICs detected, " - "ignoring module parameters!\n", - ap->name, ACE_MAX_MOD_PARMS); - } else if (board_idx >= 0) { - if (tx_coal_tick[board_idx]) - writel(tx_coal_tick[board_idx], - ®s->TuneTxCoalTicks); - if (max_tx_desc[board_idx]) - writel(max_tx_desc[board_idx], ®s->TuneMaxTxDesc); - - if (rx_coal_tick[board_idx]) - writel(rx_coal_tick[board_idx], - ®s->TuneRxCoalTicks); - if (max_rx_desc[board_idx]) - writel(max_rx_desc[board_idx], ®s->TuneMaxRxDesc); - - if (trace[board_idx]) - writel(trace[board_idx], ®s->TuneTrace); - - if ((tx_ratio[board_idx] > 0) && (tx_ratio[board_idx] < 64)) - writel(tx_ratio[board_idx], ®s->TxBufRat); - } - - /* - * Default link parameters - */ - tmp = LNK_ENABLE | LNK_FULL_DUPLEX | LNK_1000MB | LNK_100MB | - LNK_10MB | LNK_RX_FLOW_CTL_Y | LNK_NEG_FCTL | LNK_NEGOTIATE; - if(ap->version >= 2) - tmp |= LNK_TX_FLOW_CTL_Y; - - /* - * Override link default parameters - */ - if ((board_idx >= 0) && link_state[board_idx]) { - int option = link_state[board_idx]; - - tmp = LNK_ENABLE; - - if (option & 0x01) { - printk(KERN_INFO "%s: Setting half duplex link\n", - ap->name); - tmp &= ~LNK_FULL_DUPLEX; - } - if (option & 0x02) - tmp &= ~LNK_NEGOTIATE; - if (option & 0x10) - tmp |= LNK_10MB; - if (option & 0x20) - tmp |= LNK_100MB; - if (option & 0x40) - tmp |= LNK_1000MB; - if ((option & 0x70) == 0) { - printk(KERN_WARNING "%s: No media speed specified, " - "forcing auto negotiation\n", ap->name); - tmp |= LNK_NEGOTIATE | LNK_1000MB | - LNK_100MB | LNK_10MB; - } - if ((option & 0x100) == 0) - tmp |= LNK_NEG_FCTL; - else - printk(KERN_INFO "%s: Disabling flow control " - "negotiation\n", ap->name); - if (option & 0x200) - tmp |= LNK_RX_FLOW_CTL_Y; - if ((option & 0x400) && (ap->version >= 2)) { - printk(KERN_INFO "%s: Enabling TX flow control\n", - ap->name); - tmp |= LNK_TX_FLOW_CTL_Y; - } - } - - ap->link = tmp; - writel(tmp, ®s->TuneLink); - if (ap->version >= 2) - writel(tmp, ®s->TuneFastLink); - - writel(ap->firmware_start, ®s->Pc); - - writel(0, ®s->Mb0Lo); - - /* - * Set tx_csm before we start receiving interrupts, otherwise - * the interrupt handler might think it is supposed to process - * tx ints before we are up and running, which may cause a null - * pointer access in the int handler. - */ - ap->cur_rx = 0; - ap->tx_prd = *(ap->tx_csm) = ap->tx_ret_csm = 0; - - wmb(); - ace_set_txprd(regs, ap, 0); - writel(0, ®s->RxRetCsm); - - /* - * Enable DMA engine now. - * If we do this sooner, Mckinley box pukes. - * I assume it's because Tigon II DMA engine wants to check - * *something* even before the CPU is started. - */ - writel(1, ®s->AssistState); /* enable DMA */ - - /* - * Start the NIC CPU - */ - writel(readl(®s->CpuCtrl) & ~(CPU_HALT|CPU_TRACE), ®s->CpuCtrl); - readl(®s->CpuCtrl); - - /* - * Wait for the firmware to spin up - max 3 seconds. - */ - myjif = jiffies + 3 * HZ; - while (time_before(jiffies, myjif) && !ap->fw_running) - cpu_relax(); - - if (!ap->fw_running) { - printk(KERN_ERR "%s: Firmware NOT running!\n", ap->name); - - ace_dump_trace(ap); - writel(readl(®s->CpuCtrl) | CPU_HALT, ®s->CpuCtrl); - readl(®s->CpuCtrl); - - /* aman@sgi.com - account for badly behaving firmware/NIC: - * - have observed that the NIC may continue to generate - * interrupts for some reason; attempt to stop it - halt - * second CPU for Tigon II cards, and also clear Mb0 - * - if we're a module, we'll fail to load if this was - * the only GbE card in the system => if the kernel does - * see an interrupt from the NIC, code to handle it is - * gone and OOps! - so free_irq also - */ - if (ap->version >= 2) - writel(readl(®s->CpuBCtrl) | CPU_HALT, - ®s->CpuBCtrl); - writel(0, ®s->Mb0Lo); - readl(®s->Mb0Lo); - - ecode = -EBUSY; - goto init_error; - } - - /* - * We load the ring here as there seem to be no way to tell the - * firmware to wipe the ring without re-initializing it. - */ - if (!test_and_set_bit(0, &ap->std_refill_busy)) - ace_load_std_rx_ring(dev, RX_RING_SIZE); - else - printk(KERN_ERR "%s: Someone is busy refilling the RX ring\n", - ap->name); - if (ap->version >= 2) { - if (!test_and_set_bit(0, &ap->mini_refill_busy)) - ace_load_mini_rx_ring(dev, RX_MINI_SIZE); - else - printk(KERN_ERR "%s: Someone is busy refilling " - "the RX mini ring\n", ap->name); - } - return 0; - - init_error: - ace_init_cleanup(dev); - return ecode; -} - - -static void ace_set_rxtx_parms(struct net_device *dev, int jumbo) -{ - struct ace_private *ap = netdev_priv(dev); - struct ace_regs __iomem *regs = ap->regs; - int board_idx = ap->board_idx; - - if (board_idx >= 0) { - if (!jumbo) { - if (!tx_coal_tick[board_idx]) - writel(DEF_TX_COAL, ®s->TuneTxCoalTicks); - if (!max_tx_desc[board_idx]) - writel(DEF_TX_MAX_DESC, ®s->TuneMaxTxDesc); - if (!rx_coal_tick[board_idx]) - writel(DEF_RX_COAL, ®s->TuneRxCoalTicks); - if (!max_rx_desc[board_idx]) - writel(DEF_RX_MAX_DESC, ®s->TuneMaxRxDesc); - if (!tx_ratio[board_idx]) - writel(DEF_TX_RATIO, ®s->TxBufRat); - } else { - if (!tx_coal_tick[board_idx]) - writel(DEF_JUMBO_TX_COAL, - ®s->TuneTxCoalTicks); - if (!max_tx_desc[board_idx]) - writel(DEF_JUMBO_TX_MAX_DESC, - ®s->TuneMaxTxDesc); - if (!rx_coal_tick[board_idx]) - writel(DEF_JUMBO_RX_COAL, - ®s->TuneRxCoalTicks); - if (!max_rx_desc[board_idx]) - writel(DEF_JUMBO_RX_MAX_DESC, - ®s->TuneMaxRxDesc); - if (!tx_ratio[board_idx]) - writel(DEF_JUMBO_TX_RATIO, ®s->TxBufRat); - } - } -} - - -static void ace_watchdog(struct net_device *data, unsigned int txqueue) -{ - struct net_device *dev = data; - struct ace_private *ap = netdev_priv(dev); - struct ace_regs __iomem *regs = ap->regs; - - /* - * We haven't received a stats update event for more than 2.5 - * seconds and there is data in the transmit queue, thus we - * assume the card is stuck. - */ - if (*ap->tx_csm != ap->tx_ret_csm) { - printk(KERN_WARNING "%s: Transmitter is stuck, %08x\n", - dev->name, (unsigned int)readl(®s->HostCtrl)); - /* This can happen due to ieee flow control. */ - } else { - printk(KERN_DEBUG "%s: BUG... transmitter died. Kicking it.\n", - dev->name); -#if 0 - netif_wake_queue(dev); -#endif - } -} - - -static void ace_bh_work(struct work_struct *work) -{ - struct ace_private *ap = from_work(ap, work, ace_bh_work); - struct net_device *dev = ap->ndev; - int cur_size; - - cur_size = atomic_read(&ap->cur_rx_bufs); - if ((cur_size < RX_LOW_STD_THRES) && - !test_and_set_bit(0, &ap->std_refill_busy)) { -#ifdef DEBUG - printk("refilling buffers (current %i)\n", cur_size); -#endif - ace_load_std_rx_ring(dev, RX_RING_SIZE - cur_size); - } - - if (ap->version >= 2) { - cur_size = atomic_read(&ap->cur_mini_bufs); - if ((cur_size < RX_LOW_MINI_THRES) && - !test_and_set_bit(0, &ap->mini_refill_busy)) { -#ifdef DEBUG - printk("refilling mini buffers (current %i)\n", - cur_size); -#endif - ace_load_mini_rx_ring(dev, RX_MINI_SIZE - cur_size); - } - } - - cur_size = atomic_read(&ap->cur_jumbo_bufs); - if (ap->jumbo && (cur_size < RX_LOW_JUMBO_THRES) && - !test_and_set_bit(0, &ap->jumbo_refill_busy)) { -#ifdef DEBUG - printk("refilling jumbo buffers (current %i)\n", cur_size); -#endif - ace_load_jumbo_rx_ring(dev, RX_JUMBO_SIZE - cur_size); - } - ap->bh_work_pending = 0; -} - - -/* - * Copy the contents of the NIC's trace buffer to kernel memory. - */ -static void ace_dump_trace(struct ace_private *ap) -{ -#if 0 - if (!ap->trace_buf) - if (!(ap->trace_buf = kmalloc(ACE_TRACE_SIZE, GFP_KERNEL))) - return; -#endif -} - - -/* - * Load the standard rx ring. - * - * Loading rings is safe without holding the spin lock since this is - * done only before the device is enabled, thus no interrupts are - * generated and by the interrupt handler/bh handler. - */ -static void ace_load_std_rx_ring(struct net_device *dev, int nr_bufs) -{ - struct ace_private *ap = netdev_priv(dev); - struct ace_regs __iomem *regs = ap->regs; - short i, idx; - - - prefetchw(&ap->cur_rx_bufs); - - idx = ap->rx_std_skbprd; - - for (i = 0; i < nr_bufs; i++) { - struct sk_buff *skb; - struct rx_desc *rd; - dma_addr_t mapping; - - skb = netdev_alloc_skb_ip_align(dev, ACE_STD_BUFSIZE); - if (!skb) - break; - - mapping = dma_map_page(&ap->pdev->dev, - virt_to_page(skb->data), - offset_in_page(skb->data), - ACE_STD_BUFSIZE, DMA_FROM_DEVICE); - ap->skb->rx_std_skbuff[idx].skb = skb; - dma_unmap_addr_set(&ap->skb->rx_std_skbuff[idx], - mapping, mapping); - - rd = &ap->rx_std_ring[idx]; - set_aceaddr(&rd->addr, mapping); - rd->size = ACE_STD_BUFSIZE; - rd->idx = idx; - idx = (idx + 1) % RX_STD_RING_ENTRIES; - } - - if (!i) - goto error_out; - - atomic_add(i, &ap->cur_rx_bufs); - ap->rx_std_skbprd = idx; - - if (ACE_IS_TIGON_I(ap)) { - struct cmd cmd; - cmd.evt = C_SET_RX_PRD_IDX; - cmd.code = 0; - cmd.idx = ap->rx_std_skbprd; - ace_issue_cmd(regs, &cmd); - } else { - writel(idx, ®s->RxStdPrd); - wmb(); - } - - out: - clear_bit(0, &ap->std_refill_busy); - return; - - error_out: - printk(KERN_INFO "Out of memory when allocating " - "standard receive buffers\n"); - goto out; -} - - -static void ace_load_mini_rx_ring(struct net_device *dev, int nr_bufs) -{ - struct ace_private *ap = netdev_priv(dev); - struct ace_regs __iomem *regs = ap->regs; - short i, idx; - - prefetchw(&ap->cur_mini_bufs); - - idx = ap->rx_mini_skbprd; - for (i = 0; i < nr_bufs; i++) { - struct sk_buff *skb; - struct rx_desc *rd; - dma_addr_t mapping; - - skb = netdev_alloc_skb_ip_align(dev, ACE_MINI_BUFSIZE); - if (!skb) - break; - - mapping = dma_map_page(&ap->pdev->dev, - virt_to_page(skb->data), - offset_in_page(skb->data), - ACE_MINI_BUFSIZE, DMA_FROM_DEVICE); - ap->skb->rx_mini_skbuff[idx].skb = skb; - dma_unmap_addr_set(&ap->skb->rx_mini_skbuff[idx], - mapping, mapping); - - rd = &ap->rx_mini_ring[idx]; - set_aceaddr(&rd->addr, mapping); - rd->size = ACE_MINI_BUFSIZE; - rd->idx = idx; - idx = (idx + 1) % RX_MINI_RING_ENTRIES; - } - - if (!i) - goto error_out; - - atomic_add(i, &ap->cur_mini_bufs); - - ap->rx_mini_skbprd = idx; - - writel(idx, ®s->RxMiniPrd); - wmb(); - - out: - clear_bit(0, &ap->mini_refill_busy); - return; - error_out: - printk(KERN_INFO "Out of memory when allocating " - "mini receive buffers\n"); - goto out; -} - - -/* - * Load the jumbo rx ring, this may happen at any time if the MTU - * is changed to a value > 1500. - */ -static void ace_load_jumbo_rx_ring(struct net_device *dev, int nr_bufs) -{ - struct ace_private *ap = netdev_priv(dev); - struct ace_regs __iomem *regs = ap->regs; - short i, idx; - - idx = ap->rx_jumbo_skbprd; - - for (i = 0; i < nr_bufs; i++) { - struct sk_buff *skb; - struct rx_desc *rd; - dma_addr_t mapping; - - skb = netdev_alloc_skb_ip_align(dev, ACE_JUMBO_BUFSIZE); - if (!skb) - break; - - mapping = dma_map_page(&ap->pdev->dev, - virt_to_page(skb->data), - offset_in_page(skb->data), - ACE_JUMBO_BUFSIZE, DMA_FROM_DEVICE); - ap->skb->rx_jumbo_skbuff[idx].skb = skb; - dma_unmap_addr_set(&ap->skb->rx_jumbo_skbuff[idx], - mapping, mapping); - - rd = &ap->rx_jumbo_ring[idx]; - set_aceaddr(&rd->addr, mapping); - rd->size = ACE_JUMBO_BUFSIZE; - rd->idx = idx; - idx = (idx + 1) % RX_JUMBO_RING_ENTRIES; - } - - if (!i) - goto error_out; - - atomic_add(i, &ap->cur_jumbo_bufs); - ap->rx_jumbo_skbprd = idx; - - if (ACE_IS_TIGON_I(ap)) { - struct cmd cmd; - cmd.evt = C_SET_RX_JUMBO_PRD_IDX; - cmd.code = 0; - cmd.idx = ap->rx_jumbo_skbprd; - ace_issue_cmd(regs, &cmd); - } else { - writel(idx, ®s->RxJumboPrd); - wmb(); - } - - out: - clear_bit(0, &ap->jumbo_refill_busy); - return; - error_out: - if (net_ratelimit()) - printk(KERN_INFO "Out of memory when allocating " - "jumbo receive buffers\n"); - goto out; -} - - -/* - * All events are considered to be slow (RX/TX ints do not generate - * events) and are handled here, outside the main interrupt handler, - * to reduce the size of the handler. - */ -static u32 ace_handle_event(struct net_device *dev, u32 evtcsm, u32 evtprd) -{ - struct ace_private *ap; - - ap = netdev_priv(dev); - - while (evtcsm != evtprd) { - switch (ap->evt_ring[evtcsm].evt) { - case E_FW_RUNNING: - printk(KERN_INFO "%s: Firmware up and running\n", - ap->name); - ap->fw_running = 1; - wmb(); - break; - case E_STATS_UPDATED: - break; - case E_LNK_STATE: - { - u16 code = ap->evt_ring[evtcsm].code; - switch (code) { - case E_C_LINK_UP: - { - u32 state = readl(&ap->regs->GigLnkState); - printk(KERN_WARNING "%s: Optical link UP " - "(%s Duplex, Flow Control: %s%s)\n", - ap->name, - state & LNK_FULL_DUPLEX ? "Full":"Half", - state & LNK_TX_FLOW_CTL_Y ? "TX " : "", - state & LNK_RX_FLOW_CTL_Y ? "RX" : ""); - break; - } - case E_C_LINK_DOWN: - printk(KERN_WARNING "%s: Optical link DOWN\n", - ap->name); - break; - case E_C_LINK_10_100: - printk(KERN_WARNING "%s: 10/100BaseT link " - "UP\n", ap->name); - break; - default: - printk(KERN_ERR "%s: Unknown optical link " - "state %02x\n", ap->name, code); - } - break; - } - case E_ERROR: - switch(ap->evt_ring[evtcsm].code) { - case E_C_ERR_INVAL_CMD: - printk(KERN_ERR "%s: invalid command error\n", - ap->name); - break; - case E_C_ERR_UNIMP_CMD: - printk(KERN_ERR "%s: unimplemented command " - "error\n", ap->name); - break; - case E_C_ERR_BAD_CFG: - printk(KERN_ERR "%s: bad config error\n", - ap->name); - break; - default: - printk(KERN_ERR "%s: unknown error %02x\n", - ap->name, ap->evt_ring[evtcsm].code); - } - break; - case E_RESET_JUMBO_RNG: - { - int i; - for (i = 0; i < RX_JUMBO_RING_ENTRIES; i++) { - if (ap->skb->rx_jumbo_skbuff[i].skb) { - ap->rx_jumbo_ring[i].size = 0; - set_aceaddr(&ap->rx_jumbo_ring[i].addr, 0); - dev_kfree_skb(ap->skb->rx_jumbo_skbuff[i].skb); - ap->skb->rx_jumbo_skbuff[i].skb = NULL; - } - } - - if (ACE_IS_TIGON_I(ap)) { - struct cmd cmd; - cmd.evt = C_SET_RX_JUMBO_PRD_IDX; - cmd.code = 0; - cmd.idx = 0; - ace_issue_cmd(ap->regs, &cmd); - } else { - writel(0, &((ap->regs)->RxJumboPrd)); - wmb(); - } - - ap->jumbo = 0; - ap->rx_jumbo_skbprd = 0; - printk(KERN_INFO "%s: Jumbo ring flushed\n", - ap->name); - clear_bit(0, &ap->jumbo_refill_busy); - break; - } - default: - printk(KERN_ERR "%s: Unhandled event 0x%02x\n", - ap->name, ap->evt_ring[evtcsm].evt); - } - evtcsm = (evtcsm + 1) % EVT_RING_ENTRIES; - } - - return evtcsm; -} - - -static void ace_rx_int(struct net_device *dev, u32 rxretprd, u32 rxretcsm) -{ - struct ace_private *ap = netdev_priv(dev); - u32 idx; - int mini_count = 0, std_count = 0; - - idx = rxretcsm; - - prefetchw(&ap->cur_rx_bufs); - prefetchw(&ap->cur_mini_bufs); - - while (idx != rxretprd) { - struct ring_info *rip; - struct sk_buff *skb; - struct rx_desc *retdesc; - u32 skbidx; - int bd_flags, desc_type, mapsize; - u16 csum; - - - /* make sure the rx descriptor isn't read before rxretprd */ - if (idx == rxretcsm) - rmb(); - - retdesc = &ap->rx_return_ring[idx]; - skbidx = retdesc->idx; - bd_flags = retdesc->flags; - desc_type = bd_flags & (BD_FLG_JUMBO | BD_FLG_MINI); - - switch(desc_type) { - /* - * Normal frames do not have any flags set - * - * Mini and normal frames arrive frequently, - * so use a local counter to avoid doing - * atomic operations for each packet arriving. - */ - case 0: - rip = &ap->skb->rx_std_skbuff[skbidx]; - mapsize = ACE_STD_BUFSIZE; - std_count++; - break; - case BD_FLG_JUMBO: - rip = &ap->skb->rx_jumbo_skbuff[skbidx]; - mapsize = ACE_JUMBO_BUFSIZE; - atomic_dec(&ap->cur_jumbo_bufs); - break; - case BD_FLG_MINI: - rip = &ap->skb->rx_mini_skbuff[skbidx]; - mapsize = ACE_MINI_BUFSIZE; - mini_count++; - break; - default: - printk(KERN_INFO "%s: unknown frame type (0x%02x) " - "returned by NIC\n", dev->name, - retdesc->flags); - goto error; - } - - skb = rip->skb; - rip->skb = NULL; - dma_unmap_page(&ap->pdev->dev, dma_unmap_addr(rip, mapping), - mapsize, DMA_FROM_DEVICE); - skb_put(skb, retdesc->size); - - /* - * Fly baby, fly! - */ - csum = retdesc->tcp_udp_csum; - - skb->protocol = eth_type_trans(skb, dev); - - /* - * Instead of forcing the poor tigon mips cpu to calculate - * pseudo hdr checksum, we do this ourselves. - */ - if (bd_flags & BD_FLG_TCP_UDP_SUM) { - skb->csum = htons(csum); - skb->ip_summed = CHECKSUM_COMPLETE; - } else { - skb_checksum_none_assert(skb); - } - - /* send it up */ - if ((bd_flags & BD_FLG_VLAN_TAG)) - __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), retdesc->vlan); - netif_rx(skb); - - dev->stats.rx_packets++; - dev->stats.rx_bytes += retdesc->size; - - idx = (idx + 1) % RX_RETURN_RING_ENTRIES; - } - - atomic_sub(std_count, &ap->cur_rx_bufs); - if (!ACE_IS_TIGON_I(ap)) - atomic_sub(mini_count, &ap->cur_mini_bufs); - - out: - /* - * According to the documentation RxRetCsm is obsolete with - * the 12.3.x Firmware - my Tigon I NICs seem to disagree! - */ - if (ACE_IS_TIGON_I(ap)) { - writel(idx, &ap->regs->RxRetCsm); - } - ap->cur_rx = idx; - - return; - error: - idx = rxretprd; - goto out; -} - - -static inline void ace_tx_int(struct net_device *dev, - u32 txcsm, u32 idx) -{ - struct ace_private *ap = netdev_priv(dev); - - do { - struct sk_buff *skb; - struct tx_ring_info *info; - - info = ap->skb->tx_skbuff + idx; - skb = info->skb; - - if (dma_unmap_len(info, maplen)) { - dma_unmap_page(&ap->pdev->dev, - dma_unmap_addr(info, mapping), - dma_unmap_len(info, maplen), - DMA_TO_DEVICE); - dma_unmap_len_set(info, maplen, 0); - } - - if (skb) { - dev->stats.tx_packets++; - dev->stats.tx_bytes += skb->len; - dev_consume_skb_irq(skb); - info->skb = NULL; - } - - idx = (idx + 1) % ACE_TX_RING_ENTRIES(ap); - } while (idx != txcsm); - - if (netif_queue_stopped(dev)) - netif_wake_queue(dev); - - wmb(); - ap->tx_ret_csm = txcsm; - - /* So... tx_ret_csm is advanced _after_ check for device wakeup. - * - * We could try to make it before. In this case we would get - * the following race condition: hard_start_xmit on other cpu - * enters after we advanced tx_ret_csm and fills space, - * which we have just freed, so that we make illegal device wakeup. - * There is no good way to workaround this (at entry - * to ace_start_xmit detects this condition and prevents - * ring corruption, but it is not a good workaround.) - * - * When tx_ret_csm is advanced after, we wake up device _only_ - * if we really have some space in ring (though the core doing - * hard_start_xmit can see full ring for some period and has to - * synchronize.) Superb. - * BUT! We get another subtle race condition. hard_start_xmit - * may think that ring is full between wakeup and advancing - * tx_ret_csm and will stop device instantly! It is not so bad. - * We are guaranteed that there is something in ring, so that - * the next irq will resume transmission. To speedup this we could - * mark descriptor, which closes ring with BD_FLG_COAL_NOW - * (see ace_start_xmit). - * - * Well, this dilemma exists in all lock-free devices. - * We, following scheme used in drivers by Donald Becker, - * select the least dangerous. - * --ANK - */ -} - - -static irqreturn_t ace_interrupt(int irq, void *dev_id) -{ - struct net_device *dev = (struct net_device *)dev_id; - struct ace_private *ap = netdev_priv(dev); - struct ace_regs __iomem *regs = ap->regs; - u32 idx; - u32 txcsm, rxretcsm, rxretprd; - u32 evtcsm, evtprd; - - /* - * In case of PCI shared interrupts or spurious interrupts, - * we want to make sure it is actually our interrupt before - * spending any time in here. - */ - if (!(readl(®s->HostCtrl) & IN_INT)) - return IRQ_NONE; - - /* - * ACK intr now. Otherwise we will lose updates to rx_ret_prd, - * which happened _after_ rxretprd = *ap->rx_ret_prd; but before - * writel(0, ®s->Mb0Lo). - * - * "IRQ avoidance" recommended in docs applies to IRQs served - * threads and it is wrong even for that case. - */ - writel(0, ®s->Mb0Lo); - readl(®s->Mb0Lo); - - /* - * There is no conflict between transmit handling in - * start_xmit and receive processing, thus there is no reason - * to take a spin lock for RX handling. Wait until we start - * working on the other stuff - hey we don't need a spin lock - * anymore. - */ - rxretprd = *ap->rx_ret_prd; - rxretcsm = ap->cur_rx; - - if (rxretprd != rxretcsm) - ace_rx_int(dev, rxretprd, rxretcsm); - - txcsm = *ap->tx_csm; - idx = ap->tx_ret_csm; - - if (txcsm != idx) { - /* - * If each skb takes only one descriptor this check degenerates - * to identity, because new space has just been opened. - * But if skbs are fragmented we must check that this index - * update releases enough of space, otherwise we just - * wait for device to make more work. - */ - if (!tx_ring_full(ap, txcsm, ap->tx_prd)) - ace_tx_int(dev, txcsm, idx); - } - - evtcsm = readl(®s->EvtCsm); - evtprd = *ap->evt_prd; - - if (evtcsm != evtprd) { - evtcsm = ace_handle_event(dev, evtcsm, evtprd); - writel(evtcsm, ®s->EvtCsm); - } - - /* - * This has to go last in the interrupt handler and run with - * the spin lock released ... what lock? - */ - if (netif_running(dev)) { - int cur_size; - int run_bh_work = 0; - - cur_size = atomic_read(&ap->cur_rx_bufs); - if (cur_size < RX_LOW_STD_THRES) { - if ((cur_size < RX_PANIC_STD_THRES) && - !test_and_set_bit(0, &ap->std_refill_busy)) { -#ifdef DEBUG - printk("low on std buffers %i\n", cur_size); -#endif - ace_load_std_rx_ring(dev, - RX_RING_SIZE - cur_size); - } else - run_bh_work = 1; - } - - if (!ACE_IS_TIGON_I(ap)) { - cur_size = atomic_read(&ap->cur_mini_bufs); - if (cur_size < RX_LOW_MINI_THRES) { - if ((cur_size < RX_PANIC_MINI_THRES) && - !test_and_set_bit(0, - &ap->mini_refill_busy)) { -#ifdef DEBUG - printk("low on mini buffers %i\n", - cur_size); -#endif - ace_load_mini_rx_ring(dev, - RX_MINI_SIZE - cur_size); - } else - run_bh_work = 1; - } - } - - if (ap->jumbo) { - cur_size = atomic_read(&ap->cur_jumbo_bufs); - if (cur_size < RX_LOW_JUMBO_THRES) { - if ((cur_size < RX_PANIC_JUMBO_THRES) && - !test_and_set_bit(0, - &ap->jumbo_refill_busy)){ -#ifdef DEBUG - printk("low on jumbo buffers %i\n", - cur_size); -#endif - ace_load_jumbo_rx_ring(dev, - RX_JUMBO_SIZE - cur_size); - } else - run_bh_work = 1; - } - } - if (run_bh_work && !ap->bh_work_pending) { - ap->bh_work_pending = 1; - queue_work(system_bh_wq, &ap->ace_bh_work); - } - } - - return IRQ_HANDLED; -} - -static int ace_open(struct net_device *dev) -{ - struct ace_private *ap = netdev_priv(dev); - struct ace_regs __iomem *regs = ap->regs; - struct cmd cmd; - - if (!(ap->fw_running)) { - printk(KERN_WARNING "%s: Firmware not running!\n", dev->name); - return -EBUSY; - } - - writel(dev->mtu + ETH_HLEN + 4, ®s->IfMtu); - - cmd.evt = C_CLEAR_STATS; - cmd.code = 0; - cmd.idx = 0; - ace_issue_cmd(regs, &cmd); - - cmd.evt = C_HOST_STATE; - cmd.code = C_C_STACK_UP; - cmd.idx = 0; - ace_issue_cmd(regs, &cmd); - - if (ap->jumbo && - !test_and_set_bit(0, &ap->jumbo_refill_busy)) - ace_load_jumbo_rx_ring(dev, RX_JUMBO_SIZE); - - if (dev->flags & IFF_PROMISC) { - cmd.evt = C_SET_PROMISC_MODE; - cmd.code = C_C_PROMISC_ENABLE; - cmd.idx = 0; - ace_issue_cmd(regs, &cmd); - - ap->promisc = 1; - }else - ap->promisc = 0; - ap->mcast_all = 0; - -#if 0 - cmd.evt = C_LNK_NEGOTIATION; - cmd.code = 0; - cmd.idx = 0; - ace_issue_cmd(regs, &cmd); -#endif - - netif_start_queue(dev); - - /* - * Setup the bottom half rx ring refill handler - */ - INIT_WORK(&ap->ace_bh_work, ace_bh_work); - return 0; -} - - -static int ace_close(struct net_device *dev) -{ - struct ace_private *ap = netdev_priv(dev); - struct ace_regs __iomem *regs = ap->regs; - struct cmd cmd; - unsigned long flags; - short i; - - /* - * Without (or before) releasing irq and stopping hardware, this - * is an absolute non-sense, by the way. It will be reset instantly - * by the first irq. - */ - netif_stop_queue(dev); - - - if (ap->promisc) { - cmd.evt = C_SET_PROMISC_MODE; - cmd.code = C_C_PROMISC_DISABLE; - cmd.idx = 0; - ace_issue_cmd(regs, &cmd); - ap->promisc = 0; - } - - cmd.evt = C_HOST_STATE; - cmd.code = C_C_STACK_DOWN; - cmd.idx = 0; - ace_issue_cmd(regs, &cmd); - - cancel_work_sync(&ap->ace_bh_work); - - /* - * Make sure one CPU is not processing packets while - * buffers are being released by another. - */ - - local_irq_save(flags); - ace_mask_irq(dev); - - for (i = 0; i < ACE_TX_RING_ENTRIES(ap); i++) { - struct sk_buff *skb; - struct tx_ring_info *info; - - info = ap->skb->tx_skbuff + i; - skb = info->skb; - - if (dma_unmap_len(info, maplen)) { - if (ACE_IS_TIGON_I(ap)) { - /* NB: TIGON_1 is special, tx_ring is in io space */ - struct tx_desc __iomem *tx; - tx = (__force struct tx_desc __iomem *) &ap->tx_ring[i]; - writel(0, &tx->addr.addrhi); - writel(0, &tx->addr.addrlo); - writel(0, &tx->flagsize); - } else - memset(ap->tx_ring + i, 0, - sizeof(struct tx_desc)); - dma_unmap_page(&ap->pdev->dev, - dma_unmap_addr(info, mapping), - dma_unmap_len(info, maplen), - DMA_TO_DEVICE); - dma_unmap_len_set(info, maplen, 0); - } - if (skb) { - dev_kfree_skb(skb); - info->skb = NULL; - } - } - - if (ap->jumbo) { - cmd.evt = C_RESET_JUMBO_RNG; - cmd.code = 0; - cmd.idx = 0; - ace_issue_cmd(regs, &cmd); - } - - ace_unmask_irq(dev); - local_irq_restore(flags); - - return 0; -} - - -static inline dma_addr_t -ace_map_tx_skb(struct ace_private *ap, struct sk_buff *skb, - struct sk_buff *tail, u32 idx) -{ - dma_addr_t mapping; - struct tx_ring_info *info; - - mapping = dma_map_page(&ap->pdev->dev, virt_to_page(skb->data), - offset_in_page(skb->data), skb->len, - DMA_TO_DEVICE); - - info = ap->skb->tx_skbuff + idx; - info->skb = tail; - dma_unmap_addr_set(info, mapping, mapping); - dma_unmap_len_set(info, maplen, skb->len); - return mapping; -} - - -static inline void -ace_load_tx_bd(struct ace_private *ap, struct tx_desc *desc, u64 addr, - u32 flagsize, u32 vlan_tag) -{ -#if !USE_TX_COAL_NOW - flagsize &= ~BD_FLG_COAL_NOW; -#endif - - if (ACE_IS_TIGON_I(ap)) { - struct tx_desc __iomem *io = (__force struct tx_desc __iomem *) desc; - writel(addr >> 32, &io->addr.addrhi); - writel(addr & 0xffffffff, &io->addr.addrlo); - writel(flagsize, &io->flagsize); - writel(vlan_tag, &io->vlanres); - } else { - desc->addr.addrhi = addr >> 32; - desc->addr.addrlo = addr; - desc->flagsize = flagsize; - desc->vlanres = vlan_tag; - } -} - - -static netdev_tx_t ace_start_xmit(struct sk_buff *skb, - struct net_device *dev) -{ - struct ace_private *ap = netdev_priv(dev); - struct ace_regs __iomem *regs = ap->regs; - struct tx_desc *desc; - u32 idx, flagsize; - unsigned long maxjiff = jiffies + 3*HZ; - -restart: - idx = ap->tx_prd; - - if (tx_ring_full(ap, ap->tx_ret_csm, idx)) - goto overflow; - - if (!skb_shinfo(skb)->nr_frags) { - dma_addr_t mapping; - u32 vlan_tag = 0; - - mapping = ace_map_tx_skb(ap, skb, skb, idx); - flagsize = (skb->len << 16) | (BD_FLG_END); - if (skb->ip_summed == CHECKSUM_PARTIAL) - flagsize |= BD_FLG_TCP_UDP_SUM; - if (skb_vlan_tag_present(skb)) { - flagsize |= BD_FLG_VLAN_TAG; - vlan_tag = skb_vlan_tag_get(skb); - } - desc = ap->tx_ring + idx; - idx = (idx + 1) % ACE_TX_RING_ENTRIES(ap); - - /* Look at ace_tx_int for explanations. */ - if (tx_ring_full(ap, ap->tx_ret_csm, idx)) - flagsize |= BD_FLG_COAL_NOW; - - ace_load_tx_bd(ap, desc, mapping, flagsize, vlan_tag); - } else { - dma_addr_t mapping; - u32 vlan_tag = 0; - int i; - - mapping = ace_map_tx_skb(ap, skb, NULL, idx); - flagsize = (skb_headlen(skb) << 16); - if (skb->ip_summed == CHECKSUM_PARTIAL) - flagsize |= BD_FLG_TCP_UDP_SUM; - if (skb_vlan_tag_present(skb)) { - flagsize |= BD_FLG_VLAN_TAG; - vlan_tag = skb_vlan_tag_get(skb); - } - - ace_load_tx_bd(ap, ap->tx_ring + idx, mapping, flagsize, vlan_tag); - - idx = (idx + 1) % ACE_TX_RING_ENTRIES(ap); - - for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { - const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; - struct tx_ring_info *info; - - info = ap->skb->tx_skbuff + idx; - desc = ap->tx_ring + idx; - - mapping = skb_frag_dma_map(&ap->pdev->dev, frag, 0, - skb_frag_size(frag), - DMA_TO_DEVICE); - - flagsize = skb_frag_size(frag) << 16; - if (skb->ip_summed == CHECKSUM_PARTIAL) - flagsize |= BD_FLG_TCP_UDP_SUM; - idx = (idx + 1) % ACE_TX_RING_ENTRIES(ap); - - if (i == skb_shinfo(skb)->nr_frags - 1) { - flagsize |= BD_FLG_END; - if (tx_ring_full(ap, ap->tx_ret_csm, idx)) - flagsize |= BD_FLG_COAL_NOW; - - /* - * Only the last fragment frees - * the skb! - */ - info->skb = skb; - } else { - info->skb = NULL; - } - dma_unmap_addr_set(info, mapping, mapping); - dma_unmap_len_set(info, maplen, skb_frag_size(frag)); - ace_load_tx_bd(ap, desc, mapping, flagsize, vlan_tag); - } - } - - wmb(); - ap->tx_prd = idx; - ace_set_txprd(regs, ap, idx); - - if (flagsize & BD_FLG_COAL_NOW) { - netif_stop_queue(dev); - - /* - * A TX-descriptor producer (an IRQ) might have gotten - * between, making the ring free again. Since xmit is - * serialized, this is the only situation we have to - * re-test. - */ - if (!tx_ring_full(ap, ap->tx_ret_csm, idx)) - netif_wake_queue(dev); - } - - return NETDEV_TX_OK; - -overflow: - /* - * This race condition is unavoidable with lock-free drivers. - * We wake up the queue _before_ tx_prd is advanced, so that we can - * enter hard_start_xmit too early, while tx ring still looks closed. - * This happens ~1-4 times per 100000 packets, so that we can allow - * to loop syncing to other CPU. Probably, we need an additional - * wmb() in ace_tx_intr as well. - * - * Note that this race is relieved by reserving one more entry - * in tx ring than it is necessary (see original non-SG driver). - * However, with SG we need to reserve 2*MAX_SKB_FRAGS+1, which - * is already overkill. - * - * Alternative is to return with 1 not throttling queue. In this - * case loop becomes longer, no more useful effects. - */ - if (time_before(jiffies, maxjiff)) { - barrier(); - cpu_relax(); - goto restart; - } - - /* The ring is stuck full. */ - printk(KERN_WARNING "%s: Transmit ring stuck full\n", dev->name); - return NETDEV_TX_BUSY; -} - - -static int ace_change_mtu(struct net_device *dev, int new_mtu) -{ - struct ace_private *ap = netdev_priv(dev); - struct ace_regs __iomem *regs = ap->regs; - - writel(new_mtu + ETH_HLEN + 4, ®s->IfMtu); - WRITE_ONCE(dev->mtu, new_mtu); - - if (new_mtu > ACE_STD_MTU) { - if (!(ap->jumbo)) { - printk(KERN_INFO "%s: Enabling Jumbo frame " - "support\n", dev->name); - ap->jumbo = 1; - if (!test_and_set_bit(0, &ap->jumbo_refill_busy)) - ace_load_jumbo_rx_ring(dev, RX_JUMBO_SIZE); - ace_set_rxtx_parms(dev, 1); - } - } else { - while (test_and_set_bit(0, &ap->jumbo_refill_busy)); - ace_sync_irq(dev->irq); - ace_set_rxtx_parms(dev, 0); - if (ap->jumbo) { - struct cmd cmd; - - cmd.evt = C_RESET_JUMBO_RNG; - cmd.code = 0; - cmd.idx = 0; - ace_issue_cmd(regs, &cmd); - } - } - - return 0; -} - -static int ace_get_link_ksettings(struct net_device *dev, - struct ethtool_link_ksettings *cmd) -{ - struct ace_private *ap = netdev_priv(dev); - struct ace_regs __iomem *regs = ap->regs; - u32 link; - u32 supported; - - memset(cmd, 0, sizeof(struct ethtool_link_ksettings)); - - supported = (SUPPORTED_10baseT_Half | SUPPORTED_10baseT_Full | - SUPPORTED_100baseT_Half | SUPPORTED_100baseT_Full | - SUPPORTED_1000baseT_Half | SUPPORTED_1000baseT_Full | - SUPPORTED_Autoneg | SUPPORTED_FIBRE); - - cmd->base.port = PORT_FIBRE; - - link = readl(®s->GigLnkState); - if (link & LNK_1000MB) { - cmd->base.speed = SPEED_1000; - } else { - link = readl(®s->FastLnkState); - if (link & LNK_100MB) - cmd->base.speed = SPEED_100; - else if (link & LNK_10MB) - cmd->base.speed = SPEED_10; - else - cmd->base.speed = 0; - } - if (link & LNK_FULL_DUPLEX) - cmd->base.duplex = DUPLEX_FULL; - else - cmd->base.duplex = DUPLEX_HALF; - - if (link & LNK_NEGOTIATE) - cmd->base.autoneg = AUTONEG_ENABLE; - else - cmd->base.autoneg = AUTONEG_DISABLE; - -#if 0 - /* - * Current struct ethtool_cmd is insufficient - */ - ecmd->trace = readl(®s->TuneTrace); - - ecmd->txcoal = readl(®s->TuneTxCoalTicks); - ecmd->rxcoal = readl(®s->TuneRxCoalTicks); -#endif - - ethtool_convert_legacy_u32_to_link_mode(cmd->link_modes.supported, - supported); - - return 0; -} - -static int ace_set_link_ksettings(struct net_device *dev, - const struct ethtool_link_ksettings *cmd) -{ - struct ace_private *ap = netdev_priv(dev); - struct ace_regs __iomem *regs = ap->regs; - u32 link, speed; - - link = readl(®s->GigLnkState); - if (link & LNK_1000MB) - speed = SPEED_1000; - else { - link = readl(®s->FastLnkState); - if (link & LNK_100MB) - speed = SPEED_100; - else if (link & LNK_10MB) - speed = SPEED_10; - else - speed = SPEED_100; - } - - link = LNK_ENABLE | LNK_1000MB | LNK_100MB | LNK_10MB | - LNK_RX_FLOW_CTL_Y | LNK_NEG_FCTL; - if (!ACE_IS_TIGON_I(ap)) - link |= LNK_TX_FLOW_CTL_Y; - if (cmd->base.autoneg == AUTONEG_ENABLE) - link |= LNK_NEGOTIATE; - if (cmd->base.speed != speed) { - link &= ~(LNK_1000MB | LNK_100MB | LNK_10MB); - switch (cmd->base.speed) { - case SPEED_1000: - link |= LNK_1000MB; - break; - case SPEED_100: - link |= LNK_100MB; - break; - case SPEED_10: - link |= LNK_10MB; - break; - } - } - - if (cmd->base.duplex == DUPLEX_FULL) - link |= LNK_FULL_DUPLEX; - - if (link != ap->link) { - struct cmd cmd; - printk(KERN_INFO "%s: Renegotiating link state\n", - dev->name); - - ap->link = link; - writel(link, ®s->TuneLink); - if (!ACE_IS_TIGON_I(ap)) - writel(link, ®s->TuneFastLink); - wmb(); - - cmd.evt = C_LNK_NEGOTIATION; - cmd.code = 0; - cmd.idx = 0; - ace_issue_cmd(regs, &cmd); - } - return 0; -} - -static void ace_get_drvinfo(struct net_device *dev, - struct ethtool_drvinfo *info) -{ - struct ace_private *ap = netdev_priv(dev); - - strscpy(info->driver, "acenic", sizeof(info->driver)); - snprintf(info->fw_version, sizeof(info->version), "%i.%i.%i", - ap->firmware_major, ap->firmware_minor, ap->firmware_fix); - - if (ap->pdev) - strscpy(info->bus_info, pci_name(ap->pdev), - sizeof(info->bus_info)); - -} - -/* - * Set the hardware MAC address. - */ -static int ace_set_mac_addr(struct net_device *dev, void *p) -{ - struct ace_private *ap = netdev_priv(dev); - struct ace_regs __iomem *regs = ap->regs; - struct sockaddr *addr=p; - const u8 *da; - struct cmd cmd; - - if(netif_running(dev)) - return -EBUSY; - - eth_hw_addr_set(dev, addr->sa_data); - - da = (const u8 *)dev->dev_addr; - - writel(da[0] << 8 | da[1], ®s->MacAddrHi); - writel((da[2] << 24) | (da[3] << 16) | (da[4] << 8) | da[5], - ®s->MacAddrLo); - - cmd.evt = C_SET_MAC_ADDR; - cmd.code = 0; - cmd.idx = 0; - ace_issue_cmd(regs, &cmd); - - return 0; -} - - -static void ace_set_multicast_list(struct net_device *dev) -{ - struct ace_private *ap = netdev_priv(dev); - struct ace_regs __iomem *regs = ap->regs; - struct cmd cmd; - - if ((dev->flags & IFF_ALLMULTI) && !(ap->mcast_all)) { - cmd.evt = C_SET_MULTICAST_MODE; - cmd.code = C_C_MCAST_ENABLE; - cmd.idx = 0; - ace_issue_cmd(regs, &cmd); - ap->mcast_all = 1; - } else if (ap->mcast_all) { - cmd.evt = C_SET_MULTICAST_MODE; - cmd.code = C_C_MCAST_DISABLE; - cmd.idx = 0; - ace_issue_cmd(regs, &cmd); - ap->mcast_all = 0; - } - - if ((dev->flags & IFF_PROMISC) && !(ap->promisc)) { - cmd.evt = C_SET_PROMISC_MODE; - cmd.code = C_C_PROMISC_ENABLE; - cmd.idx = 0; - ace_issue_cmd(regs, &cmd); - ap->promisc = 1; - }else if (!(dev->flags & IFF_PROMISC) && (ap->promisc)) { - cmd.evt = C_SET_PROMISC_MODE; - cmd.code = C_C_PROMISC_DISABLE; - cmd.idx = 0; - ace_issue_cmd(regs, &cmd); - ap->promisc = 0; - } - - /* - * For the time being multicast relies on the upper layers - * filtering it properly. The Firmware does not allow one to - * set the entire multicast list at a time and keeping track of - * it here is going to be messy. - */ - if (!netdev_mc_empty(dev) && !ap->mcast_all) { - cmd.evt = C_SET_MULTICAST_MODE; - cmd.code = C_C_MCAST_ENABLE; - cmd.idx = 0; - ace_issue_cmd(regs, &cmd); - }else if (!ap->mcast_all) { - cmd.evt = C_SET_MULTICAST_MODE; - cmd.code = C_C_MCAST_DISABLE; - cmd.idx = 0; - ace_issue_cmd(regs, &cmd); - } -} - - -static struct net_device_stats *ace_get_stats(struct net_device *dev) -{ - struct ace_private *ap = netdev_priv(dev); - struct ace_mac_stats __iomem *mac_stats = - (struct ace_mac_stats __iomem *)ap->regs->Stats; - - dev->stats.rx_missed_errors = readl(&mac_stats->drop_space); - dev->stats.multicast = readl(&mac_stats->kept_mc); - dev->stats.collisions = readl(&mac_stats->coll); - - return &dev->stats; -} - - -static void ace_copy(struct ace_regs __iomem *regs, const __be32 *src, - u32 dest, int size) -{ - void __iomem *tdest; - short tsize, i; - - if (size <= 0) - return; - - while (size > 0) { - tsize = min_t(u32, ((~dest & (ACE_WINDOW_SIZE - 1)) + 1), - min_t(u32, size, ACE_WINDOW_SIZE)); - tdest = (void __iomem *) ®s->Window + - (dest & (ACE_WINDOW_SIZE - 1)); - writel(dest & ~(ACE_WINDOW_SIZE - 1), ®s->WinBase); - for (i = 0; i < (tsize / 4); i++) { - /* Firmware is big-endian */ - writel(be32_to_cpup(src), tdest); - src++; - tdest += 4; - dest += 4; - size -= 4; - } - } -} - - -static void ace_clear(struct ace_regs __iomem *regs, u32 dest, int size) -{ - void __iomem *tdest; - short tsize = 0, i; - - if (size <= 0) - return; - - while (size > 0) { - tsize = min_t(u32, ((~dest & (ACE_WINDOW_SIZE - 1)) + 1), - min_t(u32, size, ACE_WINDOW_SIZE)); - tdest = (void __iomem *) ®s->Window + - (dest & (ACE_WINDOW_SIZE - 1)); - writel(dest & ~(ACE_WINDOW_SIZE - 1), ®s->WinBase); - - for (i = 0; i < (tsize / 4); i++) { - writel(0, tdest + i*4); - } - - dest += tsize; - size -= tsize; - } -} - - -/* - * Download the firmware into the SRAM on the NIC - * - * This operation requires the NIC to be halted and is performed with - * interrupts disabled and with the spinlock hold. - */ -static int ace_load_firmware(struct net_device *dev) -{ - const struct firmware *fw; - const char *fw_name = "acenic/tg2.bin"; - struct ace_private *ap = netdev_priv(dev); - struct ace_regs __iomem *regs = ap->regs; - const __be32 *fw_data; - u32 load_addr; - int ret; - - if (!(readl(®s->CpuCtrl) & CPU_HALTED)) { - printk(KERN_ERR "%s: trying to download firmware while the " - "CPU is running!\n", ap->name); - return -EFAULT; - } - - if (ACE_IS_TIGON_I(ap)) - fw_name = "acenic/tg1.bin"; - - ret = request_firmware(&fw, fw_name, &ap->pdev->dev); - if (ret) { - printk(KERN_ERR "%s: Failed to load firmware \"%s\"\n", - ap->name, fw_name); - return ret; - } - - fw_data = (void *)fw->data; - - /* Firmware blob starts with version numbers, followed by - load and start address. Remainder is the blob to be loaded - contiguously from load address. We don't bother to represent - the BSS/SBSS sections any more, since we were clearing the - whole thing anyway. */ - ap->firmware_major = fw->data[0]; - ap->firmware_minor = fw->data[1]; - ap->firmware_fix = fw->data[2]; - - ap->firmware_start = be32_to_cpu(fw_data[1]); - if (ap->firmware_start < 0x4000 || ap->firmware_start >= 0x80000) { - printk(KERN_ERR "%s: bogus load address %08x in \"%s\"\n", - ap->name, ap->firmware_start, fw_name); - ret = -EINVAL; - goto out; - } - - load_addr = be32_to_cpu(fw_data[2]); - if (load_addr < 0x4000 || load_addr >= 0x80000) { - printk(KERN_ERR "%s: bogus load address %08x in \"%s\"\n", - ap->name, load_addr, fw_name); - ret = -EINVAL; - goto out; - } - - /* - * Do not try to clear more than 512KiB or we end up seeing - * funny things on NICs with only 512KiB SRAM - */ - ace_clear(regs, 0x2000, 0x80000-0x2000); - ace_copy(regs, &fw_data[3], load_addr, fw->size-12); - out: - release_firmware(fw); - return ret; -} - - -/* - * The eeprom on the AceNIC is an Atmel i2c EEPROM. - * - * Accessing the EEPROM is `interesting' to say the least - don't read - * this code right after dinner. - * - * This is all about black magic and bit-banging the device .... I - * wonder in what hospital they have put the guy who designed the i2c - * specs. - * - * Oh yes, this is only the beginning! - * - * Thanks to Stevarino Webinski for helping tracking down the bugs in the - * code i2c readout code by beta testing all my hacks. - */ -static void eeprom_start(struct ace_regs __iomem *regs) -{ - u32 local; - - readl(®s->LocalCtrl); - udelay(ACE_SHORT_DELAY); - local = readl(®s->LocalCtrl); - local |= EEPROM_DATA_OUT | EEPROM_WRITE_ENABLE; - writel(local, ®s->LocalCtrl); - readl(®s->LocalCtrl); - mb(); - udelay(ACE_SHORT_DELAY); - local |= EEPROM_CLK_OUT; - writel(local, ®s->LocalCtrl); - readl(®s->LocalCtrl); - mb(); - udelay(ACE_SHORT_DELAY); - local &= ~EEPROM_DATA_OUT; - writel(local, ®s->LocalCtrl); - readl(®s->LocalCtrl); - mb(); - udelay(ACE_SHORT_DELAY); - local &= ~EEPROM_CLK_OUT; - writel(local, ®s->LocalCtrl); - readl(®s->LocalCtrl); - mb(); -} - - -static void eeprom_prep(struct ace_regs __iomem *regs, u8 magic) -{ - short i; - u32 local; - - udelay(ACE_SHORT_DELAY); - local = readl(®s->LocalCtrl); - local &= ~EEPROM_DATA_OUT; - local |= EEPROM_WRITE_ENABLE; - writel(local, ®s->LocalCtrl); - readl(®s->LocalCtrl); - mb(); - - for (i = 0; i < 8; i++, magic <<= 1) { - udelay(ACE_SHORT_DELAY); - if (magic & 0x80) - local |= EEPROM_DATA_OUT; - else - local &= ~EEPROM_DATA_OUT; - writel(local, ®s->LocalCtrl); - readl(®s->LocalCtrl); - mb(); - - udelay(ACE_SHORT_DELAY); - local |= EEPROM_CLK_OUT; - writel(local, ®s->LocalCtrl); - readl(®s->LocalCtrl); - mb(); - udelay(ACE_SHORT_DELAY); - local &= ~(EEPROM_CLK_OUT | EEPROM_DATA_OUT); - writel(local, ®s->LocalCtrl); - readl(®s->LocalCtrl); - mb(); - } -} - - -static int eeprom_check_ack(struct ace_regs __iomem *regs) -{ - int state; - u32 local; - - local = readl(®s->LocalCtrl); - local &= ~EEPROM_WRITE_ENABLE; - writel(local, ®s->LocalCtrl); - readl(®s->LocalCtrl); - mb(); - udelay(ACE_LONG_DELAY); - local |= EEPROM_CLK_OUT; - writel(local, ®s->LocalCtrl); - readl(®s->LocalCtrl); - mb(); - udelay(ACE_SHORT_DELAY); - /* sample data in middle of high clk */ - state = (readl(®s->LocalCtrl) & EEPROM_DATA_IN) != 0; - udelay(ACE_SHORT_DELAY); - mb(); - writel(readl(®s->LocalCtrl) & ~EEPROM_CLK_OUT, ®s->LocalCtrl); - readl(®s->LocalCtrl); - mb(); - - return state; -} - - -static void eeprom_stop(struct ace_regs __iomem *regs) -{ - u32 local; - - udelay(ACE_SHORT_DELAY); - local = readl(®s->LocalCtrl); - local |= EEPROM_WRITE_ENABLE; - writel(local, ®s->LocalCtrl); - readl(®s->LocalCtrl); - mb(); - udelay(ACE_SHORT_DELAY); - local &= ~EEPROM_DATA_OUT; - writel(local, ®s->LocalCtrl); - readl(®s->LocalCtrl); - mb(); - udelay(ACE_SHORT_DELAY); - local |= EEPROM_CLK_OUT; - writel(local, ®s->LocalCtrl); - readl(®s->LocalCtrl); - mb(); - udelay(ACE_SHORT_DELAY); - local |= EEPROM_DATA_OUT; - writel(local, ®s->LocalCtrl); - readl(®s->LocalCtrl); - mb(); - udelay(ACE_LONG_DELAY); - local &= ~EEPROM_CLK_OUT; - writel(local, ®s->LocalCtrl); - mb(); -} - - -/* - * Read a whole byte from the EEPROM. - */ -static int read_eeprom_byte(struct net_device *dev, unsigned long offset) -{ - struct ace_private *ap = netdev_priv(dev); - struct ace_regs __iomem *regs = ap->regs; - unsigned long flags; - u32 local; - int result = 0; - short i; - - /* - * Don't take interrupts on this CPU will bit banging - * the %#%#@$ I2C device - */ - local_irq_save(flags); - - eeprom_start(regs); - - eeprom_prep(regs, EEPROM_WRITE_SELECT); - if (eeprom_check_ack(regs)) { - local_irq_restore(flags); - printk(KERN_ERR "%s: Unable to sync eeprom\n", ap->name); - result = -EIO; - goto eeprom_read_error; - } - - eeprom_prep(regs, (offset >> 8) & 0xff); - if (eeprom_check_ack(regs)) { - local_irq_restore(flags); - printk(KERN_ERR "%s: Unable to set address byte 0\n", - ap->name); - result = -EIO; - goto eeprom_read_error; - } - - eeprom_prep(regs, offset & 0xff); - if (eeprom_check_ack(regs)) { - local_irq_restore(flags); - printk(KERN_ERR "%s: Unable to set address byte 1\n", - ap->name); - result = -EIO; - goto eeprom_read_error; - } - - eeprom_start(regs); - eeprom_prep(regs, EEPROM_READ_SELECT); - if (eeprom_check_ack(regs)) { - local_irq_restore(flags); - printk(KERN_ERR "%s: Unable to set READ_SELECT\n", - ap->name); - result = -EIO; - goto eeprom_read_error; - } - - for (i = 0; i < 8; i++) { - local = readl(®s->LocalCtrl); - local &= ~EEPROM_WRITE_ENABLE; - writel(local, ®s->LocalCtrl); - readl(®s->LocalCtrl); - udelay(ACE_LONG_DELAY); - mb(); - local |= EEPROM_CLK_OUT; - writel(local, ®s->LocalCtrl); - readl(®s->LocalCtrl); - mb(); - udelay(ACE_SHORT_DELAY); - /* sample data mid high clk */ - result = (result << 1) | - ((readl(®s->LocalCtrl) & EEPROM_DATA_IN) != 0); - udelay(ACE_SHORT_DELAY); - mb(); - local = readl(®s->LocalCtrl); - local &= ~EEPROM_CLK_OUT; - writel(local, ®s->LocalCtrl); - readl(®s->LocalCtrl); - udelay(ACE_SHORT_DELAY); - mb(); - if (i == 7) { - local |= EEPROM_WRITE_ENABLE; - writel(local, ®s->LocalCtrl); - readl(®s->LocalCtrl); - mb(); - udelay(ACE_SHORT_DELAY); - } - } - - local |= EEPROM_DATA_OUT; - writel(local, ®s->LocalCtrl); - readl(®s->LocalCtrl); - mb(); - udelay(ACE_SHORT_DELAY); - writel(readl(®s->LocalCtrl) | EEPROM_CLK_OUT, ®s->LocalCtrl); - readl(®s->LocalCtrl); - udelay(ACE_LONG_DELAY); - writel(readl(®s->LocalCtrl) & ~EEPROM_CLK_OUT, ®s->LocalCtrl); - readl(®s->LocalCtrl); - mb(); - udelay(ACE_SHORT_DELAY); - eeprom_stop(regs); - - local_irq_restore(flags); - out: - return result; - - eeprom_read_error: - printk(KERN_ERR "%s: Unable to read eeprom byte 0x%02lx\n", - ap->name, offset); - goto out; -} - -module_pci_driver(acenic_pci_driver); diff --git a/drivers/net/ethernet/alteon/acenic.h b/drivers/net/ethernet/alteon/acenic.h deleted file mode 100644 index 0e45a97b9c9b..000000000000 --- a/drivers/net/ethernet/alteon/acenic.h +++ /dev/null @@ -1,791 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _ACENIC_H_ -#define _ACENIC_H_ -#include <linux/interrupt.h> -#include <linux/workqueue.h> - -/* - * Generate TX index update each time, when TX ring is closed. - * Normally, this is not useful, because results in more dma (and irqs - * without TX_COAL_INTS_ONLY). - */ -#define USE_TX_COAL_NOW 0 - -/* - * Addressing: - * - * The Tigon uses 64-bit host addresses, regardless of their actual - * length, and it expects a big-endian format. For 32 bit systems the - * upper 32 bits of the address are simply ignored (zero), however for - * little endian 64 bit systems (Alpha) this looks strange with the - * two parts of the address word being swapped. - * - * The addresses are split in two 32 bit words for all architectures - * as some of them are in PCI shared memory and it is necessary to use - * readl/writel to access them. - * - * The addressing code is derived from Pete Wyckoff's work, but - * modified to deal properly with readl/writel usage. - */ - -struct ace_regs { - u32 pad0[16]; /* PCI control registers */ - - u32 HostCtrl; /* 0x40 */ - u32 LocalCtrl; - - u32 pad1[2]; - - u32 MiscCfg; /* 0x50 */ - - u32 pad2[2]; - - u32 PciState; - - u32 pad3[2]; /* 0x60 */ - - u32 WinBase; - u32 WinData; - - u32 pad4[12]; /* 0x70 */ - - u32 DmaWriteState; /* 0xa0 */ - u32 pad5[3]; - u32 DmaReadState; /* 0xb0 */ - - u32 pad6[26]; - - u32 AssistState; - - u32 pad7[8]; /* 0x120 */ - - u32 CpuCtrl; /* 0x140 */ - u32 Pc; - - u32 pad8[3]; - - u32 SramAddr; /* 0x154 */ - u32 SramData; - - u32 pad9[49]; - - u32 MacRxState; /* 0x220 */ - - u32 pad10[7]; - - u32 CpuBCtrl; /* 0x240 */ - u32 PcB; - - u32 pad11[3]; - - u32 SramBAddr; /* 0x254 */ - u32 SramBData; - - u32 pad12[105]; - - u32 pad13[32]; /* 0x400 */ - u32 Stats[32]; - - u32 Mb0Hi; /* 0x500 */ - u32 Mb0Lo; - u32 Mb1Hi; - u32 CmdPrd; - u32 Mb2Hi; - u32 TxPrd; - u32 Mb3Hi; - u32 RxStdPrd; - u32 Mb4Hi; - u32 RxJumboPrd; - u32 Mb5Hi; - u32 RxMiniPrd; - u32 Mb6Hi; - u32 Mb6Lo; - u32 Mb7Hi; - u32 Mb7Lo; - u32 Mb8Hi; - u32 Mb8Lo; - u32 Mb9Hi; - u32 Mb9Lo; - u32 MbAHi; - u32 MbALo; - u32 MbBHi; - u32 MbBLo; - u32 MbCHi; - u32 MbCLo; - u32 MbDHi; - u32 MbDLo; - u32 MbEHi; - u32 MbELo; - u32 MbFHi; - u32 MbFLo; - - u32 pad14[32]; - - u32 MacAddrHi; /* 0x600 */ - u32 MacAddrLo; - u32 InfoPtrHi; - u32 InfoPtrLo; - u32 MultiCastHi; /* 0x610 */ - u32 MultiCastLo; - u32 ModeStat; - u32 DmaReadCfg; - u32 DmaWriteCfg; /* 0x620 */ - u32 TxBufRat; - u32 EvtCsm; - u32 CmdCsm; - u32 TuneRxCoalTicks;/* 0x630 */ - u32 TuneTxCoalTicks; - u32 TuneStatTicks; - u32 TuneMaxTxDesc; - u32 TuneMaxRxDesc; /* 0x640 */ - u32 TuneTrace; - u32 TuneLink; - u32 TuneFastLink; - u32 TracePtr; /* 0x650 */ - u32 TraceStrt; - u32 TraceLen; - u32 IfIdx; - u32 IfMtu; /* 0x660 */ - u32 MaskInt; - u32 GigLnkState; - u32 FastLnkState; - u32 pad16[4]; /* 0x670 */ - u32 RxRetCsm; /* 0x680 */ - - u32 pad17[31]; - - u32 CmdRng[64]; /* 0x700 */ - u32 Window[0x200]; -}; - - -typedef struct { - u32 addrhi; - u32 addrlo; -} aceaddr; - - -#define ACE_WINDOW_SIZE 0x800 - -#define ACE_JUMBO_MTU 9000 -#define ACE_STD_MTU 1500 - -#define ACE_TRACE_SIZE 0x8000 - -/* - * Host control register bits. - */ - -#define IN_INT 0x01 -#define CLR_INT 0x02 -#define HW_RESET 0x08 -#define BYTE_SWAP 0x10 -#define WORD_SWAP 0x20 -#define MASK_INTS 0x40 - -/* - * Local control register bits. - */ - -#define EEPROM_DATA_IN 0x800000 -#define EEPROM_DATA_OUT 0x400000 -#define EEPROM_WRITE_ENABLE 0x200000 -#define EEPROM_CLK_OUT 0x100000 - -#define EEPROM_BASE 0xa0000000 - -#define EEPROM_WRITE_SELECT 0xa0 -#define EEPROM_READ_SELECT 0xa1 - -#define SRAM_BANK_512K 0x200 - - -/* - * udelay() values for when clocking the eeprom - */ -#define ACE_SHORT_DELAY 2 -#define ACE_LONG_DELAY 4 - - -/* - * Misc Config bits - */ - -#define SYNC_SRAM_TIMING 0x100000 - - -/* - * CPU state bits. - */ - -#define CPU_RESET 0x01 -#define CPU_TRACE 0x02 -#define CPU_PROM_FAILED 0x10 -#define CPU_HALT 0x00010000 -#define CPU_HALTED 0xffff0000 - - -/* - * PCI State bits. - */ - -#define DMA_READ_MAX_4 0x04 -#define DMA_READ_MAX_16 0x08 -#define DMA_READ_MAX_32 0x0c -#define DMA_READ_MAX_64 0x10 -#define DMA_READ_MAX_128 0x14 -#define DMA_READ_MAX_256 0x18 -#define DMA_READ_MAX_1K 0x1c -#define DMA_WRITE_MAX_4 0x20 -#define DMA_WRITE_MAX_16 0x40 -#define DMA_WRITE_MAX_32 0x60 -#define DMA_WRITE_MAX_64 0x80 -#define DMA_WRITE_MAX_128 0xa0 -#define DMA_WRITE_MAX_256 0xc0 -#define DMA_WRITE_MAX_1K 0xe0 -#define DMA_READ_WRITE_MASK 0xfc -#define MEM_READ_MULTIPLE 0x00020000 -#define PCI_66MHZ 0x00080000 -#define PCI_32BIT 0x00100000 -#define DMA_WRITE_ALL_ALIGN 0x00800000 -#define READ_CMD_MEM 0x06000000 -#define WRITE_CMD_MEM 0x70000000 - - -/* - * Mode status - */ - -#define ACE_BYTE_SWAP_BD 0x02 -#define ACE_WORD_SWAP_BD 0x04 /* not actually used */ -#define ACE_WARN 0x08 -#define ACE_BYTE_SWAP_DMA 0x10 -#define ACE_NO_JUMBO_FRAG 0x200 -#define ACE_FATAL 0x40000000 - - -/* - * DMA config - */ - -#define DMA_THRESH_1W 0x10 -#define DMA_THRESH_2W 0x20 -#define DMA_THRESH_4W 0x40 -#define DMA_THRESH_8W 0x80 -#define DMA_THRESH_16W 0x100 -#define DMA_THRESH_32W 0x0 /* not described in doc, but exists. */ - - -/* - * Tuning parameters - */ - -#define TICKS_PER_SEC 1000000 - - -/* - * Link bits - */ - -#define LNK_PREF 0x00008000 -#define LNK_10MB 0x00010000 -#define LNK_100MB 0x00020000 -#define LNK_1000MB 0x00040000 -#define LNK_FULL_DUPLEX 0x00080000 -#define LNK_HALF_DUPLEX 0x00100000 -#define LNK_TX_FLOW_CTL_Y 0x00200000 -#define LNK_NEG_ADVANCED 0x00400000 -#define LNK_RX_FLOW_CTL_Y 0x00800000 -#define LNK_NIC 0x01000000 -#define LNK_JAM 0x02000000 -#define LNK_JUMBO 0x04000000 -#define LNK_ALTEON 0x08000000 -#define LNK_NEG_FCTL 0x10000000 -#define LNK_NEGOTIATE 0x20000000 -#define LNK_ENABLE 0x40000000 -#define LNK_UP 0x80000000 - - -/* - * Event definitions - */ - -#define EVT_RING_ENTRIES 256 -#define EVT_RING_SIZE (EVT_RING_ENTRIES * sizeof(struct event)) - -struct event { -#ifdef __LITTLE_ENDIAN_BITFIELD - u32 idx:12; - u32 code:12; - u32 evt:8; -#else - u32 evt:8; - u32 code:12; - u32 idx:12; -#endif - u32 pad; -}; - - -/* - * Events - */ - -#define E_FW_RUNNING 0x01 -#define E_STATS_UPDATED 0x04 - -#define E_STATS_UPDATE 0x04 - -#define E_LNK_STATE 0x06 -#define E_C_LINK_UP 0x01 -#define E_C_LINK_DOWN 0x02 -#define E_C_LINK_10_100 0x03 - -#define E_ERROR 0x07 -#define E_C_ERR_INVAL_CMD 0x01 -#define E_C_ERR_UNIMP_CMD 0x02 -#define E_C_ERR_BAD_CFG 0x03 - -#define E_MCAST_LIST 0x08 -#define E_C_MCAST_ADDR_ADD 0x01 -#define E_C_MCAST_ADDR_DEL 0x02 - -#define E_RESET_JUMBO_RNG 0x09 - - -/* - * Commands - */ - -#define CMD_RING_ENTRIES 64 - -struct cmd { -#ifdef __LITTLE_ENDIAN_BITFIELD - u32 idx:12; - u32 code:12; - u32 evt:8; -#else - u32 evt:8; - u32 code:12; - u32 idx:12; -#endif -}; - - -#define C_HOST_STATE 0x01 -#define C_C_STACK_UP 0x01 -#define C_C_STACK_DOWN 0x02 - -#define C_FDR_FILTERING 0x02 -#define C_C_FDR_FILT_ENABLE 0x01 -#define C_C_FDR_FILT_DISABLE 0x02 - -#define C_SET_RX_PRD_IDX 0x03 -#define C_UPDATE_STATS 0x04 -#define C_RESET_JUMBO_RNG 0x05 -#define C_ADD_MULTICAST_ADDR 0x08 -#define C_DEL_MULTICAST_ADDR 0x09 - -#define C_SET_PROMISC_MODE 0x0a -#define C_C_PROMISC_ENABLE 0x01 -#define C_C_PROMISC_DISABLE 0x02 - -#define C_LNK_NEGOTIATION 0x0b -#define C_C_NEGOTIATE_BOTH 0x00 -#define C_C_NEGOTIATE_GIG 0x01 -#define C_C_NEGOTIATE_10_100 0x02 - -#define C_SET_MAC_ADDR 0x0c -#define C_CLEAR_PROFILE 0x0d - -#define C_SET_MULTICAST_MODE 0x0e -#define C_C_MCAST_ENABLE 0x01 -#define C_C_MCAST_DISABLE 0x02 - -#define C_CLEAR_STATS 0x0f -#define C_SET_RX_JUMBO_PRD_IDX 0x10 -#define C_REFRESH_STATS 0x11 - - -/* - * Descriptor flags - */ -#define BD_FLG_TCP_UDP_SUM 0x01 -#define BD_FLG_IP_SUM 0x02 -#define BD_FLG_END 0x04 -#define BD_FLG_MORE 0x08 -#define BD_FLG_JUMBO 0x10 -#define BD_FLG_UCAST 0x20 -#define BD_FLG_MCAST 0x40 -#define BD_FLG_BCAST 0x60 -#define BD_FLG_TYP_MASK 0x60 -#define BD_FLG_IP_FRAG 0x80 -#define BD_FLG_IP_FRAG_END 0x100 -#define BD_FLG_VLAN_TAG 0x200 -#define BD_FLG_FRAME_ERROR 0x400 -#define BD_FLG_COAL_NOW 0x800 -#define BD_FLG_MINI 0x1000 - - -/* - * Ring Control block flags - */ -#define RCB_FLG_TCP_UDP_SUM 0x01 -#define RCB_FLG_IP_SUM 0x02 -#define RCB_FLG_NO_PSEUDO_HDR 0x08 -#define RCB_FLG_VLAN_ASSIST 0x10 -#define RCB_FLG_COAL_INT_ONLY 0x20 -#define RCB_FLG_TX_HOST_RING 0x40 -#define RCB_FLG_IEEE_SNAP_SUM 0x80 -#define RCB_FLG_EXT_RX_BD 0x100 -#define RCB_FLG_RNG_DISABLE 0x200 - - -/* - * TX ring - maximum TX ring entries for Tigon I's is 128 - */ -#define MAX_TX_RING_ENTRIES 256 -#define TIGON_I_TX_RING_ENTRIES 128 -#define TX_RING_SIZE (MAX_TX_RING_ENTRIES * sizeof(struct tx_desc)) -#define TX_RING_BASE 0x3800 - -struct tx_desc{ - aceaddr addr; - u32 flagsize; -#if 0 -/* - * This is in PCI shared mem and must be accessed with readl/writel - * real layout is: - */ -#if __LITTLE_ENDIAN - u16 flags; - u16 size; - u16 vlan; - u16 reserved; -#else - u16 size; - u16 flags; - u16 reserved; - u16 vlan; -#endif -#endif - u32 vlanres; -}; - - -#define RX_STD_RING_ENTRIES 512 -#define RX_STD_RING_SIZE (RX_STD_RING_ENTRIES * sizeof(struct rx_desc)) - -#define RX_JUMBO_RING_ENTRIES 256 -#define RX_JUMBO_RING_SIZE (RX_JUMBO_RING_ENTRIES *sizeof(struct rx_desc)) - -#define RX_MINI_RING_ENTRIES 1024 -#define RX_MINI_RING_SIZE (RX_MINI_RING_ENTRIES *sizeof(struct rx_desc)) - -#define RX_RETURN_RING_ENTRIES 2048 -#define RX_RETURN_RING_SIZE (RX_MAX_RETURN_RING_ENTRIES * \ - sizeof(struct rx_desc)) - -struct rx_desc{ - aceaddr addr; -#ifdef __LITTLE_ENDIAN - u16 size; - u16 idx; -#else - u16 idx; - u16 size; -#endif -#ifdef __LITTLE_ENDIAN - u16 flags; - u16 type; -#else - u16 type; - u16 flags; -#endif -#ifdef __LITTLE_ENDIAN - u16 tcp_udp_csum; - u16 ip_csum; -#else - u16 ip_csum; - u16 tcp_udp_csum; -#endif -#ifdef __LITTLE_ENDIAN - u16 vlan; - u16 err_flags; -#else - u16 err_flags; - u16 vlan; -#endif - u32 reserved; - u32 opague; -}; - - -/* - * This struct is shared with the NIC firmware. - */ -struct ring_ctrl { - aceaddr rngptr; -#ifdef __LITTLE_ENDIAN - u16 flags; - u16 max_len; -#else - u16 max_len; - u16 flags; -#endif - u32 pad; -}; - - -struct ace_mac_stats { - u32 excess_colls; - u32 coll_1; - u32 coll_2; - u32 coll_3; - u32 coll_4; - u32 coll_5; - u32 coll_6; - u32 coll_7; - u32 coll_8; - u32 coll_9; - u32 coll_10; - u32 coll_11; - u32 coll_12; - u32 coll_13; - u32 coll_14; - u32 coll_15; - u32 late_coll; - u32 defers; - u32 crc_err; - u32 underrun; - u32 crs_err; - u32 pad[3]; - u32 drop_ula; - u32 drop_mc; - u32 drop_fc; - u32 drop_space; - u32 coll; - u32 kept_bc; - u32 kept_mc; - u32 kept_uc; -}; - - -struct ace_info { - union { - u32 stats[256]; - } s; - struct ring_ctrl evt_ctrl; - struct ring_ctrl cmd_ctrl; - struct ring_ctrl tx_ctrl; - struct ring_ctrl rx_std_ctrl; - struct ring_ctrl rx_jumbo_ctrl; - struct ring_ctrl rx_mini_ctrl; - struct ring_ctrl rx_return_ctrl; - aceaddr evt_prd_ptr; - aceaddr rx_ret_prd_ptr; - aceaddr tx_csm_ptr; - aceaddr stats2_ptr; -}; - - -struct ring_info { - struct sk_buff *skb; - DEFINE_DMA_UNMAP_ADDR(mapping); -}; - - -/* - * Funny... As soon as we add maplen on alpha, it starts to work - * much slower. Hmm... is it because struct does not fit to one cacheline? - * So, split tx_ring_info. - */ -struct tx_ring_info { - struct sk_buff *skb; - DEFINE_DMA_UNMAP_ADDR(mapping); - DEFINE_DMA_UNMAP_LEN(maplen); -}; - - -/* - * struct ace_skb holding the rings of skb's. This is an awful lot of - * pointers, but I don't see any other smart mode to do this in an - * efficient manner ;-( - */ -struct ace_skb -{ - struct tx_ring_info tx_skbuff[MAX_TX_RING_ENTRIES]; - struct ring_info rx_std_skbuff[RX_STD_RING_ENTRIES]; - struct ring_info rx_mini_skbuff[RX_MINI_RING_ENTRIES]; - struct ring_info rx_jumbo_skbuff[RX_JUMBO_RING_ENTRIES]; -}; - - -/* - * Struct private for the AceNIC. - * - * Elements are grouped so variables used by the tx handling goes - * together, and will go into the same cache lines etc. in order to - * avoid cache line contention between the rx and tx handling on SMP. - * - * Frequently accessed variables are put at the beginning of the - * struct to help the compiler generate better/shorter code. - */ -struct ace_private -{ - struct net_device *ndev; /* backpointer */ - struct ace_info *info; - struct ace_regs __iomem *regs; /* register base */ - struct ace_skb *skb; - dma_addr_t info_dma; /* 32/64 bit */ - - int version, link; - int promisc, mcast_all; - - /* - * TX elements - */ - struct tx_desc *tx_ring; - u32 tx_prd; - volatile u32 tx_ret_csm; - int tx_ring_entries; - - /* - * RX elements - */ - unsigned long std_refill_busy - __attribute__ ((aligned (SMP_CACHE_BYTES))); - unsigned long mini_refill_busy, jumbo_refill_busy; - atomic_t cur_rx_bufs; - atomic_t cur_mini_bufs; - atomic_t cur_jumbo_bufs; - u32 rx_std_skbprd, rx_mini_skbprd, rx_jumbo_skbprd; - u32 cur_rx; - - struct rx_desc *rx_std_ring; - struct rx_desc *rx_jumbo_ring; - struct rx_desc *rx_mini_ring; - struct rx_desc *rx_return_ring; - - int bh_work_pending, jumbo; - struct work_struct ace_bh_work; - - struct event *evt_ring; - - volatile u32 *evt_prd, *rx_ret_prd, *tx_csm; - - dma_addr_t tx_ring_dma; /* 32/64 bit */ - dma_addr_t rx_ring_base_dma; - dma_addr_t evt_ring_dma; - dma_addr_t evt_prd_dma, rx_ret_prd_dma, tx_csm_dma; - - unsigned char *trace_buf; - struct pci_dev *pdev; - struct net_device *next; - volatile int fw_running; - int board_idx; - u16 pci_command; - u8 pci_latency; - const char *name; -#ifdef INDEX_DEBUG - spinlock_t debug_lock - __attribute__ ((aligned (SMP_CACHE_BYTES))); - u32 last_tx, last_std_rx, last_mini_rx; -#endif - u8 firmware_major; - u8 firmware_minor; - u8 firmware_fix; - u32 firmware_start; -}; - - -#define TX_RESERVED MAX_SKB_FRAGS - -static inline int tx_space (struct ace_private *ap, u32 csm, u32 prd) -{ - return (csm - prd - 1) & (ACE_TX_RING_ENTRIES(ap) - 1); -} - -#define tx_free(ap) tx_space((ap)->tx_ret_csm, (ap)->tx_prd, ap) -#define tx_ring_full(ap, csm, prd) (tx_space(ap, csm, prd) <= TX_RESERVED) - -static inline void set_aceaddr(aceaddr *aa, dma_addr_t addr) -{ - u64 baddr = (u64) addr; - aa->addrlo = baddr & 0xffffffff; - aa->addrhi = baddr >> 32; - wmb(); -} - - -static inline void ace_set_txprd(struct ace_regs __iomem *regs, - struct ace_private *ap, u32 value) -{ -#ifdef INDEX_DEBUG - unsigned long flags; - spin_lock_irqsave(&ap->debug_lock, flags); - writel(value, ®s->TxPrd); - if (value == ap->last_tx) - printk(KERN_ERR "AceNIC RACE ALERT! writing identical value " - "to tx producer (%i)\n", value); - ap->last_tx = value; - spin_unlock_irqrestore(&ap->debug_lock, flags); -#else - writel(value, ®s->TxPrd); -#endif - wmb(); -} - - -static inline void ace_mask_irq(struct net_device *dev) -{ - struct ace_private *ap = netdev_priv(dev); - struct ace_regs __iomem *regs = ap->regs; - - if (ACE_IS_TIGON_I(ap)) - writel(1, ®s->MaskInt); - else - writel(readl(®s->HostCtrl) | MASK_INTS, ®s->HostCtrl); - - ace_sync_irq(dev->irq); -} - - -static inline void ace_unmask_irq(struct net_device *dev) -{ - struct ace_private *ap = netdev_priv(dev); - struct ace_regs __iomem *regs = ap->regs; - - if (ACE_IS_TIGON_I(ap)) - writel(0, ®s->MaskInt); - else - writel(readl(®s->HostCtrl) & ~MASK_INTS, ®s->HostCtrl); -} - - -/* - * Prototypes - */ -static int ace_init(struct net_device *dev); -static void ace_load_std_rx_ring(struct net_device *dev, int nr_bufs); -static void ace_load_mini_rx_ring(struct net_device *dev, int nr_bufs); -static void ace_load_jumbo_rx_ring(struct net_device *dev, int nr_bufs); -static irqreturn_t ace_interrupt(int irq, void *dev_id); -static int ace_load_firmware(struct net_device *dev); -static int ace_open(struct net_device *dev); -static netdev_tx_t ace_start_xmit(struct sk_buff *skb, - struct net_device *dev); -static int ace_close(struct net_device *dev); -static void ace_bh_work(struct work_struct *work); -static void ace_dump_trace(struct ace_private *ap); -static void ace_set_multicast_list(struct net_device *dev); -static int ace_change_mtu(struct net_device *dev, int new_mtu); -static int ace_set_mac_addr(struct net_device *dev, void *p); -static void ace_set_rxtx_parms(struct net_device *dev, int jumbo); -static int ace_allocate_descriptors(struct net_device *dev); -static void ace_free_descriptors(struct net_device *dev); -static void ace_init_cleanup(struct net_device *dev); -static struct net_device_stats *ace_get_stats(struct net_device *dev); -static int read_eeprom_byte(struct net_device *dev, unsigned long offset); - -#endif /* _ACENIC_H_ */ diff --git a/drivers/net/ethernet/amd/declance.c b/drivers/net/ethernet/amd/declance.c index e6d56fcdc1dd..c7d47ca603a8 100644 --- a/drivers/net/ethernet/amd/declance.c +++ b/drivers/net/ethernet/amd/declance.c @@ -726,8 +726,10 @@ out: static irqreturn_t lance_dma_merr_int(int irq, void *dev_id) { struct net_device *dev = dev_id; + u64 ldp = ioasic_read(IO_REG_LANCE_DMA_P); - printk(KERN_ERR "%s: DMA error\n", dev->name); + pr_err_ratelimited("%s: DMA error at %#010llx\n", dev->name, + (ldp & 0x1f) << 29 | (ldp & 0xffffffe0) >> 3); return IRQ_HANDLED; } diff --git a/drivers/net/ethernet/amd/xgbe/xgbe-common.h b/drivers/net/ethernet/amd/xgbe/xgbe-common.h index 80c2c27ac9dc..66807d67e984 100644 --- a/drivers/net/ethernet/amd/xgbe/xgbe-common.h +++ b/drivers/net/ethernet/amd/xgbe/xgbe-common.h @@ -330,6 +330,10 @@ #define MAC_ISR_SMI_WIDTH 1 #define MAC_ISR_TSIS_INDEX 12 #define MAC_ISR_TSIS_WIDTH 1 +#define MAC_ISR_LS_INDEX 24 +#define MAC_ISR_LS_WIDTH 2 +#define MAC_ISR_LSI_INDEX 0 +#define MAC_ISR_LSI_WIDTH 1 #define MAC_MACA1HR_AE_INDEX 31 #define MAC_MACA1HR_AE_WIDTH 1 #define MAC_MDIOIER_SNGLCOMPIE_INDEX 12 @@ -832,6 +836,8 @@ #define PCS_V2_YC_WINDOW_SELECT 0x18064 #define PCS_V3_RN_WINDOW_DEF 0xf8078 #define PCS_V3_RN_WINDOW_SELECT 0xf807c +#define PCS_P100a_WINDOW_DEF 0x8060 +#define PCS_P100a_WINDOW_SELECT 0x8080 #define PCS_RN_SMN_BASE_ADDR 0x11e00000 #define PCS_RN_PORT_ADDR_SIZE 0x100000 @@ -968,7 +974,7 @@ #define XP_PROP_0_PORT_MODE_INDEX 8 #define XP_PROP_0_PORT_MODE_WIDTH 4 #define XP_PROP_0_PORT_SPEEDS_INDEX 22 -#define XP_PROP_0_PORT_SPEEDS_WIDTH 5 +#define XP_PROP_0_PORT_SPEEDS_WIDTH 6 #define XP_PROP_1_MAX_RX_DMA_INDEX 24 #define XP_PROP_1_MAX_RX_DMA_WIDTH 5 #define XP_PROP_1_MAX_RX_QUEUES_INDEX 8 diff --git a/drivers/net/ethernet/amd/xgbe/xgbe-dev.c b/drivers/net/ethernet/amd/xgbe/xgbe-dev.c index c04a9c76bd40..2de974213090 100644 --- a/drivers/net/ethernet/amd/xgbe/xgbe-dev.c +++ b/drivers/net/ethernet/amd/xgbe/xgbe-dev.c @@ -718,20 +718,25 @@ static void xgbe_disable_ecc_sec(struct xgbe_prv_data *pdata, static int xgbe_set_speed(struct xgbe_prv_data *pdata, int speed) { - unsigned int ss; + unsigned int ss, ver; switch (speed) { case SPEED_10: - ss = 0x07; + ss = XGBE_MAC_SS_10M; break; case SPEED_1000: - ss = 0x03; + ss = XGBE_MAC_SS_1G; break; case SPEED_2500: - ss = 0x02; + /* P100a uses XGMII mode for 2.5G, older platforms use GMII */ + ver = XGMAC_GET_BITS(pdata->hw_feat.version, MAC_VR, SNPSVER); + if (ver == XGBE_MAC_VER_33) + ss = XGBE_MAC_SS_2_5G_XGMII; + else + ss = XGBE_MAC_SS_2_5G_GMII; break; case SPEED_10000: - ss = 0x00; + ss = XGBE_MAC_SS_10G; break; default: return -EINVAL; @@ -1070,6 +1075,8 @@ static void xgbe_get_pcs_index_and_offset(struct xgbe_prv_data *pdata, unsigned int *index, unsigned int *offset) { + unsigned int ver; + /* The PCS registers are accessed using mmio. The underlying * management interface uses indirect addressing to access the MMD * register sets. This requires accessing of the PCS register in two @@ -1081,7 +1088,27 @@ static void xgbe_get_pcs_index_and_offset(struct xgbe_prv_data *pdata, */ mmd_address <<= 1; *index = mmd_address & ~pdata->xpcs_window_mask; - *offset = pdata->xpcs_window + (mmd_address & pdata->xpcs_window_mask); + + ver = XGMAC_GET_BITS(pdata->hw_feat.version, MAC_VR, SNPSVER); + + /* The P100a platform uses a different memory mapping scheme for XPCS + * register access. The offset calculation differs between platforms: + * + * For P100a platforms: The offset is calculated by adding the + * mmd_address to the xpcs_window and then applying the xpcs_window_mask + * For older platforms: The offset is calculated by applying the + * xpcs_window_mask to the mmd_address and then adding it to the + * xpcs_window. + * + * This is critical because using the wrong calculation causes register + * accesses to target the wrong registers, leading to incorrect behavior + */ + if (ver == XGBE_MAC_VER_33) + *offset = (pdata->xpcs_window + mmd_address) & + pdata->xpcs_window_mask; + else + *offset = pdata->xpcs_window + + (mmd_address & pdata->xpcs_window_mask); } static int xgbe_read_mmd_regs_v3(struct xgbe_prv_data *pdata, int prtad, @@ -3159,7 +3186,16 @@ static void xgbe_txq_prepare_tx_stop(struct xgbe_prv_data *pdata, /* The Tx engine cannot be stopped if it is actively processing * packets. Wait for the Tx queue to empty the Tx fifo. Don't * wait forever though... + * + * Optimization: Skip the wait when link is down. Hardware won't + * complete TX processing, so waiting serves no purpose and only + * delays interface shutdown. Descriptors will be reclaimed via + * the force-cleanup path in tx_poll. */ + + if (!pdata->phy.link) + return; + tx_timeout = jiffies + (XGBE_DMA_STOP_TIMEOUT * HZ); while (time_before(jiffies, tx_timeout)) { tx_status = XGMAC_MTL_IOREAD(pdata, queue, MTL_Q_TQDR); @@ -3240,28 +3276,83 @@ static void xgbe_enable_tx(struct xgbe_prv_data *pdata) XGMAC_IOWRITE_BITS(pdata, MAC_TCR, TE, 1); } -static void xgbe_disable_tx(struct xgbe_prv_data *pdata) +/** + * xgbe_wait_for_dma_tx_complete - Wait for DMA to complete pending TX + * @pdata: driver private data + * + * Wait for the DMA TX channels to complete all pending descriptors. + * This ensures no frames are in-flight before we disable the transmitter. + * If link is down, return immediately as TX will never complete. + * + * Return: 0 on success, -ETIMEDOUT on timeout + */ +static int xgbe_wait_for_dma_tx_complete(struct xgbe_prv_data *pdata) { + struct xgbe_channel *channel; + struct xgbe_ring *ring; + unsigned long timeout; unsigned int i; + bool complete; - /* Prepare for Tx DMA channel stop */ - for (i = 0; i < pdata->tx_q_count; i++) - xgbe_prepare_tx_stop(pdata, i); + /* If link is down, TX will never complete - skip waiting */ + if (!pdata->phy.link) + return 0; - /* Disable MAC Tx */ - XGMAC_IOWRITE_BITS(pdata, MAC_TCR, TE, 0); + timeout = jiffies + (XGBE_DMA_STOP_TIMEOUT * HZ); - /* Disable each Tx queue */ - for (i = 0; i < pdata->tx_q_count; i++) - XGMAC_MTL_IOWRITE_BITS(pdata, i, MTL_Q_TQOMR, TXQEN, 0); + do { + complete = true; - /* Disable each Tx DMA channel */ + for (i = 0; i < pdata->channel_count; i++) { + channel = pdata->channel[i]; + ring = channel->tx_ring; + if (!ring) + continue; + + /* Check if DMA has processed all descriptors */ + if (ring->dirty != ring->cur) { + complete = false; + break; + } + } + + if (complete) + return 0; + + usleep_range(100, 200); + } while (time_before(jiffies, timeout)); + + netif_warn(pdata, drv, pdata->netdev, + "timeout waiting for DMA TX to complete\n"); + return -ETIMEDOUT; +} + +static void xgbe_disable_tx(struct xgbe_prv_data *pdata) +{ + unsigned int i; + + /* Step 1: Wait for DMA to complete pending descriptors */ + xgbe_wait_for_dma_tx_complete(pdata); + + /* Step 2: Disable each Tx DMA channel to stop + * processing new descriptors + */ for (i = 0; i < pdata->channel_count; i++) { if (!pdata->channel[i]->tx_ring) break; - XGMAC_DMA_IOWRITE_BITS(pdata->channel[i], DMA_CH_TCR, ST, 0); } + + /* Step 3: Wait for MTL TX queues to drain */ + for (i = 0; i < pdata->tx_q_count; i++) + xgbe_prepare_tx_stop(pdata, i); + + /* Step 4: Disable MTL TX queues */ + for (i = 0; i < pdata->tx_q_count; i++) + XGMAC_MTL_IOWRITE_BITS(pdata, i, MTL_Q_TQOMR, TXQEN, 0); + + /* Step 5: Disable MAC TX last */ + XGMAC_IOWRITE_BITS(pdata, MAC_TCR, TE, 0); } static void xgbe_prepare_rx_stop(struct xgbe_prv_data *pdata, diff --git a/drivers/net/ethernet/amd/xgbe/xgbe-drv.c b/drivers/net/ethernet/amd/xgbe/xgbe-drv.c index 23beea48ae26..2d6d00e3689b 100644 --- a/drivers/net/ethernet/amd/xgbe/xgbe-drv.c +++ b/drivers/net/ethernet/amd/xgbe/xgbe-drv.c @@ -607,11 +607,33 @@ static void xgbe_service_timer(struct timer_list *t) struct xgbe_prv_data *pdata = timer_container_of(pdata, t, service_timer); struct xgbe_channel *channel; + unsigned int poll_interval; unsigned int i; queue_work(pdata->dev_workqueue, &pdata->service_work); - mod_timer(&pdata->service_timer, jiffies + HZ); + /* Adaptive link status polling for fast failure detection: + * + * - When carrier is UP: poll every 100ms for rapid link-down detection + * Enables sub-second response to link failures, minimizing traffic + * loss. + * + * - When carrier is DOWN: poll every 1s to conserve CPU resources + * Link-up events are less time-critical. + * + * The 100ms active polling interval balances responsiveness with + * efficiency: + * - Provides ~100-200ms link-down detection (10x faster than 1s + * polling) + * - Minimal CPU overhead (1% vs 0.1% with 1s polling) + * - Enables fast failover in link aggregation deployments + */ + if (netif_running(pdata->netdev) && netif_carrier_ok(pdata->netdev)) + poll_interval = msecs_to_jiffies(100); /* 100ms when up */ + else + poll_interval = HZ; /* 1 second when down */ + + mod_timer(&pdata->service_timer, jiffies + poll_interval); if (!pdata->tx_usecs) return; @@ -1116,69 +1138,62 @@ static int xgbe_phy_reset(struct xgbe_prv_data *pdata) return pdata->phy_if.phy_reset(pdata); } -int xgbe_powerdown(struct net_device *netdev, unsigned int caller) +int xgbe_powerdown(struct net_device *netdev) { struct xgbe_prv_data *pdata = netdev_priv(netdev); struct xgbe_hw_if *hw_if = &pdata->hw_if; - DBGPR("-->xgbe_powerdown\n"); - - if (!netif_running(netdev) || - (caller == XGMAC_IOCTL_CONTEXT && pdata->power_down)) { - netdev_alert(netdev, "Device is already powered down\n"); - DBGPR("<--xgbe_powerdown\n"); + if (!netif_running(netdev)) { + netdev_dbg(netdev, "Device is not running, skipping powerdown\n"); return -EINVAL; } - if (caller == XGMAC_DRIVER_CONTEXT) - netif_device_detach(netdev); + if (pdata->power_down) { + netdev_dbg(netdev, "Device is already powered down\n"); + return -EINVAL; + } + netif_device_detach(netdev); netif_tx_stop_all_queues(netdev); xgbe_stop_timers(pdata); flush_workqueue(pdata->dev_workqueue); + xgbe_napi_disable(pdata, 0); + hw_if->powerdown_tx(pdata); hw_if->powerdown_rx(pdata); - xgbe_napi_disable(pdata, 0); - pdata->power_down = 1; - DBGPR("<--xgbe_powerdown\n"); - return 0; } -int xgbe_powerup(struct net_device *netdev, unsigned int caller) +int xgbe_powerup(struct net_device *netdev) { struct xgbe_prv_data *pdata = netdev_priv(netdev); struct xgbe_hw_if *hw_if = &pdata->hw_if; - DBGPR("-->xgbe_powerup\n"); - - if (!netif_running(netdev) || - (caller == XGMAC_IOCTL_CONTEXT && !pdata->power_down)) { - netdev_alert(netdev, "Device is already powered up\n"); - DBGPR("<--xgbe_powerup\n"); + if (!netif_running(netdev)) { + netdev_dbg(netdev, "Device is not running, skipping powerup\n"); return -EINVAL; } - pdata->power_down = 0; - - xgbe_napi_enable(pdata, 0); + if (!pdata->power_down) { + netdev_dbg(netdev, "Device is already powered up\n"); + return -EINVAL; + } hw_if->powerup_tx(pdata); hw_if->powerup_rx(pdata); - if (caller == XGMAC_DRIVER_CONTEXT) - netif_device_attach(netdev); + xgbe_napi_enable(pdata, 0); netif_tx_start_all_queues(netdev); - xgbe_start_timers(pdata); + netif_device_attach(netdev); - DBGPR("<--xgbe_powerup\n"); + pdata->power_down = 0; return 0; } @@ -2154,6 +2169,7 @@ static int xgbe_tx_poll(struct xgbe_channel *channel) struct net_device *netdev = pdata->netdev; struct netdev_queue *txq; int processed = 0; + int force_cleanup; unsigned int tx_packets = 0, tx_bytes = 0; unsigned int cur; @@ -2170,13 +2186,41 @@ static int xgbe_tx_poll(struct xgbe_channel *channel) txq = netdev_get_tx_queue(netdev, channel->queue_index); + /* Smart descriptor cleanup during link-down conditions. + * + * When link is down, hardware stops processing TX descriptors (OWN bit + * remains set). Enable intelligent cleanup to reclaim these abandoned + * descriptors and maintain TX queue health. + * + * This cleanup mechanism enables: + * - Continuous TX queue availability for new packets when link recovers + * - Clean resource management (skbs, DMA mappings, descriptors) + * - Fast failover in link aggregation scenarios + */ + force_cleanup = !pdata->phy.link; + while ((processed < XGBE_TX_DESC_MAX_PROC) && (ring->dirty != cur)) { rdata = XGBE_GET_DESC_DATA(ring, ring->dirty); rdesc = rdata->rdesc; - if (!hw_if->tx_complete(rdesc)) - break; + if (!hw_if->tx_complete(rdesc)) { + if (!force_cleanup) + break; + /* Link-down descriptor cleanup: reclaim abandoned + * resources. + * + * Hardware has stopped processing this descriptor, so + * perform intelligent cleanup to free skbs and reclaim + * descriptors for future use when link recovers. + * + * These are not counted as successful transmissions + * since packets never reached the wire. + */ + netif_dbg(pdata, tx_err, netdev, + "force-freeing stuck TX desc %u (link down)\n", + ring->dirty); + } /* Make sure descriptor fields are read after reading the OWN * bit */ @@ -2185,9 +2229,13 @@ static int xgbe_tx_poll(struct xgbe_channel *channel) if (netif_msg_tx_done(pdata)) xgbe_dump_tx_desc(pdata, ring, ring->dirty, 1, 0); - if (hw_if->is_last_desc(rdesc)) { - tx_packets += rdata->tx.packets; - tx_bytes += rdata->tx.bytes; + /* Only count packets actually transmitted (not force-cleaned) + */ + if (!force_cleanup || hw_if->is_last_desc(rdesc)) { + if (hw_if->is_last_desc(rdesc)) { + tx_packets += rdata->tx.packets; + tx_bytes += rdata->tx.bytes; + } } /* Free the SKB and reset the descriptor for re-use */ diff --git a/drivers/net/ethernet/amd/xgbe/xgbe-mdio.c b/drivers/net/ethernet/amd/xgbe/xgbe-mdio.c index 7675bb98f029..fa0df6181207 100644 --- a/drivers/net/ethernet/amd/xgbe/xgbe-mdio.c +++ b/drivers/net/ethernet/amd/xgbe/xgbe-mdio.c @@ -1047,11 +1047,29 @@ static void xgbe_phy_adjust_link(struct xgbe_prv_data *pdata) if (pdata->phy_link != pdata->phy.link) { new_state = 1; pdata->phy_link = pdata->phy.link; + + /* Link is coming up - wake TX queues */ + netif_tx_wake_all_queues(pdata->netdev); } } else if (pdata->phy_link) { new_state = 1; pdata->phy_link = 0; pdata->phy_speed = SPEED_UNKNOWN; + + /* Proactive TX queue management on link-down. + * + * Immediately stop TX queues to enable clean link-down + * handling: + * - Prevents queueing packets that can't be transmitted + * - Allows orderly descriptor cleanup by NAPI poll + * - Enables rapid failover in link aggregation configurations + * + * Note: We do NOT call netdev_tx_reset_queue() here because + * NAPI poll may still be running and would trigger BQL + * assertion. BQL state is cleaned up naturally during + * descriptor reclamation. + */ + netif_tx_stop_all_queues(pdata->netdev); } if (new_state && netif_msg_link(pdata)) diff --git a/drivers/net/ethernet/amd/xgbe/xgbe-pci.c b/drivers/net/ethernet/amd/xgbe/xgbe-pci.c index e3e1dca9856a..dbdd791380a4 100644 --- a/drivers/net/ethernet/amd/xgbe/xgbe-pci.c +++ b/drivers/net/ethernet/amd/xgbe/xgbe-pci.c @@ -168,6 +168,14 @@ static int xgbe_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) rdev = pci_get_domain_bus_and_slot(0, 0, PCI_DEVFN(0, 0)); if (rdev && rdev->vendor == PCI_VENDOR_ID_AMD) { switch (rdev->device) { + case XGBE_P100a_PCI_DEVICE_ID: + pdata->xpcs_window_def_reg = PCS_P100a_WINDOW_DEF; + pdata->xpcs_window_sel_reg = PCS_P100a_WINDOW_SELECT; + + /* P100a devices do not need rrc and cdr workaround */ + pdata->vdata->an_cdr_workaround = 0; + pdata->vdata->enable_rrc = 0; + break; case XGBE_RV_PCI_DEVICE_ID: pdata->xpcs_window_def_reg = PCS_V2_RV_WINDOW_DEF; pdata->xpcs_window_sel_reg = PCS_V2_RV_WINDOW_SELECT; @@ -352,35 +360,102 @@ static void xgbe_pci_remove(struct pci_dev *pdev) xgbe_free_pdata(pdata); } -static int __maybe_unused xgbe_pci_suspend(struct device *dev) +static void xgbe_pci_synchronize_irqs(struct xgbe_prv_data *pdata) +{ + unsigned int i; + + /* Synchronize main device interrupt */ + synchronize_irq(pdata->dev_irq); + + /* Synchronize ECC interrupt if separate from main device interrupt */ + if (pdata->vdata->ecc_support && pdata->dev_irq != pdata->ecc_irq) + synchronize_irq(pdata->ecc_irq); + + /* Synchronize I2C interrupt if separate from main device interrupt */ + if (pdata->vdata->i2c_support && pdata->dev_irq != pdata->i2c_irq) + synchronize_irq(pdata->i2c_irq); + + /* Synchronize AN interrupt if separate from main device interrupt */ + if (pdata->dev_irq != pdata->an_irq) + synchronize_irq(pdata->an_irq); + + /* Synchronize per-channel DMA interrupts */ + if (pdata->per_channel_irq) { + for (i = 0; i < pdata->channel_count; i++) + synchronize_irq(pdata->channel[i]->dma_irq); + } +} + +static int xgbe_pci_suspend(struct device *dev) { struct xgbe_prv_data *pdata = dev_get_drvdata(dev); struct net_device *netdev = pdata->netdev; + struct pci_dev *pdev = to_pci_dev(dev); int ret = 0; if (netif_running(netdev)) - ret = xgbe_powerdown(netdev, XGMAC_DRIVER_CONTEXT); + ret = xgbe_powerdown(netdev); + + /* Disable all device interrupts to prevent spurious wakeups */ + XP_IOWRITE(pdata, XP_INT_EN, 0x0); + + /* Ensure no IRQ handlers are still executing before powering down. + * This prevents race conditions where an IRQ handler could access + * invalid register state after the device is disabled. + */ + xgbe_pci_synchronize_irqs(pdata); + /* Set PHY to low-power mode */ pdata->lpm_ctrl = XMDIO_READ(pdata, MDIO_MMD_PCS, MDIO_CTRL1); pdata->lpm_ctrl |= MDIO_CTRL1_LPOWER; XMDIO_WRITE(pdata, MDIO_MMD_PCS, MDIO_CTRL1, pdata->lpm_ctrl); + /* Disable bus mastering to prevent DMA activity */ + pci_clear_master(pdev); + + /* Save PCI configuration state and disable device */ + pci_save_state(pdev); + pci_disable_device(pdev); + + /* Disable wake from D3 - required for S0i3 deep sleep */ + pci_wake_from_d3(pdev, false); + pci_set_power_state(pdev, PCI_D3hot); + return ret; } -static int __maybe_unused xgbe_pci_resume(struct device *dev) +static int xgbe_pci_resume(struct device *dev) { struct xgbe_prv_data *pdata = dev_get_drvdata(dev); struct net_device *netdev = pdata->netdev; + struct pci_dev *pdev = to_pci_dev(dev); int ret = 0; + /* Restore PCI power state */ + pci_set_power_state(pdev, PCI_D0); + + /* Restore PCI configuration state */ + pci_restore_state(pdev); + + /* Enable PCI device */ + ret = pci_enable_device(pdev); + if (ret) { + dev_err(dev, "pci_enable_device failed: %d\n", ret); + return ret; + } + + /* Re-enable bus mastering */ + pci_set_master(pdev); + + /* Re-enable all device interrupts */ XP_IOWRITE(pdata, XP_INT_EN, 0x1fffff); + /* Clear PHY low-power mode */ pdata->lpm_ctrl &= ~MDIO_CTRL1_LPOWER; XMDIO_WRITE(pdata, MDIO_MMD_PCS, MDIO_CTRL1, pdata->lpm_ctrl); if (netif_running(netdev)) { - ret = xgbe_powerup(netdev, XGMAC_DRIVER_CONTEXT); + ret = xgbe_powerup(netdev); /* Schedule a restart in case the link or phy state changed * while we were powered down. @@ -453,16 +528,16 @@ static const struct pci_device_id xgbe_pci_table[] = { }; MODULE_DEVICE_TABLE(pci, xgbe_pci_table); -static SIMPLE_DEV_PM_OPS(xgbe_pci_pm_ops, xgbe_pci_suspend, xgbe_pci_resume); +static DEFINE_SIMPLE_DEV_PM_OPS(xgbe_pci_pm_ops, + xgbe_pci_suspend, + xgbe_pci_resume); static struct pci_driver xgbe_driver = { .name = XGBE_DRV_NAME, .id_table = xgbe_pci_table, .probe = xgbe_pci_probe, .remove = xgbe_pci_remove, - .driver = { - .pm = &xgbe_pci_pm_ops, - } + .driver.pm = pm_sleep_ptr(&xgbe_pci_pm_ops), }; int xgbe_pci_init(void) diff --git a/drivers/net/ethernet/amd/xgbe/xgbe-phy-v2.c b/drivers/net/ethernet/amd/xgbe/xgbe-phy-v2.c index b8cf6ccfe641..59a074ed312a 100644 --- a/drivers/net/ethernet/amd/xgbe/xgbe-phy-v2.c +++ b/drivers/net/ethernet/amd/xgbe/xgbe-phy-v2.c @@ -2183,7 +2183,11 @@ static bool enable_rx_adap(struct xgbe_prv_data *pdata, enum xgbe_mode mode) /* Rx-Adaptation is not supported on older platforms(< 0x30H) */ ver = XGMAC_GET_BITS(pdata->hw_feat.version, MAC_VR, SNPSVER); - if (ver < 0x30) + if (ver < XGBE_MAC_VER_30) + return false; + + /* Rx adaptation not yet supported on P100a */ + if (ver == XGBE_MAC_VER_33) return false; /* Re-driver models 4223 && 4227 do not support Rx-Adaptation */ @@ -2309,12 +2313,24 @@ static void xgbe_phy_kr_mode(struct xgbe_prv_data *pdata) static void xgbe_phy_kx_2500_mode(struct xgbe_prv_data *pdata) { + unsigned int ver; + struct xgbe_phy_data *phy_data = pdata->phy_data; xgbe_phy_set_redrv_mode(pdata); - /* 2.5G/KX */ - xgbe_phy_perform_ratechange(pdata, XGBE_MB_CMD_SET_2_5G, XGBE_MB_SUBCMD_NONE); + ver = XGMAC_GET_BITS(pdata->hw_feat.version, MAC_VR, SNPSVER); + + /* + * P100a uses XGMII mode for 2.5G which requires the 2.5G_KX subcommand. + * Older platforms use GMII mode. + */ + if (ver == XGBE_MAC_VER_33) + xgbe_phy_perform_ratechange(pdata, XGBE_MB_CMD_SET_2_5G, + XGBE_MB_SUBCMD_2_5G_KX); + else + xgbe_phy_perform_ratechange(pdata, XGBE_MB_CMD_SET_2_5G, + XGBE_MB_SUBCMD_NONE); phy_data->cur_mode = XGBE_MODE_KX_2500; diff --git a/drivers/net/ethernet/amd/xgbe/xgbe-platform.c b/drivers/net/ethernet/amd/xgbe/xgbe-platform.c index 47d53e59ccf6..bca0dbff80c1 100644 --- a/drivers/net/ethernet/amd/xgbe/xgbe-platform.c +++ b/drivers/net/ethernet/amd/xgbe/xgbe-platform.c @@ -252,13 +252,9 @@ static int xgbe_platform_probe(struct platform_device *pdev) dev_dbg(dev, "sir1_regs = %p\n", pdata->sir1_regs); /* Retrieve the MAC address */ - ret = device_property_read_u8_array(dev, XGBE_MAC_ADDR_PROPERTY, - pdata->mac_addr, - sizeof(pdata->mac_addr)); - if (ret || !is_valid_ether_addr(pdata->mac_addr)) { - dev_err(dev, "invalid %s property\n", XGBE_MAC_ADDR_PROPERTY); - if (!ret) - ret = -EINVAL; + ret = device_get_mac_address(dev, pdata->mac_addr); + if (ret) { + dev_err(dev, "invalid MAC address property\n"); goto err_io; } @@ -384,7 +380,7 @@ static int xgbe_platform_suspend(struct device *dev) DBGPR("-->xgbe_suspend\n"); if (netif_running(netdev)) - ret = xgbe_powerdown(netdev, XGMAC_DRIVER_CONTEXT); + ret = xgbe_powerdown(netdev); pdata->lpm_ctrl = XMDIO_READ(pdata, MDIO_MMD_PCS, MDIO_CTRL1); pdata->lpm_ctrl |= MDIO_CTRL1_LPOWER; @@ -407,7 +403,7 @@ static int xgbe_platform_resume(struct device *dev) XMDIO_WRITE(pdata, MDIO_MMD_PCS, MDIO_CTRL1, pdata->lpm_ctrl); if (netif_running(netdev)) { - ret = xgbe_powerup(netdev, XGMAC_DRIVER_CONTEXT); + ret = xgbe_powerup(netdev); /* Schedule a restart in case the link or phy state changed * while we were powered down. diff --git a/drivers/net/ethernet/amd/xgbe/xgbe.h b/drivers/net/ethernet/amd/xgbe/xgbe.h index 438033a71523..60b7e53206d1 100644 --- a/drivers/net/ethernet/amd/xgbe/xgbe.h +++ b/drivers/net/ethernet/amd/xgbe/xgbe.h @@ -96,7 +96,6 @@ min_t(unsigned int, IEEE_8021QAZ_MAX_TCS, (_cnt)) /* Common property names */ -#define XGBE_MAC_ADDR_PROPERTY "mac-address" #define XGBE_PHY_MODE_PROPERTY "phy-mode" #define XGBE_DMA_IRQS_PROPERTY "amd,per-channel-interrupt" #define XGBE_SPEEDSET_PROPERTY "amd,speed-set" @@ -146,10 +145,6 @@ #define XGBE_MAX_PPS_OUT 4 #define XGBE_MAX_AUX_SNAP 4 -/* Driver PMT macros */ -#define XGMAC_DRIVER_CONTEXT 1 -#define XGMAC_IOCTL_CONTEXT 2 - #define XGMAC_FIFO_MIN_ALLOC 2048 #define XGMAC_FIFO_UNIT 256 #define XGMAC_FIFO_ALIGN(_x) \ @@ -262,11 +257,31 @@ #define XGBE_RV_PCI_DEVICE_ID 0x15d0 #define XGBE_YC_PCI_DEVICE_ID 0x14b5 #define XGBE_RN_PCI_DEVICE_ID 0x1630 +#define XGBE_P100a_PCI_DEVICE_ID 0x1122 /* Generic low and high masks */ #define XGBE_GEN_HI_MASK GENMASK(31, 16) #define XGBE_GEN_LO_MASK GENMASK(15, 0) +/* MAC hardware version numbers (SNPSVER field in MAC_VR register) */ +#define XGBE_MAC_VER_30 0x30 /* Baseline Rx adaptation support */ +#define XGBE_MAC_VER_33 0x33 /* P100a platform */ + +/* MAC Speed Select (SS) values for MAC_TCR register + * These values are written to the SS field to configure link speed. + * Note: P100a uses XGMII mode (0x06) for 2.5G instead of GMII (0x02) + */ +/* Note: 100M and 2.5G GMII share the same value (0x02) but are + * differentiated by the mode/interface type at the PHY level + */ + +#define XGBE_MAC_SS_10G 0x00 /* 10Gbps - XGMII mode */ +#define XGBE_MAC_SS_2_5G_GMII 0x02 /* 2.5Gbps - GMII mode (YC) */ +#define XGBE_MAC_SS_2_5G_XGMII 0x06 /* 2.5Gbps - XGMII mode (P100a) */ +#define XGBE_MAC_SS_1G 0x03 /* 1Gbps */ +#define XGBE_MAC_SS_100M 0x02 /* 100Mbps */ +#define XGBE_MAC_SS_10M 0x07 /* 10Mbps */ + struct xgbe_prv_data; struct xgbe_packet_data { @@ -558,7 +573,10 @@ enum xgbe_mb_subcmd { XGBE_MB_SUBCMD_10MBITS = 0, XGBE_MB_SUBCMD_100MBITS, XGBE_MB_SUBCMD_1G_SGMII, - XGBE_MB_SUBCMD_1G_KX + XGBE_MB_SUBCMD_1G_KX, + + /* 2.5GbE Mode subcommands */ + XGBE_MB_SUBCMD_2_5G_KX = 1 }; struct xgbe_phy { @@ -1290,8 +1308,8 @@ void xgbe_dump_rx_desc(struct xgbe_prv_data *, struct xgbe_ring *, unsigned int); void xgbe_print_pkt(struct net_device *, struct sk_buff *, bool); void xgbe_get_all_hw_features(struct xgbe_prv_data *); -int xgbe_powerup(struct net_device *, unsigned int); -int xgbe_powerdown(struct net_device *, unsigned int); +int xgbe_powerup(struct net_device *netdev); +int xgbe_powerdown(struct net_device *netdev); void xgbe_init_rx_coalesce(struct xgbe_prv_data *); void xgbe_init_tx_coalesce(struct xgbe_prv_data *); void xgbe_restart_dev(struct xgbe_prv_data *pdata); diff --git a/drivers/net/ethernet/aquantia/atlantic/aq_ethtool.c b/drivers/net/ethernet/aquantia/atlantic/aq_ethtool.c index a6e1826dd5d7..420af958d486 100644 --- a/drivers/net/ethernet/aquantia/atlantic/aq_ethtool.c +++ b/drivers/net/ethernet/aquantia/atlantic/aq_ethtool.c @@ -983,6 +983,38 @@ static int aq_ethtool_set_phy_tunable(struct net_device *ndev, return err; } +static bool aq_ethtool_can_read_module_eeprom(struct aq_nic_s *aq_nic) +{ + return aq_nic->aq_fw_ops->read_module_eeprom || + aq_nic->aq_hw_ops->hw_read_module_eeprom; +} + +static int aq_ethtool_read_module_eeprom(struct aq_nic_s *aq_nic, u8 dev_addr, + u8 reg_start_addr, int len, u8 *data) +{ + const struct aq_fw_ops *fw_ops = aq_nic->aq_fw_ops; + const struct aq_hw_ops *hw_ops = aq_nic->aq_hw_ops; + int err = -EOPNOTSUPP; + + if (fw_ops->read_module_eeprom) { + err = fw_ops->read_module_eeprom(aq_nic->aq_hw, dev_addr, + reg_start_addr, len, data); + + /* If the only error is that the firmware version doesn't + * support reading EEPROM, we can still attempt to read it + * directly from the hardware if supported. + */ + if (err != -EOPNOTSUPP) + return err; + } + + if (hw_ops->hw_read_module_eeprom) + err = hw_ops->hw_read_module_eeprom(aq_nic->aq_hw, dev_addr, + reg_start_addr, len, data); + + return err; +} + static int aq_ethtool_get_module_info(struct net_device *ndev, struct ethtool_modinfo *modinfo) { @@ -992,16 +1024,18 @@ static int aq_ethtool_get_module_info(struct net_device *ndev, /* Module EEPROM is only supported for controllers with external PHY */ if (aq_nic->aq_nic_cfg.aq_hw_caps->media_type != AQ_HW_MEDIA_TYPE_FIBRE || - !aq_nic->aq_hw_ops->hw_read_module_eeprom) + !aq_ethtool_can_read_module_eeprom(aq_nic)) return -EOPNOTSUPP; - err = aq_nic->aq_hw_ops->hw_read_module_eeprom(aq_nic->aq_hw, - SFF_8472_ID_ADDR, SFF_8472_COMP_ADDR, 1, &compliance_val); + err = aq_ethtool_read_module_eeprom(aq_nic, SFF_8472_ID_ADDR, + SFF_8472_COMP_ADDR, 1, + &compliance_val); if (err) return err; - err = aq_nic->aq_hw_ops->hw_read_module_eeprom(aq_nic->aq_hw, - SFF_8472_ID_ADDR, SFF_8472_DOM_TYPE_ADDR, 1, &dom_type); + err = aq_ethtool_read_module_eeprom(aq_nic, SFF_8472_ID_ADDR, + SFF_8472_DOM_TYPE_ADDR, 1, + &dom_type); if (err) return err; @@ -1022,7 +1056,7 @@ static int aq_ethtool_get_module_eeprom(struct net_device *ndev, unsigned int first, last, len; int err; - if (!aq_nic->aq_hw_ops->hw_read_module_eeprom) + if (!aq_ethtool_can_read_module_eeprom(aq_nic)) return -EOPNOTSUPP; first = ee->offset; @@ -1032,8 +1066,8 @@ static int aq_ethtool_get_module_eeprom(struct net_device *ndev, len = min(last, ETH_MODULE_SFF_8079_LEN); len -= first; - err = aq_nic->aq_hw_ops->hw_read_module_eeprom(aq_nic->aq_hw, - SFF_8472_ID_ADDR, first, len, data); + err = aq_ethtool_read_module_eeprom(aq_nic, SFF_8472_ID_ADDR, + first, len, data); if (err) return err; @@ -1045,8 +1079,9 @@ static int aq_ethtool_get_module_eeprom(struct net_device *ndev, len -= first; first -= ETH_MODULE_SFF_8079_LEN; - err = aq_nic->aq_hw_ops->hw_read_module_eeprom(aq_nic->aq_hw, - SFF_8472_DIAGNOSTICS_ADDR, first, len, data); + err = aq_ethtool_read_module_eeprom(aq_nic, + SFF_8472_DIAGNOSTICS_ADDR, + first, len, data); if (err) return err; } diff --git a/drivers/net/ethernet/aquantia/atlantic/aq_hw.h b/drivers/net/ethernet/aquantia/atlantic/aq_hw.h index 4e66fd9b2ab1..57ea59026a2c 100644 --- a/drivers/net/ethernet/aquantia/atlantic/aq_hw.h +++ b/drivers/net/ethernet/aquantia/atlantic/aq_hw.h @@ -404,6 +404,9 @@ struct aq_fw_ops { int (*send_macsec_req)(struct aq_hw_s *self, struct macsec_msg_fw_request *msg, struct macsec_msg_fw_response *resp); + + int (*read_module_eeprom)(struct aq_hw_s *self, u8 dev_addr, + u8 reg_start_addr, int len, u8 *data); }; #endif /* AQ_HW_H */ diff --git a/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_utils.h b/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_utils.h index f6b990b7f5b4..404c84adad4a 100644 --- a/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_utils.h +++ b/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_utils.h @@ -319,6 +319,13 @@ struct __packed hw_atl_utils_settings { u32 media_detect; }; +struct __packed smbus_request { + u32 msg_id; + u32 device_id; + u32 address; + u32 length; +}; + enum macsec_msg_type { macsec_cfg_msg = 0, macsec_add_rx_sc_msg, diff --git a/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_utils_fw2x.c b/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_utils_fw2x.c index 4d4cfbc91e19..2cac0d9670bf 100644 --- a/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_utils_fw2x.c +++ b/drivers/net/ethernet/aquantia/atlantic/hw_atl/hw_atl_utils_fw2x.c @@ -703,6 +703,85 @@ static int aq_fw2x_send_macsec_req(struct aq_hw_s *hw, return err; } +static int aq_fw2x_read_module_eeprom(struct aq_hw_s *self, u8 dev_addr, + u8 reg_start_addr, int len, u8 *data) +{ + u32 low_status, orig_low_status, low_req = 0; + u32 res_bytes_remain_cnt = len % sizeof(u32); + u32 res_dword_cnt = len / sizeof(u32); + struct smbus_request request = { 0 }; + u32 req_dword_cnt; + u32 result = 0; + u32 caps_lo; + u32 offset; + int err; + + caps_lo = aq_fw2x_get_link_capabilities(self); + if (!(caps_lo & BIT(CAPS_LO_SMBUS_READ))) + return -EOPNOTSUPP; + + request.msg_id = 0; + request.device_id = dev_addr; + request.address = reg_start_addr; + request.length = len; + + /* Write SMBUS request to cfg memory */ + req_dword_cnt = DIV_ROUND_UP(sizeof(request), sizeof(u32)); + err = hw_atl_write_fwcfg_dwords(self, (void *)&request, req_dword_cnt); + if (err < 0) + return err; + + /* Toggle 0x368.CAPS_LO_SMBUS_READ bit */ + low_req = aq_hw_read_reg(self, HW_ATL_FW2X_MPI_CONTROL_ADDR); + orig_low_status = low_req & BIT(CAPS_LO_SMBUS_READ); + low_req ^= BIT(CAPS_LO_SMBUS_READ); + aq_hw_write_reg(self, HW_ATL_FW2X_MPI_CONTROL_ADDR, low_req); + + /* Wait FW to report back */ + err = readx_poll_timeout_atomic(aq_fw2x_state_get, self, low_status, + orig_low_status != (low_status & + BIT(CAPS_LO_SMBUS_READ)), + 10U, 100000U); + if (err) + return err; + + /* Read status of read operation */ + offset = self->rpc_addr + sizeof(u32); + err = hw_atl_utils_fw_downld_dwords(self, offset, &result, + sizeof(result) / sizeof(u32)); + if (err < 0) + return err; + if (result) + return -EIO; + + /* Read response full DWORD data */ + if (res_dword_cnt) { + offset = self->rpc_addr + sizeof(u32) * 2; + err = hw_atl_utils_fw_downld_dwords(self, offset, (u32 *)data, + res_dword_cnt); + if (err < 0) + return err; + } + + /* Read response trailing bytes data */ + if (res_bytes_remain_cnt) { + u32 bytes_remain_val = 0; + + offset = self->rpc_addr + + (sizeof(u32) * 2) + + (res_dword_cnt * sizeof(u32)); + err = hw_atl_utils_fw_downld_dwords(self, offset, + &bytes_remain_val, 1); + if (err < 0) + return err; + + memcpy(data + len - res_bytes_remain_cnt, + &bytes_remain_val, res_bytes_remain_cnt); + } + + return 0; +} + const struct aq_fw_ops aq_fw_2x_ops = { .init = aq_fw2x_init, .deinit = aq_fw2x_deinit, @@ -729,4 +808,5 @@ const struct aq_fw_ops aq_fw_2x_ops = { .adjust_ptp = aq_fw3x_adjust_ptp, .get_link_capabilities = aq_fw2x_get_link_capabilities, .send_macsec_req = aq_fw2x_send_macsec_req, + .read_module_eeprom = aq_fw2x_read_module_eeprom, }; diff --git a/drivers/net/ethernet/broadcom/Kconfig b/drivers/net/ethernet/broadcom/Kconfig index 9787c1857e13..4287edc7ddd6 100644 --- a/drivers/net/ethernet/broadcom/Kconfig +++ b/drivers/net/ethernet/broadcom/Kconfig @@ -96,7 +96,7 @@ config BNX2 config CNIC tristate "QLogic CNIC support" - depends on PCI && (IPV6 || IPV6=n) + depends on PCI select BNX2 select UIO help @@ -272,6 +272,7 @@ config BCMASP depends on OF select PHYLIB select MDIO_BCM_UNIMAC + select PAGE_POOL help This configuration enables the Broadcom ASP 2.0 Ethernet controller driver which is present in Broadcom STB SoCs such as 72165. diff --git a/drivers/net/ethernet/broadcom/asp2/bcmasp.h b/drivers/net/ethernet/broadcom/asp2/bcmasp.h index 29cd87335ec8..8c8ffaeadc79 100644 --- a/drivers/net/ethernet/broadcom/asp2/bcmasp.h +++ b/drivers/net/ethernet/broadcom/asp2/bcmasp.h @@ -6,6 +6,7 @@ #include <linux/phy.h> #include <linux/io-64-nonatomic-hi-lo.h> #include <uapi/linux/ethtool.h> +#include <net/page_pool/helpers.h> #define ASP_INTR2_OFFSET 0x1000 #define ASP_INTR2_STATUS 0x0 @@ -298,16 +299,19 @@ struct bcmasp_intf { void __iomem *rx_edpkt_cfg; void __iomem *rx_edpkt_dma; int rx_edpkt_index; - int rx_buf_order; struct bcmasp_desc *rx_edpkt_cpu; dma_addr_t rx_edpkt_dma_addr; dma_addr_t rx_edpkt_dma_read; dma_addr_t rx_edpkt_dma_valid; - /* RX buffer prefetcher ring*/ + /* Streaming RX data ring (RBUF_4K mode) */ void *rx_ring_cpu; dma_addr_t rx_ring_dma; dma_addr_t rx_ring_dma_valid; + int rx_buf_order; + + /* Page pool for recycling RX SKB data pages */ + struct page_pool *rx_page_pool; struct napi_struct rx_napi; struct bcmasp_res res; diff --git a/drivers/net/ethernet/broadcom/asp2/bcmasp_intf.c b/drivers/net/ethernet/broadcom/asp2/bcmasp_intf.c index b368ec2fea43..ec63f50a849e 100644 --- a/drivers/net/ethernet/broadcom/asp2/bcmasp_intf.c +++ b/drivers/net/ethernet/broadcom/asp2/bcmasp_intf.c @@ -15,6 +15,7 @@ #include <linux/platform_device.h> #include <net/ip.h> #include <net/ipv6.h> +#include <net/page_pool/helpers.h> #include "bcmasp.h" #include "bcmasp_intf_defs.h" @@ -482,10 +483,14 @@ static int bcmasp_rx_poll(struct napi_struct *napi, int budget) struct bcmasp_desc *desc; struct sk_buff *skb; dma_addr_t valid; + struct page *page; void *data; u64 flags; u32 len; + /* Hardware advances DMA_VALID as it writes each descriptor + * (RBUF_4K streaming mode); software chases with rx_edpkt_dma_read. + */ valid = rx_edpkt_dma_rq(intf, RX_EDPKT_DMA_VALID) + 1; if (valid == intf->rx_edpkt_dma_addr + DESC_RING_SIZE) valid = intf->rx_edpkt_dma_addr; @@ -493,12 +498,12 @@ static int bcmasp_rx_poll(struct napi_struct *napi, int budget) while ((processed < budget) && (valid != intf->rx_edpkt_dma_read)) { desc = &intf->rx_edpkt_cpu[intf->rx_edpkt_index]; - /* Ensure that descriptor has been fully written to DRAM by - * hardware before reading by the CPU + /* Ensure the descriptor has been fully written to DRAM by + * the hardware before the CPU reads it. */ rmb(); - /* Calculate virt addr by offsetting from physical addr */ + /* Locate the packet data inside the streaming ring buffer. */ data = intf->rx_ring_cpu + (DESC_ADDR(desc->buf) - intf->rx_ring_dma); @@ -524,19 +529,38 @@ static int bcmasp_rx_poll(struct napi_struct *napi, int budget) len = desc->size; - skb = napi_alloc_skb(napi, len); - if (!skb) { + /* Allocate a page pool page as the SKB data area so the + * kernel can recycle it efficiently after the packet is + * consumed, avoiding repeated slab allocations. + */ + page = page_pool_dev_alloc_pages(intf->rx_page_pool); + if (!page) { u64_stats_update_begin(&stats->syncp); u64_stats_inc(&stats->rx_dropped); u64_stats_update_end(&stats->syncp); intf->mib.alloc_rx_skb_failed++; + goto next; + } + skb = napi_build_skb(page_address(page), PAGE_SIZE); + if (!skb) { + u64_stats_update_begin(&stats->syncp); + u64_stats_inc(&stats->rx_dropped); + u64_stats_update_end(&stats->syncp); + intf->mib.alloc_rx_skb_failed++; + page_pool_recycle_direct(intf->rx_page_pool, page); goto next; } + /* Reserve headroom then copy the full descriptor payload + * (hardware prepends a 2-byte alignment pad at the start). + */ + skb_reserve(skb, NET_SKB_PAD); skb_put(skb, len); memcpy(skb->data, data, len); + skb_mark_for_recycle(skb); + /* Skip the 2-byte hardware alignment pad. */ skb_pull(skb, 2); len -= 2; if (likely(intf->crc_fwd)) { @@ -558,6 +582,7 @@ static int bcmasp_rx_poll(struct napi_struct *napi, int budget) u64_stats_update_end(&stats->syncp); next: + /* Return this portion of the streaming ring buffer to HW. */ rx_edpkt_cfg_wq(intf, (DESC_ADDR(desc->buf) + desc->size), RX_EDPKT_RING_BUFFER_READ); @@ -661,12 +686,31 @@ static void bcmasp_adj_link(struct net_device *dev) phy_print_status(phydev); } -static int bcmasp_alloc_buffers(struct bcmasp_intf *intf) +static struct page_pool * +bcmasp_rx_page_pool_create(struct bcmasp_intf *intf) +{ + struct page_pool_params pp_params = { + .order = 0, + .flags = 0, + .pool_size = NUM_4K_BUFFERS, + .nid = NUMA_NO_NODE, + .dev = &intf->parent->pdev->dev, + .napi = &intf->rx_napi, + .netdev = intf->ndev, + .offset = 0, + .max_len = PAGE_SIZE, + }; + + return page_pool_create(&pp_params); +} + +static int bcmasp_alloc_rx_buffers(struct bcmasp_intf *intf) { struct device *kdev = &intf->parent->pdev->dev; struct page *buffer_pg; + int ret; - /* Alloc RX */ + /* Contiguous streaming ring that hardware writes packet data into. */ intf->rx_buf_order = get_order(RING_BUFFER_SIZE); buffer_pg = alloc_pages(GFP_KERNEL, intf->rx_buf_order); if (!buffer_pg) @@ -675,13 +719,55 @@ static int bcmasp_alloc_buffers(struct bcmasp_intf *intf) intf->rx_ring_cpu = page_to_virt(buffer_pg); intf->rx_ring_dma = dma_map_page(kdev, buffer_pg, 0, RING_BUFFER_SIZE, DMA_FROM_DEVICE); - if (dma_mapping_error(kdev, intf->rx_ring_dma)) - goto free_rx_buffer; + if (dma_mapping_error(kdev, intf->rx_ring_dma)) { + ret = -ENOMEM; + goto free_ring_pages; + } + + /* Page pool for SKB data areas (copy targets, not DMA buffers). */ + intf->rx_page_pool = bcmasp_rx_page_pool_create(intf); + if (IS_ERR(intf->rx_page_pool)) { + ret = PTR_ERR(intf->rx_page_pool); + intf->rx_page_pool = NULL; + goto free_ring_dma; + } + + return 0; + +free_ring_dma: + dma_unmap_page(kdev, intf->rx_ring_dma, RING_BUFFER_SIZE, + DMA_FROM_DEVICE); +free_ring_pages: + __free_pages(buffer_pg, intf->rx_buf_order); + return ret; +} + +static void bcmasp_reclaim_rx_buffers(struct bcmasp_intf *intf) +{ + struct device *kdev = &intf->parent->pdev->dev; + + page_pool_destroy(intf->rx_page_pool); + intf->rx_page_pool = NULL; + dma_unmap_page(kdev, intf->rx_ring_dma, RING_BUFFER_SIZE, + DMA_FROM_DEVICE); + __free_pages(virt_to_page(intf->rx_ring_cpu), intf->rx_buf_order); +} + +static int bcmasp_alloc_buffers(struct bcmasp_intf *intf) +{ + struct device *kdev = &intf->parent->pdev->dev; + int ret; + + /* Alloc RX */ + ret = bcmasp_alloc_rx_buffers(intf); + if (ret) + return ret; intf->rx_edpkt_cpu = dma_alloc_coherent(kdev, DESC_RING_SIZE, - &intf->rx_edpkt_dma_addr, GFP_KERNEL); + &intf->rx_edpkt_dma_addr, + GFP_KERNEL); if (!intf->rx_edpkt_cpu) - goto free_rx_buffer_dma; + goto free_rx_buffers; /* Alloc TX */ intf->tx_spb_cpu = dma_alloc_coherent(kdev, DESC_RING_SIZE, @@ -701,11 +787,8 @@ free_tx_spb_dma: free_rx_edpkt_dma: dma_free_coherent(kdev, DESC_RING_SIZE, intf->rx_edpkt_cpu, intf->rx_edpkt_dma_addr); -free_rx_buffer_dma: - dma_unmap_page(kdev, intf->rx_ring_dma, RING_BUFFER_SIZE, - DMA_FROM_DEVICE); -free_rx_buffer: - __free_pages(buffer_pg, intf->rx_buf_order); +free_rx_buffers: + bcmasp_reclaim_rx_buffers(intf); return -ENOMEM; } @@ -717,9 +800,7 @@ static void bcmasp_reclaim_free_buffers(struct bcmasp_intf *intf) /* RX buffers */ dma_free_coherent(kdev, DESC_RING_SIZE, intf->rx_edpkt_cpu, intf->rx_edpkt_dma_addr); - dma_unmap_page(kdev, intf->rx_ring_dma, RING_BUFFER_SIZE, - DMA_FROM_DEVICE); - __free_pages(virt_to_page(intf->rx_ring_cpu), intf->rx_buf_order); + bcmasp_reclaim_rx_buffers(intf); /* TX buffers */ dma_free_coherent(kdev, DESC_RING_SIZE, intf->tx_spb_cpu, @@ -738,7 +819,7 @@ static void bcmasp_init_rx(struct bcmasp_intf *intf) /* Make sure channels are disabled */ rx_edpkt_cfg_wl(intf, 0x0, RX_EDPKT_CFG_ENABLE); - /* Rx SPB */ + /* Streaming data ring: hardware writes raw packet bytes here. */ rx_edpkt_cfg_wq(intf, intf->rx_ring_dma, RX_EDPKT_RING_BUFFER_READ); rx_edpkt_cfg_wq(intf, intf->rx_ring_dma, RX_EDPKT_RING_BUFFER_WRITE); rx_edpkt_cfg_wq(intf, intf->rx_ring_dma, RX_EDPKT_RING_BUFFER_BASE); @@ -747,7 +828,9 @@ static void bcmasp_init_rx(struct bcmasp_intf *intf) rx_edpkt_cfg_wq(intf, intf->rx_ring_dma_valid, RX_EDPKT_RING_BUFFER_VALID); - /* EDPKT */ + /* EDPKT descriptor ring: hardware fills descriptors pointing into + * the streaming ring buffer above (RBUF_4K mode). + */ rx_edpkt_cfg_wl(intf, (RX_EDPKT_CFG_CFG0_RBUF_4K << RX_EDPKT_CFG_CFG0_DBUF_SHIFT) | (RX_EDPKT_CFG_CFG0_64_ALN << diff --git a/drivers/net/ethernet/broadcom/asp2/bcmasp_intf_defs.h b/drivers/net/ethernet/broadcom/asp2/bcmasp_intf_defs.h index af7418348e81..0318f257452a 100644 --- a/drivers/net/ethernet/broadcom/asp2/bcmasp_intf_defs.h +++ b/drivers/net/ethernet/broadcom/asp2/bcmasp_intf_defs.h @@ -246,6 +246,10 @@ ((((intf)->channel - 6) * 0x14) + 0xa2000) #define RX_SPB_TOP_BLKOUT 0x00 +/* + * Number of 4 KB pages that make up the contiguous RBUF_4K streaming ring + * and the page pool used as copy-target SKB data areas. + */ #define NUM_4K_BUFFERS 32 #define RING_BUFFER_SIZE (PAGE_SIZE * NUM_4K_BUFFERS) diff --git a/drivers/net/ethernet/broadcom/bnge/Makefile b/drivers/net/ethernet/broadcom/bnge/Makefile index fa604ee20264..8e07cb307d21 100644 --- a/drivers/net/ethernet/broadcom/bnge/Makefile +++ b/drivers/net/ethernet/broadcom/bnge/Makefile @@ -11,4 +11,5 @@ bng_en-y := bnge_core.o \ bnge_netdev.o \ bnge_ethtool.o \ bnge_auxr.o \ - bnge_txrx.o + bnge_txrx.o \ + bnge_link.o diff --git a/drivers/net/ethernet/broadcom/bnge/bnge.h b/drivers/net/ethernet/broadcom/bnge/bnge.h index f376913aa321..f21cff651fd4 100644 --- a/drivers/net/ethernet/broadcom/bnge/bnge.h +++ b/drivers/net/ethernet/broadcom/bnge/bnge.h @@ -94,6 +94,11 @@ struct bnge_queue_info { u8 queue_profile; }; +#define BNGE_PHY_FLAGS2_SHIFT 8 +#define BNGE_PHY_FL_NO_FCS PORT_PHY_QCAPS_RESP_FLAGS_NO_FCS +#define BNGE_PHY_FL_NO_PAUSE \ + (PORT_PHY_QCAPS_RESP_FLAGS2_PAUSE_UNSUPPORTED << 8) + struct bnge_dev { struct device *dev; struct pci_dev *pdev; @@ -207,6 +212,11 @@ struct bnge_dev { struct bnge_auxr_priv *aux_priv; struct bnge_auxr_dev *auxr_dev; + + struct bnge_link_info link_info; + + /* Copied from flags and flags2 in hwrm_port_phy_qcaps_output */ + u32 phy_flags; }; static inline bool bnge_is_roce_en(struct bnge_dev *bd) diff --git a/drivers/net/ethernet/broadcom/bnge/bnge_auxr.c b/drivers/net/ethernet/broadcom/bnge/bnge_auxr.c index b942076762ef..67e93e17d4d9 100644 --- a/drivers/net/ethernet/broadcom/bnge/bnge_auxr.c +++ b/drivers/net/ethernet/broadcom/bnge/bnge_auxr.c @@ -194,6 +194,7 @@ void bnge_rdma_aux_device_add(struct bnge_dev *bd) dev_warn(bd->dev, "Failed to add auxiliary device for ROCE\n"); auxiliary_device_uninit(aux_dev); bd->flags &= ~BNGE_EN_ROCE; + return; } bd->auxr_dev->net = bd->netdev; diff --git a/drivers/net/ethernet/broadcom/bnge/bnge_core.c b/drivers/net/ethernet/broadcom/bnge/bnge_core.c index b4090283df0f..1c14c5fe8d61 100644 --- a/drivers/net/ethernet/broadcom/bnge/bnge_core.c +++ b/drivers/net/ethernet/broadcom/bnge/bnge_core.c @@ -10,6 +10,7 @@ #include "bnge_devlink.h" #include "bnge_hwrm.h" #include "bnge_hwrm_lib.h" +#include "bnge_link.h" MODULE_LICENSE("GPL"); MODULE_DESCRIPTION(DRV_SUMMARY); diff --git a/drivers/net/ethernet/broadcom/bnge/bnge_ethtool.c b/drivers/net/ethernet/broadcom/bnge/bnge_ethtool.c index 569371c1b4f2..2467e44de291 100644 --- a/drivers/net/ethernet/broadcom/bnge/bnge_ethtool.c +++ b/drivers/net/ethernet/broadcom/bnge/bnge_ethtool.c @@ -11,6 +11,294 @@ #include "bnge.h" #include "bnge_ethtool.h" +#include "bnge_hwrm_lib.h" + +static int bnge_nway_reset(struct net_device *dev) +{ + struct bnge_net *bn = netdev_priv(dev); + struct bnge_dev *bd = bn->bd; + bool set_pause = false; + int rc = 0; + + if (!BNGE_PHY_CFG_ABLE(bd)) + return -EOPNOTSUPP; + + if (!(bn->eth_link_info.autoneg & BNGE_AUTONEG_SPEED)) + return -EINVAL; + + if (!(bd->phy_flags & BNGE_PHY_FL_NO_PAUSE)) + set_pause = true; + + if (netif_running(dev)) + rc = bnge_hwrm_set_link_setting(bn, set_pause); + + return rc; +} + +static const char * const bnge_ring_q_stats_str[] = { + "ucast_packets", + "mcast_packets", + "bcast_packets", + "ucast_bytes", + "mcast_bytes", + "bcast_bytes", +}; + +static const char * const bnge_ring_tpa2_stats_str[] = { + "tpa_eligible_pkt", + "tpa_eligible_bytes", + "tpa_pkt", + "tpa_bytes", + "tpa_errors", + "tpa_events", +}; + +#define BNGE_RX_PORT_STATS_ENTRY(suffix) \ + { BNGE_RX_STATS_OFFSET(rx_##suffix), "rxp_" __stringify(suffix) } + +#define BNGE_TX_PORT_STATS_ENTRY(suffix) \ + { BNGE_TX_STATS_OFFSET(tx_##suffix), "txp_" __stringify(suffix) } + +#define BNGE_RX_STATS_EXT_ENTRY(counter) \ + { BNGE_RX_STATS_EXT_OFFSET(counter), __stringify(counter) } + +#define BNGE_TX_STATS_EXT_ENTRY(counter) \ + { BNGE_TX_STATS_EXT_OFFSET(counter), __stringify(counter) } + +#define BNGE_RX_STATS_EXT_PFC_ENTRY(n) \ + BNGE_RX_STATS_EXT_ENTRY(pfc_pri##n##_rx_duration_us), \ + BNGE_RX_STATS_EXT_ENTRY(pfc_pri##n##_rx_transitions) + +#define BNGE_TX_STATS_EXT_PFC_ENTRY(n) \ + BNGE_TX_STATS_EXT_ENTRY(pfc_pri##n##_tx_duration_us), \ + BNGE_TX_STATS_EXT_ENTRY(pfc_pri##n##_tx_transitions) + +#define BNGE_RX_STATS_EXT_PFC_ENTRIES \ + BNGE_RX_STATS_EXT_PFC_ENTRY(0), \ + BNGE_RX_STATS_EXT_PFC_ENTRY(1), \ + BNGE_RX_STATS_EXT_PFC_ENTRY(2), \ + BNGE_RX_STATS_EXT_PFC_ENTRY(3), \ + BNGE_RX_STATS_EXT_PFC_ENTRY(4), \ + BNGE_RX_STATS_EXT_PFC_ENTRY(5), \ + BNGE_RX_STATS_EXT_PFC_ENTRY(6), \ + BNGE_RX_STATS_EXT_PFC_ENTRY(7) + +#define BNGE_TX_STATS_EXT_PFC_ENTRIES \ + BNGE_TX_STATS_EXT_PFC_ENTRY(0), \ + BNGE_TX_STATS_EXT_PFC_ENTRY(1), \ + BNGE_TX_STATS_EXT_PFC_ENTRY(2), \ + BNGE_TX_STATS_EXT_PFC_ENTRY(3), \ + BNGE_TX_STATS_EXT_PFC_ENTRY(4), \ + BNGE_TX_STATS_EXT_PFC_ENTRY(5), \ + BNGE_TX_STATS_EXT_PFC_ENTRY(6), \ + BNGE_TX_STATS_EXT_PFC_ENTRY(7) + +#define BNGE_RX_STATS_EXT_COS_ENTRY(n) \ + BNGE_RX_STATS_EXT_ENTRY(rx_bytes_cos##n), \ + BNGE_RX_STATS_EXT_ENTRY(rx_packets_cos##n) + +#define BNGE_TX_STATS_EXT_COS_ENTRY(n) \ + BNGE_TX_STATS_EXT_ENTRY(tx_bytes_cos##n), \ + BNGE_TX_STATS_EXT_ENTRY(tx_packets_cos##n) + +#define BNGE_RX_STATS_EXT_COS_ENTRIES \ + BNGE_RX_STATS_EXT_COS_ENTRY(0), \ + BNGE_RX_STATS_EXT_COS_ENTRY(1), \ + BNGE_RX_STATS_EXT_COS_ENTRY(2), \ + BNGE_RX_STATS_EXT_COS_ENTRY(3), \ + BNGE_RX_STATS_EXT_COS_ENTRY(4), \ + BNGE_RX_STATS_EXT_COS_ENTRY(5), \ + BNGE_RX_STATS_EXT_COS_ENTRY(6), \ + BNGE_RX_STATS_EXT_COS_ENTRY(7) \ + +#define BNGE_TX_STATS_EXT_COS_ENTRIES \ + BNGE_TX_STATS_EXT_COS_ENTRY(0), \ + BNGE_TX_STATS_EXT_COS_ENTRY(1), \ + BNGE_TX_STATS_EXT_COS_ENTRY(2), \ + BNGE_TX_STATS_EXT_COS_ENTRY(3), \ + BNGE_TX_STATS_EXT_COS_ENTRY(4), \ + BNGE_TX_STATS_EXT_COS_ENTRY(5), \ + BNGE_TX_STATS_EXT_COS_ENTRY(6), \ + BNGE_TX_STATS_EXT_COS_ENTRY(7) \ + +#define BNGE_RX_STATS_EXT_DISCARD_COS_ENTRY(n) \ + BNGE_RX_STATS_EXT_ENTRY(rx_discard_bytes_cos##n), \ + BNGE_RX_STATS_EXT_ENTRY(rx_discard_packets_cos##n) + +#define BNGE_RX_STATS_EXT_DISCARD_COS_ENTRIES \ + BNGE_RX_STATS_EXT_DISCARD_COS_ENTRY(0), \ + BNGE_RX_STATS_EXT_DISCARD_COS_ENTRY(1), \ + BNGE_RX_STATS_EXT_DISCARD_COS_ENTRY(2), \ + BNGE_RX_STATS_EXT_DISCARD_COS_ENTRY(3), \ + BNGE_RX_STATS_EXT_DISCARD_COS_ENTRY(4), \ + BNGE_RX_STATS_EXT_DISCARD_COS_ENTRY(5), \ + BNGE_RX_STATS_EXT_DISCARD_COS_ENTRY(6), \ + BNGE_RX_STATS_EXT_DISCARD_COS_ENTRY(7) + +#define BNGE_RX_STATS_PRI_ENTRY(counter, n) \ + { BNGE_RX_STATS_EXT_OFFSET(counter##_cos0), \ + __stringify(counter##_pri##n) } + +#define BNGE_TX_STATS_PRI_ENTRY(counter, n) \ + { BNGE_TX_STATS_EXT_OFFSET(counter##_cos0), \ + __stringify(counter##_pri##n) } + +#define BNGE_RX_STATS_PRI_ENTRIES(counter) \ + BNGE_RX_STATS_PRI_ENTRY(counter, 0), \ + BNGE_RX_STATS_PRI_ENTRY(counter, 1), \ + BNGE_RX_STATS_PRI_ENTRY(counter, 2), \ + BNGE_RX_STATS_PRI_ENTRY(counter, 3), \ + BNGE_RX_STATS_PRI_ENTRY(counter, 4), \ + BNGE_RX_STATS_PRI_ENTRY(counter, 5), \ + BNGE_RX_STATS_PRI_ENTRY(counter, 6), \ + BNGE_RX_STATS_PRI_ENTRY(counter, 7) + +#define BNGE_TX_STATS_PRI_ENTRIES(counter) \ + BNGE_TX_STATS_PRI_ENTRY(counter, 0), \ + BNGE_TX_STATS_PRI_ENTRY(counter, 1), \ + BNGE_TX_STATS_PRI_ENTRY(counter, 2), \ + BNGE_TX_STATS_PRI_ENTRY(counter, 3), \ + BNGE_TX_STATS_PRI_ENTRY(counter, 4), \ + BNGE_TX_STATS_PRI_ENTRY(counter, 5), \ + BNGE_TX_STATS_PRI_ENTRY(counter, 6), \ + BNGE_TX_STATS_PRI_ENTRY(counter, 7) + +#define NUM_RING_Q_HW_STATS ARRAY_SIZE(bnge_ring_q_stats_str) + +static const struct { + long offset; + char string[ETH_GSTRING_LEN]; +} bnge_tx_port_stats_ext_arr[] = { + BNGE_TX_STATS_EXT_COS_ENTRIES, + BNGE_TX_STATS_EXT_PFC_ENTRIES, +}; + +static const struct { + long base_off; + char string[ETH_GSTRING_LEN]; +} bnge_rx_bytes_pri_arr[] = { + BNGE_RX_STATS_PRI_ENTRIES(rx_bytes), +}; + +static const struct { + long base_off; + char string[ETH_GSTRING_LEN]; +} bnge_rx_pkts_pri_arr[] = { + BNGE_RX_STATS_PRI_ENTRIES(rx_packets), +}; + +static const struct { + long base_off; + char string[ETH_GSTRING_LEN]; +} bnge_tx_bytes_pri_arr[] = { + BNGE_TX_STATS_PRI_ENTRIES(tx_bytes), +}; + +static const struct { + long base_off; + char string[ETH_GSTRING_LEN]; +} bnge_tx_pkts_pri_arr[] = { + BNGE_TX_STATS_PRI_ENTRIES(tx_packets), +}; + +static const struct { + long offset; + char string[ETH_GSTRING_LEN]; +} bnge_port_stats_arr[] = { + BNGE_RX_PORT_STATS_ENTRY(good_vlan_frames), + BNGE_RX_PORT_STATS_ENTRY(mtu_err_frames), + BNGE_RX_PORT_STATS_ENTRY(tagged_frames), + BNGE_RX_PORT_STATS_ENTRY(double_tagged_frames), + BNGE_RX_PORT_STATS_ENTRY(pfc_ena_frames_pri0), + BNGE_RX_PORT_STATS_ENTRY(pfc_ena_frames_pri1), + BNGE_RX_PORT_STATS_ENTRY(pfc_ena_frames_pri2), + BNGE_RX_PORT_STATS_ENTRY(pfc_ena_frames_pri3), + BNGE_RX_PORT_STATS_ENTRY(pfc_ena_frames_pri4), + BNGE_RX_PORT_STATS_ENTRY(pfc_ena_frames_pri5), + BNGE_RX_PORT_STATS_ENTRY(pfc_ena_frames_pri6), + BNGE_RX_PORT_STATS_ENTRY(pfc_ena_frames_pri7), + BNGE_RX_PORT_STATS_ENTRY(eee_lpi_events), + BNGE_RX_PORT_STATS_ENTRY(eee_lpi_duration), + BNGE_RX_PORT_STATS_ENTRY(runt_bytes), + BNGE_RX_PORT_STATS_ENTRY(runt_frames), + + BNGE_TX_PORT_STATS_ENTRY(good_vlan_frames), + BNGE_TX_PORT_STATS_ENTRY(jabber_frames), + BNGE_TX_PORT_STATS_ENTRY(fcs_err_frames), + BNGE_TX_PORT_STATS_ENTRY(pfc_ena_frames_pri0), + BNGE_TX_PORT_STATS_ENTRY(pfc_ena_frames_pri1), + BNGE_TX_PORT_STATS_ENTRY(pfc_ena_frames_pri2), + BNGE_TX_PORT_STATS_ENTRY(pfc_ena_frames_pri3), + BNGE_TX_PORT_STATS_ENTRY(pfc_ena_frames_pri4), + BNGE_TX_PORT_STATS_ENTRY(pfc_ena_frames_pri5), + BNGE_TX_PORT_STATS_ENTRY(pfc_ena_frames_pri6), + BNGE_TX_PORT_STATS_ENTRY(pfc_ena_frames_pri7), + BNGE_TX_PORT_STATS_ENTRY(eee_lpi_events), + BNGE_TX_PORT_STATS_ENTRY(eee_lpi_duration), + BNGE_TX_PORT_STATS_ENTRY(xthol_frames), +}; + +static const struct { + long offset; + char string[ETH_GSTRING_LEN]; +} bnge_port_stats_ext_arr[] = { + BNGE_RX_STATS_EXT_ENTRY(continuous_pause_events), + BNGE_RX_STATS_EXT_ENTRY(resume_pause_events), + BNGE_RX_STATS_EXT_ENTRY(continuous_roce_pause_events), + BNGE_RX_STATS_EXT_ENTRY(resume_roce_pause_events), + BNGE_RX_STATS_EXT_COS_ENTRIES, + BNGE_RX_STATS_EXT_PFC_ENTRIES, + BNGE_RX_STATS_EXT_ENTRY(rx_bits), + BNGE_RX_STATS_EXT_ENTRY(rx_buffer_passed_threshold), + BNGE_RX_STATS_EXT_DISCARD_COS_ENTRIES, + BNGE_RX_STATS_EXT_ENTRY(rx_filter_miss), +}; + +static int bnge_get_num_tpa_ring_stats(struct bnge_dev *bd) +{ + if (BNGE_SUPPORTS_TPA(bd)) + return BNGE_NUM_TPA_RING_STATS; + return 0; +} + +#define BNGE_NUM_PORT_STATS ARRAY_SIZE(bnge_port_stats_arr) +#define BNGE_NUM_STATS_PRI \ + (ARRAY_SIZE(bnge_rx_bytes_pri_arr) + \ + ARRAY_SIZE(bnge_rx_pkts_pri_arr) + \ + ARRAY_SIZE(bnge_tx_bytes_pri_arr) + \ + ARRAY_SIZE(bnge_tx_pkts_pri_arr)) + +static int bnge_get_num_ring_stats(struct bnge_dev *bd) +{ + int rx, tx; + + rx = NUM_RING_Q_HW_STATS + bnge_get_num_tpa_ring_stats(bd); + tx = NUM_RING_Q_HW_STATS; + return rx * bd->rx_nr_rings + + tx * bd->tx_nr_rings_per_tc; +} + +static u32 bnge_get_num_stats(struct bnge_net *bn) +{ + u32 num_stats = bnge_get_num_ring_stats(bn->bd); + u32 len; + + if (bn->flags & BNGE_FLAG_PORT_STATS) + num_stats += BNGE_NUM_PORT_STATS; + + if (bn->flags & BNGE_FLAG_PORT_STATS_EXT) { + len = min_t(u32, bn->fw_rx_stats_ext_size, + ARRAY_SIZE(bnge_port_stats_ext_arr)); + num_stats += len; + len = min_t(u32, bn->fw_tx_stats_ext_size, + ARRAY_SIZE(bnge_tx_port_stats_ext_arr)); + num_stats += len; + if (bn->pri2cos_valid) + num_stats += BNGE_NUM_STATS_PRI; + } + + return num_stats; +} static void bnge_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info) @@ -23,8 +311,452 @@ static void bnge_get_drvinfo(struct net_device *dev, strscpy(info->bus_info, pci_name(bd->pdev), sizeof(info->bus_info)); } +static int bnge_get_sset_count(struct net_device *dev, int sset) +{ + struct bnge_net *bn = netdev_priv(dev); + + switch (sset) { + case ETH_SS_STATS: + return bnge_get_num_stats(bn); + default: + return -EOPNOTSUPP; + } +} + +static bool is_rx_ring(struct bnge_dev *bd, u16 ring_num) +{ + return ring_num < bd->rx_nr_rings; +} + +static bool is_tx_ring(struct bnge_dev *bd, u16 ring_num) +{ + u16 tx_base = 0; + + if (!(bd->flags & BNGE_EN_SHARED_CHNL)) + tx_base = bd->rx_nr_rings; + + return ring_num >= tx_base && ring_num < (tx_base + bd->tx_nr_rings_per_tc); +} + +static void bnge_get_ethtool_stats(struct net_device *dev, + struct ethtool_stats *stats, u64 *buf) +{ + struct bnge_net *bn = netdev_priv(dev); + struct bnge_dev *bd = bn->bd; + u32 tpa_stats; + u32 i, j = 0; + + if (!bn->bnapi) { + j += bnge_get_num_ring_stats(bd); + goto skip_ring_stats; + } + + tpa_stats = bnge_get_num_tpa_ring_stats(bd); + for (i = 0; i < bd->nq_nr_rings; i++) { + struct bnge_napi *bnapi = bn->bnapi[i]; + struct bnge_nq_ring_info *nqr; + u64 *sw_stats; + int k; + + nqr = &bnapi->nq_ring; + sw_stats = nqr->stats.sw_stats; + + if (is_rx_ring(bd, i)) { + buf[j++] = BNGE_GET_RING_STATS64(sw_stats, rx_ucast_pkts); + buf[j++] = BNGE_GET_RING_STATS64(sw_stats, rx_mcast_pkts); + buf[j++] = BNGE_GET_RING_STATS64(sw_stats, rx_bcast_pkts); + buf[j++] = BNGE_GET_RING_STATS64(sw_stats, rx_ucast_bytes); + buf[j++] = BNGE_GET_RING_STATS64(sw_stats, rx_mcast_bytes); + buf[j++] = BNGE_GET_RING_STATS64(sw_stats, rx_bcast_bytes); + } + if (is_tx_ring(bd, i)) { + buf[j++] = BNGE_GET_RING_STATS64(sw_stats, tx_ucast_pkts); + buf[j++] = BNGE_GET_RING_STATS64(sw_stats, tx_mcast_pkts); + buf[j++] = BNGE_GET_RING_STATS64(sw_stats, tx_bcast_pkts); + buf[j++] = BNGE_GET_RING_STATS64(sw_stats, tx_ucast_bytes); + buf[j++] = BNGE_GET_RING_STATS64(sw_stats, tx_mcast_bytes); + buf[j++] = BNGE_GET_RING_STATS64(sw_stats, tx_bcast_bytes); + } + if (!tpa_stats || !is_rx_ring(bd, i)) + continue; + + k = BNGE_NUM_RX_RING_STATS + BNGE_NUM_TX_RING_STATS; + for (; k < BNGE_NUM_RX_RING_STATS + BNGE_NUM_TX_RING_STATS + + tpa_stats; j++, k++) + buf[j] = sw_stats[k]; + } + +skip_ring_stats: + if (bn->flags & BNGE_FLAG_PORT_STATS) { + u64 *port_stats = bn->port_stats.sw_stats; + + for (i = 0; i < BNGE_NUM_PORT_STATS; i++, j++) + buf[j] = *(port_stats + bnge_port_stats_arr[i].offset); + } + if (bn->flags & BNGE_FLAG_PORT_STATS_EXT) { + u64 *rx_port_stats_ext = bn->rx_port_stats_ext.sw_stats; + u64 *tx_port_stats_ext = bn->tx_port_stats_ext.sw_stats; + u32 len; + + len = min_t(u32, bn->fw_rx_stats_ext_size, + ARRAY_SIZE(bnge_port_stats_ext_arr)); + for (i = 0; i < len; i++, j++) { + buf[j] = *(rx_port_stats_ext + + bnge_port_stats_ext_arr[i].offset); + } + len = min_t(u32, bn->fw_tx_stats_ext_size, + ARRAY_SIZE(bnge_tx_port_stats_ext_arr)); + for (i = 0; i < len; i++, j++) { + buf[j] = *(tx_port_stats_ext + + bnge_tx_port_stats_ext_arr[i].offset); + } + if (bn->pri2cos_valid) { + for (i = 0; i < 8; i++, j++) { + long n = bnge_rx_bytes_pri_arr[i].base_off + + bn->pri2cos_idx[i]; + + buf[j] = *(rx_port_stats_ext + n); + } + for (i = 0; i < 8; i++, j++) { + long n = bnge_rx_pkts_pri_arr[i].base_off + + bn->pri2cos_idx[i]; + + buf[j] = *(rx_port_stats_ext + n); + } + for (i = 0; i < 8; i++, j++) { + long n = bnge_tx_bytes_pri_arr[i].base_off + + bn->pri2cos_idx[i]; + + buf[j] = *(tx_port_stats_ext + n); + } + for (i = 0; i < 8; i++, j++) { + long n = bnge_tx_pkts_pri_arr[i].base_off + + bn->pri2cos_idx[i]; + + buf[j] = *(tx_port_stats_ext + n); + } + } + } +} + +static void bnge_get_strings(struct net_device *dev, u32 stringset, u8 *buf) +{ + struct bnge_net *bn = netdev_priv(dev); + struct bnge_dev *bd = bn->bd; + u32 i, j, num_str; + const char *str; + + switch (stringset) { + case ETH_SS_STATS: + for (i = 0; i < bd->nq_nr_rings; i++) { + if (is_rx_ring(bd, i)) + for (j = 0; j < NUM_RING_Q_HW_STATS; j++) { + str = bnge_ring_q_stats_str[j]; + ethtool_sprintf(&buf, "rxq%d_%s", i, + str); + } + if (is_tx_ring(bd, i)) + for (j = 0; j < NUM_RING_Q_HW_STATS; j++) { + str = bnge_ring_q_stats_str[j]; + ethtool_sprintf(&buf, "txq%d_%s", i, + str); + } + num_str = bnge_get_num_tpa_ring_stats(bd); + if (!num_str || !is_rx_ring(bd, i)) + continue; + + for (j = 0; j < num_str; j++) { + str = bnge_ring_tpa2_stats_str[j]; + ethtool_sprintf(&buf, "rxq%d_%s", i, str); + } + } + + if (bn->flags & BNGE_FLAG_PORT_STATS) + for (i = 0; i < BNGE_NUM_PORT_STATS; i++) { + str = bnge_port_stats_arr[i].string; + ethtool_puts(&buf, str); + } + + if (bn->flags & BNGE_FLAG_PORT_STATS_EXT) { + u32 len; + + len = min_t(u32, bn->fw_rx_stats_ext_size, + ARRAY_SIZE(bnge_port_stats_ext_arr)); + for (i = 0; i < len; i++) { + str = bnge_port_stats_ext_arr[i].string; + ethtool_puts(&buf, str); + } + + len = min_t(u32, bn->fw_tx_stats_ext_size, + ARRAY_SIZE(bnge_tx_port_stats_ext_arr)); + for (i = 0; i < len; i++) { + str = bnge_tx_port_stats_ext_arr[i].string; + ethtool_puts(&buf, str); + } + + if (bn->pri2cos_valid) { + for (i = 0; i < 8; i++) { + str = bnge_rx_bytes_pri_arr[i].string; + ethtool_puts(&buf, str); + } + + for (i = 0; i < 8; i++) { + str = bnge_rx_pkts_pri_arr[i].string; + ethtool_puts(&buf, str); + } + + for (i = 0; i < 8; i++) { + str = bnge_tx_bytes_pri_arr[i].string; + ethtool_puts(&buf, str); + } + + for (i = 0; i < 8; i++) { + str = bnge_tx_pkts_pri_arr[i].string; + ethtool_puts(&buf, str); + } + } + } + break; + default: + netdev_err(bd->netdev, "%s invalid request %x\n", + __func__, stringset); + break; + } +} + +static void bnge_get_eth_phy_stats(struct net_device *dev, + struct ethtool_eth_phy_stats *phy_stats) +{ + struct bnge_net *bn = netdev_priv(dev); + u64 *rx; + + if (!(bn->flags & BNGE_FLAG_PORT_STATS_EXT)) + return; + + rx = bn->rx_port_stats_ext.sw_stats; + phy_stats->SymbolErrorDuringCarrier = + *(rx + BNGE_RX_STATS_EXT_OFFSET(rx_pcs_symbol_err)); +} + +static void bnge_get_eth_mac_stats(struct net_device *dev, + struct ethtool_eth_mac_stats *mac_stats) +{ + struct bnge_net *bn = netdev_priv(dev); + u64 *rx, *tx; + + if (!(bn->flags & BNGE_FLAG_PORT_STATS)) + return; + + rx = bn->port_stats.sw_stats; + tx = bn->port_stats.sw_stats + BNGE_TX_PORT_STATS_BYTE_OFFSET / 8; + + mac_stats->FramesReceivedOK = + BNGE_GET_RX_PORT_STATS64(rx, rx_good_frames); + mac_stats->FramesTransmittedOK = + BNGE_GET_TX_PORT_STATS64(tx, tx_good_frames); + mac_stats->FrameCheckSequenceErrors = + BNGE_GET_RX_PORT_STATS64(rx, rx_fcs_err_frames); + mac_stats->AlignmentErrors = + BNGE_GET_RX_PORT_STATS64(rx, rx_align_err_frames); + mac_stats->OutOfRangeLengthField = + BNGE_GET_RX_PORT_STATS64(rx, rx_oor_len_frames); + mac_stats->OctetsReceivedOK = BNGE_GET_RX_PORT_STATS64(rx, rx_bytes); + mac_stats->OctetsTransmittedOK = BNGE_GET_TX_PORT_STATS64(tx, tx_bytes); + mac_stats->MulticastFramesReceivedOK = + BNGE_GET_RX_PORT_STATS64(rx, rx_mcast_frames); + mac_stats->BroadcastFramesReceivedOK = + BNGE_GET_RX_PORT_STATS64(rx, rx_bcast_frames); + mac_stats->MulticastFramesXmittedOK = + BNGE_GET_TX_PORT_STATS64(tx, tx_mcast_frames); + mac_stats->BroadcastFramesXmittedOK = + BNGE_GET_TX_PORT_STATS64(tx, tx_bcast_frames); + mac_stats->FrameTooLongErrors = + BNGE_GET_RX_PORT_STATS64(rx, rx_ovrsz_frames); +} + +static void bnge_get_eth_ctrl_stats(struct net_device *dev, + struct ethtool_eth_ctrl_stats *ctrl_stats) +{ + struct bnge_net *bn = netdev_priv(dev); + u64 *rx; + + if (!(bn->flags & BNGE_FLAG_PORT_STATS)) + return; + + rx = bn->port_stats.sw_stats; + ctrl_stats->MACControlFramesReceived = + BNGE_GET_RX_PORT_STATS64(rx, rx_ctrl_frames); +} + +static void bnge_get_pause_stats(struct net_device *dev, + struct ethtool_pause_stats *pause_stats) +{ + struct bnge_net *bn = netdev_priv(dev); + u64 *rx, *tx; + + if (!(bn->flags & BNGE_FLAG_PORT_STATS)) + return; + + rx = bn->port_stats.sw_stats; + tx = bn->port_stats.sw_stats + BNGE_TX_PORT_STATS_BYTE_OFFSET / 8; + + pause_stats->rx_pause_frames = + BNGE_GET_RX_PORT_STATS64(rx, rx_pause_frames); + pause_stats->tx_pause_frames = + BNGE_GET_TX_PORT_STATS64(tx, tx_pause_frames); +} + +static const struct ethtool_rmon_hist_range bnge_rmon_ranges[] = { + { 0, 64 }, + { 65, 127 }, + { 128, 255 }, + { 256, 511 }, + { 512, 1023 }, + { 1024, 1518 }, + { 1519, 2047 }, + { 2048, 4095 }, + { 4096, 9216 }, + { 9217, 16383 }, + {} +}; + +static void bnge_get_rmon_stats(struct net_device *dev, + struct ethtool_rmon_stats *rmon_stats, + const struct ethtool_rmon_hist_range **ranges) +{ + struct bnge_net *bn = netdev_priv(dev); + u64 *rx, *tx; + + if (!(bn->flags & BNGE_FLAG_PORT_STATS)) + return; + + rx = bn->port_stats.sw_stats; + tx = bn->port_stats.sw_stats + BNGE_TX_PORT_STATS_BYTE_OFFSET / 8; + + rmon_stats->jabbers = BNGE_GET_RX_PORT_STATS64(rx, rx_jbr_frames); + rmon_stats->oversize_pkts = + BNGE_GET_RX_PORT_STATS64(rx, rx_ovrsz_frames); + rmon_stats->undersize_pkts = + BNGE_GET_RX_PORT_STATS64(rx, rx_undrsz_frames); + + rmon_stats->hist[0] = BNGE_GET_RX_PORT_STATS64(rx, rx_64b_frames); + rmon_stats->hist[1] = BNGE_GET_RX_PORT_STATS64(rx, rx_65b_127b_frames); + rmon_stats->hist[2] = BNGE_GET_RX_PORT_STATS64(rx, rx_128b_255b_frames); + rmon_stats->hist[3] = BNGE_GET_RX_PORT_STATS64(rx, rx_256b_511b_frames); + rmon_stats->hist[4] = + BNGE_GET_RX_PORT_STATS64(rx, rx_512b_1023b_frames); + rmon_stats->hist[5] = + BNGE_GET_RX_PORT_STATS64(rx, rx_1024b_1518b_frames); + rmon_stats->hist[6] = + BNGE_GET_RX_PORT_STATS64(rx, rx_1519b_2047b_frames); + rmon_stats->hist[7] = + BNGE_GET_RX_PORT_STATS64(rx, rx_2048b_4095b_frames); + rmon_stats->hist[8] = + BNGE_GET_RX_PORT_STATS64(rx, rx_4096b_9216b_frames); + rmon_stats->hist[9] = + BNGE_GET_RX_PORT_STATS64(rx, rx_9217b_16383b_frames); + + rmon_stats->hist_tx[0] = BNGE_GET_TX_PORT_STATS64(tx, tx_64b_frames); + rmon_stats->hist_tx[1] = + BNGE_GET_TX_PORT_STATS64(tx, tx_65b_127b_frames); + rmon_stats->hist_tx[2] = + BNGE_GET_TX_PORT_STATS64(tx, tx_128b_255b_frames); + rmon_stats->hist_tx[3] = + BNGE_GET_TX_PORT_STATS64(tx, tx_256b_511b_frames); + rmon_stats->hist_tx[4] = + BNGE_GET_TX_PORT_STATS64(tx, tx_512b_1023b_frames); + rmon_stats->hist_tx[5] = + BNGE_GET_TX_PORT_STATS64(tx, tx_1024b_1518b_frames); + rmon_stats->hist_tx[6] = + BNGE_GET_TX_PORT_STATS64(tx, tx_1519b_2047b_frames); + rmon_stats->hist_tx[7] = + BNGE_GET_TX_PORT_STATS64(tx, tx_2048b_4095b_frames); + rmon_stats->hist_tx[8] = + BNGE_GET_TX_PORT_STATS64(tx, tx_4096b_9216b_frames); + rmon_stats->hist_tx[9] = + BNGE_GET_TX_PORT_STATS64(tx, tx_9217b_16383b_frames); + + *ranges = bnge_rmon_ranges; +} + +static void bnge_get_pauseparam(struct net_device *dev, + struct ethtool_pauseparam *epause) +{ + struct bnge_net *bn = netdev_priv(dev); + struct bnge_dev *bd = bn->bd; + + if (bd->phy_flags & BNGE_PHY_FL_NO_PAUSE) { + epause->autoneg = 0; + epause->rx_pause = 0; + epause->tx_pause = 0; + return; + } + + epause->autoneg = !!(bn->eth_link_info.autoneg & + BNGE_AUTONEG_FLOW_CTRL); + epause->rx_pause = !!(bn->eth_link_info.req_flow_ctrl & + BNGE_LINK_PAUSE_RX); + epause->tx_pause = !!(bn->eth_link_info.req_flow_ctrl & + BNGE_LINK_PAUSE_TX); +} + +static int bnge_set_pauseparam(struct net_device *dev, + struct ethtool_pauseparam *epause) +{ + struct bnge_ethtool_link_info old_elink_info, *elink_info; + struct bnge_net *bn = netdev_priv(dev); + struct bnge_dev *bd = bn->bd; + int rc = 0; + + if (!BNGE_PHY_CFG_ABLE(bd) || (bd->phy_flags & BNGE_PHY_FL_NO_PAUSE)) + return -EOPNOTSUPP; + + elink_info = &bn->eth_link_info; + old_elink_info = *elink_info; + + if (epause->autoneg) { + if (!(elink_info->autoneg & BNGE_AUTONEG_SPEED)) + return -EINVAL; + + elink_info->autoneg |= BNGE_AUTONEG_FLOW_CTRL; + } else { + if (elink_info->autoneg & BNGE_AUTONEG_FLOW_CTRL) + elink_info->force_link_chng = true; + elink_info->autoneg &= ~BNGE_AUTONEG_FLOW_CTRL; + } + + elink_info->req_flow_ctrl = 0; + if (epause->rx_pause) + elink_info->req_flow_ctrl |= BNGE_LINK_PAUSE_RX; + if (epause->tx_pause) + elink_info->req_flow_ctrl |= BNGE_LINK_PAUSE_TX; + + if (netif_running(dev)) { + rc = bnge_hwrm_set_pause(bn); + if (rc) + *elink_info = old_elink_info; + } + + return rc; +} + static const struct ethtool_ops bnge_ethtool_ops = { + .cap_link_lanes_supported = 1, + .get_link_ksettings = bnge_get_link_ksettings, + .set_link_ksettings = bnge_set_link_ksettings, .get_drvinfo = bnge_get_drvinfo, + .get_link = bnge_get_link, + .nway_reset = bnge_nway_reset, + .get_pauseparam = bnge_get_pauseparam, + .set_pauseparam = bnge_set_pauseparam, + .get_sset_count = bnge_get_sset_count, + .get_strings = bnge_get_strings, + .get_ethtool_stats = bnge_get_ethtool_stats, + .get_eth_phy_stats = bnge_get_eth_phy_stats, + .get_eth_mac_stats = bnge_get_eth_mac_stats, + .get_eth_ctrl_stats = bnge_get_eth_ctrl_stats, + .get_pause_stats = bnge_get_pause_stats, + .get_rmon_stats = bnge_get_rmon_stats, }; void bnge_set_ethtool_ops(struct net_device *dev) diff --git a/drivers/net/ethernet/broadcom/bnge/bnge_hwrm_lib.c b/drivers/net/ethernet/broadcom/bnge/bnge_hwrm_lib.c index c46da3413417..eb11800f5573 100644 --- a/drivers/net/ethernet/broadcom/bnge/bnge_hwrm_lib.c +++ b/drivers/net/ethernet/broadcom/bnge/bnge_hwrm_lib.c @@ -14,6 +14,14 @@ #include "bnge_hwrm_lib.h" #include "bnge_rmem.h" #include "bnge_resc.h" +#include "bnge_netdev.h" + +static const u16 bnge_async_events_arr[] = { + ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE, + ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CHANGE, + ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE, + ASYNC_EVENT_CMPL_EVENT_ID_PORT_PHY_CFG_CHANGE, +}; int bnge_hwrm_ver_get(struct bnge_dev *bd) { @@ -166,10 +174,12 @@ int bnge_hwrm_fw_set_time(struct bnge_dev *bd) int bnge_hwrm_func_drv_rgtr(struct bnge_dev *bd) { + DECLARE_BITMAP(async_events_bmap, 256); struct hwrm_func_drv_rgtr_output *resp; struct hwrm_func_drv_rgtr_input *req; + u32 events[8]; u32 flags; - int rc; + int rc, i; rc = bnge_hwrm_req_init(bd, req, HWRM_FUNC_DRV_RGTR); if (rc) @@ -190,6 +200,14 @@ int bnge_hwrm_func_drv_rgtr(struct bnge_dev *bd) req->ver_min = cpu_to_le16(DRV_VER_MIN); req->ver_upd = cpu_to_le16(DRV_VER_UPD); + memset(async_events_bmap, 0, sizeof(async_events_bmap)); + for (i = 0; i < ARRAY_SIZE(bnge_async_events_arr); i++) + __set_bit(bnge_async_events_arr[i], async_events_bmap); + + bitmap_to_arr32(events, async_events_bmap, 256); + for (i = 0; i < ARRAY_SIZE(req->async_event_fwd); i++) + req->async_event_fwd[i] |= cpu_to_le32(events[i]); + resp = bnge_hwrm_req_hold(bd, req); rc = bnge_hwrm_req_send(bd, req); if (!rc) { @@ -577,7 +595,7 @@ int bnge_hwrm_func_qcaps(struct bnge_dev *bd) struct hwrm_func_qcaps_output *resp; struct hwrm_func_qcaps_input *req; struct bnge_pf_info *pf = &bd->pf; - u32 flags; + u32 flags, flags_ext; int rc; rc = bnge_hwrm_req_init(bd, req, HWRM_FUNC_QCAPS); @@ -595,6 +613,12 @@ int bnge_hwrm_func_qcaps(struct bnge_dev *bd) bd->flags |= BNGE_EN_ROCE_V1; if (flags & FUNC_QCAPS_RESP_FLAGS_ROCE_V2_SUPPORTED) bd->flags |= BNGE_EN_ROCE_V2; + if (flags & FUNC_QCAPS_RESP_FLAGS_EXT_STATS_SUPPORTED) + bd->fw_cap |= BNGE_FW_CAP_EXT_STATS_SUPPORTED; + + flags_ext = le32_to_cpu(resp->flags_ext); + if (flags_ext & FUNC_QCAPS_RESP_FLAGS_EXT_EXT_HW_STATS_SUPPORTED) + bd->fw_cap |= BNGE_FW_CAP_EXT_HW_STATS_SUPPORTED; pf->fw_fid = le16_to_cpu(resp->fid); pf->port_id = le16_to_cpu(resp->port_id); @@ -981,6 +1005,220 @@ void bnge_hwrm_vnic_ctx_free_one(struct bnge_dev *bd, vnic->fw_rss_cos_lb_ctx[ctx_idx] = INVALID_HW_RING_ID; } +static bool bnge_phy_qcaps_no_speed(struct hwrm_port_phy_qcaps_output *resp) +{ + return !resp->supported_speeds2_auto_mode && + !resp->supported_speeds2_force_mode; +} + +int bnge_hwrm_phy_qcaps(struct bnge_dev *bd) +{ + struct bnge_link_info *link_info = &bd->link_info; + struct hwrm_port_phy_qcaps_output *resp; + struct hwrm_port_phy_qcaps_input *req; + int rc; + + rc = bnge_hwrm_req_init(bd, req, HWRM_PORT_PHY_QCAPS); + if (rc) + return rc; + + resp = bnge_hwrm_req_hold(bd, req); + rc = bnge_hwrm_req_send(bd, req); + if (rc) + goto hwrm_phy_qcaps_exit; + + bd->phy_flags = resp->flags | + (le16_to_cpu(resp->flags2) << BNGE_PHY_FLAGS2_SHIFT); + + if (bnge_phy_qcaps_no_speed(resp)) { + link_info->phy_enabled = false; + netdev_warn(bd->netdev, "Ethernet link disabled\n"); + } else if (!link_info->phy_enabled) { + link_info->phy_enabled = true; + netdev_info(bd->netdev, "Ethernet link enabled\n"); + /* Phy re-enabled, reprobe the speeds */ + link_info->support_auto_speeds2 = 0; + } + + /* Firmware may report 0 for autoneg supported speeds when no + * SFP module is present. Skip the update to preserve the + * current supported speeds -- storing 0 would cause autoneg + * default fallback to advertise nothing. + */ + if (resp->supported_speeds2_auto_mode) + link_info->support_auto_speeds2 = + le16_to_cpu(resp->supported_speeds2_auto_mode); + + bd->port_count = resp->port_cnt; + +hwrm_phy_qcaps_exit: + bnge_hwrm_req_drop(bd, req); + return rc; +} + +int bnge_hwrm_set_link_setting(struct bnge_net *bn, bool set_pause) +{ + struct hwrm_port_phy_cfg_input *req; + struct bnge_dev *bd = bn->bd; + int rc; + + rc = bnge_hwrm_req_init(bd, req, HWRM_PORT_PHY_CFG); + if (rc) + return rc; + + if (set_pause) + bnge_hwrm_set_pause_common(bn, req); + + bnge_hwrm_set_link_common(bn, req); + + rc = bnge_hwrm_req_send(bd, req); + if (!rc) + bn->eth_link_info.force_link_chng = false; + + return rc; +} + +int bnge_update_link(struct bnge_net *bn, bool chng_link_state) +{ + struct hwrm_port_phy_qcfg_output *resp; + struct hwrm_port_phy_qcfg_input *req; + struct bnge_link_info *link_info; + struct bnge_dev *bd = bn->bd; + bool support_changed; + u8 link_state; + int rc; + + link_info = &bd->link_info; + link_state = link_info->link_state; + + rc = bnge_hwrm_req_init(bd, req, HWRM_PORT_PHY_QCFG); + if (rc) + return rc; + + resp = bnge_hwrm_req_hold(bd, req); + rc = bnge_hwrm_req_send(bd, req); + if (rc) { + bnge_hwrm_req_drop(bd, req); + return rc; + } + + memcpy(&link_info->phy_qcfg_resp, resp, sizeof(*resp)); + link_info->phy_link_status = resp->link; + link_info->duplex = resp->duplex_state; + link_info->pause = resp->pause; + link_info->auto_mode = resp->auto_mode; + link_info->auto_pause_setting = resp->auto_pause; + link_info->lp_pause = resp->link_partner_adv_pause; + link_info->force_pause_setting = resp->force_pause; + link_info->duplex_setting = resp->duplex_cfg; + if (link_info->phy_link_status == BNGE_LINK_LINK) { + link_info->link_speed = le16_to_cpu(resp->link_speed); + link_info->active_lanes = resp->active_lanes; + } else { + link_info->link_speed = 0; + link_info->active_lanes = 0; + } + link_info->force_link_speed2 = le16_to_cpu(resp->force_link_speeds2); + link_info->support_speeds2 = le16_to_cpu(resp->support_speeds2); + link_info->auto_link_speeds2 = le16_to_cpu(resp->auto_link_speeds2); + link_info->lp_auto_link_speeds = + le16_to_cpu(resp->link_partner_adv_speeds); + link_info->media_type = resp->media_type; + link_info->phy_type = resp->phy_type; + link_info->phy_addr = resp->eee_config_phy_addr & + PORT_PHY_QCFG_RESP_PHY_ADDR_MASK; + link_info->module_status = resp->module_status; + + link_info->fec_cfg = le16_to_cpu(resp->fec_cfg); + link_info->active_fec_sig_mode = resp->active_fec_signal_mode; + + if (chng_link_state) { + if (link_info->phy_link_status == BNGE_LINK_LINK) + link_info->link_state = BNGE_LINK_STATE_UP; + else + link_info->link_state = BNGE_LINK_STATE_DOWN; + if (link_state != link_info->link_state) + bnge_report_link(bd); + } else { + /* always link down if not required to update link state */ + link_info->link_state = BNGE_LINK_STATE_DOWN; + } + bnge_hwrm_req_drop(bd, req); + + if (!BNGE_PHY_CFG_ABLE(bd)) + return 0; + + support_changed = bnge_support_speed_dropped(bn); + if (support_changed && (bn->eth_link_info.autoneg & BNGE_AUTONEG_SPEED)) + rc = bnge_hwrm_set_link_setting(bn, true); + return rc; +} + +int bnge_hwrm_set_pause(struct bnge_net *bn) +{ + struct bnge_ethtool_link_info *elink_info = &bn->eth_link_info; + struct hwrm_port_phy_cfg_input *req; + struct bnge_dev *bd = bn->bd; + bool pause_autoneg; + int rc; + + rc = bnge_hwrm_req_init(bd, req, HWRM_PORT_PHY_CFG); + if (rc) + return rc; + + pause_autoneg = !!(elink_info->autoneg & BNGE_AUTONEG_FLOW_CTRL); + + /* Prepare PHY pause-advertisement or forced-pause settings. */ + bnge_hwrm_set_pause_common(bn, req); + + /* Prepare speed/autoneg settings */ + if (pause_autoneg || elink_info->force_link_chng) + bnge_hwrm_set_link_common(bn, req); + + rc = bnge_hwrm_req_send(bd, req); + if (!rc && !pause_autoneg) { + /* Since changing of pause setting, with pause autoneg off, + * doesn't trigger any link change event, the driver needs to + * update the current MAC pause upon successful return of the + * phy_cfg command. + */ + bd->link_info.force_pause_setting = + bd->link_info.pause = elink_info->req_flow_ctrl; + bd->link_info.auto_pause_setting = 0; + if (!elink_info->force_link_chng) + bnge_report_link(bd); + } + if (!rc) + elink_info->force_link_chng = false; + + return rc; +} + +int bnge_hwrm_shutdown_link(struct bnge_dev *bd) +{ + struct hwrm_port_phy_cfg_input *req; + int rc; + + if (!BNGE_PHY_CFG_ABLE(bd)) + return 0; + + rc = bnge_hwrm_req_init(bd, req, HWRM_PORT_PHY_CFG); + if (rc) + return rc; + + req->flags = cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_FORCE_LINK_DWN); + rc = bnge_hwrm_req_send(bd, req); + if (!rc) { + /* Device is not obliged to link down in certain scenarios, + * even when forced. Setting the state unknown is consistent + * with driver startup and will force link state to be + * reported during subsequent open based on PORT_PHY_QCFG. + */ + bd->link_info.link_state = BNGE_LINK_STATE_UNKNOWN; + } + return rc; +} + void bnge_hwrm_stat_ctx_free(struct bnge_net *bn) { struct hwrm_stat_ctx_free_input *req; @@ -1248,3 +1486,145 @@ int bnge_hwrm_vnic_set_tpa(struct bnge_dev *bd, struct bnge_vnic_info *vnic, return bnge_hwrm_req_send(bd, req); } + +int bnge_hwrm_func_qstat_ext(struct bnge_dev *bd, struct bnge_stats_mem *stats) +{ + struct hwrm_func_qstats_ext_output *resp; + struct hwrm_func_qstats_ext_input *req; + __le64 *hw_masks; + int rc; + + if (!(bd->fw_cap & BNGE_FW_CAP_EXT_HW_STATS_SUPPORTED)) + return -EOPNOTSUPP; + + rc = bnge_hwrm_req_init(bd, req, HWRM_FUNC_QSTATS_EXT); + if (rc) + return rc; + + req->fid = cpu_to_le16(0xffff); + req->flags = FUNC_QSTATS_EXT_REQ_FLAGS_COUNTER_MASK; + + resp = bnge_hwrm_req_hold(bd, req); + rc = bnge_hwrm_req_send(bd, req); + if (!rc) { + hw_masks = &resp->rx_ucast_pkts; + bnge_copy_hw_masks(stats->hw_masks, hw_masks, stats->len / 8); + } + bnge_hwrm_req_drop(bd, req); + return rc; +} + +int bnge_hwrm_port_qstats_ext(struct bnge_dev *bd, u8 flags) +{ + struct hwrm_queue_pri2cos_qcfg_output *resp_qc; + struct bnge_net *bn = netdev_priv(bd->netdev); + struct hwrm_queue_pri2cos_qcfg_input *req_qc; + struct hwrm_port_qstats_ext_output *resp_qs; + struct hwrm_port_qstats_ext_input *req_qs; + struct bnge_pf_info *pf = &bd->pf; + u32 tx_stat_size; + int rc; + + if (!(bn->flags & BNGE_FLAG_PORT_STATS_EXT)) + return 0; + + if (flags && !(bd->fw_cap & BNGE_FW_CAP_EXT_HW_STATS_SUPPORTED)) + return -EOPNOTSUPP; + + rc = bnge_hwrm_req_init(bd, req_qs, HWRM_PORT_QSTATS_EXT); + if (rc) + return rc; + + req_qs->flags = flags; + req_qs->port_id = cpu_to_le16(pf->port_id); + req_qs->rx_stat_size = cpu_to_le16(sizeof(struct rx_port_stats_ext)); + req_qs->rx_stat_host_addr = + cpu_to_le64(bn->rx_port_stats_ext.hw_stats_map); + tx_stat_size = bn->tx_port_stats_ext.hw_stats ? + sizeof(struct tx_port_stats_ext) : 0; + req_qs->tx_stat_size = cpu_to_le16(tx_stat_size); + req_qs->tx_stat_host_addr = + cpu_to_le64(bn->tx_port_stats_ext.hw_stats_map); + resp_qs = bnge_hwrm_req_hold(bd, req_qs); + rc = bnge_hwrm_req_send(bd, req_qs); + if (!rc) { + bn->fw_rx_stats_ext_size = + le16_to_cpu(resp_qs->rx_stat_size) / 8; + bn->fw_tx_stats_ext_size = tx_stat_size ? + le16_to_cpu(resp_qs->tx_stat_size) / 8 : 0; + } else { + bn->fw_rx_stats_ext_size = 0; + bn->fw_tx_stats_ext_size = 0; + } + bnge_hwrm_req_drop(bd, req_qs); + + if (flags) + return rc; + + if (bn->fw_tx_stats_ext_size <= + offsetof(struct tx_port_stats_ext, pfc_pri0_tx_duration_us) / 8) { + bn->pri2cos_valid = false; + return rc; + } + + rc = bnge_hwrm_req_init(bd, req_qc, HWRM_QUEUE_PRI2COS_QCFG); + if (rc) + return rc; + + req_qc->flags = cpu_to_le32(QUEUE_PRI2COS_QCFG_REQ_FLAGS_IVLAN); + + resp_qc = bnge_hwrm_req_hold(bd, req_qc); + rc = bnge_hwrm_req_send(bd, req_qc); + if (!rc) { + u8 *pri2cos; + int i, j; + + pri2cos = &resp_qc->pri0_cos_queue_id; + for (i = 0; i < 8; i++) { + u8 queue_id = pri2cos[i]; + u8 queue_idx; + + /* Per port queue IDs start from 0, 10, 20, etc */ + queue_idx = queue_id % 10; + if (queue_idx >= BNGE_MAX_QUEUE) { + bn->pri2cos_valid = false; + rc = -EINVAL; + goto drop_req; + } + for (j = 0; j < bd->max_q; j++) { + if (bd->q_ids[j] == queue_id) + bn->pri2cos_idx[i] = queue_idx; + } + } + bn->pri2cos_valid = true; + } +drop_req: + bnge_hwrm_req_drop(bd, req_qc); + return rc; +} + +int bnge_hwrm_port_qstats(struct bnge_dev *bd, u8 flags) +{ + struct bnge_net *bn = netdev_priv(bd->netdev); + struct hwrm_port_qstats_input *req; + struct bnge_pf_info *pf = &bd->pf; + int rc; + + if (!(bn->flags & BNGE_FLAG_PORT_STATS)) + return 0; + + if (flags && !(bd->fw_cap & BNGE_FW_CAP_EXT_HW_STATS_SUPPORTED)) + return -EOPNOTSUPP; + + rc = bnge_hwrm_req_init(bd, req, HWRM_PORT_QSTATS); + if (rc) + return rc; + + req->flags = flags; + req->port_id = cpu_to_le16(pf->port_id); + req->tx_stat_host_addr = cpu_to_le64(bn->port_stats.hw_stats_map + + BNGE_TX_PORT_STATS_BYTE_OFFSET); + req->rx_stat_host_addr = cpu_to_le64(bn->port_stats.hw_stats_map); + + return bnge_hwrm_req_send(bd, req); +} diff --git a/drivers/net/ethernet/broadcom/bnge/bnge_hwrm_lib.h b/drivers/net/ethernet/broadcom/bnge/bnge_hwrm_lib.h index 38b046237feb..3501de7a89b9 100644 --- a/drivers/net/ethernet/broadcom/bnge/bnge_hwrm_lib.h +++ b/drivers/net/ethernet/broadcom/bnge/bnge_hwrm_lib.h @@ -57,4 +57,12 @@ int hwrm_ring_alloc_send_msg(struct bnge_net *bn, int bnge_hwrm_set_async_event_cr(struct bnge_dev *bd, int idx); int bnge_hwrm_vnic_set_tpa(struct bnge_dev *bd, struct bnge_vnic_info *vnic, u32 tpa_flags); +int bnge_update_link(struct bnge_net *bn, bool chng_link_state); +int bnge_hwrm_phy_qcaps(struct bnge_dev *bd); +int bnge_hwrm_set_link_setting(struct bnge_net *bn, bool set_pause); +int bnge_hwrm_set_pause(struct bnge_net *bn); +int bnge_hwrm_shutdown_link(struct bnge_dev *bd); +int bnge_hwrm_port_qstats(struct bnge_dev *bd, u8 flags); +int bnge_hwrm_port_qstats_ext(struct bnge_dev *bd, u8 flags); +int bnge_hwrm_func_qstat_ext(struct bnge_dev *bd, struct bnge_stats_mem *stats); #endif /* _BNGE_HWRM_LIB_H_ */ diff --git a/drivers/net/ethernet/broadcom/bnge/bnge_link.c b/drivers/net/ethernet/broadcom/bnge/bnge_link.c new file mode 100644 index 000000000000..3b0dfcf27376 --- /dev/null +++ b/drivers/net/ethernet/broadcom/bnge/bnge_link.c @@ -0,0 +1,1132 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) 2026 Broadcom. + +#include <linux/linkmode.h> + +#include "bnge.h" +#include "bnge_link.h" +#include "bnge_hwrm_lib.h" + +enum bnge_media_type { + BNGE_MEDIA_UNKNOWN = 0, + BNGE_MEDIA_CR, + BNGE_MEDIA_SR, + BNGE_MEDIA_LR_ER_FR, + BNGE_MEDIA_KR, + __BNGE_MEDIA_END, +}; + +static const enum bnge_media_type bnge_phy_types[] = { + [PORT_PHY_QCFG_RESP_PHY_TYPE_100G_BASECR4] = BNGE_MEDIA_CR, + [PORT_PHY_QCFG_RESP_PHY_TYPE_100G_BASESR4] = BNGE_MEDIA_SR, + [PORT_PHY_QCFG_RESP_PHY_TYPE_100G_BASELR4] = BNGE_MEDIA_LR_ER_FR, + [PORT_PHY_QCFG_RESP_PHY_TYPE_100G_BASEER4] = BNGE_MEDIA_LR_ER_FR, + [PORT_PHY_QCFG_RESP_PHY_TYPE_100G_BASESR10] = BNGE_MEDIA_SR, + [PORT_PHY_QCFG_RESP_PHY_TYPE_200G_BASECR4] = BNGE_MEDIA_CR, + [PORT_PHY_QCFG_RESP_PHY_TYPE_200G_BASESR4] = BNGE_MEDIA_SR, + [PORT_PHY_QCFG_RESP_PHY_TYPE_200G_BASELR4] = BNGE_MEDIA_LR_ER_FR, + [PORT_PHY_QCFG_RESP_PHY_TYPE_200G_BASEER4] = BNGE_MEDIA_LR_ER_FR, + [PORT_PHY_QCFG_RESP_PHY_TYPE_50G_BASECR] = BNGE_MEDIA_CR, + [PORT_PHY_QCFG_RESP_PHY_TYPE_50G_BASESR] = BNGE_MEDIA_SR, + [PORT_PHY_QCFG_RESP_PHY_TYPE_50G_BASELR] = BNGE_MEDIA_LR_ER_FR, + [PORT_PHY_QCFG_RESP_PHY_TYPE_50G_BASEER] = BNGE_MEDIA_LR_ER_FR, + [PORT_PHY_QCFG_RESP_PHY_TYPE_100G_BASECR2] = BNGE_MEDIA_CR, + [PORT_PHY_QCFG_RESP_PHY_TYPE_100G_BASESR2] = BNGE_MEDIA_SR, + [PORT_PHY_QCFG_RESP_PHY_TYPE_100G_BASELR2] = BNGE_MEDIA_LR_ER_FR, + [PORT_PHY_QCFG_RESP_PHY_TYPE_100G_BASEER2] = BNGE_MEDIA_LR_ER_FR, + [PORT_PHY_QCFG_RESP_PHY_TYPE_100G_BASECR] = BNGE_MEDIA_CR, + [PORT_PHY_QCFG_RESP_PHY_TYPE_100G_BASESR] = BNGE_MEDIA_SR, + [PORT_PHY_QCFG_RESP_PHY_TYPE_100G_BASELR] = BNGE_MEDIA_LR_ER_FR, + [PORT_PHY_QCFG_RESP_PHY_TYPE_100G_BASEER] = BNGE_MEDIA_LR_ER_FR, + [PORT_PHY_QCFG_RESP_PHY_TYPE_200G_BASECR2] = BNGE_MEDIA_CR, + [PORT_PHY_QCFG_RESP_PHY_TYPE_200G_BASESR2] = BNGE_MEDIA_SR, + [PORT_PHY_QCFG_RESP_PHY_TYPE_200G_BASELR2] = BNGE_MEDIA_LR_ER_FR, + [PORT_PHY_QCFG_RESP_PHY_TYPE_200G_BASEER2] = BNGE_MEDIA_LR_ER_FR, + [PORT_PHY_QCFG_RESP_PHY_TYPE_400G_BASECR8] = BNGE_MEDIA_CR, + [PORT_PHY_QCFG_RESP_PHY_TYPE_400G_BASESR8] = BNGE_MEDIA_SR, + [PORT_PHY_QCFG_RESP_PHY_TYPE_400G_BASELR8] = BNGE_MEDIA_LR_ER_FR, + [PORT_PHY_QCFG_RESP_PHY_TYPE_400G_BASEER8] = BNGE_MEDIA_LR_ER_FR, + [PORT_PHY_QCFG_RESP_PHY_TYPE_400G_BASECR4] = BNGE_MEDIA_CR, + [PORT_PHY_QCFG_RESP_PHY_TYPE_400G_BASESR4] = BNGE_MEDIA_SR, + [PORT_PHY_QCFG_RESP_PHY_TYPE_400G_BASELR4] = BNGE_MEDIA_LR_ER_FR, + [PORT_PHY_QCFG_RESP_PHY_TYPE_400G_BASEER4] = BNGE_MEDIA_LR_ER_FR, +}; + +static u32 bnge_fw_to_ethtool_speed(u16 fw_link_speed) +{ + switch (fw_link_speed) { + case BNGE_LINK_SPEED_50GB: + case BNGE_LINK_SPEED_50GB_PAM4: + return SPEED_50000; + case BNGE_LINK_SPEED_100GB: + case BNGE_LINK_SPEED_100GB_PAM4: + case BNGE_LINK_SPEED_100GB_PAM4_112: + return SPEED_100000; + case BNGE_LINK_SPEED_200GB: + case BNGE_LINK_SPEED_200GB_PAM4: + case BNGE_LINK_SPEED_200GB_PAM4_112: + return SPEED_200000; + case BNGE_LINK_SPEED_400GB: + case BNGE_LINK_SPEED_400GB_PAM4: + case BNGE_LINK_SPEED_400GB_PAM4_112: + return SPEED_400000; + case BNGE_LINK_SPEED_800GB: + case BNGE_LINK_SPEED_800GB_PAM4_112: + return SPEED_800000; + default: + return SPEED_UNKNOWN; + } +} + +static void bnge_set_auto_speed(struct bnge_net *bn) +{ + struct bnge_ethtool_link_info *elink_info = &bn->eth_link_info; + struct bnge_link_info *link_info; + + link_info = &bn->bd->link_info; + elink_info->advertising = link_info->auto_link_speeds2; +} + +static void bnge_set_force_speed(struct bnge_net *bn) +{ + struct bnge_ethtool_link_info *elink_info = &bn->eth_link_info; + struct bnge_link_info *link_info; + + link_info = &bn->bd->link_info; + elink_info->req_link_speed = link_info->force_link_speed2; + switch (elink_info->req_link_speed) { + case BNGE_LINK_SPEED_50GB_PAM4: + case BNGE_LINK_SPEED_100GB_PAM4: + case BNGE_LINK_SPEED_200GB_PAM4: + case BNGE_LINK_SPEED_400GB_PAM4: + elink_info->req_signal_mode = BNGE_SIG_MODE_PAM4; + break; + case BNGE_LINK_SPEED_100GB_PAM4_112: + case BNGE_LINK_SPEED_200GB_PAM4_112: + case BNGE_LINK_SPEED_400GB_PAM4_112: + case BNGE_LINK_SPEED_800GB_PAM4_112: + elink_info->req_signal_mode = BNGE_SIG_MODE_PAM4_112; + break; + default: + elink_info->req_signal_mode = BNGE_SIG_MODE_NRZ; + break; + } +} + +void bnge_init_ethtool_link_settings(struct bnge_net *bn) +{ + struct bnge_ethtool_link_info *elink_info = &bn->eth_link_info; + struct bnge_link_info *link_info; + struct bnge_dev *bd = bn->bd; + + link_info = &bd->link_info; + + if (BNGE_AUTO_MODE(link_info->auto_mode)) { + elink_info->autoneg = BNGE_AUTONEG_SPEED; + if (link_info->auto_pause_setting & + PORT_PHY_QCFG_RESP_AUTO_PAUSE_AUTONEG_PAUSE) + elink_info->autoneg |= BNGE_AUTONEG_FLOW_CTRL; + bnge_set_auto_speed(bn); + } else { + elink_info->autoneg = 0; + elink_info->advertising = 0; + bnge_set_force_speed(bn); + elink_info->req_duplex = link_info->duplex_setting; + } + if (elink_info->autoneg & BNGE_AUTONEG_FLOW_CTRL) + elink_info->req_flow_ctrl = + link_info->auto_pause_setting & BNGE_LINK_PAUSE_BOTH; + else + elink_info->req_flow_ctrl = link_info->force_pause_setting; +} + +int bnge_probe_phy(struct bnge_net *bn, bool fw_dflt) +{ + struct bnge_dev *bd = bn->bd; + int rc; + + bd->phy_flags = 0; + rc = bnge_hwrm_phy_qcaps(bd); + if (rc) { + netdev_err(bn->netdev, + "Probe PHY can't get PHY qcaps (rc: %d)\n", rc); + return rc; + } + if (bd->phy_flags & BNGE_PHY_FL_NO_FCS) + bn->netdev->priv_flags |= IFF_SUPP_NOFCS; + else + bn->netdev->priv_flags &= ~IFF_SUPP_NOFCS; + if (!fw_dflt) + return 0; + + rc = bnge_update_link(bn, false); + if (rc) { + netdev_err(bn->netdev, "Probe PHY can't update link (rc: %d)\n", + rc); + return rc; + } + bnge_init_ethtool_link_settings(bn); + + return 0; +} + +void bnge_hwrm_set_link_common(struct bnge_net *bn, + struct hwrm_port_phy_cfg_input *req) +{ + struct bnge_ethtool_link_info *elink_info = &bn->eth_link_info; + + if (elink_info->autoneg & BNGE_AUTONEG_SPEED) { + req->auto_mode |= PORT_PHY_CFG_REQ_AUTO_MODE_SPEED_MASK; + req->enables |= cpu_to_le32(BNGE_PHY_AUTO_SPEEDS2_MASK); + req->auto_link_speeds2_mask = + cpu_to_le16(elink_info->advertising); + req->enables |= cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_MODE); + req->flags |= cpu_to_le32(BNGE_PHY_FLAGS_RESTART_AUTO); + } else { + req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_FORCE); + req->force_link_speeds2 = + cpu_to_le16(elink_info->req_link_speed); + req->enables |= + cpu_to_le32(BNGE_PHY_FLAGS_ENA_FORCE_SPEEDS2); + netif_info(bn, link, bn->netdev, + "Forcing FW speed2: %d\n", + (u32)elink_info->req_link_speed); + } + + /* tell FW that the setting takes effect immediately */ + req->flags |= cpu_to_le32(PORT_PHY_CFG_REQ_FLAGS_RESET_PHY); +} + +static bool bnge_auto_speed_updated(struct bnge_net *bn) +{ + struct bnge_ethtool_link_info *elink_info = &bn->eth_link_info; + struct bnge_link_info *link_info = &bn->bd->link_info; + + return elink_info->advertising != link_info->auto_link_speeds2; +} + +void bnge_hwrm_set_pause_common(struct bnge_net *bn, + struct hwrm_port_phy_cfg_input *req) +{ + if (bn->eth_link_info.autoneg & BNGE_AUTONEG_FLOW_CTRL) { + req->auto_pause = PORT_PHY_CFG_REQ_AUTO_PAUSE_AUTONEG_PAUSE; + if (bn->eth_link_info.req_flow_ctrl & BNGE_LINK_PAUSE_RX) + req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_RX; + if (bn->eth_link_info.req_flow_ctrl & BNGE_LINK_PAUSE_TX) + req->auto_pause |= PORT_PHY_CFG_REQ_AUTO_PAUSE_TX; + req->enables |= + cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE); + } else { + if (bn->eth_link_info.req_flow_ctrl & BNGE_LINK_PAUSE_RX) + req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_RX; + if (bn->eth_link_info.req_flow_ctrl & BNGE_LINK_PAUSE_TX) + req->force_pause |= PORT_PHY_CFG_REQ_FORCE_PAUSE_TX; + req->enables |= + cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_FORCE_PAUSE); + req->auto_pause = req->force_pause; + req->enables |= + cpu_to_le32(PORT_PHY_CFG_REQ_ENABLES_AUTO_PAUSE); + } +} + +static bool bnge_force_speed_updated(struct bnge_net *bn) +{ + struct bnge_ethtool_link_info *elink_info = &bn->eth_link_info; + struct bnge_link_info *link_info = &bn->bd->link_info; + + return elink_info->req_link_speed != link_info->force_link_speed2; +} + +int bnge_update_phy_setting(struct bnge_net *bn) +{ + struct bnge_ethtool_link_info *elink_info; + struct bnge_link_info *link_info; + struct bnge_dev *bd = bn->bd; + bool update_pause = false; + bool update_link = false; + bool hw_pause_autoneg; + bool pause_autoneg; + int rc; + + link_info = &bd->link_info; + elink_info = &bn->eth_link_info; + rc = bnge_update_link(bn, true); + if (rc) { + netdev_err(bn->netdev, "failed to update link (rc: %d)\n", + rc); + return rc; + } + + pause_autoneg = !!(elink_info->autoneg & BNGE_AUTONEG_FLOW_CTRL); + hw_pause_autoneg = !!(link_info->auto_pause_setting & + PORT_PHY_QCFG_RESP_AUTO_PAUSE_AUTONEG_PAUSE); + + /* Check if pause autonegotiation state has changed */ + if (pause_autoneg != hw_pause_autoneg) { + update_pause = true; + } else if (pause_autoneg) { + /* If pause autoneg is enabled, check if the + * requested RX/TX bits changed + */ + if ((link_info->auto_pause_setting & BNGE_LINK_PAUSE_BOTH) != + elink_info->req_flow_ctrl) + update_pause = true; + } else { + /* If pause autoneg is disabled, check if the + * forced RX/TX bits changed + */ + if (link_info->force_pause_setting != elink_info->req_flow_ctrl) + update_pause = true; + } + + /* Force update if link change is requested */ + if (elink_info->force_link_chng) + update_pause = true; + + /* Check if link speed or duplex settings have changed */ + if (!(elink_info->autoneg & BNGE_AUTONEG_SPEED)) { + if (BNGE_AUTO_MODE(link_info->auto_mode) || + bnge_force_speed_updated(bn) || + elink_info->req_duplex != link_info->duplex_setting) + update_link = true; + } else { + if (link_info->auto_mode == BNGE_LINK_AUTO_NONE || + bnge_auto_speed_updated(bn)) + update_link = true; + } + + /* The last close may have shut down the link, so need to call + * PHY_CFG to bring it back up. + */ + if (!BNGE_LINK_IS_UP(bd)) + update_link = true; + + if (update_link) + rc = bnge_hwrm_set_link_setting(bn, update_pause); + else if (update_pause) + rc = bnge_hwrm_set_pause(bn); + + if (rc) { + netdev_err(bn->netdev, + "failed to update PHY setting (rc: %d)\n", rc); + return rc; + } + + return 0; +} + +void bnge_get_port_module_status(struct bnge_net *bn) +{ + struct hwrm_port_phy_qcfg_output *resp; + struct bnge_link_info *link_info; + struct bnge_dev *bd = bn->bd; + u8 module_status; + + link_info = &bd->link_info; + resp = &link_info->phy_qcfg_resp; + + if (bnge_update_link(bn, true)) + return; + + module_status = link_info->module_status; + switch (module_status) { + case PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX: + case PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN: + case PORT_PHY_QCFG_RESP_MODULE_STATUS_WARNINGMSG: + netdev_warn(bn->netdev, + "Unqualified SFP+ module detected on port %d\n", + bd->pf.port_id); + netdev_warn(bn->netdev, "Module part number %.*s\n", + (int)sizeof(resp->phy_vendor_partnumber), + resp->phy_vendor_partnumber); + if (module_status == PORT_PHY_QCFG_RESP_MODULE_STATUS_DISABLETX) + netdev_warn(bn->netdev, "TX is disabled\n"); + if (module_status == PORT_PHY_QCFG_RESP_MODULE_STATUS_PWRDOWN) + netdev_warn(bn->netdev, "SFP+ module is shut down\n"); + break; + } +} + +static void bnge_set_default_adv_speeds(struct bnge_net *bn) +{ + struct bnge_ethtool_link_info *elink_info = &bn->eth_link_info; + struct bnge_link_info *link_info = &bn->bd->link_info; + + elink_info->advertising = link_info->support_auto_speeds2; +} + +static bool bnge_support_dropped(u16 advertising, u16 supported) +{ + return (advertising & ~supported) != 0; +} + +bool bnge_support_speed_dropped(struct bnge_net *bn) +{ + struct bnge_ethtool_link_info *elink_info = &bn->eth_link_info; + struct bnge_link_info *link_info = &bn->bd->link_info; + + /* Check if any advertised speeds are no longer supported. The caller + * holds the netdev instance lock, so we can modify link_info settings. + */ + if (bnge_support_dropped(elink_info->advertising, + link_info->support_auto_speeds2)) { + elink_info->advertising = link_info->support_auto_speeds2; + return true; + } + return false; +} + +static char *bnge_report_fec(struct bnge_link_info *link_info) +{ + u8 active_fec = link_info->active_fec_sig_mode & + PORT_PHY_QCFG_RESP_ACTIVE_FEC_MASK; + + switch (active_fec) { + default: + case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_NONE_ACTIVE: + return "None"; + case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_CLAUSE74_ACTIVE: + return "Clause 74 BaseR"; + case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_CLAUSE91_ACTIVE: + return "Clause 91 RS(528,514)"; + case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS544_1XN_ACTIVE: + return "Clause 91 RS544_1XN"; + case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS544_IEEE_ACTIVE: + return "Clause 91 RS(544,514)"; + case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS272_1XN_ACTIVE: + return "Clause 91 RS272_1XN"; + case PORT_PHY_QCFG_RESP_ACTIVE_FEC_FEC_RS272_IEEE_ACTIVE: + return "Clause 91 RS(272,257)"; + } +} + +void bnge_report_link(struct bnge_dev *bd) +{ + if (BNGE_LINK_IS_UP(bd)) { + const char *signal = ""; + const char *flow_ctrl; + const char *duplex; + u32 speed; + u16 fec; + + netif_carrier_on(bd->netdev); + speed = bnge_fw_to_ethtool_speed(bd->link_info.link_speed); + if (speed == SPEED_UNKNOWN) { + netdev_info(bd->netdev, + "NIC Link is Up, speed unknown\n"); + return; + } + if (bd->link_info.duplex == BNGE_LINK_DUPLEX_FULL) + duplex = "full"; + else + duplex = "half"; + if (bd->link_info.pause == BNGE_LINK_PAUSE_BOTH) + flow_ctrl = "ON - receive & transmit"; + else if (bd->link_info.pause == BNGE_LINK_PAUSE_TX) + flow_ctrl = "ON - transmit"; + else if (bd->link_info.pause == BNGE_LINK_PAUSE_RX) + flow_ctrl = "ON - receive"; + else + flow_ctrl = "none"; + if (bd->link_info.phy_qcfg_resp.option_flags & + PORT_PHY_QCFG_RESP_OPTION_FLAGS_SIGNAL_MODE_KNOWN) { + u8 sig_mode = bd->link_info.active_fec_sig_mode & + PORT_PHY_QCFG_RESP_SIGNAL_MODE_MASK; + switch (sig_mode) { + case PORT_PHY_QCFG_RESP_SIGNAL_MODE_NRZ: + signal = "(NRZ) "; + break; + case PORT_PHY_QCFG_RESP_SIGNAL_MODE_PAM4: + signal = "(PAM4 56Gbps) "; + break; + case PORT_PHY_QCFG_RESP_SIGNAL_MODE_PAM4_112: + signal = "(PAM4 112Gbps) "; + break; + default: + break; + } + } + netdev_info(bd->netdev, "NIC Link is Up, %u Mbps %s%s duplex, Flow control: %s\n", + speed, signal, duplex, flow_ctrl); + fec = bd->link_info.fec_cfg; + if (!(fec & PORT_PHY_QCFG_RESP_FEC_CFG_FEC_NONE_SUPPORTED)) + netdev_info(bd->netdev, "FEC autoneg %s encoding: %s\n", + (fec & BNGE_FEC_AUTONEG) ? "on" : "off", + bnge_report_fec(&bd->link_info)); + } else { + netif_carrier_off(bd->netdev); + netdev_info(bd->netdev, "NIC Link is Down\n"); + } +} + +static void bnge_get_ethtool_modes(struct bnge_net *bn, + struct ethtool_link_ksettings *lk_ksettings) +{ + struct bnge_ethtool_link_info *elink_info; + struct bnge_link_info *link_info; + struct bnge_dev *bd = bn->bd; + + elink_info = &bn->eth_link_info; + link_info = &bd->link_info; + + if (!(bd->phy_flags & BNGE_PHY_FL_NO_PAUSE)) { + linkmode_set_bit(ETHTOOL_LINK_MODE_Pause_BIT, + lk_ksettings->link_modes.supported); + linkmode_set_bit(ETHTOOL_LINK_MODE_Asym_Pause_BIT, + lk_ksettings->link_modes.supported); + } + + if (link_info->support_auto_speeds2) + linkmode_set_bit(ETHTOOL_LINK_MODE_Autoneg_BIT, + lk_ksettings->link_modes.supported); + + if (~elink_info->autoneg & BNGE_AUTONEG_FLOW_CTRL) + return; + + if (link_info->auto_pause_setting & BNGE_LINK_PAUSE_RX) + linkmode_set_bit(ETHTOOL_LINK_MODE_Pause_BIT, + lk_ksettings->link_modes.advertising); + if (hweight8(link_info->auto_pause_setting & BNGE_LINK_PAUSE_BOTH) == 1) + linkmode_set_bit(ETHTOOL_LINK_MODE_Asym_Pause_BIT, + lk_ksettings->link_modes.advertising); + if (link_info->lp_pause & BNGE_LINK_PAUSE_RX) + linkmode_set_bit(ETHTOOL_LINK_MODE_Pause_BIT, + lk_ksettings->link_modes.lp_advertising); + if (hweight8(link_info->lp_pause & BNGE_LINK_PAUSE_BOTH) == 1) + linkmode_set_bit(ETHTOOL_LINK_MODE_Asym_Pause_BIT, + lk_ksettings->link_modes.lp_advertising); +} + +u32 bnge_get_link(struct net_device *dev) +{ + struct bnge_net *bn = netdev_priv(dev); + + return BNGE_LINK_IS_UP(bn->bd); +} + +static enum bnge_media_type +bnge_get_media(struct bnge_link_info *link_info) +{ + switch (link_info->media_type) { + case PORT_PHY_QCFG_RESP_MEDIA_TYPE_DAC: + return BNGE_MEDIA_CR; + default: + if (link_info->phy_type < ARRAY_SIZE(bnge_phy_types)) + return bnge_phy_types[link_info->phy_type]; + return BNGE_MEDIA_UNKNOWN; + } +} + +enum bnge_link_speed_indices { + BNGE_LINK_SPEED_UNKNOWN = 0, + BNGE_LINK_SPEED_50GB_IDX, + BNGE_LINK_SPEED_100GB_IDX, + BNGE_LINK_SPEED_200GB_IDX, + BNGE_LINK_SPEED_400GB_IDX, + BNGE_LINK_SPEED_800GB_IDX, + __BNGE_LINK_SPEED_END +}; + +static enum bnge_link_speed_indices bnge_fw_speed_idx(u16 speed) +{ + switch (speed) { + case BNGE_LINK_SPEED_50GB: + case BNGE_LINK_SPEED_50GB_PAM4: + return BNGE_LINK_SPEED_50GB_IDX; + case BNGE_LINK_SPEED_100GB: + case BNGE_LINK_SPEED_100GB_PAM4: + case BNGE_LINK_SPEED_100GB_PAM4_112: + return BNGE_LINK_SPEED_100GB_IDX; + case BNGE_LINK_SPEED_200GB: + case BNGE_LINK_SPEED_200GB_PAM4: + case BNGE_LINK_SPEED_200GB_PAM4_112: + return BNGE_LINK_SPEED_200GB_IDX; + case BNGE_LINK_SPEED_400GB: + case BNGE_LINK_SPEED_400GB_PAM4: + case BNGE_LINK_SPEED_400GB_PAM4_112: + return BNGE_LINK_SPEED_400GB_IDX; + case BNGE_LINK_SPEED_800GB: + case BNGE_LINK_SPEED_800GB_PAM4_112: + return BNGE_LINK_SPEED_800GB_IDX; + default: + return BNGE_LINK_SPEED_UNKNOWN; + } +} + +/* Compile-time link mode mapping table. + * Indexed by [speed_idx][sig_mode][media]. + */ +#define BNGE_LINK_M(speed, sig, media, lm) \ + [BNGE_LINK_SPEED_##speed##_IDX] \ + [BNGE_SIG_MODE_##sig] \ + [BNGE_MEDIA_##media] = ETHTOOL_LINK_MODE_##lm##_Full_BIT + +static const enum ethtool_link_mode_bit_indices +bnge_link_modes[__BNGE_LINK_SPEED_END] + [BNGE_SIG_MODE_MAX] + [__BNGE_MEDIA_END] = { + /* 50GB PAM4 */ + BNGE_LINK_M(50GB, PAM4, CR, 50000baseCR), + BNGE_LINK_M(50GB, PAM4, SR, 50000baseSR), + BNGE_LINK_M(50GB, PAM4, LR_ER_FR, 50000baseLR_ER_FR), + BNGE_LINK_M(50GB, PAM4, KR, 50000baseKR), + + /* 100GB NRZ */ + BNGE_LINK_M(100GB, NRZ, CR, 100000baseCR4), + BNGE_LINK_M(100GB, NRZ, SR, 100000baseSR4), + BNGE_LINK_M(100GB, NRZ, LR_ER_FR, 100000baseLR4_ER4), + BNGE_LINK_M(100GB, NRZ, KR, 100000baseKR4), + + /* 100GB PAM4 */ + BNGE_LINK_M(100GB, PAM4, CR, 100000baseCR2), + BNGE_LINK_M(100GB, PAM4, SR, 100000baseSR2), + BNGE_LINK_M(100GB, PAM4, LR_ER_FR, 100000baseLR2_ER2_FR2), + BNGE_LINK_M(100GB, PAM4, KR, 100000baseKR2), + + /* 100GB PAM4_112 */ + BNGE_LINK_M(100GB, PAM4_112, CR, 100000baseCR), + BNGE_LINK_M(100GB, PAM4_112, SR, 100000baseSR), + BNGE_LINK_M(100GB, PAM4_112, LR_ER_FR, 100000baseLR_ER_FR), + BNGE_LINK_M(100GB, PAM4_112, KR, 100000baseKR), + + /* 200GB PAM4 */ + BNGE_LINK_M(200GB, PAM4, CR, 200000baseCR4), + BNGE_LINK_M(200GB, PAM4, SR, 200000baseSR4), + BNGE_LINK_M(200GB, PAM4, LR_ER_FR, 200000baseLR4_ER4_FR4), + BNGE_LINK_M(200GB, PAM4, KR, 200000baseKR4), + + /* 200GB PAM4_112 */ + BNGE_LINK_M(200GB, PAM4_112, CR, 200000baseCR2), + BNGE_LINK_M(200GB, PAM4_112, SR, 200000baseSR2), + BNGE_LINK_M(200GB, PAM4_112, LR_ER_FR, 200000baseLR2_ER2_FR2), + BNGE_LINK_M(200GB, PAM4_112, KR, 200000baseKR2), + + /* 400GB PAM4 */ + BNGE_LINK_M(400GB, PAM4, CR, 400000baseCR8), + BNGE_LINK_M(400GB, PAM4, SR, 400000baseSR8), + BNGE_LINK_M(400GB, PAM4, LR_ER_FR, 400000baseLR8_ER8_FR8), + BNGE_LINK_M(400GB, PAM4, KR, 400000baseKR8), + + /* 400GB PAM4_112 */ + BNGE_LINK_M(400GB, PAM4_112, CR, 400000baseCR4), + BNGE_LINK_M(400GB, PAM4_112, SR, 400000baseSR4), + BNGE_LINK_M(400GB, PAM4_112, LR_ER_FR, 400000baseLR4_ER4_FR4), + BNGE_LINK_M(400GB, PAM4_112, KR, 400000baseKR4), + + /* 800GB PAM4_112 */ + BNGE_LINK_M(800GB, PAM4_112, CR, 800000baseCR8), + BNGE_LINK_M(800GB, PAM4_112, SR, 800000baseSR8), + BNGE_LINK_M(800GB, PAM4_112, KR, 800000baseKR8), +}; + +#define BNGE_LINK_MODE_UNKNOWN -1 + +static enum ethtool_link_mode_bit_indices +bnge_get_link_mode(struct bnge_net *bn) +{ + enum ethtool_link_mode_bit_indices link_mode; + struct bnge_ethtool_link_info *elink_info; + enum bnge_link_speed_indices speed; + struct bnge_link_info *link_info; + struct bnge_dev *bd = bn->bd; + enum bnge_media_type media; + u8 sig_mode; + + elink_info = &bn->eth_link_info; + link_info = &bd->link_info; + + if (link_info->phy_link_status != BNGE_LINK_LINK) + return BNGE_LINK_MODE_UNKNOWN; + + media = bnge_get_media(link_info); + if (BNGE_AUTO_MODE(link_info->auto_mode)) { + speed = bnge_fw_speed_idx(link_info->link_speed); + sig_mode = link_info->active_fec_sig_mode & + PORT_PHY_QCFG_RESP_SIGNAL_MODE_MASK; + } else { + speed = bnge_fw_speed_idx(elink_info->req_link_speed); + sig_mode = elink_info->req_signal_mode; + } + if (sig_mode >= BNGE_SIG_MODE_MAX) + return BNGE_LINK_MODE_UNKNOWN; + + /* Since ETHTOOL_LINK_MODE_10baseT_Half_BIT is defined as 0 and + * not actually supported, the zeroes in this map can be safely + * used to represent unknown link modes. + */ + link_mode = bnge_link_modes[speed][sig_mode][media]; + if (!link_mode) + return BNGE_LINK_MODE_UNKNOWN; + + return link_mode; +} + +static const u16 bnge_nrz_speeds2_masks[__BNGE_LINK_SPEED_END] = { + [BNGE_LINK_SPEED_100GB_IDX] = BNGE_LINK_SPEEDS2_MSK_100GB, +}; + +static const u16 bnge_pam4_speeds2_masks[__BNGE_LINK_SPEED_END] = { + [BNGE_LINK_SPEED_50GB_IDX] = BNGE_LINK_SPEEDS2_MSK_50GB_PAM4, + [BNGE_LINK_SPEED_100GB_IDX] = BNGE_LINK_SPEEDS2_MSK_100GB_PAM4, + [BNGE_LINK_SPEED_200GB_IDX] = BNGE_LINK_SPEEDS2_MSK_200GB_PAM4, + [BNGE_LINK_SPEED_400GB_IDX] = BNGE_LINK_SPEEDS2_MSK_400GB_PAM4, +}; + +static const u16 bnge_pam4_112_speeds2_masks[__BNGE_LINK_SPEED_END] = { + [BNGE_LINK_SPEED_100GB_IDX] = BNGE_LINK_SPEEDS2_MSK_100GB_PAM4_112, + [BNGE_LINK_SPEED_200GB_IDX] = BNGE_LINK_SPEEDS2_MSK_200GB_PAM4_112, + [BNGE_LINK_SPEED_400GB_IDX] = BNGE_LINK_SPEEDS2_MSK_400GB_PAM4_112, + [BNGE_LINK_SPEED_800GB_IDX] = BNGE_LINK_SPEEDS2_MSK_800GB_PAM4_112, +}; + +static enum bnge_link_speed_indices +bnge_encoding_speed_idx(u8 sig_mode, u16 speed_msk) +{ + const u16 *speeds; + int idx, len; + + switch (sig_mode) { + case BNGE_SIG_MODE_NRZ: + speeds = bnge_nrz_speeds2_masks; + len = ARRAY_SIZE(bnge_nrz_speeds2_masks); + break; + case BNGE_SIG_MODE_PAM4: + speeds = bnge_pam4_speeds2_masks; + len = ARRAY_SIZE(bnge_pam4_speeds2_masks); + break; + case BNGE_SIG_MODE_PAM4_112: + speeds = bnge_pam4_112_speeds2_masks; + len = ARRAY_SIZE(bnge_pam4_112_speeds2_masks); + break; + default: + return BNGE_LINK_SPEED_UNKNOWN; + } + + for (idx = 0; idx < len; idx++) { + if (speeds[idx] == speed_msk) + return idx; + } + + return BNGE_LINK_SPEED_UNKNOWN; +} + +#define BNGE_FW_SPEED_MSK_BITS 16 + +static void +__bnge_get_ethtool_speeds(unsigned long fw_mask, enum bnge_media_type media, + u8 sig_mode, unsigned long *et_mask) +{ + enum ethtool_link_mode_bit_indices link_mode; + enum bnge_link_speed_indices speed; + u8 bit; + + for_each_set_bit(bit, &fw_mask, BNGE_FW_SPEED_MSK_BITS) { + speed = bnge_encoding_speed_idx(sig_mode, 1 << bit); + if (!speed) + continue; + + link_mode = bnge_link_modes[speed][sig_mode][media]; + if (!link_mode) + continue; + + linkmode_set_bit(link_mode, et_mask); + } +} + +static void +bnge_get_ethtool_speeds(unsigned long fw_mask, enum bnge_media_type media, + u8 sig_mode, unsigned long *et_mask) +{ + if (media) { + __bnge_get_ethtool_speeds(fw_mask, media, sig_mode, et_mask); + return; + } + + /* list speeds for all media if unknown */ + for (media = 1; media < __BNGE_MEDIA_END; media++) + __bnge_get_ethtool_speeds(fw_mask, media, sig_mode, et_mask); +} + +static void +bnge_get_all_ethtool_support_speeds(struct bnge_dev *bd, + enum bnge_media_type media, + struct ethtool_link_ksettings *lk_ksettings) +{ + u16 sp = bd->link_info.support_speeds2; + + bnge_get_ethtool_speeds(sp, media, BNGE_SIG_MODE_NRZ, + lk_ksettings->link_modes.supported); + bnge_get_ethtool_speeds(sp, media, BNGE_SIG_MODE_PAM4, + lk_ksettings->link_modes.supported); + bnge_get_ethtool_speeds(sp, media, BNGE_SIG_MODE_PAM4_112, + lk_ksettings->link_modes.supported); +} + +static void +bnge_get_all_ethtool_adv_speeds(struct bnge_net *bn, + enum bnge_media_type media, + struct ethtool_link_ksettings *lk_ksettings) +{ + u16 sp = bn->eth_link_info.advertising; + + bnge_get_ethtool_speeds(sp, media, BNGE_SIG_MODE_NRZ, + lk_ksettings->link_modes.advertising); + bnge_get_ethtool_speeds(sp, media, BNGE_SIG_MODE_PAM4, + lk_ksettings->link_modes.advertising); + bnge_get_ethtool_speeds(sp, media, BNGE_SIG_MODE_PAM4_112, + lk_ksettings->link_modes.advertising); +} + +static void +bnge_get_all_ethtool_lp_speeds(struct bnge_dev *bd, + enum bnge_media_type media, + struct ethtool_link_ksettings *lk_ksettings) +{ + u16 sp = bd->link_info.lp_auto_link_speeds; + + bnge_get_ethtool_speeds(sp, media, BNGE_SIG_MODE_NRZ, + lk_ksettings->link_modes.lp_advertising); + bnge_get_ethtool_speeds(sp, media, BNGE_SIG_MODE_PAM4, + lk_ksettings->link_modes.lp_advertising); + bnge_get_ethtool_speeds(sp, media, BNGE_SIG_MODE_PAM4_112, + lk_ksettings->link_modes.lp_advertising); +} + +static void bnge_update_speed(u32 *delta, bool installed_media, u16 *speeds, + u16 speed_msk, const unsigned long *et_mask, + enum ethtool_link_mode_bit_indices mode) +{ + bool mode_desired = linkmode_test_bit(mode, et_mask); + + if (!mode || !mode_desired) + return; + + /* installed media takes priority; for non-installed media, only allow + * one change per fw_speed bit (many to one mapping). + */ + if (installed_media || !(*delta & speed_msk)) { + *speeds |= speed_msk; + *delta |= speed_msk; + } +} + +static void bnge_set_ethtool_speeds(struct bnge_net *bn, + const unsigned long *et_mask) +{ + struct bnge_ethtool_link_info *elink_info = &bn->eth_link_info; + enum bnge_media_type media; + u32 delta_pam4_112 = 0; + u32 delta_pam4 = 0; + u32 delta_nrz = 0; + int i, m; + + elink_info->advertising = 0; + + media = bnge_get_media(&bn->bd->link_info); + for (i = 1; i < __BNGE_LINK_SPEED_END; i++) { + /* accept any legal media from user */ + for (m = 1; m < __BNGE_MEDIA_END; m++) { + bnge_update_speed(&delta_nrz, m == media, + &elink_info->advertising, + bnge_nrz_speeds2_masks[i], et_mask, + bnge_link_modes[i][BNGE_SIG_MODE_NRZ][m]); + bnge_update_speed(&delta_pam4, m == media, + &elink_info->advertising, + bnge_pam4_speeds2_masks[i], et_mask, + bnge_link_modes[i][BNGE_SIG_MODE_PAM4][m]); + bnge_update_speed(&delta_pam4_112, m == media, + &elink_info->advertising, + bnge_pam4_112_speeds2_masks[i], + et_mask, + bnge_link_modes[i][BNGE_SIG_MODE_PAM4_112][m]); + } + } +} + +static void +bnge_fw_to_ethtool_advertised_fec(struct bnge_link_info *link_info, + struct ethtool_link_ksettings *lk_ksettings) +{ + u16 fec_cfg = link_info->fec_cfg; + + if ((fec_cfg & BNGE_FEC_NONE) || !(fec_cfg & BNGE_FEC_AUTONEG)) { + linkmode_set_bit(ETHTOOL_LINK_MODE_FEC_NONE_BIT, + lk_ksettings->link_modes.advertising); + return; + } + if (fec_cfg & BNGE_FEC_ENC_BASE_R) + linkmode_set_bit(ETHTOOL_LINK_MODE_FEC_BASER_BIT, + lk_ksettings->link_modes.advertising); + if (fec_cfg & BNGE_FEC_ENC_RS) + linkmode_set_bit(ETHTOOL_LINK_MODE_FEC_RS_BIT, + lk_ksettings->link_modes.advertising); + if (fec_cfg & BNGE_FEC_ENC_LLRS) + linkmode_set_bit(ETHTOOL_LINK_MODE_FEC_LLRS_BIT, + lk_ksettings->link_modes.advertising); +} + +static void +bnge_fw_to_ethtool_support_fec(struct bnge_link_info *link_info, + struct ethtool_link_ksettings *lk_ksettings) +{ + u16 fec_cfg = link_info->fec_cfg; + + if (fec_cfg & BNGE_FEC_NONE) { + linkmode_set_bit(ETHTOOL_LINK_MODE_FEC_NONE_BIT, + lk_ksettings->link_modes.supported); + return; + } + if (fec_cfg & BNGE_FEC_ENC_BASE_R_CAP) + linkmode_set_bit(ETHTOOL_LINK_MODE_FEC_BASER_BIT, + lk_ksettings->link_modes.supported); + if (fec_cfg & BNGE_FEC_ENC_RS_CAP) + linkmode_set_bit(ETHTOOL_LINK_MODE_FEC_RS_BIT, + lk_ksettings->link_modes.supported); + if (fec_cfg & BNGE_FEC_ENC_LLRS_CAP) + linkmode_set_bit(ETHTOOL_LINK_MODE_FEC_LLRS_BIT, + lk_ksettings->link_modes.supported); +} + +static void bnge_get_default_speeds(struct bnge_net *bn, + struct ethtool_link_ksettings *lk_ksettings) +{ + struct bnge_ethtool_link_info *elink_info = &bn->eth_link_info; + struct ethtool_link_settings *base = &lk_ksettings->base; + struct bnge_link_info *link_info; + struct bnge_dev *bd = bn->bd; + + link_info = &bd->link_info; + + if (link_info->link_state == BNGE_LINK_STATE_UP) { + base->speed = bnge_fw_to_ethtool_speed(link_info->link_speed); + base->duplex = DUPLEX_HALF; + if (link_info->duplex & BNGE_LINK_DUPLEX_FULL) + base->duplex = DUPLEX_FULL; + lk_ksettings->lanes = link_info->active_lanes; + } else if (!elink_info->autoneg) { + base->speed = + bnge_fw_to_ethtool_speed(elink_info->req_link_speed); + base->duplex = DUPLEX_HALF; + if (elink_info->req_duplex == BNGE_LINK_DUPLEX_FULL) + base->duplex = DUPLEX_FULL; + } +} + +int bnge_get_link_ksettings(struct net_device *dev, + struct ethtool_link_ksettings *lk_ksettings) +{ + struct ethtool_link_settings *base = &lk_ksettings->base; + enum ethtool_link_mode_bit_indices link_mode; + struct bnge_net *bn = netdev_priv(dev); + struct bnge_link_info *link_info; + struct bnge_dev *bd = bn->bd; + enum bnge_media_type media; + + ethtool_link_ksettings_zero_link_mode(lk_ksettings, lp_advertising); + ethtool_link_ksettings_zero_link_mode(lk_ksettings, advertising); + ethtool_link_ksettings_zero_link_mode(lk_ksettings, supported); + base->duplex = DUPLEX_UNKNOWN; + base->speed = SPEED_UNKNOWN; + link_info = &bd->link_info; + + bnge_get_ethtool_modes(bn, lk_ksettings); + media = bnge_get_media(link_info); + bnge_get_all_ethtool_support_speeds(bd, media, lk_ksettings); + bnge_fw_to_ethtool_support_fec(link_info, lk_ksettings); + link_mode = bnge_get_link_mode(bn); + if (link_mode != BNGE_LINK_MODE_UNKNOWN) + ethtool_params_from_link_mode(lk_ksettings, link_mode); + else + bnge_get_default_speeds(bn, lk_ksettings); + + if (bn->eth_link_info.autoneg) { + bnge_fw_to_ethtool_advertised_fec(link_info, lk_ksettings); + linkmode_set_bit(ETHTOOL_LINK_MODE_Autoneg_BIT, + lk_ksettings->link_modes.advertising); + base->autoneg = AUTONEG_ENABLE; + bnge_get_all_ethtool_adv_speeds(bn, media, lk_ksettings); + if (link_info->phy_link_status == BNGE_LINK_LINK) + bnge_get_all_ethtool_lp_speeds(bd, media, lk_ksettings); + } else { + base->autoneg = AUTONEG_DISABLE; + } + + linkmode_set_bit(ETHTOOL_LINK_MODE_FIBRE_BIT, + lk_ksettings->link_modes.supported); + linkmode_set_bit(ETHTOOL_LINK_MODE_FIBRE_BIT, + lk_ksettings->link_modes.advertising); + + if (link_info->media_type == PORT_PHY_QCFG_RESP_MEDIA_TYPE_DAC) + base->port = PORT_DA; + else + base->port = PORT_FIBRE; + base->phy_address = link_info->phy_addr; + + return 0; +} + +static bool bnge_lanes_match(u32 user_lanes, u32 supported_lanes) +{ + /* 0 means lanes unspecified (auto) */ + return !user_lanes || user_lanes == supported_lanes; +} + +static int +bnge_force_link_speed(struct net_device *dev, u32 ethtool_speed, u32 user_lanes) +{ + struct bnge_ethtool_link_info *elink_info; + struct bnge_net *bn = netdev_priv(dev); + u8 sig_mode = BNGE_SIG_MODE_NRZ; + u16 support_spds2; + u16 fw_speed = 0; + + elink_info = &bn->eth_link_info; + support_spds2 = bn->bd->link_info.support_speeds2; + + switch (ethtool_speed) { + case SPEED_50000: + if (bnge_lanes_match(user_lanes, 1) && + (support_spds2 & BNGE_LINK_SPEEDS2_MSK_50GB_PAM4)) { + fw_speed = BNGE_LINK_SPEED_50GB_PAM4; + sig_mode = BNGE_SIG_MODE_PAM4; + } + break; + case SPEED_100000: + if (bnge_lanes_match(user_lanes, 4) && + (support_spds2 & BNGE_LINK_SPEEDS2_MSK_100GB)) { + fw_speed = PORT_PHY_CFG_REQ_FORCE_LINK_SPEED_100GB; + } else if (bnge_lanes_match(user_lanes, 2) && + (support_spds2 & BNGE_LINK_SPEEDS2_MSK_100GB_PAM4)) { + fw_speed = BNGE_LINK_SPEED_100GB_PAM4; + sig_mode = BNGE_SIG_MODE_PAM4; + } else if (bnge_lanes_match(user_lanes, 1) && + (support_spds2 & BNGE_LINK_SPEEDS2_MSK_100GB_PAM4_112)) { + fw_speed = BNGE_LINK_SPEED_100GB_PAM4_112; + sig_mode = BNGE_SIG_MODE_PAM4_112; + } + break; + case SPEED_200000: + if (bnge_lanes_match(user_lanes, 4) && + (support_spds2 & BNGE_LINK_SPEEDS2_MSK_200GB_PAM4)) { + fw_speed = BNGE_LINK_SPEED_200GB_PAM4; + sig_mode = BNGE_SIG_MODE_PAM4; + } else if (bnge_lanes_match(user_lanes, 2) && + (support_spds2 & BNGE_LINK_SPEEDS2_MSK_200GB_PAM4_112)) { + fw_speed = BNGE_LINK_SPEED_200GB_PAM4_112; + sig_mode = BNGE_SIG_MODE_PAM4_112; + } + break; + case SPEED_400000: + if (bnge_lanes_match(user_lanes, 8) && + (support_spds2 & BNGE_LINK_SPEEDS2_MSK_400GB_PAM4)) { + fw_speed = BNGE_LINK_SPEED_400GB_PAM4; + sig_mode = BNGE_SIG_MODE_PAM4; + } else if (bnge_lanes_match(user_lanes, 4) && + (support_spds2 & BNGE_LINK_SPEEDS2_MSK_400GB_PAM4_112)) { + fw_speed = BNGE_LINK_SPEED_400GB_PAM4_112; + sig_mode = BNGE_SIG_MODE_PAM4_112; + } + break; + case SPEED_800000: + if (bnge_lanes_match(user_lanes, 8) && + (support_spds2 & BNGE_LINK_SPEEDS2_MSK_800GB_PAM4_112)) { + fw_speed = BNGE_LINK_SPEED_800GB_PAM4_112; + sig_mode = BNGE_SIG_MODE_PAM4_112; + } + break; + default: + break; + } + + if (!fw_speed) { + if (user_lanes) + netdev_err(dev, "unsupported speed or number of lanes!\n"); + else + netdev_err(dev, "unsupported speed!\n"); + return -EINVAL; + } + + if (elink_info->req_link_speed == fw_speed && + elink_info->req_signal_mode == sig_mode && + elink_info->autoneg == 0) + return -EALREADY; + + elink_info->req_link_speed = fw_speed; + elink_info->req_signal_mode = sig_mode; + elink_info->req_duplex = BNGE_LINK_DUPLEX_FULL; + elink_info->autoneg = 0; + elink_info->advertising = 0; + + return 0; +} + +int bnge_set_link_ksettings(struct net_device *dev, + const struct ethtool_link_ksettings *lk_ksettings) +{ + const struct ethtool_link_settings *base = &lk_ksettings->base; + struct bnge_ethtool_link_info old_elink_info; + struct bnge_ethtool_link_info *elink_info; + struct bnge_net *bn = netdev_priv(dev); + struct bnge_dev *bd = bn->bd; + bool set_pause = false; + int rc = 0; + + if (!BNGE_PHY_CFG_ABLE(bd)) + return -EOPNOTSUPP; + + elink_info = &bn->eth_link_info; + old_elink_info = *elink_info; + + if (base->autoneg == AUTONEG_ENABLE) { + bnge_set_ethtool_speeds(bn, + lk_ksettings->link_modes.advertising); + elink_info->autoneg |= BNGE_AUTONEG_SPEED; + if (!elink_info->advertising) + bnge_set_default_adv_speeds(bn); + /* any change to autoneg will cause link change, therefore the + * driver should put back the original pause setting in autoneg + */ + if (!(bd->phy_flags & BNGE_PHY_FL_NO_PAUSE)) + set_pause = true; + } else { + if (base->duplex == DUPLEX_HALF) { + netdev_err(dev, "HALF DUPLEX is not supported!\n"); + rc = -EINVAL; + goto set_setting_exit; + } + rc = bnge_force_link_speed(dev, base->speed, + lk_ksettings->lanes); + if (rc) { + if (rc == -EALREADY) + rc = 0; + goto set_setting_exit; + } + } + + if (netif_running(dev)) { + rc = bnge_hwrm_set_link_setting(bn, set_pause); + if (rc) + *elink_info = old_elink_info; + } + +set_setting_exit: + return rc; +} + +void bnge_link_async_event_process(struct bnge_net *bn, u16 event_id) +{ + switch (event_id) { + case ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE: + set_bit(BNGE_LINK_SPEED_CHNG_SP_EVENT, &bn->sp_event); + break; + case ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CHANGE: + case ASYNC_EVENT_CMPL_EVENT_ID_PORT_PHY_CFG_CHANGE: + set_bit(BNGE_LINK_CFG_CHANGE_SP_EVENT, &bn->sp_event); + break; + case ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE: + set_bit(BNGE_LINK_CHNG_SP_EVENT, &bn->sp_event); + break; + default: + break; + } +} diff --git a/drivers/net/ethernet/broadcom/bnge/bnge_link.h b/drivers/net/ethernet/broadcom/bnge/bnge_link.h new file mode 100644 index 000000000000..ccaf343aa018 --- /dev/null +++ b/drivers/net/ethernet/broadcom/bnge/bnge_link.h @@ -0,0 +1,173 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (c) 2026 Broadcom */ + +#ifndef _BNGE_LINK_H_ +#define _BNGE_LINK_H_ + +#include <linux/ethtool.h> + +#define BNGE_PHY_CFG_ABLE(bd) \ + ((bd)->link_info.phy_enabled) + +#define BNGE_PHY_AUTO_SPEEDS2_MASK \ + PORT_PHY_CFG_REQ_ENABLES_AUTO_LINK_SPEEDS2_MASK +#define BNGE_PHY_FLAGS_RESTART_AUTO \ + PORT_PHY_CFG_REQ_FLAGS_RESTART_AUTONEG +#define BNGE_PHY_FLAGS_ENA_FORCE_SPEEDS2 \ + PORT_PHY_CFG_REQ_ENABLES_FORCE_LINK_SPEEDS2 + +#define BNGE_LINK_LINK PORT_PHY_QCFG_RESP_LINK_LINK + +enum bnge_link_state { + BNGE_LINK_STATE_UNKNOWN, + BNGE_LINK_STATE_DOWN, + BNGE_LINK_STATE_UP, +}; + +#define BNGE_LINK_IS_UP(bd) \ + ((bd)->link_info.link_state == BNGE_LINK_STATE_UP) + +#define BNGE_LINK_DUPLEX_FULL PORT_PHY_QCFG_RESP_DUPLEX_STATE_FULL + +#define BNGE_LINK_PAUSE_TX PORT_PHY_QCFG_RESP_PAUSE_TX +#define BNGE_LINK_PAUSE_RX PORT_PHY_QCFG_RESP_PAUSE_RX +#define BNGE_LINK_PAUSE_BOTH (PORT_PHY_QCFG_RESP_PAUSE_RX | \ + PORT_PHY_QCFG_RESP_PAUSE_TX) + +#define BNGE_LINK_AUTO_NONE PORT_PHY_QCFG_RESP_AUTO_MODE_NONE +#define BNGE_LINK_AUTO_MSK PORT_PHY_QCFG_RESP_AUTO_MODE_SPEED_MASK +#define BNGE_AUTO_MODE(mode) ((mode) > BNGE_LINK_AUTO_NONE && \ + (mode) <= BNGE_LINK_AUTO_MSK) + +#define BNGE_LINK_SPEED_50GB PORT_PHY_QCFG_RESP_LINK_SPEED_50GB +#define BNGE_LINK_SPEED_100GB PORT_PHY_QCFG_RESP_LINK_SPEED_100GB +#define BNGE_LINK_SPEED_200GB PORT_PHY_QCFG_RESP_LINK_SPEED_200GB +#define BNGE_LINK_SPEED_400GB PORT_PHY_QCFG_RESP_LINK_SPEED_400GB +#define BNGE_LINK_SPEED_800GB PORT_PHY_QCFG_RESP_LINK_SPEED_800GB + +#define BNGE_LINK_SPEEDS2_MSK_50GB PORT_PHY_QCFG_RESP_SUPPORT_SPEEDS2_50GB +#define BNGE_LINK_SPEEDS2_MSK_100GB PORT_PHY_QCFG_RESP_SUPPORT_SPEEDS2_100GB +#define BNGE_LINK_SPEEDS2_MSK_50GB_PAM4 \ + PORT_PHY_QCFG_RESP_SUPPORT_SPEEDS2_50GB_PAM4_56 +#define BNGE_LINK_SPEEDS2_MSK_100GB_PAM4 \ + PORT_PHY_QCFG_RESP_SUPPORT_SPEEDS2_100GB_PAM4_56 +#define BNGE_LINK_SPEEDS2_MSK_200GB_PAM4 \ + PORT_PHY_QCFG_RESP_SUPPORT_SPEEDS2_200GB_PAM4_56 +#define BNGE_LINK_SPEEDS2_MSK_400GB_PAM4 \ + PORT_PHY_QCFG_RESP_SUPPORT_SPEEDS2_400GB_PAM4_56 +#define BNGE_LINK_SPEEDS2_MSK_100GB_PAM4_112 \ + PORT_PHY_QCFG_RESP_SUPPORT_SPEEDS2_100GB_PAM4_112 +#define BNGE_LINK_SPEEDS2_MSK_200GB_PAM4_112 \ + PORT_PHY_QCFG_RESP_SUPPORT_SPEEDS2_200GB_PAM4_112 +#define BNGE_LINK_SPEEDS2_MSK_400GB_PAM4_112 \ + PORT_PHY_QCFG_RESP_SUPPORT_SPEEDS2_400GB_PAM4_112 +#define BNGE_LINK_SPEEDS2_MSK_800GB_PAM4_112 \ + PORT_PHY_QCFG_RESP_SUPPORT_SPEEDS2_800GB_PAM4_112 + +#define BNGE_LINK_SPEED_50GB_PAM4 \ + PORT_PHY_CFG_REQ_FORCE_LINK_SPEEDS2_50GB_PAM4_56 +#define BNGE_LINK_SPEED_100GB_PAM4 \ + PORT_PHY_CFG_REQ_FORCE_LINK_SPEEDS2_100GB_PAM4_56 +#define BNGE_LINK_SPEED_200GB_PAM4 \ + PORT_PHY_CFG_REQ_FORCE_LINK_SPEEDS2_200GB_PAM4_56 +#define BNGE_LINK_SPEED_400GB_PAM4 \ + PORT_PHY_CFG_REQ_FORCE_LINK_SPEEDS2_400GB_PAM4_56 +#define BNGE_LINK_SPEED_100GB_PAM4_112 \ + PORT_PHY_CFG_REQ_FORCE_LINK_SPEEDS2_100GB_PAM4_112 +#define BNGE_LINK_SPEED_200GB_PAM4_112 \ + PORT_PHY_CFG_REQ_FORCE_LINK_SPEEDS2_200GB_PAM4_112 +#define BNGE_LINK_SPEED_400GB_PAM4_112 \ + PORT_PHY_CFG_REQ_FORCE_LINK_SPEEDS2_400GB_PAM4_112 +#define BNGE_LINK_SPEED_800GB_PAM4_112 \ + PORT_PHY_CFG_REQ_FORCE_LINK_SPEEDS2_800GB_PAM4_112 + +#define BNGE_FEC_NONE PORT_PHY_QCFG_RESP_FEC_CFG_FEC_NONE_SUPPORTED +#define BNGE_FEC_AUTONEG PORT_PHY_QCFG_RESP_FEC_CFG_FEC_AUTONEG_ENABLED +#define BNGE_FEC_ENC_BASE_R_CAP \ + PORT_PHY_QCFG_RESP_FEC_CFG_FEC_CLAUSE74_SUPPORTED +#define BNGE_FEC_ENC_BASE_R PORT_PHY_QCFG_RESP_FEC_CFG_FEC_CLAUSE74_ENABLED +#define BNGE_FEC_ENC_RS_CAP \ + PORT_PHY_QCFG_RESP_FEC_CFG_FEC_CLAUSE91_SUPPORTED +#define BNGE_FEC_ENC_LLRS_CAP \ + (PORT_PHY_QCFG_RESP_FEC_CFG_FEC_RS272_1XN_SUPPORTED | \ + PORT_PHY_QCFG_RESP_FEC_CFG_FEC_RS272_IEEE_SUPPORTED) +#define BNGE_FEC_ENC_RS \ + (PORT_PHY_QCFG_RESP_FEC_CFG_FEC_CLAUSE91_ENABLED | \ + PORT_PHY_QCFG_RESP_FEC_CFG_FEC_RS544_1XN_ENABLED | \ + PORT_PHY_QCFG_RESP_FEC_CFG_FEC_RS544_IEEE_ENABLED) +#define BNGE_FEC_ENC_LLRS \ + (PORT_PHY_QCFG_RESP_FEC_CFG_FEC_RS272_1XN_ENABLED | \ + PORT_PHY_QCFG_RESP_FEC_CFG_FEC_RS272_IEEE_ENABLED) + +struct bnge_link_info { + u8 phy_type; + u8 media_type; + u8 phy_addr; + u8 phy_link_status; + bool phy_enabled; + + u8 link_state; + u8 active_lanes; + u8 duplex; + u8 pause; + u8 lp_pause; + u8 auto_pause_setting; + u8 force_pause_setting; + u8 duplex_setting; + u8 auto_mode; + u16 link_speed; + u16 support_speeds2; + u16 auto_link_speeds2; + u16 support_auto_speeds2; + u16 lp_auto_link_speeds; + u16 force_link_speed2; + + u8 module_status; + u8 active_fec_sig_mode; + u16 fec_cfg; + + /* A copy of phy_qcfg output used to report link + * info to VF + */ + struct hwrm_port_phy_qcfg_output phy_qcfg_resp; + + bool phy_retry; + unsigned long phy_retry_expires; +}; + +#define BNGE_AUTONEG_SPEED 1 +#define BNGE_AUTONEG_FLOW_CTRL 2 + +#define BNGE_SIG_MODE_NRZ PORT_PHY_QCFG_RESP_SIGNAL_MODE_NRZ +#define BNGE_SIG_MODE_PAM4 PORT_PHY_QCFG_RESP_SIGNAL_MODE_PAM4 +#define BNGE_SIG_MODE_PAM4_112 PORT_PHY_QCFG_RESP_SIGNAL_MODE_PAM4_112 +#define BNGE_SIG_MODE_MAX (PORT_PHY_QCFG_RESP_SIGNAL_MODE_LAST + 1) + +struct bnge_ethtool_link_info { + /* copy of requested setting from ethtool cmd */ + u8 autoneg; + u8 req_signal_mode; + u8 req_duplex; + u8 req_flow_ctrl; + u16 req_link_speed; + u16 advertising; /* user adv setting */ + bool force_link_chng; +}; + +void bnge_hwrm_set_link_common(struct bnge_net *bn, + struct hwrm_port_phy_cfg_input *req); +void bnge_hwrm_set_pause_common(struct bnge_net *bn, + struct hwrm_port_phy_cfg_input *req); +int bnge_update_phy_setting(struct bnge_net *bn); +void bnge_get_port_module_status(struct bnge_net *bn); +void bnge_report_link(struct bnge_dev *bd); +bool bnge_support_speed_dropped(struct bnge_net *bn); +void bnge_init_ethtool_link_settings(struct bnge_net *bn); +int bnge_probe_phy(struct bnge_net *bn, bool fw_dflt); +int bnge_set_link_ksettings(struct net_device *dev, + const struct ethtool_link_ksettings *lk_ksettings); +int bnge_get_link_ksettings(struct net_device *dev, + struct ethtool_link_ksettings *lk_ksettings); +u32 bnge_get_link(struct net_device *dev); +void bnge_link_async_event_process(struct bnge_net *bn, u16 event_id); +#endif /* _BNGE_LINK_H_ */ diff --git a/drivers/net/ethernet/broadcom/bnge/bnge_netdev.c b/drivers/net/ethernet/broadcom/bnge/bnge_netdev.c index a20dc3ca640c..70768193004c 100644 --- a/drivers/net/ethernet/broadcom/bnge/bnge_netdev.c +++ b/drivers/net/ethernet/broadcom/bnge/bnge_netdev.c @@ -39,6 +39,10 @@ static void bnge_free_stats_mem(struct bnge_net *bn, { struct bnge_dev *bd = bn->bd; + kfree(stats->hw_masks); + stats->hw_masks = NULL; + kfree(stats->sw_stats); + stats->sw_stats = NULL; if (stats->hw_stats) { dma_free_coherent(bd->dev, stats->len, stats->hw_stats, stats->hw_stats_map); @@ -47,7 +51,7 @@ static void bnge_free_stats_mem(struct bnge_net *bn, } static int bnge_alloc_stats_mem(struct bnge_net *bn, - struct bnge_stats_mem *stats) + struct bnge_stats_mem *stats, bool alloc_masks) { struct bnge_dev *bd = bn->bd; @@ -56,7 +60,23 @@ static int bnge_alloc_stats_mem(struct bnge_net *bn, if (!stats->hw_stats) return -ENOMEM; + stats->sw_stats = kzalloc(stats->len, GFP_KERNEL); + if (!stats->sw_stats) + goto stats_mem_err; + + if (alloc_masks) { + stats->hw_masks = kzalloc(stats->len, GFP_KERNEL); + if (!stats->hw_masks) + goto stats_mem_err; + } + + u64_stats_init(&stats->syncp); + return 0; + +stats_mem_err: + bnge_free_stats_mem(bn, stats); + return -ENOMEM; } static void bnge_free_ring_stats(struct bnge_net *bn) @@ -75,6 +95,107 @@ static void bnge_free_ring_stats(struct bnge_net *bn) } } +static void bnge_fill_masks(u64 *mask_arr, u64 mask, int count) +{ + int i; + + for (i = 0; i < count; i++) + mask_arr[i] = mask; +} + +void bnge_copy_hw_masks(u64 *mask_arr, __le64 *hw_mask_arr, int count) +{ + int i; + + for (i = 0; i < count; i++) + mask_arr[i] = le64_to_cpu(hw_mask_arr[i]); +} + +static void bnge_init_stats(struct bnge_net *bn) +{ + struct bnge_napi *bnapi = bn->bnapi[0]; + struct bnge_nq_ring_info *nqr; + struct bnge_stats_mem *stats; + struct bnge_dev *bd = bn->bd; + __le64 *rx_stats, *tx_stats; + int rc, rx_count, tx_count; + u64 *rx_masks, *tx_masks; + u8 flags; + + nqr = &bnapi->nq_ring; + stats = &nqr->stats; + rc = bnge_hwrm_func_qstat_ext(bd, stats); + if (rc) { + u64 mask = (1ULL << 48) - 1; + + bnge_fill_masks(stats->hw_masks, mask, stats->len / 8); + } + + if (bn->flags & BNGE_FLAG_PORT_STATS) { + stats = &bn->port_stats; + rx_stats = stats->hw_stats; + rx_masks = stats->hw_masks; + rx_count = sizeof(struct rx_port_stats) / 8; + tx_stats = rx_stats + BNGE_TX_PORT_STATS_BYTE_OFFSET / 8; + tx_masks = rx_masks + BNGE_TX_PORT_STATS_BYTE_OFFSET / 8; + tx_count = sizeof(struct tx_port_stats) / 8; + + flags = PORT_QSTATS_REQ_FLAGS_COUNTER_MASK; + rc = bnge_hwrm_port_qstats(bd, flags); + if (rc) { + u64 mask = (1ULL << 40) - 1; + + bnge_fill_masks(rx_masks, mask, rx_count); + bnge_fill_masks(tx_masks, mask, tx_count); + } else { + bnge_copy_hw_masks(rx_masks, rx_stats, rx_count); + bnge_copy_hw_masks(tx_masks, tx_stats, tx_count); + } + bnge_hwrm_port_qstats(bd, 0); + } + + if (bn->flags & BNGE_FLAG_PORT_STATS_EXT) { + stats = &bn->rx_port_stats_ext; + rx_stats = stats->hw_stats; + rx_masks = stats->hw_masks; + rx_count = sizeof(struct rx_port_stats_ext) / 8; + stats = &bn->tx_port_stats_ext; + tx_stats = stats->hw_stats; + tx_masks = stats->hw_masks; + tx_count = sizeof(struct tx_port_stats_ext) / 8; + + flags = PORT_QSTATS_EXT_REQ_FLAGS_COUNTER_MASK; + rc = bnge_hwrm_port_qstats_ext(bd, flags); + if (rc) { + u64 mask = (1ULL << 40) - 1; + + bnge_fill_masks(rx_masks, mask, rx_count); + if (tx_stats) + bnge_fill_masks(tx_masks, mask, tx_count); + } else { + bnge_copy_hw_masks(rx_masks, rx_stats, rx_count); + if (tx_stats) + bnge_copy_hw_masks(tx_masks, tx_stats, + tx_count); + } + bnge_hwrm_port_qstats_ext(bd, 0); + } +} + +static void bnge_free_port_ext_stats(struct bnge_net *bn) +{ + bn->flags &= ~BNGE_FLAG_PORT_STATS_EXT; + bnge_free_stats_mem(bn, &bn->rx_port_stats_ext); + bnge_free_stats_mem(bn, &bn->tx_port_stats_ext); +} + +static void bnge_free_port_stats(struct bnge_net *bn) +{ + bn->flags &= ~BNGE_FLAG_PORT_STATS; + bnge_free_stats_mem(bn, &bn->port_stats); + bnge_free_port_ext_stats(bn); +} + static int bnge_alloc_ring_stats(struct bnge_net *bn) { struct bnge_dev *bd = bn->bd; @@ -88,12 +209,13 @@ static int bnge_alloc_ring_stats(struct bnge_net *bn) struct bnge_nq_ring_info *nqr = &bnapi->nq_ring; nqr->stats.len = size; - rc = bnge_alloc_stats_mem(bn, &nqr->stats); + rc = bnge_alloc_stats_mem(bn, &nqr->stats, !i); if (rc) goto err_free_ring_stats; nqr->hw_stats_ctx_id = INVALID_STATS_CTX_ID; } + return 0; err_free_ring_stats: @@ -101,6 +223,227 @@ err_free_ring_stats: return rc; } +static void bnge_alloc_port_ext_stats(struct bnge_net *bn) +{ + struct bnge_dev *bd = bn->bd; + int rc; + + if (!(bd->fw_cap & BNGE_FW_CAP_EXT_STATS_SUPPORTED)) + return; + + if (!bn->rx_port_stats_ext.hw_stats) { + bn->rx_port_stats_ext.len = sizeof(struct rx_port_stats_ext); + rc = bnge_alloc_stats_mem(bn, &bn->rx_port_stats_ext, true); + /* Extended stats are optional */ + if (rc) + return; + } + + if (!bn->tx_port_stats_ext.hw_stats) { + bn->tx_port_stats_ext.len = sizeof(struct tx_port_stats_ext); + rc = bnge_alloc_stats_mem(bn, &bn->tx_port_stats_ext, true); + /* Extended stats are optional */ + if (rc) { + bnge_free_port_ext_stats(bn); + return; + } + } + bn->flags |= BNGE_FLAG_PORT_STATS_EXT; +} + +static int bnge_alloc_port_stats(struct bnge_net *bn) +{ + int rc; + + if (!bn->port_stats.hw_stats) { + bn->port_stats.len = BNGE_PORT_STATS_SIZE; + rc = bnge_alloc_stats_mem(bn, &bn->port_stats, true); + if (rc) + return rc; + + bn->flags |= BNGE_FLAG_PORT_STATS; + } + + bnge_alloc_port_ext_stats(bn); + return 0; +} + +void __bnge_queue_sp_work(struct bnge_net *bn) +{ + queue_work(bn->bnge_pf_wq, &bn->sp_task); +} + +static void bnge_queue_sp_work(struct bnge_net *bn, unsigned int event) +{ + set_bit(event, &bn->sp_event); + __bnge_queue_sp_work(bn); +} + +static void bnge_timer(struct timer_list *t) +{ + struct bnge_net *bn = timer_container_of(bn, t, timer); + struct bnge_dev *bd = bn->bd; + + if (!netif_running(bn->netdev) || + !test_bit(BNGE_STATE_OPEN, &bd->state)) + return; + + if (READ_ONCE(bd->link_info.phy_retry)) { + if (time_after(jiffies, READ_ONCE(bd->link_info.phy_retry_expires))) { + WRITE_ONCE(bd->link_info.phy_retry, false); + netdev_warn(bn->netdev, "failed to update PHY settings after maximum retries.\n"); + } else { + bnge_queue_sp_work(bn, BNGE_UPDATE_PHY_SP_EVENT); + } + } + + if (BNGE_LINK_IS_UP(bd) && bn->stats_coal_ticks) + bnge_queue_sp_work(bn, BNGE_PERIODIC_STATS_SP_EVENT); + + mod_timer(&bn->timer, jiffies + bn->current_interval); +} + +static void bnge_add_one_ctr(u64 hw, u64 *sw, u64 mask) +{ + u64 sw_tmp, sw_val; + + hw &= mask; + sw_val = READ_ONCE(*sw); + sw_tmp = (sw_val & ~mask) | hw; + if (hw < (sw_val & mask)) + sw_tmp += mask + 1; + WRITE_ONCE(*sw, sw_tmp); +} + +static void __bnge_accumulate_stats(__le64 *hw_stats, u64 *sw_stats, u64 *masks, + int count, struct u64_stats_sync *syncp) +{ + unsigned long flags; + int i; + + flags = u64_stats_update_begin_irqsave(syncp); + for (i = 0; i < count; i++) { + u64 hw = le64_to_cpu(READ_ONCE(hw_stats[i])); + + if (masks[i] == -1ULL) + WRITE_ONCE(sw_stats[i], hw); + else + bnge_add_one_ctr(hw, &sw_stats[i], masks[i]); + } + u64_stats_update_end_irqrestore(syncp, flags); +} + +static void bnge_accumulate_stats(struct bnge_stats_mem *stats) +{ + if (!stats->hw_stats) + return; + + __bnge_accumulate_stats(stats->hw_stats, stats->sw_stats, + stats->hw_masks, stats->len / 8, &stats->syncp); +} + +static void bnge_accumulate_all_stats(struct bnge_dev *bd) +{ + struct bnge_net *bn = netdev_priv(bd->netdev); + struct bnge_stats_mem *ring0_stats = NULL; + int i; + + for (i = 0; i < bd->nq_nr_rings; i++) { + struct bnge_napi *bnapi = bn->bnapi[i]; + struct bnge_nq_ring_info *nqr; + struct bnge_stats_mem *stats; + + nqr = &bnapi->nq_ring; + stats = &nqr->stats; + + if (!ring0_stats) + ring0_stats = &bn->bnapi[0]->nq_ring.stats; + + __bnge_accumulate_stats(stats->hw_stats, stats->sw_stats, + ring0_stats->hw_masks, + ring0_stats->len / 8, &stats->syncp); + } + + if (bn->flags & BNGE_FLAG_PORT_STATS) { + struct bnge_stats_mem *stats = &bn->port_stats; + __le64 *hw_stats = stats->hw_stats; + u64 *sw_stats = stats->sw_stats; + u64 *masks = stats->hw_masks; + u16 cnt; + + cnt = sizeof(struct rx_port_stats) / 8; + __bnge_accumulate_stats(hw_stats, sw_stats, masks, cnt, + &stats->syncp); + + hw_stats += BNGE_TX_PORT_STATS_BYTE_OFFSET / 8; + sw_stats += BNGE_TX_PORT_STATS_BYTE_OFFSET / 8; + masks += BNGE_TX_PORT_STATS_BYTE_OFFSET / 8; + cnt = sizeof(struct tx_port_stats) / 8; + __bnge_accumulate_stats(hw_stats, sw_stats, masks, cnt, + &stats->syncp); + } + + if (bn->flags & BNGE_FLAG_PORT_STATS_EXT) { + bnge_accumulate_stats(&bn->rx_port_stats_ext); + bnge_accumulate_stats(&bn->tx_port_stats_ext); + } +} + +static void bnge_sp_task(struct work_struct *work) +{ + struct bnge_net *bn = container_of(work, struct bnge_net, sp_task); + bool speed_chng, cfg_chng, link_chng; + struct bnge_dev *bd = bn->bd; + + netdev_lock(bn->netdev); + if (!test_bit(BNGE_STATE_OPEN, &bd->state)) { + netdev_unlock(bn->netdev); + return; + } + + if (test_and_clear_bit(BNGE_PERIODIC_STATS_SP_EVENT, &bn->sp_event)) { + bnge_hwrm_port_qstats(bd, 0); + bnge_hwrm_port_qstats_ext(bd, 0); + bnge_accumulate_all_stats(bd); + } + + if (test_and_clear_bit(BNGE_UPDATE_PHY_SP_EVENT, &bn->sp_event)) { + int rc; + + rc = bnge_update_phy_setting(bn); + if (rc) { + netdev_warn(bn->netdev, "update PHY settings retry failed\n"); + } else { + WRITE_ONCE(bd->link_info.phy_retry, false); + netdev_info(bn->netdev, "update PHY settings retry succeeded\n"); + } + } + + speed_chng = test_and_clear_bit(BNGE_LINK_SPEED_CHNG_SP_EVENT, + &bn->sp_event); + cfg_chng = test_and_clear_bit(BNGE_LINK_CFG_CHANGE_SP_EVENT, + &bn->sp_event); + link_chng = test_and_clear_bit(BNGE_LINK_CHNG_SP_EVENT, + &bn->sp_event); + + if (speed_chng || cfg_chng || link_chng) { + int rc; + + if (speed_chng) + bnge_hwrm_phy_qcaps(bd); + + rc = bnge_update_link(bn, true); + if (rc) + netdev_err(bn->netdev, "SP task cannot update link (rc: %d)\n", + rc); + + if (speed_chng || cfg_chng) + bnge_init_ethtool_link_settings(bn); + } + + netdev_unlock(bn->netdev); +} + static void bnge_free_nq_desc_arr(struct bnge_nq_ring_info *nqr) { struct bnge_ring_struct *ring = &nqr->ring_struct; @@ -947,6 +1290,8 @@ static int bnge_alloc_core(struct bnge_net *bn) if (rc) goto err_free_core; + bnge_init_stats(bn); + rc = bnge_alloc_vnics(bn); if (rc) goto err_free_core; @@ -2466,6 +2811,8 @@ static void bnge_tx_enable(struct bnge_net *bn) /* Make sure napi polls see @dev_state change */ synchronize_net(); netif_tx_wake_all_queues(bn->netdev); + if (BNGE_LINK_IS_UP(bn->bd)) + netif_carrier_on(bn->netdev); } static int bnge_open_core(struct bnge_net *bn) @@ -2502,11 +2849,28 @@ static int bnge_open_core(struct bnge_net *bn) bnge_enable_napi(bn); + rc = bnge_update_phy_setting(bn); + if (rc) { + netdev_warn(bn->netdev, "failed to update PHY settings (rc: %d)\n", + rc); + WRITE_ONCE(bd->link_info.phy_retry_expires, jiffies + 5 * HZ); + WRITE_ONCE(bd->link_info.phy_retry, true); + } else { + WRITE_ONCE(bd->link_info.phy_retry, false); + } + set_bit(BNGE_STATE_OPEN, &bd->state); + set_bit(BNGE_STATE_STATS_ENABLE, &bn->state); bnge_enable_int(bn); bnge_tx_enable(bn); + + mod_timer(&bn->timer, jiffies + bn->current_interval); + + /* Poll link status and check for SFP+ module status */ + bnge_get_port_module_status(bn); + return 0; err_free_irq: @@ -2535,6 +2899,184 @@ static int bnge_shutdown_nic(struct bnge_net *bn) return 0; } +static void bnge_add_prev_ring_stats64(struct bnge_net *bn, + struct rtnl_link_stats64 *stats) +{ + struct netdev_queue_stats_rx *rx_save = &bn->rxq_prv_stats; + struct netdev_queue_stats_tx *tx_save = &bn->txq_prv_stats; + struct rtnl_link_stats64 *stats64_save = &bn->prv_stats64; + + stats->rx_packets += rx_save->packets; + stats->tx_packets += tx_save->packets; + stats->rx_bytes += rx_save->bytes; + stats->tx_bytes += tx_save->bytes; + stats->rx_missed_errors += rx_save->hw_drop_overruns; + stats->tx_dropped += tx_save->hw_drop_errors; + + stats->multicast += stats64_save->multicast; +} + +static void bnge_get_ring_stats64(struct bnge_dev *bd, + struct rtnl_link_stats64 *stats) +{ + struct bnge_net *bn = netdev_priv(bd->netdev); + int i; + + for (i = 0; i < bd->nq_nr_rings; i++) { + struct bnge_napi *bnapi = bn->bnapi[i]; + u64 tx_bytes, tx_packets, tx_dropped; + u64 multicast, rx_missed_errors; + struct bnge_nq_ring_info *nqr; + u64 rx_bytes, rx_packets; + unsigned int start; + u64 *sw; + + nqr = &bnapi->nq_ring; + sw = nqr->stats.sw_stats; + + do { + start = u64_stats_fetch_begin(&nqr->stats.syncp); + + rx_packets = BNGE_GET_RING_STATS64(sw, rx_ucast_pkts); + rx_packets += BNGE_GET_RING_STATS64(sw, rx_mcast_pkts); + rx_packets += BNGE_GET_RING_STATS64(sw, rx_bcast_pkts); + + tx_packets = BNGE_GET_RING_STATS64(sw, tx_ucast_pkts); + tx_packets += BNGE_GET_RING_STATS64(sw, tx_mcast_pkts); + tx_packets += BNGE_GET_RING_STATS64(sw, tx_bcast_pkts); + + rx_bytes = BNGE_GET_RING_STATS64(sw, rx_ucast_bytes); + rx_bytes += BNGE_GET_RING_STATS64(sw, rx_mcast_bytes); + rx_bytes += BNGE_GET_RING_STATS64(sw, rx_bcast_bytes); + + tx_bytes = BNGE_GET_RING_STATS64(sw, tx_ucast_bytes); + tx_bytes += BNGE_GET_RING_STATS64(sw, tx_mcast_bytes); + tx_bytes += BNGE_GET_RING_STATS64(sw, tx_bcast_bytes); + + multicast = BNGE_GET_RING_STATS64(sw, rx_mcast_pkts); + rx_missed_errors = + BNGE_GET_RING_STATS64(sw, rx_discard_pkts); + tx_dropped = + BNGE_GET_RING_STATS64(sw, tx_error_pkts); + } while (u64_stats_fetch_retry(&nqr->stats.syncp, start)); + + stats->rx_packets += rx_packets; + stats->tx_packets += tx_packets; + stats->rx_bytes += rx_bytes; + stats->tx_bytes += tx_bytes; + stats->multicast += multicast; + stats->rx_missed_errors += rx_missed_errors; + stats->tx_dropped += tx_dropped; + } +} + +static void bnge_get_port_stats64(struct bnge_net *bn, + struct rtnl_link_stats64 *stats) +{ + unsigned int start; + u64 *tx, *rx; + + rx = bn->port_stats.sw_stats; + tx = bn->port_stats.sw_stats + BNGE_TX_PORT_STATS_BYTE_OFFSET / 8; + + do { + start = u64_stats_fetch_begin(&bn->port_stats.syncp); + + stats->rx_crc_errors = + BNGE_GET_RX_PORT_STATS64(rx, rx_fcs_err_frames); + stats->rx_frame_errors = + BNGE_GET_RX_PORT_STATS64(rx, rx_align_err_frames); + stats->rx_length_errors = + BNGE_GET_RX_PORT_STATS64(rx, rx_undrsz_frames) + + BNGE_GET_RX_PORT_STATS64(rx, rx_ovrsz_frames) + + BNGE_GET_RX_PORT_STATS64(rx, rx_runt_frames); + stats->rx_errors = + BNGE_GET_RX_PORT_STATS64(rx, rx_false_carrier_frames) + + BNGE_GET_RX_PORT_STATS64(rx, rx_jbr_frames); + stats->collisions = + BNGE_GET_TX_PORT_STATS64(tx, tx_total_collisions); + stats->tx_fifo_errors = + BNGE_GET_TX_PORT_STATS64(tx, tx_fifo_underruns); + stats->tx_errors = BNGE_GET_TX_PORT_STATS64(tx, tx_err); + } while (u64_stats_fetch_retry(&bn->port_stats.syncp, start)); +} + +static void bnge_fill_prev_stats64(struct bnge_net *bn, + struct rtnl_link_stats64 *stats) +{ + struct netdev_queue_stats_rx *rx_save = &bn->rxq_prv_stats; + struct netdev_queue_stats_tx *tx_save = &bn->txq_prv_stats; + struct rtnl_link_stats64 *stats64_save = &bn->prv_stats64; + + stats->rx_packets = rx_save->packets; + stats->tx_packets = tx_save->packets; + stats->rx_bytes = rx_save->bytes; + stats->tx_bytes = tx_save->bytes; + stats->rx_missed_errors = rx_save->hw_drop_overruns; + stats->tx_dropped = tx_save->hw_drop_errors; + stats->multicast = stats64_save->multicast; +} + +static void bnge_get_stats64(struct net_device *dev, + struct rtnl_link_stats64 *stats) +{ + struct bnge_net *bn = netdev_priv(dev); + + if (bn->flags & BNGE_FLAG_PORT_STATS) + bnge_get_port_stats64(bn, stats); + + spin_lock_bh(&bn->stats_lock); + if (!test_bit(BNGE_STATE_STATS_ENABLE, &bn->state)) { + bnge_fill_prev_stats64(bn, stats); + spin_unlock_bh(&bn->stats_lock); + return; + } + + bnge_get_ring_stats64(bn->bd, stats); + bnge_add_prev_ring_stats64(bn, stats); + spin_unlock_bh(&bn->stats_lock); +} + +static void bnge_save_ring_stats(struct bnge_net *bn) +{ + struct netdev_queue_stats_rx *rx_save = &bn->rxq_prv_stats; + struct netdev_queue_stats_tx *tx_save = &bn->txq_prv_stats; + struct rtnl_link_stats64 *stats64_save = &bn->prv_stats64; + int i; + + for (i = 0; i < bn->bd->nq_nr_rings; i++) { + struct bnge_napi *bnapi = bn->bnapi[i]; + struct bnge_nq_ring_info *nqr; + u64 *sw; + + nqr = &bnapi->nq_ring; + sw = nqr->stats.sw_stats; + + rx_save->packets += BNGE_GET_RING_STATS64(sw, rx_ucast_pkts); + rx_save->packets += BNGE_GET_RING_STATS64(sw, rx_mcast_pkts); + rx_save->packets += BNGE_GET_RING_STATS64(sw, rx_bcast_pkts); + rx_save->bytes += BNGE_GET_RING_STATS64(sw, rx_ucast_bytes); + rx_save->bytes += BNGE_GET_RING_STATS64(sw, rx_mcast_bytes); + rx_save->bytes += BNGE_GET_RING_STATS64(sw, rx_bcast_bytes); + rx_save->hw_drop_overruns += BNGE_GET_RING_STATS64(sw, rx_discard_pkts); + rx_save->hw_drops += BNGE_GET_RING_STATS64(sw, rx_error_pkts) + + BNGE_GET_RING_STATS64(sw, rx_discard_pkts); + + tx_save->packets += BNGE_GET_RING_STATS64(sw, tx_ucast_pkts); + tx_save->packets += BNGE_GET_RING_STATS64(sw, tx_mcast_pkts); + tx_save->packets += BNGE_GET_RING_STATS64(sw, tx_bcast_pkts); + tx_save->bytes += BNGE_GET_RING_STATS64(sw, tx_ucast_bytes); + tx_save->bytes += BNGE_GET_RING_STATS64(sw, tx_mcast_bytes); + tx_save->bytes += BNGE_GET_RING_STATS64(sw, tx_bcast_bytes); + tx_save->hw_drop_errors += BNGE_GET_RING_STATS64(sw, tx_error_pkts); + tx_save->hw_drops += BNGE_GET_RING_STATS64(sw, tx_discard_pkts) + + BNGE_GET_RING_STATS64(sw, tx_error_pkts); + + stats64_save->multicast += + BNGE_GET_RING_STATS64(sw, rx_mcast_pkts); + } +} + static void bnge_close_core(struct bnge_net *bn) { struct bnge_dev *bd = bn->bd; @@ -2542,8 +3084,17 @@ static void bnge_close_core(struct bnge_net *bn) bnge_tx_disable(bn); clear_bit(BNGE_STATE_OPEN, &bd->state); + + timer_delete_sync(&bn->timer); bnge_shutdown_nic(bn); bnge_disable_napi(bn); + + /* Save ring stats before shutdown */ + spin_lock_bh(&bn->stats_lock); + bnge_save_ring_stats(bn); + clear_bit(BNGE_STATE_STATS_ENABLE, &bn->state); + spin_unlock_bh(&bn->stats_lock); + bnge_free_all_rings_bufs(bn); bnge_free_irq(bn); bnge_del_napi(bn); @@ -2556,14 +3107,96 @@ static int bnge_close(struct net_device *dev) struct bnge_net *bn = netdev_priv(dev); bnge_close_core(bn); + bnge_hwrm_shutdown_link(bn->bd); + bn->sp_event = 0; return 0; } +static void bnge_get_queue_stats_rx(struct net_device *dev, int i, + struct netdev_queue_stats_rx *stats) +{ + struct bnge_net *bn = netdev_priv(dev); + struct bnge_nq_ring_info *nqr; + u64 *sw; + + if (!bn->bnapi) + return; + + nqr = &bn->bnapi[i]->nq_ring; + sw = nqr->stats.sw_stats; + + stats->packets = 0; + stats->packets += BNGE_GET_RING_STATS64(sw, rx_ucast_pkts); + stats->packets += BNGE_GET_RING_STATS64(sw, rx_mcast_pkts); + stats->packets += BNGE_GET_RING_STATS64(sw, rx_bcast_pkts); + + stats->bytes = 0; + stats->bytes += BNGE_GET_RING_STATS64(sw, rx_ucast_bytes); + stats->bytes += BNGE_GET_RING_STATS64(sw, rx_mcast_bytes); + stats->bytes += BNGE_GET_RING_STATS64(sw, rx_bcast_bytes); + + stats->hw_drop_overruns = BNGE_GET_RING_STATS64(sw, rx_discard_pkts); + stats->hw_drops = BNGE_GET_RING_STATS64(sw, rx_error_pkts) + + stats->hw_drop_overruns; +} + +static void bnge_get_queue_stats_tx(struct net_device *dev, int i, + struct netdev_queue_stats_tx *stats) +{ + struct bnge_net *bn = netdev_priv(dev); + struct bnge_napi *bnapi; + u64 *sw; + + if (!bn->tx_ring) + return; + + bnapi = bn->tx_ring[bn->tx_ring_map[i]].bnapi; + sw = bnapi->nq_ring.stats.sw_stats; + + stats->packets = 0; + stats->packets += BNGE_GET_RING_STATS64(sw, tx_ucast_pkts); + stats->packets += BNGE_GET_RING_STATS64(sw, tx_mcast_pkts); + stats->packets += BNGE_GET_RING_STATS64(sw, tx_bcast_pkts); + + stats->bytes = 0; + stats->bytes += BNGE_GET_RING_STATS64(sw, tx_ucast_bytes); + stats->bytes += BNGE_GET_RING_STATS64(sw, tx_mcast_bytes); + stats->bytes += BNGE_GET_RING_STATS64(sw, tx_bcast_bytes); + + stats->hw_drop_errors = BNGE_GET_RING_STATS64(sw, tx_error_pkts); + stats->hw_drops = BNGE_GET_RING_STATS64(sw, tx_discard_pkts) + + stats->hw_drop_errors; +} + +static void bnge_get_base_stats(struct net_device *dev, + struct netdev_queue_stats_rx *rx, + struct netdev_queue_stats_tx *tx) +{ + struct bnge_net *bn = netdev_priv(dev); + + rx->packets = bn->rxq_prv_stats.packets; + rx->bytes = bn->rxq_prv_stats.bytes; + rx->hw_drops = bn->rxq_prv_stats.hw_drops; + rx->hw_drop_overruns = bn->rxq_prv_stats.hw_drop_overruns; + + tx->packets = bn->txq_prv_stats.packets; + tx->bytes = bn->txq_prv_stats.bytes; + tx->hw_drops = bn->txq_prv_stats.hw_drops; + tx->hw_drop_errors = bn->txq_prv_stats.hw_drop_errors; +} + +static const struct netdev_stat_ops bnge_stat_ops = { + .get_queue_stats_rx = bnge_get_queue_stats_rx, + .get_queue_stats_tx = bnge_get_queue_stats_tx, + .get_base_stats = bnge_get_base_stats, +}; + static const struct net_device_ops bnge_netdev_ops = { .ndo_open = bnge_open, .ndo_stop = bnge_close, .ndo_start_xmit = bnge_start_xmit, + .ndo_get_stats64 = bnge_get_stats64, .ndo_features_check = bnge_features_check, }; @@ -2705,6 +3338,7 @@ int bnge_netdev_alloc(struct bnge_dev *bd, int max_irqs) bd->netdev = netdev; netdev->netdev_ops = &bnge_netdev_ops; + netdev->stat_ops = &bnge_stat_ops; bnge_set_ethtool_ops(netdev); @@ -2774,6 +3408,18 @@ int bnge_netdev_alloc(struct bnge_dev *bd, int max_irqs) if (bd->tso_max_segs) netif_set_tso_max_segs(netdev, bd->tso_max_segs); + INIT_WORK(&bn->sp_task, bnge_sp_task); + timer_setup(&bn->timer, bnge_timer, 0); + bn->current_interval = BNGE_TIMER_INTERVAL; + + bn->bnge_pf_wq = alloc_ordered_workqueue("bnge_pf_wq-%s", 0, + dev_name(bd->dev)); + if (!bn->bnge_pf_wq) { + netdev_err(netdev, "Unable to create workqueue.\n"); + rc = -ENOMEM; + goto err_netdev; + } + bn->rx_ring_size = BNGE_DEFAULT_RX_RING_SIZE; bn->tx_ring_size = BNGE_DEFAULT_TX_RING_SIZE; bn->rx_dir = DMA_FROM_DEVICE; @@ -2785,15 +3431,28 @@ int bnge_netdev_alloc(struct bnge_dev *bd, int max_irqs) bnge_init_l2_fltr_tbl(bn); bnge_init_mac_addr(bd); + rc = bnge_probe_phy(bn, true); + if (rc) + goto err_free_workq; + + rc = bnge_alloc_port_stats(bn); + if (rc) + goto err_free_workq; + spin_lock_init(&bn->stats_lock); + netdev->request_ops_lock = true; rc = register_netdev(netdev); if (rc) { dev_err(bd->dev, "Register netdev failed rc: %d\n", rc); - goto err_netdev; + goto err_free_port_stats; } return 0; +err_free_port_stats: + bnge_free_port_stats(bn); +err_free_workq: + destroy_workqueue(bn->bnge_pf_wq); err_netdev: free_netdev(netdev); return rc; @@ -2802,8 +3461,19 @@ err_netdev: void bnge_netdev_free(struct bnge_dev *bd) { struct net_device *netdev = bd->netdev; + struct bnge_net *bn; + + bn = netdev_priv(netdev); unregister_netdev(netdev); + + timer_shutdown_sync(&bn->timer); + cancel_work_sync(&bn->sp_task); + bn->sp_event = 0; + destroy_workqueue(bn->bnge_pf_wq); + + bnge_free_port_stats(bn); + free_netdev(netdev); bd->netdev = NULL; } diff --git a/drivers/net/ethernet/broadcom/bnge/bnge_netdev.h b/drivers/net/ethernet/broadcom/bnge/bnge_netdev.h index 70f1a7c24814..f4636b5b0cf3 100644 --- a/drivers/net/ethernet/broadcom/bnge/bnge_netdev.h +++ b/drivers/net/ethernet/broadcom/bnge/bnge_netdev.h @@ -7,8 +7,11 @@ #include <linux/bnge/hsi.h> #include <linux/io-64-nonatomic-lo-hi.h> #include <linux/refcount.h> +#include <linux/u64_stats_sync.h> +#include <net/netdev_queues.h> #include "bnge_db.h" #include "bnge_hw_def.h" +#include "bnge_link.h" struct tx_bd { __le32 tx_bd_len_flags_type; @@ -223,6 +226,62 @@ struct bnge_tpa_info { #define BNGE_NQ_HDL_IDX(hdl) ((hdl) & BNGE_NQ_HDL_IDX_MASK) #define BNGE_NQ_HDL_TYPE(hdl) (((hdl) & BNGE_NQ_HDL_TYPE_MASK) >> \ BNGE_NQ_HDL_TYPE_SHIFT) +#define BNGE_GET_RING_STATS64(sw, counter) \ + (*((sw) + offsetof(struct ctx_hw_stats, counter) / 8)) + +#define BNGE_GET_RX_PORT_STATS64(sw, counter) \ + (*((sw) + offsetof(struct rx_port_stats, counter) / 8)) + +#define BNGE_GET_TX_PORT_STATS64(sw, counter) \ + (*((sw) + offsetof(struct tx_port_stats, counter) / 8)) + +#define BNGE_PORT_STATS_SIZE \ + (sizeof(struct rx_port_stats) + sizeof(struct tx_port_stats) + 1024) + +#define BNGE_TX_PORT_STATS_BYTE_OFFSET \ + (sizeof(struct rx_port_stats) + 512) + +#define BNGE_RX_STATS_OFFSET(counter) \ + (offsetof(struct rx_port_stats, counter) / 8) + +#define BNGE_TX_STATS_OFFSET(counter) \ + ((offsetof(struct tx_port_stats, counter) + \ + BNGE_TX_PORT_STATS_BYTE_OFFSET) / 8) + +#define BNGE_RX_STATS_EXT_OFFSET(counter) \ + (offsetof(struct rx_port_stats_ext, counter) / 8) + +#define BNGE_TX_STATS_EXT_OFFSET(counter) \ + (offsetof(struct tx_port_stats_ext, counter) / 8) + +struct bnge_stats_mem { + u64 *sw_stats; + u64 *hw_masks; + void *hw_stats; + dma_addr_t hw_stats_map; + u32 len; + struct u64_stats_sync syncp; +}; + +enum bnge_net_state { + BNGE_STATE_NAPI_DISABLED, + BNGE_STATE_STATS_ENABLE, +}; + +#define BNGE_TIMER_INTERVAL HZ + +enum bnge_net_flag { + BNGE_FLAG_PORT_STATS = BIT(0), + BNGE_FLAG_PORT_STATS_EXT = BIT(1), +}; + +enum bnge_sp_event { + BNGE_LINK_CHNG_SP_EVENT, + BNGE_LINK_SPEED_CHNG_SP_EVENT, + BNGE_LINK_CFG_CHANGE_SP_EVENT, + BNGE_UPDATE_PHY_SP_EVENT, + BNGE_PERIODIC_STATS_SP_EVENT, +}; struct bnge_net { struct bnge_dev *bd; @@ -281,13 +340,36 @@ struct bnge_net { u32 stats_coal_ticks; unsigned long state; -#define BNGE_STATE_NAPI_DISABLED 0 u32 msg_enable; u16 max_tpa; __be16 vxlan_port; __be16 nge_port; __be16 vxlan_gpe_port; + + unsigned int current_interval; + struct timer_list timer; + struct workqueue_struct *bnge_pf_wq; + struct work_struct sp_task; + unsigned long sp_event; + + struct bnge_ethtool_link_info eth_link_info; + + u64 flags; + + struct bnge_stats_mem port_stats; + struct bnge_stats_mem rx_port_stats_ext; + struct bnge_stats_mem tx_port_stats_ext; + u16 fw_rx_stats_ext_size; + u16 fw_tx_stats_ext_size; + + struct netdev_queue_stats_rx rxq_prv_stats; + struct netdev_queue_stats_tx txq_prv_stats; + struct rtnl_link_stats64 prv_stats64; + spinlock_t stats_lock; + + u8 pri2cos_idx[8]; + bool pri2cos_valid; }; #define BNGE_DEFAULT_RX_RING_SIZE 511 @@ -353,14 +435,6 @@ void bnge_set_ring_params(struct bnge_dev *bd); bnge_writeq(bd, (db)->db_key64 | DBR_TYPE_NQ_ARM | \ DB_RING_IDX(db, idx), (db)->doorbell) -struct bnge_stats_mem { - u64 *sw_stats; - u64 *hw_masks; - void *hw_stats; - dma_addr_t hw_stats_map; - int len; -}; - struct nqe_cn { __le16 type; #define NQ_CN_TYPE_MASK 0x3fUL @@ -566,4 +640,6 @@ u8 *__bnge_alloc_rx_frag(struct bnge_net *bn, dma_addr_t *mapping, struct bnge_rx_ring_info *rxr, gfp_t gfp); int bnge_alloc_rx_netmem(struct bnge_net *bn, struct bnge_rx_ring_info *rxr, u16 prod, gfp_t gfp); +void __bnge_queue_sp_work(struct bnge_net *bn); +void bnge_copy_hw_masks(u64 *mask_arr, __le64 *hw_mask_arr, int count); #endif /* _BNGE_NETDEV_H_ */ diff --git a/drivers/net/ethernet/broadcom/bnge/bnge_txrx.c b/drivers/net/ethernet/broadcom/bnge/bnge_txrx.c index a2616f037557..7d45e057f2e8 100644 --- a/drivers/net/ethernet/broadcom/bnge/bnge_txrx.c +++ b/drivers/net/ethernet/broadcom/bnge/bnge_txrx.c @@ -1128,6 +1128,29 @@ static void __bnge_poll_work_done(struct bnge_net *bn, struct bnge_napi *bnapi, } } +static void bnge_async_event_process(struct bnge_net *bn, + struct hwrm_async_event_cmpl *cmpl) +{ + u16 event_id = le16_to_cpu(cmpl->event_id); + u32 data1 = le32_to_cpu(cmpl->event_data1); + u32 data2 = le32_to_cpu(cmpl->event_data2); + + netdev_dbg(bn->netdev, "hwrm event 0x%x {0x%x, 0x%x}\n", + event_id, data1, data2); + + switch (event_id) { + case ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE: + case ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CHANGE: + case ASYNC_EVENT_CMPL_EVENT_ID_PORT_PHY_CFG_CHANGE: + case ASYNC_EVENT_CMPL_EVENT_ID_LINK_STATUS_CHANGE: + bnge_link_async_event_process(bn, event_id); + break; + default: + return; + } + __bnge_queue_sp_work(bn); +} + static void bnge_hwrm_update_token(struct bnge_dev *bd, u16 seq_id, enum bnge_hwrm_wait_state state) @@ -1146,7 +1169,7 @@ bnge_hwrm_update_token(struct bnge_dev *bd, u16 seq_id, dev_err(bd->dev, "Invalid hwrm seq id %d\n", seq_id); } -static int bnge_hwrm_handler(struct bnge_dev *bd, struct tx_cmp *txcmp) +static int bnge_hwrm_handler(struct bnge_net *bn, struct tx_cmp *txcmp) { struct hwrm_cmpl *h_cmpl = (struct hwrm_cmpl *)txcmp; u16 cmpl_type = TX_CMP_TYPE(txcmp), seq_id; @@ -1154,10 +1177,14 @@ static int bnge_hwrm_handler(struct bnge_dev *bd, struct tx_cmp *txcmp) switch (cmpl_type) { case CMPL_BASE_TYPE_HWRM_DONE: seq_id = le16_to_cpu(h_cmpl->sequence_id); - bnge_hwrm_update_token(bd, seq_id, BNGE_HWRM_COMPLETE); + bnge_hwrm_update_token(bn->bd, seq_id, BNGE_HWRM_COMPLETE); break; case CMPL_BASE_TYPE_HWRM_ASYNC_EVENT: + bnge_async_event_process(bn, + (struct hwrm_async_event_cmpl *)txcmp); + break; + default: break; } @@ -1235,7 +1262,7 @@ static int __bnge_poll_work(struct bnge_net *bn, struct bnge_cp_ring_info *cpr, } else if (unlikely(cmp_type == CMPL_BASE_TYPE_HWRM_DONE || cmp_type == CMPL_BASE_TYPE_HWRM_FWD_REQ || cmp_type == CMPL_BA_TY_HWRM_ASY_EVT)) { - bnge_hwrm_handler(bn->bd, txcmp); + bnge_hwrm_handler(bn, txcmp); } raw_cons = NEXT_RAW_CMP(raw_cons); @@ -1355,7 +1382,7 @@ int bnge_napi_poll(struct napi_struct *napi, int budget) budget - work_done); nqr->has_more_work |= cpr->has_more_work; } else { - bnge_hwrm_handler(bn->bd, (struct tx_cmp *)nqcmp); + bnge_hwrm_handler(bn, (struct tx_cmp *)nqcmp); } raw_cons = NEXT_RAW_CMP(raw_cons); } diff --git a/drivers/net/ethernet/broadcom/bnxt/Makefile b/drivers/net/ethernet/broadcom/bnxt/Makefile index ba6c239d52fa..debef78c8b6d 100644 --- a/drivers/net/ethernet/broadcom/bnxt/Makefile +++ b/drivers/net/ethernet/broadcom/bnxt/Makefile @@ -1,7 +1,7 @@ # SPDX-License-Identifier: GPL-2.0-only obj-$(CONFIG_BNXT) += bnxt_en.o -bnxt_en-y := bnxt.o bnxt_hwrm.o bnxt_sriov.o bnxt_ethtool.o bnxt_dcb.o bnxt_ulp.o bnxt_xdp.o bnxt_ptp.o bnxt_vfr.o bnxt_devlink.o bnxt_dim.o bnxt_coredump.o +bnxt_en-y := bnxt.o bnxt_hwrm.o bnxt_sriov.o bnxt_ethtool.o bnxt_dcb.o bnxt_ulp.o bnxt_xdp.o bnxt_ptp.o bnxt_vfr.o bnxt_devlink.o bnxt_dim.o bnxt_coredump.o bnxt_gso.o bnxt_en-$(CONFIG_BNXT_FLOWER_OFFLOAD) += bnxt_tc.o bnxt_en-$(CONFIG_DEBUG_FS) += bnxt_debugfs.o bnxt_en-$(CONFIG_BNXT_HWMON) += bnxt_hwmon.o diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.c b/drivers/net/ethernet/broadcom/bnxt/bnxt.c index 3f775196ef81..2715632115a5 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.c @@ -74,6 +74,8 @@ #include "bnxt_debugfs.h" #include "bnxt_coredump.h" #include "bnxt_hwmon.h" +#include "bnxt_gso.h" +#include <net/tso.h> #define BNXT_TX_TIMEOUT (5 * HZ) #define BNXT_DEF_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_HW | \ @@ -447,7 +449,7 @@ const u16 bnxt_lhint_arr[] = { TX_BD_FLAGS_LHINT_2048_AND_LARGER, }; -static u16 bnxt_xmit_get_cfa_action(struct sk_buff *skb) +u16 bnxt_xmit_get_cfa_action(struct sk_buff *skb) { struct metadata_dst *md_dst = skb_metadata_dst(skb); @@ -506,6 +508,11 @@ static netdev_tx_t bnxt_start_xmit(struct sk_buff *skb, struct net_device *dev) } } #endif + if (skb_is_gso(skb) && + (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) && + !(bp->flags & BNXT_FLAG_UDP_GSO_CAP)) + return bnxt_sw_udp_gso_xmit(bp, txr, txq, skb); + free_size = bnxt_tx_avail(bp, txr); if (unlikely(free_size < skb_shinfo(skb)->nr_frags + 2)) { /* We must have raced with NAPI cleanup */ @@ -656,6 +663,7 @@ normal_tx: goto tx_free; dma_unmap_addr_set(tx_buf, mapping, mapping); + dma_unmap_len_set(tx_buf, len, len); flags = (len << TX_BD_LEN_SHIFT) | TX_BD_TYPE_LONG_TX_BD | TX_BD_CNT(last_frag + 2); @@ -663,10 +671,9 @@ normal_tx: txbd->tx_bd_opaque = SET_TX_OPAQUE(bp, txr, prod, 2 + last_frag); prod = NEXT_TX(prod); - txbd1 = (struct tx_bd_ext *) - &txr->tx_desc_ring[TX_RING(bp, prod)][TX_IDX(prod)]; + txbd1 = bnxt_init_ext_bd(bp, txr, prod, lflags, vlan_tag_flags, + cfa_action); - txbd1->tx_bd_hsize_lflags = lflags; if (skb_is_gso(skb)) { bool udp_gso = !!(skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4); u32 hdr_len; @@ -693,7 +700,6 @@ normal_tx: } else if (skb->ip_summed == CHECKSUM_PARTIAL) { txbd1->tx_bd_hsize_lflags |= cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM); - txbd1->tx_bd_mss = 0; } length >>= 9; @@ -706,9 +712,6 @@ normal_tx: flags |= bnxt_lhint_arr[length]; txbd->tx_bd_len_flags_type = cpu_to_le32(flags); - txbd1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags); - txbd1->tx_bd_cfa_action = - cpu_to_le32(cfa_action << TX_BD_CFA_ACTION_SHIFT); txbd0 = txbd; for (i = 0; i < last_frag; i++) { frag = &skb_shinfo(skb)->frags[i]; @@ -725,6 +728,7 @@ normal_tx: tx_buf = &txr->tx_buf_ring[RING_TX(bp, prod)]; netmem_dma_unmap_addr_set(skb_frag_netmem(frag), tx_buf, mapping, mapping); + dma_unmap_len_set(tx_buf, len, len); txbd->tx_bd_haddr = cpu_to_le64(mapping); @@ -814,17 +818,19 @@ static bool __bnxt_tx_int(struct bnxt *bp, struct bnxt_tx_ring_info *txr, u16 hw_cons = txr->tx_hw_cons; unsigned int tx_bytes = 0; u16 cons = txr->tx_cons; - skb_frag_t *frag; + unsigned int dma_len; + dma_addr_t dma_addr; int tx_pkts = 0; bool rc = false; while (RING_TX(bp, cons) != hw_cons) { - struct bnxt_sw_tx_bd *tx_buf; + struct bnxt_sw_tx_bd *tx_buf, *head_buf; struct sk_buff *skb; bool is_ts_pkt; int j, last; tx_buf = &txr->tx_buf_ring[RING_TX(bp, cons)]; + head_buf = tx_buf; skb = tx_buf->skb; if (unlikely(!skb)) { @@ -849,20 +855,44 @@ static bool __bnxt_tx_int(struct bnxt *bp, struct bnxt_tx_ring_info *txr, goto next_tx_int; } - dma_unmap_single(&pdev->dev, dma_unmap_addr(tx_buf, mapping), - skb_headlen(skb), DMA_TO_DEVICE); + if (dma_unmap_len(tx_buf, len)) { + dma_addr = dma_unmap_addr(tx_buf, mapping); + dma_len = dma_unmap_len(tx_buf, len); + + dma_unmap_single(&pdev->dev, dma_addr, dma_len, + DMA_TO_DEVICE); + } + last = tx_buf->nr_frags; for (j = 0; j < last; j++) { - frag = &skb_shinfo(skb)->frags[j]; cons = NEXT_TX(cons); tx_buf = &txr->tx_buf_ring[RING_TX(bp, cons)]; - netmem_dma_unmap_page_attrs(&pdev->dev, - dma_unmap_addr(tx_buf, - mapping), - skb_frag_size(frag), - DMA_TO_DEVICE, 0); + if (dma_unmap_len(tx_buf, len)) { + dma_addr = dma_unmap_addr(tx_buf, mapping); + dma_len = dma_unmap_len(tx_buf, len); + + netmem_dma_unmap_page_attrs(&pdev->dev, + dma_addr, dma_len, + DMA_TO_DEVICE, 0); + } + } + + if (unlikely(head_buf->is_sw_gso)) { + u16 inline_cons = txr->tx_inline_cons + 1; + + WRITE_ONCE(txr->tx_inline_cons, inline_cons); + if (head_buf->is_sw_gso == BNXT_SW_GSO_LAST) { + tso_dma_map_complete(&pdev->dev, + &head_buf->sw_gso_cstate); + } else { + tx_pkts--; + tx_bytes -= skb->len; + skb = NULL; + } + head_buf->is_sw_gso = 0; } + if (unlikely(is_ts_pkt)) { if (BNXT_CHIP_P5(bp)) { /* PTP worker takes ownership of the skb */ @@ -2077,23 +2107,6 @@ vlan_err: return NULL; } -static enum pkt_hash_types bnxt_rss_ext_op(struct bnxt *bp, - struct rx_cmp *rxcmp) -{ - u8 ext_op; - - ext_op = RX_CMP_V3_HASH_TYPE(bp, rxcmp); - switch (ext_op) { - case EXT_OP_INNER_4: - case EXT_OP_OUTER_4: - case EXT_OP_INNFL_3: - case EXT_OP_OUTFL_3: - return PKT_HASH_TYPE_L4; - default: - return PKT_HASH_TYPE_L3; - } -} - /* returns the following: * 1 - 1 packet successfully received * 0 - successful TPA_START, packet not completed yet @@ -2112,13 +2125,13 @@ static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, u32 tmp_raw_cons = *raw_cons; u16 cons, prod, cp_cons = RING_CMP(tmp_raw_cons); struct skb_shared_info *sinfo; + struct bnxt_xdp_buff bnxt_xdp; struct bnxt_sw_rx_bd *rx_buf; unsigned int len; u8 *data_ptr, agg_bufs, cmp_type; bool xdp_active = false; dma_addr_t dma_addr; struct sk_buff *skb; - struct xdp_buff xdp; u32 flags, misc; u32 cmpl_ts; void *data; @@ -2231,9 +2244,14 @@ static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, dma_addr = rx_buf->mapping; if (bnxt_xdp_attached(bp, rxr)) { - bnxt_xdp_buff_init(bp, rxr, cons, data_ptr, len, &xdp); + bnxt_xdp.rxcmp = rxcmp; + bnxt_xdp.rxcmp1 = rxcmp1; + bnxt_xdp.cmp_type = cmp_type; + + bnxt_xdp_buff_init(bp, rxr, cons, data_ptr, len, &bnxt_xdp.xdp); if (agg_bufs) { - u32 frag_len = bnxt_rx_agg_netmems_xdp(bp, cpr, &xdp, + u32 frag_len = bnxt_rx_agg_netmems_xdp(bp, cpr, + &bnxt_xdp.xdp, cp_cons, agg_bufs, false); @@ -2245,12 +2263,13 @@ static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, } if (xdp_active) { - if (bnxt_rx_xdp(bp, rxr, cons, &xdp, data, &data_ptr, &len, event)) { + if (bnxt_rx_xdp(bp, rxr, cons, &bnxt_xdp.xdp, data, &data_ptr, + &len, event)) { rc = 1; goto next_rx; } - if (xdp_buff_has_frags(&xdp)) { - sinfo = xdp_get_shared_info_from_buff(&xdp); + if (xdp_buff_has_frags(&bnxt_xdp.xdp)) { + sinfo = xdp_get_shared_info_from_buff(&bnxt_xdp.xdp); agg_bufs = sinfo->nr_frags; } else { agg_bufs = 0; @@ -2261,7 +2280,8 @@ static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, if (!xdp_active) skb = bnxt_copy_skb(bnapi, data_ptr, len, dma_addr); else - skb = bnxt_copy_xdp(bnapi, &xdp, len, dma_addr); + skb = bnxt_copy_xdp(bnapi, &bnxt_xdp.xdp, len, + dma_addr); bnxt_reuse_rx_data(rxr, cons, data); if (!skb) { if (agg_bufs) { @@ -2269,7 +2289,8 @@ static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, bnxt_reuse_rx_agg_bufs(cpr, cp_cons, 0, agg_bufs, false); else - bnxt_xdp_buff_frags_free(rxr, &xdp); + bnxt_xdp_buff_frags_free(rxr, + &bnxt_xdp.xdp); } goto oom_next_rx; } @@ -2293,10 +2314,11 @@ static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, if (!skb) goto oom_next_rx; } else { - skb = bnxt_xdp_build_skb(bp, skb, agg_bufs, rxr, &xdp); + skb = bnxt_xdp_build_skb(bp, skb, agg_bufs, + rxr, &bnxt_xdp.xdp); if (!skb) { /* we should be able to free the old skb here */ - bnxt_xdp_buff_frags_free(rxr, &xdp); + bnxt_xdp_buff_frags_free(rxr, &bnxt_xdp.xdp); goto oom_next_rx; } } @@ -3407,19 +3429,23 @@ static void bnxt_free_one_tx_ring_skbs(struct bnxt *bp, { int i, max_idx; struct pci_dev *pdev = bp->pdev; + unsigned int dma_len; + dma_addr_t dma_addr; max_idx = bp->tx_nr_pages * TX_DESC_CNT; for (i = 0; i < max_idx;) { struct bnxt_sw_tx_bd *tx_buf = &txr->tx_buf_ring[i]; + struct bnxt_sw_tx_bd *head_buf = tx_buf; struct sk_buff *skb; int j, last; if (idx < bp->tx_nr_rings_xdp && tx_buf->action == XDP_REDIRECT) { - dma_unmap_single(&pdev->dev, - dma_unmap_addr(tx_buf, mapping), - dma_unmap_len(tx_buf, len), + dma_addr = dma_unmap_addr(tx_buf, mapping); + dma_len = dma_unmap_len(tx_buf, len); + + dma_unmap_single(&pdev->dev, dma_addr, dma_len, DMA_TO_DEVICE); xdp_return_frame(tx_buf->xdpf); tx_buf->action = 0; @@ -3442,25 +3468,43 @@ static void bnxt_free_one_tx_ring_skbs(struct bnxt *bp, continue; } - dma_unmap_single(&pdev->dev, - dma_unmap_addr(tx_buf, mapping), - skb_headlen(skb), - DMA_TO_DEVICE); + if (dma_unmap_len(tx_buf, len)) { + dma_addr = dma_unmap_addr(tx_buf, mapping); + dma_len = dma_unmap_len(tx_buf, len); + + dma_unmap_single(&pdev->dev, dma_addr, dma_len, + DMA_TO_DEVICE); + } last = tx_buf->nr_frags; i += 2; for (j = 0; j < last; j++, i++) { int ring_idx = i & bp->tx_ring_mask; - skb_frag_t *frag = &skb_shinfo(skb)->frags[j]; tx_buf = &txr->tx_buf_ring[ring_idx]; - netmem_dma_unmap_page_attrs(&pdev->dev, - dma_unmap_addr(tx_buf, - mapping), - skb_frag_size(frag), - DMA_TO_DEVICE, 0); + if (dma_unmap_len(tx_buf, len)) { + dma_addr = dma_unmap_addr(tx_buf, mapping); + dma_len = dma_unmap_len(tx_buf, len); + + netmem_dma_unmap_page_attrs(&pdev->dev, + dma_addr, dma_len, + DMA_TO_DEVICE, 0); + } } - dev_kfree_skb(skb); + if (head_buf->is_sw_gso) { + u16 inline_cons = txr->tx_inline_cons + 1; + + WRITE_ONCE(txr->tx_inline_cons, inline_cons); + if (head_buf->is_sw_gso == BNXT_SW_GSO_LAST) { + tso_dma_map_complete(&pdev->dev, + &head_buf->sw_gso_cstate); + } else { + skb = NULL; + } + head_buf->is_sw_gso = 0; + } + if (skb) + dev_kfree_skb(skb); } netdev_tx_reset_queue(netdev_get_tx_queue(bp->dev, idx)); } @@ -3973,6 +4017,39 @@ static int bnxt_alloc_rx_rings(struct bnxt *bp) return rc; } +static void bnxt_free_tx_inline_buf(struct bnxt_tx_ring_info *txr, + struct pci_dev *pdev) +{ + if (!txr->tx_inline_buf) + return; + + dma_unmap_single(&pdev->dev, txr->tx_inline_dma, + txr->tx_inline_size, DMA_TO_DEVICE); + kfree(txr->tx_inline_buf); + txr->tx_inline_buf = NULL; + txr->tx_inline_size = 0; +} + +static int bnxt_alloc_tx_inline_buf(struct bnxt_tx_ring_info *txr, + struct pci_dev *pdev, + unsigned int size) +{ + txr->tx_inline_buf = kmalloc(size, GFP_KERNEL); + if (!txr->tx_inline_buf) + return -ENOMEM; + + txr->tx_inline_dma = dma_map_single(&pdev->dev, txr->tx_inline_buf, + size, DMA_TO_DEVICE); + if (dma_mapping_error(&pdev->dev, txr->tx_inline_dma)) { + kfree(txr->tx_inline_buf); + txr->tx_inline_buf = NULL; + return -ENOMEM; + } + txr->tx_inline_size = size; + + return 0; +} + static void bnxt_free_tx_rings(struct bnxt *bp) { int i; @@ -3991,6 +4068,8 @@ static void bnxt_free_tx_rings(struct bnxt *bp) txr->tx_push = NULL; } + bnxt_free_tx_inline_buf(txr, pdev); + ring = &txr->tx_ring_struct; bnxt_free_ring(bp, &ring->ring_mem); @@ -4056,6 +4135,13 @@ static int bnxt_alloc_tx_rings(struct bnxt *bp) sizeof(struct tx_push_bd); txr->data_mapping = cpu_to_le64(mapping); } + if (!(bp->flags & BNXT_FLAG_UDP_GSO_CAP)) { + rc = bnxt_alloc_tx_inline_buf(txr, pdev, + BNXT_SW_USO_MAX_SEGS * + TSO_HEADER_SIZE); + if (rc) + return rc; + } qidx = bp->tc_to_qidx[j]; ring->queue_id = bp->q_info[qidx].queue_id; spin_lock_init(&txr->xdp_tx_lock); @@ -4594,10 +4680,13 @@ static int bnxt_init_rx_rings(struct bnxt *bp) static int bnxt_init_tx_rings(struct bnxt *bp) { + netdev_features_t features; u16 i; + features = bp->dev->features; + bp->tx_wake_thresh = max_t(int, bp->tx_ring_size / 2, - BNXT_MIN_TX_DESC_CNT); + bnxt_min_tx_desc_cnt(bp, features)); for (i = 0; i < bp->tx_nr_rings; i++) { struct bnxt_tx_ring_info *txr = &bp->tx_ring[i]; @@ -8120,8 +8209,7 @@ static int __bnxt_reserve_rings(struct bnxt *bp) (bnxt_get_nr_rss_ctxs(bp, bp->rx_nr_rings) != bnxt_get_nr_rss_ctxs(bp, rx_rings) || bnxt_get_max_rss_ring(bp) >= rx_rings)) { - netdev_warn(bp->dev, "RSS table entries reverting to default\n"); - bp->dev->priv_flags &= ~IFF_RXFH_CONFIGURED; + ethtool_rxfh_indir_lost(bp->dev); } } bp->rx_nr_rings = rx_rings; @@ -13256,7 +13344,7 @@ static void __bnxt_close_nic(struct bnxt *bp, bool irq_re_init, /* Save ring stats before shutdown */ if (bp->bnapi && irq_re_init) { bnxt_get_ring_stats(bp, &bp->net_stats_prev); - bnxt_get_ring_err_stats(bp, &bp->ring_err_stats_prev); + bnxt_get_ring_drv_stats(bp, &bp->ring_drv_stats_prev); } if (irq_re_init) { bnxt_free_irq(bp); @@ -13501,8 +13589,8 @@ bnxt_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats) clear_bit(BNXT_STATE_READ_STATS, &bp->state); } -static void bnxt_get_one_ring_err_stats(struct bnxt *bp, - struct bnxt_total_ring_err_stats *stats, +static void bnxt_get_one_ring_drv_stats(struct bnxt *bp, + struct bnxt_total_ring_drv_stats *stats, struct bnxt_cp_ring_info *cpr) { struct bnxt_sw_stats *sw_stats = cpr->sw_stats; @@ -13523,13 +13611,13 @@ static void bnxt_get_one_ring_err_stats(struct bnxt *bp, stats->total_missed_irqs += sw_stats->cmn.missed_irqs; } -void bnxt_get_ring_err_stats(struct bnxt *bp, - struct bnxt_total_ring_err_stats *stats) +void bnxt_get_ring_drv_stats(struct bnxt *bp, + struct bnxt_total_ring_drv_stats *stats) { int i; for (i = 0; i < bp->cp_nr_rings; i++) - bnxt_get_one_ring_err_stats(bp, stats, &bp->bnapi[i]->cp_ring); + bnxt_get_one_ring_drv_stats(bp, stats, &bp->bnapi[i]->cp_ring); } static bool bnxt_mc_list_updated(struct bnxt *bp, u32 *rx_mask) @@ -13797,6 +13885,11 @@ static netdev_features_t bnxt_fix_features(struct net_device *dev, if ((features & NETIF_F_NTUPLE) && !bnxt_rfs_capable(bp, false)) features &= ~NETIF_F_NTUPLE; + if ((features & NETIF_F_GSO_UDP_L4) && + !(bp->flags & BNXT_FLAG_UDP_GSO_CAP) && + bp->tx_ring_size < 2 * BNXT_SW_USO_MAX_DESCS) + features &= ~NETIF_F_GSO_UDP_L4; + if ((bp->flags & BNXT_FLAG_NO_AGG_RINGS) || bp->xdp_prog) features &= ~(NETIF_F_LRO | NETIF_F_GRO_HW); @@ -13842,6 +13935,9 @@ static int bnxt_set_features(struct net_device *dev, netdev_features_t features) int rc = 0; bool re_init = false; + bp->tx_wake_thresh = max_t(int, bp->tx_ring_size / 2, + bnxt_min_tx_desc_cnt(bp, features)); + flags &= ~BNXT_FLAG_ALL_CONFIG_FEATS; if (features & NETIF_F_GRO_HW) flags |= BNXT_FLAG_GRO; @@ -15920,6 +16016,10 @@ static const struct net_device_ops bnxt_netdev_ops = { .ndo_hwtstamp_set = bnxt_hwtstamp_set, }; +static const struct xdp_metadata_ops bnxt_xdp_metadata_ops = { + .xmo_rx_hash = bnxt_xdp_rx_hash, +}; + static void bnxt_get_queue_stats_rx(struct net_device *dev, int i, struct netdev_queue_stats_rx *stats) { @@ -15980,9 +16080,9 @@ static void bnxt_get_base_stats(struct net_device *dev, rx->packets = bp->net_stats_prev.rx_packets; rx->bytes = bp->net_stats_prev.rx_bytes; - rx->alloc_fail = bp->ring_err_stats_prev.rx_total_oom_discards; - rx->hw_gro_packets = bp->ring_err_stats_prev.rx_total_hw_gro_packets; - rx->hw_gro_wire_packets = bp->ring_err_stats_prev.rx_total_hw_gro_wire_packets; + rx->alloc_fail = bp->ring_drv_stats_prev.rx_total_oom_discards; + rx->hw_gro_packets = bp->ring_drv_stats_prev.rx_total_hw_gro_packets; + rx->hw_gro_wire_packets = bp->ring_drv_stats_prev.rx_total_hw_gro_wire_packets; tx->packets = bp->net_stats_prev.tx_packets; tx->bytes = bp->net_stats_prev.tx_bytes; @@ -16815,6 +16915,7 @@ static int bnxt_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) goto init_err_free; dev->netdev_ops = &bnxt_netdev_ops; + dev->xdp_metadata_ops = &bnxt_xdp_metadata_ops; dev->stat_ops = &bnxt_stat_ops; dev->watchdog_timeo = BNXT_TX_TIMEOUT; dev->ethtool_ops = &bnxt_ethtool_ops; @@ -16862,8 +16963,7 @@ static int bnxt_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) NETIF_F_GSO_UDP_TUNNEL_CSUM | NETIF_F_GSO_GRE_CSUM | NETIF_F_GSO_PARTIAL | NETIF_F_RXHASH | NETIF_F_RXCSUM | NETIF_F_GRO; - if (bp->flags & BNXT_FLAG_UDP_GSO_CAP) - dev->hw_features |= NETIF_F_GSO_UDP_L4; + dev->hw_features |= NETIF_F_GSO_UDP_L4; if (BNXT_SUPPORTS_TPA(bp)) dev->hw_features |= NETIF_F_LRO; @@ -16896,8 +16996,15 @@ static int bnxt_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) dev->priv_flags |= IFF_UNICAST_FLT; netif_set_tso_max_size(dev, GSO_MAX_SIZE); - if (bp->tso_max_segs) + if (!(bp->flags & BNXT_FLAG_UDP_GSO_CAP)) { + u16 max_segs = BNXT_SW_USO_MAX_SEGS; + + if (bp->tso_max_segs) + max_segs = min_t(u16, max_segs, bp->tso_max_segs); + netif_set_tso_max_segs(dev, max_segs); + } else if (bp->tso_max_segs) { netif_set_tso_max_segs(dev, bp->tso_max_segs); + } dev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT | NETDEV_XDP_ACT_RX_SG; diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.h b/drivers/net/ethernet/broadcom/bnxt/bnxt.h index 4bc7f7aeaab3..fe50576ae525 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt.h +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.h @@ -11,6 +11,8 @@ #ifndef BNXT_H #define BNXT_H +#include <net/tso.h> + #define DRV_MODULE_NAME "bnxt_en" /* DO NOT CHANGE DRV_VER_* defines @@ -232,6 +234,7 @@ struct rx_cmp { #define RX_CMP_FLAGS_ITYPE_UDP (3 << 12) #define RX_CMP_FLAGS_ITYPE_FCOE (4 << 12) #define RX_CMP_FLAGS_ITYPE_ROCE (5 << 12) + #define RX_CMP_FLAGS_ITYPE_ICMP (7 << 12) #define RX_CMP_FLAGS_ITYPE_PTP_WO_TS (8 << 12) #define RX_CMP_FLAGS_ITYPE_PTP_W_TS (9 << 12) #define RX_CMP_LEN (0xffff << 16) @@ -311,6 +314,7 @@ struct rx_cmp_ext { #define RX_CMP_FLAGS2_T_IP_CS_CALC (0x1 << 2) #define RX_CMP_FLAGS2_T_L4_CS_CALC (0x1 << 3) #define RX_CMP_FLAGS2_META_FORMAT_VLAN (0x1 << 4) + #define RX_CMP_FLAGS2_IP_TYPE (0x1 << 8) __le32 rx_cmp_meta_data; #define RX_CMP_FLAGS2_METADATA_TCI_MASK 0xffff #define RX_CMP_FLAGS2_METADATA_VID_MASK 0xfff @@ -890,14 +894,19 @@ struct bnxt_sw_tx_bd { struct page *page; u8 is_ts_pkt; u8 is_push; + u8 is_sw_gso; u8 action; unsigned short nr_frags; union { u16 rx_prod; u16 txts_prod; }; + struct tso_dma_map_completion_state sw_gso_cstate; }; +#define BNXT_SW_GSO_MID 1 +#define BNXT_SW_GSO_LAST 2 + struct bnxt_sw_rx_bd { void *data; u8 *data_ptr; @@ -994,6 +1003,12 @@ struct bnxt_tx_ring_info { dma_addr_t tx_push_mapping; __le64 data_mapping; + void *tx_inline_buf; + dma_addr_t tx_inline_dma; + unsigned int tx_inline_size; + u16 tx_inline_prod; + u16 tx_inline_cons; + #define BNXT_DEV_STATE_CLOSING 0x1 u32 dev_state; @@ -1147,7 +1162,7 @@ struct bnxt_sw_stats { struct bnxt_cmn_sw_stats cmn; }; -struct bnxt_total_ring_err_stats { +struct bnxt_total_ring_drv_stats { u64 rx_total_l4_csum_errors; u64 rx_total_resets; u64 rx_total_buf_errors; @@ -2572,7 +2587,7 @@ struct bnxt { u8 pri2cos_idx[8]; u8 pri2cos_valid; - struct bnxt_total_ring_err_stats ring_err_stats_prev; + struct bnxt_total_ring_drv_stats ring_drv_stats_prev; u16 hwrm_max_req_len; u16 hwrm_max_ext_req_len; @@ -2834,6 +2849,24 @@ static inline u32 bnxt_tx_avail(struct bnxt *bp, return bp->tx_ring_size - (used & bp->tx_ring_mask); } +static inline struct tx_bd_ext * +bnxt_init_ext_bd(struct bnxt *bp, struct bnxt_tx_ring_info *txr, + u16 prod, __le32 lflags, u32 vlan_tag_flags, + u32 cfa_action) +{ + struct tx_bd_ext *txbd1; + + txbd1 = (struct tx_bd_ext *) + &txr->tx_desc_ring[TX_RING(bp, prod)][TX_IDX(prod)]; + txbd1->tx_bd_hsize_lflags = lflags; + txbd1->tx_bd_mss = 0; + txbd1->tx_bd_cfa_meta = cpu_to_le32(vlan_tag_flags); + txbd1->tx_bd_cfa_action = + cpu_to_le32(cfa_action << TX_BD_CFA_ACTION_SHIFT); + + return txbd1; +} + static inline void bnxt_writeq(struct bnxt *bp, u64 val, volatile void __iomem *addr) { @@ -2900,6 +2933,23 @@ static inline bool bnxt_sriov_cfg(struct bnxt *bp) #endif } +static inline enum pkt_hash_types bnxt_rss_ext_op(struct bnxt *bp, + const struct rx_cmp *rxcmp) +{ + u8 ext_op; + + ext_op = RX_CMP_V3_HASH_TYPE(bp, rxcmp); + switch (ext_op) { + case EXT_OP_INNER_4: + case EXT_OP_OUTER_4: + case EXT_OP_INNFL_3: + case EXT_OP_OUTFL_3: + return PKT_HASH_TYPE_L4; + default: + return PKT_HASH_TYPE_L3; + } +} + extern const u16 bnxt_bstore_to_trace[]; extern const u16 bnxt_lhint_arr[]; @@ -2950,6 +3000,7 @@ unsigned int bnxt_get_avail_cp_rings_for_en(struct bnxt *bp); int bnxt_reserve_rings(struct bnxt *bp, bool irq_re_init); void bnxt_tx_disable(struct bnxt *bp); void bnxt_tx_enable(struct bnxt *bp); +u16 bnxt_xmit_get_cfa_action(struct sk_buff *skb); void bnxt_sched_reset_txr(struct bnxt *bp, struct bnxt_tx_ring_info *txr, u16 curr); void bnxt_report_link(struct bnxt *bp); @@ -2974,8 +3025,8 @@ int bnxt_half_open_nic(struct bnxt *bp); void bnxt_half_close_nic(struct bnxt *bp); void bnxt_reenable_sriov(struct bnxt *bp); void bnxt_close_nic(struct bnxt *, bool, bool); -void bnxt_get_ring_err_stats(struct bnxt *bp, - struct bnxt_total_ring_err_stats *stats); +void bnxt_get_ring_drv_stats(struct bnxt *bp, + struct bnxt_total_ring_drv_stats *stats); bool bnxt_rfs_capable(struct bnxt *bp, bool new_rss_ctx); int bnxt_dbg_hwrm_rd_reg(struct bnxt *bp, u32 reg_off, u16 num_words, u32 *reg_buf); diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c index 0407aa1b3190..9ded88196bb4 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c @@ -33,6 +33,7 @@ #include "bnxt_xdp.h" #include "bnxt_ptp.h" #include "bnxt_ethtool.h" +#include "bnxt_gso.h" #include "bnxt_nvm_defs.h" /* NVRAM content constant and structure defs */ #include "bnxt_fw_hdr.h" /* Firmware hdr constant and structure defs */ #include "bnxt_coredump.h" @@ -340,7 +341,7 @@ enum { RX_NETPOLL_DISCARDS, }; -static const char *const bnxt_ring_err_stats_arr[] = { +static const char *const bnxt_ring_drv_stats_arr[] = { "rx_total_l4_csum_errors", "rx_total_resets", "rx_total_buf_errors", @@ -500,7 +501,7 @@ static const struct { BNXT_TX_STATS_PRI_ENTRIES(tx_packets), }; -#define BNXT_NUM_RING_ERR_STATS ARRAY_SIZE(bnxt_ring_err_stats_arr) +#define BNXT_NUM_RING_DRV_STATS ARRAY_SIZE(bnxt_ring_drv_stats_arr) #define BNXT_NUM_PORT_STATS ARRAY_SIZE(bnxt_port_stats_arr) #define BNXT_NUM_STATS_PRI \ (ARRAY_SIZE(bnxt_rx_bytes_pri_arr) + \ @@ -539,7 +540,7 @@ static int bnxt_get_num_stats(struct bnxt *bp) int num_stats = bnxt_get_num_ring_stats(bp); int len; - num_stats += BNXT_NUM_RING_ERR_STATS; + num_stats += BNXT_NUM_RING_DRV_STATS; if (bp->flags & BNXT_FLAG_PORT_STATS) num_stats += BNXT_NUM_PORT_STATS; @@ -594,7 +595,7 @@ static bool is_tx_ring(struct bnxt *bp, int ring_num) static void bnxt_get_ethtool_stats(struct net_device *dev, struct ethtool_stats *stats, u64 *buf) { - struct bnxt_total_ring_err_stats ring_err_stats = {0}; + struct bnxt_total_ring_drv_stats ring_drv_stats = {0}; struct bnxt *bp = netdev_priv(dev); u64 *curr, *prev; u32 tpa_stats; @@ -643,12 +644,12 @@ skip_tpa_ring_stats: buf[j] = sw[k]; } - bnxt_get_ring_err_stats(bp, &ring_err_stats); + bnxt_get_ring_drv_stats(bp, &ring_drv_stats); skip_ring_stats: - curr = &ring_err_stats.rx_total_l4_csum_errors; - prev = &bp->ring_err_stats_prev.rx_total_l4_csum_errors; - for (i = 0; i < BNXT_NUM_RING_ERR_STATS; i++, j++, curr++, prev++) + curr = &ring_drv_stats.rx_total_l4_csum_errors; + prev = &bp->ring_drv_stats_prev.rx_total_l4_csum_errors; + for (i = 0; i < BNXT_NUM_RING_DRV_STATS; i++, j++, curr++, prev++) buf[j] = *curr + *prev; if (bp->flags & BNXT_FLAG_PORT_STATS) { @@ -758,8 +759,8 @@ skip_tpa_stats: ethtool_sprintf(&buf, "[%d]: %s", i, str); } } - for (i = 0; i < BNXT_NUM_RING_ERR_STATS; i++) - ethtool_puts(&buf, bnxt_ring_err_stats_arr[i]); + for (i = 0; i < BNXT_NUM_RING_DRV_STATS; i++) + ethtool_puts(&buf, bnxt_ring_drv_stats_arr[i]); if (bp->flags & BNXT_FLAG_PORT_STATS) for (i = 0; i < BNXT_NUM_PORT_STATS; i++) { @@ -852,12 +853,18 @@ static int bnxt_set_ringparam(struct net_device *dev, u8 tcp_data_split = kernel_ering->tcp_data_split; struct bnxt *bp = netdev_priv(dev); u8 hds_config_mod; + int rc; if ((ering->rx_pending > BNXT_MAX_RX_DESC_CNT) || (ering->tx_pending > BNXT_MAX_TX_DESC_CNT) || (ering->tx_pending < BNXT_MIN_TX_DESC_CNT)) return -EINVAL; + if ((dev->features & NETIF_F_GSO_UDP_L4) && + !(bp->flags & BNXT_FLAG_UDP_GSO_CAP) && + ering->tx_pending < 2 * BNXT_SW_USO_MAX_DESCS) + return -EINVAL; + hds_config_mod = tcp_data_split != dev->cfg->hds_config; if (tcp_data_split == ETHTOOL_TCP_DATA_SPLIT_DISABLED && hds_config_mod) return -EINVAL; @@ -882,9 +889,17 @@ static int bnxt_set_ringparam(struct net_device *dev, bp->tx_ring_size = ering->tx_pending; bnxt_set_ring_params(bp); - if (netif_running(dev)) - return bnxt_open_nic(bp, false, false); + if (netif_running(dev)) { + rc = bnxt_open_nic(bp, false, false); + if (rc) + return rc; + } + /* ring size changes may affect features (SW USO requires a minimum + * ring size), so recalculate features to ensure the correct features + * are blocked/available. + */ + netdev_update_features(dev); return 0; } @@ -942,6 +957,7 @@ static int bnxt_set_channels(struct net_device *dev, { struct bnxt *bp = netdev_priv(dev); int req_tx_rings, req_rx_rings, tcs; + u32 new_tbl_size = 0, old_tbl_size; bool sh = false; int tx_xdp = 0; int rc = 0; @@ -976,19 +992,33 @@ static int bnxt_set_channels(struct net_device *dev, tx_xdp = req_rx_rings; } - if (bnxt_get_nr_rss_ctxs(bp, req_rx_rings) != - bnxt_get_nr_rss_ctxs(bp, bp->rx_nr_rings) && - (netif_is_rxfh_configured(dev) || bp->num_rss_ctx)) { - netdev_warn(dev, "RSS table size change required, RSS table entries must be default (with no additional RSS contexts present) to proceed\n"); - return -EINVAL; - } - rc = bnxt_check_rings(bp, req_tx_rings, req_rx_rings, sh, tcs, tx_xdp); if (rc) { netdev_warn(dev, "Unable to allocate the requested rings\n"); return rc; } + /* RSS table size only changes on P5 chips with older firmware; + * newer firmware always uses the largest table size. + */ + if (bnxt_get_nr_rss_ctxs(bp, req_rx_rings) != + bnxt_get_nr_rss_ctxs(bp, bp->rx_nr_rings)) { + new_tbl_size = bnxt_get_nr_rss_ctxs(bp, req_rx_rings) * + BNXT_RSS_TABLE_ENTRIES_P5; + old_tbl_size = bnxt_get_rxfh_indir_size(dev); + + if (!ethtool_rxfh_indir_can_resize(dev, bp->rss_indir_tbl, + old_tbl_size, + new_tbl_size)) { + netdev_warn(dev, "RSS table resize not possible\n"); + return -EINVAL; + } + + rc = ethtool_rxfh_ctxs_can_resize(dev, new_tbl_size); + if (rc) + return rc; + } + if (netif_running(dev)) { if (BNXT_PF(bp)) { /* TODO CHIMP_FW: Send message to all VF's @@ -998,6 +1028,12 @@ static int bnxt_set_channels(struct net_device *dev, bnxt_close_nic(bp, true, false); } + if (new_tbl_size) { + ethtool_rxfh_indir_resize(dev, bp->rss_indir_tbl, + old_tbl_size, new_tbl_size); + ethtool_rxfh_ctxs_resize(dev, new_tbl_size); + } + if (sh) { bp->flags |= BNXT_FLAG_SHARED_RINGS; bp->rx_nr_rings = channel->combined_count; diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_gso.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_gso.c new file mode 100644 index 000000000000..f317f60414e8 --- /dev/null +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_gso.c @@ -0,0 +1,240 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* Broadcom NetXtreme-C/E network driver. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation. + */ + +#include <linux/pci.h> +#include <linux/netdevice.h> +#include <linux/skbuff.h> +#include <net/netdev_queues.h> +#include <net/ip.h> +#include <net/ipv6.h> +#include <net/udp.h> +#include <net/tso.h> +#include <linux/bnxt/hsi.h> + +#include "bnxt.h" +#include "bnxt_gso.h" + +static u32 bnxt_sw_gso_lhint(unsigned int len) +{ + if (len <= 512) + return TX_BD_FLAGS_LHINT_512_AND_SMALLER; + else if (len <= 1023) + return TX_BD_FLAGS_LHINT_512_TO_1023; + else if (len <= 2047) + return TX_BD_FLAGS_LHINT_1024_TO_2047; + else + return TX_BD_FLAGS_LHINT_2048_AND_LARGER; +} + +netdev_tx_t bnxt_sw_udp_gso_xmit(struct bnxt *bp, + struct bnxt_tx_ring_info *txr, + struct netdev_queue *txq, + struct sk_buff *skb) +{ + unsigned int last_unmap_len __maybe_unused = 0; + dma_addr_t last_unmap_addr __maybe_unused = 0; + struct bnxt_sw_tx_bd *last_unmap_buf = NULL; + unsigned int hdr_len, mss, num_segs; + struct pci_dev *pdev = bp->pdev; + unsigned int total_payload; + struct tso_dma_map map; + u32 vlan_tag_flags = 0; + int i, bds_needed; + struct tso_t tso; + u16 cfa_action; + __le32 csum; + u16 prod; + + hdr_len = tso_start(skb, &tso); + mss = skb_shinfo(skb)->gso_size; + total_payload = skb->len - hdr_len; + num_segs = DIV_ROUND_UP(total_payload, mss); + + if (unlikely(num_segs <= 1)) + goto drop; + + /* Upper bound on the number of descriptors needed. + * + * Each segment uses 1 long BD + 1 ext BD + payload BDs, which is + * at most num_segs + nr_frags (each frag boundary crossing adds at + * most 1 extra BD). + */ + bds_needed = 3 * num_segs + skb_shinfo(skb)->nr_frags + 1; + + if (unlikely(bnxt_tx_avail(bp, txr) < bds_needed)) { + netif_txq_try_stop(txq, bnxt_tx_avail(bp, txr), + bp->tx_wake_thresh); + return NETDEV_TX_BUSY; + } + + /* BD backpressure alone cannot prevent overwriting in-flight + * headers in the inline buffer. Check slot availability directly. + */ + if (!netif_txq_maybe_stop(txq, bnxt_inline_avail(txr), + num_segs, num_segs)) + return NETDEV_TX_BUSY; + + if (unlikely(tso_dma_map_init(&map, &pdev->dev, skb, hdr_len))) + goto drop; + + cfa_action = bnxt_xmit_get_cfa_action(skb); + if (skb_vlan_tag_present(skb)) { + vlan_tag_flags = TX_BD_CFA_META_KEY_VLAN | + skb_vlan_tag_get(skb); + if (skb->vlan_proto == htons(ETH_P_8021Q)) + vlan_tag_flags |= 1 << TX_BD_CFA_META_TPID_SHIFT; + } + + csum = cpu_to_le32(TX_BD_FLAGS_TCP_UDP_CHKSUM); + if (!tso.ipv6) + csum |= cpu_to_le32(TX_BD_FLAGS_IP_CKSUM); + + prod = txr->tx_prod; + + for (i = 0; i < num_segs; i++) { + unsigned int seg_payload = min_t(unsigned int, mss, + total_payload - i * mss); + u16 slot = (txr->tx_inline_prod + i) & + (BNXT_SW_USO_MAX_SEGS - 1); + struct bnxt_sw_tx_bd *tx_buf; + unsigned int mapping_len; + dma_addr_t this_hdr_dma; + unsigned int chunk_len; + unsigned int offset; + dma_addr_t dma_addr; + struct tx_bd *txbd; + struct udphdr *uh; + void *this_hdr; + int bd_count; + bool last; + u32 flags; + + last = (i == num_segs - 1); + offset = slot * TSO_HEADER_SIZE; + this_hdr = txr->tx_inline_buf + offset; + this_hdr_dma = txr->tx_inline_dma + offset; + + tso_build_hdr(skb, this_hdr, &tso, seg_payload, last); + + /* Zero stale csum fields copied from the original skb; + * HW offload recomputes from scratch. + */ + uh = this_hdr + skb_transport_offset(skb); + uh->check = 0; + if (!tso.ipv6) { + struct iphdr *iph = this_hdr + skb_network_offset(skb); + + iph->check = 0; + } + + dma_sync_single_for_device(&pdev->dev, this_hdr_dma, + hdr_len, DMA_TO_DEVICE); + + bd_count = tso_dma_map_count(&map, seg_payload); + + tx_buf = &txr->tx_buf_ring[RING_TX(bp, prod)]; + txbd = &txr->tx_desc_ring[TX_RING(bp, prod)][TX_IDX(prod)]; + + tx_buf->skb = skb; + tx_buf->nr_frags = bd_count; + tx_buf->is_push = 0; + tx_buf->is_ts_pkt = 0; + + dma_unmap_addr_set(tx_buf, mapping, this_hdr_dma); + dma_unmap_len_set(tx_buf, len, 0); + + if (last) { + tx_buf->is_sw_gso = BNXT_SW_GSO_LAST; + tso_dma_map_completion_save(&map, &tx_buf->sw_gso_cstate); + } else { + tx_buf->is_sw_gso = BNXT_SW_GSO_MID; + } + + flags = (hdr_len << TX_BD_LEN_SHIFT) | + TX_BD_TYPE_LONG_TX_BD | + TX_BD_CNT(2 + bd_count); + + flags |= bnxt_sw_gso_lhint(hdr_len + seg_payload); + + txbd->tx_bd_len_flags_type = cpu_to_le32(flags); + txbd->tx_bd_haddr = cpu_to_le64(this_hdr_dma); + txbd->tx_bd_opaque = SET_TX_OPAQUE(bp, txr, prod, + 2 + bd_count); + + prod = NEXT_TX(prod); + bnxt_init_ext_bd(bp, txr, prod, csum, + vlan_tag_flags, cfa_action); + + /* set dma_unmap_len on the LAST BD touching each + * region. Since completions are in-order, the last segment + * completes after all earlier ones, so the unmap is safe. + */ + while (tso_dma_map_next(&map, &dma_addr, &chunk_len, + &mapping_len, seg_payload)) { + prod = NEXT_TX(prod); + txbd = &txr->tx_desc_ring[TX_RING(bp, prod)][TX_IDX(prod)]; + tx_buf = &txr->tx_buf_ring[RING_TX(bp, prod)]; + + txbd->tx_bd_haddr = cpu_to_le64(dma_addr); + dma_unmap_addr_set(tx_buf, mapping, dma_addr); + dma_unmap_len_set(tx_buf, len, 0); + tx_buf->skb = NULL; + tx_buf->is_sw_gso = 0; + + if (mapping_len) { + if (last_unmap_buf) { + dma_unmap_addr_set(last_unmap_buf, + mapping, + last_unmap_addr); + dma_unmap_len_set(last_unmap_buf, + len, + last_unmap_len); + } + last_unmap_addr = dma_addr; + last_unmap_len = mapping_len; + } + last_unmap_buf = tx_buf; + + flags = chunk_len << TX_BD_LEN_SHIFT; + txbd->tx_bd_len_flags_type = cpu_to_le32(flags); + txbd->tx_bd_opaque = 0; + + seg_payload -= chunk_len; + } + + txbd->tx_bd_len_flags_type |= + cpu_to_le32(TX_BD_FLAGS_PACKET_END); + + prod = NEXT_TX(prod); + } + + if (last_unmap_buf) { + dma_unmap_addr_set(last_unmap_buf, mapping, last_unmap_addr); + dma_unmap_len_set(last_unmap_buf, len, last_unmap_len); + } + + txr->tx_inline_prod += num_segs; + + netdev_tx_sent_queue(txq, skb->len); + + WRITE_ONCE(txr->tx_prod, prod); + /* Sync BDs before doorbell */ + wmb(); + bnxt_db_write(bp, &txr->tx_db, prod); + + if (unlikely(bnxt_tx_avail(bp, txr) <= bp->tx_wake_thresh)) + netif_txq_try_stop(txq, bnxt_tx_avail(bp, txr), + bp->tx_wake_thresh); + + return NETDEV_TX_OK; + +drop: + dev_kfree_skb_any(skb); + dev_core_stats_tx_dropped_inc(bp->dev); + return NETDEV_TX_OK; +} diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_gso.h b/drivers/net/ethernet/broadcom/bnxt/bnxt_gso.h new file mode 100644 index 000000000000..47528c20f311 --- /dev/null +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_gso.h @@ -0,0 +1,46 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Broadcom NetXtreme-C/E network driver. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation. + */ + +#ifndef BNXT_GSO_H +#define BNXT_GSO_H + +/* Maximum segments the stack may send in a single SW USO skb. + * This caps gso_max_segs for NICs without HW USO support. + */ +#define BNXT_SW_USO_MAX_SEGS 64 + +/* Worst-case TX descriptors consumed by one SW USO packet: + * Each segment: 1 long BD + 1 ext BD + payload BDs. + * Total payload BDs across all segs <= num_segs + nr_frags (each frag + * boundary crossing adds at most 1 extra BD). + * So: 3 * max_segs + MAX_SKB_FRAGS + 1 = 3 * 64 + 17 + 1 = 210. + */ +#define BNXT_SW_USO_MAX_DESCS (3 * BNXT_SW_USO_MAX_SEGS + MAX_SKB_FRAGS + 1) + +static inline u16 bnxt_inline_avail(struct bnxt_tx_ring_info *txr) +{ + return BNXT_SW_USO_MAX_SEGS - + (u16)(txr->tx_inline_prod - READ_ONCE(txr->tx_inline_cons)); +} + +static inline int bnxt_min_tx_desc_cnt(struct bnxt *bp, + netdev_features_t features) +{ + if (!(bp->flags & BNXT_FLAG_UDP_GSO_CAP) && + (features & NETIF_F_GSO_UDP_L4)) + return BNXT_SW_USO_MAX_DESCS; + return BNXT_MIN_TX_DESC_CNT; +} + +netdev_tx_t bnxt_sw_udp_gso_xmit(struct bnxt *bp, + struct bnxt_tx_ring_info *txr, + struct netdev_queue *txq, + struct sk_buff *skb); + +#endif diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c index bebe37e139c9..9e5009be8e98 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c @@ -471,3 +471,60 @@ bnxt_xdp_build_skb(struct bnxt *bp, struct sk_buff *skb, u8 num_frags, xdp_buff_get_skb_flags(xdp)); return skb; } + +int bnxt_xdp_rx_hash(const struct xdp_md *ctx, u32 *hash, + enum xdp_rss_hash_type *rss_type) +{ + const struct bnxt_xdp_buff *xdp = (void *)ctx; + const struct rx_cmp_ext *rxcmp1 = xdp->rxcmp1; + const struct rx_cmp *rxcmp = xdp->rxcmp; + enum xdp_rss_hash_type hash_type = 0; + u32 itypes; + + if (!rxcmp || !RX_CMP_HASH_VALID(rxcmp)) + return -ENODATA; + + *hash = le32_to_cpu(rxcmp->rx_cmp_rss_hash); + + if (!rxcmp1) { + *rss_type = XDP_RSS_TYPE_L2; + return 0; + } + + if (xdp->cmp_type == CMP_TYPE_RX_L2_CMP) { + itypes = RX_CMP_ITYPES(rxcmp); + if (rxcmp1->rx_cmp_flags2 & + cpu_to_le32(RX_CMP_FLAGS2_IP_TYPE)) { + hash_type |= XDP_RSS_TYPE_L3_IPV6; + } else { + hash_type |= XDP_RSS_TYPE_L3_IPV4; + } + + switch (itypes) { + case RX_CMP_FLAGS_ITYPE_TCP: + hash_type |= XDP_RSS_L4 | XDP_RSS_L4_TCP; + break; + case RX_CMP_FLAGS_ITYPE_UDP: + hash_type |= XDP_RSS_L4 | XDP_RSS_L4_UDP; + break; + case RX_CMP_FLAGS_ITYPE_ICMP: + hash_type |= XDP_RSS_L4 | XDP_RSS_L4_ICMP; + break; + default: + break; + } + } else if (xdp->cmp_type == CMP_TYPE_RX_L2_V3_CMP) { + struct bnxt *bp = netdev_priv(xdp->xdp.rxq->dev); + + if (rxcmp1->rx_cmp_flags2 & cpu_to_le32(RX_CMP_FLAGS2_IP_TYPE)) + hash_type |= XDP_RSS_TYPE_L3_IPV6; + else + hash_type |= XDP_RSS_TYPE_L3_IPV4; + + if (bnxt_rss_ext_op(bp, rxcmp) == PKT_HASH_TYPE_L4) + hash_type |= XDP_RSS_L4; + } + + *rss_type = hash_type; + return 0; +} diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.h b/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.h index 8933a0dec09a..fb4f9143929f 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.h +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.h @@ -12,6 +12,13 @@ DECLARE_STATIC_KEY_FALSE(bnxt_xdp_locking_key); +struct bnxt_xdp_buff { + struct xdp_buff xdp; + struct rx_cmp *rxcmp; + struct rx_cmp_ext *rxcmp1; + u8 cmp_type; +}; + struct bnxt_sw_tx_bd *bnxt_xmit_bd(struct bnxt *bp, struct bnxt_tx_ring_info *txr, dma_addr_t mapping, u32 len, @@ -34,4 +41,7 @@ void bnxt_xdp_buff_frags_free(struct bnxt_rx_ring_info *rxr, struct sk_buff *bnxt_xdp_build_skb(struct bnxt *bp, struct sk_buff *skb, u8 num_frags, struct bnxt_rx_ring_info *rxr, struct xdp_buff *xdp); +int bnxt_xdp_rx_hash(const struct xdp_md *ctx, u32 *hash, + enum xdp_rss_hash_type *rss_type); + #endif diff --git a/drivers/net/ethernet/broadcom/genet/bcmgenet.c b/drivers/net/ethernet/broadcom/genet/bcmgenet.c index 482a31e7b72b..54f71b1e85fc 100644 --- a/drivers/net/ethernet/broadcom/genet/bcmgenet.c +++ b/drivers/net/ethernet/broadcom/genet/bcmgenet.c @@ -1819,15 +1819,15 @@ static struct enet_cb *bcmgenet_put_txcb(struct bcmgenet_priv *priv, { struct enet_cb *tx_cb_ptr; - tx_cb_ptr = ring->cbs; - tx_cb_ptr += ring->write_ptr - ring->cb_ptr; - /* Rewinding local write pointer */ if (ring->write_ptr == ring->cb_ptr) ring->write_ptr = ring->end_ptr; else ring->write_ptr--; + tx_cb_ptr = ring->cbs; + tx_cb_ptr += ring->write_ptr - ring->cb_ptr; + return tx_cb_ptr; } @@ -1985,6 +1985,7 @@ static unsigned int bcmgenet_tx_reclaim(struct net_device *dev, drop = (ring->prod_index - ring->c_index) & DMA_C_INDEX_MASK; released += drop; ring->prod_index = ring->c_index & DMA_C_INDEX_MASK; + ring->free_bds += drop; while (drop--) { cb_ptr = bcmgenet_put_txcb(priv, ring); skb = cb_ptr->skb; @@ -1996,6 +1997,7 @@ static unsigned int bcmgenet_tx_reclaim(struct net_device *dev, } if (skb) dev_consume_skb_any(skb); + netdev_tx_reset_queue(netdev_get_tx_queue(dev, ring->index)); bcmgenet_tdma_ring_writel(priv, ring->index, ring->prod_index, TDMA_PROD_INDEX); wr_ptr = ring->write_ptr * WORDS_PER_BD(priv); @@ -3475,27 +3477,23 @@ static void bcmgenet_dump_tx_queue(struct bcmgenet_tx_ring *ring) static void bcmgenet_timeout(struct net_device *dev, unsigned int txqueue) { struct bcmgenet_priv *priv = netdev_priv(dev); - u32 int1_enable = 0; - unsigned int q; + struct bcmgenet_tx_ring *ring = &priv->tx_rings[txqueue]; + struct netdev_queue *txq = netdev_get_tx_queue(dev, txqueue); netif_dbg(priv, tx_err, dev, "bcmgenet_timeout\n"); - for (q = 0; q <= priv->hw_params->tx_queues; q++) - bcmgenet_dump_tx_queue(&priv->tx_rings[q]); - - bcmgenet_tx_reclaim_all(dev); + bcmgenet_dump_tx_queue(ring); - for (q = 0; q <= priv->hw_params->tx_queues; q++) - int1_enable |= (1 << q); + bcmgenet_tx_reclaim(dev, ring, true); - /* Re-enable TX interrupts if disabled */ - bcmgenet_intrl2_1_writel(priv, int1_enable, INTRL2_CPU_MASK_CLEAR); + /* Re-enable the TX interrupt for this ring */ + bcmgenet_intrl2_1_writel(priv, 1 << txqueue, INTRL2_CPU_MASK_CLEAR); - netif_trans_update(dev); + txq_trans_cond_update(txq); - BCMGENET_STATS64_INC((&priv->tx_rings[txqueue].stats64), errors); + BCMGENET_STATS64_INC((&ring->stats64), errors); - netif_tx_wake_all_queues(dev); + netif_tx_wake_queue(txq); } #define MAX_MDF_FILTER 17 diff --git a/drivers/net/ethernet/cadence/macb.h b/drivers/net/ethernet/cadence/macb.h index 87414a2ddf6e..2de56017ee0d 100644 --- a/drivers/net/ethernet/cadence/macb.h +++ b/drivers/net/ethernet/cadence/macb.h @@ -170,6 +170,10 @@ #define GEM_PCSANNPTX 0x021c /* PCS AN Next Page TX */ #define GEM_PCSANNPLP 0x0220 /* PCS AN Next Page LP */ #define GEM_PCSANEXTSTS 0x023c /* PCS AN Extended Status */ +#define GEM_RXLPI 0x0270 /* RX LPI Transitions */ +#define GEM_RXLPITIME 0x0274 /* RX LPI Time */ +#define GEM_TXLPI 0x0278 /* TX LPI Transitions */ +#define GEM_TXLPITIME 0x027c /* TX LPI Time */ #define GEM_DCFG1 0x0280 /* Design Config 1 */ #define GEM_DCFG2 0x0284 /* Design Config 2 */ #define GEM_DCFG3 0x0288 /* Design Config 3 */ @@ -305,6 +309,8 @@ #define MACB_IRXFCS_SIZE 1 /* GEM specific NCR bitfields. */ +#define GEM_TXLPIEN_OFFSET 19 +#define GEM_TXLPIEN_SIZE 1 #define GEM_ENABLE_HS_MAC_OFFSET 31 #define GEM_ENABLE_HS_MAC_SIZE 1 @@ -518,6 +524,8 @@ #define GEM_IRQCOR_SIZE 1 #define GEM_DBWDEF_OFFSET 25 #define GEM_DBWDEF_SIZE 3 +#define GEM_USERIO_OFFSET 9 +#define GEM_USERIO_SIZE 1 #define GEM_NO_PCS_OFFSET 0 #define GEM_NO_PCS_SIZE 1 @@ -779,6 +787,10 @@ #define MACB_CAPS_DMA_PTP BIT(22) #define MACB_CAPS_RSC BIT(23) #define MACB_CAPS_NO_LSO BIT(24) +#define MACB_CAPS_EEE BIT(25) +#define MACB_CAPS_USRIO_HAS_MII BIT(26) +#define MACB_CAPS_USRIO_HAS_REFCLK_SOURCE BIT(27) +#define MACB_CAPS_USRIO_HAS_TSUCLK_SOURCE BIT(28) /* LSO settings */ #define MACB_LSO_UFO_ENABLE 0x01 @@ -1043,6 +1055,10 @@ struct gem_stats { u64 rx_ip_header_checksum_errors; u64 rx_tcp_checksum_errors; u64 rx_udp_checksum_errors; + u64 rx_lpi_transitions; + u64 rx_lpi_time; + u64 tx_lpi_transitions; + u64 tx_lpi_time; }; /* Describes the name and offset of an individual statistic register, as @@ -1142,6 +1158,10 @@ static const struct gem_statistic gem_statistics[] = { GEM_BIT(NDS_RXERR)), GEM_STAT_TITLE_BITS(RXUDPCCNT, "rx_udp_checksum_errors", GEM_BIT(NDS_RXERR)), + GEM_STAT_TITLE(RXLPI, "rx_lpi_transitions"), + GEM_STAT_TITLE(RXLPITIME, "rx_lpi_time"), + GEM_STAT_TITLE(TXLPI, "tx_lpi_transitions"), + GEM_STAT_TITLE(TXLPITIME, "tx_lpi_time"), }; #define GEM_STATS_LEN ARRAY_SIZE(gem_statistics) @@ -1210,7 +1230,10 @@ struct macb_usrio_config { u32 rmii; u32 rgmii; u32 refclk; + u32 clken; u32 hdfctlen; + u32 tsu_source; + bool refclk_default_external; }; struct macb_config { @@ -1357,6 +1380,11 @@ struct macb { struct work_struct hresp_err_bh_work; + /* EEE / LPI state */ + bool eee_active; + struct delayed_work tx_lpi_work; + u32 tx_lpi_timer; + int rx_bd_rd_prefetch; int tx_bd_rd_prefetch; @@ -1446,6 +1474,13 @@ static inline bool macb_dma_ptp(struct macb *bp) bp->caps & MACB_CAPS_DMA_PTP; } +static inline void macb_queue_isr_clear(struct macb *bp, + struct macb_queue *queue, u32 value) +{ + if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) + queue_writel(queue, ISR, value); +} + /** * struct macb_platform_data - platform data for MACB Ethernet used for PCI registration * @pclk: platform clock diff --git a/drivers/net/ethernet/cadence/macb_main.c b/drivers/net/ethernet/cadence/macb_main.c index 99e7d5cf3786..a12aa21244e8 100644 --- a/drivers/net/ethernet/cadence/macb_main.c +++ b/drivers/net/ethernet/cadence/macb_main.c @@ -10,6 +10,7 @@ #include <linux/clk-provider.h> #include <linux/clk.h> #include <linux/crc32.h> +#include <linux/delay.h> #include <linux/dma-mapping.h> #include <linux/etherdevice.h> #include <linux/firmware/xlnx-zynqmp.h> @@ -49,6 +50,7 @@ struct sifive_fu540_macb_mgmt { #define MACB_RX_BUFFER_SIZE 128 #define RX_BUFFER_MULTIPLE 64 /* bytes */ +#define RX_BUFFER_MAX (0xFF * RX_BUFFER_MULTIPLE) /* 16320 bytes */ #define DEFAULT_RX_RING_SIZE 512 /* must be power of 2 */ #define MIN_RX_RING_SIZE 64 @@ -68,6 +70,10 @@ struct sifive_fu540_macb_mgmt { #define MACB_TX_INT_FLAGS (MACB_TX_ERR_FLAGS | MACB_BIT(TCOMP) \ | MACB_BIT(TXUBR)) +#define MACB_INT_MISC_FLAGS (MACB_TX_ERR_FLAGS | MACB_BIT(RXUBR) | \ + MACB_BIT(ISR_ROVR) | MACB_BIT(HRESP) | \ + GEM_BIT(WOL) | MACB_BIT(WOL)) + /* Max length of transmit frame must be a multiple of 8 bytes */ #define MACB_TX_LEN_ALIGN 8 #define MACB_MAX_TX_LEN ((unsigned int)((1 << MACB_TX_FRMLEN_SIZE) - 1) & ~((unsigned int)(MACB_TX_LEN_ALIGN - 1))) @@ -558,10 +564,21 @@ static int macb_usx_pcs_config(struct phylink_pcs *pcs, return 0; } +static unsigned int macb_pcs_inband_caps(struct phylink_pcs *pcs, + phy_interface_t interface) +{ + return LINK_INBAND_DISABLE | LINK_INBAND_ENABLE; +} + static void macb_pcs_get_state(struct phylink_pcs *pcs, unsigned int neg_mode, struct phylink_link_state *state) { - state->link = 0; + struct macb *bp = container_of(pcs, struct macb, phylink_sgmii_pcs); + u16 bmsr, lpa; + + bmsr = gem_readl(bp, PCSSTS); + lpa = gem_readl(bp, PCSANLPBASE); + phylink_mii_c22_pcs_decode_state(state, neg_mode, bmsr, lpa); } static void macb_pcs_an_restart(struct phylink_pcs *pcs) @@ -575,6 +592,26 @@ static int macb_pcs_config(struct phylink_pcs *pcs, const unsigned long *advertising, bool permit_pause_to_mac) { + struct macb *bp = container_of(pcs, struct macb, phylink_sgmii_pcs); + u32 old, new; + + old = gem_readl(bp, PCSANADV); + new = phylink_mii_c22_pcs_encode_advertisement(interface, advertising); + if (new != -EINVAL && old != new) + gem_writel(bp, PCSANADV, new); + + /* Disable AN if it's not to be used, enable otherwise. + * Must be written after PCSSEL is set in NCFGR which is done in + * macb_mac_config(), otherwise writes will not take effect. + */ + old = gem_readl(bp, PCSCNTRL); + if (neg_mode == PHYLINK_PCS_NEG_INBAND_ENABLED) + new = old | BMCR_ANENABLE; + else + new = old & ~BMCR_ANENABLE; + if (old != new) + gem_writel(bp, PCSCNTRL, new); + return 0; } @@ -585,11 +622,113 @@ static const struct phylink_pcs_ops macb_phylink_usx_pcs_ops = { }; static const struct phylink_pcs_ops macb_phylink_pcs_ops = { + .pcs_inband_caps = macb_pcs_inband_caps, .pcs_get_state = macb_pcs_get_state, .pcs_an_restart = macb_pcs_an_restart, .pcs_config = macb_pcs_config, }; +static bool macb_tx_lpi_set(struct macb *bp, bool enable) +{ + u32 old, ncr; + + lockdep_assert_held(&bp->lock); + + ncr = macb_readl(bp, NCR); + old = ncr; + if (enable) + ncr |= GEM_BIT(TXLPIEN); + else + ncr &= ~GEM_BIT(TXLPIEN); + if (old != ncr) + macb_writel(bp, NCR, ncr); + + return old != ncr; +} + +static bool macb_tx_all_queues_idle(struct macb *bp) +{ + struct macb_queue *queue; + unsigned int q; + + for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { + if (READ_ONCE(queue->tx_head) != READ_ONCE(queue->tx_tail)) + return false; + } + return true; +} + +static void macb_tx_lpi_work_fn(struct work_struct *work) +{ + struct macb *bp = container_of(work, struct macb, tx_lpi_work.work); + unsigned long flags; + + spin_lock_irqsave(&bp->lock, flags); + if (bp->eee_active && macb_tx_all_queues_idle(bp)) + macb_tx_lpi_set(bp, true); + spin_unlock_irqrestore(&bp->lock, flags); +} + +static void macb_tx_lpi_schedule(struct macb *bp) +{ + if (bp->eee_active) + mod_delayed_work(system_wq, &bp->tx_lpi_work, + usecs_to_jiffies(bp->tx_lpi_timer)); +} + +/* Wake from LPI before transmitting. The MAC must deassert TXLPIEN + * and wait for the PHY to exit LPI before any frame can be sent. + * IEEE 802.3az Tw_sys is ~17us for 1000BASE-T, ~30us for 100BASE-TX; + * we use a conservative 50us. + */ +static void macb_tx_lpi_wake(struct macb *bp) +{ + lockdep_assert_held(&bp->lock); + + if (!bp->eee_active) + return; + + if (!macb_tx_lpi_set(bp, false)) + return; + + cancel_delayed_work(&bp->tx_lpi_work); + udelay(50); +} + +static void macb_mac_disable_tx_lpi(struct phylink_config *config) +{ + struct net_device *ndev = to_net_dev(config->dev); + struct macb *bp = netdev_priv(ndev); + unsigned long flags; + + cancel_delayed_work_sync(&bp->tx_lpi_work); + + spin_lock_irqsave(&bp->lock, flags); + bp->eee_active = false; + macb_tx_lpi_set(bp, false); + spin_unlock_irqrestore(&bp->lock, flags); +} + +static int macb_mac_enable_tx_lpi(struct phylink_config *config, u32 timer, + bool tx_clk_stop) +{ + struct net_device *ndev = to_net_dev(config->dev); + struct macb *bp = netdev_priv(ndev); + unsigned long flags; + + spin_lock_irqsave(&bp->lock, flags); + bp->tx_lpi_timer = timer; + bp->eee_active = true; + spin_unlock_irqrestore(&bp->lock, flags); + + /* Defer initial LPI entry by 1 second after link-up per + * IEEE 802.3az section 22.7a. + */ + mod_delayed_work(system_wq, &bp->tx_lpi_work, msecs_to_jiffies(1000)); + + return 0; +} + static void macb_mac_config(struct phylink_config *config, unsigned int mode, const struct phylink_link_state *state) { @@ -629,22 +768,6 @@ static void macb_mac_config(struct phylink_config *config, unsigned int mode, if (old_ncr ^ ncr) macb_or_gem_writel(bp, NCR, ncr); - /* Disable AN for SGMII fixed link configuration, enable otherwise. - * Must be written after PCSSEL is set in NCFGR, - * otherwise writes will not take effect. - */ - if (macb_is_gem(bp) && state->interface == PHY_INTERFACE_MODE_SGMII) { - u32 pcsctrl, old_pcsctrl; - - old_pcsctrl = gem_readl(bp, PCSCNTRL); - if (mode == MLO_AN_FIXED) - pcsctrl = old_pcsctrl & ~GEM_BIT(PCSAUTONEG); - else - pcsctrl = old_pcsctrl | GEM_BIT(PCSAUTONEG); - if (old_pcsctrl != pcsctrl) - gem_writel(bp, PCSCNTRL, pcsctrl); - } - spin_unlock_irqrestore(&bp->lock, flags); } @@ -845,6 +968,8 @@ static const struct phylink_mac_ops macb_phylink_ops = { .mac_config = macb_mac_config, .mac_link_down = macb_mac_link_down, .mac_link_up = macb_mac_link_up, + .mac_disable_tx_lpi = macb_mac_disable_tx_lpi, + .mac_enable_tx_lpi = macb_mac_enable_tx_lpi, }; static bool macb_phy_handle_exists(struct device_node *dn) @@ -909,6 +1034,12 @@ static int macb_mii_probe(struct net_device *dev) if (bp->phy_interface == PHY_INTERFACE_MODE_SGMII) { bp->phylink_config.poll_fixed_state = true; bp->phylink_config.get_fixed_state = macb_get_pcs_fixed_state; + /* The PCSAUTONEG bit in PCSCNTRL is on out of reset. Setting + * default_an_inband to true tells phylink to turn it off only + * if necessary (e.g. a fixed link or a PHY that doesn't support + * inband). + */ + bp->phylink_config.default_an_inband = true; } bp->phylink_config.mac_capabilities = MAC_ASYM_PAUSE | @@ -940,6 +1071,18 @@ static int macb_mii_probe(struct net_device *dev) } } + /* Configure EEE LPI if supported */ + if (bp->caps & MACB_CAPS_EEE) { + __set_bit(PHY_INTERFACE_MODE_MII, + bp->phylink_config.lpi_interfaces); + __set_bit(PHY_INTERFACE_MODE_GMII, + bp->phylink_config.lpi_interfaces); + phy_interface_set_rgmii(bp->phylink_config.lpi_interfaces); + bp->phylink_config.lpi_capabilities = MAC_100FD | MAC_1000FD; + bp->phylink_config.lpi_timer_default = 250000; + bp->phylink_config.eee_enabled_default = true; + } + bp->phylink = phylink_create(&bp->phylink_config, bp->pdev->dev.fwnode, bp->phy_interface, &macb_phylink_ops); if (IS_ERR(bp->phylink)) { @@ -1336,6 +1479,9 @@ static int macb_tx_complete(struct macb_queue *queue, int budget) netif_wake_subqueue(bp->dev, queue_index); spin_unlock_irqrestore(&queue->tx_ptr_lock, flags); + if (packets) + macb_tx_lpi_schedule(bp); + return packets; } @@ -1745,8 +1891,7 @@ static int macb_rx_poll(struct napi_struct *napi, int budget) */ if (macb_rx_pending(queue)) { queue_writel(queue, IDR, bp->rx_intr_mask); - if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) - queue_writel(queue, ISR, MACB_BIT(RCOMP)); + macb_queue_isr_clear(bp, queue, MACB_BIT(RCOMP)); netdev_vdbg(bp->dev, "poll: packets pending, reschedule\n"); napi_schedule(napi); } @@ -1833,8 +1978,7 @@ static int macb_tx_poll(struct napi_struct *napi, int budget) */ if (macb_tx_complete_pending(queue)) { queue_writel(queue, IDR, MACB_BIT(TCOMP)); - if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) - queue_writel(queue, ISR, MACB_BIT(TCOMP)); + macb_queue_isr_clear(bp, queue, MACB_BIT(TCOMP)); netdev_vdbg(bp->dev, "TX poll: packets pending, reschedule\n"); napi_schedule(napi); } @@ -1882,62 +2026,92 @@ static void macb_hresp_error_task(struct work_struct *work) netif_tx_start_all_queues(dev); } -static irqreturn_t macb_wol_interrupt(int irq, void *dev_id) +static void macb_wol_interrupt(struct macb_queue *queue, u32 status) { - struct macb_queue *queue = dev_id; struct macb *bp = queue->bp; - u32 status; - - status = queue_readl(queue, ISR); - - if (unlikely(!status)) - return IRQ_NONE; - spin_lock(&bp->lock); - - if (status & MACB_BIT(WOL)) { - queue_writel(queue, IDR, MACB_BIT(WOL)); - macb_writel(bp, WOL, 0); - netdev_vdbg(bp->dev, "MACB WoL: queue = %u, isr = 0x%08lx\n", - (unsigned int)(queue - bp->queues), - (unsigned long)status); - if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) - queue_writel(queue, ISR, MACB_BIT(WOL)); - pm_wakeup_event(&bp->pdev->dev, 0); - } + queue_writel(queue, IDR, MACB_BIT(WOL)); + macb_writel(bp, WOL, 0); + netdev_vdbg(bp->dev, "MACB WoL: queue = %u, isr = 0x%08lx\n", + (unsigned int)(queue - bp->queues), + (unsigned long)status); + macb_queue_isr_clear(bp, queue, MACB_BIT(WOL)); + pm_wakeup_event(&bp->pdev->dev, 0); +} - spin_unlock(&bp->lock); +static void gem_wol_interrupt(struct macb_queue *queue, u32 status) +{ + struct macb *bp = queue->bp; - return IRQ_HANDLED; + queue_writel(queue, IDR, GEM_BIT(WOL)); + gem_writel(bp, WOL, 0); + netdev_vdbg(bp->dev, "GEM WoL: queue = %u, isr = 0x%08lx\n", + (unsigned int)(queue - bp->queues), + (unsigned long)status); + macb_queue_isr_clear(bp, queue, GEM_BIT(WOL)); + pm_wakeup_event(&bp->pdev->dev, 0); } -static irqreturn_t gem_wol_interrupt(int irq, void *dev_id) +static int macb_interrupt_misc(struct macb_queue *queue, u32 status) { - struct macb_queue *queue = dev_id; struct macb *bp = queue->bp; - u32 status; + struct net_device *dev; + u32 ctrl; - status = queue_readl(queue, ISR); + dev = bp->dev; - if (unlikely(!status)) - return IRQ_NONE; + if (unlikely(status & (MACB_TX_ERR_FLAGS))) { + queue_writel(queue, IDR, MACB_TX_INT_FLAGS); + schedule_work(&queue->tx_error_task); + macb_queue_isr_clear(bp, queue, MACB_TX_ERR_FLAGS); + return -1; + } - spin_lock(&bp->lock); + /* Link change detection isn't possible with RMII, so we'll + * add that if/when we get our hands on a full-blown MII PHY. + */ - if (status & GEM_BIT(WOL)) { - queue_writel(queue, IDR, GEM_BIT(WOL)); - gem_writel(bp, WOL, 0); - netdev_vdbg(bp->dev, "GEM WoL: queue = %u, isr = 0x%08lx\n", - (unsigned int)(queue - bp->queues), - (unsigned long)status); - if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) - queue_writel(queue, ISR, GEM_BIT(WOL)); - pm_wakeup_event(&bp->pdev->dev, 0); + /* There is a hardware issue under heavy load where DMA can + * stop, this causes endless "used buffer descriptor read" + * interrupts but it can be cleared by re-enabling RX. See + * the at91rm9200 manual, section 41.3.1 or the Zynq manual + * section 16.7.4 for details. RXUBR is only enabled for + * these two versions. + */ + if (status & MACB_BIT(RXUBR)) { + ctrl = macb_readl(bp, NCR); + macb_writel(bp, NCR, ctrl & ~MACB_BIT(RE)); + wmb(); + macb_writel(bp, NCR, ctrl | MACB_BIT(RE)); + macb_queue_isr_clear(bp, queue, MACB_BIT(RXUBR)); } - spin_unlock(&bp->lock); + if (status & MACB_BIT(ISR_ROVR)) { + /* We missed at least one packet */ + spin_lock(&bp->stats_lock); + if (macb_is_gem(bp)) + bp->hw_stats.gem.rx_overruns++; + else + bp->hw_stats.macb.rx_overruns++; + spin_unlock(&bp->stats_lock); + macb_queue_isr_clear(bp, queue, MACB_BIT(ISR_ROVR)); + } - return IRQ_HANDLED; + if (status & MACB_BIT(HRESP)) { + queue_work(system_bh_wq, &bp->hresp_err_bh_work); + netdev_err(dev, "DMA bus error: HRESP not OK\n"); + macb_queue_isr_clear(bp, queue, MACB_BIT(HRESP)); + } + + if (macb_is_gem(bp)) { + if (status & GEM_BIT(WOL)) + gem_wol_interrupt(queue, status); + } else { + if (status & MACB_BIT(WOL)) + macb_wol_interrupt(queue, status); + } + + return 0; } static irqreturn_t macb_interrupt(int irq, void *dev_id) @@ -1945,7 +2119,7 @@ static irqreturn_t macb_interrupt(int irq, void *dev_id) struct macb_queue *queue = dev_id; struct macb *bp = queue->bp; struct net_device *dev = bp->dev; - u32 status, ctrl; + u32 status; status = queue_readl(queue, ISR); @@ -1958,8 +2132,7 @@ static irqreturn_t macb_interrupt(int irq, void *dev_id) /* close possible race with dev_close */ if (unlikely(!netif_running(dev))) { queue_writel(queue, IDR, -1); - if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) - queue_writel(queue, ISR, -1); + macb_queue_isr_clear(bp, queue, -1); break; } @@ -1975,84 +2148,27 @@ static irqreturn_t macb_interrupt(int irq, void *dev_id) * now. */ queue_writel(queue, IDR, bp->rx_intr_mask); - if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) - queue_writel(queue, ISR, MACB_BIT(RCOMP)); - - if (napi_schedule_prep(&queue->napi_rx)) { - netdev_vdbg(bp->dev, "scheduling RX softirq\n"); - __napi_schedule(&queue->napi_rx); - } + macb_queue_isr_clear(bp, queue, MACB_BIT(RCOMP)); + napi_schedule_irqoff(&queue->napi_rx); } if (status & (MACB_BIT(TCOMP) | MACB_BIT(TXUBR))) { queue_writel(queue, IDR, MACB_BIT(TCOMP)); - if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) - queue_writel(queue, ISR, MACB_BIT(TCOMP) | - MACB_BIT(TXUBR)); - + macb_queue_isr_clear(bp, queue, MACB_BIT(TCOMP) | + MACB_BIT(TXUBR)); if (status & MACB_BIT(TXUBR)) { queue->txubr_pending = true; wmb(); // ensure softirq can see update } - if (napi_schedule_prep(&queue->napi_tx)) { - netdev_vdbg(bp->dev, "scheduling TX softirq\n"); - __napi_schedule(&queue->napi_tx); - } - } - - if (unlikely(status & (MACB_TX_ERR_FLAGS))) { - queue_writel(queue, IDR, MACB_TX_INT_FLAGS); - schedule_work(&queue->tx_error_task); - - if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) - queue_writel(queue, ISR, MACB_TX_ERR_FLAGS); - - break; - } - - /* Link change detection isn't possible with RMII, so we'll - * add that if/when we get our hands on a full-blown MII PHY. - */ - - /* There is a hardware issue under heavy load where DMA can - * stop, this causes endless "used buffer descriptor read" - * interrupts but it can be cleared by re-enabling RX. See - * the at91rm9200 manual, section 41.3.1 or the Zynq manual - * section 16.7.4 for details. RXUBR is only enabled for - * these two versions. - */ - if (status & MACB_BIT(RXUBR)) { - ctrl = macb_readl(bp, NCR); - macb_writel(bp, NCR, ctrl & ~MACB_BIT(RE)); - wmb(); - macb_writel(bp, NCR, ctrl | MACB_BIT(RE)); - - if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) - queue_writel(queue, ISR, MACB_BIT(RXUBR)); - } - - if (status & MACB_BIT(ISR_ROVR)) { - /* We missed at least one packet */ - spin_lock(&bp->stats_lock); - if (macb_is_gem(bp)) - bp->hw_stats.gem.rx_overruns++; - else - bp->hw_stats.macb.rx_overruns++; - spin_unlock(&bp->stats_lock); - - if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) - queue_writel(queue, ISR, MACB_BIT(ISR_ROVR)); + napi_schedule_irqoff(&queue->napi_tx); } - if (status & MACB_BIT(HRESP)) { - queue_work(system_bh_wq, &bp->hresp_err_bh_work); - netdev_err(dev, "DMA bus error: HRESP not OK\n"); + if (unlikely(status & MACB_INT_MISC_FLAGS)) + if (macb_interrupt_misc(queue, status)) + break; - if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) - queue_writel(queue, ISR, MACB_BIT(HRESP)); - } status = queue_readl(queue, ISR); } @@ -2442,6 +2558,7 @@ static netdev_tx_t macb_start_xmit(struct sk_buff *skb, struct net_device *dev) skb->len); spin_lock(&bp->lock); + macb_tx_lpi_wake(bp); macb_writel(bp, NCR, macb_readl(bp, NCR) | MACB_BIT(TSTART)); spin_unlock(&bp->lock); @@ -2459,7 +2576,7 @@ static void macb_init_rx_buffer_size(struct macb *bp, size_t size) if (!macb_is_gem(bp)) { bp->rx_buffer_size = MACB_RX_BUFFER_SIZE; } else { - bp->rx_buffer_size = size; + bp->rx_buffer_size = MIN(size, RX_BUFFER_MAX); if (bp->rx_buffer_size % RX_BUFFER_MULTIPLE) { netdev_dbg(bp->dev, @@ -2746,8 +2863,7 @@ static void macb_reset_hw(struct macb *bp) for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { queue_writel(queue, IDR, -1); queue_readl(queue, ISR); - if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) - queue_writel(queue, ISR, -1); + macb_queue_isr_clear(bp, queue, -1); } } @@ -3107,6 +3223,8 @@ static int macb_close(struct net_device *dev) netdev_tx_reset_queue(netdev_get_tx_queue(dev, q)); } + cancel_delayed_work_sync(&bp->tx_lpi_work); + phylink_stop(bp->phylink); phylink_disconnect_phy(bp->phylink); @@ -3242,7 +3360,6 @@ static int gem_get_sset_count(struct net_device *dev, int sset) static void gem_get_ethtool_strings(struct net_device *dev, u32 sset, u8 *p) { - char stat_string[ETH_GSTRING_LEN]; struct macb *bp = netdev_priv(dev); struct macb_queue *queue; unsigned int i; @@ -3255,11 +3372,8 @@ static void gem_get_ethtool_strings(struct net_device *dev, u32 sset, u8 *p) ETH_GSTRING_LEN); for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) { - for (i = 0; i < QUEUE_STATS_LEN; i++, p += ETH_GSTRING_LEN) { - snprintf(stat_string, ETH_GSTRING_LEN, "q%d_%s", - q, queue_statistics[i].stat_string); - memcpy(p, stat_string, ETH_GSTRING_LEN); - } + for (i = 0; i < QUEUE_STATS_LEN; i++) + ethtool_sprintf(&p, "q%u_%s", q, queue_statistics[i].stat_string); } break; } @@ -3624,15 +3738,14 @@ static unsigned int gem_get_tsu_rate(struct macb *bp) struct clk *tsu_clk; unsigned int tsu_rate; - tsu_clk = devm_clk_get(&bp->pdev->dev, "tsu_clk"); - if (!IS_ERR(tsu_clk)) - tsu_rate = clk_get_rate(tsu_clk); - /* try pclk instead */ - else if (!IS_ERR(bp->pclk)) { + if (!IS_ERR_OR_NULL(bp->tsu_clk)) { + tsu_rate = clk_get_rate(bp->tsu_clk); + } else { tsu_clk = bp->pclk; tsu_rate = clk_get_rate(tsu_clk); - } else - return -ENOTSUPP; + dev_warn(&bp->pdev->dev, "devicetree missing tsu_clk, using pclk as fallback\n"); + } + return tsu_rate; } @@ -4024,6 +4137,20 @@ static const struct ethtool_ops macb_ethtool_ops = { .set_ringparam = macb_set_ringparam, }; +static int macb_get_eee(struct net_device *dev, struct ethtool_keee *eee) +{ + struct macb *bp = netdev_priv(dev); + + return phylink_ethtool_get_eee(bp->phylink, eee); +} + +static int macb_set_eee(struct net_device *dev, struct ethtool_keee *eee) +{ + struct macb *bp = netdev_priv(dev); + + return phylink_ethtool_set_eee(bp->phylink, eee); +} + static const struct ethtool_ops gem_ethtool_ops = { .get_regs_len = macb_get_regs_len, .get_regs = macb_get_regs, @@ -4045,6 +4172,9 @@ static const struct ethtool_ops gem_ethtool_ops = { .get_rxnfc = gem_get_rxnfc, .set_rxnfc = gem_set_rxnfc, .get_rx_ring_count = gem_get_rx_ring_count, + .nway_reset = phy_ethtool_nway_reset, + .get_eee = macb_get_eee, + .set_eee = macb_set_eee, }; static int macb_ioctl(struct net_device *dev, struct ifreq *rq, int cmd) @@ -4418,14 +4548,12 @@ static const struct net_device_ops macb_netdev_ops = { static void macb_configure_caps(struct macb *bp, const struct macb_config *dt_conf) { - struct device_node *np = bp->pdev->dev.of_node; - bool refclk_ext; u32 dcfg; - refclk_ext = of_property_read_bool(np, "cdns,refclk-ext"); + bp->caps = dt_conf->caps; - if (dt_conf) - bp->caps = dt_conf->caps; + if (!dt_conf->usrio) + bp->caps |= MACB_CAPS_USRIO_DISABLED; if (hw_is_gem(bp->regs, bp->native_io)) { bp->caps |= MACB_CAPS_MACB_IS_GEM; @@ -4435,6 +4563,8 @@ static void macb_configure_caps(struct macb *bp, bp->caps |= MACB_CAPS_ISR_CLEAR_ON_WRITE; if (GEM_BFEXT(NO_PCS, dcfg) == 0) bp->caps |= MACB_CAPS_PCS; + if (!(dcfg & GEM_BIT(USERIO))) + bp->caps |= MACB_CAPS_USRIO_DISABLED; dcfg = gem_readl(bp, DCFG12); if (GEM_BFEXT(HIGH_SPEED, dcfg) == 1) bp->caps |= MACB_CAPS_HIGH_SPEED; @@ -4456,9 +4586,6 @@ static void macb_configure_caps(struct macb *bp, } } - if (refclk_ext) - bp->caps |= MACB_CAPS_USRIO_HAS_CLKEN; - dev_dbg(&bp->pdev->dev, "Cadence caps 0x%08x\n", bp->caps); } @@ -4497,9 +4624,9 @@ static void macb_clks_disable(struct clk *pclk, struct clk *hclk, struct clk *tx clk_bulk_disable_unprepare(ARRAY_SIZE(clks), clks); } -static int macb_clk_init(struct platform_device *pdev, struct clk **pclk, - struct clk **hclk, struct clk **tx_clk, - struct clk **rx_clk, struct clk **tsu_clk) +static int macb_clk_init_dflt(struct platform_device *pdev, struct clk **pclk, + struct clk **hclk, struct clk **tx_clk, + struct clk **rx_clk, struct clk **tsu_clk) { struct macb_platform_data *pdata; int err; @@ -4582,7 +4709,20 @@ err_disable_pclk: return err; } -static int macb_init(struct platform_device *pdev) +static int macb_clk_init(struct platform_device *pdev, struct clk **pclk, + struct clk **hclk, struct clk **tx_clk, + struct clk **rx_clk, struct clk **tsu_clk, + const struct macb_config *config) +{ + if (config->clk_init) + return config->clk_init(pdev, pclk, hclk, tx_clk, rx_clk, + tsu_clk); + else + return macb_clk_init_dflt(pdev, pclk, hclk, tx_clk, rx_clk, + tsu_clk); +} + +static int macb_init_dflt(struct platform_device *pdev) { struct net_device *dev = platform_get_drvdata(pdev); unsigned int hw_q, q; @@ -4603,7 +4743,7 @@ static int macb_init(struct platform_device *pdev) queue->bp = bp; spin_lock_init(&queue->tx_ptr_lock); netif_napi_add(dev, &queue->napi_rx, macb_rx_poll); - netif_napi_add(dev, &queue->napi_tx, macb_tx_poll); + netif_napi_add_tx(dev, &queue->napi_tx, macb_tx_poll); if (hw_q) { queue->ISR = GEM_ISR(hw_q - 1); queue->IER = GEM_IER(hw_q - 1); @@ -4713,16 +4853,52 @@ static int macb_init(struct platform_device *pdev) if (!(bp->caps & MACB_CAPS_USRIO_DISABLED)) { val = 0; - if (phy_interface_mode_is_rgmii(bp->phy_interface)) - val = bp->usrio->rgmii; - else if (bp->phy_interface == PHY_INTERFACE_MODE_RMII && - (bp->caps & MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII)) - val = bp->usrio->rmii; - else if (!(bp->caps & MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII)) - val = bp->usrio->mii; + if (bp->caps & MACB_CAPS_USRIO_HAS_MII) { + if (phy_interface_mode_is_rgmii(bp->phy_interface)) + val = bp->usrio->rgmii; + else if (bp->phy_interface == PHY_INTERFACE_MODE_RMII && + (bp->caps & MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII)) + val = bp->usrio->rmii; + else if (!(bp->caps & MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII)) + val = bp->usrio->mii; + } if (bp->caps & MACB_CAPS_USRIO_HAS_CLKEN) - val |= bp->usrio->refclk; + val |= bp->usrio->clken; + + if (bp->caps & MACB_CAPS_USRIO_HAS_REFCLK_SOURCE) { + const char *prop; + bool refclk_ext; + int ret; + + /* Default to whatever was set in the match data for + * this device. There's two properties for refclk + * control, but the boolean one is deprecated so is + * a lower priority to check, no device should have + * both. + */ + refclk_ext = bp->usrio->refclk_default_external; + + ret = of_property_read_string(pdev->dev.of_node, + "cdns,refclk-source", &prop); + if (!ret) { + if (!strcmp(prop, "external")) + refclk_ext = true; + else + refclk_ext = false; + } else { + ret = of_property_read_bool(pdev->dev.of_node, + "cdns,refclk-ext"); + if (ret) + refclk_ext = true; + } + + if (refclk_ext) + val |= bp->usrio->refclk; + } + + if (bp->caps & MACB_CAPS_USRIO_HAS_TSUCLK_SOURCE) + val |= bp->usrio->tsu_source; macb_or_gem_writel(bp, USRIO, val); } @@ -4737,11 +4913,20 @@ static int macb_init(struct platform_device *pdev) return 0; } -static const struct macb_usrio_config macb_default_usrio = { +static int macb_init(struct platform_device *pdev, + const struct macb_config *config) +{ + if (config->init) + return config->init(pdev); + else + return macb_init_dflt(pdev); +} + +static const struct macb_usrio_config at91_default_usrio = { .mii = MACB_BIT(MII), .rmii = MACB_BIT(RMII), .rgmii = GEM_BIT(RGMII), - .refclk = MACB_BIT(CLKEN), + .clken = MACB_BIT(CLKEN), }; #if defined(CONFIG_OF) @@ -5190,7 +5375,7 @@ static int fu540_c000_clk_init(struct platform_device *pdev, struct clk **pclk, struct clk_init_data init; int err = 0; - err = macb_clk_init(pdev, pclk, hclk, tx_clk, rx_clk, tsu_clk); + err = macb_clk_init_dflt(pdev, pclk, hclk, tx_clk, rx_clk, tsu_clk); if (err) return err; @@ -5237,7 +5422,7 @@ static int fu540_c000_init(struct platform_device *pdev) if (IS_ERR(mgmt->reg)) return PTR_ERR(mgmt->reg); - return macb_init(pdev); + return macb_init_dflt(pdev); } static int init_reset_optional(struct platform_device *pdev) @@ -5287,7 +5472,7 @@ static int init_reset_optional(struct platform_device *pdev) return dev_err_probe(&pdev->dev, ret, "failed to reset controller"); } - ret = macb_init(pdev); + ret = macb_init_dflt(pdev); err_out_phy_exit: if (ret) @@ -5312,130 +5497,134 @@ static int eyeq5_init(struct platform_device *pdev) if (ret) return dev_err_probe(dev, ret, "failed to init PHY\n"); - ret = macb_init(pdev); + ret = macb_init_dflt(pdev); if (ret) phy_exit(bp->phy); return ret; } -static const struct macb_usrio_config sama7g5_usrio = { +static const struct macb_usrio_config mpfs_usrio = { + .tsu_source = 0, +}; + +static const struct macb_usrio_config sama7g5_gem_usrio = { + .mii = 0, + .rmii = 1, + .rgmii = 2, + .refclk = BIT(2), + .refclk_default_external = false, + .hdfctlen = BIT(6), +}; + +static const struct macb_usrio_config sama7g5_emac_usrio = { .mii = 0, .rmii = 1, .rgmii = 2, .refclk = BIT(2), + .refclk_default_external = true, .hdfctlen = BIT(6), }; static const struct macb_config fu540_c000_config = { .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | MACB_CAPS_JUMBO | - MACB_CAPS_GEM_HAS_PTP, + MACB_CAPS_GEM_HAS_PTP | MACB_CAPS_USRIO_HAS_MII, .dma_burst_length = 16, .clk_init = fu540_c000_clk_init, .init = fu540_c000_init, .jumbo_max_len = 10240, - .usrio = &macb_default_usrio, + .usrio = &at91_default_usrio, }; static const struct macb_config at91sam9260_config = { - .caps = MACB_CAPS_USRIO_HAS_CLKEN | MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII, - .clk_init = macb_clk_init, - .init = macb_init, - .usrio = &macb_default_usrio, + .caps = MACB_CAPS_USRIO_HAS_CLKEN | MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII | + MACB_CAPS_USRIO_HAS_MII, + .usrio = &at91_default_usrio, }; static const struct macb_config sama5d3macb_config = { .caps = MACB_CAPS_SG_DISABLED | - MACB_CAPS_USRIO_HAS_CLKEN | MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII, - .clk_init = macb_clk_init, - .init = macb_init, - .usrio = &macb_default_usrio, + MACB_CAPS_USRIO_HAS_CLKEN | MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII | + MACB_CAPS_USRIO_HAS_MII, + .usrio = &at91_default_usrio, }; static const struct macb_config pc302gem_config = { - .caps = MACB_CAPS_SG_DISABLED | MACB_CAPS_GIGABIT_MODE_AVAILABLE, + .caps = MACB_CAPS_SG_DISABLED | MACB_CAPS_GIGABIT_MODE_AVAILABLE | + MACB_CAPS_USRIO_HAS_MII, .dma_burst_length = 16, - .clk_init = macb_clk_init, - .init = macb_init, - .usrio = &macb_default_usrio, + .usrio = &at91_default_usrio, }; static const struct macb_config sama5d2_config = { - .caps = MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII | MACB_CAPS_JUMBO, + .caps = MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII | MACB_CAPS_JUMBO | + MACB_CAPS_USRIO_HAS_MII, .dma_burst_length = 16, - .clk_init = macb_clk_init, - .init = macb_init, .jumbo_max_len = 10240, - .usrio = &macb_default_usrio, + .usrio = &at91_default_usrio, }; static const struct macb_config sama5d29_config = { - .caps = MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII | MACB_CAPS_GEM_HAS_PTP, + .caps = MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII | MACB_CAPS_GEM_HAS_PTP | + MACB_CAPS_USRIO_HAS_MII, .dma_burst_length = 16, - .clk_init = macb_clk_init, - .init = macb_init, - .usrio = &macb_default_usrio, + .usrio = &at91_default_usrio, }; static const struct macb_config sama5d3_config = { .caps = MACB_CAPS_SG_DISABLED | MACB_CAPS_GIGABIT_MODE_AVAILABLE | - MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII | MACB_CAPS_JUMBO, + MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII | MACB_CAPS_JUMBO | + MACB_CAPS_USRIO_HAS_MII, .dma_burst_length = 16, - .clk_init = macb_clk_init, - .init = macb_init, .jumbo_max_len = 10240, - .usrio = &macb_default_usrio, + .usrio = &at91_default_usrio, }; static const struct macb_config sama5d4_config = { - .caps = MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII, + .caps = MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII | + MACB_CAPS_USRIO_HAS_MII, .dma_burst_length = 4, - .clk_init = macb_clk_init, - .init = macb_init, - .usrio = &macb_default_usrio, + .usrio = &at91_default_usrio, }; static const struct macb_config emac_config = { - .caps = MACB_CAPS_NEEDS_RSTONUBR | MACB_CAPS_MACB_IS_EMAC, + .caps = MACB_CAPS_NEEDS_RSTONUBR | MACB_CAPS_MACB_IS_EMAC | + MACB_CAPS_USRIO_HAS_MII, .clk_init = at91ether_clk_init, .init = at91ether_init, - .usrio = &macb_default_usrio, + .usrio = &at91_default_usrio, }; static const struct macb_config np4_config = { .caps = MACB_CAPS_USRIO_DISABLED, - .clk_init = macb_clk_init, - .init = macb_init, - .usrio = &macb_default_usrio, }; static const struct macb_config zynqmp_config = { .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | MACB_CAPS_JUMBO | - MACB_CAPS_GEM_HAS_PTP | MACB_CAPS_BD_RD_PREFETCH, + MACB_CAPS_GEM_HAS_PTP | MACB_CAPS_BD_RD_PREFETCH | + MACB_CAPS_USRIO_HAS_MII, .dma_burst_length = 16, - .clk_init = macb_clk_init, .init = init_reset_optional, .jumbo_max_len = 10240, - .usrio = &macb_default_usrio, + .usrio = &at91_default_usrio, }; static const struct macb_config zynq_config = { .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | MACB_CAPS_NO_GIGABIT_HALF | - MACB_CAPS_NEEDS_RSTONUBR, + MACB_CAPS_NEEDS_RSTONUBR | + MACB_CAPS_USRIO_HAS_MII, .dma_burst_length = 16, - .clk_init = macb_clk_init, - .init = macb_init, - .usrio = &macb_default_usrio, + .usrio = &at91_default_usrio, }; static const struct macb_config mpfs_config = { .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | MACB_CAPS_JUMBO | - MACB_CAPS_GEM_HAS_PTP, + MACB_CAPS_GEM_HAS_PTP | + MACB_CAPS_USRIO_HAS_TSUCLK_SOURCE, .dma_burst_length = 16, - .clk_init = macb_clk_init, .init = init_reset_optional, - .usrio = &macb_default_usrio, + .usrio = &mpfs_usrio, .max_tx_length = 4040, /* Cadence Erratum 1686 */ .jumbo_max_len = 4040, }; @@ -5443,57 +5632,63 @@ static const struct macb_config mpfs_config = { static const struct macb_config sama7g5_gem_config = { .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | MACB_CAPS_CLK_HW_CHG | MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII | - MACB_CAPS_MIIONRGMII | MACB_CAPS_GEM_HAS_PTP, + MACB_CAPS_USRIO_HAS_REFCLK_SOURCE | + MACB_CAPS_MIIONRGMII | MACB_CAPS_GEM_HAS_PTP | + MACB_CAPS_USRIO_HAS_MII, .dma_burst_length = 16, - .clk_init = macb_clk_init, - .init = macb_init, - .usrio = &sama7g5_usrio, + .usrio = &sama7g5_gem_usrio, }; static const struct macb_config sama7g5_emac_config = { .caps = MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII | - MACB_CAPS_USRIO_HAS_CLKEN | MACB_CAPS_MIIONRGMII | - MACB_CAPS_GEM_HAS_PTP, + MACB_CAPS_MIIONRGMII | + MACB_CAPS_USRIO_HAS_REFCLK_SOURCE | + MACB_CAPS_GEM_HAS_PTP | + MACB_CAPS_USRIO_HAS_MII, .dma_burst_length = 16, - .clk_init = macb_clk_init, - .init = macb_init, - .usrio = &sama7g5_usrio, + .usrio = &sama7g5_emac_usrio, }; static const struct macb_config versal_config = { .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | MACB_CAPS_JUMBO | MACB_CAPS_GEM_HAS_PTP | MACB_CAPS_BD_RD_PREFETCH | MACB_CAPS_NEED_TSUCLK | MACB_CAPS_QUEUE_DISABLE | - MACB_CAPS_QBV, + MACB_CAPS_QBV | + MACB_CAPS_USRIO_HAS_MII, .dma_burst_length = 16, - .clk_init = macb_clk_init, .init = init_reset_optional, .jumbo_max_len = 10240, - .usrio = &macb_default_usrio, + .usrio = &at91_default_usrio, }; static const struct macb_config eyeq5_config = { .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | MACB_CAPS_JUMBO | MACB_CAPS_GEM_HAS_PTP | MACB_CAPS_QUEUE_DISABLE | - MACB_CAPS_NO_LSO, + MACB_CAPS_NO_LSO | MACB_CAPS_EEE, .dma_burst_length = 16, - .clk_init = macb_clk_init, .init = eyeq5_init, .jumbo_max_len = 10240, - .usrio = &macb_default_usrio, }; static const struct macb_config raspberrypi_rp1_config = { .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | MACB_CAPS_CLK_HW_CHG | MACB_CAPS_JUMBO | - MACB_CAPS_GEM_HAS_PTP, + MACB_CAPS_GEM_HAS_PTP | + MACB_CAPS_EEE | + MACB_CAPS_USRIO_HAS_MII, .dma_burst_length = 16, - .clk_init = macb_clk_init, - .init = macb_init, - .usrio = &macb_default_usrio, + .usrio = &at91_default_usrio, .jumbo_max_len = 10240, }; +static const struct macb_config pic64hpsc_config = { + .caps = MACB_CAPS_GIGABIT_MODE_AVAILABLE | MACB_CAPS_JUMBO | + MACB_CAPS_GEM_HAS_PTP | MACB_CAPS_USRIO_DISABLED, + .dma_burst_length = 16, + .init = init_reset_optional, + .jumbo_max_len = 16383, +}; + static const struct of_device_id macb_dt_ids[] = { { .compatible = "cdns,at91sam9260-macb", .data = &at91sam9260_config }, { .compatible = "cdns,macb" }, @@ -5512,6 +5707,7 @@ static const struct of_device_id macb_dt_ids[] = { { .compatible = "cdns,zynq-gem", .data = &zynq_config }, /* deprecated */ { .compatible = "sifive,fu540-c000-gem", .data = &fu540_c000_config }, { .compatible = "microchip,mpfs-macb", .data = &mpfs_config }, + { .compatible = "microchip,pic64hpsc-gem", .data = &pic64hpsc_config}, { .compatible = "microchip,sama7g5-gem", .data = &sama7g5_gem_config }, { .compatible = "microchip,sama7g5-emac", .data = &sama7g5_emac_config }, { .compatible = "mobileye,eyeq5-gem", .data = &eyeq5_config }, @@ -5529,9 +5725,7 @@ static const struct macb_config default_gem_config = { MACB_CAPS_JUMBO | MACB_CAPS_GEM_HAS_PTP, .dma_burst_length = 16, - .clk_init = macb_clk_init, - .init = macb_init, - .usrio = &macb_default_usrio, + .usrio = &at91_default_usrio, .jumbo_max_len = 10240, }; @@ -5559,7 +5753,8 @@ static int macb_probe(struct platform_device *pdev) if (!macb_config) macb_config = &default_gem_config; - err = macb_config->clk_init(pdev, &pclk, &hclk, &tx_clk, &rx_clk, &tsu_clk); + err = macb_clk_init(pdev, &pclk, &hclk, &tx_clk, &rx_clk, &tsu_clk, + macb_config); if (err) return err; @@ -5619,6 +5814,13 @@ static int macb_probe(struct platform_device *pdev) bp->usrio = macb_config->usrio; + if (of_property_read_bool(bp->pdev->dev.of_node, "cdns,timer-adjust") && + IS_ENABLED(CONFIG_MACB_USE_HWSTAMP)) { + dev_err(&pdev->dev, "Timer adjust mode is not supported\n"); + err = -EINVAL; + goto err_out_free_netdev; + } + /* By default we set to partial store and forward mode for zynqmp. * Disable if not set in devicetree. */ @@ -5665,7 +5867,8 @@ static int macb_probe(struct platform_device *pdev) /* MTU range: 68 - 1518 or 10240 */ dev->min_mtu = GEM_MTU_MIN_SIZE; if ((bp->caps & MACB_CAPS_JUMBO) && bp->jumbo_max_len) - dev->max_mtu = bp->jumbo_max_len - ETH_HLEN - ETH_FCS_LEN; + dev->max_mtu = MIN(bp->jumbo_max_len, RX_BUFFER_MAX) - + ETH_HLEN - ETH_FCS_LEN; else dev->max_mtu = 1536 - ETH_HLEN - ETH_FCS_LEN; @@ -5699,7 +5902,7 @@ static int macb_probe(struct platform_device *pdev) bp->phy_interface = interface; /* IP specific init */ - err = macb_config->init(pdev); + err = macb_init(pdev, macb_config); if (err) goto err_out_free_netdev; @@ -5716,6 +5919,7 @@ static int macb_probe(struct platform_device *pdev) } INIT_WORK(&bp->hresp_err_bh_work, macb_hresp_error_task); + INIT_DELAYED_WORK(&bp->tx_lpi_work, macb_tx_lpi_work_fn); netdev_info(dev, "Cadence %s rev 0x%08x at 0x%08lx irq %d (%pM)\n", macb_is_gem(bp) ? "GEM" : "MACB", macb_readl(bp, MID), @@ -5759,6 +5963,7 @@ static void macb_remove(struct platform_device *pdev) mdiobus_free(bp->mii_bus); device_set_wakeup_enable(&bp->pdev->dev, 0); + cancel_delayed_work_sync(&bp->tx_lpi_work); cancel_work_sync(&bp->hresp_err_bh_work); pm_runtime_disable(&pdev->dev); pm_runtime_dont_use_autosuspend(&pdev->dev); @@ -5778,7 +5983,6 @@ static int __maybe_unused macb_suspend(struct device *dev) unsigned long flags; u32 tmp, ifa_local; unsigned int q; - int err; if (!device_may_wakeup(&bp->dev->dev)) phy_exit(bp->phy); @@ -5827,8 +6031,7 @@ static int __maybe_unused macb_suspend(struct device *dev) /* Disable all interrupts */ queue_writel(queue, IDR, -1); queue_readl(queue, ISR); - if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) - queue_writel(queue, ISR, -1); + macb_queue_isr_clear(bp, queue, -1); } /* Enable Receive engine */ macb_writel(bp, NCR, tmp | MACB_BIT(RE)); @@ -5842,39 +6045,15 @@ static int __maybe_unused macb_suspend(struct device *dev) /* write IP address into register */ tmp |= MACB_BFEXT(IP, ifa_local); } - spin_unlock_irqrestore(&bp->lock, flags); - /* Change interrupt handler and - * Enable WoL IRQ on queue 0 - */ - devm_free_irq(dev, bp->queues[0].irq, bp->queues); if (macb_is_gem(bp)) { - err = devm_request_irq(dev, bp->queues[0].irq, gem_wol_interrupt, - IRQF_SHARED, netdev->name, bp->queues); - if (err) { - dev_err(dev, - "Unable to request IRQ %d (error %d)\n", - bp->queues[0].irq, err); - return err; - } - spin_lock_irqsave(&bp->lock, flags); queue_writel(bp->queues, IER, GEM_BIT(WOL)); gem_writel(bp, WOL, tmp); - spin_unlock_irqrestore(&bp->lock, flags); } else { - err = devm_request_irq(dev, bp->queues[0].irq, macb_wol_interrupt, - IRQF_SHARED, netdev->name, bp->queues); - if (err) { - dev_err(dev, - "Unable to request IRQ %d (error %d)\n", - bp->queues[0].irq, err); - return err; - } - spin_lock_irqsave(&bp->lock, flags); queue_writel(bp->queues, IER, MACB_BIT(WOL)); macb_writel(bp, WOL, tmp); - spin_unlock_irqrestore(&bp->lock, flags); } + spin_unlock_irqrestore(&bp->lock, flags); enable_irq_wake(bp->queues[0].irq); } @@ -5916,7 +6095,6 @@ static int __maybe_unused macb_resume(struct device *dev) struct macb_queue *queue; unsigned long flags; unsigned int q; - int err; if (!device_may_wakeup(&bp->dev->dev)) phy_init(bp->phy); @@ -5939,21 +6117,9 @@ static int __maybe_unused macb_resume(struct device *dev) } /* Clear ISR on queue 0 */ queue_readl(bp->queues, ISR); - if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE) - queue_writel(bp->queues, ISR, -1); + macb_queue_isr_clear(bp, bp->queues, -1); spin_unlock_irqrestore(&bp->lock, flags); - /* Replace interrupt handler on queue 0 */ - devm_free_irq(dev, bp->queues[0].irq, bp->queues); - err = devm_request_irq(dev, bp->queues[0].irq, macb_interrupt, - IRQF_SHARED, netdev->name, bp->queues); - if (err) { - dev_err(dev, - "Unable to request IRQ %d (error %d)\n", - bp->queues[0].irq, err); - return err; - } - disable_irq_wake(bp->queues[0].irq); /* Now make sure we disable phy before moving diff --git a/drivers/net/ethernet/chelsio/Kconfig b/drivers/net/ethernet/chelsio/Kconfig index c931ec8cac40..96d7779cd2f0 100644 --- a/drivers/net/ethernet/chelsio/Kconfig +++ b/drivers/net/ethernet/chelsio/Kconfig @@ -68,7 +68,7 @@ config CHELSIO_T3 config CHELSIO_T4 tristate "Chelsio Communications T4/T5/T6 Ethernet support" - depends on PCI && (IPV6 || IPV6=n) && (TLS || TLS=n) + depends on PCI && (TLS || TLS=n) depends on PTP_1588_CLOCK_OPTIONAL select FW_LOADER select MDIO diff --git a/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls.h b/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls.h index 21e0dfeff158..1de5744a49b0 100644 --- a/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls.h +++ b/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls.h @@ -567,7 +567,7 @@ void chtls_shutdown(struct sock *sk, int how); void chtls_destroy_sock(struct sock *sk); int chtls_sendmsg(struct sock *sk, struct msghdr *msg, size_t size); int chtls_recvmsg(struct sock *sk, struct msghdr *msg, - size_t len, int flags, int *addr_len); + size_t len, int flags); void chtls_splice_eof(struct socket *sock); int send_tx_flowc_wr(struct sock *sk, int compl, u32 snd_nxt, u32 rcv_nxt); diff --git a/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_io.c b/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_io.c index ee19933e2cca..c8e99409a52a 100644 --- a/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_io.c +++ b/drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_io.c @@ -1332,7 +1332,7 @@ static void chtls_cleanup_rbuf(struct sock *sk, int copied) } static int chtls_pt_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, - int flags, int *addr_len) + int flags) { struct chtls_sock *csk = rcu_dereference_sk_user_data(sk); struct chtls_hws *hws = &csk->tlshws; @@ -1656,7 +1656,7 @@ found_ok_skb: } int chtls_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, - int flags, int *addr_len) + int flags) { struct tcp_sock *tp = tcp_sk(sk); struct chtls_sock *csk; @@ -1670,7 +1670,7 @@ int chtls_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, buffers_freed = 0; if (unlikely(flags & MSG_OOB)) - return tcp_prot.recvmsg(sk, msg, len, flags, addr_len); + return tcp_prot.recvmsg(sk, msg, len, flags); if (unlikely(flags & MSG_PEEK)) return peekmsg(sk, msg, len, flags); @@ -1684,7 +1684,7 @@ int chtls_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, csk = rcu_dereference_sk_user_data(sk); if (is_tls_rx(csk)) - return chtls_pt_recvmsg(sk, msg, len, flags, addr_len); + return chtls_pt_recvmsg(sk, msg, len, flags); timeo = sock_rcvtimeo(sk, flags & MSG_DONTWAIT); target = sock_rcvlowat(sk, flags & MSG_WAITALL, len); diff --git a/drivers/net/ethernet/cisco/enic/enic.h b/drivers/net/ethernet/cisco/enic/enic.h index 301b3f3114af..08472420f3a1 100644 --- a/drivers/net/ethernet/cisco/enic/enic.h +++ b/drivers/net/ethernet/cisco/enic/enic.h @@ -22,6 +22,45 @@ #define DRV_NAME "enic" #define DRV_DESCRIPTION "Cisco VIC Ethernet NIC Driver" +#define PCI_SUBDEV_ID_CISCO_VIC_1225 0x085 +#define PCI_SUBDEV_ID_CISCO_VIC_1225T 0x0CE +#define PCI_SUBDEV_ID_CISCO_VIC_1227 0x12E +#define PCI_SUBDEV_ID_CISCO_VIC_1227T 0x139 +#define PCI_SUBDEV_ID_CISCO_VIC_1240 0x084 +#define PCI_SUBDEV_ID_CISCO_VIC_1280 0x04F +#define PCI_SUBDEV_ID_CISCO_VIC_1285 0x0CD + +#define PCI_SUBDEV_ID_CISCO_VIC_1340 0x12C +#define PCI_SUBDEV_ID_CISCO_VIC_1380 0x137 +#define PCI_SUBDEV_ID_CISCO_VIC_1385 0x14D +#define PCI_SUBDEV_ID_CISCO_VIC_1387 0x15D + +#define PCI_SUBDEV_ID_CISCO_VIC_1440 0x0215 +#define PCI_SUBDEV_ID_CISCO_VIC_1455 0x0217 +#define PCI_SUBDEV_ID_CISCO_VIC_1457 0x0218 +#define PCI_SUBDEV_ID_CISCO_VIC_1467 0x02AF +#define PCI_SUBDEV_ID_CISCO_VIC_1477 0x2B0 +#define PCI_SUBDEV_ID_CISCO_VIC_1480 0x0216 +#define PCI_SUBDEV_ID_CISCO_VIC_1485 0x0219 +#define PCI_SUBDEV_ID_CISCO_VIC_1487 0x021A +#define PCI_SUBDEV_ID_CISCO_VIC_1495 0x024A +#define PCI_SUBDEV_ID_CISCO_VIC_1497 0x024B +#define PCI_SUBDEV_ID_CISCO_VIC_14425 0x02CF +#define PCI_SUBDEV_ID_CISCO_VIC_14825 0x02D0 + +#define PCI_SUBDEV_ID_CISCO_VIC_15230 0x02DF +#define PCI_SUBDEV_ID_CISCO_VIC_15231 0x02DB +#define PCI_SUBDEV_ID_CISCO_VIC_15235 0x02E4 +#define PCI_SUBDEV_ID_CISCO_VIC_15237 0x02F3 +#define PCI_SUBDEV_ID_CISCO_VIC_15238 0x02E8 +#define PCI_SUBDEV_ID_CISCO_VIC_15411 0x02DC +#define PCI_SUBDEV_ID_CISCO_VIC_15412 0x02E2 +#define PCI_SUBDEV_ID_CISCO_VIC_15420 0x02DE +#define PCI_SUBDEV_ID_CISCO_VIC_15422 0x02E1 +#define PCI_SUBDEV_ID_CISCO_VIC_15425 0x02F2 +#define PCI_SUBDEV_ID_CISCO_VIC_15427 0x02E0 +#define PCI_SUBDEV_ID_CISCO_VIC_15428 0x02DD + #define ENIC_BARS_MAX 6 #define ENIC_WQ_MAX 256 @@ -186,6 +225,13 @@ struct enic_rq { struct page_pool *pool; } ____cacheline_aligned; +enum enic_vf_type { + ENIC_VF_TYPE_NONE, + ENIC_VF_TYPE_V1, + ENIC_VF_TYPE_USNIC, + ENIC_VF_TYPE_V2, +}; + /* Per-instance private data structure */ struct enic { struct net_device *netdev; @@ -213,6 +259,8 @@ struct enic { #ifdef CONFIG_PCI_IOV u16 num_vfs; #endif + enum enic_vf_type vf_type; + unsigned int enable_count; spinlock_t enic_api_lock; bool enic_api_busy; struct enic_port_profile *pp; @@ -241,6 +289,13 @@ struct enic { u8 rss_key[ENIC_RSS_LEN]; struct vnic_gen_stats gen_stats; enum ext_cq ext_cq; + + /* Admin channel resources for SR-IOV MBOX */ + bool has_admin_channel; + struct vnic_wq admin_wq; + struct vnic_rq admin_rq; + struct vnic_cq admin_cq[2]; + struct vnic_intr admin_intr; }; static inline struct net_device *vnic_get_netdev(struct vnic_dev *vdev) @@ -258,6 +313,8 @@ static inline struct net_device *vnic_get_netdev(struct vnic_dev *vdev) dev_warn(&(vdev)->pdev->dev, fmt, ##__VA_ARGS__) #define vdev_info(vdev, fmt, ...) \ dev_info(&(vdev)->pdev->dev, fmt, ##__VA_ARGS__) +#define vdev_dbg(vdev, fmt, ...) \ + dev_dbg(&(vdev)->pdev->dev, fmt, ##__VA_ARGS__) #define vdev_neterr(vdev, fmt, ...) \ netdev_err(vnic_get_netdev(vdev), fmt, ##__VA_ARGS__) diff --git a/drivers/net/ethernet/cisco/enic/enic_dev.c b/drivers/net/ethernet/cisco/enic/enic_dev.c index 2cbae7c6cc3d..659787f73cf1 100644 --- a/drivers/net/ethernet/cisco/enic/enic_dev.c +++ b/drivers/net/ethernet/cisco/enic/enic_dev.c @@ -131,10 +131,13 @@ int enic_dev_set_ig_vlan_rewrite_mode(struct enic *enic) int enic_dev_enable(struct enic *enic) { - int err; + int err = 0; spin_lock_bh(&enic->devcmd_lock); - err = vnic_dev_enable_wait(enic->vdev); + if (enic->enable_count == 0) + err = vnic_dev_enable_wait(enic->vdev); + if (!err) + enic->enable_count++; spin_unlock_bh(&enic->devcmd_lock); return err; @@ -142,10 +145,16 @@ int enic_dev_enable(struct enic *enic) int enic_dev_disable(struct enic *enic) { - int err; + int err = 0; spin_lock_bh(&enic->devcmd_lock); - err = vnic_dev_disable(enic->vdev); + if (enic->enable_count == 0) { + spin_unlock_bh(&enic->devcmd_lock); + return 0; + } + enic->enable_count--; + if (enic->enable_count == 0) + err = vnic_dev_disable(enic->vdev); spin_unlock_bh(&enic->devcmd_lock); return err; diff --git a/drivers/net/ethernet/cisco/enic/enic_ethtool.c b/drivers/net/ethernet/cisco/enic/enic_ethtool.c index 471613899ec0..61541fb8db7b 100644 --- a/drivers/net/ethernet/cisco/enic/enic_ethtool.c +++ b/drivers/net/ethernet/cisco/enic/enic_ethtool.c @@ -126,19 +126,160 @@ static void enic_intr_coal_set_rx(struct enic *enic, u32 timer) } } +static void enic_get_supp_adv_media_type(struct net_device *netdev, + struct ethtool_link_ksettings *ecmd) +{ + struct enic *enic = netdev_priv(netdev); + struct ethtool_link_settings *base = &ecmd->base; + u16 sub_dev_id = 0; + + base->port = PORT_OTHER; + + if (enic->pdev) + sub_dev_id = enic->pdev->subsystem_device; + + switch (sub_dev_id) { + case PCI_SUBDEV_ID_CISCO_VIC_1225: + case PCI_SUBDEV_ID_CISCO_VIC_1227: + ethtool_link_ksettings_add_link_mode(ecmd, supported, + 10000baseSR_Full); + ethtool_link_ksettings_add_link_mode(ecmd, supported, FIBRE); + base->port = PORT_FIBRE; + break; + case PCI_SUBDEV_ID_CISCO_VIC_1285: + ethtool_link_ksettings_add_link_mode(ecmd, supported, + 40000baseSR4_Full); + ethtool_link_ksettings_add_link_mode(ecmd, supported, FIBRE); + base->port = PORT_FIBRE; + break; + case PCI_SUBDEV_ID_CISCO_VIC_1225T: + case PCI_SUBDEV_ID_CISCO_VIC_1227T: + ethtool_link_ksettings_add_link_mode(ecmd, supported, + 10000baseT_Full); + ethtool_link_ksettings_add_link_mode(ecmd, supported, FIBRE); + base->port = PORT_FIBRE; + break; + case PCI_SUBDEV_ID_CISCO_VIC_1385: + case PCI_SUBDEV_ID_CISCO_VIC_1387: + ethtool_link_ksettings_add_link_mode(ecmd, supported, + 10000baseSR_Full); + ethtool_link_ksettings_add_link_mode(ecmd, supported, + 10000baseLR_Full); + ethtool_link_ksettings_add_link_mode(ecmd, supported, + 40000baseSR4_Full); + ethtool_link_ksettings_add_link_mode(ecmd, supported, + 40000baseLR4_Full); + ethtool_link_ksettings_add_link_mode(ecmd, supported, FIBRE); + base->port = PORT_FIBRE; + break; + case PCI_SUBDEV_ID_CISCO_VIC_1477: + case PCI_SUBDEV_ID_CISCO_VIC_1485: + case PCI_SUBDEV_ID_CISCO_VIC_1487: + case PCI_SUBDEV_ID_CISCO_VIC_1495: + case PCI_SUBDEV_ID_CISCO_VIC_1497: + ethtool_link_ksettings_add_link_mode(ecmd, supported, + 40000baseCR4_Full); + ethtool_link_ksettings_add_link_mode(ecmd, supported, + 40000baseSR4_Full); + ethtool_link_ksettings_add_link_mode(ecmd, supported, + 40000baseLR4_Full); + ethtool_link_ksettings_add_link_mode(ecmd, supported, + 100000baseSR4_Full); + ethtool_link_ksettings_add_link_mode(ecmd, supported, + 100000baseCR4_Full); + ethtool_link_ksettings_add_link_mode(ecmd, supported, FIBRE); + base->port = PORT_FIBRE; + break; + case PCI_SUBDEV_ID_CISCO_VIC_15235: + case PCI_SUBDEV_ID_CISCO_VIC_15237: + case PCI_SUBDEV_ID_CISCO_VIC_15238: + ethtool_link_ksettings_add_link_mode(ecmd, supported, + 40000baseCR4_Full); + ethtool_link_ksettings_add_link_mode(ecmd, supported, + 40000baseSR4_Full); + ethtool_link_ksettings_add_link_mode(ecmd, supported, + 40000baseLR4_Full); + ethtool_link_ksettings_add_link_mode(ecmd, supported, + 100000baseSR4_Full); + ethtool_link_ksettings_add_link_mode(ecmd, supported, + 100000baseCR4_Full); + ethtool_link_ksettings_add_link_mode(ecmd, supported, + 200000baseSR4_Full); + ethtool_link_ksettings_add_link_mode(ecmd, supported, + 200000baseDR4_Full); + ethtool_link_ksettings_add_link_mode(ecmd, supported, + 200000baseLR4_ER4_FR4_Full); + ethtool_link_ksettings_add_link_mode(ecmd, supported, FIBRE); + base->port = PORT_FIBRE; + break; + case PCI_SUBDEV_ID_CISCO_VIC_1455: + case PCI_SUBDEV_ID_CISCO_VIC_1457: + case PCI_SUBDEV_ID_CISCO_VIC_1467: + ethtool_link_ksettings_add_link_mode(ecmd, supported, + 10000baseT_Full); + ethtool_link_ksettings_add_link_mode(ecmd, supported, + 25000baseSR_Full); + ethtool_link_ksettings_add_link_mode(ecmd, supported, FIBRE); + base->port = PORT_FIBRE; + break; + case PCI_SUBDEV_ID_CISCO_VIC_15428: + case PCI_SUBDEV_ID_CISCO_VIC_15427: + case PCI_SUBDEV_ID_CISCO_VIC_15425: + ethtool_link_ksettings_add_link_mode(ecmd, supported, + 10000baseT_Full); + ethtool_link_ksettings_add_link_mode(ecmd, supported, + 25000baseSR_Full); + ethtool_link_ksettings_add_link_mode(ecmd, supported, + 50000baseSR_Full); + ethtool_link_ksettings_add_link_mode(ecmd, supported, FIBRE); + base->port = PORT_FIBRE; + break; + /* Do not mention port type as FIBRE for blade VICs */ + case PCI_SUBDEV_ID_CISCO_VIC_1240: + case PCI_SUBDEV_ID_CISCO_VIC_1280: + ethtool_link_ksettings_add_link_mode(ecmd, supported, + 10000baseKR_Full); + break; + case PCI_SUBDEV_ID_CISCO_VIC_1340: + case PCI_SUBDEV_ID_CISCO_VIC_1380: + case PCI_SUBDEV_ID_CISCO_VIC_1440: /* 10G/40G KR */ + case PCI_SUBDEV_ID_CISCO_VIC_1480: /* 10G/40G KR */ + ethtool_link_ksettings_add_link_mode(ecmd, supported, + 10000baseKR_Full); + ethtool_link_ksettings_add_link_mode(ecmd, supported, + 40000baseKR4_Full); + break; + case PCI_SUBDEV_ID_CISCO_VIC_14425: /* 25G KR */ + case PCI_SUBDEV_ID_CISCO_VIC_14825: /* 25G KR */ + case PCI_SUBDEV_ID_CISCO_VIC_15420: /* 25G KR */ + case PCI_SUBDEV_ID_CISCO_VIC_15422: /* 25G KR */ + ethtool_link_ksettings_add_link_mode(ecmd, supported, + 25000baseKR_Full); + break; + case PCI_SUBDEV_ID_CISCO_VIC_15411: /* 10G KR */ + case PCI_SUBDEV_ID_CISCO_VIC_15412: /* 10G KR */ + ethtool_link_ksettings_add_link_mode(ecmd, supported, + 10000baseKR_Full); + break; + case PCI_SUBDEV_ID_CISCO_VIC_15231: /* 25G/100G/200G KR */ + case PCI_SUBDEV_ID_CISCO_VIC_15230: /* 25G/100G/200G KR */ + ethtool_link_ksettings_add_link_mode(ecmd, supported, + 25000baseKR_Full); + ethtool_link_ksettings_add_link_mode(ecmd, supported, + 100000baseKR_Full); + ethtool_link_ksettings_add_link_mode(ecmd, supported, + 200000baseKR4_Full); + break; + } +} + static int enic_get_ksettings(struct net_device *netdev, struct ethtool_link_ksettings *ecmd) { struct enic *enic = netdev_priv(netdev); struct ethtool_link_settings *base = &ecmd->base; - ethtool_link_ksettings_add_link_mode(ecmd, supported, - 10000baseT_Full); - ethtool_link_ksettings_add_link_mode(ecmd, supported, FIBRE); - ethtool_link_ksettings_add_link_mode(ecmd, advertising, - 10000baseT_Full); - ethtool_link_ksettings_add_link_mode(ecmd, advertising, FIBRE); - base->port = PORT_FIBRE; + enic_get_supp_adv_media_type(netdev, ecmd); if (netif_carrier_ok(netdev)) { base->speed = vnic_dev_port_speed(enic->vdev); diff --git a/drivers/net/ethernet/cisco/enic/enic_main.c b/drivers/net/ethernet/cisco/enic/enic_main.c index e839081f9ee4..e7125b818087 100644 --- a/drivers/net/ethernet/cisco/enic/enic_main.c +++ b/drivers/net/ethernet/cisco/enic/enic_main.c @@ -66,12 +66,15 @@ #define PCI_DEVICE_ID_CISCO_VIC_ENET 0x0043 /* ethernet vnic */ #define PCI_DEVICE_ID_CISCO_VIC_ENET_DYN 0x0044 /* enet dynamic vnic */ #define PCI_DEVICE_ID_CISCO_VIC_ENET_VF 0x0071 /* enet SRIOV VF */ +#define PCI_DEVICE_ID_CISCO_VIC_ENET_VF_V2 0x02b7 /* enet SRIOV V2 VF */ +#define PCI_DEVICE_ID_CISCO_VIC_ENET_VF_USNIC 0x00cf /* enet USNIC VF */ /* Supported devices */ static const struct pci_device_id enic_id_table[] = { { PCI_VDEVICE(CISCO, PCI_DEVICE_ID_CISCO_VIC_ENET) }, { PCI_VDEVICE(CISCO, PCI_DEVICE_ID_CISCO_VIC_ENET_DYN) }, { PCI_VDEVICE(CISCO, PCI_DEVICE_ID_CISCO_VIC_ENET_VF) }, + { PCI_VDEVICE(CISCO, PCI_DEVICE_ID_CISCO_VIC_ENET_VF_V2) }, { 0, } /* end of table */ }; @@ -307,7 +310,8 @@ int enic_sriov_enabled(struct enic *enic) static int enic_is_sriov_vf(struct enic *enic) { - return enic->pdev->device == PCI_DEVICE_ID_CISCO_VIC_ENET_VF; + return enic->pdev->device == PCI_DEVICE_ID_CISCO_VIC_ENET_VF || + enic->pdev->device == PCI_DEVICE_ID_CISCO_VIC_ENET_VF_V2; } int enic_is_valid_vf(struct enic *enic, int vf) @@ -1746,7 +1750,11 @@ static int enic_open(struct net_device *netdev) if (vnic_dev_get_intr_mode(enic->vdev) == VNIC_DEV_INTR_MODE_MSIX) for (i = 0; i < enic->wq_count; i++) napi_enable(&enic->napi[enic_cq_wq(enic, i)]); - enic_dev_enable(enic); + err = enic_dev_enable(enic); + if (err) { + netdev_err(netdev, "Failed to enable device: %d\n", err); + goto err_out_dev_enable; + } for (i = 0; i < enic->intr_count; i++) vnic_intr_unmask(&enic->intr[i]); @@ -1756,6 +1764,17 @@ static int enic_open(struct net_device *netdev) return 0; +err_out_dev_enable: + for (i = 0; i < enic->rq_count; i++) + napi_disable(&enic->napi[i]); + if (vnic_dev_get_intr_mode(enic->vdev) == VNIC_DEV_INTR_MODE_MSIX) + for (i = 0; i < enic->wq_count; i++) + napi_disable(&enic->napi[enic_cq_wq(enic, i)]); + netif_tx_disable(netdev); + if (!enic_is_dynamic(enic) && !enic_is_sriov_vf(enic)) + enic_dev_del_station_addr(enic); + for (i = 0; i < enic->wq_count; i++) + vnic_wq_disable(&enic->wq[i].vwq); err_out_free_rq: for (i = 0; i < enic->rq_count; i++) { ret = vnic_rq_disable(&enic->rq[i].vrq); @@ -2618,6 +2637,41 @@ static void enic_iounmap(struct enic *enic) iounmap(enic->bar[i].vaddr); } +#ifdef CONFIG_PCI_IOV +static void enic_sriov_detect_vf_type(struct enic *enic) +{ + struct pci_dev *pdev = enic->pdev; + int pos; + u16 vf_dev_id; + + if (enic_is_sriov_vf(enic) || enic_is_dynamic(enic)) + return; + + pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_SRIOV); + if (!pos) { + enic->vf_type = ENIC_VF_TYPE_NONE; + return; + } + + pci_read_config_word(pdev, pos + PCI_SRIOV_VF_DID, &vf_dev_id); + + switch (vf_dev_id) { + case PCI_DEVICE_ID_CISCO_VIC_ENET_VF: + enic->vf_type = ENIC_VF_TYPE_V1; + break; + case PCI_DEVICE_ID_CISCO_VIC_ENET_VF_USNIC: + enic->vf_type = ENIC_VF_TYPE_USNIC; + break; + case PCI_DEVICE_ID_CISCO_VIC_ENET_VF_V2: + enic->vf_type = ENIC_VF_TYPE_V2; + break; + default: + enic->vf_type = ENIC_VF_TYPE_NONE; + break; + } +} +#endif + static int enic_probe(struct pci_dev *pdev, const struct pci_device_id *ent) { struct device *dev = &pdev->dev; @@ -2731,6 +2785,7 @@ static int enic_probe(struct pci_dev *pdev, const struct pci_device_id *ent) num_pps = enic->num_vfs; } } + enic_sriov_detect_vf_type(enic); #endif /* Allocate structure for port profiles */ diff --git a/drivers/net/ethernet/cisco/enic/enic_res.c b/drivers/net/ethernet/cisco/enic/enic_res.c index bbd3143ed73e..2b7545d6a67f 100644 --- a/drivers/net/ethernet/cisco/enic/enic_res.c +++ b/drivers/net/ethernet/cisco/enic/enic_res.c @@ -205,10 +205,18 @@ void enic_get_res_counts(struct enic *enic) enic->cq_count = enic->cq_avail; enic->intr_count = enic->intr_avail; + enic->has_admin_channel = + vnic_dev_get_res_count(enic->vdev, RES_TYPE_ADMIN_WQ) >= 1 && + vnic_dev_get_res_count(enic->vdev, RES_TYPE_ADMIN_RQ) >= 1 && + vnic_dev_get_res_count(enic->vdev, RES_TYPE_ADMIN_CQ) >= + ARRAY_SIZE(enic->admin_cq) && + vnic_dev_get_res_count(enic->vdev, RES_TYPE_SRIOV_INTR) >= 1; + dev_info(enic_get_dev(enic), - "vNIC resources avail: wq %d rq %d cq %d intr %d\n", + "vNIC resources avail: wq %d rq %d cq %d intr %d admin %s\n", enic->wq_avail, enic->rq_avail, - enic->cq_avail, enic->intr_avail); + enic->cq_avail, enic->intr_avail, + enic->has_admin_channel ? "yes" : "no"); } void enic_init_vnic_resources(struct enic *enic) diff --git a/drivers/net/ethernet/cisco/enic/vnic_cq.c b/drivers/net/ethernet/cisco/enic/vnic_cq.c index 27c885e91552..5a0dbb816223 100644 --- a/drivers/net/ethernet/cisco/enic/vnic_cq.c +++ b/drivers/net/ethernet/cisco/enic/vnic_cq.c @@ -20,13 +20,14 @@ void vnic_cq_free(struct vnic_cq *cq) cq->ctrl = NULL; } -int vnic_cq_alloc(struct vnic_dev *vdev, struct vnic_cq *cq, unsigned int index, - unsigned int desc_count, unsigned int desc_size) +int vnic_cq_alloc_with_type(struct vnic_dev *vdev, struct vnic_cq *cq, + unsigned int index, unsigned int desc_count, + unsigned int desc_size, unsigned int res_type) { cq->index = index; cq->vdev = vdev; - cq->ctrl = vnic_dev_get_res(vdev, RES_TYPE_CQ, index); + cq->ctrl = vnic_dev_get_res(vdev, res_type, index); if (!cq->ctrl) { vdev_err(vdev, "Failed to hook CQ[%d] resource\n", index); return -EINVAL; @@ -35,6 +36,13 @@ int vnic_cq_alloc(struct vnic_dev *vdev, struct vnic_cq *cq, unsigned int index, return vnic_dev_alloc_desc_ring(vdev, &cq->ring, desc_count, desc_size); } +int vnic_cq_alloc(struct vnic_dev *vdev, struct vnic_cq *cq, unsigned int index, + unsigned int desc_count, unsigned int desc_size) +{ + return vnic_cq_alloc_with_type(vdev, cq, index, desc_count, desc_size, + RES_TYPE_CQ); +} + void vnic_cq_init(struct vnic_cq *cq, unsigned int flow_control_enable, unsigned int color_enable, unsigned int cq_head, unsigned int cq_tail, unsigned int cq_tail_color, unsigned int interrupt_enable, diff --git a/drivers/net/ethernet/cisco/enic/vnic_cq.h b/drivers/net/ethernet/cisco/enic/vnic_cq.h index 0e37f5d5e527..d46d4d2ef6bb 100644 --- a/drivers/net/ethernet/cisco/enic/vnic_cq.h +++ b/drivers/net/ethernet/cisco/enic/vnic_cq.h @@ -73,6 +73,9 @@ static inline void vnic_cq_inc_to_clean(struct vnic_cq *cq) void vnic_cq_free(struct vnic_cq *cq); int vnic_cq_alloc(struct vnic_dev *vdev, struct vnic_cq *cq, unsigned int index, unsigned int desc_count, unsigned int desc_size); +int vnic_cq_alloc_with_type(struct vnic_dev *vdev, struct vnic_cq *cq, + unsigned int index, unsigned int desc_count, + unsigned int desc_size, unsigned int res_type); void vnic_cq_init(struct vnic_cq *cq, unsigned int flow_control_enable, unsigned int color_enable, unsigned int cq_head, unsigned int cq_tail, unsigned int cq_tail_color, unsigned int interrupt_enable, diff --git a/drivers/net/ethernet/cisco/enic/vnic_dev.c b/drivers/net/ethernet/cisco/enic/vnic_dev.c index c72452749f5e..c8d657e97094 100644 --- a/drivers/net/ethernet/cisco/enic/vnic_dev.c +++ b/drivers/net/ethernet/cisco/enic/vnic_dev.c @@ -77,6 +77,9 @@ static int vnic_dev_discover_res(struct vnic_dev *vdev, u32 count = ioread32(&r->count); u32 len; + vdev_dbg(vdev, "res type %u bar %u offset 0x%x count %u\n", + type, bar_num, bar_offset, count); + r++; if (bar_num >= num_bars) @@ -90,6 +93,9 @@ static int vnic_dev_discover_res(struct vnic_dev *vdev, case RES_TYPE_RQ: case RES_TYPE_CQ: case RES_TYPE_INTR_CTRL: + case RES_TYPE_ADMIN_WQ: + case RES_TYPE_ADMIN_RQ: + case RES_TYPE_ADMIN_CQ: /* each count is stride bytes long */ len = count * VNIC_RES_STRIDE; if (len + bar_offset > bar[bar_num].len) { @@ -102,6 +108,7 @@ static int vnic_dev_discover_res(struct vnic_dev *vdev, case RES_TYPE_INTR_PBA_LEGACY: case RES_TYPE_DEVCMD: case RES_TYPE_DEVCMD2: + case RES_TYPE_SRIOV_INTR: len = count; break; default: @@ -135,6 +142,9 @@ void __iomem *vnic_dev_get_res(struct vnic_dev *vdev, enum vnic_res_type type, case RES_TYPE_RQ: case RES_TYPE_CQ: case RES_TYPE_INTR_CTRL: + case RES_TYPE_ADMIN_WQ: + case RES_TYPE_ADMIN_RQ: + case RES_TYPE_ADMIN_CQ: return (char __iomem *)vdev->res[type].vaddr + index * VNIC_RES_STRIDE; default: diff --git a/drivers/net/ethernet/cisco/enic/vnic_intr.c b/drivers/net/ethernet/cisco/enic/vnic_intr.c index 25319f072a04..010ad8c2108d 100644 --- a/drivers/net/ethernet/cisco/enic/vnic_intr.c +++ b/drivers/net/ethernet/cisco/enic/vnic_intr.c @@ -19,13 +19,13 @@ void vnic_intr_free(struct vnic_intr *intr) intr->ctrl = NULL; } -int vnic_intr_alloc(struct vnic_dev *vdev, struct vnic_intr *intr, - unsigned int index) +int vnic_intr_alloc_with_type(struct vnic_dev *vdev, struct vnic_intr *intr, + unsigned int index, unsigned int res_type) { intr->index = index; intr->vdev = vdev; - intr->ctrl = vnic_dev_get_res(vdev, RES_TYPE_INTR_CTRL, index); + intr->ctrl = vnic_dev_get_res(vdev, res_type, index); if (!intr->ctrl) { vdev_err(vdev, "Failed to hook INTR[%d].ctrl resource\n", index); @@ -35,6 +35,12 @@ int vnic_intr_alloc(struct vnic_dev *vdev, struct vnic_intr *intr, return 0; } +int vnic_intr_alloc(struct vnic_dev *vdev, struct vnic_intr *intr, + unsigned int index) +{ + return vnic_intr_alloc_with_type(vdev, intr, index, RES_TYPE_INTR_CTRL); +} + void vnic_intr_init(struct vnic_intr *intr, u32 coalescing_timer, unsigned int coalescing_type, unsigned int mask_on_assertion) { diff --git a/drivers/net/ethernet/cisco/enic/vnic_intr.h b/drivers/net/ethernet/cisco/enic/vnic_intr.h index 33a72aa10b26..2bc2dbffdb80 100644 --- a/drivers/net/ethernet/cisco/enic/vnic_intr.h +++ b/drivers/net/ethernet/cisco/enic/vnic_intr.h @@ -89,6 +89,8 @@ static inline u32 vnic_intr_legacy_pba(u32 __iomem *legacy_pba) void vnic_intr_free(struct vnic_intr *intr); int vnic_intr_alloc(struct vnic_dev *vdev, struct vnic_intr *intr, unsigned int index); +int vnic_intr_alloc_with_type(struct vnic_dev *vdev, struct vnic_intr *intr, + unsigned int index, unsigned int res_type); void vnic_intr_init(struct vnic_intr *intr, u32 coalescing_timer, unsigned int coalescing_type, unsigned int mask_on_assertion); void vnic_intr_coalescing_timer_set(struct vnic_intr *intr, diff --git a/drivers/net/ethernet/cisco/enic/vnic_resource.h b/drivers/net/ethernet/cisco/enic/vnic_resource.h index b4776e334d63..d327821fa9b9 100644 --- a/drivers/net/ethernet/cisco/enic/vnic_resource.h +++ b/drivers/net/ethernet/cisco/enic/vnic_resource.h @@ -42,6 +42,10 @@ enum vnic_res_type { RES_TYPE_DEPRECATED1, /* Old version of devcmd 2 */ RES_TYPE_DEPRECATED2, /* Old version of devcmd 2 */ RES_TYPE_DEVCMD2, /* Device control region */ + RES_TYPE_SRIOV_INTR = 45, /* SR-IOV VF interrupt */ + RES_TYPE_ADMIN_WQ = 49, /* Admin channel WQ */ + RES_TYPE_ADMIN_RQ, /* Admin channel RQ */ + RES_TYPE_ADMIN_CQ, /* Admin channel CQ */ RES_TYPE_MAX, /* Count of resource types */ }; diff --git a/drivers/net/ethernet/cisco/enic/vnic_rq.c b/drivers/net/ethernet/cisco/enic/vnic_rq.c index 5ae80551f17c..a662d9fd1199 100644 --- a/drivers/net/ethernet/cisco/enic/vnic_rq.c +++ b/drivers/net/ethernet/cisco/enic/vnic_rq.c @@ -69,15 +69,16 @@ void vnic_rq_free(struct vnic_rq *rq) rq->ctrl = NULL; } -int vnic_rq_alloc(struct vnic_dev *vdev, struct vnic_rq *rq, unsigned int index, - unsigned int desc_count, unsigned int desc_size) +int vnic_rq_alloc_with_type(struct vnic_dev *vdev, struct vnic_rq *rq, + unsigned int index, unsigned int desc_count, + unsigned int desc_size, unsigned int res_type) { int err; rq->index = index; rq->vdev = vdev; - rq->ctrl = vnic_dev_get_res(vdev, RES_TYPE_RQ, index); + rq->ctrl = vnic_dev_get_res(vdev, res_type, index); if (!rq->ctrl) { vdev_err(vdev, "Failed to hook RQ[%d] resource\n", index); return -EINVAL; @@ -98,6 +99,13 @@ int vnic_rq_alloc(struct vnic_dev *vdev, struct vnic_rq *rq, unsigned int index, return 0; } +int vnic_rq_alloc(struct vnic_dev *vdev, struct vnic_rq *rq, unsigned int index, + unsigned int desc_count, unsigned int desc_size) +{ + return vnic_rq_alloc_with_type(vdev, rq, index, desc_count, desc_size, + RES_TYPE_RQ); +} + static void vnic_rq_init_start(struct vnic_rq *rq, unsigned int cq_index, unsigned int fetch_index, unsigned int posted_index, unsigned int error_interrupt_enable, diff --git a/drivers/net/ethernet/cisco/enic/vnic_rq.h b/drivers/net/ethernet/cisco/enic/vnic_rq.h index a1cdd729caec..9fc2090eac44 100644 --- a/drivers/net/ethernet/cisco/enic/vnic_rq.h +++ b/drivers/net/ethernet/cisco/enic/vnic_rq.h @@ -196,6 +196,9 @@ static inline int vnic_rq_fill(struct vnic_rq *rq, void vnic_rq_free(struct vnic_rq *rq); int vnic_rq_alloc(struct vnic_dev *vdev, struct vnic_rq *rq, unsigned int index, unsigned int desc_count, unsigned int desc_size); +int vnic_rq_alloc_with_type(struct vnic_dev *vdev, struct vnic_rq *rq, + unsigned int index, unsigned int desc_count, + unsigned int desc_size, unsigned int res_type); void vnic_rq_init(struct vnic_rq *rq, unsigned int cq_index, unsigned int error_interrupt_enable, unsigned int error_interrupt_offset); diff --git a/drivers/net/ethernet/cisco/enic/vnic_wq.c b/drivers/net/ethernet/cisco/enic/vnic_wq.c index 29c7900349b2..5a20bdc62141 100644 --- a/drivers/net/ethernet/cisco/enic/vnic_wq.c +++ b/drivers/net/ethernet/cisco/enic/vnic_wq.c @@ -72,15 +72,16 @@ void vnic_wq_free(struct vnic_wq *wq) wq->ctrl = NULL; } -int vnic_wq_alloc(struct vnic_dev *vdev, struct vnic_wq *wq, unsigned int index, - unsigned int desc_count, unsigned int desc_size) +int vnic_wq_alloc_with_type(struct vnic_dev *vdev, struct vnic_wq *wq, + unsigned int index, unsigned int desc_count, + unsigned int desc_size, unsigned int res_type) { int err; wq->index = index; wq->vdev = vdev; - wq->ctrl = vnic_dev_get_res(vdev, RES_TYPE_WQ, index); + wq->ctrl = vnic_dev_get_res(vdev, res_type, index); if (!wq->ctrl) { vdev_err(vdev, "Failed to hook WQ[%d] resource\n", index); return -EINVAL; @@ -101,6 +102,13 @@ int vnic_wq_alloc(struct vnic_dev *vdev, struct vnic_wq *wq, unsigned int index, return 0; } +int vnic_wq_alloc(struct vnic_dev *vdev, struct vnic_wq *wq, unsigned int index, + unsigned int desc_count, unsigned int desc_size) +{ + return vnic_wq_alloc_with_type(vdev, wq, index, desc_count, desc_size, + RES_TYPE_WQ); +} + int enic_wq_devcmd2_alloc(struct vnic_dev *vdev, struct vnic_wq *wq, unsigned int desc_count, unsigned int desc_size) { diff --git a/drivers/net/ethernet/cisco/enic/vnic_wq.h b/drivers/net/ethernet/cisco/enic/vnic_wq.h index 3bb4758100ba..1d448b7a9ba2 100644 --- a/drivers/net/ethernet/cisco/enic/vnic_wq.h +++ b/drivers/net/ethernet/cisco/enic/vnic_wq.h @@ -165,6 +165,9 @@ static inline void vnic_wq_service(struct vnic_wq *wq, void vnic_wq_free(struct vnic_wq *wq); int vnic_wq_alloc(struct vnic_dev *vdev, struct vnic_wq *wq, unsigned int index, unsigned int desc_count, unsigned int desc_size); +int vnic_wq_alloc_with_type(struct vnic_dev *vdev, struct vnic_wq *wq, + unsigned int index, unsigned int desc_count, + unsigned int desc_size, unsigned int res_type); void vnic_wq_init(struct vnic_wq *wq, unsigned int cq_index, unsigned int error_interrupt_enable, unsigned int error_interrupt_offset); diff --git a/drivers/net/ethernet/freescale/Makefile b/drivers/net/ethernet/freescale/Makefile index de7b31842233..d0a259e47960 100644 --- a/drivers/net/ethernet/freescale/Makefile +++ b/drivers/net/ethernet/freescale/Makefile @@ -22,6 +22,5 @@ ucc_geth_driver-objs := ucc_geth.o ucc_geth_ethtool.o obj-$(CONFIG_FSL_FMAN) += fman/ obj-$(CONFIG_FSL_DPAA_ETH) += dpaa/ -obj-$(CONFIG_FSL_DPAA2_ETH) += dpaa2/ - +obj-y += dpaa2/ obj-y += enetc/ diff --git a/drivers/net/ethernet/freescale/dpaa2/Kconfig b/drivers/net/ethernet/freescale/dpaa2/Kconfig index d029b69c3f18..36280e5d99e1 100644 --- a/drivers/net/ethernet/freescale/dpaa2/Kconfig +++ b/drivers/net/ethernet/freescale/dpaa2/Kconfig @@ -34,6 +34,10 @@ config FSL_DPAA2_SWITCH tristate "Freescale DPAA2 Ethernet Switch" depends on BRIDGE || BRIDGE=n depends on NET_SWITCHDEV + depends on FSL_MC_BUS && FSL_MC_DPIO + select PHYLINK + select PCS_LYNX + select FSL_XGMAC_MDIO help Driver for Freescale DPAA2 Ethernet Switch. This driver manages switch objects discovered on the Freeescale MC bus. diff --git a/drivers/net/ethernet/freescale/dpaa2/Makefile b/drivers/net/ethernet/freescale/dpaa2/Makefile index 1b05ba8d1cbf..5f74be76434f 100644 --- a/drivers/net/ethernet/freescale/dpaa2/Makefile +++ b/drivers/net/ethernet/freescale/dpaa2/Makefile @@ -3,15 +3,16 @@ # Makefile for the Freescale DPAA2 Ethernet controller # -obj-$(CONFIG_FSL_DPAA2_ETH) += fsl-dpaa2-eth.o +obj-$(CONFIG_FSL_DPAA2_ETH) += fsl-dpaa2-eth.o fsl-dpaa2-mac.o obj-$(CONFIG_FSL_DPAA2_PTP_CLOCK) += fsl-dpaa2-ptp.o -obj-$(CONFIG_FSL_DPAA2_SWITCH) += fsl-dpaa2-switch.o +obj-$(CONFIG_FSL_DPAA2_SWITCH) += fsl-dpaa2-switch.o fsl-dpaa2-mac.o -fsl-dpaa2-eth-objs := dpaa2-eth.o dpaa2-ethtool.o dpni.o dpaa2-mac.o dpmac.o dpaa2-eth-devlink.o dpaa2-xsk.o +fsl-dpaa2-eth-objs := dpaa2-eth.o dpaa2-ethtool.o dpni.o dpaa2-eth-devlink.o dpaa2-xsk.o fsl-dpaa2-eth-${CONFIG_FSL_DPAA2_ETH_DCB} += dpaa2-eth-dcb.o fsl-dpaa2-eth-${CONFIG_DEBUG_FS} += dpaa2-eth-debugfs.o fsl-dpaa2-ptp-objs := dpaa2-ptp.o dprtc.o -fsl-dpaa2-switch-objs := dpaa2-switch.o dpaa2-switch-ethtool.o dpsw.o dpaa2-switch-flower.o dpaa2-mac.o dpmac.o +fsl-dpaa2-switch-objs := dpaa2-switch.o dpaa2-switch-ethtool.o dpsw.o dpaa2-switch-flower.o +fsl-dpaa2-mac-objs += dpaa2-mac.o dpmac.o # Needed by the tracing framework CFLAGS_dpaa2-eth.o := -I$(src) diff --git a/drivers/net/ethernet/freescale/dpaa2/dpaa2-ethtool.c b/drivers/net/ethernet/freescale/dpaa2/dpaa2-ethtool.c index baab4f1c908d..59f5c778df38 100644 --- a/drivers/net/ethernet/freescale/dpaa2/dpaa2-ethtool.c +++ b/drivers/net/ethernet/freescale/dpaa2/dpaa2-ethtool.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) /* Copyright 2014-2016 Freescale Semiconductor Inc. - * Copyright 2016-2022 NXP + * Copyright 2016-2022, 2024-2026 NXP */ #include <linux/net_tstamp.h> @@ -938,6 +938,61 @@ static void dpaa2_eth_get_channels(struct net_device *net_dev, channels->other_count; } +static void +dpaa2_eth_get_rmon_stats(struct net_device *net_dev, + struct ethtool_rmon_stats *rmon_stats, + const struct ethtool_rmon_hist_range **ranges) +{ + struct dpaa2_eth_priv *priv = netdev_priv(net_dev); + + mutex_lock(&priv->mac_lock); + + if (dpaa2_eth_has_mac(priv)) + dpaa2_mac_get_rmon_stats(priv->mac, rmon_stats, ranges); + + mutex_unlock(&priv->mac_lock); +} + +static void dpaa2_eth_get_pause_stats(struct net_device *net_dev, + struct ethtool_pause_stats *pause_stats) +{ + struct dpaa2_eth_priv *priv = netdev_priv(net_dev); + + mutex_lock(&priv->mac_lock); + + if (dpaa2_eth_has_mac(priv)) + dpaa2_mac_get_pause_stats(priv->mac, pause_stats); + + mutex_unlock(&priv->mac_lock); +} + +static void dpaa2_eth_get_ctrl_stats(struct net_device *net_dev, + struct ethtool_eth_ctrl_stats *ctrl_stats) +{ + struct dpaa2_eth_priv *priv = netdev_priv(net_dev); + + mutex_lock(&priv->mac_lock); + + if (dpaa2_eth_has_mac(priv)) + dpaa2_mac_get_ctrl_stats(priv->mac, ctrl_stats); + + mutex_unlock(&priv->mac_lock); +} + +static void +dpaa2_eth_get_eth_mac_stats(struct net_device *net_dev, + struct ethtool_eth_mac_stats *eth_mac_stats) +{ + struct dpaa2_eth_priv *priv = netdev_priv(net_dev); + + mutex_lock(&priv->mac_lock); + + if (dpaa2_eth_has_mac(priv)) + dpaa2_mac_get_eth_mac_stats(priv->mac, eth_mac_stats); + + mutex_unlock(&priv->mac_lock); +} + const struct ethtool_ops dpaa2_ethtool_ops = { .supported_coalesce_params = ETHTOOL_COALESCE_RX_USECS | ETHTOOL_COALESCE_USE_ADAPTIVE_RX, @@ -962,4 +1017,8 @@ const struct ethtool_ops dpaa2_ethtool_ops = { .get_coalesce = dpaa2_eth_get_coalesce, .set_coalesce = dpaa2_eth_set_coalesce, .get_channels = dpaa2_eth_get_channels, + .get_rmon_stats = dpaa2_eth_get_rmon_stats, + .get_pause_stats = dpaa2_eth_get_pause_stats, + .get_eth_ctrl_stats = dpaa2_eth_get_ctrl_stats, + .get_eth_mac_stats = dpaa2_eth_get_eth_mac_stats, }; diff --git a/drivers/net/ethernet/freescale/dpaa2/dpaa2-mac.c b/drivers/net/ethernet/freescale/dpaa2/dpaa2-mac.c index 422ce13a7c94..1f80a527264a 100644 --- a/drivers/net/ethernet/freescale/dpaa2/dpaa2-mac.c +++ b/drivers/net/ethernet/freescale/dpaa2/dpaa2-mac.c @@ -1,5 +1,5 @@ // SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) -/* Copyright 2019 NXP */ +/* Copyright 2019, 2024-2026 NXP */ #include <linux/acpi.h> #include <linux/pcs-lynx.h> @@ -15,7 +15,195 @@ #define DPMAC_PROTOCOL_CHANGE_VER_MAJOR 4 #define DPMAC_PROTOCOL_CHANGE_VER_MINOR 8 +#define DPMAC_STATS_BUNDLE_VER_MAJOR 4 +#define DPMAC_STATS_BUNDLE_VER_MINOR 10 + +#define DPMAC_STANDARD_STATS_VER_MAJOR 4 +#define DPMAC_STANDARD_STATS_VER_MINOR 11 + #define DPAA2_MAC_FEATURE_PROTOCOL_CHANGE BIT(0) +#define DPAA2_MAC_FEATURE_STATS_BUNDLE BIT(1) +#define DPAA2_MAC_FEATURE_STANDARD_STATS BIT(2) + +struct dpmac_counter { + enum dpmac_counter_id id; + size_t offset; + const char *name; +}; + +#define DPMAC_COUNTER(counter_id, struct_name, struct_offset) \ + { \ + .id = counter_id, \ + .offset = offsetof(struct_name, struct_offset), \ + } + +#define DPMAC_UNSTRUCTURED_COUNTER(counter_id, counter_name) \ + { \ + .id = counter_id, \ + .name = counter_name, \ + } + +#define DPMAC_RMON_COUNTER(counter_id, struct_offset) \ + DPMAC_COUNTER(counter_id, struct ethtool_rmon_stats, struct_offset) + +#define DPMAC_PAUSE_COUNTER(counter_id, struct_offset) \ + DPMAC_COUNTER(counter_id, struct ethtool_pause_stats, struct_offset) + +#define DPMAC_CTRL_COUNTER(counter_id, struct_offset) \ + DPMAC_COUNTER(counter_id, struct ethtool_eth_ctrl_stats, struct_offset) + +#define DPMAC_MAC_COUNTER(counter_id, struct_offset) \ + DPMAC_COUNTER(counter_id, struct ethtool_eth_mac_stats, struct_offset) + +static const struct dpmac_counter dpaa2_mac_ethtool_stats[] = { + DPMAC_UNSTRUCTURED_COUNTER(DPMAC_CNT_ING_ALL_FRAME, "[mac] rx all frames"), + DPMAC_UNSTRUCTURED_COUNTER(DPMAC_CNT_ING_GOOD_FRAME, "[mac] rx frames ok"), + DPMAC_UNSTRUCTURED_COUNTER(DPMAC_CNT_ING_ERR_FRAME, "[mac] rx frame errors"), + DPMAC_UNSTRUCTURED_COUNTER(DPMAC_CNT_ING_FRAME_DISCARD, "[mac] rx frame discards"), + DPMAC_UNSTRUCTURED_COUNTER(DPMAC_CNT_ING_UCAST_FRAME, "[mac] rx u-cast"), + DPMAC_UNSTRUCTURED_COUNTER(DPMAC_CNT_ING_BCAST_FRAME, "[mac] rx b-cast"), + DPMAC_UNSTRUCTURED_COUNTER(DPMAC_CNT_ING_MCAST_FRAME, "[mac] rx m-cast"), + DPMAC_UNSTRUCTURED_COUNTER(DPMAC_CNT_ING_FRAME_64, "[mac] rx 64 bytes"), + DPMAC_UNSTRUCTURED_COUNTER(DPMAC_CNT_ING_FRAME_127, "[mac] rx 65-127 bytes"), + DPMAC_UNSTRUCTURED_COUNTER(DPMAC_CNT_ING_FRAME_255, "[mac] rx 128-255 bytes"), + DPMAC_UNSTRUCTURED_COUNTER(DPMAC_CNT_ING_FRAME_511, "[mac] rx 256-511 bytes"), + DPMAC_UNSTRUCTURED_COUNTER(DPMAC_CNT_ING_FRAME_1023, "[mac] rx 512-1023 bytes"), + DPMAC_UNSTRUCTURED_COUNTER(DPMAC_CNT_ING_FRAME_1518, "[mac] rx 1024-1518 bytes"), + DPMAC_UNSTRUCTURED_COUNTER(DPMAC_CNT_ING_FRAME_1519_MAX, "[mac] rx 1519-max bytes"), + DPMAC_UNSTRUCTURED_COUNTER(DPMAC_CNT_ING_FRAG, "[mac] rx frags"), + DPMAC_UNSTRUCTURED_COUNTER(DPMAC_CNT_ING_JABBER, "[mac] rx jabber"), + DPMAC_UNSTRUCTURED_COUNTER(DPMAC_CNT_ING_ALIGN_ERR, "[mac] rx align errors"), + DPMAC_UNSTRUCTURED_COUNTER(DPMAC_CNT_ING_OVERSIZED, "[mac] rx oversized"), + DPMAC_UNSTRUCTURED_COUNTER(DPMAC_CNT_ING_VALID_PAUSE_FRAME, "[mac] rx pause"), + DPMAC_UNSTRUCTURED_COUNTER(DPMAC_CNT_ING_BYTE, "[mac] rx bytes"), + DPMAC_UNSTRUCTURED_COUNTER(DPMAC_CNT_EGR_GOOD_FRAME, "[mac] tx frames ok"), + DPMAC_UNSTRUCTURED_COUNTER(DPMAC_CNT_EGR_UCAST_FRAME, "[mac] tx u-cast"), + DPMAC_UNSTRUCTURED_COUNTER(DPMAC_CNT_EGR_MCAST_FRAME, "[mac] tx m-cast"), + DPMAC_UNSTRUCTURED_COUNTER(DPMAC_CNT_EGR_BCAST_FRAME, "[mac] tx b-cast"), + DPMAC_UNSTRUCTURED_COUNTER(DPMAC_CNT_EGR_ERR_FRAME, "[mac] tx frame errors"), + DPMAC_UNSTRUCTURED_COUNTER(DPMAC_CNT_EGR_UNDERSIZED, "[mac] tx undersized"), + DPMAC_UNSTRUCTURED_COUNTER(DPMAC_CNT_EGR_VALID_PAUSE_FRAME, "[mac] tx b-pause"), + DPMAC_UNSTRUCTURED_COUNTER(DPMAC_CNT_EGR_BYTE, "[mac] tx bytes"), +}; + +#define DPAA2_MAC_NUM_ETHTOOL_STATS ARRAY_SIZE(dpaa2_mac_ethtool_stats) + +static const struct dpmac_counter dpaa2_mac_rmon_stats[] = { + DPMAC_RMON_COUNTER(DPMAC_CNT_ING_FRAME_64, hist[0]), + DPMAC_RMON_COUNTER(DPMAC_CNT_ING_FRAME_127, hist[1]), + DPMAC_RMON_COUNTER(DPMAC_CNT_ING_FRAME_255, hist[2]), + DPMAC_RMON_COUNTER(DPMAC_CNT_ING_FRAME_511, hist[3]), + DPMAC_RMON_COUNTER(DPMAC_CNT_ING_FRAME_1023, hist[4]), + DPMAC_RMON_COUNTER(DPMAC_CNT_ING_FRAME_1518, hist[5]), + DPMAC_RMON_COUNTER(DPMAC_CNT_ING_FRAME_1519_MAX, hist[6]), + DPMAC_RMON_COUNTER(DPMAC_CNT_EGR_FRAME_64, hist_tx[0]), + DPMAC_RMON_COUNTER(DPMAC_CNT_EGR_FRAME_127, hist_tx[1]), + DPMAC_RMON_COUNTER(DPMAC_CNT_EGR_FRAME_255, hist_tx[2]), + DPMAC_RMON_COUNTER(DPMAC_CNT_EGR_FRAME_511, hist_tx[3]), + DPMAC_RMON_COUNTER(DPMAC_CNT_EGR_FRAME_1023, hist_tx[4]), + DPMAC_RMON_COUNTER(DPMAC_CNT_EGR_FRAME_1518, hist_tx[5]), + DPMAC_RMON_COUNTER(DPMAC_CNT_EGR_FRAME_1519_MAX, hist_tx[6]), + DPMAC_RMON_COUNTER(DPMAC_CNT_ING_UNDERSIZED, undersize_pkts), + DPMAC_RMON_COUNTER(DPMAC_CNT_ING_OVERSIZED, oversize_pkts), + DPMAC_RMON_COUNTER(DPMAC_CNT_ING_FRAG, fragments), + DPMAC_RMON_COUNTER(DPMAC_CNT_ING_JABBER, jabbers), +}; + +#define DPAA2_MAC_NUM_RMON_STATS ARRAY_SIZE(dpaa2_mac_rmon_stats) + +static const struct dpmac_counter dpaa2_mac_pause_stats[] = { + DPMAC_PAUSE_COUNTER(DPMAC_CNT_ING_VALID_PAUSE_FRAME, rx_pause_frames), + DPMAC_PAUSE_COUNTER(DPMAC_CNT_EGR_VALID_PAUSE_FRAME, tx_pause_frames), +}; + +#define DPAA2_MAC_NUM_PAUSE_STATS ARRAY_SIZE(dpaa2_mac_pause_stats) + +static const struct dpmac_counter dpaa2_mac_eth_ctrl_stats[] = { + DPMAC_CTRL_COUNTER(DPMAC_CNT_ING_CONTROL_FRAME, MACControlFramesReceived), + DPMAC_CTRL_COUNTER(DPMAC_CNT_EGR_CONTROL_FRAME, MACControlFramesTransmitted), +}; + +#define DPAA2_MAC_NUM_ETH_CTRL_STATS ARRAY_SIZE(dpaa2_mac_eth_ctrl_stats) + +static const struct dpmac_counter dpaa2_mac_eth_mac_stats[] = { + DPMAC_MAC_COUNTER(DPMAC_CNT_EGR_GOOD_FRAME, FramesTransmittedOK), + DPMAC_MAC_COUNTER(DPMAC_CNT_ING_GOOD_FRAME, FramesReceivedOK), + DPMAC_MAC_COUNTER(DPMAC_CNT_ING_FCS_ERR, FrameCheckSequenceErrors), + DPMAC_MAC_COUNTER(DPMAC_CNT_ING_ALIGN_ERR, AlignmentErrors), + DPMAC_MAC_COUNTER(DPMAC_CNT_EGR_ALL_BYTE, OctetsTransmittedOK), + DPMAC_MAC_COUNTER(DPMAC_CNT_EGR_ERR_FRAME, FramesLostDueToIntMACXmitError), + DPMAC_MAC_COUNTER(DPMAC_CNT_ING_ALL_BYTE, OctetsReceivedOK), + DPMAC_MAC_COUNTER(DPMAC_CNT_ING_FRAME_DISCARD_NOT_TRUNC, FramesLostDueToIntMACRcvError), + DPMAC_MAC_COUNTER(DPMAC_CNT_EGR_MCAST_FRAME, MulticastFramesXmittedOK), + DPMAC_MAC_COUNTER(DPMAC_CNT_EGR_BCAST_FRAME, BroadcastFramesXmittedOK), + DPMAC_MAC_COUNTER(DPMAC_CNT_ING_MCAST_FRAME, MulticastFramesReceivedOK), + DPMAC_MAC_COUNTER(DPMAC_CNT_ING_BCAST_FRAME, BroadcastFramesReceivedOK), +}; + +#define DPAA2_MAC_NUM_ETH_MAC_STATS ARRAY_SIZE(dpaa2_mac_eth_mac_stats) + +static void dpaa2_mac_setup_stats(struct dpaa2_mac *mac, + struct dpaa2_mac_stats *stats, + size_t num_stats, + const struct dpmac_counter *counters) +{ + struct device *dev = mac->net_dev->dev.parent; + size_t size_idx, size_values; + __le32 *cnt_idx; + + size_idx = num_stats * sizeof(u32); + stats->idx_dma_mem = dma_alloc_noncoherent(dev, size_idx, + &stats->idx_iova, + DMA_TO_DEVICE, + GFP_KERNEL); + if (!stats->idx_dma_mem) + goto out; + + size_values = num_stats * sizeof(u64); + stats->values_dma_mem = dma_alloc_noncoherent(dev, size_values, + &stats->values_iova, + DMA_FROM_DEVICE, + GFP_KERNEL); + if (!stats->values_dma_mem) + goto err_alloc_values; + + cnt_idx = stats->idx_dma_mem; + for (size_t i = 0; i < num_stats; i++) + *cnt_idx++ = cpu_to_le32((u32)(counters[i].id)); + + dma_sync_single_for_device(dev, stats->idx_iova, size_idx, + DMA_TO_DEVICE); + + return; + +err_alloc_values: + dma_free_noncoherent(dev, num_stats * sizeof(u32), stats->idx_dma_mem, + stats->idx_iova, DMA_TO_DEVICE); +out: + stats->idx_dma_mem = NULL; + stats->values_dma_mem = NULL; +} + +static void dpaa2_mac_clear_stats(struct dpaa2_mac *mac, + struct dpaa2_mac_stats *stats, + size_t num_stats) +{ + struct device *dev = mac->net_dev->dev.parent; + + if (stats->idx_dma_mem) { + dma_free_noncoherent(dev, num_stats * sizeof(u32), + stats->idx_dma_mem, + stats->idx_iova, DMA_TO_DEVICE); + stats->idx_dma_mem = NULL; + } + + if (stats->values_dma_mem) { + dma_free_noncoherent(dev, num_stats * sizeof(u64), + stats->values_dma_mem, + stats->values_iova, DMA_FROM_DEVICE); + stats->values_dma_mem = NULL; + } +} static int dpaa2_mac_cmp_ver(struct dpaa2_mac *mac, u16 ver_major, u16 ver_minor) @@ -32,6 +220,14 @@ static void dpaa2_mac_detect_features(struct dpaa2_mac *mac) if (dpaa2_mac_cmp_ver(mac, DPMAC_PROTOCOL_CHANGE_VER_MAJOR, DPMAC_PROTOCOL_CHANGE_VER_MINOR) >= 0) mac->features |= DPAA2_MAC_FEATURE_PROTOCOL_CHANGE; + + if (dpaa2_mac_cmp_ver(mac, DPMAC_STATS_BUNDLE_VER_MAJOR, + DPMAC_STATS_BUNDLE_VER_MINOR) >= 0) + mac->features |= DPAA2_MAC_FEATURE_STATS_BUNDLE; + + if (dpaa2_mac_cmp_ver(mac, DPMAC_STANDARD_STATS_VER_MAJOR, + DPMAC_STANDARD_STATS_VER_MINOR) >= 0) + mac->features |= DPAA2_MAC_FEATURE_STANDARD_STATS; } static int phy_mode(enum dpmac_eth_if eth_if, phy_interface_t *if_mode) @@ -348,6 +544,7 @@ void dpaa2_mac_start(struct dpaa2_mac *mac) phylink_start(mac->phylink); } +EXPORT_SYMBOL_GPL(dpaa2_mac_start); void dpaa2_mac_stop(struct dpaa2_mac *mac) { @@ -358,6 +555,7 @@ void dpaa2_mac_stop(struct dpaa2_mac *mac) if (mac->serdes_phy) phy_power_off(mac->serdes_phy); } +EXPORT_SYMBOL_GPL(dpaa2_mac_stop); int dpaa2_mac_connect(struct dpaa2_mac *mac) { @@ -450,6 +648,7 @@ err_pcs_destroy: return err; } +EXPORT_SYMBOL_GPL(dpaa2_mac_connect); void dpaa2_mac_disconnect(struct dpaa2_mac *mac) { @@ -462,6 +661,7 @@ void dpaa2_mac_disconnect(struct dpaa2_mac *mac) of_phy_put(mac->serdes_phy); mac->serdes_phy = NULL; } +EXPORT_SYMBOL_GPL(dpaa2_mac_disconnect); int dpaa2_mac_open(struct dpaa2_mac *mac) { @@ -504,77 +704,226 @@ int dpaa2_mac_open(struct dpaa2_mac *mac) mac->fw_node = fw_node; net_dev->dev.of_node = to_of_node(mac->fw_node); + if (mac->features & DPAA2_MAC_FEATURE_STATS_BUNDLE) + dpaa2_mac_setup_stats(mac, &mac->ethtool_stats, + DPAA2_MAC_NUM_ETHTOOL_STATS, + dpaa2_mac_ethtool_stats); + + if (mac->features & DPAA2_MAC_FEATURE_STANDARD_STATS) { + dpaa2_mac_setup_stats(mac, &mac->rmon_stats, + DPAA2_MAC_NUM_RMON_STATS, + dpaa2_mac_rmon_stats); + + dpaa2_mac_setup_stats(mac, &mac->pause_stats, + DPAA2_MAC_NUM_PAUSE_STATS, + dpaa2_mac_pause_stats); + + dpaa2_mac_setup_stats(mac, &mac->eth_ctrl_stats, + DPAA2_MAC_NUM_ETH_CTRL_STATS, + dpaa2_mac_eth_ctrl_stats); + + dpaa2_mac_setup_stats(mac, &mac->eth_mac_stats, + DPAA2_MAC_NUM_ETH_MAC_STATS, + dpaa2_mac_eth_mac_stats); + } + return 0; err_close_dpmac: dpmac_close(mac->mc_io, 0, dpmac_dev->mc_handle); return err; } +EXPORT_SYMBOL_GPL(dpaa2_mac_open); void dpaa2_mac_close(struct dpaa2_mac *mac) { struct fsl_mc_device *dpmac_dev = mac->mc_dev; + if (mac->features & DPAA2_MAC_FEATURE_STATS_BUNDLE) + dpaa2_mac_clear_stats(mac, &mac->ethtool_stats, + DPAA2_MAC_NUM_ETHTOOL_STATS); + + if (mac->features & DPAA2_MAC_FEATURE_STANDARD_STATS) { + dpaa2_mac_clear_stats(mac, &mac->rmon_stats, + DPAA2_MAC_NUM_RMON_STATS); + dpaa2_mac_clear_stats(mac, &mac->pause_stats, + DPAA2_MAC_NUM_PAUSE_STATS); + dpaa2_mac_clear_stats(mac, &mac->eth_ctrl_stats, + DPAA2_MAC_NUM_ETH_CTRL_STATS); + dpaa2_mac_clear_stats(mac, &mac->eth_mac_stats, + DPAA2_MAC_NUM_ETH_MAC_STATS); + } + dpmac_close(mac->mc_io, 0, dpmac_dev->mc_handle); if (mac->fw_node) fwnode_handle_put(mac->fw_node); } +EXPORT_SYMBOL_GPL(dpaa2_mac_close); -static char dpaa2_mac_ethtool_stats[][ETH_GSTRING_LEN] = { - [DPMAC_CNT_ING_ALL_FRAME] = "[mac] rx all frames", - [DPMAC_CNT_ING_GOOD_FRAME] = "[mac] rx frames ok", - [DPMAC_CNT_ING_ERR_FRAME] = "[mac] rx frame errors", - [DPMAC_CNT_ING_FRAME_DISCARD] = "[mac] rx frame discards", - [DPMAC_CNT_ING_UCAST_FRAME] = "[mac] rx u-cast", - [DPMAC_CNT_ING_BCAST_FRAME] = "[mac] rx b-cast", - [DPMAC_CNT_ING_MCAST_FRAME] = "[mac] rx m-cast", - [DPMAC_CNT_ING_FRAME_64] = "[mac] rx 64 bytes", - [DPMAC_CNT_ING_FRAME_127] = "[mac] rx 65-127 bytes", - [DPMAC_CNT_ING_FRAME_255] = "[mac] rx 128-255 bytes", - [DPMAC_CNT_ING_FRAME_511] = "[mac] rx 256-511 bytes", - [DPMAC_CNT_ING_FRAME_1023] = "[mac] rx 512-1023 bytes", - [DPMAC_CNT_ING_FRAME_1518] = "[mac] rx 1024-1518 bytes", - [DPMAC_CNT_ING_FRAME_1519_MAX] = "[mac] rx 1519-max bytes", - [DPMAC_CNT_ING_FRAG] = "[mac] rx frags", - [DPMAC_CNT_ING_JABBER] = "[mac] rx jabber", - [DPMAC_CNT_ING_ALIGN_ERR] = "[mac] rx align errors", - [DPMAC_CNT_ING_OVERSIZED] = "[mac] rx oversized", - [DPMAC_CNT_ING_VALID_PAUSE_FRAME] = "[mac] rx pause", - [DPMAC_CNT_ING_BYTE] = "[mac] rx bytes", - [DPMAC_CNT_EGR_GOOD_FRAME] = "[mac] tx frames ok", - [DPMAC_CNT_EGR_UCAST_FRAME] = "[mac] tx u-cast", - [DPMAC_CNT_EGR_MCAST_FRAME] = "[mac] tx m-cast", - [DPMAC_CNT_EGR_BCAST_FRAME] = "[mac] tx b-cast", - [DPMAC_CNT_EGR_ERR_FRAME] = "[mac] tx frame errors", - [DPMAC_CNT_EGR_UNDERSIZED] = "[mac] tx undersized", - [DPMAC_CNT_EGR_VALID_PAUSE_FRAME] = "[mac] tx b-pause", - [DPMAC_CNT_EGR_BYTE] = "[mac] tx bytes", +static void dpaa2_mac_transfer_stats(const struct dpmac_counter *counters, + size_t num_counters, void *s, + __le64 *cnt_values) +{ + for (size_t i = 0; i < num_counters; i++) { + u64 *p = s + counters[i].offset; + + *p = le64_to_cpu(cnt_values[i]); + } +} + +static const struct ethtool_rmon_hist_range dpaa2_mac_rmon_ranges[] = { + { 64, 64 }, + { 65, 127 }, + { 128, 255 }, + { 256, 511 }, + { 512, 1023 }, + { 1024, 1518 }, + { 1519, DPAA2_ETH_MFL }, + {}, }; -#define DPAA2_MAC_NUM_STATS ARRAY_SIZE(dpaa2_mac_ethtool_stats) +static void dpaa2_mac_get_standard_stats(struct dpaa2_mac *mac, + struct dpaa2_mac_stats *stats, + size_t num_cnt, + const struct dpmac_counter *counters, + void *s) +{ + struct device *dev = mac->net_dev->dev.parent; + struct fsl_mc_device *dpmac_dev = mac->mc_dev; + size_t values_size = num_cnt * sizeof(u64); + int err; + + if (!(mac->features & DPAA2_MAC_FEATURE_STANDARD_STATS)) + return; + + if (!stats->idx_dma_mem || !stats->values_dma_mem) + return; + + dma_sync_single_for_device(dev, stats->values_iova, values_size, + DMA_FROM_DEVICE); + + err = dpmac_get_statistics(mac->mc_io, 0, dpmac_dev->mc_handle, + stats->idx_iova, stats->values_iova, + num_cnt); + if (err) { + netdev_err(mac->net_dev, "%s: dpmac_get_statistics() = %d\n", + __func__, err); + return; + } + + dma_sync_single_for_cpu(dev, stats->values_iova, values_size, + DMA_FROM_DEVICE); + + dpaa2_mac_transfer_stats(counters, num_cnt, s, stats->values_dma_mem); +} + +void dpaa2_mac_get_rmon_stats(struct dpaa2_mac *mac, + struct ethtool_rmon_stats *s, + const struct ethtool_rmon_hist_range **ranges) +{ + if (s->src != ETHTOOL_MAC_STATS_SRC_AGGREGATE) + return; + + dpaa2_mac_get_standard_stats(mac, &mac->rmon_stats, + DPAA2_MAC_NUM_RMON_STATS, + dpaa2_mac_rmon_stats, s); + + *ranges = dpaa2_mac_rmon_ranges; +} +EXPORT_SYMBOL_GPL(dpaa2_mac_get_rmon_stats); + +void dpaa2_mac_get_pause_stats(struct dpaa2_mac *mac, + struct ethtool_pause_stats *s) +{ + if (s->src != ETHTOOL_MAC_STATS_SRC_AGGREGATE) + return; + + dpaa2_mac_get_standard_stats(mac, &mac->pause_stats, + DPAA2_MAC_NUM_PAUSE_STATS, + dpaa2_mac_pause_stats, s); +} +EXPORT_SYMBOL_GPL(dpaa2_mac_get_pause_stats); + +void dpaa2_mac_get_ctrl_stats(struct dpaa2_mac *mac, + struct ethtool_eth_ctrl_stats *s) +{ + if (s->src != ETHTOOL_MAC_STATS_SRC_AGGREGATE) + return; + + dpaa2_mac_get_standard_stats(mac, &mac->eth_ctrl_stats, + DPAA2_MAC_NUM_ETH_CTRL_STATS, + dpaa2_mac_eth_ctrl_stats, s); +} +EXPORT_SYMBOL_GPL(dpaa2_mac_get_ctrl_stats); + +void dpaa2_mac_get_eth_mac_stats(struct dpaa2_mac *mac, + struct ethtool_eth_mac_stats *s) +{ + if (s->src != ETHTOOL_MAC_STATS_SRC_AGGREGATE) + return; + + dpaa2_mac_get_standard_stats(mac, &mac->eth_mac_stats, + DPAA2_MAC_NUM_ETH_MAC_STATS, + dpaa2_mac_eth_mac_stats, s); +} +EXPORT_SYMBOL_GPL(dpaa2_mac_get_eth_mac_stats); int dpaa2_mac_get_sset_count(void) { - return DPAA2_MAC_NUM_STATS; + return DPAA2_MAC_NUM_ETHTOOL_STATS; } +EXPORT_SYMBOL_GPL(dpaa2_mac_get_sset_count); void dpaa2_mac_get_strings(u8 **data) { int i; - for (i = 0; i < DPAA2_MAC_NUM_STATS; i++) - ethtool_puts(data, dpaa2_mac_ethtool_stats[i]); + for (i = 0; i < DPAA2_MAC_NUM_ETHTOOL_STATS; i++) + ethtool_puts(data, dpaa2_mac_ethtool_stats[i].name); } +EXPORT_SYMBOL_GPL(dpaa2_mac_get_strings); void dpaa2_mac_get_ethtool_stats(struct dpaa2_mac *mac, u64 *data) { + size_t values_size = DPAA2_MAC_NUM_ETHTOOL_STATS * sizeof(u64); + struct device *dev = mac->net_dev->dev.parent; struct fsl_mc_device *dpmac_dev = mac->mc_dev; + __le64 *cnt_values; int i, err; u64 value; - for (i = 0; i < DPAA2_MAC_NUM_STATS; i++) { + if (!(mac->features & DPAA2_MAC_FEATURE_STATS_BUNDLE)) + goto fallback; + + if (!mac->ethtool_stats.idx_dma_mem || + !mac->ethtool_stats.values_dma_mem) + goto fallback; + + dma_sync_single_for_device(dev, mac->ethtool_stats.values_iova, + values_size, DMA_FROM_DEVICE); + + err = dpmac_get_statistics(mac->mc_io, 0, dpmac_dev->mc_handle, + mac->ethtool_stats.idx_iova, + mac->ethtool_stats.values_iova, + DPAA2_MAC_NUM_ETHTOOL_STATS); + if (err) + goto fallback; + + dma_sync_single_for_cpu(dev, mac->ethtool_stats.values_iova, + values_size, DMA_FROM_DEVICE); + + cnt_values = mac->ethtool_stats.values_dma_mem; + for (i = 0; i < DPAA2_MAC_NUM_ETHTOOL_STATS; i++) + *(data + i) = le64_to_cpu(*cnt_values++); + + return; + +fallback: + + /* Fallback and retrieve each counter one by one */ + for (i = 0; i < DPAA2_MAC_NUM_ETHTOOL_STATS; i++) { err = dpmac_get_counter(mac->mc_io, 0, dpmac_dev->mc_handle, - i, &value); + dpaa2_mac_ethtool_stats[i].id, &value); if (err) { netdev_err_once(mac->net_dev, "dpmac_get_counter error %d\n", err); @@ -584,3 +933,7 @@ void dpaa2_mac_get_ethtool_stats(struct dpaa2_mac *mac, u64 *data) *(data + i) = value; } } +EXPORT_SYMBOL_GPL(dpaa2_mac_get_ethtool_stats); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("DPAA2 Ethernet MAC library"); diff --git a/drivers/net/ethernet/freescale/dpaa2/dpaa2-mac.h b/drivers/net/ethernet/freescale/dpaa2/dpaa2-mac.h index 53f8d106d11e..98c725f609e9 100644 --- a/drivers/net/ethernet/freescale/dpaa2/dpaa2-mac.h +++ b/drivers/net/ethernet/freescale/dpaa2/dpaa2-mac.h @@ -1,5 +1,5 @@ /* SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) */ -/* Copyright 2019 NXP */ +/* Copyright 2019, 2024-2026 NXP */ #ifndef DPAA2_MAC_H #define DPAA2_MAC_H @@ -11,6 +11,12 @@ #include "dpmac.h" #include "dpmac-cmd.h" +struct dpaa2_mac_stats { + __le32 *idx_dma_mem; + __le64 *values_dma_mem; + dma_addr_t idx_iova, values_iova; +}; + struct dpaa2_mac { struct fsl_mc_device *mc_dev; struct dpmac_link_state state; @@ -28,6 +34,12 @@ struct dpaa2_mac { struct fwnode_handle *fw_node; struct phy *serdes_phy; + + struct dpaa2_mac_stats ethtool_stats; + struct dpaa2_mac_stats rmon_stats; + struct dpaa2_mac_stats pause_stats; + struct dpaa2_mac_stats eth_ctrl_stats; + struct dpaa2_mac_stats eth_mac_stats; }; static inline bool dpaa2_mac_is_type_phy(struct dpaa2_mac *mac) @@ -53,6 +65,19 @@ void dpaa2_mac_get_strings(u8 **data); void dpaa2_mac_get_ethtool_stats(struct dpaa2_mac *mac, u64 *data); +void dpaa2_mac_get_rmon_stats(struct dpaa2_mac *mac, + struct ethtool_rmon_stats *s, + const struct ethtool_rmon_hist_range **ranges); + +void dpaa2_mac_get_pause_stats(struct dpaa2_mac *mac, + struct ethtool_pause_stats *s); + +void dpaa2_mac_get_ctrl_stats(struct dpaa2_mac *mac, + struct ethtool_eth_ctrl_stats *s); + +void dpaa2_mac_get_eth_mac_stats(struct dpaa2_mac *mac, + struct ethtool_eth_mac_stats *s); + void dpaa2_mac_start(struct dpaa2_mac *mac); void dpaa2_mac_stop(struct dpaa2_mac *mac); diff --git a/drivers/net/ethernet/freescale/dpaa2/dpaa2-switch-ethtool.c b/drivers/net/ethernet/freescale/dpaa2/dpaa2-switch-ethtool.c index a888f6e6e9b0..f5d9321c7ef9 100644 --- a/drivers/net/ethernet/freescale/dpaa2/dpaa2-switch-ethtool.c +++ b/drivers/net/ethernet/freescale/dpaa2/dpaa2-switch-ethtool.c @@ -3,7 +3,7 @@ * DPAA2 Ethernet Switch ethtool support * * Copyright 2014-2016 Freescale Semiconductor Inc. - * Copyright 2017-2018 NXP + * Copyright 2017-2018, 2024-2026 NXP * */ @@ -210,6 +210,49 @@ static void dpaa2_switch_ethtool_get_stats(struct net_device *netdev, mutex_unlock(&port_priv->mac_lock); } +static void +dpaa2_switch_get_rmon_stats(struct net_device *netdev, + struct ethtool_rmon_stats *rmon_stats, + const struct ethtool_rmon_hist_range **ranges) +{ + struct ethsw_port_priv *port_priv = netdev_priv(netdev); + + mutex_lock(&port_priv->mac_lock); + + if (dpaa2_switch_port_has_mac(port_priv)) + dpaa2_mac_get_rmon_stats(port_priv->mac, rmon_stats, ranges); + + mutex_unlock(&port_priv->mac_lock); +} + +static void +dpaa2_switch_get_ctrl_stats(struct net_device *net_dev, + struct ethtool_eth_ctrl_stats *ctrl_stats) +{ + struct ethsw_port_priv *port_priv = netdev_priv(net_dev); + + mutex_lock(&port_priv->mac_lock); + + if (dpaa2_switch_port_has_mac(port_priv)) + dpaa2_mac_get_ctrl_stats(port_priv->mac, ctrl_stats); + + mutex_unlock(&port_priv->mac_lock); +} + +static void +dpaa2_switch_get_eth_mac_stats(struct net_device *net_dev, + struct ethtool_eth_mac_stats *eth_mac_stats) +{ + struct ethsw_port_priv *port_priv = netdev_priv(net_dev); + + mutex_lock(&port_priv->mac_lock); + + if (dpaa2_switch_port_has_mac(port_priv)) + dpaa2_mac_get_eth_mac_stats(port_priv->mac, eth_mac_stats); + + mutex_unlock(&port_priv->mac_lock); +} + const struct ethtool_ops dpaa2_switch_port_ethtool_ops = { .get_drvinfo = dpaa2_switch_get_drvinfo, .get_link = ethtool_op_get_link, @@ -218,4 +261,7 @@ const struct ethtool_ops dpaa2_switch_port_ethtool_ops = { .get_strings = dpaa2_switch_ethtool_get_strings, .get_ethtool_stats = dpaa2_switch_ethtool_get_stats, .get_sset_count = dpaa2_switch_ethtool_get_sset_count, + .get_rmon_stats = dpaa2_switch_get_rmon_stats, + .get_eth_ctrl_stats = dpaa2_switch_get_ctrl_stats, + .get_eth_mac_stats = dpaa2_switch_get_eth_mac_stats, }; diff --git a/drivers/net/ethernet/freescale/dpaa2/dpmac-cmd.h b/drivers/net/ethernet/freescale/dpaa2/dpmac-cmd.h index e9ac2ecef3be..a864a99a0f75 100644 --- a/drivers/net/ethernet/freescale/dpaa2/dpmac-cmd.h +++ b/drivers/net/ethernet/freescale/dpaa2/dpmac-cmd.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) */ /* Copyright 2013-2016 Freescale Semiconductor Inc. - * Copyright 2019 NXP + * Copyright 2019, 2024-2026 NXP */ #ifndef _FSL_DPMAC_CMD_H #define _FSL_DPMAC_CMD_H @@ -28,6 +28,8 @@ #define DPMAC_CMDID_SET_PROTOCOL DPMAC_CMD(0x0c7) +#define DPMAC_CMDID_GET_STATISTICS DPMAC_CMD(0x0c8) + /* Macros for accessing command fields smaller than 1byte */ #define DPMAC_MASK(field) \ GENMASK(DPMAC_##field##_SHIFT + DPMAC_##field##_SIZE - 1, \ @@ -82,4 +84,11 @@ struct dpmac_rsp_get_api_version { struct dpmac_cmd_set_protocol { u8 eth_if; }; + +struct dpmac_cmd_get_statistics { + __le64 iova_cnt; + __le64 iova_values; + __le32 num_cnt; +}; + #endif /* _FSL_DPMAC_CMD_H */ diff --git a/drivers/net/ethernet/freescale/dpaa2/dpmac.c b/drivers/net/ethernet/freescale/dpaa2/dpmac.c index f440a4c3b70c..efb9864d051f 100644 --- a/drivers/net/ethernet/freescale/dpaa2/dpmac.c +++ b/drivers/net/ethernet/freescale/dpaa2/dpmac.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) /* Copyright 2013-2016 Freescale Semiconductor Inc. - * Copyright 2019 NXP + * Copyright 2019, 2024-2026 NXP */ #include <linux/fsl/mc.h> #include "dpmac.h" @@ -235,3 +235,32 @@ int dpmac_set_protocol(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, return mc_send_command(mc_io, &cmd); } + +/** + * dpmac_get_statistics() - Get MAC statistics + * @mc_io: Pointer to opaque I/O object + * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' + * @token: Token of DPMAC object + * @iova_cnt: IOVA containing the requested MAC counters formatted as an + * array of __le32 representing the dpmac_counter_id. + * @iova_values: IOVA containing the values for all the requested counters + * formatted as an array of __le64. + * @num_cnt: Number of counters requested + * + * Return: '0' on Success; Error code otherwise. + */ +int dpmac_get_statistics(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, + u64 iova_cnt, u64 iova_values, u32 num_cnt) +{ + struct dpmac_cmd_get_statistics *cmd_params; + struct fsl_mc_command cmd = { 0 }; + + cmd.header = mc_encode_cmd_header(DPMAC_CMDID_GET_STATISTICS, + cmd_flags, token); + cmd_params = (struct dpmac_cmd_get_statistics *)cmd.params; + cmd_params->iova_cnt = cpu_to_le64(iova_cnt); + cmd_params->iova_values = cpu_to_le64(iova_values); + cmd_params->num_cnt = cpu_to_le32(num_cnt); + + return mc_send_command(mc_io, &cmd); +} diff --git a/drivers/net/ethernet/freescale/dpaa2/dpmac.h b/drivers/net/ethernet/freescale/dpaa2/dpmac.h index 17488819ef68..b5276168b869 100644 --- a/drivers/net/ethernet/freescale/dpaa2/dpmac.h +++ b/drivers/net/ethernet/freescale/dpaa2/dpmac.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) */ /* Copyright 2013-2016 Freescale Semiconductor Inc. - * Copyright 2019 NXP + * Copyright 2019, 2024-2026 NXP */ #ifndef __FSL_DPMAC_H #define __FSL_DPMAC_H @@ -170,6 +170,58 @@ int dpmac_set_link_state(struct fsl_mc_io *mc_io, * pause frames. * @DPMAC_CNT_EGR_GOOD_FRAME: counts frames transmitted without error, including * pause frames. + * @DPMAC_CNT_EGR_FRAME_64: counts transmitted 64-bytes frames, good or bad. + * @DPMAC_CNT_EGR_FRAME_127: counts transmitted 65 to 127-bytes frames, good or + * bad. + * @DPMAC_CNT_EGR_FRAME_255: counts transmitted 128 to 255-bytes frames, good + * or bad. + * @DPMAC_CNT_EGR_FRAME_511: counts transmitted 256 to 511-bytes frames, good + * or bad. + * @DPMAC_CNT_EGR_FRAME_1023: counts transmitted 512 to 1023-bytes frames, good + * or bad. + * @DPMAC_CNT_EGR_FRAME_1518: counts transmitted 1024 to 1518-bytes frames, + * good or bad. + * @DPMAC_CNT_EGR_FRAME_1519_MAX: counts transmitted 1519-bytes frames and + * larger (up to max frame length specified), + * good or bad. + * @DPMAC_CNT_ING_ALL_BYTE: counts bytes received in both good and bad packets + * @DPMAC_CNT_ING_FCS_ERR: counts frames received with a CRC-32 error but the + * frame is otherwise of correct length + * @DPMAC_CNT_ING_VLAN_FRAME: counts the received VLAN tagged frames which are + * valid. + * @DPMAC_CNT_ING_UNDERSIZED: counts received frames which were less than 64 + * bytes long and with a good CRC. + * @DPMAC_CNT_ING_CONTROL_FRAME: counts received control frames (type 0x8808) + * but not pause frames. + * @DPMAC_CNT_ING_FRAME_DISCARD_NOT_TRUNC: counts the fully dropped frames (not + * truncated) due to internal errors of + * the MAC client. Occurs when a + * receive FIFO overflows. + * @DPMAC_CNT_EGR_ALL_BYTE: counts transmitted bytes in both good and bad + * packets. + * @DPMAC_CNT_EGR_FCS_ERR: counts trasmitted frames with a CRC-32 error except + * for underflows. + * @DPMAC_CNT_EGR_VLAN_FRAME: counts the transmitted VLAN tagged frames which + * are valid. + * @DPMAC_CNT_EGR_ALL_FRAME: counts all trasmitted frames, good or bad. + * @DPMAC_CNT_EGR_CONTROL_FRAME: counts transmitted control frames (type + * 0x8808) but not pause frames. + * @DPMAC_CNT_ING_PFC0: number of PFC frames received for class 0. + * @DPMAC_CNT_ING_PFC1: number of PFC frames received for class 1. + * @DPMAC_CNT_ING_PFC2: number of PFC frames received for class 2. + * @DPMAC_CNT_ING_PFC3: number of PFC frames received for class 3. + * @DPMAC_CNT_ING_PFC4: number of PFC frames received for class 4. + * @DPMAC_CNT_ING_PFC5: number of PFC frames received for class 5. + * @DPMAC_CNT_ING_PFC6: number of PFC frames received for class 6. + * @DPMAC_CNT_ING_PFC7: number of PFC frames received for class 7. + * @DPMAC_CNT_EGR_PFC0: number of PFC frames transmitted for class 0. + * @DPMAC_CNT_EGR_PFC1: number of PFC frames transmitted for class 1. + * @DPMAC_CNT_EGR_PFC2: number of PFC frames transmitted for class 2. + * @DPMAC_CNT_EGR_PFC3: number of PFC frames transmitted for class 3. + * @DPMAC_CNT_EGR_PFC4: number of PFC frames transmitted for class 4. + * @DPMAC_CNT_EGR_PFC5: number of PFC frames transmitted for class 5. + * @DPMAC_CNT_EGR_PFC6: number of PFC frames transmitted for class 6. + * @DPMAC_CNT_EGR_PFC7: number of PFC frames transmitted for class 7. */ enum dpmac_counter_id { DPMAC_CNT_ING_FRAME_64, @@ -199,7 +251,41 @@ enum dpmac_counter_id { DPMAC_CNT_EGR_UCAST_FRAME, DPMAC_CNT_EGR_ERR_FRAME, DPMAC_CNT_ING_GOOD_FRAME, - DPMAC_CNT_EGR_GOOD_FRAME + DPMAC_CNT_EGR_GOOD_FRAME, + DPMAC_CNT_EGR_FRAME_64, + DPMAC_CNT_EGR_FRAME_127, + DPMAC_CNT_EGR_FRAME_255, + DPMAC_CNT_EGR_FRAME_511, + DPMAC_CNT_EGR_FRAME_1023, + DPMAC_CNT_EGR_FRAME_1518, + DPMAC_CNT_EGR_FRAME_1519_MAX, + DPMAC_CNT_ING_ALL_BYTE, + DPMAC_CNT_ING_FCS_ERR, + DPMAC_CNT_ING_VLAN_FRAME, + DPMAC_CNT_ING_UNDERSIZED, + DPMAC_CNT_ING_CONTROL_FRAME, + DPMAC_CNT_ING_FRAME_DISCARD_NOT_TRUNC, + DPMAC_CNT_EGR_ALL_BYTE, + DPMAC_CNT_EGR_FCS_ERR, + DPMAC_CNT_EGR_VLAN_FRAME, + DPMAC_CNT_EGR_ALL_FRAME, + DPMAC_CNT_EGR_CONTROL_FRAME, + DPMAC_CNT_ING_PFC0, + DPMAC_CNT_ING_PFC1, + DPMAC_CNT_ING_PFC2, + DPMAC_CNT_ING_PFC3, + DPMAC_CNT_ING_PFC4, + DPMAC_CNT_ING_PFC5, + DPMAC_CNT_ING_PFC6, + DPMAC_CNT_ING_PFC7, + DPMAC_CNT_EGR_PFC0, + DPMAC_CNT_EGR_PFC1, + DPMAC_CNT_EGR_PFC2, + DPMAC_CNT_EGR_PFC3, + DPMAC_CNT_EGR_PFC4, + DPMAC_CNT_EGR_PFC5, + DPMAC_CNT_EGR_PFC6, + DPMAC_CNT_EGR_PFC7, }; int dpmac_get_counter(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, @@ -210,4 +296,8 @@ int dpmac_get_api_version(struct fsl_mc_io *mc_io, u32 cmd_flags, int dpmac_set_protocol(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, enum dpmac_eth_if protocol); + +int dpmac_get_statistics(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, + u64 iova_cnt, u64 iova_values, u32 num_cnt); + #endif /* __FSL_DPMAC_H */ diff --git a/drivers/net/ethernet/freescale/enetc/enetc.h b/drivers/net/ethernet/freescale/enetc/enetc.h index aecd40aeef9c..e663bb5e614e 100644 --- a/drivers/net/ethernet/freescale/enetc/enetc.h +++ b/drivers/net/ethernet/freescale/enetc/enetc.h @@ -264,6 +264,8 @@ struct enetc_msg_swbd { }; #define ENETC_REV1 0x1 +#define ENETC_REV4 0x4 + enum enetc_errata { ENETC_ERR_VLAN_ISOL = BIT(0), ENETC_ERR_UCMCSWP = BIT(1), diff --git a/drivers/net/ethernet/freescale/enetc/enetc4_hw.h b/drivers/net/ethernet/freescale/enetc/enetc4_hw.h index 719c88ceb801..f18437556a0e 100644 --- a/drivers/net/ethernet/freescale/enetc/enetc4_hw.h +++ b/drivers/net/ethernet/freescale/enetc/enetc4_hw.h @@ -64,6 +64,9 @@ #define ENETC4_PPAUONTR 0x108 #define ENETC4_PPAUOFFTR 0x10c +/* Port ingress congestion DRa (a=0,1,2,3) discard count register */ +#define ENETC4_PICDRDCR(a) ((a) * 0x10 + 0x140) + /* Port Station interface promiscuous MAC mode register */ #define ENETC4_PSIPMMR 0x200 #define PSIPMMR_SI_MAC_UP(a) BIT(a) /* a = SI index */ @@ -72,6 +75,12 @@ /* Port Station interface promiscuous VLAN mode register */ #define ENETC4_PSIPVMR 0x204 +/* Port broadcast frames dropped due to MAC filtering register */ +#define ENETC4_PBFDSIR 0x208 + +/* Port frame drop MAC source address pruning register */ +#define ENETC4_PFDMSAPR 0x20c + /* Port RSS key register n. n = 0,1,2,...,9 */ #define ENETC4_PRSSKR(n) ((n) * 0x4 + 0x250) @@ -79,6 +88,12 @@ #define ENETC4_PSIMAFCAPR 0x280 #define PSIMAFCAPR_NUM_MAC_AFTE GENMASK(11, 0) +/* Port unicast frames dropped due to MAC filtering register */ +#define ENETC4_PUFDMFR 0x284 + +/* Port multicast frames dropped due to MAC filtering register */ +#define ENETC4_PMFDMFR 0x288 + /* Port station interface VLAN filtering capability register */ #define ENETC4_PSIVLANFCAPR 0x2c0 #define PSIVLANFCAPR_NUM_VLAN_FTE GENMASK(11, 0) @@ -87,6 +102,15 @@ #define ENETC4_PSIVLANFMR 0x2c4 #define PSIVLANFMR_VS BIT(0) +/* Port unicast frames dropped VLAN filtering register */ +#define ENETC4_PUFDVFR 0x2d0 + +/* Port multicast frames dropped VLAN filtering register */ +#define ENETC4_PMFDVFR 0x2d4 + +/* Port broadcast frames dropped VLAN filtering register */ +#define ENETC4_PBFDVFR 0x2d8 + /* Port Station interface a primary MAC address registers */ #define ENETC4_PSIPMAR0(a) ((a) * 0x80 + 0x2000) #define ENETC4_PSIPMAR1(a) ((a) * 0x80 + 0x2004) @@ -141,6 +165,18 @@ #define ENETC4_PSR 0x4104 #define PSR_RX_BUSY BIT(1) +/* Port Rx discard count register */ +#define ENETC4_PRXDCR 0x41c0 + +/* Port Rx discard count read-reset register */ +#define ENETC4_PRXDCRRR 0x41c4 + +/* Port Rx discard count reason register 0 */ +#define ENETC4_PRXDCRR0 0x41c8 + +/* Port Rx discard count reason register 1 */ +#define ENETC4_PRXDCRR1 0x41cc + /* Port traffic class a transmit maximum SDU register */ #define ENETC4_PTCTMSDUR(a) ((a) * 0x20 + 0x4208) #define PTCTMSDUR_MAXSDU GENMASK(15, 0) @@ -196,6 +232,162 @@ #define PM_SINGLE_STEP_OFFSET_SET(o) FIELD_PREP(PM_SINGLE_STEP_OFFSET, o) #define PM_SINGLE_STEP_EN BIT(31) +/* Port MAC 0/1 Receive Ethernet Octets Counter */ +#define ENETC4_PM_REOCT(mac) (0x5100 + (mac) * 0x400) + +/* Port MAC 0/1 Receive Octets Counter */ +#define ENETC4_PM_ROCT(mac) (0x5108 + (mac) * 0x400) + +/* Port MAC 0/1 Receive Alignment Error Counter Register */ +#define ENETC4_PM_RALN(mac) (0x5110 + (mac) * 0x400) + +/* Port MAC 0/1 Receive Valid Pause Frame Counter */ +#define ENETC4_PM_RXPF(mac) (0x5118 + (mac) * 0x400) + +/* Port MAC 0/1 Receive Frame Counter */ +#define ENETC4_PM_RFRM(mac) (0x5120 + (mac) * 0x400) + +/* Port MAC 0/1 Receive Frame Check Sequence Error Counter */ +#define ENETC4_PM_RFCS(mac) (0x5128 + (mac) * 0x400) + +/* Port MAC 0/1 Receive VLAN Frame Counter */ +#define ENETC4_PM_RVLAN(mac) (0x5130 + (mac) * 0x400) + +/* Port MAC 0/1 Receive Frame Error Counter */ +#define ENETC4_PM_RERR(mac) (0x5138 + (mac) * 0x400) + +/* Port MAC 0/1 Receive Unicast Frame Counter */ +#define ENETC4_PM_RUCA(mac) (0x5140 + (mac) * 0x400) + +/* Port MAC 0/1 Receive Multicast Frame Counter */ +#define ENETC4_PM_RMCA(mac) (0x5148 + (mac) * 0x400) + +/* Port MAC 0/1 Receive Broadcast Frame Counter */ +#define ENETC4_PM_RBCA(mac) (0x5150 + (mac) * 0x400) + +/* Port MAC 0/1 Receive Dropped Packets Counter */ +#define ENETC4_PM_RDRP(mac) (0x5158 + (mac) * 0x400) + +/* Port MAC 0/1 Receive Packets Counter */ +#define ENETC4_PM_RPKT(mac) (0x5160 + (mac) * 0x400) + +/* Port MAC 0/1 Receive Undersized Packet Counter */ +#define ENETC4_PM_RUND(mac) (0x5168 + (mac) * 0x400) + +/* Port MAC 0/1 Receive 64-Octet Packet Counter */ +#define ENETC4_PM_R64(mac) (0x5170 + (mac) * 0x400) + +/* Port MAC 0/1 Receive 65 to 127-Octet Packet Counter */ +#define ENETC4_PM_R127(mac) (0x5178 + (mac) * 0x400) + +/* Port MAC 0/1 Receive 128 to 255-Octet Packet Counter */ +#define ENETC4_PM_R255(mac) (0x5180 + (mac) * 0x400) + +/* Port MAC 0/1 Receive 256 to 511-Octet Packet Counter */ +#define ENETC4_PM_R511(mac) (0x5188 + (mac) * 0x400) + +/* Port MAC 0/1 Receive 512 to 1023-Octet Packet Counter */ +#define ENETC4_PM_R1023(mac) (0x5190 + (mac) * 0x400) + +/* Port MAC 0/1 Receive 1024 to 1522-Octet Packet Counter */ +#define ENETC4_PM_R1522(mac) (0x5198 + (mac) * 0x400) + +/* Port MAC 0/1 Receive 1523 to Max-Octet Packet Counter */ +#define ENETC4_PM_R1523X(mac) (0x51a0 + (mac) * 0x400) + +/* Port MAC 0/1 Receive Oversized Packet Counter */ +#define ENETC4_PM_ROVR(mac) (0x51a8 + (mac) * 0x400) + +/* Port MAC 0/1 Receive Jabber Packet Counter */ +#define ENETC4_PM_RJBR(mac) (0x51b0 + (mac) * 0x400) + +/* Port MAC 0/1 Receive Fragment Packet Counter */ +#define ENETC4_PM_RFRG(mac) (0x51b8 + (mac) * 0x400) + +/* Port MAC 0/1 Receive Control Packet Counter */ +#define ENETC4_PM_RCNP(mac) (0x51c0 + (mac) * 0x400) + +/* Port MAC 0/1 Receive Dropped Not Truncated Packets Counter */ +#define ENETC4_PM_RDRNTP(mac) (0x51c8 + (mac) * 0x400) + +/* Port MAC 0/1 Transmit Ethernet Octets Counter */ +#define ENETC4_PM_TEOCT(mac) (0x5200 + (mac) * 0x400) + +/* Port MAC 0/1 Transmit Octets Counter */ +#define ENETC4_PM_TOCT(mac) (0x5208 + (mac) * 0x400) + +/* Port MAC 0/1 Transmit Valid Pause Frame Counter */ +#define ENETC4_PM_TXPF(mac) (0x5218 + (mac) * 0x400) + +/* Port MAC 0/1 Transmit Frame Counter */ +#define ENETC4_PM_TFRM(mac) (0x5220 + (mac) * 0x400) + +/* Port MAC 0/1 Transmit Frame Check Sequence Error Counter */ +#define ENETC4_PM_TFCS(mac) (0x5228 + (mac) * 0x400) + +/* Port MAC 0/1 Transmit VLAN Frame Counter */ +#define ENETC4_PM_TVLAN(mac) (0x5230 + (mac) * 0x400) + +/* Port MAC 0/1 Transmit Frame Error Counter */ +#define ENETC4_PM_TERR(mac) (0x5238 + (mac) * 0x400) + +/* Port MAC 0/1 Transmit Unicast Frame Counter */ +#define ENETC4_PM_TUCA(mac) (0x5240 + (mac) * 0x400) + +/* Port MAC 0/1 Transmit Multicast Frame Counter */ +#define ENETC4_PM_TMCA(mac) (0x5248 + (mac) * 0x400) + +/* Port MAC 0/1 Transmit Broadcast Frame Counter */ +#define ENETC4_PM_TBCA(mac) (0x5250 + (mac) * 0x400) + +/* Port MAC 0/1 Transmit Packets Counter */ +#define ENETC4_PM_TPKT(mac) (0x5260 + (mac) * 0x400) + +/* Port MAC 0/1 Transmit Undersized Packet Counter */ +#define ENETC4_PM_TUND(mac) (0x5268 + (mac) * 0x400) + +/* Port MAC 0/1 Transmit 64-Octet Packet Counter */ +#define ENETC4_PM_T64(mac) (0x5270 + (mac) * 0x400) + +/* Port MAC 0/1 Transmit 65 to 127-Octet Packet Counter */ +#define ENETC4_PM_T127(mac) (0x5278 + (mac) * 0x400) + +/* Port MAC 0/1 Transmit 128 to 255-Octet Packet Counter */ +#define ENETC4_PM_T255(mac) (0x5280 + (mac) * 0x400) + +/* Port MAC 0/1 Transmit 256 to 511-Octet Packet Counter */ +#define ENETC4_PM_T511(mac) (0x5288 + (mac) * 0x400) + +/* Port MAC 0/1 Transmit 512 to 1023-Octet Packet Counter */ +#define ENETC4_PM_T1023(mac) (0x5290 + (mac) * 0x400) + +/* Port MAC 0/1 Transmit 1024 to 1522-Octet Packet Counter */ +#define ENETC4_PM_T1522(mac) (0x5298 + (mac) * 0x400) + +/* Port MAC 0/1 Transmit 1523 to TX_MTU-Octet Packet Counter */ +#define ENETC4_PM_T1523X(mac) (0x52a0 + (mac) * 0x400) + +/* Port MAC 0/1 Transmit Control Packet Counter */ +#define ENETC4_PM_TCNP(mac) (0x52c0 + (mac) * 0x400) + +/* Port MAC 0/1 Transmit Deferred Packet Counter */ +#define ENETC4_PM_TDFR(mac) (0x52d0 + (mac) * 0x400) + +/* Port MAC 0/1 Transmit Multiple Collisions Counter */ +#define ENETC4_PM_TMCOL(mac) (0x52d8 + (mac) * 0x400) + +/* Port MAC 0/1 Transmit Single Collision */ +#define ENETC4_PM_TSCOL(mac) (0x52e0 + (mac) * 0x400) + +/* Port MAC 0/1 Transmit Late Collision Counter */ +#define ENETC4_PM_TLCOL(mac) (0x52e8 + (mac) * 0x400) + +/* Port MAC 0/1 Transmit Excessive Collisions Counter */ +#define ENETC4_PM_TECOL(mac) (0x52f0 + (mac) * 0x400) + +/* Port MAC 0/1 Transmit Invalid Octets Counter */ +#define ENETC4_PM_TIOCT(mac) (0x52f8 + (mac) * 0x400) + /* Port MAC 0 Interface Mode Control Register */ #define ENETC4_PM_IF_MODE(mac) (0x5300 + (mac) * 0x400) #define PM_IF_MODE_IFMODE GENMASK(2, 0) diff --git a/drivers/net/ethernet/freescale/enetc/enetc_ethtool.c b/drivers/net/ethernet/freescale/enetc/enetc_ethtool.c index 7c17acaf7a38..71f376ef1be1 100644 --- a/drivers/net/ethernet/freescale/enetc/enetc_ethtool.c +++ b/drivers/net/ethernet/freescale/enetc/enetc_ethtool.c @@ -124,79 +124,40 @@ static const struct { { ENETC_SITFRM, "SI tx frames" }, { ENETC_SITUCA, "SI tx u-cast frames" }, { ENETC_SITMCA, "SI tx m-cast frames" }, - { ENETC_RBDCR(0), "Rx ring 0 discarded frames" }, - { ENETC_RBDCR(1), "Rx ring 1 discarded frames" }, - { ENETC_RBDCR(2), "Rx ring 2 discarded frames" }, - { ENETC_RBDCR(3), "Rx ring 3 discarded frames" }, - { ENETC_RBDCR(4), "Rx ring 4 discarded frames" }, - { ENETC_RBDCR(5), "Rx ring 5 discarded frames" }, - { ENETC_RBDCR(6), "Rx ring 6 discarded frames" }, - { ENETC_RBDCR(7), "Rx ring 7 discarded frames" }, - { ENETC_RBDCR(8), "Rx ring 8 discarded frames" }, - { ENETC_RBDCR(9), "Rx ring 9 discarded frames" }, - { ENETC_RBDCR(10), "Rx ring 10 discarded frames" }, - { ENETC_RBDCR(11), "Rx ring 11 discarded frames" }, - { ENETC_RBDCR(12), "Rx ring 12 discarded frames" }, - { ENETC_RBDCR(13), "Rx ring 13 discarded frames" }, - { ENETC_RBDCR(14), "Rx ring 14 discarded frames" }, - { ENETC_RBDCR(15), "Rx ring 15 discarded frames" }, }; static const struct { int reg; char name[ETH_GSTRING_LEN] __nonstring; -} enetc_pm_counters[] = { - { ENETC_PM_REOCT(0), "MAC rx ethernet octets" }, - { ENETC_PM_RALN(0), "MAC rx alignment errors" }, - { ENETC_PM_RXPF(0), "MAC rx valid pause frames" }, - { ENETC_PM_RFRM(0), "MAC rx valid frames" }, - { ENETC_PM_RFCS(0), "MAC rx fcs errors" }, - { ENETC_PM_RVLAN(0), "MAC rx VLAN frames" }, - { ENETC_PM_RERR(0), "MAC rx frame errors" }, - { ENETC_PM_RUCA(0), "MAC rx unicast frames" }, - { ENETC_PM_RMCA(0), "MAC rx multicast frames" }, - { ENETC_PM_RBCA(0), "MAC rx broadcast frames" }, - { ENETC_PM_RDRP(0), "MAC rx dropped packets" }, - { ENETC_PM_RPKT(0), "MAC rx packets" }, - { ENETC_PM_RUND(0), "MAC rx undersized packets" }, - { ENETC_PM_R64(0), "MAC rx 64 byte packets" }, - { ENETC_PM_R127(0), "MAC rx 65-127 byte packets" }, - { ENETC_PM_R255(0), "MAC rx 128-255 byte packets" }, - { ENETC_PM_R511(0), "MAC rx 256-511 byte packets" }, - { ENETC_PM_R1023(0), "MAC rx 512-1023 byte packets" }, - { ENETC_PM_R1522(0), "MAC rx 1024-1522 byte packets" }, - { ENETC_PM_R1523X(0), "MAC rx 1523 to max-octet packets" }, - { ENETC_PM_ROVR(0), "MAC rx oversized packets" }, - { ENETC_PM_RJBR(0), "MAC rx jabber packets" }, - { ENETC_PM_RFRG(0), "MAC rx fragment packets" }, - { ENETC_PM_RCNP(0), "MAC rx control packets" }, - { ENETC_PM_RDRNTP(0), "MAC rx fifo drop" }, - { ENETC_PM_TEOCT(0), "MAC tx ethernet octets" }, - { ENETC_PM_TOCT(0), "MAC tx octets" }, - { ENETC_PM_TCRSE(0), "MAC tx carrier sense errors" }, - { ENETC_PM_TXPF(0), "MAC tx valid pause frames" }, - { ENETC_PM_TFRM(0), "MAC tx frames" }, - { ENETC_PM_TFCS(0), "MAC tx fcs errors" }, - { ENETC_PM_TVLAN(0), "MAC tx VLAN frames" }, - { ENETC_PM_TERR(0), "MAC tx frame errors" }, - { ENETC_PM_TUCA(0), "MAC tx unicast frames" }, - { ENETC_PM_TMCA(0), "MAC tx multicast frames" }, - { ENETC_PM_TBCA(0), "MAC tx broadcast frames" }, - { ENETC_PM_TPKT(0), "MAC tx packets" }, - { ENETC_PM_TUND(0), "MAC tx undersized packets" }, - { ENETC_PM_T64(0), "MAC tx 64 byte packets" }, - { ENETC_PM_T127(0), "MAC tx 65-127 byte packets" }, - { ENETC_PM_T255(0), "MAC tx 128-255 byte packets" }, - { ENETC_PM_T511(0), "MAC tx 256-511 byte packets" }, - { ENETC_PM_T1023(0), "MAC tx 512-1023 byte packets" }, - { ENETC_PM_T1522(0), "MAC tx 1024-1522 byte packets" }, - { ENETC_PM_T1523X(0), "MAC tx 1523 to max-octet packets" }, - { ENETC_PM_TCNP(0), "MAC tx control packets" }, - { ENETC_PM_TDFR(0), "MAC tx deferred packets" }, - { ENETC_PM_TMCOL(0), "MAC tx multiple collisions" }, - { ENETC_PM_TSCOL(0), "MAC tx single collisions" }, - { ENETC_PM_TLCOL(0), "MAC tx late collisions" }, - { ENETC_PM_TECOL(0), "MAC tx excessive collisions" }, +} enetc_emac_counters[] = { + { ENETC_PM_RVLAN(0), "eMAC rx VLAN frames" }, + { ENETC_PM_RERR(0), "eMAC rx frame errors" }, + { ENETC_PM_RUCA(0), "eMAC rx unicast frames" }, + { ENETC_PM_RDRP(0), "eMAC rx dropped packets" }, + { ENETC_PM_RPKT(0), "eMAC rx packets" }, + { ENETC_PM_TOCT(0), "eMAC tx octets" }, + { ENETC_PM_TFCS(0), "eMAC tx fcs errors" }, + { ENETC_PM_TVLAN(0), "eMAC tx VLAN frames" }, + { ENETC_PM_TUCA(0), "eMAC tx unicast frames" }, + { ENETC_PM_TPKT(0), "eMAC tx packets" }, + { ENETC_PM_TUND(0), "eMAC tx undersized packets" }, +}; + +static const struct { + int reg; + char name[ETH_GSTRING_LEN] __nonstring; +} enetc_pmac_counters[] = { + { ENETC_PM_RVLAN(1), "pMAC rx VLAN frames" }, + { ENETC_PM_RERR(1), "pMAC rx frame errors" }, + { ENETC_PM_RUCA(1), "pMAC rx unicast frames" }, + { ENETC_PM_RDRP(1), "pMAC rx dropped packets" }, + { ENETC_PM_RPKT(1), "pMAC rx packets" }, + { ENETC_PM_TOCT(1), "pMAC tx octets" }, + { ENETC_PM_TFCS(1), "pMAC tx fcs errors" }, + { ENETC_PM_TVLAN(1), "pMAC tx VLAN frames" }, + { ENETC_PM_TUCA(1), "pMAC tx unicast frames" }, + { ENETC_PM_TPKT(1), "pMAC tx packets" }, + { ENETC_PM_TUND(1), "pMAC tx undersized packets" }, }; static const struct { @@ -216,6 +177,65 @@ static const struct { { ENETC_PICDR(3), "ICM DR3 discarded frames" }, }; +static const struct { + int reg; + char name[ETH_GSTRING_LEN] __nonstring; +} enetc4_emac_counters[] = { + { ENETC4_PM_ROCT(0), "eMAC rx octets" }, + { ENETC4_PM_RVLAN(0), "eMAC rx VLAN frames" }, + { ENETC4_PM_RERR(0), "eMAC rx frame errors" }, + { ENETC4_PM_RUCA(0), "eMAC rx unicast frames" }, + { ENETC4_PM_RDRP(0), "eMAC rx dropped packets" }, + { ENETC4_PM_RPKT(0), "eMAC rx packets" }, + { ENETC4_PM_TOCT(0), "eMAC tx octets" }, + { ENETC4_PM_TVLAN(0), "eMAC tx VLAN frames" }, + { ENETC4_PM_TFCS(0), "eMAC tx fcs errors" }, + { ENETC4_PM_TUCA(0), "eMAC tx unicast frames" }, + { ENETC4_PM_TPKT(0), "eMAC tx packets" }, + { ENETC4_PM_TUND(0), "eMAC tx undersized packets" }, + { ENETC4_PM_TIOCT(0), "eMAC tx invalid octets" }, +}; + +static const struct { + int reg; + char name[ETH_GSTRING_LEN] __nonstring; +} enetc4_pmac_counters[] = { + { ENETC4_PM_ROCT(1), "pMAC rx octets" }, + { ENETC4_PM_RVLAN(1), "pMAC rx VLAN frames" }, + { ENETC4_PM_RERR(1), "pMAC rx frame errors" }, + { ENETC4_PM_RUCA(1), "pMAC rx unicast frames" }, + { ENETC4_PM_RDRP(1), "pMAC rx dropped packets" }, + { ENETC4_PM_RPKT(1), "pMAC rx packets" }, + { ENETC4_PM_TOCT(1), "pMAC tx octets" }, + { ENETC4_PM_TVLAN(1), "pMAC tx VLAN frames" }, + { ENETC4_PM_TFCS(1), "pMAC tx fcs errors" }, + { ENETC4_PM_TUCA(1), "pMAC tx unicast frames" }, + { ENETC4_PM_TPKT(1), "pMAC tx packets" }, + { ENETC4_PM_TUND(1), "pMAC tx undersized packets" }, + { ENETC4_PM_TIOCT(1), "pMAC tx invalid octets" }, +}; + +static const struct { + int reg; + char name[ETH_GSTRING_LEN] __nonstring; +} enetc4_port_counters[] = { + { ENETC4_PICDRDCR(0), "ICM DR0 discarded frames" }, + { ENETC4_PICDRDCR(1), "ICM DR1 discarded frames" }, + { ENETC4_PICDRDCR(2), "ICM DR2 discarded frames" }, + { ENETC4_PICDRDCR(3), "ICM DR3 discarded frames" }, + { ENETC4_PUFDMFR, "MAC filter discarded unicast" }, + { ENETC4_PMFDMFR, "MAC filter discarded multicast" }, + { ENETC4_PBFDSIR, "MAC filter discarded broadcast" }, + { ENETC4_PFDMSAPR, "MAC SA pruning discarded frames" }, + { ENETC4_PUFDVFR, "VLAN filter discarded unicast" }, + { ENETC4_PMFDVFR, "VLAN filter discarded multicast" }, + { ENETC4_PBFDVFR, "VLAN filter discarded broadcast" }, + { ENETC4_PRXDCR, "MAC rx discarded frames" }, + { ENETC4_PRXDCRRR, "MAC rx discard read-reset" }, + { ENETC4_PRXDCRR0, "MAC rx discard reason 0" }, + { ENETC4_PRXDCRR1, "MAC rx discard reason 1" }, +}; + static const char rx_ring_stats[][ETH_GSTRING_LEN] = { "Rx ring %2d frames", "Rx ring %2d alloc errors", @@ -224,6 +244,7 @@ static const char rx_ring_stats[][ETH_GSTRING_LEN] = { "Rx ring %2d recycle failures", "Rx ring %2d redirects", "Rx ring %2d redirect failures", + "Rx ring %2d discarded frames", }; static const char tx_ring_stats[][ETH_GSTRING_LEN] = { @@ -236,6 +257,7 @@ static const char tx_ring_stats[][ETH_GSTRING_LEN] = { static int enetc_get_sset_count(struct net_device *ndev, int sset) { struct enetc_ndev_priv *priv = netdev_priv(ndev); + struct enetc_si *si = priv->si; int len; if (sset != ETH_SS_STATS) @@ -245,18 +267,69 @@ static int enetc_get_sset_count(struct net_device *ndev, int sset) ARRAY_SIZE(tx_ring_stats) * priv->num_tx_rings + ARRAY_SIZE(rx_ring_stats) * priv->num_rx_rings; - if (!enetc_si_is_pf(priv->si)) + if (!enetc_si_is_pf(si)) return len; - len += ARRAY_SIZE(enetc_port_counters); - len += ARRAY_SIZE(enetc_pm_counters); + if (is_enetc_rev1(si)) { + len += ARRAY_SIZE(enetc_port_counters); + len += ARRAY_SIZE(enetc_emac_counters); + if (si->hw_features & ENETC_SI_F_QBU) + len += ARRAY_SIZE(enetc_pmac_counters); + } else { + len += ARRAY_SIZE(enetc4_port_counters); + + if (enetc_is_pseudo_mac(si)) + return len; + + len += ARRAY_SIZE(enetc4_emac_counters); + if (si->hw_features & ENETC_SI_F_QBU) + len += ARRAY_SIZE(enetc4_pmac_counters); + } return len; } +static void enetc_get_pf_strings(struct enetc_si *si, u8 *data) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(enetc_port_counters); i++) + ethtool_cpy(&data, enetc_port_counters[i].name); + + for (i = 0; i < ARRAY_SIZE(enetc_emac_counters); i++) + ethtool_cpy(&data, enetc_emac_counters[i].name); + + if (!(si->hw_features & ENETC_SI_F_QBU)) + return; + + for (i = 0; i < ARRAY_SIZE(enetc_pmac_counters); i++) + ethtool_cpy(&data, enetc_pmac_counters[i].name); +} + +static void enetc4_get_pf_strings(struct enetc_si *si, u8 *data) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(enetc4_port_counters); i++) + ethtool_cpy(&data, enetc4_port_counters[i].name); + + if (enetc_is_pseudo_mac(si)) + return; + + for (i = 0; i < ARRAY_SIZE(enetc4_emac_counters); i++) + ethtool_cpy(&data, enetc4_emac_counters[i].name); + + if (!(si->hw_features & ENETC_SI_F_QBU)) + return; + + for (i = 0; i < ARRAY_SIZE(enetc4_pmac_counters); i++) + ethtool_cpy(&data, enetc4_pmac_counters[i].name); +} + static void enetc_get_strings(struct net_device *ndev, u32 stringset, u8 *data) { struct enetc_ndev_priv *priv = netdev_priv(ndev); + struct enetc_si *si = priv->si; int i, j; switch (stringset) { @@ -270,24 +343,63 @@ static void enetc_get_strings(struct net_device *ndev, u32 stringset, u8 *data) for (j = 0; j < ARRAY_SIZE(rx_ring_stats); j++) ethtool_sprintf(&data, rx_ring_stats[j], i); - if (!enetc_si_is_pf(priv->si)) + if (!enetc_si_is_pf(si)) break; - for (i = 0; i < ARRAY_SIZE(enetc_port_counters); i++) - ethtool_cpy(&data, enetc_port_counters[i].name); - - for (i = 0; i < ARRAY_SIZE(enetc_pm_counters); i++) - ethtool_cpy(&data, enetc_pm_counters[i].name); + if (is_enetc_rev1(si)) + enetc_get_pf_strings(si, data); + else + enetc4_get_pf_strings(si, data); break; } } +static void enetc_pf_get_ethtool_stats(struct enetc_si *si, int *o, u64 *data) +{ + struct enetc_hw *hw = &si->hw; + int i; + + for (i = 0; i < ARRAY_SIZE(enetc_port_counters); i++) + data[(*o)++] = enetc_port_rd(hw, enetc_port_counters[i].reg); + + for (i = 0; i < ARRAY_SIZE(enetc_emac_counters); i++) + data[(*o)++] = enetc_port_rd64(hw, enetc_emac_counters[i].reg); + + if (!(si->hw_features & ENETC_SI_F_QBU)) + return; + + for (i = 0; i < ARRAY_SIZE(enetc_pmac_counters); i++) + data[(*o)++] = enetc_port_rd64(hw, enetc_pmac_counters[i].reg); +} + +static void enetc4_pf_get_ethtool_stats(struct enetc_si *si, int *o, u64 *data) +{ + struct enetc_hw *hw = &si->hw; + int i; + + for (i = 0; i < ARRAY_SIZE(enetc4_port_counters); i++) + data[(*o)++] = enetc_port_rd(hw, enetc4_port_counters[i].reg); + + if (enetc_is_pseudo_mac(si)) + return; + + for (i = 0; i < ARRAY_SIZE(enetc4_emac_counters); i++) + data[(*o)++] = enetc_port_rd64(hw, enetc4_emac_counters[i].reg); + + if (!(si->hw_features & ENETC_SI_F_QBU)) + return; + + for (i = 0; i < ARRAY_SIZE(enetc4_pmac_counters); i++) + data[(*o)++] = enetc_port_rd64(hw, enetc4_pmac_counters[i].reg); +} + static void enetc_get_ethtool_stats(struct net_device *ndev, struct ethtool_stats *stats, u64 *data) { struct enetc_ndev_priv *priv = netdev_priv(ndev); - struct enetc_hw *hw = &priv->si->hw; + struct enetc_si *si = priv->si; + struct enetc_hw *hw = &si->hw; int i, o = 0; for (i = 0; i < ARRAY_SIZE(enetc_si_counters); i++) @@ -308,39 +420,50 @@ static void enetc_get_ethtool_stats(struct net_device *ndev, data[o++] = priv->rx_ring[i]->stats.recycle_failures; data[o++] = priv->rx_ring[i]->stats.xdp_redirect; data[o++] = priv->rx_ring[i]->stats.xdp_redirect_failures; + data[o++] = enetc_rd(hw, ENETC_RBDCR(i)); } - if (!enetc_si_is_pf(priv->si)) + if (!enetc_si_is_pf(si)) return; - for (i = 0; i < ARRAY_SIZE(enetc_port_counters); i++) - data[o++] = enetc_port_rd(hw, enetc_port_counters[i].reg); - - for (i = 0; i < ARRAY_SIZE(enetc_pm_counters); i++) - data[o++] = enetc_port_rd64(hw, enetc_pm_counters[i].reg); + if (is_enetc_rev1(si)) + enetc_pf_get_ethtool_stats(si, &o, data); + else + enetc4_pf_get_ethtool_stats(si, &o, data); } -static void enetc_pause_stats(struct enetc_hw *hw, int mac, +static void enetc_pause_stats(struct enetc_si *si, int mac, struct ethtool_pause_stats *pause_stats) { - pause_stats->tx_pause_frames = enetc_port_rd64(hw, ENETC_PM_TXPF(mac)); - pause_stats->rx_pause_frames = enetc_port_rd64(hw, ENETC_PM_RXPF(mac)); + struct enetc_hw *hw = &si->hw; + + switch (si->pdev->revision) { + case ENETC_REV1: + pause_stats->tx_pause_frames = enetc_port_rd64(hw, ENETC_PM_TXPF(mac)); + pause_stats->rx_pause_frames = enetc_port_rd64(hw, ENETC_PM_RXPF(mac)); + break; + case ENETC_REV4: + pause_stats->tx_pause_frames = enetc_port_rd64(hw, ENETC4_PM_TXPF(mac)); + pause_stats->rx_pause_frames = enetc_port_rd64(hw, ENETC4_PM_RXPF(mac)); + break; + default: + break; + } } static void enetc_get_pause_stats(struct net_device *ndev, struct ethtool_pause_stats *pause_stats) { struct enetc_ndev_priv *priv = netdev_priv(ndev); - struct enetc_hw *hw = &priv->si->hw; struct enetc_si *si = priv->si; switch (pause_stats->src) { case ETHTOOL_MAC_STATS_SRC_EMAC: - enetc_pause_stats(hw, 0, pause_stats); + enetc_pause_stats(si, 0, pause_stats); break; case ETHTOOL_MAC_STATS_SRC_PMAC: if (si->hw_features & ENETC_SI_F_QBU) - enetc_pause_stats(hw, 1, pause_stats); + enetc_pause_stats(si, 1, pause_stats); break; case ETHTOOL_MAC_STATS_SRC_AGGREGATE: ethtool_aggregate_pause_stats(ndev, pause_stats); @@ -371,11 +494,45 @@ static void enetc_mac_stats(struct enetc_hw *hw, int mac, s->BroadcastFramesReceivedOK = enetc_port_rd64(hw, ENETC_PM_RBCA(mac)); } -static void enetc_ctrl_stats(struct enetc_hw *hw, int mac, +static void enetc4_mac_stats(struct enetc_hw *hw, int mac, + struct ethtool_eth_mac_stats *s) +{ + s->FramesTransmittedOK = enetc_port_rd64(hw, ENETC4_PM_TFRM(mac)); + s->SingleCollisionFrames = enetc_port_rd64(hw, ENETC4_PM_TSCOL(mac)); + s->MultipleCollisionFrames = enetc_port_rd64(hw, ENETC4_PM_TMCOL(mac)); + s->FramesReceivedOK = enetc_port_rd64(hw, ENETC4_PM_RFRM(mac)); + s->FrameCheckSequenceErrors = enetc_port_rd64(hw, ENETC4_PM_RFCS(mac)); + s->AlignmentErrors = enetc_port_rd64(hw, ENETC4_PM_RALN(mac)); + s->OctetsTransmittedOK = enetc_port_rd64(hw, ENETC4_PM_TEOCT(mac)); + s->FramesWithDeferredXmissions = enetc_port_rd64(hw, ENETC4_PM_TDFR(mac)); + s->LateCollisions = enetc_port_rd64(hw, ENETC4_PM_TLCOL(mac)); + s->FramesAbortedDueToXSColls = enetc_port_rd64(hw, ENETC4_PM_TECOL(mac)); + s->FramesLostDueToIntMACXmitError = enetc_port_rd64(hw, ENETC4_PM_TERR(mac)); + s->OctetsReceivedOK = enetc_port_rd64(hw, ENETC4_PM_REOCT(mac)); + s->FramesLostDueToIntMACRcvError = enetc_port_rd64(hw, ENETC4_PM_RDRNTP(mac)); + s->MulticastFramesXmittedOK = enetc_port_rd64(hw, ENETC4_PM_TMCA(mac)); + s->BroadcastFramesXmittedOK = enetc_port_rd64(hw, ENETC4_PM_TBCA(mac)); + s->MulticastFramesReceivedOK = enetc_port_rd64(hw, ENETC4_PM_RMCA(mac)); + s->BroadcastFramesReceivedOK = enetc_port_rd64(hw, ENETC4_PM_RBCA(mac)); +} + +static void enetc_ctrl_stats(struct enetc_si *si, int mac, struct ethtool_eth_ctrl_stats *s) { - s->MACControlFramesTransmitted = enetc_port_rd64(hw, ENETC_PM_TCNP(mac)); - s->MACControlFramesReceived = enetc_port_rd64(hw, ENETC_PM_RCNP(mac)); + struct enetc_hw *hw = &si->hw; + + switch (si->pdev->revision) { + case ENETC_REV1: + s->MACControlFramesTransmitted = enetc_port_rd64(hw, ENETC_PM_TCNP(mac)); + s->MACControlFramesReceived = enetc_port_rd64(hw, ENETC_PM_RCNP(mac)); + break; + case ENETC_REV4: + s->MACControlFramesTransmitted = enetc_port_rd64(hw, ENETC4_PM_TCNP(mac)); + s->MACControlFramesReceived = enetc_port_rd64(hw, ENETC4_PM_RCNP(mac)); + break; + default: + break; + } } static const struct ethtool_rmon_hist_range enetc_rmon_ranges[] = { @@ -414,20 +571,61 @@ static void enetc_rmon_stats(struct enetc_hw *hw, int mac, s->hist_tx[6] = enetc_port_rd64(hw, ENETC_PM_T1523X(mac)); } +static void enetc4_rmon_stats(struct enetc_hw *hw, int mac, + struct ethtool_rmon_stats *s) +{ + s->undersize_pkts = enetc_port_rd64(hw, ENETC4_PM_RUND(mac)); + s->oversize_pkts = enetc_port_rd64(hw, ENETC4_PM_ROVR(mac)); + s->fragments = enetc_port_rd64(hw, ENETC4_PM_RFRG(mac)); + s->jabbers = enetc_port_rd64(hw, ENETC4_PM_RJBR(mac)); + + s->hist[0] = enetc_port_rd64(hw, ENETC4_PM_R64(mac)); + s->hist[1] = enetc_port_rd64(hw, ENETC4_PM_R127(mac)); + s->hist[2] = enetc_port_rd64(hw, ENETC4_PM_R255(mac)); + s->hist[3] = enetc_port_rd64(hw, ENETC4_PM_R511(mac)); + s->hist[4] = enetc_port_rd64(hw, ENETC4_PM_R1023(mac)); + s->hist[5] = enetc_port_rd64(hw, ENETC4_PM_R1522(mac)); + s->hist[6] = enetc_port_rd64(hw, ENETC4_PM_R1523X(mac)); + + s->hist_tx[0] = enetc_port_rd64(hw, ENETC4_PM_T64(mac)); + s->hist_tx[1] = enetc_port_rd64(hw, ENETC4_PM_T127(mac)); + s->hist_tx[2] = enetc_port_rd64(hw, ENETC4_PM_T255(mac)); + s->hist_tx[3] = enetc_port_rd64(hw, ENETC4_PM_T511(mac)); + s->hist_tx[4] = enetc_port_rd64(hw, ENETC4_PM_T1023(mac)); + s->hist_tx[5] = enetc_port_rd64(hw, ENETC4_PM_T1522(mac)); + s->hist_tx[6] = enetc_port_rd64(hw, ENETC4_PM_T1523X(mac)); +} + +static void enetc_get_mac_stats(struct enetc_si *si, int mac, + struct ethtool_eth_mac_stats *mac_stats) +{ + struct enetc_hw *hw = &si->hw; + + switch (si->pdev->revision) { + case ENETC_REV1: + enetc_mac_stats(hw, mac, mac_stats); + break; + case ENETC_REV4: + enetc4_mac_stats(hw, mac, mac_stats); + break; + default: + break; + } +} + static void enetc_get_eth_mac_stats(struct net_device *ndev, struct ethtool_eth_mac_stats *mac_stats) { struct enetc_ndev_priv *priv = netdev_priv(ndev); - struct enetc_hw *hw = &priv->si->hw; struct enetc_si *si = priv->si; switch (mac_stats->src) { case ETHTOOL_MAC_STATS_SRC_EMAC: - enetc_mac_stats(hw, 0, mac_stats); + enetc_get_mac_stats(si, 0, mac_stats); break; case ETHTOOL_MAC_STATS_SRC_PMAC: if (si->hw_features & ENETC_SI_F_QBU) - enetc_mac_stats(hw, 1, mac_stats); + enetc_get_mac_stats(si, 1, mac_stats); break; case ETHTOOL_MAC_STATS_SRC_AGGREGATE: ethtool_aggregate_mac_stats(ndev, mac_stats); @@ -481,16 +679,15 @@ static void enetc_get_eth_ctrl_stats(struct net_device *ndev, struct ethtool_eth_ctrl_stats *ctrl_stats) { struct enetc_ndev_priv *priv = netdev_priv(ndev); - struct enetc_hw *hw = &priv->si->hw; struct enetc_si *si = priv->si; switch (ctrl_stats->src) { case ETHTOOL_MAC_STATS_SRC_EMAC: - enetc_ctrl_stats(hw, 0, ctrl_stats); + enetc_ctrl_stats(si, 0, ctrl_stats); break; case ETHTOOL_MAC_STATS_SRC_PMAC: if (si->hw_features & ENETC_SI_F_QBU) - enetc_ctrl_stats(hw, 1, ctrl_stats); + enetc_ctrl_stats(si, 1, ctrl_stats); break; case ETHTOOL_MAC_STATS_SRC_AGGREGATE: ethtool_aggregate_ctrl_stats(ndev, ctrl_stats); @@ -498,23 +695,39 @@ static void enetc_get_eth_ctrl_stats(struct net_device *ndev, } } +static void enetc_get_mac_rmon_stats(struct enetc_si *si, int mac, + struct ethtool_rmon_stats *rmon_stats) +{ + struct enetc_hw *hw = &si->hw; + + switch (si->pdev->revision) { + case ENETC_REV1: + enetc_rmon_stats(hw, mac, rmon_stats); + break; + case ENETC_REV4: + enetc4_rmon_stats(hw, mac, rmon_stats); + break; + default: + break; + } +} + static void enetc_get_rmon_stats(struct net_device *ndev, struct ethtool_rmon_stats *rmon_stats, const struct ethtool_rmon_hist_range **ranges) { struct enetc_ndev_priv *priv = netdev_priv(ndev); - struct enetc_hw *hw = &priv->si->hw; struct enetc_si *si = priv->si; *ranges = enetc_rmon_ranges; switch (rmon_stats->src) { case ETHTOOL_MAC_STATS_SRC_EMAC: - enetc_rmon_stats(hw, 0, rmon_stats); + enetc_get_mac_rmon_stats(si, 0, rmon_stats); break; case ETHTOOL_MAC_STATS_SRC_PMAC: if (si->hw_features & ENETC_SI_F_QBU) - enetc_rmon_stats(hw, 1, rmon_stats); + enetc_get_mac_rmon_stats(si, 1, rmon_stats); break; case ETHTOOL_MAC_STATS_SRC_AGGREGATE: ethtool_aggregate_rmon_stats(ndev, rmon_stats); @@ -1356,6 +1569,9 @@ const struct ethtool_ops enetc4_ppm_ethtool_ops = { .supported_coalesce_params = ETHTOOL_COALESCE_USECS | ETHTOOL_COALESCE_MAX_FRAMES | ETHTOOL_COALESCE_USE_ADAPTIVE_RX, + .get_sset_count = enetc_get_sset_count, + .get_strings = enetc_get_strings, + .get_ethtool_stats = enetc_get_ethtool_stats, .get_eth_mac_stats = enetc_ppm_get_eth_mac_stats, .get_rx_ring_count = enetc_get_rx_ring_count, .get_rxfh_key_size = enetc_get_rxfh_key_size, @@ -1398,6 +1614,13 @@ const struct ethtool_ops enetc4_pf_ethtool_ops = { .supported_coalesce_params = ETHTOOL_COALESCE_USECS | ETHTOOL_COALESCE_MAX_FRAMES | ETHTOOL_COALESCE_USE_ADAPTIVE_RX, + .get_sset_count = enetc_get_sset_count, + .get_strings = enetc_get_strings, + .get_ethtool_stats = enetc_get_ethtool_stats, + .get_pause_stats = enetc_get_pause_stats, + .get_rmon_stats = enetc_get_rmon_stats, + .get_eth_ctrl_stats = enetc_get_eth_ctrl_stats, + .get_eth_mac_stats = enetc_get_eth_mac_stats, .get_ringparam = enetc_get_ringparam, .get_coalesce = enetc_get_coalesce, .set_coalesce = enetc_set_coalesce, diff --git a/drivers/net/ethernet/freescale/enetc/ntmp.c b/drivers/net/ethernet/freescale/enetc/ntmp.c index 0c1d343253bf..703752995e93 100644 --- a/drivers/net/ethernet/freescale/enetc/ntmp.c +++ b/drivers/net/ethernet/freescale/enetc/ntmp.c @@ -227,7 +227,7 @@ static const char *ntmp_table_name(int tbl_id) return "RSS Table"; default: return "Unknown Table"; - }; + } } static int ntmp_delete_entry_by_id(struct ntmp_user *user, int tbl_id, diff --git a/drivers/net/ethernet/freescale/fs_enet/fs_enet-main.c b/drivers/net/ethernet/freescale/fs_enet/fs_enet-main.c index 2cbcc29408e3..d3c772ed5fc9 100644 --- a/drivers/net/ethernet/freescale/fs_enet/fs_enet-main.c +++ b/drivers/net/ethernet/freescale/fs_enet/fs_enet-main.c @@ -951,7 +951,9 @@ static int fs_enet_probe(struct platform_device *ofdev) spin_lock_init(&fep->lock); spin_lock_init(&fep->tx_lock); - of_get_ethdev_address(ofdev->dev.of_node, ndev); + ret = of_get_ethdev_address(ofdev->dev.of_node, ndev); + if (ret == -EPROBE_DEFER) + goto out_cleanup_data; ret = fep->ops->allocate_bd(ndev); if (ret) diff --git a/drivers/net/ethernet/freescale/ucc_geth.c b/drivers/net/ethernet/freescale/ucc_geth.c index 3607681650ce..7af4b5e3f38e 100644 --- a/drivers/net/ethernet/freescale/ucc_geth.c +++ b/drivers/net/ethernet/freescale/ucc_geth.c @@ -3470,14 +3470,13 @@ static int ucc_geth_probe(struct platform_device* ofdev) phy_node = of_parse_phandle(np, "phy-handle", 0); if (phy_node) { prop = of_get_property(phy_node, "interface", NULL); + of_node_put(phy_node); if (prop) { dev_err(&ofdev->dev, "Device-tree property 'interface' is no longer supported. Please use 'phy-connection-type' instead."); - of_node_put(phy_node); err = -EINVAL; goto err_put_tbi; } - of_node_put(phy_node); } err = of_get_phy_mode(np, &phy_interface); diff --git a/drivers/net/ethernet/google/gve/gve.h b/drivers/net/ethernet/google/gve/gve.h index cbdf3a842cfe..1d66d3834f7e 100644 --- a/drivers/net/ethernet/google/gve/gve.h +++ b/drivers/net/ethernet/google/gve/gve.h @@ -79,8 +79,6 @@ #define GVE_DEFAULT_HEADER_BUFFER_SIZE 128 -#define DQO_QPL_DEFAULT_TX_PAGES 512 - /* Maximum TSO size supported on DQO */ #define GVE_DQO_TX_MAX 0x3FFFF @@ -711,6 +709,7 @@ struct gve_ptype_lut { /* Parameters for allocating resources for tx queues */ struct gve_tx_alloc_rings_cfg { struct gve_tx_queue_config *qcfg; + u16 pages_per_qpl; u16 num_xdp_rings; @@ -726,6 +725,7 @@ struct gve_rx_alloc_rings_cfg { /* tx config is also needed to determine QPL ids */ struct gve_rx_queue_config *qcfg_rx; struct gve_tx_queue_config *qcfg_tx; + u16 pages_per_qpl; u16 ring_size; u16 packet_buffer_size; @@ -816,7 +816,8 @@ struct gve_priv { u16 min_rx_desc_cnt; bool modify_ring_size_enabled; bool default_min_ring_size; - u16 tx_pages_per_qpl; /* Suggested number of pages per qpl for TX queues by NIC */ + u16 tx_pages_per_qpl; + u16 rx_pages_per_qpl; u64 max_registered_pages; u64 num_registered_pages; /* num pages registered with NIC */ struct bpf_prog *xdp_prog; /* XDP BPF program */ @@ -1150,14 +1151,6 @@ static inline u32 gve_rx_start_qpl_id(const struct gve_tx_queue_config *tx_cfg) return gve_get_rx_qpl_id(tx_cfg, 0); } -static inline u32 gve_get_rx_pages_per_qpl_dqo(u32 rx_desc_cnt) -{ - /* For DQO, page count should be more than ring size for - * out-of-order completions. Set it to two times of ring size. - */ - return 2 * rx_desc_cnt; -} - /* Returns the correct dma direction for tx and rx qpls */ static inline enum dma_data_direction gve_qpl_dma_dir(struct gve_priv *priv, int id) @@ -1308,6 +1301,9 @@ int gve_reset(struct gve_priv *priv, bool attempt_teardown); void gve_get_curr_alloc_cfgs(struct gve_priv *priv, struct gve_tx_alloc_rings_cfg *tx_alloc_cfg, struct gve_rx_alloc_rings_cfg *rx_alloc_cfg); +void gve_update_num_qpl_pages(struct gve_priv *priv, + struct gve_rx_alloc_rings_cfg *rx_alloc_cfg, + struct gve_tx_alloc_rings_cfg *tx_alloc_cfg); int gve_adjust_config(struct gve_priv *priv, struct gve_tx_alloc_rings_cfg *tx_alloc_cfg, struct gve_rx_alloc_rings_cfg *rx_alloc_cfg); diff --git a/drivers/net/ethernet/google/gve/gve_adminq.c b/drivers/net/ethernet/google/gve/gve_adminq.c index b72cc0fa2ba2..08587bf40ed4 100644 --- a/drivers/net/ethernet/google/gve/gve_adminq.c +++ b/drivers/net/ethernet/google/gve/gve_adminq.c @@ -791,7 +791,7 @@ static void gve_adminq_get_create_rx_queue_cmd(struct gve_priv *priv, cmd->create_rx_queue.rx_buff_ring_size = cpu_to_be16(priv->rx_desc_cnt); cmd->create_rx_queue.enable_rsc = - !!(priv->dev->features & NETIF_F_LRO); + !!(priv->dev->features & NETIF_F_GRO_HW); if (priv->header_split_enabled) cmd->create_rx_queue.header_buffer_size = cpu_to_be16(priv->header_buf_size); @@ -970,14 +970,6 @@ static void gve_enable_supported_features(struct gve_priv *priv, priv->dev->max_mtu = be16_to_cpu(dev_op_jumbo_frames->max_mtu); } - /* Override pages for qpl for DQO-QPL */ - if (dev_op_dqo_qpl) { - priv->tx_pages_per_qpl = - be16_to_cpu(dev_op_dqo_qpl->tx_pages_per_qpl); - if (priv->tx_pages_per_qpl == 0) - priv->tx_pages_per_qpl = DQO_QPL_DEFAULT_TX_PAGES; - } - if (dev_op_buffer_sizes && (supported_features_mask & GVE_SUP_BUFFER_SIZES_MASK)) { priv->max_rx_buffer_size = @@ -997,12 +989,10 @@ static void gve_enable_supported_features(struct gve_priv *priv, if (dev_op_modify_ring && (supported_features_mask & GVE_SUP_MODIFY_RING_MASK)) { priv->modify_ring_size_enabled = true; - - /* max ring size for DQO QPL should not be overwritten because of device limit */ - if (priv->queue_format != GVE_DQO_QPL_FORMAT) { - priv->max_rx_desc_cnt = be16_to_cpu(dev_op_modify_ring->max_rx_ring_size); - priv->max_tx_desc_cnt = be16_to_cpu(dev_op_modify_ring->max_tx_ring_size); - } + priv->max_rx_desc_cnt = + be16_to_cpu(dev_op_modify_ring->max_rx_ring_size); + priv->max_tx_desc_cnt = + be16_to_cpu(dev_op_modify_ring->max_tx_ring_size); if (priv->default_min_ring_size) { /* If device hasn't provided minimums, use default minimums */ priv->min_tx_desc_cnt = GVE_DEFAULT_MIN_TX_RING_SIZE; @@ -1127,9 +1117,13 @@ int gve_adminq_describe_device(struct gve_priv *priv) gve_set_default_rss_sizes(priv); - /* DQO supports LRO. */ - if (!gve_is_gqi(priv)) - priv->dev->hw_features |= NETIF_F_LRO; + /* DQO supports HW-GRO and UDP_GSO */ + if (gve_is_dqo(priv)) { + u64 additional_features = NETIF_F_GRO_HW | NETIF_F_GSO_UDP_L4; + + priv->dev->hw_features |= additional_features; + priv->dev->features |= additional_features; + } priv->max_registered_pages = be64_to_cpu(descriptor->max_registered_pages); diff --git a/drivers/net/ethernet/google/gve/gve_buffer_mgmt_dqo.c b/drivers/net/ethernet/google/gve/gve_buffer_mgmt_dqo.c index 0e2b703c673a..6880d1531ba5 100644 --- a/drivers/net/ethernet/google/gve/gve_buffer_mgmt_dqo.c +++ b/drivers/net/ethernet/google/gve/gve_buffer_mgmt_dqo.c @@ -133,7 +133,7 @@ int gve_alloc_qpl_page_dqo(struct gve_rx_ring *rx, u32 idx; idx = rx->dqo.next_qpl_page_idx; - if (idx >= gve_get_rx_pages_per_qpl_dqo(priv->rx_desc_cnt)) { + if (idx >= priv->rx_pages_per_qpl) { net_err_ratelimited("%s: Out of QPL pages\n", priv->dev->name); return -ENOMEM; diff --git a/drivers/net/ethernet/google/gve/gve_main.c b/drivers/net/ethernet/google/gve/gve_main.c index 9eb4b3614c4f..424d973c97f2 100644 --- a/drivers/net/ethernet/google/gve/gve_main.c +++ b/drivers/net/ethernet/google/gve/gve_main.c @@ -11,6 +11,7 @@ #include <linux/filter.h> #include <linux/interrupt.h> #include <linux/irq.h> +#include <linux/math64.h> #include <linux/module.h> #include <linux/pci.h> #include <linux/sched.h> @@ -966,6 +967,7 @@ static void gve_tx_get_curr_alloc_cfg(struct gve_priv *priv, cfg->qcfg = &priv->tx_cfg; cfg->raw_addressing = !gve_is_qpl(priv); cfg->ring_size = priv->tx_desc_cnt; + cfg->pages_per_qpl = priv->tx_pages_per_qpl; cfg->num_xdp_rings = cfg->qcfg->num_xdp_queues; cfg->tx = priv->tx; } @@ -997,12 +999,48 @@ static void gve_tx_start_rings(struct gve_priv *priv, int num_rings) } } +void gve_update_num_qpl_pages(struct gve_priv *priv, + struct gve_rx_alloc_rings_cfg *rx_alloc_cfg, + struct gve_tx_alloc_rings_cfg *tx_alloc_cfg) +{ + u64 ideal_tx_pages, ideal_rx_pages; + u16 tx_num_queues, rx_num_queues; + u64 max_pages, tx_pages; + + if (priv->queue_format == GVE_GQI_QPL_FORMAT) { + rx_alloc_cfg->pages_per_qpl = rx_alloc_cfg->ring_size; + } else if (priv->queue_format == GVE_DQO_QPL_FORMAT) { + /* + * We want 2 pages per RX descriptor and half a page per TX + * descriptor, which means the fraction ideal_tx_pages / + * (ideal_tx_pages + ideal_rx_pages) of the pages we allocate + * should be for TX. Shrink proportionally as necessary to avoid + * allocating more than max_registered_pages total pages. + */ + tx_num_queues = tx_alloc_cfg->qcfg->num_queues; + rx_num_queues = rx_alloc_cfg->qcfg_rx->num_queues; + + ideal_tx_pages = tx_alloc_cfg->ring_size * tx_num_queues / 2; + ideal_rx_pages = rx_alloc_cfg->ring_size * rx_num_queues * 2; + max_pages = min(priv->max_registered_pages, + ideal_tx_pages + ideal_rx_pages); + + tx_pages = div64_u64(max_pages * ideal_tx_pages, + ideal_tx_pages + ideal_rx_pages); + tx_alloc_cfg->pages_per_qpl = div_u64(tx_pages, tx_num_queues); + rx_alloc_cfg->pages_per_qpl = div_u64(max_pages - tx_pages, + rx_num_queues); + } +} + static int gve_queues_mem_alloc(struct gve_priv *priv, struct gve_tx_alloc_rings_cfg *tx_alloc_cfg, struct gve_rx_alloc_rings_cfg *rx_alloc_cfg) { int err; + gve_update_num_qpl_pages(priv, rx_alloc_cfg, tx_alloc_cfg); + if (gve_is_gqi(priv)) err = gve_tx_alloc_rings_gqi(priv, tx_alloc_cfg); else @@ -1293,6 +1331,7 @@ static void gve_rx_get_curr_alloc_cfg(struct gve_priv *priv, cfg->raw_addressing = !gve_is_qpl(priv); cfg->enable_header_split = priv->header_split_enabled; cfg->ring_size = priv->rx_desc_cnt; + cfg->pages_per_qpl = priv->rx_pages_per_qpl; cfg->packet_buffer_size = priv->rx_cfg.packet_buffer_size; cfg->rx = priv->rx; cfg->xdp = !!cfg->qcfg_tx->num_xdp_queues; @@ -1372,6 +1411,8 @@ static int gve_queues_start(struct gve_priv *priv, priv->rx_cfg = *rx_alloc_cfg->qcfg_rx; priv->tx_desc_cnt = tx_alloc_cfg->ring_size; priv->rx_desc_cnt = rx_alloc_cfg->ring_size; + priv->tx_pages_per_qpl = tx_alloc_cfg->pages_per_qpl; + priv->rx_pages_per_qpl = rx_alloc_cfg->pages_per_qpl; gve_tx_start_rings(priv, gve_num_tx_queues(priv)); gve_rx_start_rings(priv, rx_alloc_cfg->qcfg_rx->num_queues); @@ -1717,9 +1758,9 @@ static int gve_verify_xdp_configuration(struct net_device *dev, struct gve_priv *priv = netdev_priv(dev); u16 max_xdp_mtu; - if (dev->features & NETIF_F_LRO) { + if (dev->features & NETIF_F_GRO_HW) { NL_SET_ERR_MSG_MOD(extack, - "XDP is not supported when LRO is on."); + "XDP is not supported when HW-GRO is on."); return -EOPNOTSUPP; } @@ -2136,12 +2177,13 @@ static int gve_set_features(struct net_device *netdev, gve_get_curr_alloc_cfgs(priv, &tx_alloc_cfg, &rx_alloc_cfg); - if ((netdev->features & NETIF_F_LRO) != (features & NETIF_F_LRO)) { - netdev->features ^= NETIF_F_LRO; - if (priv->xdp_prog && (netdev->features & NETIF_F_LRO)) { + if ((netdev->features & NETIF_F_GRO_HW) != + (features & NETIF_F_GRO_HW)) { + netdev->features ^= NETIF_F_GRO_HW; + if (priv->xdp_prog && (netdev->features & NETIF_F_GRO_HW)) { netdev_warn(netdev, - "XDP is not supported when LRO is on.\n"); - err = -EOPNOTSUPP; + "HW-GRO is not supported when XDP is on."); + err = -EOPNOTSUPP; goto revert_features; } if (netif_running(netdev)) { diff --git a/drivers/net/ethernet/google/gve/gve_rx.c b/drivers/net/ethernet/google/gve/gve_rx.c index 4eb6ed29d3cc..81ea800e66e9 100644 --- a/drivers/net/ethernet/google/gve/gve_rx.c +++ b/drivers/net/ethernet/google/gve/gve_rx.c @@ -278,7 +278,6 @@ int gve_rx_alloc_ring_gqi(struct gve_priv *priv, struct device *hdev = &priv->pdev->dev; u32 slots = cfg->ring_size; int filled_pages; - int qpl_page_cnt; u32 qpl_id = 0; size_t bytes; int err; @@ -314,10 +313,8 @@ int gve_rx_alloc_ring_gqi(struct gve_priv *priv, if (!rx->data.raw_addressing) { qpl_id = gve_get_rx_qpl_id(cfg->qcfg_tx, rx->q_num); - qpl_page_cnt = cfg->ring_size; - rx->data.qpl = gve_alloc_queue_page_list(priv, qpl_id, - qpl_page_cnt); + cfg->pages_per_qpl); if (!rx->data.qpl) { err = -ENOMEM; goto abort_with_copy_pool; diff --git a/drivers/net/ethernet/google/gve/gve_rx_dqo.c b/drivers/net/ethernet/google/gve/gve_rx_dqo.c index c706c7932159..7924dce719e2 100644 --- a/drivers/net/ethernet/google/gve/gve_rx_dqo.c +++ b/drivers/net/ethernet/google/gve/gve_rx_dqo.c @@ -218,7 +218,6 @@ int gve_rx_alloc_ring_dqo(struct gve_priv *priv, { struct device *hdev = &priv->pdev->dev; struct page_pool *pool; - int qpl_page_cnt; size_t size; u32 qpl_id; @@ -246,7 +245,7 @@ int gve_rx_alloc_ring_dqo(struct gve_priv *priv, XSK_CHECK_PRIV_TYPE(struct gve_xdp_buff); rx->dqo.num_buf_states = cfg->raw_addressing ? buffer_queue_slots : - gve_get_rx_pages_per_qpl_dqo(cfg->ring_size); + cfg->pages_per_qpl; rx->dqo.buf_states = kvcalloc_node(rx->dqo.num_buf_states, sizeof(rx->dqo.buf_states[0]), GFP_KERNEL, priv->numa_node); @@ -281,10 +280,9 @@ int gve_rx_alloc_ring_dqo(struct gve_priv *priv, rx->dqo.page_pool = pool; } else { qpl_id = gve_get_rx_qpl_id(cfg->qcfg_tx, rx->q_num); - qpl_page_cnt = gve_get_rx_pages_per_qpl_dqo(cfg->ring_size); rx->dqo.qpl = gve_alloc_queue_page_list(priv, qpl_id, - qpl_page_cnt); + cfg->pages_per_qpl); if (!rx->dqo.qpl) goto err; rx->dqo.next_qpl_page_idx = 0; @@ -944,11 +942,18 @@ static int gve_rx_complete_rsc(struct sk_buff *skb, struct gve_ptype ptype) { struct skb_shared_info *shinfo = skb_shinfo(skb); + int rsc_segments, rsc_seg_len, hdr_len; + skb_frag_t *frag; + void *va; - /* Only TCP is supported right now. */ + /* HW-GRO only coalesces TCP. */ if (ptype.l4_type != GVE_L4_TYPE_TCP) return -EINVAL; + rsc_seg_len = le16_to_cpu(desc->rsc_seg_len); + if (!rsc_seg_len) + return 0; + switch (ptype.l3_type) { case GVE_L3_TYPE_IPV4: shinfo->gso_type = SKB_GSO_TCPV4; @@ -960,7 +965,31 @@ static int gve_rx_complete_rsc(struct sk_buff *skb, return -EINVAL; } - shinfo->gso_size = le16_to_cpu(desc->rsc_seg_len); + if (skb_headlen(skb)) { + /* With header-split, payload is in the non-linear part */ + rsc_segments = DIV_ROUND_UP(skb->data_len, rsc_seg_len); + } else { + /* HW-GRO packets are guaranteed to have complete TCP/IP + * headers in frag[0] when header-split is not enabled. + */ + frag = &skb_shinfo(skb)->frags[0]; + va = skb_frag_address(frag); + hdr_len = + eth_get_headlen(skb->dev, va, skb_frag_size(frag)); + rsc_segments = DIV_ROUND_UP(skb->len - hdr_len, rsc_seg_len); + skb_copy_to_linear_data(skb, va, hdr_len); + skb_frag_size_sub(frag, hdr_len); + /* Verify we didn't empty the fragment completely as that could + * otherwise lead to page leaks. + */ + DEBUG_NET_WARN_ON_ONCE(!skb_frag_size(frag)); + skb_frag_off_add(frag, hdr_len); + skb->data_len -= hdr_len; + skb->tail += hdr_len; + } + shinfo->gso_size = rsc_seg_len; + shinfo->gso_segs = rsc_segments; + return 0; } @@ -993,7 +1022,7 @@ static int gve_rx_complete_skb(struct gve_rx_ring *rx, struct napi_struct *napi, return err; } - if (skb_headlen(rx->ctx.skb_head) == 0) + if (rx->ctx.skb_head == napi->skb) napi_gro_frags(napi); else napi_gro_receive(napi, rx->ctx.skb_head); diff --git a/drivers/net/ethernet/google/gve/gve_tx.c b/drivers/net/ethernet/google/gve/gve_tx.c index 0b856e269eac..79338fd9bf66 100644 --- a/drivers/net/ethernet/google/gve/gve_tx.c +++ b/drivers/net/ethernet/google/gve/gve_tx.c @@ -264,7 +264,6 @@ static int gve_tx_alloc_ring_gqi(struct gve_priv *priv, int idx) { struct device *hdev = &priv->pdev->dev; - int qpl_page_cnt; u32 qpl_id = 0; size_t bytes; @@ -291,10 +290,8 @@ static int gve_tx_alloc_ring_gqi(struct gve_priv *priv, tx->dev = hdev; if (!tx->raw_addressing) { qpl_id = gve_tx_qpl_id(priv, tx->q_num); - qpl_page_cnt = priv->tx_pages_per_qpl; - tx->tx_fifo.qpl = gve_alloc_queue_page_list(priv, qpl_id, - qpl_page_cnt); + cfg->pages_per_qpl); if (!tx->tx_fifo.qpl) goto abort_with_desc; diff --git a/drivers/net/ethernet/google/gve/gve_tx_dqo.c b/drivers/net/ethernet/google/gve/gve_tx_dqo.c index 4db9d4f9ed6f..80ab0a449ff5 100644 --- a/drivers/net/ethernet/google/gve/gve_tx_dqo.c +++ b/drivers/net/ethernet/google/gve/gve_tx_dqo.c @@ -311,7 +311,6 @@ static int gve_tx_alloc_ring_dqo(struct gve_priv *priv, { struct device *hdev = &priv->pdev->dev; int num_pending_packets; - int qpl_page_cnt; size_t bytes; u32 qpl_id; int i; @@ -392,10 +391,9 @@ static int gve_tx_alloc_ring_dqo(struct gve_priv *priv, if (!cfg->raw_addressing) { qpl_id = gve_tx_qpl_id(priv, tx->q_num); - qpl_page_cnt = priv->tx_pages_per_qpl; tx->dqo.qpl = gve_alloc_queue_page_list(priv, qpl_id, - qpl_page_cnt); + cfg->pages_per_qpl); if (!tx->dqo.qpl) goto err; @@ -572,9 +570,11 @@ static void gve_tx_fill_pkt_desc_dqo(struct gve_tx_ring *tx, u32 *desc_idx, */ static int gve_prep_tso(struct sk_buff *skb) { + struct skb_shared_info *shinfo = skb_shinfo(skb); + u32 paylen, l4_start; struct tcphdr *tcp; + struct udphdr *udp; int header_len; - u32 paylen; int err; /* Note: HW requires MSS (gso_size) to be <= 9728 and the total length @@ -585,21 +585,34 @@ static int gve_prep_tso(struct sk_buff *skb) * - Kernel will not produce a TSO larger than 64k */ - if (unlikely(skb_shinfo(skb)->gso_size < GVE_TX_MIN_TSO_MSS_DQO)) + if (unlikely(shinfo->gso_size < GVE_TX_MIN_TSO_MSS_DQO)) return -1; - if (!(skb_shinfo(skb)->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6))) - return -EINVAL; - /* Needed because we will modify header. */ err = skb_cow_head(skb, 0); if (err < 0) return err; - tcp = tcp_hdr(skb); - paylen = skb->len - skb_transport_offset(skb); - csum_replace_by_diff(&tcp->check, (__force __wsum)htonl(paylen)); - header_len = skb_tcp_all_headers(skb); + l4_start = skb_transport_offset(skb); + paylen = skb->len - l4_start; + + switch (shinfo->gso_type) { + case SKB_GSO_TCPV4: + case SKB_GSO_TCPV6: + tcp = tcp_hdr(skb); + csum_replace_by_diff(&tcp->check, + (__force __wsum)htonl(paylen)); + header_len = skb_tcp_all_headers(skb); + break; + case SKB_GSO_UDP_L4: + udp = udp_hdr(skb); + csum_replace_by_diff(&udp->check, + (__force __wsum)htonl(paylen)); + header_len = sizeof(struct udphdr) + l4_start; + break; + default: + return -EINVAL; + } if (unlikely(header_len > GVE_TX_MAX_HDR_SIZE_DQO)) return -EINVAL; diff --git a/drivers/net/ethernet/huawei/hinic3/Makefile b/drivers/net/ethernet/huawei/hinic3/Makefile index 26c05ecf31c9..4ebad3a4f943 100644 --- a/drivers/net/ethernet/huawei/hinic3/Makefile +++ b/drivers/net/ethernet/huawei/hinic3/Makefile @@ -6,6 +6,7 @@ obj-$(CONFIG_HINIC3) += hinic3.o hinic3-objs := hinic3_cmdq.o \ hinic3_common.o \ hinic3_eqs.o \ + hinic3_ethtool.o \ hinic3_filter.o \ hinic3_hw_cfg.o \ hinic3_hw_comm.o \ diff --git a/drivers/net/ethernet/huawei/hinic3/hinic3_cmdq.c b/drivers/net/ethernet/huawei/hinic3/hinic3_cmdq.c index bc5b80c31693..1b0ba244a68f 100644 --- a/drivers/net/ethernet/huawei/hinic3/hinic3_cmdq.c +++ b/drivers/net/ethernet/huawei/hinic3/hinic3_cmdq.c @@ -6,12 +6,14 @@ #include <linux/dma-mapping.h> #include "hinic3_cmdq.h" +#include "hinic3_eqs.h" #include "hinic3_hwdev.h" #include "hinic3_hwif.h" #include "hinic3_mbox.h" #define CMDQ_BUF_SIZE 2048 #define CMDQ_WQEBB_SIZE 64 +#define CMDQ_WQE_HEAD_LEN 32 #define CMDQ_CMD_TIMEOUT 5000 #define CMDQ_ENABLE_WAIT_TIMEOUT 300 @@ -61,6 +63,10 @@ #define CMDQ_DB_HEAD_SET(val, member) \ FIELD_PREP(CMDQ_DB_HEAD_##member##_MASK, val) +#define SAVED_DATA_ARM_MASK BIT(31) +#define SAVED_DATA_SET(val, member) \ + FIELD_PREP(SAVED_DATA_##member##_MASK, val) + #define CMDQ_CEQE_TYPE_MASK GENMASK(2, 0) #define CMDQ_CEQE_GET(val, member) \ FIELD_GET(CMDQ_CEQE_##member##_MASK, le32_to_cpu(val)) @@ -84,6 +90,10 @@ enum cmdq_data_format { CMDQ_DATA_DIRECT = 1, }; +enum cmdq_scmd_type { + CMDQ_SET_ARM_CMD = 2, +}; + enum cmdq_ctrl_sect_len { CMDQ_CTRL_SECT_LEN = 1, CMDQ_CTRL_DIRECT_SECT_LEN = 2, @@ -106,6 +116,20 @@ enum cmdq_cmd_type { #define CMDQ_WQE_NUM_WQEBBS 1 +static void hinic3_dump_cmdq_wqe_head(struct hinic3_hwdev *hwdev, + struct cmdq_wqe *wqe) +{ + u32 *data = (u32 *)wqe; + u32 i; + + for (i = 0; i < (CMDQ_WQE_HEAD_LEN / sizeof(u32)); i += 0x4) { + dev_dbg(hwdev->dev, + "wqe data: 0x%08x, 0x%08x, 0x%08x, 0x%08x\n", + *(data + i), *(data + i + 0x1), *(data + i + 0x2), + *(data + i + 0x3)); + } +} + static struct cmdq_wqe *cmdq_read_wqe(struct hinic3_wq *wq, u16 *ci) { if (hinic3_wq_get_used(wq) == 0) @@ -166,6 +190,11 @@ static void cmdq_clear_cmd_buf(struct hinic3_cmdq_cmd_info *cmd_info, hinic3_free_cmd_buf(hwdev, cmd_info->buf_in); cmd_info->buf_in = NULL; } + + if (cmd_info->buf_out) { + hinic3_free_cmd_buf(hwdev, cmd_info->buf_out); + cmd_info->buf_out = NULL; + } } static void clear_wqe_complete_bit(struct hinic3_cmdq *cmdq, @@ -189,6 +218,20 @@ static void clear_wqe_complete_bit(struct hinic3_cmdq *cmdq, hinic3_wq_put_wqebbs(&cmdq->wq, CMDQ_WQE_NUM_WQEBBS); } +static int cmdq_arm_ceq_handler(struct hinic3_cmdq *cmdq, + struct cmdq_wqe *wqe, u16 ci) +{ + struct cmdq_ctrl *ctrl = &wqe->wqe_scmd.ctrl; + __le32 ctrl_info = ctrl->ctrl_info; + + if (!CMDQ_WQE_COMPLETED(ctrl_info)) + return -EBUSY; + + clear_wqe_complete_bit(cmdq, wqe, ci); + + return 0; +} + static void cmdq_update_cmd_status(struct hinic3_cmdq *cmdq, u16 prod_idx, struct cmdq_wqe *wqe) { @@ -252,11 +295,17 @@ void hinic3_cmdq_ceq_handler(struct hinic3_hwdev *hwdev, __le32 ceqe_data) case HINIC3_CMD_TYPE_TIMEOUT: dev_warn(hwdev->dev, "Cmdq timeout, q_id: %u, ci: %u\n", cmdq_type, ci); + hinic3_dump_cmdq_wqe_head(hwdev, wqe); fallthrough; case HINIC3_CMD_TYPE_FAKE_TIMEOUT: cmdq_clear_cmd_buf(cmd_info, hwdev); clear_wqe_complete_bit(cmdq, wqe, ci); break; + case HINIC3_CMD_TYPE_SET_ARM: + /* arm_bit was set until here */ + if (cmdq_arm_ceq_handler(cmdq, wqe, ci)) + return; + break; default: /* only arm bit is using scmd wqe, * the other wqe is lcmd @@ -283,6 +332,18 @@ void hinic3_cmdq_ceq_handler(struct hinic3_hwdev *hwdev, __le32 ceqe_data) } } +static int cmdq_params_valid(const struct hinic3_hwdev *hwdev, + const struct hinic3_cmd_buf *buf_in) +{ + if (le16_to_cpu(buf_in->size) > CMDQ_BUF_SIZE) { + dev_err(hwdev->dev, "Invalid CMDQ buffer size: 0x%x\n", + le16_to_cpu(buf_in->size)); + return -EINVAL; + } + + return 0; +} + static int wait_cmdqs_enable(struct hinic3_cmdqs *cmdqs) { unsigned long end; @@ -292,7 +353,8 @@ static int wait_cmdqs_enable(struct hinic3_cmdqs *cmdqs) if (cmdqs->status & HINIC3_CMDQ_ENABLE) return 0; usleep_range(1000, 2000); - } while (time_before(jiffies, end) && !cmdqs->disable_flag); + } while (time_before(jiffies, end) && !cmdqs->disable_flag && + cmdqs->hwdev->chip_present_flag); cmdqs->disable_flag = 1; @@ -356,6 +418,7 @@ static void cmdq_prepare_wqe_ctrl(struct cmdq_wqe *wqe, u8 wrapped, enum cmdq_bufdesc_len buf_len) { struct cmdq_header *hdr = CMDQ_WQE_HEADER(wqe); + __le32 saved_data = hdr->saved_data; enum cmdq_ctrl_sect_len ctrl_len; struct cmdq_wqe_lcmd *wqe_lcmd; struct cmdq_wqe_scmd *wqe_scmd; @@ -386,6 +449,11 @@ static void cmdq_prepare_wqe_ctrl(struct cmdq_wqe *wqe, u8 wrapped, CMDQ_WQE_HDR_SET(3, COMPLETE_SECT_LEN) | CMDQ_WQE_HDR_SET(ctrl_len, CTRL_LEN) | CMDQ_WQE_HDR_SET(wrapped, HW_BUSY_BIT)); + + saved_data &= ~cpu_to_le32(SAVED_DATA_ARM_MASK); + if (cmd == CMDQ_SET_ARM_CMD && mod == MGMT_MOD_COMM) + saved_data |= cpu_to_le32(SAVED_DATA_SET(1, ARM)); + hdr->saved_data = saved_data; } static void cmdq_set_lcmd_wqe(struct cmdq_wqe *wqe, @@ -485,12 +553,15 @@ static int wait_cmdq_sync_cmd_completion(struct hinic3_cmdq *cmdq, clear_cmd_info(cmd_info, saved_cmd_info); spin_unlock_bh(&cmdq->cmdq_lock); + hinic3_dump_ceq_info(cmdq->hwdev); + return err; } -static int cmdq_sync_cmd_direct_resp(struct hinic3_cmdq *cmdq, u8 mod, u8 cmd, - struct hinic3_cmd_buf *buf_in, - __le64 *out_param) +static int cmdq_sync_cmd_exec(struct hinic3_cmdq *cmdq, u8 mod, u8 cmd, + struct hinic3_cmd_buf *buf_in, + struct hinic3_cmd_buf *buf_out, + __le64 *out_param, u8 cmd_type, u8 wqe_cmd) { struct hinic3_cmdq_cmd_info *cmd_info, saved_cmd_info; int cmpt_code = CMDQ_SEND_CMPT_CODE; @@ -520,31 +591,35 @@ static int cmdq_sync_cmd_direct_resp(struct hinic3_cmdq *cmdq, u8 mod, u8 cmd, cmd_info = &cmdq->cmd_infos[curr_prod_idx]; init_completion(&done); refcount_inc(&buf_in->ref_cnt); - cmd_info->cmd_type = HINIC3_CMD_TYPE_DIRECT_RESP; + if (buf_out) + refcount_inc(&buf_out->ref_cnt); + + cmd_info->cmd_type = cmd_type; cmd_info->done = &done; cmd_info->errcode = &errcode; cmd_info->direct_resp = out_param; cmd_info->cmpt_code = &cmpt_code; cmd_info->buf_in = buf_in; + if (buf_out) + cmd_info->buf_out = buf_out; + saved_cmd_info = *cmd_info; - cmdq_set_lcmd_wqe(&wqe, CMDQ_CMD_DIRECT_RESP, buf_in, NULL, + cmdq_set_lcmd_wqe(&wqe, wqe_cmd, buf_in, buf_out, wrapped, mod, cmd, curr_prod_idx); cmdq_wqe_fill(curr_wqe, &wqe); (cmd_info->cmdq_msg_id)++; curr_msg_id = cmd_info->cmdq_msg_id; - cmdq_set_db(cmdq, HINIC3_CMDQ_SYNC, next_prod_idx); + cmdq_set_db(cmdq, cmdq->cmdq_type, next_prod_idx); spin_unlock_bh(&cmdq->cmdq_lock); err = wait_cmdq_sync_cmd_completion(cmdq, cmd_info, &saved_cmd_info, curr_msg_id, curr_prod_idx, curr_wqe, CMDQ_CMD_TIMEOUT); - if (err) { + if (err) dev_err(cmdq->hwdev->dev, "Cmdq sync command timeout, mod: %u, cmd: %u, prod idx: 0x%x\n", mod, cmd, curr_prod_idx); - err = -ETIMEDOUT; - } if (cmpt_code == CMDQ_FORCE_STOP_CMPT_CODE) { dev_dbg(cmdq->hwdev->dev, @@ -557,22 +632,114 @@ static int cmdq_sync_cmd_direct_resp(struct hinic3_cmdq *cmdq, u8 mod, u8 cmd, return err ? err : errcode; } +static int cmdq_sync_cmd_direct_resp(struct hinic3_cmdq *cmdq, u8 mod, u8 cmd, + struct hinic3_cmd_buf *buf_in, + __le64 *out_param) +{ + return cmdq_sync_cmd_exec(cmdq, mod, cmd, buf_in, NULL, out_param, + HINIC3_CMD_TYPE_DIRECT_RESP, + CMDQ_CMD_DIRECT_RESP); +} + +static int cmdq_sync_cmd_detail_resp(struct hinic3_cmdq *cmdq, u8 mod, u8 cmd, + struct hinic3_cmd_buf *buf_in, + struct hinic3_cmd_buf *buf_out, + __le64 *out_param) +{ + return cmdq_sync_cmd_exec(cmdq, mod, cmd, buf_in, buf_out, out_param, + HINIC3_CMD_TYPE_SGE_RESP, + CMDQ_CMD_SGE_RESP); +} + +int hinic3_cmd_buf_pair_init(struct hinic3_hwdev *hwdev, + struct hinic3_cmd_buf_pair *pair) +{ + pair->in = hinic3_alloc_cmd_buf(hwdev); + if (!pair->in) + goto err_out; + + pair->out = hinic3_alloc_cmd_buf(hwdev); + if (!pair->out) + goto err_free_cmd_buf_in; + + return 0; + +err_free_cmd_buf_in: + hinic3_free_cmd_buf(hwdev, pair->in); +err_out: + return -ENOMEM; +} + +void hinic3_cmd_buf_pair_uninit(struct hinic3_hwdev *hwdev, + struct hinic3_cmd_buf_pair *pair) +{ + hinic3_free_cmd_buf(hwdev, pair->in); + hinic3_free_cmd_buf(hwdev, pair->out); +} + int hinic3_cmdq_direct_resp(struct hinic3_hwdev *hwdev, u8 mod, u8 cmd, struct hinic3_cmd_buf *buf_in, __le64 *out_param) { struct hinic3_cmdqs *cmdqs; int err; + if (!hwdev->chip_present_flag) + return -ETIMEDOUT; + + err = cmdq_params_valid(hwdev, buf_in); + if (err) { + dev_err(hwdev->dev, "Invalid CMDQ parameters\n"); + goto err_out; + } + cmdqs = hwdev->cmdqs; err = wait_cmdqs_enable(cmdqs); if (err) { dev_err(hwdev->dev, "Cmdq is disabled\n"); - return err; + goto err_out; } err = cmdq_sync_cmd_direct_resp(&cmdqs->cmdq[HINIC3_CMDQ_SYNC], mod, cmd, buf_in, out_param); + if (err) + goto err_out; + + return 0; +err_out: + return err; +} + +int hinic3_cmdq_detail_resp(struct hinic3_hwdev *hwdev, u8 mod, u8 cmd, + struct hinic3_cmd_buf *buf_in, + struct hinic3_cmd_buf *buf_out, __le64 *out_param) +{ + struct hinic3_cmdqs *cmdqs; + int err; + + if (!hwdev->chip_present_flag) + return -ETIMEDOUT; + + err = cmdq_params_valid(hwdev, buf_in); + if (err) + goto err_out; + + cmdqs = hwdev->cmdqs; + + err = wait_cmdqs_enable(cmdqs); + if (err) { + dev_err(hwdev->dev, "Cmdq is disabled\n"); + goto err_out; + } + + err = cmdq_sync_cmd_detail_resp(&cmdqs->cmdq[HINIC3_CMDQ_SYNC], + mod, cmd, buf_in, buf_out, out_param); + if (err) + goto err_out; + + return 0; + +err_out: return err; } @@ -758,7 +925,8 @@ static int init_cmdqs(struct hinic3_hwdev *hwdev) static void cmdq_flush_sync_cmd(struct hinic3_cmdq_cmd_info *cmd_info) { - if (cmd_info->cmd_type != HINIC3_CMD_TYPE_DIRECT_RESP) + if (cmd_info->cmd_type != HINIC3_CMD_TYPE_DIRECT_RESP && + cmd_info->cmd_type != HINIC3_CMD_TYPE_SGE_RESP) return; cmd_info->cmd_type = HINIC3_CMD_TYPE_FORCE_STOP; @@ -785,7 +953,8 @@ static void hinic3_cmdq_flush_cmd(struct hinic3_cmdq *cmdq) while (cmdq_read_wqe(&cmdq->wq, &ci)) { hinic3_wq_put_wqebbs(&cmdq->wq, CMDQ_WQE_NUM_WQEBBS); cmd_info = &cmdq->cmd_infos[ci]; - if (cmd_info->cmd_type == HINIC3_CMD_TYPE_DIRECT_RESP) + if (cmd_info->cmd_type == HINIC3_CMD_TYPE_DIRECT_RESP || + cmd_info->cmd_type == HINIC3_CMD_TYPE_SGE_RESP) cmdq_flush_sync_cmd(cmd_info); } spin_unlock_bh(&cmdq->cmdq_lock); diff --git a/drivers/net/ethernet/huawei/hinic3/hinic3_cmdq.h b/drivers/net/ethernet/huawei/hinic3/hinic3_cmdq.h index f99c386a2780..29bad2fae7ba 100644 --- a/drivers/net/ethernet/huawei/hinic3/hinic3_cmdq.h +++ b/drivers/net/ethernet/huawei/hinic3/hinic3_cmdq.h @@ -85,7 +85,9 @@ enum hinic3_cmdq_status { enum hinic3_cmdq_cmd_type { HINIC3_CMD_TYPE_NONE, + HINIC3_CMD_TYPE_SET_ARM, HINIC3_CMD_TYPE_DIRECT_RESP, + HINIC3_CMD_TYPE_SGE_RESP, HINIC3_CMD_TYPE_FAKE_TIMEOUT, HINIC3_CMD_TYPE_TIMEOUT, HINIC3_CMD_TYPE_FORCE_STOP, @@ -98,6 +100,11 @@ struct hinic3_cmd_buf { refcount_t ref_cnt; }; +struct hinic3_cmd_buf_pair { + struct hinic3_cmd_buf *in; + struct hinic3_cmd_buf *out; +}; + struct hinic3_cmdq_cmd_info { enum hinic3_cmdq_cmd_type cmd_type; struct completion *done; @@ -107,6 +114,7 @@ struct hinic3_cmdq_cmd_info { __le64 *direct_resp; u64 cmdq_msg_id; struct hinic3_cmd_buf *buf_in; + struct hinic3_cmd_buf *buf_out; }; struct hinic3_cmdq { @@ -146,8 +154,15 @@ void hinic3_free_cmd_buf(struct hinic3_hwdev *hwdev, struct hinic3_cmd_buf *cmd_buf); void hinic3_cmdq_ceq_handler(struct hinic3_hwdev *hwdev, __le32 ceqe_data); +int hinic3_cmd_buf_pair_init(struct hinic3_hwdev *hwdev, + struct hinic3_cmd_buf_pair *pair); +void hinic3_cmd_buf_pair_uninit(struct hinic3_hwdev *hwdev, + struct hinic3_cmd_buf_pair *pair); int hinic3_cmdq_direct_resp(struct hinic3_hwdev *hwdev, u8 mod, u8 cmd, struct hinic3_cmd_buf *buf_in, __le64 *out_param); +int hinic3_cmdq_detail_resp(struct hinic3_hwdev *hwdev, u8 mod, u8 cmd, + struct hinic3_cmd_buf *buf_in, + struct hinic3_cmd_buf *buf_out, __le64 *out_param); void hinic3_cmdq_flush_sync_cmd(struct hinic3_hwdev *hwdev); int hinic3_reinit_cmdq_ctxts(struct hinic3_hwdev *hwdev); diff --git a/drivers/net/ethernet/huawei/hinic3/hinic3_common.c b/drivers/net/ethernet/huawei/hinic3/hinic3_common.c index fe4778d152cf..b28576debdc8 100644 --- a/drivers/net/ethernet/huawei/hinic3/hinic3_common.c +++ b/drivers/net/ethernet/huawei/hinic3/hinic3_common.c @@ -59,10 +59,14 @@ int hinic3_wait_for_timeout(void *priv_data, wait_cpl_handler handler, enum hinic3_wait_return ret; int err; - err = read_poll_timeout(handler, ret, ret == HINIC3_WAIT_PROCESS_CPL, + err = read_poll_timeout(handler, ret, + !(ret & HINIC3_WAIT_PROCESS_WAITING), wait_once_us, wait_total_ms * USEC_PER_MSEC, false, priv_data); + if (ret == HINIC3_WAIT_PROCESS_ERR) + return -EIO; + return err; } diff --git a/drivers/net/ethernet/huawei/hinic3/hinic3_common.h b/drivers/net/ethernet/huawei/hinic3/hinic3_common.h index a8fabfae90fb..c892439fa3cd 100644 --- a/drivers/net/ethernet/huawei/hinic3/hinic3_common.h +++ b/drivers/net/ethernet/huawei/hinic3/hinic3_common.h @@ -21,6 +21,7 @@ struct hinic3_dma_addr_align { enum hinic3_wait_return { HINIC3_WAIT_PROCESS_CPL = 0, HINIC3_WAIT_PROCESS_WAITING = 1, + HINIC3_WAIT_PROCESS_ERR = 2, }; struct hinic3_sge { diff --git a/drivers/net/ethernet/huawei/hinic3/hinic3_csr.h b/drivers/net/ethernet/huawei/hinic3/hinic3_csr.h index f7083a6e7df9..0e32ff34919e 100644 --- a/drivers/net/ethernet/huawei/hinic3/hinic3_csr.h +++ b/drivers/net/ethernet/huawei/hinic3/hinic3_csr.h @@ -76,9 +76,11 @@ #define HINIC3_CSR_AEQ_CTRL_0_ADDR (HINIC3_CFG_REGS_FLAG + 0x200) #define HINIC3_CSR_AEQ_CTRL_1_ADDR (HINIC3_CFG_REGS_FLAG + 0x204) +#define HINIC3_CSR_AEQ_CONS_IDX_ADDR (HINIC3_CFG_REGS_FLAG + 0x208) #define HINIC3_CSR_AEQ_PROD_IDX_ADDR (HINIC3_CFG_REGS_FLAG + 0x20C) #define HINIC3_CSR_AEQ_CI_SIMPLE_INDIR_ADDR (HINIC3_CFG_REGS_FLAG + 0x50) +#define HINIC3_CSR_CEQ_CONS_IDX_ADDR (HINIC3_CFG_REGS_FLAG + 0x288) #define HINIC3_CSR_CEQ_PROD_IDX_ADDR (HINIC3_CFG_REGS_FLAG + 0x28c) #define HINIC3_CSR_CEQ_CI_SIMPLE_INDIR_ADDR (HINIC3_CFG_REGS_FLAG + 0x54) diff --git a/drivers/net/ethernet/huawei/hinic3/hinic3_eqs.c b/drivers/net/ethernet/huawei/hinic3/hinic3_eqs.c index 13a0c6b07660..b8ac1d7bd82b 100644 --- a/drivers/net/ethernet/huawei/hinic3/hinic3_eqs.c +++ b/drivers/net/ethernet/huawei/hinic3/hinic3_eqs.c @@ -56,6 +56,10 @@ #define EQ_CI_SIMPLE_INDIR_SET(val, member) \ FIELD_PREP(EQ_CI_SIMPLE_INDIR_##member##_MASK, val) +#define EQ_CONS_IDX_REG_ADDR(eq) \ + (((eq)->type == HINIC3_AEQ) ? \ + HINIC3_CSR_AEQ_CONS_IDX_ADDR : HINIC3_CSR_CEQ_CONS_IDX_ADDR) + #define EQ_CI_SIMPLE_INDIR_REG_ADDR(eq) \ (((eq)->type == HINIC3_AEQ) ? \ HINIC3_CSR_AEQ_CI_SIMPLE_INDIR_ADDR : \ @@ -353,6 +357,7 @@ static irqreturn_t ceq_interrupt(int irq, void *data) struct hinic3_eq *ceq = data; int err; + ceq->soft_intr_jif = jiffies; /* clear resend timer counters */ hinic3_msix_intr_clear_resend_bit(ceq->hwdev, ceq->msix_entry_idx, EQ_MSIX_RESEND_TIMER_CLEAR); @@ -713,6 +718,39 @@ void hinic3_aeqs_free(struct hinic3_hwdev *hwdev) kfree(aeqs); } +void hinic3_dump_aeq_info(struct hinic3_hwdev *hwdev) +{ + const struct hinic3_aeq_elem *aeqe_pos; + u32 addr, ci, pi, ctrl0, idx; + struct hinic3_eq *eq; + int q_id; + + for (q_id = 0; q_id < hwdev->aeqs->num_aeqs; q_id++) { + eq = &hwdev->aeqs->aeq[q_id]; + /* Indirect access should set q_id first */ + hinic3_hwif_write_reg(eq->hwdev->hwif, + HINIC3_EQ_INDIR_IDX_ADDR(eq->type), + eq->q_id); + + addr = HINIC3_CSR_AEQ_CTRL_0_ADDR; + + ctrl0 = hinic3_hwif_read_reg(hwdev->hwif, addr); + + idx = hinic3_hwif_read_reg(hwdev->hwif, + HINIC3_EQ_INDIR_IDX_ADDR(eq->type)); + + addr = EQ_CONS_IDX_REG_ADDR(eq); + ci = hinic3_hwif_read_reg(hwdev->hwif, addr); + addr = EQ_PROD_IDX_REG_ADDR(eq); + pi = hinic3_hwif_read_reg(hwdev->hwif, addr); + aeqe_pos = get_curr_aeq_elem(eq); + dev_err(hwdev->dev, + "Aeq id: %d, idx: %u, ctrl0: 0x%08x, ci: 0x%08x, pi: 0x%x, work_state: 0x%x, wrap: %u, desc: 0x%x swci:0x%x\n", + q_id, idx, ctrl0, ci, pi, work_busy(&eq->aeq_work), + eq->wrapped, be32_to_cpu(aeqe_pos->desc), eq->cons_idx); + } +} + int hinic3_ceqs_init(struct hinic3_hwdev *hwdev, u16 num_ceqs, struct msix_entry *msix_entries) { @@ -773,3 +811,30 @@ void hinic3_ceqs_free(struct hinic3_hwdev *hwdev) kfree(ceqs); } + +void hinic3_dump_ceq_info(struct hinic3_hwdev *hwdev) +{ + struct hinic3_eq *eq; + u32 addr, ci, pi; + int q_id; + + for (q_id = 0; q_id < hwdev->ceqs->num_ceqs; q_id++) { + eq = &hwdev->ceqs->ceq[q_id]; + /* Indirect access should set q_id first */ + hinic3_hwif_write_reg(eq->hwdev->hwif, + HINIC3_EQ_INDIR_IDX_ADDR(eq->type), + eq->q_id); + + addr = EQ_CONS_IDX_REG_ADDR(eq); + ci = hinic3_hwif_read_reg(hwdev->hwif, addr); + addr = EQ_PROD_IDX_REG_ADDR(eq); + pi = hinic3_hwif_read_reg(hwdev->hwif, addr); + dev_err(hwdev->dev, + "Ceq id: %d, ci: 0x%08x, sw_ci: 0x%08x, pi: 0x%x, wrap: %u, ceqe: 0x%x\n", + q_id, ci, eq->cons_idx, pi, + eq->wrapped, be32_to_cpu(*get_curr_ceq_elem(eq))); + + dev_err(hwdev->dev, "Ceq last response soft interrupt time: %u\n", + jiffies_to_msecs(jiffies - eq->soft_intr_jif)); + } +} diff --git a/drivers/net/ethernet/huawei/hinic3/hinic3_eqs.h b/drivers/net/ethernet/huawei/hinic3/hinic3_eqs.h index 005a6e0745b3..c0fa237b270b 100644 --- a/drivers/net/ethernet/huawei/hinic3/hinic3_eqs.h +++ b/drivers/net/ethernet/huawei/hinic3/hinic3_eqs.h @@ -56,6 +56,8 @@ struct hinic3_eq { u16 msix_entry_idx; char irq_name[HINIC3_EQ_IRQ_NAME_LEN]; struct work_struct aeq_work; + + u64 soft_intr_jif; }; struct hinic3_aeq_elem { @@ -110,6 +112,8 @@ int hinic3_aeq_register_cb(struct hinic3_hwdev *hwdev, hinic3_aeq_event_cb hwe_cb); void hinic3_aeq_unregister_cb(struct hinic3_hwdev *hwdev, enum hinic3_aeq_type event); +void hinic3_dump_aeq_info(struct hinic3_hwdev *hwdev); + int hinic3_ceqs_init(struct hinic3_hwdev *hwdev, u16 num_ceqs, struct msix_entry *msix_entries); void hinic3_ceqs_free(struct hinic3_hwdev *hwdev); @@ -118,5 +122,6 @@ int hinic3_ceq_register_cb(struct hinic3_hwdev *hwdev, hinic3_ceq_event_cb callback); void hinic3_ceq_unregister_cb(struct hinic3_hwdev *hwdev, enum hinic3_ceq_event event); +void hinic3_dump_ceq_info(struct hinic3_hwdev *hwdev); #endif diff --git a/drivers/net/ethernet/huawei/hinic3/hinic3_ethtool.c b/drivers/net/ethernet/huawei/hinic3/hinic3_ethtool.c new file mode 100644 index 000000000000..90fc16288de9 --- /dev/null +++ b/drivers/net/ethernet/huawei/hinic3/hinic3_ethtool.c @@ -0,0 +1,425 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. + +#include <linux/kernel.h> +#include <linux/pci.h> +#include <linux/device.h> +#include <linux/module.h> +#include <linux/types.h> +#include <linux/errno.h> +#include <linux/etherdevice.h> +#include <linux/netdevice.h> +#include <linux/ethtool.h> + +#include "hinic3_lld.h" +#include "hinic3_hw_comm.h" +#include "hinic3_nic_dev.h" +#include "hinic3_nic_cfg.h" + +#define HINIC3_MGMT_VERSION_MAX_LEN 32 + +static void hinic3_get_drvinfo(struct net_device *netdev, + struct ethtool_drvinfo *info) +{ + struct hinic3_nic_dev *nic_dev = netdev_priv(netdev); + u8 mgmt_ver[HINIC3_MGMT_VERSION_MAX_LEN]; + struct pci_dev *pdev = nic_dev->pdev; + int err; + + strscpy(info->driver, HINIC3_NIC_DRV_NAME, sizeof(info->driver)); + strscpy(info->bus_info, pci_name(pdev), sizeof(info->bus_info)); + + err = hinic3_get_mgmt_version(nic_dev->hwdev, mgmt_ver, + HINIC3_MGMT_VERSION_MAX_LEN); + if (err) { + netdev_err(netdev, "Failed to get fw version\n"); + return; + } + + snprintf(info->fw_version, sizeof(info->fw_version), "%s", mgmt_ver); +} + +static u32 hinic3_get_msglevel(struct net_device *netdev) +{ + struct hinic3_nic_dev *nic_dev = netdev_priv(netdev); + + return nic_dev->msg_enable; +} + +static void hinic3_set_msglevel(struct net_device *netdev, u32 data) +{ + struct hinic3_nic_dev *nic_dev = netdev_priv(netdev); + + nic_dev->msg_enable = data; + + netdev_dbg(netdev, "Set message level: 0x%x\n", data); +} + +static const u32 hinic3_link_mode_ge[] = { + ETHTOOL_LINK_MODE_1000baseT_Full_BIT, + ETHTOOL_LINK_MODE_1000baseKX_Full_BIT, + ETHTOOL_LINK_MODE_1000baseX_Full_BIT, +}; + +static const u32 hinic3_link_mode_10ge_base_r[] = { + ETHTOOL_LINK_MODE_10000baseKR_Full_BIT, + ETHTOOL_LINK_MODE_10000baseR_FEC_BIT, + ETHTOOL_LINK_MODE_10000baseCR_Full_BIT, + ETHTOOL_LINK_MODE_10000baseSR_Full_BIT, + ETHTOOL_LINK_MODE_10000baseLR_Full_BIT, + ETHTOOL_LINK_MODE_10000baseLRM_Full_BIT, +}; + +static const u32 hinic3_link_mode_25ge_base_r[] = { + ETHTOOL_LINK_MODE_25000baseCR_Full_BIT, + ETHTOOL_LINK_MODE_25000baseKR_Full_BIT, + ETHTOOL_LINK_MODE_25000baseSR_Full_BIT, +}; + +static const u32 hinic3_link_mode_40ge_base_r4[] = { + ETHTOOL_LINK_MODE_40000baseKR4_Full_BIT, + ETHTOOL_LINK_MODE_40000baseCR4_Full_BIT, + ETHTOOL_LINK_MODE_40000baseSR4_Full_BIT, + ETHTOOL_LINK_MODE_40000baseLR4_Full_BIT, +}; + +static const u32 hinic3_link_mode_50ge_base_r[] = { + ETHTOOL_LINK_MODE_50000baseKR_Full_BIT, + ETHTOOL_LINK_MODE_50000baseSR_Full_BIT, + ETHTOOL_LINK_MODE_50000baseCR_Full_BIT, +}; + +static const u32 hinic3_link_mode_50ge_base_r2[] = { + ETHTOOL_LINK_MODE_50000baseCR2_Full_BIT, + ETHTOOL_LINK_MODE_50000baseKR2_Full_BIT, + ETHTOOL_LINK_MODE_50000baseSR2_Full_BIT, +}; + +static const u32 hinic3_link_mode_100ge_base_r[] = { + ETHTOOL_LINK_MODE_100000baseKR_Full_BIT, + ETHTOOL_LINK_MODE_100000baseSR_Full_BIT, + ETHTOOL_LINK_MODE_100000baseCR_Full_BIT, +}; + +static const u32 hinic3_link_mode_100ge_base_r2[] = { + ETHTOOL_LINK_MODE_100000baseKR2_Full_BIT, + ETHTOOL_LINK_MODE_100000baseSR2_Full_BIT, + ETHTOOL_LINK_MODE_100000baseCR2_Full_BIT, +}; + +static const u32 hinic3_link_mode_100ge_base_r4[] = { + ETHTOOL_LINK_MODE_100000baseKR4_Full_BIT, + ETHTOOL_LINK_MODE_100000baseSR4_Full_BIT, + ETHTOOL_LINK_MODE_100000baseCR4_Full_BIT, + ETHTOOL_LINK_MODE_100000baseLR4_ER4_Full_BIT, +}; + +static const u32 hinic3_link_mode_200ge_base_r2[] = { + ETHTOOL_LINK_MODE_200000baseKR2_Full_BIT, + ETHTOOL_LINK_MODE_200000baseSR2_Full_BIT, + ETHTOOL_LINK_MODE_200000baseCR2_Full_BIT, +}; + +static const u32 hinic3_link_mode_200ge_base_r4[] = { + ETHTOOL_LINK_MODE_200000baseKR4_Full_BIT, + ETHTOOL_LINK_MODE_200000baseSR4_Full_BIT, + ETHTOOL_LINK_MODE_200000baseCR4_Full_BIT, +}; + +struct hw2ethtool_link_mode { + const u32 *link_mode_bit_arr; + u32 arr_size; + u32 speed; +}; + +static const struct hw2ethtool_link_mode + hw2ethtool_link_mode_table[LINK_MODE_MAX_NUMBERS] = { + [LINK_MODE_GE] = { + .link_mode_bit_arr = hinic3_link_mode_ge, + .arr_size = ARRAY_SIZE(hinic3_link_mode_ge), + .speed = SPEED_1000, + }, + [LINK_MODE_10GE_BASE_R] = { + .link_mode_bit_arr = hinic3_link_mode_10ge_base_r, + .arr_size = ARRAY_SIZE(hinic3_link_mode_10ge_base_r), + .speed = SPEED_10000, + }, + [LINK_MODE_25GE_BASE_R] = { + .link_mode_bit_arr = hinic3_link_mode_25ge_base_r, + .arr_size = ARRAY_SIZE(hinic3_link_mode_25ge_base_r), + .speed = SPEED_25000, + }, + [LINK_MODE_40GE_BASE_R4] = { + .link_mode_bit_arr = hinic3_link_mode_40ge_base_r4, + .arr_size = ARRAY_SIZE(hinic3_link_mode_40ge_base_r4), + .speed = SPEED_40000, + }, + [LINK_MODE_50GE_BASE_R] = { + .link_mode_bit_arr = hinic3_link_mode_50ge_base_r, + .arr_size = ARRAY_SIZE(hinic3_link_mode_50ge_base_r), + .speed = SPEED_50000, + }, + [LINK_MODE_50GE_BASE_R2] = { + .link_mode_bit_arr = hinic3_link_mode_50ge_base_r2, + .arr_size = ARRAY_SIZE(hinic3_link_mode_50ge_base_r2), + .speed = SPEED_50000, + }, + [LINK_MODE_100GE_BASE_R] = { + .link_mode_bit_arr = hinic3_link_mode_100ge_base_r, + .arr_size = ARRAY_SIZE(hinic3_link_mode_100ge_base_r), + .speed = SPEED_100000, + }, + [LINK_MODE_100GE_BASE_R2] = { + .link_mode_bit_arr = hinic3_link_mode_100ge_base_r2, + .arr_size = ARRAY_SIZE(hinic3_link_mode_100ge_base_r2), + .speed = SPEED_100000, + }, + [LINK_MODE_100GE_BASE_R4] = { + .link_mode_bit_arr = hinic3_link_mode_100ge_base_r4, + .arr_size = ARRAY_SIZE(hinic3_link_mode_100ge_base_r4), + .speed = SPEED_100000, + }, + [LINK_MODE_200GE_BASE_R2] = { + .link_mode_bit_arr = hinic3_link_mode_200ge_base_r2, + .arr_size = ARRAY_SIZE(hinic3_link_mode_200ge_base_r2), + .speed = SPEED_200000, + }, + [LINK_MODE_200GE_BASE_R4] = { + .link_mode_bit_arr = hinic3_link_mode_200ge_base_r4, + .arr_size = ARRAY_SIZE(hinic3_link_mode_200ge_base_r4), + .speed = SPEED_200000, + }, +}; + +#define GET_SUPPORTED_MODE 0 +#define GET_ADVERTISED_MODE 1 + +struct hinic3_link_settings { + __ETHTOOL_DECLARE_LINK_MODE_MASK(supported); + __ETHTOOL_DECLARE_LINK_MODE_MASK(advertising); + + u32 speed; + u8 duplex; + u8 port; + u8 autoneg; +}; + +#define HINIC3_ADD_SUPPORTED_LINK_MODE(ecmd, mode) \ + set_bit(ETHTOOL_LINK_##mode##_BIT, (ecmd)->supported) +#define HINIC3_ADD_ADVERTISED_LINK_MODE(ecmd, mode) \ + set_bit(ETHTOOL_LINK_##mode##_BIT, (ecmd)->advertising) + +static void hinic3_add_speed_link_mode(unsigned long *bitmap, u32 mode) +{ + u32 i; + + for (i = 0; i < hw2ethtool_link_mode_table[mode].arr_size; i++) { + if (hw2ethtool_link_mode_table[mode].link_mode_bit_arr[i] >= + __ETHTOOL_LINK_MODE_MASK_NBITS) + continue; + + set_bit(hw2ethtool_link_mode_table[mode].link_mode_bit_arr[i], + bitmap); + } +} + +/* Related to enum mag_cmd_port_speed */ +static const u32 hw_to_ethtool_speed[] = { + (u32)SPEED_UNKNOWN, SPEED_10, SPEED_100, SPEED_1000, SPEED_10000, + SPEED_25000, SPEED_40000, SPEED_50000, SPEED_100000, SPEED_200000 +}; + +static void +hinic3_add_ethtool_link_mode(struct hinic3_link_settings *link_settings, + u32 hw_link_mode, u32 name) +{ + unsigned long *advertising_mask = link_settings->advertising; + unsigned long *supported_mask = link_settings->supported; + u32 link_mode; + + for (link_mode = 0; link_mode < LINK_MODE_MAX_NUMBERS; link_mode++) { + if (hw_link_mode & BIT(link_mode)) { + if (name == GET_SUPPORTED_MODE) + hinic3_add_speed_link_mode(supported_mask, + link_mode); + else + hinic3_add_speed_link_mode(advertising_mask, + link_mode); + } + } +} + +static void +hinic3_link_speed_set(struct net_device *netdev, + struct hinic3_link_settings *link_settings, + struct hinic3_nic_port_info *port_info) +{ + struct hinic3_nic_dev *nic_dev = netdev_priv(netdev); + bool link_status_up; + int err; + + if (port_info->supported_mode != LINK_MODE_UNKNOWN) + hinic3_add_ethtool_link_mode(link_settings, + port_info->supported_mode, + GET_SUPPORTED_MODE); + if (port_info->advertised_mode != LINK_MODE_UNKNOWN) + hinic3_add_ethtool_link_mode(link_settings, + port_info->advertised_mode, + GET_ADVERTISED_MODE); + + err = hinic3_get_link_status(nic_dev->hwdev, &link_status_up); + if (!err && link_status_up) { + link_settings->speed = + port_info->speed < ARRAY_SIZE(hw_to_ethtool_speed) ? + hw_to_ethtool_speed[port_info->speed] : + (u32)SPEED_UNKNOWN; + + link_settings->duplex = port_info->duplex; + } else { + link_settings->speed = (u32)SPEED_UNKNOWN; + link_settings->duplex = DUPLEX_UNKNOWN; + } +} + +static void +hinic3_link_port_type_set(struct hinic3_link_settings *link_settings, + u8 port_type) +{ + switch (port_type) { + case MAG_CMD_WIRE_TYPE_ELECTRIC: + HINIC3_ADD_SUPPORTED_LINK_MODE(link_settings, MODE_TP); + HINIC3_ADD_ADVERTISED_LINK_MODE(link_settings, MODE_TP); + link_settings->port = PORT_TP; + break; + + case MAG_CMD_WIRE_TYPE_AOC: + case MAG_CMD_WIRE_TYPE_MM: + case MAG_CMD_WIRE_TYPE_SM: + HINIC3_ADD_SUPPORTED_LINK_MODE(link_settings, MODE_FIBRE); + HINIC3_ADD_ADVERTISED_LINK_MODE(link_settings, MODE_FIBRE); + link_settings->port = PORT_FIBRE; + break; + + case MAG_CMD_WIRE_TYPE_COPPER: + HINIC3_ADD_SUPPORTED_LINK_MODE(link_settings, MODE_FIBRE); + HINIC3_ADD_ADVERTISED_LINK_MODE(link_settings, MODE_FIBRE); + link_settings->port = PORT_DA; + break; + + case MAG_CMD_WIRE_TYPE_BACKPLANE: + HINIC3_ADD_SUPPORTED_LINK_MODE(link_settings, MODE_Backplane); + HINIC3_ADD_ADVERTISED_LINK_MODE(link_settings, MODE_Backplane); + link_settings->port = PORT_NONE; + break; + + default: + link_settings->port = PORT_OTHER; + break; + } +} + +static int +hinic3_get_link_pause_settings(struct net_device *netdev, + struct hinic3_link_settings *link_settings) +{ + struct hinic3_nic_dev *nic_dev = netdev_priv(netdev); + struct hinic3_nic_pause_config nic_pause = {}; + int err; + + err = hinic3_get_pause_info(nic_dev, &nic_pause); + if (err) { + netdev_err(netdev, "Failed to get pause param from hw\n"); + return err; + } + + HINIC3_ADD_SUPPORTED_LINK_MODE(link_settings, MODE_Pause); + if (nic_pause.rx_pause && nic_pause.tx_pause) { + HINIC3_ADD_ADVERTISED_LINK_MODE(link_settings, MODE_Pause); + } else if (nic_pause.tx_pause) { + HINIC3_ADD_ADVERTISED_LINK_MODE(link_settings, + MODE_Asym_Pause); + } else if (nic_pause.rx_pause) { + HINIC3_ADD_ADVERTISED_LINK_MODE(link_settings, MODE_Pause); + HINIC3_ADD_ADVERTISED_LINK_MODE(link_settings, + MODE_Asym_Pause); + } + + return 0; +} + +static int +hinic3_get_link_settings(struct net_device *netdev, + struct hinic3_link_settings *link_settings) +{ + struct hinic3_nic_dev *nic_dev = netdev_priv(netdev); + struct hinic3_nic_port_info port_info = {}; + int err; + + err = hinic3_get_port_info(nic_dev->hwdev, &port_info); + if (err) { + netdev_err(netdev, "Failed to get port info\n"); + return err; + } + + hinic3_link_speed_set(netdev, link_settings, &port_info); + + hinic3_link_port_type_set(link_settings, port_info.port_type); + + link_settings->autoneg = port_info.autoneg_state == PORT_CFG_AN_ON ? + AUTONEG_ENABLE : AUTONEG_DISABLE; + if (port_info.autoneg_cap) + HINIC3_ADD_SUPPORTED_LINK_MODE(link_settings, MODE_Autoneg); + if (port_info.autoneg_state == PORT_CFG_AN_ON) + HINIC3_ADD_ADVERTISED_LINK_MODE(link_settings, MODE_Autoneg); + + if (!HINIC3_IS_VF(nic_dev->hwdev)) { + err = hinic3_get_link_pause_settings(netdev, link_settings); + if (err) + return err; + } + + return 0; +} + +static int +hinic3_get_link_ksettings(struct net_device *netdev, + struct ethtool_link_ksettings *link_settings) +{ + struct ethtool_link_settings *base = &link_settings->base; + struct hinic3_link_settings settings = {}; + int err; + + ethtool_link_ksettings_zero_link_mode(link_settings, supported); + ethtool_link_ksettings_zero_link_mode(link_settings, advertising); + + err = hinic3_get_link_settings(netdev, &settings); + if (err) + return err; + + bitmap_copy(link_settings->link_modes.supported, settings.supported, + __ETHTOOL_LINK_MODE_MASK_NBITS); + bitmap_copy(link_settings->link_modes.advertising, settings.advertising, + __ETHTOOL_LINK_MODE_MASK_NBITS); + + base->autoneg = settings.autoneg; + base->speed = settings.speed; + base->duplex = settings.duplex; + base->port = settings.port; + + return 0; +} + +static const struct ethtool_ops hinic3_ethtool_ops = { + .supported_coalesce_params = ETHTOOL_COALESCE_USECS | + ETHTOOL_COALESCE_PKT_RATE_RX_USECS, + .get_link_ksettings = hinic3_get_link_ksettings, + .get_drvinfo = hinic3_get_drvinfo, + .get_msglevel = hinic3_get_msglevel, + .set_msglevel = hinic3_set_msglevel, + .get_link = ethtool_op_get_link, +}; + +void hinic3_set_ethtool_ops(struct net_device *netdev) +{ + netdev->ethtool_ops = &hinic3_ethtool_ops; +} diff --git a/drivers/net/ethernet/huawei/hinic3/hinic3_hw_cfg.c b/drivers/net/ethernet/huawei/hinic3/hinic3_hw_cfg.c index ca60cf4b6282..c5dd53a8fa96 100644 --- a/drivers/net/ethernet/huawei/hinic3/hinic3_hw_cfg.c +++ b/drivers/net/ethernet/huawei/hinic3/hinic3_hw_cfg.c @@ -17,6 +17,17 @@ static void hinic3_parse_pub_res_cap(struct hinic3_hwdev *hwdev, { cap->port_id = dev_cap->port_id; cap->supp_svcs_bitmap = dev_cap->svc_cap_en; + + cap->cos_valid_bitmap = dev_cap->valid_cos_bitmap; + cap->port_cos_valid_bitmap = dev_cap->port_cos_valid_bitmap; + + if (type != HINIC3_FUNC_TYPE_VF) + cap->max_vf = dev_cap->max_vf; + else + cap->max_vf = 0; + + dev_dbg(hwdev->dev, "Port_id: 0x%x, cos_bitmap: 0x%x, Max_vf: 0x%x\n", + cap->port_id, cap->cos_valid_bitmap, cap->max_vf); } static void hinic3_parse_l2nic_res_cap(struct hinic3_hwdev *hwdev, @@ -28,6 +39,13 @@ static void hinic3_parse_l2nic_res_cap(struct hinic3_hwdev *hwdev, nic_svc_cap->max_sqs = min(dev_cap->nic_max_sq_id + 1, HINIC3_CFG_MAX_QP); + + nic_svc_cap->max_rqs = min(dev_cap->nic_max_rq_id + 1, + HINIC3_CFG_MAX_QP); + nic_svc_cap->default_num_queues = dev_cap->nic_default_num_queues; + + dev_dbg(hwdev->dev, "L2nic resource capability, max_sqs: 0x%x, max_rqs: 0x%x\n", + nic_svc_cap->max_sqs, nic_svc_cap->max_rqs); } static void hinic3_parse_dev_cap(struct hinic3_hwdev *hwdev, @@ -44,8 +62,8 @@ static void hinic3_parse_dev_cap(struct hinic3_hwdev *hwdev, hinic3_parse_l2nic_res_cap(hwdev, cap, dev_cap, type); } -static int get_cap_from_fw(struct hinic3_hwdev *hwdev, - enum hinic3_func_type type) +static int hinic3_get_cap_from_fw(struct hinic3_hwdev *hwdev, + enum hinic3_func_type type) { struct mgmt_msg_params msg_params = {}; struct cfg_cmd_dev_cap dev_cap = {}; @@ -69,6 +87,29 @@ static int get_cap_from_fw(struct hinic3_hwdev *hwdev, return 0; } +static int hinic3_get_dev_cap(struct hinic3_hwdev *hwdev) +{ + enum hinic3_func_type type = HINIC3_FUNC_TYPE(hwdev); + int err; + + switch (type) { + case HINIC3_FUNC_TYPE_PF: + case HINIC3_FUNC_TYPE_VF: + err = hinic3_get_cap_from_fw(hwdev, type); + if (err) { + dev_err(hwdev->dev, "Failed to get FW capability\n"); + return err; + } + break; + default: + dev_err(hwdev->dev, "Unsupported PCI Function type: %d\n", + type); + return -EINVAL; + } + + return 0; +} + static int hinic3_init_irq_info(struct hinic3_hwdev *hwdev) { struct hinic3_cfg_mgmt_info *cfg_mgmt = hwdev->cfg_mgmt; @@ -215,7 +256,7 @@ void hinic3_free_irq(struct hinic3_hwdev *hwdev, u32 irq_id) int hinic3_init_capability(struct hinic3_hwdev *hwdev) { - return get_cap_from_fw(hwdev, HINIC3_FUNC_TYPE_VF); + return hinic3_get_dev_cap(hwdev); } bool hinic3_support_nic(struct hinic3_hwdev *hwdev) diff --git a/drivers/net/ethernet/huawei/hinic3/hinic3_hw_cfg.h b/drivers/net/ethernet/huawei/hinic3/hinic3_hw_cfg.h index 58806199bf54..361c0b6a70f0 100644 --- a/drivers/net/ethernet/huawei/hinic3/hinic3_hw_cfg.h +++ b/drivers/net/ethernet/huawei/hinic3/hinic3_hw_cfg.h @@ -26,6 +26,8 @@ struct hinic3_irq_info { struct hinic3_nic_service_cap { u16 max_sqs; + u16 max_rqs; + u16 default_num_queues; }; /* Device capabilities */ @@ -34,6 +36,12 @@ struct hinic3_dev_cap { u16 supp_svcs_bitmap; /* Physical port */ u8 port_id; + + u8 cos_valid_bitmap; + u8 port_cos_valid_bitmap; + /* max number of VFs that PF supports */ + u16 max_vf; + struct hinic3_nic_service_cap nic_svc_cap; }; diff --git a/drivers/net/ethernet/huawei/hinic3/hinic3_hw_comm.c b/drivers/net/ethernet/huawei/hinic3/hinic3_hw_comm.c index ecfe6265954e..0dbf9cbe7f9a 100644 --- a/drivers/net/ethernet/huawei/hinic3/hinic3_hw_comm.c +++ b/drivers/net/ethernet/huawei/hinic3/hinic3_hw_comm.c @@ -292,6 +292,32 @@ int hinic3_set_cmdq_depth(struct hinic3_hwdev *hwdev, u16 cmdq_depth) return 0; } +#define HINIC3_FLR_TIMEOUT 1000 + +static enum hinic3_wait_return hinic3_check_flr_finish_handler(void *priv_data) +{ + struct hinic3_hwdev *hwdev = priv_data; + struct hinic3_hwif *hwif = hwdev->hwif; + enum hinic3_pf_status status; + + if (!hwdev->chip_present_flag) + return HINIC3_WAIT_PROCESS_ERR; + + status = hinic3_get_pf_status(hwif); + if (status == HINIC3_PF_STATUS_FLR_FINISH_FLAG) { + hinic3_set_pf_status(hwif, HINIC3_PF_STATUS_ACTIVE_FLAG); + return HINIC3_WAIT_PROCESS_CPL; + } + + return HINIC3_WAIT_PROCESS_WAITING; +} + +static int hinic3_wait_for_flr_finish(struct hinic3_hwdev *hwdev) +{ + return hinic3_wait_for_timeout(hwdev, hinic3_check_flr_finish_handler, + HINIC3_FLR_TIMEOUT, 0xa * USEC_PER_MSEC); +} + #define HINIC3_WAIT_CMDQ_IDLE_TIMEOUT 5000 static enum hinic3_wait_return check_cmdq_stop_handler(void *priv_data) @@ -300,6 +326,10 @@ static enum hinic3_wait_return check_cmdq_stop_handler(void *priv_data) enum hinic3_cmdq_type cmdq_type; struct hinic3_cmdqs *cmdqs; + /* Stop waiting when card unpresent */ + if (!hwdev->chip_present_flag) + return HINIC3_WAIT_PROCESS_ERR; + cmdqs = hwdev->cmdqs; for (cmdq_type = 0; cmdq_type < cmdqs->cmdq_num; cmdq_type++) { if (!hinic3_cmdq_idle(&cmdqs->cmdq[cmdq_type])) @@ -347,6 +377,9 @@ int hinic3_func_rx_tx_flush(struct hinic3_hwdev *hwdev) int ret = 0; int err; + if (!hwdev->chip_present_flag) + return 0; + err = wait_cmdq_stop(hwdev); if (err) { dev_warn(hwdev->dev, "CMDQ is still working, CMDQ timeout value is unreasonable\n"); @@ -382,6 +415,14 @@ int hinic3_func_rx_tx_flush(struct hinic3_hwdev *hwdev) ret = err; } + if (HINIC3_FUNC_TYPE(hwdev) != HINIC3_FUNC_TYPE_VF) { + err = hinic3_wait_for_flr_finish(hwdev); + if (err) { + dev_warn(hwdev->dev, "Wait firmware FLR timeout\n"); + ret = err; + } + } + hinic3_toggle_doorbell(hwif, ENABLE_DOORBELL); err = hinic3_reinit_cmdq_ctxts(hwdev); @@ -539,3 +580,31 @@ int hinic3_clean_root_ctxt(struct hinic3_hwdev *hwdev) return 0; } + +#define HINIC3_FW_VER_TYPE_MPU 1 + +int hinic3_get_mgmt_version(struct hinic3_hwdev *hwdev, u8 *mgmt_ver, + u8 version_size) +{ + struct comm_cmd_get_fw_version fw_ver = {}; + struct mgmt_msg_params msg_params = {}; + int err; + + fw_ver.fw_type = HINIC3_FW_VER_TYPE_MPU; + + mgmt_msg_params_init_default(&msg_params, &fw_ver, sizeof(fw_ver)); + + err = hinic3_send_mbox_to_mgmt(hwdev, MGMT_MOD_COMM, + COMM_CMD_GET_FW_VERSION, &msg_params); + + if (err || fw_ver.head.status) { + dev_err(hwdev->dev, + "Failed to get fw version, err: %d, status: 0x%x\n", + err, fw_ver.head.status); + return -EFAULT; + } + + snprintf(mgmt_ver, version_size, "%s", fw_ver.ver); + + return 0; +} diff --git a/drivers/net/ethernet/huawei/hinic3/hinic3_hw_comm.h b/drivers/net/ethernet/huawei/hinic3/hinic3_hw_comm.h index 8e4737c486b7..e672f9af5cb1 100644 --- a/drivers/net/ethernet/huawei/hinic3/hinic3_hw_comm.h +++ b/drivers/net/ethernet/huawei/hinic3/hinic3_hw_comm.h @@ -49,5 +49,7 @@ void hinic3_sync_time_to_fw(struct hinic3_hwdev *hwdev); int hinic3_set_root_ctxt(struct hinic3_hwdev *hwdev, u32 rq_depth, u32 sq_depth, int rx_buf_sz); int hinic3_clean_root_ctxt(struct hinic3_hwdev *hwdev); +int hinic3_get_mgmt_version(struct hinic3_hwdev *hwdev, u8 *mgmt_ver, + u8 version_size); #endif diff --git a/drivers/net/ethernet/huawei/hinic3/hinic3_hw_intf.h b/drivers/net/ethernet/huawei/hinic3/hinic3_hw_intf.h index 329a9c464ff9..cfc9daa3034f 100644 --- a/drivers/net/ethernet/huawei/hinic3/hinic3_hw_intf.h +++ b/drivers/net/ethernet/huawei/hinic3/hinic3_hw_intf.h @@ -114,6 +114,7 @@ enum comm_cmd { COMM_CMD_SET_DMA_ATTR = 25, /* Commands for obtaining information */ + COMM_CMD_GET_FW_VERSION = 60, COMM_CMD_SYNC_TIME = 62, COMM_CMD_SEND_BDF_INFO = 64, }; @@ -275,6 +276,17 @@ struct comm_cmd_bdf_info { u8 rsvd2[5]; }; +#define COMM_FW_VERSION_LEN 16 +#define COMM_FW_COMPILE_TIME_LEN 20 +struct comm_cmd_get_fw_version { + struct mgmt_msg_head head; + + u16 fw_type; + u16 rsvd1; + u8 ver[COMM_FW_VERSION_LEN]; + u8 time[COMM_FW_COMPILE_TIME_LEN]; +}; + /* Services supported by HW. HW uses these values when delivering events. * HW supports multiple services that are not yet supported by driver * (e.g. RoCE). diff --git a/drivers/net/ethernet/huawei/hinic3/hinic3_hwdev.c b/drivers/net/ethernet/huawei/hinic3/hinic3_hwdev.c index 0074d0c6dbaf..f44b3064ab2e 100644 --- a/drivers/net/ethernet/huawei/hinic3/hinic3_hwdev.c +++ b/drivers/net/ethernet/huawei/hinic3/hinic3_hwdev.c @@ -32,6 +32,9 @@ #define HINIC3_PCIE_PH_DISABLE 0 #define HINIC3_PCIE_MSIX_ATTR_ENTRY 0 +#define HINIC3_CHIP_PRESENT 1 +#define HINIC3_CHIP_ABSENT 0 + #define HINIC3_DEFAULT_EQ_MSIX_PENDING_LIMIT 0 #define HINIC3_DEFAULT_EQ_MSIX_COALESC_TIMER_CFG 0xFF #define HINIC3_DEFAULT_EQ_MSIX_RESEND_TIMER_CFG 7 @@ -431,6 +434,10 @@ err_free_ceqs: static void hinic3_free_cmdqs_channel(struct hinic3_hwdev *hwdev) { hinic3_comm_cmdqs_free(hwdev); + + hinic3_set_wq_page_size(hwdev, hinic3_global_func_id(hwdev), + HINIC3_MIN_PAGE_SIZE); + hinic3_ceqs_free(hwdev); } @@ -545,6 +552,7 @@ int hinic3_init_hwdev(struct pci_dev *pdev) dev_err(hwdev->dev, "Failed to init hwif\n"); goto err_free_hwdev; } + hwdev->chip_present_flag = HINIC3_CHIP_PRESENT; hwdev->workq = alloc_workqueue(HINIC3_HWDEV_WQ_NAME, WQ_MEM_RECLAIM | WQ_PERCPU, HINIC3_WQ_MAX_REQ); @@ -621,6 +629,7 @@ void hinic3_set_api_stop(struct hinic3_hwdev *hwdev) struct hinic3_recv_msg *recv_resp_msg; struct hinic3_mbox *mbox; + hwdev->chip_present_flag = HINIC3_CHIP_ABSENT; spin_lock_bh(&hwdev->channel_lock); if (HINIC3_IS_PF(hwdev) && test_bit(HINIC3_HWDEV_MGMT_INITED, &hwdev->func_state)) { diff --git a/drivers/net/ethernet/huawei/hinic3/hinic3_hwdev.h b/drivers/net/ethernet/huawei/hinic3/hinic3_hwdev.h index 9686c2600b46..4276ac136464 100644 --- a/drivers/net/ethernet/huawei/hinic3/hinic3_hwdev.h +++ b/drivers/net/ethernet/huawei/hinic3/hinic3_hwdev.h @@ -76,6 +76,7 @@ struct hinic3_hwdev { u32 wq_page_size; u8 max_cmdq; ulong func_state; + int chip_present_flag; }; struct hinic3_event_info { diff --git a/drivers/net/ethernet/huawei/hinic3/hinic3_hwif.c b/drivers/net/ethernet/huawei/hinic3/hinic3_hwif.c index 771883174b3b..90feaa225080 100644 --- a/drivers/net/ethernet/huawei/hinic3/hinic3_hwif.c +++ b/drivers/net/ethernet/huawei/hinic3/hinic3_hwif.c @@ -70,11 +70,19 @@ #define HINIC3_PPF_ELECTION_GET(val, member) \ FIELD_GET(HINIC3_PPF_ELECTION_##member##_MASK, val) +#define HINIC3_GET_REG_FLAG(reg) ((reg) & (~(HINIC3_REGS_FLAG_MASK))) #define HINIC3_GET_REG_ADDR(reg) ((reg) & (HINIC3_REGS_FLAG_MASK)) static void __iomem *hinic3_reg_addr(struct hinic3_hwif *hwif, u32 reg) { - return hwif->cfg_regs_base + HINIC3_GET_REG_ADDR(reg); + void __iomem *addr; + + if (HINIC3_GET_REG_FLAG(reg) == HINIC3_MGMT_REGS_FLAG) + addr = hwif->mgmt_regs_base + HINIC3_GET_REG_ADDR(reg); + else + addr = hwif->cfg_regs_base + HINIC3_GET_REG_ADDR(reg); + + return addr; } u32 hinic3_hwif_read_reg(struct hinic3_hwif *hwif, u32 reg) @@ -97,6 +105,8 @@ static enum hinic3_wait_return check_hwif_ready_handler(void *priv_data) u32 attr1; attr1 = hinic3_hwif_read_reg(hwdev->hwif, HINIC3_CSR_FUNC_ATTR1_ADDR); + if (attr1 == HINIC3_PCIE_LINK_DOWN) + return HINIC3_WAIT_PROCESS_ERR; return HINIC3_AF1_GET(attr1, MGMT_INIT_STATUS) ? HINIC3_WAIT_PROCESS_CPL : HINIC3_WAIT_PROCESS_WAITING; @@ -135,6 +145,7 @@ static void set_hwif_attr(struct hinic3_func_attr *attr, u32 attr0, u32 attr1, static int init_hwif_attr(struct hinic3_hwdev *hwdev) { u32 attr0, attr1, attr2, attr3, attr6; + struct hinic3_func_attr *attr; struct hinic3_hwif *hwif; hwif = hwdev->hwif; @@ -158,7 +169,15 @@ static int init_hwif_attr(struct hinic3_hwdev *hwdev) if (attr6 == HINIC3_PCIE_LINK_DOWN) return -EFAULT; - set_hwif_attr(&hwif->attr, attr0, attr1, attr2, attr3, attr6); + attr = &hwif->attr; + set_hwif_attr(attr, attr0, attr1, attr2, attr3, attr6); + + if (attr->func_type != HINIC3_FUNC_TYPE_VF && + attr->func_type != HINIC3_FUNC_TYPE_PF) { + dev_err(hwdev->dev, "unexpected func_type %u\n", + attr->func_type); + return -EINVAL; + } if (!hwif->attr.num_ceqs) { dev_err(hwdev->dev, "Ceq num cfg in fw is zero\n"); diff --git a/drivers/net/ethernet/huawei/hinic3/hinic3_lld.c b/drivers/net/ethernet/huawei/hinic3/hinic3_lld.c index a8b89eeed753..f83d5bd1041e 100644 --- a/drivers/net/ethernet/huawei/hinic3/hinic3_lld.c +++ b/drivers/net/ethernet/huawei/hinic3/hinic3_lld.c @@ -426,6 +426,7 @@ static void hinic3_remove(struct pci_dev *pdev) } static const struct pci_device_id hinic3_pci_table[] = { + {PCI_VDEVICE(HUAWEI, PCI_DEV_ID_HINIC3_PF), 0}, {PCI_VDEVICE(HUAWEI, PCI_DEV_ID_HINIC3_VF), 0}, {0, 0} diff --git a/drivers/net/ethernet/huawei/hinic3/hinic3_main.c b/drivers/net/ethernet/huawei/hinic3/hinic3_main.c index 6039fcf3c1dd..0a888fe4c975 100644 --- a/drivers/net/ethernet/huawei/hinic3/hinic3_main.c +++ b/drivers/net/ethernet/huawei/hinic3/hinic3_main.c @@ -18,6 +18,7 @@ #define HINIC3_NIC_DRV_DESC "Intelligent Network Interface Card Driver" +#define HINIC3_DEFAULT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_LINK) #define HINIC3_RX_BUF_LEN 2048 #define HINIC3_LRO_REPLENISH_THLD 256 #define HINIC3_NIC_DEV_WQ_NAME "hinic3_nic_dev_wq" @@ -143,6 +144,7 @@ static int hinic3_init_nic_dev(struct net_device *netdev, nic_dev->hwdev = hwdev; nic_dev->pdev = pdev; + nic_dev->msg_enable = HINIC3_DEFAULT_MSG_ENABLE; nic_dev->rx_buf_len = HINIC3_RX_BUF_LEN; nic_dev->lro_replenish_thld = HINIC3_LRO_REPLENISH_THLD; nic_dev->vlan_bitmap = kzalloc(HINIC3_VLAN_BITMAP_SIZE(nic_dev), @@ -241,6 +243,7 @@ static void hinic3_sw_uninit(struct net_device *netdev) static void hinic3_assign_netdev_ops(struct net_device *netdev) { hinic3_set_netdev_ops(netdev); + hinic3_set_ethtool_ops(netdev); } static void netdev_feature_init(struct net_device *netdev) diff --git a/drivers/net/ethernet/huawei/hinic3/hinic3_mbox.c b/drivers/net/ethernet/huawei/hinic3/hinic3_mbox.c index 826fa8879a11..c1bc64a50dd3 100644 --- a/drivers/net/ethernet/huawei/hinic3/hinic3_mbox.c +++ b/drivers/net/ethernet/huawei/hinic3/hinic3_mbox.c @@ -5,6 +5,7 @@ #include "hinic3_common.h" #include "hinic3_csr.h" +#include "hinic3_eqs.h" #include "hinic3_hwdev.h" #include "hinic3_hwif.h" #include "hinic3_mbox.h" @@ -50,9 +51,9 @@ #define MBOX_WB_STATUS_NOT_FINISHED 0x00 #define MBOX_STATUS_FINISHED(wb) \ - ((FIELD_PREP(MBOX_WB_STATUS_MASK, (wb))) != MBOX_WB_STATUS_NOT_FINISHED) + ((FIELD_GET(MBOX_WB_STATUS_MASK, (wb))) != MBOX_WB_STATUS_NOT_FINISHED) #define MBOX_STATUS_SUCCESS(wb) \ - ((FIELD_PREP(MBOX_WB_STATUS_MASK, (wb))) == \ + ((FIELD_GET(MBOX_WB_STATUS_MASK, (wb))) == \ MBOX_WB_STATUS_FINISHED_SUCCESS) #define MBOX_STATUS_ERRCODE(wb) \ ((wb) & MBOX_WB_ERROR_CODE_MASK) @@ -395,6 +396,7 @@ static int hinic3_mbox_pre_init(struct hinic3_hwdev *hwdev, { mbox->hwdev = hwdev; mutex_init(&mbox->mbox_send_lock); + mutex_init(&mbox->mbox_seg_send_lock); spin_lock_init(&mbox->mbox_lock); mbox->workq = create_singlethread_workqueue(HINIC3_MBOX_WQ_NAME); @@ -460,7 +462,8 @@ void hinic3_free_mbox(struct hinic3_hwdev *hwdev) destroy_workqueue(mbox->workq); free_mbox_wb_status(mbox); - hinic3_uninit_func_mbox_msg_channel(hwdev); + if (HINIC3_IS_VF(hwdev)) + hinic3_uninit_func_mbox_msg_channel(hwdev); uninit_mgmt_msg_channel(mbox); kfree(mbox); } @@ -616,6 +619,18 @@ static void write_mbox_msg_attr(struct hinic3_mbox *mbox, mbox_ctrl); } +static void hinic3_dump_mbox_reg(struct hinic3_hwdev *hwdev) +{ + u32 val; + + val = hinic3_hwif_read_reg(hwdev->hwif, + HINIC3_FUNC_CSR_MAILBOX_CONTROL_OFF); + dev_err(hwdev->dev, "Mailbox control reg: 0x%x\n", val); + val = hinic3_hwif_read_reg(hwdev->hwif, + HINIC3_FUNC_CSR_MAILBOX_INT_OFF); + dev_err(hwdev->dev, "Mailbox interrupt offset: 0x%x\n", val); +} + static u16 get_mbox_status(const struct hinic3_send_mbox *mbox) { __be64 *wb_status = mbox->wb_vaddr; @@ -633,6 +648,9 @@ static enum hinic3_wait_return check_mbox_wb_status(void *priv_data) struct hinic3_mbox *mbox = priv_data; u16 wb_status; + if (!mbox->hwdev->chip_present_flag) + return HINIC3_WAIT_PROCESS_ERR; + wb_status = get_mbox_status(&mbox->send_mbox); return MBOX_STATUS_FINISHED(wb_status) ? @@ -670,6 +688,7 @@ static int send_mbox_seg(struct hinic3_mbox *mbox, __le64 header, if (err) { dev_err(hwdev->dev, "Send mailbox segment timeout, wb status: 0x%x\n", wb_status); + hinic3_dump_mbox_reg(hwdev); return err; } @@ -706,6 +725,8 @@ static int send_mbox_msg(struct hinic3_mbox *mbox, u8 mod, u16 cmd, else rsp_aeq_id = 0; + mutex_lock(&mbox->mbox_seg_send_lock); + if (dst_func == MBOX_MGMT_FUNC_ID && !(hwdev->features[0] & MBOX_COMM_F_MBOX_SEGMENT)) { err = mbox_prepare_dma_msg(mbox, ack_type, &dma_msg, @@ -759,6 +780,8 @@ static int send_mbox_msg(struct hinic3_mbox *mbox, u8 mod, u16 cmd, } err_send: + mutex_unlock(&mbox->mbox_seg_send_lock); + return err; } @@ -774,6 +797,9 @@ static enum hinic3_wait_return check_mbox_msg_finish(void *priv_data) { struct hinic3_mbox *mbox = priv_data; + if (!mbox->hwdev->chip_present_flag) + return HINIC3_WAIT_PROCESS_ERR; + return (mbox->event_flag == MBOX_EVENT_SUCCESS) ? HINIC3_WAIT_PROCESS_CPL : HINIC3_WAIT_PROCESS_WAITING; } @@ -805,6 +831,9 @@ int hinic3_send_mbox_to_mgmt(struct hinic3_hwdev *hwdev, u8 mod, u16 cmd, u32 msg_len; int err; + if (!hwdev->chip_present_flag) + return -EPERM; + /* expect response message */ msg_desc = get_mbox_msg_desc(mbox, MBOX_MSG_RESP, MBOX_MGMT_FUNC_ID); mutex_lock(&mbox->mbox_send_lock); @@ -825,6 +854,7 @@ int hinic3_send_mbox_to_mgmt(struct hinic3_hwdev *hwdev, u8 mod, u16 cmd, if (wait_mbox_msg_completion(mbox, msg_params->timeout_ms)) { dev_err(hwdev->dev, "Send mbox msg timeout, msg_id: %u\n", msg_info.msg_id); + hinic3_dump_aeq_info(mbox->hwdev); err = -ETIMEDOUT; goto err_send; } @@ -882,6 +912,9 @@ int hinic3_send_mbox_to_mgmt_no_ack(struct hinic3_hwdev *hwdev, u8 mod, u16 cmd, struct mbox_msg_info msg_info = {}; int err; + if (!hwdev->chip_present_flag) + return -EPERM; + mutex_lock(&mbox->mbox_send_lock); err = send_mbox_msg(mbox, mod, cmd, msg_params->buf_in, msg_params->in_size, MBOX_MGMT_FUNC_ID, diff --git a/drivers/net/ethernet/huawei/hinic3/hinic3_mbox.h b/drivers/net/ethernet/huawei/hinic3/hinic3_mbox.h index e26f22d1d564..30de0c129503 100644 --- a/drivers/net/ethernet/huawei/hinic3/hinic3_mbox.h +++ b/drivers/net/ethernet/huawei/hinic3/hinic3_mbox.h @@ -114,6 +114,10 @@ struct hinic3_mbox { struct hinic3_hwdev *hwdev; /* lock for send mbox message and ack message */ struct mutex mbox_send_lock; + /* lock for send message transmission. + * The lock hierarchy is mbox_send_lock -> mbox_seg_send_lock. + */ + struct mutex mbox_seg_send_lock; struct hinic3_send_mbox send_mbox; struct mbox_dma_queue sync_msg_queue; struct mbox_dma_queue async_msg_queue; diff --git a/drivers/net/ethernet/huawei/hinic3/hinic3_mgmt_interface.h b/drivers/net/ethernet/huawei/hinic3/hinic3_mgmt_interface.h index c0c87a8c2198..c5bca3c4af96 100644 --- a/drivers/net/ethernet/huawei/hinic3/hinic3_mgmt_interface.h +++ b/drivers/net/ethernet/huawei/hinic3/hinic3_mgmt_interface.h @@ -183,7 +183,18 @@ struct l2nic_cmd_lro_timer { /* IEEE 802.1Qaz std */ #define L2NIC_DCB_COS_MAX 0x8 -struct l2nic_cmd_set_rss_ctx_tbl { +struct l2nic_cmd_pause_config { + struct mgmt_msg_head msg_head; + u8 port_id; + u8 opcode; + u16 rsvd1; + u8 auto_neg; + u8 rx_pause; + u8 tx_pause; + u8 rsvd2[5]; +}; + +struct l2nic_cmd_rss_ctx_tbl { struct mgmt_msg_head msg_head; u16 func_id; u16 rsvd1; @@ -238,6 +249,7 @@ enum l2nic_cmd { L2NIC_CMD_CFG_RSS_HASH_KEY = 63, L2NIC_CMD_CFG_RSS_HASH_ENGINE = 64, L2NIC_CMD_SET_RSS_CTX_TBL = 65, + L2NIC_CMD_CFG_PAUSE_INFO = 101, L2NIC_CMD_QOS_DCB_STATE = 110, L2NIC_CMD_FORCE_PKT_DROP = 113, L2NIC_CMD_MAX = 256, @@ -259,6 +271,8 @@ enum l2nic_ucode_cmd { enum mag_cmd { MAG_CMD_SET_PORT_ENABLE = 6, MAG_CMD_GET_LINK_STATUS = 7, + + MAG_CMD_GET_PORT_INFO = 153, }; /* firmware also use this cmd report link event to driver */ diff --git a/drivers/net/ethernet/huawei/hinic3/hinic3_netdev_ops.c b/drivers/net/ethernet/huawei/hinic3/hinic3_netdev_ops.c index cabb8523f246..da73811641a9 100644 --- a/drivers/net/ethernet/huawei/hinic3/hinic3_netdev_ops.c +++ b/drivers/net/ethernet/huawei/hinic3/hinic3_netdev_ops.c @@ -415,13 +415,17 @@ static void hinic3_vport_down(struct net_device *netdev) netif_carrier_off(netdev); netif_tx_disable(netdev); - glb_func_id = hinic3_global_func_id(nic_dev->hwdev); - hinic3_set_vport_enable(nic_dev->hwdev, glb_func_id, false); + if (nic_dev->hwdev->chip_present_flag) { + hinic3_maybe_set_port_state(netdev, false); - hinic3_flush_txqs(netdev); - /* wait to guarantee that no packets will be sent to host */ - msleep(100); - hinic3_flush_qps_res(nic_dev->hwdev); + glb_func_id = hinic3_global_func_id(nic_dev->hwdev); + hinic3_set_vport_enable(nic_dev->hwdev, glb_func_id, false); + + hinic3_flush_txqs(netdev); + /* wait to guarantee that no packets will be sent to host */ + msleep(100); + hinic3_flush_qps_res(nic_dev->hwdev); + } } static int hinic3_open(struct net_device *netdev) diff --git a/drivers/net/ethernet/huawei/hinic3/hinic3_nic_cfg.c b/drivers/net/ethernet/huawei/hinic3/hinic3_nic_cfg.c index 44abccf9cb29..de5a7984d2cb 100644 --- a/drivers/net/ethernet/huawei/hinic3/hinic3_nic_cfg.c +++ b/drivers/net/ethernet/huawei/hinic3/hinic3_nic_cfg.c @@ -639,6 +639,39 @@ int hinic3_get_link_status(struct hinic3_hwdev *hwdev, bool *link_status_up) return 0; } +int hinic3_get_port_info(struct hinic3_hwdev *hwdev, + struct hinic3_nic_port_info *port_info) +{ + struct mag_cmd_get_port_info port_msg = {}; + struct mgmt_msg_params msg_params = {}; + int err; + + port_msg.port_id = hinic3_physical_port_id(hwdev); + + mgmt_msg_params_init_default(&msg_params, &port_msg, sizeof(port_msg)); + + err = hinic3_send_mbox_to_mgmt(hwdev, MGMT_MOD_HILINK, + MAG_CMD_GET_PORT_INFO, &msg_params); + + if (err || port_msg.head.status) { + dev_err(hwdev->dev, + "Failed to get port info, err: %d, status: 0x%x\n", + err, port_msg.head.status); + return -EFAULT; + } + + port_info->autoneg_cap = port_msg.an_support; + port_info->autoneg_state = port_msg.an_en; + port_info->duplex = port_msg.duplex; + port_info->port_type = port_msg.wire_type; + port_info->speed = port_msg.speed; + port_info->fec = port_msg.fec; + port_info->supported_mode = port_msg.supported_mode; + port_info->advertised_mode = port_msg.advertised_mode; + + return 0; +} + int hinic3_set_vport_enable(struct hinic3_hwdev *hwdev, u16 func_id, bool enable) { @@ -661,3 +694,47 @@ int hinic3_set_vport_enable(struct hinic3_hwdev *hwdev, u16 func_id, return 0; } + +static int hinic3_cfg_hw_pause(struct hinic3_hwdev *hwdev, u8 opcode, + struct hinic3_nic_pause_config *nic_pause) +{ + struct l2nic_cmd_pause_config pause_info = {}; + struct mgmt_msg_params msg_params = {}; + int err; + + pause_info.port_id = hinic3_physical_port_id(hwdev); + pause_info.opcode = opcode; + if (opcode == MGMT_MSG_CMD_OP_SET) { + pause_info.auto_neg = nic_pause->auto_neg; + pause_info.rx_pause = nic_pause->rx_pause; + pause_info.tx_pause = nic_pause->tx_pause; + } + + mgmt_msg_params_init_default(&msg_params, &pause_info, + sizeof(pause_info)); + + err = hinic3_send_mbox_to_mgmt(hwdev, MGMT_MOD_L2NIC, + L2NIC_CMD_CFG_PAUSE_INFO, &msg_params); + + if (err || pause_info.msg_head.status) { + dev_err(hwdev->dev, "Failed to %s pause info, err: %d, status: 0x%x\n", + opcode == MGMT_MSG_CMD_OP_SET ? "set" : "get", + err, pause_info.msg_head.status); + return -EFAULT; + } + + if (opcode == MGMT_MSG_CMD_OP_GET) { + nic_pause->auto_neg = pause_info.auto_neg; + nic_pause->rx_pause = pause_info.rx_pause; + nic_pause->tx_pause = pause_info.tx_pause; + } + + return 0; +} + +int hinic3_get_pause_info(struct hinic3_nic_dev *nic_dev, + struct hinic3_nic_pause_config *nic_pause) +{ + return hinic3_cfg_hw_pause(nic_dev->hwdev, MGMT_MSG_CMD_OP_GET, + nic_pause); +} diff --git a/drivers/net/ethernet/huawei/hinic3/hinic3_nic_cfg.h b/drivers/net/ethernet/huawei/hinic3/hinic3_nic_cfg.h index c32eaa886e17..5d52202a8d4e 100644 --- a/drivers/net/ethernet/huawei/hinic3/hinic3_nic_cfg.h +++ b/drivers/net/ethernet/huawei/hinic3/hinic3_nic_cfg.h @@ -6,6 +6,7 @@ #include <linux/types.h> +#include "hinic3_hwif.h" #include "hinic3_hw_intf.h" #include "hinic3_mgmt_interface.h" @@ -35,6 +36,49 @@ struct hinic3_sq_attr { u64 ci_dma_base; }; +enum mag_cmd_port_an { + PORT_CFG_AN_ON = 1, +}; + +/* mag supported/advertised link mode bitmap */ +enum mag_cmd_link_mode { + LINK_MODE_GE = 0, + LINK_MODE_10GE_BASE_R = 1, + LINK_MODE_25GE_BASE_R = 2, + LINK_MODE_40GE_BASE_R4 = 3, + LINK_MODE_50GE_BASE_R = 4, + LINK_MODE_50GE_BASE_R2 = 5, + LINK_MODE_100GE_BASE_R = 6, + LINK_MODE_100GE_BASE_R2 = 7, + LINK_MODE_100GE_BASE_R4 = 8, + LINK_MODE_200GE_BASE_R2 = 9, + LINK_MODE_200GE_BASE_R4 = 10, + LINK_MODE_MAX_NUMBERS, + + LINK_MODE_UNKNOWN = 0xFFFF +}; + +struct mag_cmd_get_port_info { + struct mgmt_msg_head head; + + u8 port_id; + u8 rsvd0[3]; + + u8 wire_type; + u8 an_support; + u8 an_en; + u8 duplex; + + u8 speed; + u8 fec; + u8 lanes; + u8 rsvd1; + + u32 supported_mode; + u32 advertised_mode; + u8 rsvd2[8]; +}; + #define MAG_CMD_PORT_DISABLE 0x0 #define MAG_CMD_TX_ENABLE 0x1 #define MAG_CMD_RX_ENABLE 0x2 @@ -52,6 +96,39 @@ struct mag_cmd_set_port_enable { u8 rsvd1[3]; }; +/* xsfp wire type, refers to cmis protocol definition */ +enum mag_wire_type { + MAG_CMD_WIRE_TYPE_UNKNOWN = 0x0, + MAG_CMD_WIRE_TYPE_MM = 0x1, + MAG_CMD_WIRE_TYPE_SM = 0x2, + MAG_CMD_WIRE_TYPE_COPPER = 0x3, + MAG_CMD_WIRE_TYPE_ACC = 0x4, + MAG_CMD_WIRE_TYPE_BASET = 0x5, + MAG_CMD_WIRE_TYPE_AOC = 0x40, + MAG_CMD_WIRE_TYPE_ELECTRIC = 0x41, + MAG_CMD_WIRE_TYPE_BACKPLANE = 0x42 +}; + +#define XSFP_INFO_MAX_SIZE 640 +struct mag_cmd_get_xsfp_info { + struct mgmt_msg_head head; + + u8 port_id; + u8 wire_type; + u16 out_len; + u32 rsvd; + u8 sfp_info[XSFP_INFO_MAX_SIZE]; +}; + +struct mag_cmd_get_xsfp_present { + struct mgmt_msg_head head; + + u8 port_id; + /* 0:present, 1:absent */ + u8 abs_status; + u8 rsvd[2]; +}; + enum link_err_type { LINK_ERR_MODULE_UNRECOGENIZED, LINK_ERR_NUM, @@ -69,6 +146,34 @@ struct hinic3_port_module_event { enum link_err_type err_type; }; +struct hinic3_nic_port_info { + u8 port_type; + u8 autoneg_cap; + u8 autoneg_state; + u8 duplex; + u8 speed; + u8 fec; + u32 supported_mode; + u32 advertised_mode; +}; + +struct hinic3_nic_pause_config { + u8 auto_neg; + u8 rx_pause; + u8 tx_pause; +}; + +struct hinic3_nic_cfg { + /* Valid when pfc is disabled */ + bool pause_set; + struct hinic3_nic_pause_config nic_pause; + + u8 pfc_en; + u8 pfc_bitmap; + + struct hinic3_nic_port_info port_info; +}; + int hinic3_get_nic_feature_from_hw(struct hinic3_nic_dev *nic_dev); int hinic3_set_nic_feature_to_hw(struct hinic3_nic_dev *nic_dev); bool hinic3_test_support(struct hinic3_nic_dev *nic_dev, @@ -100,9 +205,14 @@ int hinic3_set_rx_mode(struct hinic3_hwdev *hwdev, u32 rx_mode); int hinic3_sync_dcb_state(struct hinic3_hwdev *hwdev, u8 op_code, u8 state); int hinic3_set_port_enable(struct hinic3_hwdev *hwdev, bool enable); int hinic3_get_link_status(struct hinic3_hwdev *hwdev, bool *link_status_up); +int hinic3_get_port_info(struct hinic3_hwdev *hwdev, + struct hinic3_nic_port_info *port_info); int hinic3_set_vport_enable(struct hinic3_hwdev *hwdev, u16 func_id, bool enable); int hinic3_add_vlan(struct hinic3_hwdev *hwdev, u16 vlan_id, u16 func_id); int hinic3_del_vlan(struct hinic3_hwdev *hwdev, u16 vlan_id, u16 func_id); +int hinic3_get_pause_info(struct hinic3_nic_dev *nic_dev, + struct hinic3_nic_pause_config *nic_pause); + #endif diff --git a/drivers/net/ethernet/huawei/hinic3/hinic3_nic_dev.h b/drivers/net/ethernet/huawei/hinic3/hinic3_nic_dev.h index 29189241f446..9502293ff710 100644 --- a/drivers/net/ethernet/huawei/hinic3/hinic3_nic_dev.h +++ b/drivers/net/ethernet/huawei/hinic3/hinic3_nic_dev.h @@ -101,6 +101,7 @@ struct hinic3_nic_dev { struct hinic3_hwdev *hwdev; struct hinic3_nic_io *nic_io; + u32 msg_enable; u16 max_qps; u16 rx_buf_len; u32 lro_replenish_thld; @@ -148,4 +149,6 @@ void hinic3_qps_irq_uninit(struct net_device *netdev); void hinic3_set_rx_mode_work(struct work_struct *work); void hinic3_clean_mac_list_filter(struct net_device *netdev); +void hinic3_set_ethtool_ops(struct net_device *netdev); + #endif diff --git a/drivers/net/ethernet/huawei/hinic3/hinic3_pci_id_tbl.h b/drivers/net/ethernet/huawei/hinic3/hinic3_pci_id_tbl.h index 86c88d0bb4bd..02b2b0fbecc7 100644 --- a/drivers/net/ethernet/huawei/hinic3/hinic3_pci_id_tbl.h +++ b/drivers/net/ethernet/huawei/hinic3/hinic3_pci_id_tbl.h @@ -4,6 +4,7 @@ #ifndef _HINIC3_PCI_ID_TBL_H_ #define _HINIC3_PCI_ID_TBL_H_ +#define PCI_DEV_ID_HINIC3_PF 0x0222 #define PCI_DEV_ID_HINIC3_VF 0x375F #endif diff --git a/drivers/net/ethernet/huawei/hinic3/hinic3_rss.c b/drivers/net/ethernet/huawei/hinic3/hinic3_rss.c index 4ff1b2f79838..25db74d8c7dd 100644 --- a/drivers/net/ethernet/huawei/hinic3/hinic3_rss.c +++ b/drivers/net/ethernet/huawei/hinic3/hinic3_rss.c @@ -132,7 +132,7 @@ static int hinic3_rss_set_indir_tbl(struct hinic3_hwdev *hwdev, static int hinic3_set_rss_type(struct hinic3_hwdev *hwdev, struct hinic3_rss_type rss_type) { - struct l2nic_cmd_set_rss_ctx_tbl ctx_tbl = {}; + struct l2nic_cmd_rss_ctx_tbl ctx_tbl = {}; struct mgmt_msg_params msg_params = {}; u32 ctx; int err; diff --git a/drivers/net/ethernet/huawei/hinic3/hinic3_rx.c b/drivers/net/ethernet/huawei/hinic3/hinic3_rx.c index 1236ec233b7f..309ab5901379 100644 --- a/drivers/net/ethernet/huawei/hinic3/hinic3_rx.c +++ b/drivers/net/ethernet/huawei/hinic3/hinic3_rx.c @@ -328,6 +328,7 @@ static void hinic3_rx_csum(struct hinic3_rxq *rxq, u32 offload_type, u32 ip_type = RQ_CQE_OFFOLAD_TYPE_GET(offload_type, IP_TYPE); u32 csum_err = RQ_CQE_STATUS_GET(status, CSUM_ERR); struct net_device *netdev = rxq->netdev; + bool l2_tunnel; if (!(netdev->features & NETIF_F_RXCSUM)) return; @@ -350,6 +351,12 @@ static void hinic3_rx_csum(struct hinic3_rxq *rxq, u32 offload_type, case HINIC3_RX_UDP_PKT: case HINIC3_RX_SCTP_PKT: skb->ip_summed = CHECKSUM_UNNECESSARY; + l2_tunnel = HINIC3_GET_RX_TUNNEL_PKT_FORMAT(offload_type) == + HINIC3_RX_PKT_FORMAT_VXLAN ? 1 : 0; + if (l2_tunnel) { + /* If we checked the outer header let the stack know */ + skb->csum_level = 1; + } break; default: skb->ip_summed = CHECKSUM_NONE; @@ -390,6 +397,14 @@ static int recv_one_pkt(struct hinic3_rxq *rxq, struct hinic3_rq_cqe *rx_cqe, offload_type = le32_to_cpu(rx_cqe->offload_type); hinic3_rx_csum(rxq, offload_type, status, skb); + if ((netdev->features & NETIF_F_HW_VLAN_CTAG_RX) && + RQ_CQE_OFFOLAD_TYPE_GET(offload_type, VLAN_EN)) { + u16 vid = RQ_CQE_SGE_GET(vlan_len, VLAN); + + /* if the packet is a vlan pkt, the vid may be 0 */ + __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid); + } + num_lro = RQ_CQE_STATUS_GET(status, NUM_LRO); if (num_lro) hinic3_lro_set_gso_params(skb, num_lro); diff --git a/drivers/net/ethernet/huawei/hinic3/hinic3_rx.h b/drivers/net/ethernet/huawei/hinic3/hinic3_rx.h index 31622e0a63d0..06d1b3299e7c 100644 --- a/drivers/net/ethernet/huawei/hinic3/hinic3_rx.h +++ b/drivers/net/ethernet/huawei/hinic3/hinic3_rx.h @@ -15,6 +15,9 @@ #define RQ_CQE_OFFOLAD_TYPE_GET(val, member) \ FIELD_GET(RQ_CQE_OFFOLAD_TYPE_##member##_MASK, val) +#define HINIC3_GET_RX_TUNNEL_PKT_FORMAT(offload_type) \ + RQ_CQE_OFFOLAD_TYPE_GET(offload_type, TUNNEL_PKT_FORMAT) + #define RQ_CQE_SGE_VLAN_MASK GENMASK(15, 0) #define RQ_CQE_SGE_LEN_MASK GENMASK(31, 16) #define RQ_CQE_SGE_GET(val, member) \ diff --git a/drivers/net/ethernet/ibm/ibmveth.c b/drivers/net/ethernet/ibm/ibmveth.c index 4d6b9e83e341..58cc3147afe2 100644 --- a/drivers/net/ethernet/ibm/ibmveth.c +++ b/drivers/net/ethernet/ibm/ibmveth.c @@ -355,7 +355,7 @@ hcall_failure: /* * If multi rx buffers hcall is no longer supported by FW - * e.g. in the case of Live Parttion Migration + * e.g. in the case of Live Partition Migration */ if (batch > 1 && lpar_rc == H_FUNCTION) { /* @@ -480,7 +480,7 @@ static int ibmveth_remove_buffer_from_pool(struct ibmveth_adapter *adapter, */ if (!reuse) { /* remove the skb pointer to mark free. actual freeing is done - * by upper level networking after gro_recieve + * by upper level networking after gro_receive */ adapter->rx_buff_pool[pool].skbuff[index] = NULL; diff --git a/drivers/net/ethernet/intel/i40e/i40e_ethtool.c b/drivers/net/ethernet/intel/i40e/i40e_ethtool.c index 8c27f7ba4adc..3da9ec49cc74 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_ethtool.c +++ b/drivers/net/ethernet/intel/i40e/i40e_ethtool.c @@ -3624,6 +3624,7 @@ static int i40e_set_rxfh_fields(struct net_device *netdev, ((u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(1)) << 32); DECLARE_BITMAP(flow_pctypes, FLOW_PCTYPES_SIZE); u64 i_set, i_setc; + u8 flow_id; bitmap_zero(flow_pctypes, FLOW_PCTYPES_SIZE); @@ -3707,20 +3708,14 @@ static int i40e_set_rxfh_fields(struct net_device *netdev, return -EINVAL; } - if (bitmap_weight(flow_pctypes, FLOW_PCTYPES_SIZE)) { - u8 flow_id; + for_each_set_bit(flow_id, flow_pctypes, FLOW_PCTYPES_SIZE) { + i_setc = (u64)i40e_read_rx_ctl(hw, I40E_GLQF_HASH_INSET(0, flow_id)) | + ((u64)i40e_read_rx_ctl(hw, I40E_GLQF_HASH_INSET(1, flow_id)) << 32); + i_set = i40e_get_rss_hash_bits(&pf->hw, nfc, i_setc); - for_each_set_bit(flow_id, flow_pctypes, FLOW_PCTYPES_SIZE) { - i_setc = (u64)i40e_read_rx_ctl(hw, I40E_GLQF_HASH_INSET(0, flow_id)) | - ((u64)i40e_read_rx_ctl(hw, I40E_GLQF_HASH_INSET(1, flow_id)) << 32); - i_set = i40e_get_rss_hash_bits(&pf->hw, nfc, i_setc); - - i40e_write_rx_ctl(hw, I40E_GLQF_HASH_INSET(0, flow_id), - (u32)i_set); - i40e_write_rx_ctl(hw, I40E_GLQF_HASH_INSET(1, flow_id), - (u32)(i_set >> 32)); - hena |= BIT_ULL(flow_id); - } + i40e_write_rx_ctl(hw, I40E_GLQF_HASH_INSET(0, flow_id), (u32)i_set); + i40e_write_rx_ctl(hw, I40E_GLQF_HASH_INSET(1, flow_id), (u32)(i_set >> 32)); + hena |= BIT_ULL(flow_id); } i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), (u32)hena); diff --git a/drivers/net/ethernet/intel/i40e/i40e_hmc.h b/drivers/net/ethernet/intel/i40e/i40e_hmc.h index 480e3a883cc7..967711405919 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_hmc.h +++ b/drivers/net/ethernet/intel/i40e/i40e_hmc.h @@ -4,6 +4,8 @@ #ifndef _I40E_HMC_H_ #define _I40E_HMC_H_ +#include <linux/wordpart.h> + #include "i40e_alloc.h" #include "i40e_io.h" #include "i40e_register.h" diff --git a/drivers/net/ethernet/intel/iavf/iavf_ethtool.c b/drivers/net/ethernet/intel/iavf/iavf_ethtool.c index 1cd1f3f2930a..a615d599b88e 100644 --- a/drivers/net/ethernet/intel/iavf/iavf_ethtool.c +++ b/drivers/net/ethernet/intel/iavf/iavf_ethtool.c @@ -32,7 +32,7 @@ * statistics array. Thus, every statistic string in an array should have the * same type and number of format specifiers, to be formatted by variadic * arguments to the iavf_add_stat_string() helper function. - **/ + */ struct iavf_stats { char stat_string[ETH_GSTRING_LEN]; int sizeof_stat; @@ -116,7 +116,7 @@ iavf_add_one_ethtool_stat(u64 *data, void *pointer, * the next empty location for successive calls to __iavf_add_ethtool_stats. * If pointer is null, set the data values to zero and update the pointer to * skip these stats. - **/ + */ static void __iavf_add_ethtool_stats(u64 **data, void *pointer, const struct iavf_stats stats[], @@ -140,7 +140,7 @@ __iavf_add_ethtool_stats(u64 **data, void *pointer, * * The parameter @stats is evaluated twice, so parameters with side effects * should be avoided. - **/ + */ #define iavf_add_ethtool_stats(data, pointer, stats) \ __iavf_add_ethtool_stats(data, pointer, stats, ARRAY_SIZE(stats)) @@ -157,7 +157,7 @@ __iavf_add_ethtool_stats(u64 **data, void *pointer, * buffer and update the data pointer when finished. * * This function expects to be called while under rcu_read_lock(). - **/ + */ static void iavf_add_queue_stats(u64 **data, struct iavf_ring *ring) { @@ -189,7 +189,7 @@ iavf_add_queue_stats(u64 **data, struct iavf_ring *ring) * * Format and copy the strings described by stats into the buffer pointed at * by p. - **/ + */ static void __iavf_add_stat_strings(u8 **p, const struct iavf_stats stats[], const unsigned int size, ...) { @@ -216,7 +216,7 @@ static void __iavf_add_stat_strings(u8 **p, const struct iavf_stats stats[], * The parameter @stats is evaluated twice, so parameters with side effects * should be avoided. Additionally, stats must be an array such that * ARRAY_SIZE can be called on it. - **/ + */ #define iavf_add_stat_strings(p, stats, ...) \ __iavf_add_stat_strings(p, stats, ARRAY_SIZE(stats), ## __VA_ARGS__) @@ -249,7 +249,7 @@ static const struct iavf_stats iavf_gstrings_stats[] = { * * Reports speed/duplex settings. Because this is a VF, we don't know what * kind of link we really have, so we fake it. - **/ + */ static int iavf_get_link_ksettings(struct net_device *netdev, struct ethtool_link_ksettings *cmd) { @@ -308,7 +308,7 @@ static int iavf_get_link_ksettings(struct net_device *netdev, * @sset: id of string set * * Reports size of various string tables. - **/ + */ static int iavf_get_sset_count(struct net_device *netdev, int sset) { /* Report the maximum number queues, even if not every queue is @@ -331,7 +331,7 @@ static int iavf_get_sset_count(struct net_device *netdev, int sset) * @data: pointer to data buffer * * All statistics are added to the data buffer as an array of u64. - **/ + */ static void iavf_get_ethtool_stats(struct net_device *netdev, struct ethtool_stats *stats, u64 *data) { @@ -367,7 +367,7 @@ static void iavf_get_ethtool_stats(struct net_device *netdev, * @data: buffer for string data * * Builds the statistics string table - **/ + */ static void iavf_get_stat_strings(struct net_device *netdev, u8 *data) { unsigned int i; @@ -392,7 +392,7 @@ static void iavf_get_stat_strings(struct net_device *netdev, u8 *data) * @data: buffer for string data * * Builds string tables for various string sets - **/ + */ static void iavf_get_strings(struct net_device *netdev, u32 sset, u8 *data) { switch (sset) { @@ -408,8 +408,8 @@ static void iavf_get_strings(struct net_device *netdev, u32 sset, u8 *data) * iavf_get_msglevel - Get debug message level * @netdev: network interface device structure * - * Returns current debug message level. - **/ + * Return: current debug message level. + */ static u32 iavf_get_msglevel(struct net_device *netdev) { struct iavf_adapter *adapter = netdev_priv(netdev); @@ -424,7 +424,7 @@ static u32 iavf_get_msglevel(struct net_device *netdev) * * Set current debug message level. Higher values cause the driver to * be noisier. - **/ + */ static void iavf_set_msglevel(struct net_device *netdev, u32 data) { struct iavf_adapter *adapter = netdev_priv(netdev); @@ -439,8 +439,8 @@ static void iavf_set_msglevel(struct net_device *netdev, u32 data) * @netdev: network interface device structure * @drvinfo: ethool driver info structure * - * Returns information about the driver and device for display to the user. - **/ + * Fills @drvinfo with information about the driver and device. + */ static void iavf_get_drvinfo(struct net_device *netdev, struct ethtool_drvinfo *drvinfo) { @@ -458,9 +458,9 @@ static void iavf_get_drvinfo(struct net_device *netdev, * @kernel_ring: ethtool extenal ringparam structure * @extack: netlink extended ACK report struct * - * Returns current ring parameters. TX and RX rings are reported separately, - * but the number of rings is not reported. - **/ + * Fills @ring with current ring parameters. TX and RX rings are reported + * separately, but the number of rings is not reported. + */ static void iavf_get_ringparam(struct net_device *netdev, struct ethtool_ringparam *ring, struct kernel_ethtool_ringparam *kernel_ring, @@ -483,7 +483,7 @@ static void iavf_get_ringparam(struct net_device *netdev, * * Sets ring parameters. TX and RX rings are controlled separately, but the * number of rings is not specified, so all rings get the same settings. - **/ + */ static int iavf_set_ringparam(struct net_device *netdev, struct ethtool_ringparam *ring, struct kernel_ethtool_ringparam *kernel_ring, @@ -551,7 +551,7 @@ static int iavf_set_ringparam(struct net_device *netdev, * Gets the per-queue settings for coalescence. Specifically Rx and Tx usecs * are per queue. If queue is <0 then we default to queue 0 as the * representative value. - **/ + */ static int __iavf_get_coalesce(struct net_device *netdev, struct ethtool_coalesce *ec, int queue) { @@ -588,11 +588,11 @@ static int __iavf_get_coalesce(struct net_device *netdev, * @kernel_coal: ethtool CQE mode setting structure * @extack: extack for reporting error messages * - * Returns current coalescing settings. This is referred to elsewhere in the - * driver as Interrupt Throttle Rate, as this is how the hardware describes - * this functionality. Note that if per-queue settings have been modified this - * only represents the settings of queue 0. - **/ + * Fills @ec with current coalescing settings. This is referred to elsewhere + * in the driver as Interrupt Throttle Rate, as this is how the hardware + * describes this functionality. Note that if per-queue settings have been + * modified this only represents the settings of queue 0. + */ static int iavf_get_coalesce(struct net_device *netdev, struct ethtool_coalesce *ec, struct kernel_ethtool_coalesce *kernel_coal, @@ -608,7 +608,7 @@ static int iavf_get_coalesce(struct net_device *netdev, * @queue: the queue to read * * Read specific queue's coalesce settings. - **/ + */ static int iavf_get_per_queue_coalesce(struct net_device *netdev, u32 queue, struct ethtool_coalesce *ec) { @@ -622,7 +622,7 @@ static int iavf_get_per_queue_coalesce(struct net_device *netdev, u32 queue, * @queue: the queue to modify * * Change the ITR settings for a specific queue. - **/ + */ static int iavf_set_itr_per_queue(struct iavf_adapter *adapter, struct ethtool_coalesce *ec, int queue) { @@ -680,7 +680,7 @@ static int iavf_set_itr_per_queue(struct iavf_adapter *adapter, * @queue: the queue to change * * Sets the coalesce settings for a particular queue. - **/ + */ static int __iavf_set_coalesce(struct net_device *netdev, struct ethtool_coalesce *ec, int queue) { @@ -722,7 +722,7 @@ static int __iavf_set_coalesce(struct net_device *netdev, * @extack: extack for reporting error messages * * Change current coalescing settings for every queue. - **/ + */ static int iavf_set_coalesce(struct net_device *netdev, struct ethtool_coalesce *ec, struct kernel_ethtool_coalesce *kernel_coal, @@ -1639,7 +1639,7 @@ static int iavf_set_rxnfc(struct net_device *netdev, struct ethtool_rxnfc *cmd) * @netdev: network interface device structure * * Return: number of RX rings. - **/ + */ static u32 iavf_get_rx_ring_count(struct net_device *netdev) { struct iavf_adapter *adapter = netdev_priv(netdev); @@ -1653,8 +1653,8 @@ static u32 iavf_get_rx_ring_count(struct net_device *netdev) * @cmd: ethtool rxnfc command * @rule_locs: pointer to store rule locations * - * Returns Success if the command is supported. - **/ + * Return: 0 on success, -EOPNOTSUPP if the command is not supported. + */ static int iavf_get_rxnfc(struct net_device *netdev, struct ethtool_rxnfc *cmd, u32 *rule_locs) { @@ -1684,13 +1684,13 @@ static int iavf_get_rxnfc(struct net_device *netdev, struct ethtool_rxnfc *cmd, return ret; } /** - * iavf_get_channels: get the number of channels supported by the device + * iavf_get_channels - get the number of channels supported by the device * @netdev: network interface device structure * @ch: channel information structure * * For the purposes of our device, we only use combined channels, i.e. a tx/rx * queue pair. Report one extra channel to match our "other" MSI-X vector. - **/ + */ static void iavf_get_channels(struct net_device *netdev, struct ethtool_channels *ch) { @@ -1706,14 +1706,15 @@ static void iavf_get_channels(struct net_device *netdev, } /** - * iavf_set_channels: set the new channel count + * iavf_set_channels - set the new channel count * @netdev: network interface device structure * @ch: channel information structure * - * Negotiate a new number of channels with the PF then do a reset. During - * reset we'll realloc queues and fix the RSS table. Returns 0 on success, - * negative on failure. - **/ + * Negotiate a new number of channels with the PF then do a reset. During + * reset we'll realloc queues and fix the RSS table. + * + * Return: 0 on success, negative on failure. + */ static int iavf_set_channels(struct net_device *netdev, struct ethtool_channels *ch) { @@ -1750,8 +1751,8 @@ static int iavf_set_channels(struct net_device *netdev, * iavf_get_rxfh_key_size - get the RSS hash key size * @netdev: network interface device structure * - * Returns the table size. - **/ + * Return: the RSS hash key size. + */ static u32 iavf_get_rxfh_key_size(struct net_device *netdev) { struct iavf_adapter *adapter = netdev_priv(netdev); @@ -1763,8 +1764,8 @@ static u32 iavf_get_rxfh_key_size(struct net_device *netdev) * iavf_get_rxfh_indir_size - get the rx flow hash indirection table size * @netdev: network interface device structure * - * Returns the table size. - **/ + * Return: the indirection table size. + */ static u32 iavf_get_rxfh_indir_size(struct net_device *netdev) { struct iavf_adapter *adapter = netdev_priv(netdev); @@ -1777,8 +1778,10 @@ static u32 iavf_get_rxfh_indir_size(struct net_device *netdev) * @netdev: network interface device structure * @rxfh: pointer to param struct (indir, key, hfunc) * - * Reads the indirection table directly from the hardware. Always returns 0. - **/ + * Reads the indirection table directly from the hardware. + * + * Return: 0 always. + */ static int iavf_get_rxfh(struct net_device *netdev, struct ethtool_rxfh_param *rxfh) { @@ -1806,9 +1809,9 @@ static int iavf_get_rxfh(struct net_device *netdev, * @rxfh: pointer to param struct (indir, key, hfunc) * @extack: extended ACK from the Netlink message * - * Returns -EINVAL if the table specifies an invalid queue id, otherwise - * returns 0 after programming the table. - **/ + * Return: 0 on success, -EOPNOTSUPP if the hash function is not supported, + * -EINVAL if the table specifies an invalid queue id. + */ static int iavf_set_rxfh(struct net_device *netdev, struct ethtool_rxfh_param *rxfh, struct netlink_ext_ack *extack) @@ -1885,7 +1888,7 @@ static const struct ethtool_ops iavf_ethtool_ops = { * * Sets ethtool ops struct in our netdev so that ethtool can call * our functions. - **/ + */ void iavf_set_ethtool_ops(struct net_device *netdev) { netdev->ethtool_ops = &iavf_ethtool_ops; diff --git a/drivers/net/ethernet/intel/ice/ice_ptp_hw.h b/drivers/net/ethernet/intel/ice/ice_ptp_hw.h index 5896b346e579..9bfd3e79c580 100644 --- a/drivers/net/ethernet/intel/ice/ice_ptp_hw.h +++ b/drivers/net/ethernet/intel/ice/ice_ptp_hw.h @@ -258,7 +258,7 @@ enum ice_si_cgu_out_pins { }; struct ice_cgu_pin_desc { - char *name; + const char *name; u8 index; enum dpll_pin_type type; u32 freq_supp_num; diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe.h b/drivers/net/ethernet/intel/ixgbe/ixgbe.h index 047f04045585..9b8217523fd2 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe.h +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe.h @@ -322,10 +322,11 @@ enum ixgbe_ring_state_t { __IXGBE_HANG_CHECK_ARMED, __IXGBE_TX_XDP_RING, __IXGBE_TX_DISABLED, + __IXGBE_RING_STATE_NBITS, /* must be last */ }; #define ring_uses_build_skb(ring) \ - test_bit(__IXGBE_RX_BUILD_SKB_ENABLED, &(ring)->state) + test_bit(__IXGBE_RX_BUILD_SKB_ENABLED, (ring)->state) struct ixgbe_fwd_adapter { unsigned long active_vlans[BITS_TO_LONGS(VLAN_N_VID)]; @@ -336,23 +337,23 @@ struct ixgbe_fwd_adapter { }; #define check_for_tx_hang(ring) \ - test_bit(__IXGBE_TX_DETECT_HANG, &(ring)->state) + test_bit(__IXGBE_TX_DETECT_HANG, (ring)->state) #define set_check_for_tx_hang(ring) \ - set_bit(__IXGBE_TX_DETECT_HANG, &(ring)->state) + set_bit(__IXGBE_TX_DETECT_HANG, (ring)->state) #define clear_check_for_tx_hang(ring) \ - clear_bit(__IXGBE_TX_DETECT_HANG, &(ring)->state) + clear_bit(__IXGBE_TX_DETECT_HANG, (ring)->state) #define ring_is_rsc_enabled(ring) \ - test_bit(__IXGBE_RX_RSC_ENABLED, &(ring)->state) + test_bit(__IXGBE_RX_RSC_ENABLED, (ring)->state) #define set_ring_rsc_enabled(ring) \ - set_bit(__IXGBE_RX_RSC_ENABLED, &(ring)->state) + set_bit(__IXGBE_RX_RSC_ENABLED, (ring)->state) #define clear_ring_rsc_enabled(ring) \ - clear_bit(__IXGBE_RX_RSC_ENABLED, &(ring)->state) + clear_bit(__IXGBE_RX_RSC_ENABLED, (ring)->state) #define ring_is_xdp(ring) \ - test_bit(__IXGBE_TX_XDP_RING, &(ring)->state) + test_bit(__IXGBE_TX_XDP_RING, (ring)->state) #define set_ring_xdp(ring) \ - set_bit(__IXGBE_TX_XDP_RING, &(ring)->state) + set_bit(__IXGBE_TX_XDP_RING, (ring)->state) #define clear_ring_xdp(ring) \ - clear_bit(__IXGBE_TX_XDP_RING, &(ring)->state) + clear_bit(__IXGBE_TX_XDP_RING, (ring)->state) struct ixgbe_ring { struct ixgbe_ring *next; /* pointer to next ring in q_vector */ struct ixgbe_q_vector *q_vector; /* backpointer to host q_vector */ @@ -364,7 +365,7 @@ struct ixgbe_ring { struct ixgbe_tx_buffer *tx_buffer_info; struct ixgbe_rx_buffer *rx_buffer_info; }; - unsigned long state; + DECLARE_BITMAP(state, __IXGBE_RING_STATE_NBITS); u8 __iomem *tail; dma_addr_t dma; /* phys. address of descriptor ring */ unsigned int size; /* length in bytes */ @@ -453,7 +454,7 @@ struct ixgbe_ring_feature { */ static inline unsigned int ixgbe_rx_bufsz(struct ixgbe_ring *ring) { - if (test_bit(__IXGBE_RX_3K_BUFFER, &ring->state)) + if (test_bit(__IXGBE_RX_3K_BUFFER, ring->state)) return IXGBE_RXBUFFER_3K; #if (PAGE_SIZE < 8192) if (ring_uses_build_skb(ring)) @@ -465,7 +466,7 @@ static inline unsigned int ixgbe_rx_bufsz(struct ixgbe_ring *ring) static inline unsigned int ixgbe_rx_pg_order(struct ixgbe_ring *ring) { #if (PAGE_SIZE < 8192) - if (test_bit(__IXGBE_RX_3K_BUFFER, &ring->state)) + if (test_bit(__IXGBE_RX_3K_BUFFER, ring->state)) return 1; #endif return 0; diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_lib.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_lib.c index 87abb36f50c1..1db4bd5cc2ba 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_lib.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_lib.c @@ -976,7 +976,7 @@ static int ixgbe_alloc_q_vector(struct ixgbe_adapter *adapter, * can be marked as checksum errors. */ if (adapter->hw.mac.type == ixgbe_mac_82599EB) - set_bit(__IXGBE_RX_CSUM_UDP_ZERO_ERR, &ring->state); + set_bit(__IXGBE_RX_CSUM_UDP_ZERO_ERR, ring->state); #ifdef IXGBE_FCOE if (adapter->netdev->fcoe_mtu) { @@ -984,7 +984,7 @@ static int ixgbe_alloc_q_vector(struct ixgbe_adapter *adapter, f = &adapter->ring_feature[RING_F_FCOE]; if ((rxr_idx >= f->offset) && (rxr_idx < f->offset + f->indices)) - set_bit(__IXGBE_RX_FCOE, &ring->state); + set_bit(__IXGBE_RX_FCOE, ring->state); } #endif /* IXGBE_FCOE */ diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c index 59801187a839..2646ee6f295f 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c @@ -968,7 +968,7 @@ static void ixgbe_update_xoff_rx_lfc(struct ixgbe_adapter *adapter) for (i = 0; i < adapter->num_tx_queues; i++) clear_bit(__IXGBE_HANG_CHECK_ARMED, - &adapter->tx_ring[i]->state); + adapter->tx_ring[i]->state); } static void ixgbe_update_xoff_received(struct ixgbe_adapter *adapter) @@ -1011,7 +1011,7 @@ static void ixgbe_update_xoff_received(struct ixgbe_adapter *adapter) tc = tx_ring->dcb_tc; if (xoff[tc]) - clear_bit(__IXGBE_HANG_CHECK_ARMED, &tx_ring->state); + clear_bit(__IXGBE_HANG_CHECK_ARMED, tx_ring->state); } for (i = 0; i < adapter->num_xdp_queues; i++) { @@ -1019,7 +1019,7 @@ static void ixgbe_update_xoff_received(struct ixgbe_adapter *adapter) tc = xdp_ring->dcb_tc; if (xoff[tc]) - clear_bit(__IXGBE_HANG_CHECK_ARMED, &xdp_ring->state); + clear_bit(__IXGBE_HANG_CHECK_ARMED, xdp_ring->state); } } @@ -1103,11 +1103,11 @@ static bool ixgbe_check_tx_hang(struct ixgbe_ring *tx_ring) if (tx_done_old == tx_done && tx_pending) /* make sure it is true for two checks in a row */ return test_and_set_bit(__IXGBE_HANG_CHECK_ARMED, - &tx_ring->state); + tx_ring->state); /* update completed stats and continue */ tx_ring->tx_stats.tx_done_old = tx_done; /* reset the countdown */ - clear_bit(__IXGBE_HANG_CHECK_ARMED, &tx_ring->state); + clear_bit(__IXGBE_HANG_CHECK_ARMED, tx_ring->state); return false; } @@ -1660,7 +1660,7 @@ static inline bool ixgbe_rx_is_fcoe(struct ixgbe_ring *ring, { __le16 pkt_info = rx_desc->wb.lower.lo_dword.hs_rss.pkt_info; - return test_bit(__IXGBE_RX_FCOE, &ring->state) && + return test_bit(__IXGBE_RX_FCOE, ring->state) && ((pkt_info & cpu_to_le16(IXGBE_RXDADV_PKTTYPE_ETQF_MASK)) == (cpu_to_le16(IXGBE_ETQF_FILTER_FCOE << IXGBE_RXDADV_PKTTYPE_ETQF_SHIFT))); @@ -1708,7 +1708,7 @@ static inline void ixgbe_rx_checksum(struct ixgbe_ring *ring, * checksum errors. */ if ((pkt_info & cpu_to_le16(IXGBE_RXDADV_PKTTYPE_UDP)) && - test_bit(__IXGBE_RX_CSUM_UDP_ZERO_ERR, &ring->state)) + test_bit(__IXGBE_RX_CSUM_UDP_ZERO_ERR, ring->state)) return; ring->rx_stats.csum_err++; @@ -3526,7 +3526,7 @@ static irqreturn_t ixgbe_msix_other(int irq, void *data) for (i = 0; i < adapter->num_tx_queues; i++) { struct ixgbe_ring *ring = adapter->tx_ring[i]; if (test_and_clear_bit(__IXGBE_TX_FDIR_INIT_DONE, - &ring->state)) + ring->state)) reinit_count++; } if (reinit_count) { @@ -3952,13 +3952,13 @@ void ixgbe_configure_tx_ring(struct ixgbe_adapter *adapter, if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) { ring->atr_sample_rate = adapter->atr_sample_rate; ring->atr_count = 0; - set_bit(__IXGBE_TX_FDIR_INIT_DONE, &ring->state); + set_bit(__IXGBE_TX_FDIR_INIT_DONE, ring->state); } else { ring->atr_sample_rate = 0; } /* initialize XPS */ - if (!test_and_set_bit(__IXGBE_TX_XPS_INIT_DONE, &ring->state)) { + if (!test_and_set_bit(__IXGBE_TX_XPS_INIT_DONE, ring->state)) { struct ixgbe_q_vector *q_vector = ring->q_vector; if (q_vector) @@ -3967,7 +3967,7 @@ void ixgbe_configure_tx_ring(struct ixgbe_adapter *adapter, ring->queue_index); } - clear_bit(__IXGBE_HANG_CHECK_ARMED, &ring->state); + clear_bit(__IXGBE_HANG_CHECK_ARMED, ring->state); /* reinitialize tx_buffer_info */ memset(ring->tx_buffer_info, 0, @@ -4173,7 +4173,7 @@ static void ixgbe_configure_srrctl(struct ixgbe_adapter *adapter, srrctl |= PAGE_SIZE >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; else srrctl |= xsk_buf_len >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; - } else if (test_bit(__IXGBE_RX_3K_BUFFER, &rx_ring->state)) { + } else if (test_bit(__IXGBE_RX_3K_BUFFER, rx_ring->state)) { srrctl |= IXGBE_RXBUFFER_3K >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; } else { srrctl |= IXGBE_RXBUFFER_2K >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; @@ -4558,7 +4558,7 @@ void ixgbe_configure_rx_ring(struct ixgbe_adapter *adapter, * higher than the MTU of the PF. */ if (ring_uses_build_skb(ring) && - !test_bit(__IXGBE_RX_3K_BUFFER, &ring->state)) + !test_bit(__IXGBE_RX_3K_BUFFER, ring->state)) rxdctl |= IXGBE_MAX_2K_FRAME_BUILD_SKB | IXGBE_RXDCTL_RLPML_EN; #endif @@ -4733,27 +4733,27 @@ static void ixgbe_set_rx_buffer_len(struct ixgbe_adapter *adapter) rx_ring = adapter->rx_ring[i]; clear_ring_rsc_enabled(rx_ring); - clear_bit(__IXGBE_RX_3K_BUFFER, &rx_ring->state); - clear_bit(__IXGBE_RX_BUILD_SKB_ENABLED, &rx_ring->state); + clear_bit(__IXGBE_RX_3K_BUFFER, rx_ring->state); + clear_bit(__IXGBE_RX_BUILD_SKB_ENABLED, rx_ring->state); if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED) set_ring_rsc_enabled(rx_ring); - if (test_bit(__IXGBE_RX_FCOE, &rx_ring->state)) - set_bit(__IXGBE_RX_3K_BUFFER, &rx_ring->state); + if (test_bit(__IXGBE_RX_FCOE, rx_ring->state)) + set_bit(__IXGBE_RX_3K_BUFFER, rx_ring->state); if (adapter->flags2 & IXGBE_FLAG2_RX_LEGACY) continue; - set_bit(__IXGBE_RX_BUILD_SKB_ENABLED, &rx_ring->state); + set_bit(__IXGBE_RX_BUILD_SKB_ENABLED, rx_ring->state); #if (PAGE_SIZE < 8192) if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED) - set_bit(__IXGBE_RX_3K_BUFFER, &rx_ring->state); + set_bit(__IXGBE_RX_3K_BUFFER, rx_ring->state); if (IXGBE_2K_TOO_SMALL_WITH_PADDING || (max_frame > (ETH_FRAME_LEN + ETH_FCS_LEN))) - set_bit(__IXGBE_RX_3K_BUFFER, &rx_ring->state); + set_bit(__IXGBE_RX_3K_BUFFER, rx_ring->state); #endif } } @@ -7954,10 +7954,10 @@ static void ixgbe_fdir_reinit_subtask(struct ixgbe_adapter *adapter) if (ixgbe_reinit_fdir_tables_82599(hw) == 0) { for (i = 0; i < adapter->num_tx_queues; i++) set_bit(__IXGBE_TX_FDIR_INIT_DONE, - &(adapter->tx_ring[i]->state)); + adapter->tx_ring[i]->state); for (i = 0; i < adapter->num_xdp_queues; i++) set_bit(__IXGBE_TX_FDIR_INIT_DONE, - &adapter->xdp_ring[i]->state); + adapter->xdp_ring[i]->state); /* re-enable flow director interrupts */ IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EIMS_FLOW_DIR); } else { @@ -9498,7 +9498,7 @@ netdev_tx_t ixgbe_xmit_frame_ring(struct sk_buff *skb, ixgbe_tx_csum(tx_ring, first, &ipsec_tx); /* add the ATR filter if ATR is on */ - if (test_bit(__IXGBE_TX_FDIR_INIT_DONE, &tx_ring->state)) + if (test_bit(__IXGBE_TX_FDIR_INIT_DONE, tx_ring->state)) ixgbe_atr(tx_ring, first); #ifdef IXGBE_FCOE @@ -9538,7 +9538,7 @@ static netdev_tx_t __ixgbe_xmit_frame(struct sk_buff *skb, return NETDEV_TX_OK; tx_ring = ring ? ring : adapter->tx_ring[skb_get_queue_mapping(skb)]; - if (unlikely(test_bit(__IXGBE_TX_DISABLED, &tx_ring->state))) + if (unlikely(test_bit(__IXGBE_TX_DISABLED, tx_ring->state))) return NETDEV_TX_BUSY; return ixgbe_xmit_frame_ring(skb, adapter, tx_ring); @@ -11023,7 +11023,7 @@ static int ixgbe_xdp_xmit(struct net_device *dev, int n, if (unlikely(!ring)) return -ENXIO; - if (unlikely(test_bit(__IXGBE_TX_DISABLED, &ring->state))) + if (unlikely(test_bit(__IXGBE_TX_DISABLED, ring->state))) return -ENXIO; if (static_branch_unlikely(&ixgbe_xdp_locking_key)) @@ -11129,7 +11129,7 @@ static void ixgbe_disable_txr_hw(struct ixgbe_adapter *adapter, static void ixgbe_disable_txr(struct ixgbe_adapter *adapter, struct ixgbe_ring *tx_ring) { - set_bit(__IXGBE_TX_DISABLED, &tx_ring->state); + set_bit(__IXGBE_TX_DISABLED, tx_ring->state); ixgbe_disable_txr_hw(adapter, tx_ring); } @@ -11283,9 +11283,9 @@ void ixgbe_txrx_ring_enable(struct ixgbe_adapter *adapter, int ring) ixgbe_configure_tx_ring(adapter, xdp_ring); ixgbe_configure_rx_ring(adapter, rx_ring); - clear_bit(__IXGBE_TX_DISABLED, &tx_ring->state); + clear_bit(__IXGBE_TX_DISABLED, tx_ring->state); if (xdp_ring) - clear_bit(__IXGBE_TX_DISABLED, &xdp_ring->state); + clear_bit(__IXGBE_TX_DISABLED, xdp_ring->state); /* Rx/Tx/XDP Tx share the same napi context. */ napi_enable(&rx_ring->q_vector->napi); diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c index 7b941505a9d0..89f96c463f02 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c @@ -524,7 +524,7 @@ int ixgbe_xsk_wakeup(struct net_device *dev, u32 qid, u32 flags) ring = adapter->xdp_ring[qid]; - if (test_bit(__IXGBE_TX_DISABLED, &ring->state)) + if (test_bit(__IXGBE_TX_DISABLED, ring->state)) return -ENETDOWN; if (!ring->xsk_pool) diff --git a/drivers/net/ethernet/litex/litex_liteeth.c b/drivers/net/ethernet/litex/litex_liteeth.c index 829a4b828f8e..108d0a0db206 100644 --- a/drivers/net/ethernet/litex/litex_liteeth.c +++ b/drivers/net/ethernet/litex/litex_liteeth.c @@ -232,12 +232,13 @@ static void liteeth_setup_slots(struct liteeth *priv) static int liteeth_probe(struct platform_device *pdev) { + struct device *dev = &pdev->dev; struct net_device *netdev; void __iomem *buf_base; struct liteeth *priv; int irq, err; - netdev = devm_alloc_etherdev(&pdev->dev, sizeof(*priv)); + netdev = devm_alloc_etherdev(dev, sizeof(*priv)); if (!netdev) return -ENOMEM; @@ -246,9 +247,9 @@ static int liteeth_probe(struct platform_device *pdev) priv = netdev_priv(netdev); priv->netdev = netdev; - priv->dev = &pdev->dev; + priv->dev = dev; - netdev->tstats = devm_netdev_alloc_pcpu_stats(&pdev->dev, + netdev->tstats = devm_netdev_alloc_pcpu_stats(dev, struct pcpu_sw_netstats); if (!netdev->tstats) return -ENOMEM; @@ -276,15 +277,15 @@ static int liteeth_probe(struct platform_device *pdev) priv->tx_base = buf_base + priv->num_rx_slots * priv->slot_size; priv->tx_slot = 0; - err = of_get_ethdev_address(pdev->dev.of_node, netdev); + err = of_get_ethdev_address(dev->of_node, netdev); if (err) eth_hw_addr_random(netdev); netdev->netdev_ops = &liteeth_netdev_ops; - err = register_netdev(netdev); + err = devm_register_netdev(dev, netdev); if (err) { - dev_err(&pdev->dev, "Failed to register netdev %d\n", err); + dev_err(dev, "Failed to register netdev %d\n", err); return err; } @@ -294,13 +295,6 @@ static int liteeth_probe(struct platform_device *pdev) return 0; } -static void liteeth_remove(struct platform_device *pdev) -{ - struct net_device *netdev = platform_get_drvdata(pdev); - - unregister_netdev(netdev); -} - static const struct of_device_id liteeth_of_match[] = { { .compatible = "litex,liteeth" }, { } @@ -309,7 +303,6 @@ MODULE_DEVICE_TABLE(of, liteeth_of_match); static struct platform_driver liteeth_driver = { .probe = liteeth_probe, - .remove = liteeth_remove, .driver = { .name = DRV_NAME, .of_match_table = liteeth_of_match, diff --git a/drivers/net/ethernet/marvell/mvneta.c b/drivers/net/ethernet/marvell/mvneta.c index 9ba4aef7080c..0c061fb0ed07 100644 --- a/drivers/net/ethernet/marvell/mvneta.c +++ b/drivers/net/ethernet/marvell/mvneta.c @@ -5620,6 +5620,8 @@ static int mvneta_probe(struct platform_device *pdev) } err = of_get_ethdev_address(dn, dev); + if (err == -EPROBE_DEFER) + goto err_free_stats; if (!err) { mac_from = "device tree"; } else { @@ -5755,6 +5757,7 @@ err_netdev: 1 << pp->id); mvneta_bm_put(pp->bm_priv); } +err_free_stats: free_percpu(pp->stats); err_free_ports: free_percpu(pp->ports); diff --git a/drivers/net/ethernet/marvell/octeon_ep/octep_rx.c b/drivers/net/ethernet/marvell/octeon_ep/octep_rx.c index 74de19166488..e6ebc7e44a00 100644 --- a/drivers/net/ethernet/marvell/octeon_ep/octep_rx.c +++ b/drivers/net/ethernet/marvell/octeon_ep/octep_rx.c @@ -392,7 +392,7 @@ static void octep_oq_drop_rx(struct octep_oq *oq, while (data_len > 0) { octep_oq_next_pkt(oq, buff_info, read_idx, desc_used); data_len -= oq->buffer_size; - }; + } } /** diff --git a/drivers/net/ethernet/marvell/octeon_ep_vf/octep_vf_rx.c b/drivers/net/ethernet/marvell/octeon_ep_vf/octep_vf_rx.c index b579d5b545c4..d98247408242 100644 --- a/drivers/net/ethernet/marvell/octeon_ep_vf/octep_vf_rx.c +++ b/drivers/net/ethernet/marvell/octeon_ep_vf/octep_vf_rx.c @@ -352,6 +352,11 @@ static int octep_vf_oq_check_hw_for_pkts(struct octep_vf_device *oct, return new_pkts; } +static inline u32 octep_vf_oq_next_idx(struct octep_vf_oq *oq, u32 idx) +{ + return (idx + 1 == oq->max_count) ? 0 : idx + 1; +} + /** * __octep_vf_oq_process_rx() - Process hardware Rx queue and push to stack. * @@ -409,30 +414,52 @@ static int __octep_vf_oq_process_rx(struct octep_vf_device *oct, data_offset = OCTEP_VF_OQ_RESP_HW_SIZE; rx_ol_flags = 0; } - rx_bytes += buff_info->len; - if (buff_info->len <= oq->max_single_buffer_size) { skb = napi_build_skb((void *)resp_hw, PAGE_SIZE); + if (!skb) { + oq->stats->alloc_failures++; + desc_used++; + read_idx = octep_vf_oq_next_idx(oq, read_idx); + continue; + } + rx_bytes += buff_info->len; skb_reserve(skb, data_offset); skb_put(skb, buff_info->len); - read_idx++; desc_used++; - if (read_idx == oq->max_count) - read_idx = 0; + read_idx = octep_vf_oq_next_idx(oq, read_idx); } else { struct skb_shared_info *shinfo; u16 data_len; skb = napi_build_skb((void *)resp_hw, PAGE_SIZE); + if (!skb) { + oq->stats->alloc_failures++; + desc_used++; + read_idx = octep_vf_oq_next_idx(oq, read_idx); + data_len = buff_info->len - oq->max_single_buffer_size; + while (data_len) { + dma_unmap_page(oq->dev, oq->desc_ring[read_idx].buffer_ptr, + PAGE_SIZE, DMA_FROM_DEVICE); + buff_info = (struct octep_vf_rx_buffer *) + &oq->buff_info[read_idx]; + buff_info->page = NULL; + if (data_len < oq->buffer_size) + data_len = 0; + else + data_len -= oq->buffer_size; + desc_used++; + read_idx = octep_vf_oq_next_idx(oq, read_idx); + } + continue; + } + rx_bytes += buff_info->len; skb_reserve(skb, data_offset); /* Head fragment includes response header(s); * subsequent fragments contains only data. */ skb_put(skb, oq->max_single_buffer_size); - read_idx++; desc_used++; - if (read_idx == oq->max_count) - read_idx = 0; + read_idx = octep_vf_oq_next_idx(oq, read_idx); shinfo = skb_shinfo(skb); data_len = buff_info->len - oq->max_single_buffer_size; @@ -454,10 +481,8 @@ static int __octep_vf_oq_process_rx(struct octep_vf_device *oct, buff_info->len, buff_info->len); buff_info->page = NULL; - read_idx++; desc_used++; - if (read_idx == oq->max_count) - read_idx = 0; + read_idx = octep_vf_oq_next_idx(oq, read_idx); } } diff --git a/drivers/net/ethernet/marvell/octeontx2/Kconfig b/drivers/net/ethernet/marvell/octeontx2/Kconfig index 35c4f5f64f58..47e549c581f0 100644 --- a/drivers/net/ethernet/marvell/octeontx2/Kconfig +++ b/drivers/net/ethernet/marvell/octeontx2/Kconfig @@ -33,6 +33,7 @@ config OCTEONTX2_PF select OCTEONTX2_MBOX select NET_DEVLINK select PAGE_POOL + select CRYPTO_LIB_AES if MACSEC depends on (64BIT && COMPILE_TEST) || ARM64 select DIMLIB depends on PCI diff --git a/drivers/net/ethernet/marvell/octeontx2/af/Makefile b/drivers/net/ethernet/marvell/octeontx2/af/Makefile index 244de500963e..91b7d6e96a61 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/Makefile +++ b/drivers/net/ethernet/marvell/octeontx2/af/Makefile @@ -13,4 +13,4 @@ rvu_af-y := cgx.o rvu.o rvu_cgx.o rvu_npa.o rvu_nix.o \ rvu_cpt.o rvu_devlink.o rpm.o rvu_cn10k.o rvu_switch.o \ rvu_sdp.o rvu_npc_hash.o mcs.o mcs_rvu_if.o mcs_cnf10kb.o \ rvu_rep.o cn20k/mbox_init.o cn20k/nix.o cn20k/debugfs.o \ - cn20k/npa.o + cn20k/npa.o cn20k/npc.o diff --git a/drivers/net/ethernet/marvell/octeontx2/af/cgx.c b/drivers/net/ethernet/marvell/octeontx2/af/cgx.c index 6000795823a3..4f33a816bc7a 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/cgx.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/cgx.c @@ -1822,7 +1822,7 @@ static int cgx_lmac_exit(struct cgx *cgx) continue; cgx->mac_ops->mac_pause_frm_config(cgx, lmac->lmac_id, false); cgx_configure_interrupt(cgx, lmac, lmac->lmac_id, true); - kfree(lmac->mac_to_index_bmap.bmap); + rvu_free_bitmap(&lmac->mac_to_index_bmap); rvu_free_bitmap(&lmac->rx_fc_pfvf_bmap); rvu_free_bitmap(&lmac->tx_fc_pfvf_bmap); kfree(lmac->name); diff --git a/drivers/net/ethernet/marvell/octeontx2/af/cn20k/debugfs.c b/drivers/net/ethernet/marvell/octeontx2/af/cn20k/debugfs.c index 498968bf4cf5..3debf2fae1a4 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/cn20k/debugfs.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/cn20k/debugfs.c @@ -11,7 +11,280 @@ #include <linux/pci.h> #include "struct.h" +#include "rvu.h" #include "debugfs.h" +#include "cn20k/npc.h" + +static int npc_mcam_layout_show(struct seq_file *s, void *unused) +{ + int i, j, sbd, idx0, idx1, vidx0, vidx1; + struct npc_priv_t *npc_priv; + char buf0[32], buf1[32]; + struct npc_subbank *sb; + unsigned int bw0, bw1; + bool v0, v1; + int pf1, pf2; + bool e0, e1; + void *map; + + npc_priv = s->private; + + sbd = npc_priv->subbank_depth; + + for (i = npc_priv->num_subbanks - 1; i >= 0; i--) { + sb = &npc_priv->sb[i]; + mutex_lock(&sb->lock); + + if (sb->flags & NPC_SUBBANK_FLAG_FREE) + goto next; + + bw0 = bitmap_weight(sb->b0map, npc_priv->subbank_depth); + if (sb->key_type == NPC_MCAM_KEY_X4) { + seq_printf(s, "\n\nsubbank:%u, x4, free=%u, used=%u\n", + sb->idx, sb->free_cnt, bw0); + + for (j = sbd - 1; j >= 0; j--) { + if (!test_bit(j, sb->b0map)) + continue; + + idx0 = sb->b0b + j; + map = xa_load(&npc_priv->xa_idx2pf_map, idx0); + pf1 = xa_to_value(map); + + map = xa_load(&npc_priv->xa_idx2vidx_map, idx0); + if (map) { + vidx0 = xa_to_value(map); + snprintf(buf0, sizeof(buf0), + "v:%u", vidx0); + } + + seq_printf(s, "\t%u(%#x) %s\n", idx0, pf1, + map ? buf0 : " "); + } + goto next; + } + + bw1 = bitmap_weight(sb->b1map, npc_priv->subbank_depth); + seq_printf(s, "\n\nsubbank:%u, x2, free=%u, used=%u\n", + sb->idx, sb->free_cnt, bw0 + bw1); + seq_printf(s, "bank1(%u)\t\tbank0(%u)\n", bw1, bw0); + + for (j = sbd - 1; j >= 0; j--) { + e0 = test_bit(j, sb->b0map); + e1 = test_bit(j, sb->b1map); + + if (!e1 && !e0) + continue; + + if (e1 && e0) { + idx0 = sb->b0b + j; + map = xa_load(&npc_priv->xa_idx2pf_map, idx0); + pf1 = xa_to_value(map); + + map = xa_load(&npc_priv->xa_idx2vidx_map, idx0); + v0 = !!map; + if (v0) { + vidx0 = xa_to_value(map); + snprintf(buf0, sizeof(buf0), "v:%05u", + vidx0); + } + + idx1 = sb->b1b + j; + map = xa_load(&npc_priv->xa_idx2pf_map, idx1); + pf2 = xa_to_value(map); + + map = xa_load(&npc_priv->xa_idx2vidx_map, idx1); + v1 = !!map; + if (v1) { + vidx1 = xa_to_value(map); + snprintf(buf1, sizeof(buf1), "v:%05u", + vidx1); + } + + seq_printf(s, "%05u(%#x) %s\t\t%05u(%#x) %s\n", + idx1, pf2, v1 ? buf1 : " ", + idx0, pf1, v0 ? buf0 : " "); + + continue; + } + + if (e0) { + idx0 = sb->b0b + j; + map = xa_load(&npc_priv->xa_idx2pf_map, idx0); + pf1 = xa_to_value(map); + + map = xa_load(&npc_priv->xa_idx2vidx_map, idx0); + if (map) { + vidx0 = xa_to_value(map); + snprintf(buf0, sizeof(buf0), "v:%05u", + vidx0); + } + + seq_printf(s, "\t\t \t\t%05u(%#x) %s\n", idx0, + pf1, map ? buf0 : " "); + continue; + } + + idx1 = sb->b1b + j; + map = xa_load(&npc_priv->xa_idx2pf_map, idx1); + pf1 = xa_to_value(map); + map = xa_load(&npc_priv->xa_idx2vidx_map, idx1); + if (map) { + vidx1 = xa_to_value(map); + snprintf(buf1, sizeof(buf1), "v:%05u", vidx1); + } + + seq_printf(s, "%05u(%#x) %s\n", idx1, pf1, + map ? buf1 : " "); + } +next: + mutex_unlock(&sb->lock); + } + return 0; +} + +DEFINE_SHOW_ATTRIBUTE(npc_mcam_layout); + +static int npc_mcam_default_show(struct seq_file *s, void *unused) +{ + struct npc_priv_t *npc_priv; + unsigned long index; + u16 ptr[4], pcifunc; + struct rvu *rvu; + int rc, i; + void *map; + + npc_priv = npc_priv_get(); + rvu = s->private; + + seq_puts(s, "\npcifunc\tBcast\tmcast\tpromisc\tucast\n"); + + xa_for_each(&npc_priv->xa_pf_map, index, map) { + pcifunc = index; + + for (i = 0; i < ARRAY_SIZE(ptr); i++) + ptr[i] = USHRT_MAX; + + rc = npc_cn20k_dft_rules_idx_get(rvu, pcifunc, &ptr[0], + &ptr[1], &ptr[2], &ptr[3]); + if (rc) + continue; + + seq_printf(s, "%#x\t", pcifunc); + for (i = 0; i < ARRAY_SIZE(ptr); i++) { + if (ptr[i] != USHRT_MAX) + seq_printf(s, "%u\t", ptr[i]); + else + seq_puts(s, "\t"); + } + seq_puts(s, "\n"); + } + + return 0; +} +DEFINE_SHOW_ATTRIBUTE(npc_mcam_default); + +static int npc_vidx2idx_map_show(struct seq_file *s, void *unused) +{ + struct npc_priv_t *npc_priv; + unsigned long index, start; + struct xarray *xa; + void *map; + + npc_priv = s->private; + start = npc_priv->bank_depth * 2; + xa = &npc_priv->xa_vidx2idx_map; + + seq_puts(s, "\nvidx\tmcam_idx\n"); + + xa_for_each_start(xa, index, map, start) + seq_printf(s, "%lu\t%lu\n", index, xa_to_value(map)); + return 0; +} +DEFINE_SHOW_ATTRIBUTE(npc_vidx2idx_map); + +static int npc_idx2vidx_map_show(struct seq_file *s, void *unused) +{ + struct npc_priv_t *npc_priv; + unsigned long index; + struct xarray *xa; + void *map; + + npc_priv = s->private; + xa = &npc_priv->xa_idx2vidx_map; + + seq_puts(s, "\nmidx\tvidx\n"); + + xa_for_each(xa, index, map) + seq_printf(s, "%lu\t%lu\n", index, xa_to_value(map)); + return 0; +} +DEFINE_SHOW_ATTRIBUTE(npc_idx2vidx_map); + +static int npc_defrag_show(struct seq_file *s, void *unused) +{ + struct npc_defrag_show_node *node; + struct npc_priv_t *npc_priv; + u16 sbd, bdm; + + npc_priv = s->private; + bdm = npc_priv->bank_depth - 1; + sbd = npc_priv->subbank_depth; + + seq_puts(s, "\nold(sb) -> new(sb)\t\tvidx\n"); + + mutex_lock(&npc_priv->lock); + list_for_each_entry(node, &npc_priv->defrag_lh, list) + seq_printf(s, "%u(%u)\t%u(%u)\t%u\n", node->old_midx, + (node->old_midx & bdm) / sbd, + node->new_midx, + (node->new_midx & bdm) / sbd, + node->vidx); + mutex_unlock(&npc_priv->lock); + return 0; +} + +DEFINE_SHOW_ATTRIBUTE(npc_defrag); + +int npc_cn20k_debugfs_init(struct rvu *rvu) +{ + struct npc_priv_t *npc_priv = npc_priv_get(); + struct dentry *npc_dentry; + + npc_dentry = debugfs_create_file("mcam_layout", 0444, rvu->rvu_dbg.npc, + npc_priv, &npc_mcam_layout_fops); + + if (!npc_dentry) + return -EFAULT; + + npc_dentry = debugfs_create_file("mcam_default", 0444, rvu->rvu_dbg.npc, + rvu, &npc_mcam_default_fops); + + if (!npc_dentry) + return -EFAULT; + + npc_dentry = debugfs_create_file("vidx2idx", 0444, rvu->rvu_dbg.npc, + npc_priv, &npc_vidx2idx_map_fops); + if (!npc_dentry) + return -EFAULT; + + npc_dentry = debugfs_create_file("idx2vidx", 0444, rvu->rvu_dbg.npc, + npc_priv, &npc_idx2vidx_map_fops); + if (!npc_dentry) + return -EFAULT; + + npc_dentry = debugfs_create_file("defrag", 0444, rvu->rvu_dbg.npc, + npc_priv, &npc_defrag_fops); + if (!npc_dentry) + return -EFAULT; + + return 0; +} + +void npc_cn20k_debugfs_deinit(struct rvu *rvu) +{ + debugfs_remove_recursive(rvu->rvu_dbg.npc); +} void print_nix_cn20k_sq_ctx(struct seq_file *m, struct nix_cn20k_sq_ctx_s *sq_ctx) diff --git a/drivers/net/ethernet/marvell/octeontx2/af/cn20k/debugfs.h b/drivers/net/ethernet/marvell/octeontx2/af/cn20k/debugfs.h index a2e3a2cd6edb..0c5f05883666 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/cn20k/debugfs.h +++ b/drivers/net/ethernet/marvell/octeontx2/af/cn20k/debugfs.h @@ -16,6 +16,9 @@ #include "struct.h" #include "../mbox.h" +int npc_cn20k_debugfs_init(struct rvu *rvu); +void npc_cn20k_debugfs_deinit(struct rvu *rvu); + void print_nix_cn20k_sq_ctx(struct seq_file *m, struct nix_cn20k_sq_ctx_s *sq_ctx); void print_nix_cn20k_cq_ctx(struct seq_file *m, diff --git a/drivers/net/ethernet/marvell/octeontx2/af/cn20k/mbox_init.c b/drivers/net/ethernet/marvell/octeontx2/af/cn20k/mbox_init.c index bd3aab7770dd..71401dec0d77 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/cn20k/mbox_init.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/cn20k/mbox_init.c @@ -397,6 +397,12 @@ int rvu_alloc_cint_qint_mem(struct rvu *rvu, struct rvu_pfvf *pfvf, if (is_rvu_otx2(rvu) || is_cn20k(rvu->pdev)) return 0; + /* sanity check */ + cfg = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_PFX_NIXX_CFG(0) | + (RVU_AFPF << 16)); + if (!cfg) + return 0; + ctx_cfg = rvu_read64(rvu, blkaddr, NIX_AF_CONST3); /* Alloc memory for CQINT's HW contexts */ cfg = rvu_read64(rvu, blkaddr, NIX_AF_CONST2); @@ -420,5 +426,16 @@ int rvu_alloc_cint_qint_mem(struct rvu *rvu, struct rvu_pfvf *pfvf, rvu_write64(rvu, blkaddr, NIX_AF_LFX_QINTS_BASE(nixlf), (u64)pfvf->nix_qints_ctx->iova); + rvu_write64(rvu, BLKADDR_NIX0, RVU_AF_BAR2_SEL, RVU_AF_BAR2_PFID); + rvu_write64(rvu, BLKADDR_NIX0, + AF_BAR2_ALIASX(0, NIX_GINT_INT_W1S), ALTAF_RDY); + /* wait for ack */ + err = rvu_poll_reg(rvu, BLKADDR_NIX0, + AF_BAR2_ALIASX(0, NIX_GINT_INT), ALTAF_RDY, true); + if (err) + rvu->altaf_ready = false; + else + rvu->altaf_ready = true; + return 0; } diff --git a/drivers/net/ethernet/marvell/octeontx2/af/cn20k/npc.c b/drivers/net/ethernet/marvell/octeontx2/af/cn20k/npc.c new file mode 100644 index 000000000000..7291fdb89b03 --- /dev/null +++ b/drivers/net/ethernet/marvell/octeontx2/af/cn20k/npc.c @@ -0,0 +1,4522 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Marvell RVU Admin Function driver + * + * Copyright (C) 2026 Marvell. + * + */ +#include <linux/xarray.h> +#include <linux/bitfield.h> + +#include "rvu.h" +#include "npc.h" +#include "npc_profile.h" +#include "rvu_npc_hash.h" +#include "rvu_npc.h" +#include "cn20k/npc.h" +#include "cn20k/reg.h" +#include "rvu_npc_fs.h" + +static struct npc_priv_t npc_priv = { + .num_banks = MAX_NUM_BANKS, +}; + +static const char *npc_kw_name[NPC_MCAM_KEY_MAX] = { + [NPC_MCAM_KEY_DYN] = "DYNAMIC", + [NPC_MCAM_KEY_X2] = "X2", + [NPC_MCAM_KEY_X4] = "X4", +}; + +static const char *npc_dft_rule_name[NPC_DFT_RULE_MAX_ID] = { + [NPC_DFT_RULE_PROMISC_ID] = "Promisc", + [NPC_DFT_RULE_MCAST_ID] = "Mcast", + [NPC_DFT_RULE_BCAST_ID] = "Bcast", + [NPC_DFT_RULE_UCAST_ID] = "Ucast", +}; + +#define KEX_EXTR_CFG(bytesm1, hdr_ofs, ena, key_ofs) \ + (((bytesm1) << 16) | ((hdr_ofs) << 8) | ((ena) << 7) | \ + ((key_ofs) & 0x3F)) + +#define NPC_DFT_RULE_ID_MK(pcifunc, id) \ + ((pcifunc) | FIELD_PREP(GENMASK_ULL(31, 16), id)) + +#define NPC_DFT_RULE_ID_2_PCIFUNC(rid) \ + FIELD_GET(GENMASK_ULL(15, 0), rid) + +#define NPC_DFT_RULE_ID_2_ID(rid) \ + FIELD_GET(GENMASK_ULL(31, 16), rid) + +#define NPC_DFT_RULE_PRIO 127 + +static const char cn20k_def_pfl_name[] = "default"; + +static struct npc_mcam_kex_extr npc_mkex_extr_default = { + .mkex_sign = MKEX_CN20K_SIGN, + .name = "default", + .kpu_version = NPC_KPU_PROFILE_VER, + .keyx_cfg = { + /* nibble: LA..LE (ltype only) + Error code + Channel */ + [NIX_INTF_RX] = ((u64)NPC_MCAM_KEY_DYN << 32) | + NPC_PARSE_NIBBLE_INTF_RX | + NPC_CN20K_PARSE_NIBBLE_ERRCODE, + + /* nibble: LA..LE (ltype only) */ + [NIX_INTF_TX] = ((u64)NPC_MCAM_KEY_X2 << 32) | + NPC_CN20K_PARSE_NIBBLE_INTF_TX, + }, + .intf_extr_lid = { + /* Default RX MCAM KEX profile */ + [NIX_INTF_RX] = { NPC_LID_LA, NPC_LID_LA, NPC_LID_LB, NPC_LID_LB, + NPC_LID_LC, NPC_LID_LC, NPC_LID_LD }, + [NIX_INTF_TX] = { NPC_LID_LA, NPC_LID_LA, NPC_LID_LB, NPC_LID_LB, + NPC_LID_LC, NPC_LID_LD }, + }, + .intf_extr_lt = { + /* Default RX MCAM KEX profile */ + [NIX_INTF_RX] = { + [0] = { + /* Layer A: Ethernet: */ + [NPC_LT_LA_ETHER] = + /* DMAC: 6 bytes, KW1[63:15] */ + KEX_EXTR_CFG(0x05, 0x0, 0x1, + NPC_KEXOF_DMAC + 1), + [NPC_LT_LA_CPT_HDR] = + /* DMAC: 6 bytes, KW1[63:15] */ + KEX_EXTR_CFG(0x05, 0x0, 0x1, + NPC_KEXOF_DMAC + 1), + }, + [1] = { + /* Layer A: Ethernet: */ + [NPC_LT_LA_ETHER] = + /* Ethertype: 2 bytes, KW0[63:48] */ + KEX_EXTR_CFG(0x01, 0xc, 0x1, 0x6), + [NPC_LT_LA_CPT_HDR] = + /* Ethertype: 2 bytes, KW0[63:48] */ + KEX_EXTR_CFG(0x01, 0xc, 0x1, 0x6), + }, + [2] = { + /* Layer B: Single VLAN (CTAG) */ + [NPC_LT_LB_CTAG] = + /* CTAG VLAN: 2 bytes, KW1[15:0] */ + KEX_EXTR_CFG(0x01, 0x2, 0x1, 0x8), + /* Layer B: Stacked VLAN (STAG|QinQ) */ + [NPC_LT_LB_STAG_QINQ] = + /* Outer VLAN: 2 bytes, KW1[15:0] */ + KEX_EXTR_CFG(0x01, 0x2, 0x1, 0x8), + [NPC_LT_LB_FDSA] = + /* SWITCH PORT: 1 byte, KW1[7:0] */ + KEX_EXTR_CFG(0x0, 0x1, 0x1, 0x8), + }, + [3] = { + [NPC_LT_LB_CTAG] = + /* Ethertype: 2 bytes, KW0[63:48] */ + KEX_EXTR_CFG(0x01, 0x4, 0x1, 0x6), + [NPC_LT_LB_STAG_QINQ] = + /* Ethertype: 2 bytes, KW0[63:48] */ + KEX_EXTR_CFG(0x01, 0x8, 0x1, 0x6), + [NPC_LT_LB_FDSA] = + /* Ethertype: 2 bytes, KW0[63:48] */ + KEX_EXTR_CFG(0x01, 0x4, 0x1, 0x6), + }, + [4] = { + /* Layer C: IPv4 */ + [NPC_LT_LC_IP] = + /* SIP+DIP: 8 bytes, KW3[7:0], KW2[63:8] */ + KEX_EXTR_CFG(0x07, 0xc, 0x1, 0x11), + /* Layer C: IPv6 */ + [NPC_LT_LC_IP6] = + /* Everything up to SADDR: 8 bytes, KW3[7:0], + * KW2[63:8] + */ + KEX_EXTR_CFG(0x07, 0x0, 0x1, 0x11), + }, + [5] = { + [NPC_LT_LC_IP] = + /* TOS: 1 byte, KW2[7:0] */ + KEX_EXTR_CFG(0x0, 0x1, 0x1, 0x10), + }, + [6] = { + /* Layer D:UDP */ + [NPC_LT_LD_UDP] = + /* SPORT+DPORT: 4 bytes, KW3[39:8] */ + KEX_EXTR_CFG(0x3, 0x0, 0x1, 0x19), + /* Layer D:TCP */ + [NPC_LT_LD_TCP] = + /* SPORT+DPORT: 4 bytes, KW3[39:8] */ + KEX_EXTR_CFG(0x3, 0x0, 0x1, 0x19), + }, + }, + /* Default TX MCAM KEX profile */ + [NIX_INTF_TX] = { + [0] = { + /* Layer A: NIX_INST_HDR_S + Ethernet */ + /* NIX appends 8 bytes of NIX_INST_HDR_S at the + * start of each TX packet supplied to NPC. + */ + [NPC_LT_LA_IH_NIX_ETHER] = + /* PF_FUNC: 2B , KW0 [47:32] */ + KEX_EXTR_CFG(0x01, 0x0, 0x1, 0x4), + /* Layer A: HiGig2: */ + [NPC_LT_LA_IH_NIX_HIGIG2_ETHER] = + /* PF_FUNC: 2B , KW0 [47:32] */ + KEX_EXTR_CFG(0x01, 0x0, 0x1, 0x4), + }, + [1] = { + [NPC_LT_LA_IH_NIX_ETHER] = + /* SQ_ID 3 bytes, KW1[63:16] */ + KEX_EXTR_CFG(0x02, 0x02, 0x1, 0xa), + [NPC_LT_LA_IH_NIX_HIGIG2_ETHER] = + /* VID: 2 bytes, KW1[31:16] */ + KEX_EXTR_CFG(0x01, 0x10, 0x1, 0xa), + }, + [2] = { + /* Layer B: Single VLAN (CTAG) */ + [NPC_LT_LB_CTAG] = + /* CTAG VLAN[2..3] KW0[63:48] */ + KEX_EXTR_CFG(0x01, 0x2, 0x1, 0x6), + /* Layer B: Stacked VLAN (STAG|QinQ) */ + [NPC_LT_LB_STAG_QINQ] = + /* Outer VLAN: 2 bytes, KW0[63:48] */ + KEX_EXTR_CFG(0x01, 0x2, 0x1, 0x6), + }, + [3] = { + [NPC_LT_LB_CTAG] = + /* CTAG VLAN[2..3] KW1[15:0] */ + KEX_EXTR_CFG(0x01, 0x4, 0x1, 0x8), + [NPC_LT_LB_STAG_QINQ] = + /* Outer VLAN: 2 Bytes, KW1[15:0] */ + KEX_EXTR_CFG(0x01, 0x8, 0x1, 0x8), + }, + [4] = { + /* Layer C: IPv4 */ + [NPC_LT_LC_IP] = + /* SIP+DIP: 8 bytes, KW2[63:0] */ + KEX_EXTR_CFG(0x07, 0xc, 0x1, 0x10), + /* Layer C: IPv6 */ + [NPC_LT_LC_IP6] = + /* Everything up to SADDR: 8 bytes, KW2[63:0] */ + KEX_EXTR_CFG(0x07, 0x0, 0x1, 0x10), + }, + [5] = { + /* Layer D:UDP */ + [NPC_LT_LD_UDP] = + /* SPORT+DPORT: 4 bytes, KW3[31:0] */ + KEX_EXTR_CFG(0x3, 0x0, 0x1, 0x18), + /* Layer D:TCP */ + [NPC_LT_LD_TCP] = + /* SPORT+DPORT: 4 bytes, KW3[31:0] */ + KEX_EXTR_CFG(0x3, 0x0, 0x1, 0x18), + }, + }, + }, +}; + +struct npc_mcam_kex_extr *npc_mkex_extr_default_get(void) +{ + return &npc_mkex_extr_default; +} + +static u16 npc_idx2vidx(u16 idx) +{ + unsigned long index; + void *map; + u16 vidx; + int val; + + vidx = idx; + index = idx; + + map = xa_load(&npc_priv.xa_idx2vidx_map, index); + if (!map) + goto done; + + val = xa_to_value(map); + if (val == -1) + goto done; + + vidx = val; + +done: + return vidx; +} + +static bool npc_is_vidx(u16 vidx) +{ + return vidx >= npc_priv.bank_depth * 2; +} + +static u16 npc_vidx2idx(u16 vidx) +{ + unsigned long sentinel = (unsigned long)-1; + unsigned long index; + void *map; + int val; + u16 idx; + + idx = vidx; + index = vidx; + + map = xa_load(&npc_priv.xa_vidx2idx_map, index); + if (!map) + goto done; + + val = xa_to_value(map); + if (val == sentinel) + goto done; + + idx = val; + +done: + return idx; +} + +u16 npc_cn20k_vidx2idx(u16 idx) +{ + if (!npc_priv.init_done) + return idx; + + if (!npc_is_vidx(idx)) + return idx; + + return npc_vidx2idx(idx); +} + +u16 npc_cn20k_idx2vidx(u16 idx) +{ + if (!npc_priv.init_done) + return idx; + + if (npc_is_vidx(idx)) + return idx; + + return npc_idx2vidx(idx); +} + +static int npc_vidx_maps_del_entry(struct rvu *rvu, u16 vidx, u16 *old_midx) +{ + u16 mcam_idx; + void *map; + + if (!npc_is_vidx(vidx)) { + dev_err(rvu->dev, + "%s: vidx(%u) does not map to proper mcam idx\n", + __func__, vidx); + return -ESRCH; + } + + mcam_idx = npc_vidx2idx(vidx); + + map = xa_erase(&npc_priv.xa_vidx2idx_map, vidx); + if (!map) { + dev_err(rvu->dev, + "%s: vidx(%u) does not map to proper mcam idx\n", + __func__, vidx); + return -ESRCH; + } + + map = xa_erase(&npc_priv.xa_idx2vidx_map, mcam_idx); + if (!map) { + dev_err(rvu->dev, + "%s: vidx(%u) is not valid\n", + __func__, vidx); + return -ESRCH; + } + + if (old_midx) + *old_midx = mcam_idx; + + return 0; +} + +static int npc_vidx_maps_modify(struct rvu *rvu, u16 vidx, u16 new_midx) +{ + u16 old_midx; + void *map; + int rc; + + if (!npc_is_vidx(vidx)) { + dev_err(rvu->dev, + "%s: vidx(%u) does not map to proper mcam idx\n", + __func__, vidx); + return -ESRCH; + } + + map = xa_erase(&npc_priv.xa_vidx2idx_map, vidx); + if (!map) { + dev_err(rvu->dev, + "%s: vidx(%u) could not be deleted from vidx2idx map\n", + __func__, vidx); + return -ESRCH; + } + + old_midx = xa_to_value(map); + + rc = xa_insert(&npc_priv.xa_vidx2idx_map, vidx, + xa_mk_value(new_midx), GFP_KERNEL); + if (rc) { + dev_err(rvu->dev, + "%s: vidx(%u) cannot be added to vidx2idx map\n", + __func__, vidx); + goto fail1; + } + + map = xa_erase(&npc_priv.xa_idx2vidx_map, old_midx); + if (!map) { + dev_err(rvu->dev, + "%s: old_midx(%u, vidx(%u)) cannot be added to idx2vidx map\n", + __func__, old_midx, vidx); + rc = -ESRCH; + goto fail2; + } + + rc = xa_insert(&npc_priv.xa_idx2vidx_map, new_midx, + xa_mk_value(vidx), GFP_KERNEL); + if (rc) { + dev_err(rvu->dev, + "%s: new_midx(%u, vidx(%u)) cannot be added to idx2vidx map\n", + __func__, new_midx, vidx); + goto fail3; + } + + return 0; + +fail3: + /* Restore vidx at old_midx location */ + if (xa_insert(&npc_priv.xa_idx2vidx_map, old_midx, + xa_mk_value(vidx), GFP_KERNEL)) + dev_err(rvu->dev, + "%s: Error to roll back idx2vidx old_midx=%u vidx=%u\n", + __func__, old_midx, vidx); +fail2: + /* Erase new_midx inserted at vidx */ + if (!xa_erase(&npc_priv.xa_vidx2idx_map, vidx)) + dev_err(rvu->dev, + "%s: Failed to roll back vidx2idx vidx=%u\n", + __func__, vidx); + +fail1: + /* Restore old_midx at vidx location */ + if (xa_insert(&npc_priv.xa_vidx2idx_map, vidx, + xa_mk_value(old_midx), GFP_KERNEL)) + dev_err(rvu->dev, + "%s: Failed to roll back vidx2idx to old_midx=%u, vidx=%u\n", + __func__, old_midx, vidx); + + return rc; +} + +static int npc_vidx_maps_add_entry(struct rvu *rvu, u16 mcam_idx, int pcifunc, + u16 *vidx) +{ + int rc, max, min; + u32 id; + + /* Virtual index start from maximum mcam index + 1 */ + max = npc_priv.bank_depth * 2 * 2 - 1; + min = npc_priv.bank_depth * 2; + + rc = xa_alloc(&npc_priv.xa_vidx2idx_map, &id, + xa_mk_value(mcam_idx), + XA_LIMIT(min, max), GFP_KERNEL); + if (rc) { + dev_err(rvu->dev, + "%s: Failed to add to vidx2idx map (%u)\n", + __func__, mcam_idx); + goto fail1; + } + + rc = xa_insert(&npc_priv.xa_idx2vidx_map, mcam_idx, + xa_mk_value(id), GFP_KERNEL); + if (rc) { + dev_err(rvu->dev, + "%s: Failed to add to idx2vidx map (%u)\n", + __func__, mcam_idx); + goto fail2; + } + + if (vidx) + *vidx = id; + + return 0; + +fail2: + xa_erase(&npc_priv.xa_vidx2idx_map, id); +fail1: + return rc; +} + +static void npc_config_kpmcam(struct rvu *rvu, int blkaddr, + const struct npc_kpu_profile_cam *kpucam, + int kpm, int entry) +{ + struct npc_kpu_cam cam0 = {0}; + struct npc_kpu_cam cam1 = {0}; + + cam1.state = kpucam->state & kpucam->state_mask; + cam1.dp0_data = kpucam->dp0 & kpucam->dp0_mask; + cam1.dp1_data = kpucam->dp1 & kpucam->dp1_mask; + cam1.dp2_data = kpucam->dp2 & kpucam->dp2_mask; + + cam0.state = ~kpucam->state & kpucam->state_mask; + cam0.dp0_data = ~kpucam->dp0 & kpucam->dp0_mask; + cam0.dp1_data = ~kpucam->dp1 & kpucam->dp1_mask; + cam0.dp2_data = ~kpucam->dp2 & kpucam->dp2_mask; + + rvu_write64(rvu, blkaddr, + NPC_AF_KPMX_ENTRYX_CAMX(kpm, entry, 0), *(u64 *)&cam0); + rvu_write64(rvu, blkaddr, + NPC_AF_KPMX_ENTRYX_CAMX(kpm, entry, 1), *(u64 *)&cam1); +} + +static void +npc_config_kpmaction(struct rvu *rvu, int blkaddr, + const struct npc_kpu_profile_action *kpuaction, + int kpm, int entry, bool pkind) +{ + struct npc_kpm_action0 action0 = {0}; + struct npc_kpu_action1 action1 = {0}; + u64 reg; + + action1.errlev = kpuaction->errlev; + action1.errcode = kpuaction->errcode; + action1.dp0_offset = kpuaction->dp0_offset; + action1.dp1_offset = kpuaction->dp1_offset; + action1.dp2_offset = kpuaction->dp2_offset; + + if (pkind) + reg = NPC_AF_PKINDX_ACTION1(entry); + else + reg = NPC_AF_KPMX_ENTRYX_ACTION1(kpm, entry); + + rvu_write64(rvu, blkaddr, reg, *(u64 *)&action1); + + action0.byp_count = kpuaction->bypass_count & 0x7; + action0.capture_ena = kpuaction->cap_ena & 1; + action0.parse_done = kpuaction->parse_done & 1; + action0.next_state = kpuaction->next_state & 0xf; + action0.capture_lid = kpuaction->lid & 0x7; + + /* Parser functionality will work correctly even though + * upper flag bits are silently discarded + */ + action0.capture_ltype = kpuaction->ltype & 0xf; + action0.capture_flags = kpuaction->flags & 0xf; + action0.ptr_advance = kpuaction->ptr_advance; + + action0.var_len_offset = kpuaction->offset; + action0.var_len_mask = kpuaction->mask; + action0.var_len_right = kpuaction->right & 1; + action0.var_len_shift = kpuaction->shift & 1; + + if (pkind) + reg = NPC_AF_PKINDX_ACTION0(entry); + else + reg = NPC_AF_KPMX_ENTRYX_ACTION0(kpm, entry); + + rvu_write64(rvu, blkaddr, reg, *(u64 *)&action0); +} + +static void +npc_program_single_kpm_profile(struct rvu *rvu, int blkaddr, + int kpm, int start_entry, + const struct npc_kpu_profile *profile) +{ + int entry, num_entries, max_entries; + u64 idx; + + if (profile->cam_entries != profile->action_entries) { + dev_err(rvu->dev, + "kpm%d: CAM and action entries [%d != %d] not equal\n", + kpm, profile->cam_entries, profile->action_entries); + + WARN(1, "Fatal error\n"); + return; + } + + max_entries = rvu->hw->npc_kpu_entries / 2; + entry = start_entry; + /* Program CAM match entries for previous kpm extracted data */ + num_entries = min_t(int, profile->cam_entries, max_entries); + for (idx = 0; entry < num_entries + start_entry; entry++, idx++) + npc_config_kpmcam(rvu, blkaddr, &profile->cam[idx], + kpm, entry); + + entry = start_entry; + /* Program this kpm's actions */ + num_entries = min_t(int, profile->action_entries, max_entries); + for (idx = 0; entry < num_entries + start_entry; entry++, idx++) + npc_config_kpmaction(rvu, blkaddr, &profile->action[idx], + kpm, entry, false); +} + +static void +npc_enable_kpm_entry(struct rvu *rvu, int blkaddr, int kpm, int num_entries) +{ + u64 entry_mask; + + entry_mask = npc_enable_mask(num_entries); + /* Disable first KPU_CN20K_MAX_CST_ENT entries for built-in profile */ + if (!rvu->kpu.custom) + entry_mask |= GENMASK_ULL(KPU_CN20K_MAX_CST_ENT - 1, 0); + rvu_write64(rvu, blkaddr, + NPC_AF_KPMX_ENTRY_DISX(kpm, 0), entry_mask); + if (num_entries <= 64) { + /* Disable all the entries in W1, W2 and W3 */ + rvu_write64(rvu, blkaddr, + NPC_AF_KPMX_ENTRY_DISX(kpm, 1), + npc_enable_mask(0)); + rvu_write64(rvu, blkaddr, + NPC_AF_KPMX_ENTRY_DISX(kpm, 2), + npc_enable_mask(0)); + rvu_write64(rvu, blkaddr, + NPC_AF_KPMX_ENTRY_DISX(kpm, 3), + npc_enable_mask(0)); + return; + } + + num_entries = num_entries - 64; + entry_mask = npc_enable_mask(num_entries); + rvu_write64(rvu, blkaddr, + NPC_AF_KPMX_ENTRY_DISX(kpm, 1), entry_mask); + if (num_entries <= 64) { + /* Disable all the entries in W2 and W3 */ + rvu_write64(rvu, blkaddr, + NPC_AF_KPMX_ENTRY_DISX(kpm, 2), + npc_enable_mask(0)); + rvu_write64(rvu, blkaddr, + NPC_AF_KPMX_ENTRY_DISX(kpm, 3), + npc_enable_mask(0)); + return; + } + + num_entries = num_entries - 64; + entry_mask = npc_enable_mask(num_entries); + rvu_write64(rvu, blkaddr, + NPC_AF_KPMX_ENTRY_DISX(kpm, 2), entry_mask); + if (num_entries <= 64) { + /* Disable all the entries in W3 */ + rvu_write64(rvu, blkaddr, + NPC_AF_KPMX_ENTRY_DISX(kpm, 3), + npc_enable_mask(0)); + return; + } + + num_entries = num_entries - 64; + entry_mask = npc_enable_mask(num_entries); + rvu_write64(rvu, blkaddr, + NPC_AF_KPMX_ENTRY_DISX(kpm, 3), entry_mask); +} + +#define KPU_OFFSET 8 +static void npc_program_kpm_profile(struct rvu *rvu, int blkaddr, int num_kpms) +{ + const struct npc_kpu_profile *profile1, *profile2; + int idx, total_cam_entries; + + for (idx = 0; idx < num_kpms; idx++) { + profile1 = &rvu->kpu.kpu[idx]; + npc_program_single_kpm_profile(rvu, blkaddr, idx, 0, profile1); + profile2 = &rvu->kpu.kpu[idx + KPU_OFFSET]; + npc_program_single_kpm_profile(rvu, blkaddr, idx, + profile1->cam_entries, + profile2); + total_cam_entries = profile1->cam_entries + + profile2->cam_entries; + npc_enable_kpm_entry(rvu, blkaddr, idx, total_cam_entries); + rvu_write64(rvu, blkaddr, NPC_AF_KPMX_PASS2_OFFSET(idx), + profile1->cam_entries); + /* Enable the KPUs associated with this KPM */ + rvu_write64(rvu, blkaddr, NPC_AF_KPUX_CFG(idx), 0x01); + rvu_write64(rvu, blkaddr, NPC_AF_KPUX_CFG(idx + KPU_OFFSET), + 0x01); + } +} + +void npc_cn20k_parser_profile_init(struct rvu *rvu, int blkaddr) +{ + struct rvu_hwinfo *hw = rvu->hw; + int num_pkinds, idx; + + /* Disable all KPMs and their entries */ + for (idx = 0; idx < hw->npc_kpms; idx++) { + rvu_write64(rvu, blkaddr, + NPC_AF_KPMX_ENTRY_DISX(idx, 0), ~0ULL); + rvu_write64(rvu, blkaddr, + NPC_AF_KPMX_ENTRY_DISX(idx, 1), ~0ULL); + rvu_write64(rvu, blkaddr, + NPC_AF_KPMX_ENTRY_DISX(idx, 2), ~0ULL); + rvu_write64(rvu, blkaddr, + NPC_AF_KPMX_ENTRY_DISX(idx, 3), ~0ULL); + } + + for (idx = 0; idx < hw->npc_kpus; idx++) + rvu_write64(rvu, blkaddr, NPC_AF_KPUX_CFG(idx), 0x00); + + /* Load and customize KPU profile. */ + npc_load_kpu_profile(rvu); + + /* Configure KPU and KPM mapping for second pass */ + rvu_write64(rvu, blkaddr, NPC_AF_KPM_PASS2_CFG, 0x76543210); + + /* First program IKPU profile i.e PKIND configs. + * Check HW max count to avoid configuring junk or + * writing to unsupported CSR addresses. + */ + num_pkinds = rvu->kpu.pkinds; + num_pkinds = min_t(int, hw->npc_pkinds, num_pkinds); + + for (idx = 0; idx < num_pkinds; idx++) + npc_config_kpmaction(rvu, blkaddr, &rvu->kpu.ikpu[idx], + 0, idx, true); + + /* Program KPM CAM and Action profiles */ + npc_program_kpm_profile(rvu, blkaddr, hw->npc_kpms); +} + +struct npc_priv_t *npc_priv_get(void) +{ + return &npc_priv; +} + +static void npc_program_mkex_rx(struct rvu *rvu, int blkaddr, + struct npc_mcam_kex_extr *mkex_extr, + u8 intf) +{ + u8 num_extr = rvu->hw->npc_kex_extr; + int extr, lt; + u64 val; + + if (is_npc_intf_tx(intf)) + return; + + rvu_write64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(intf), + mkex_extr->keyx_cfg[NIX_INTF_RX]); + + /* Program EXTRACTOR */ + for (extr = 0; extr < num_extr; extr++) + rvu_write64(rvu, blkaddr, + NPC_AF_INTFX_EXTRACTORX_CFG(intf, extr), + mkex_extr->intf_extr_lid[intf][extr]); + + /* Program EXTRACTOR_LTYPE */ + for (extr = 0; extr < num_extr; extr++) { + for (lt = 0; lt < NPC_MAX_LT; lt++) { + val = mkex_extr->intf_extr_lt[intf][extr][lt]; + CN20K_SET_EXTR_LT(intf, extr, lt, val); + } + } +} + +static void npc_program_mkex_tx(struct rvu *rvu, int blkaddr, + struct npc_mcam_kex_extr *mkex_extr, + u8 intf) +{ + u8 num_extr = rvu->hw->npc_kex_extr; + int extr, lt; + u64 val; + + if (is_npc_intf_rx(intf)) + return; + + rvu_write64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(intf), + mkex_extr->keyx_cfg[NIX_INTF_TX]); + + /* Program EXTRACTOR */ + for (extr = 0; extr < num_extr; extr++) + rvu_write64(rvu, blkaddr, + NPC_AF_INTFX_EXTRACTORX_CFG(intf, extr), + mkex_extr->intf_extr_lid[intf][extr]); + + /* Program EXTRACTOR_LTYPE */ + for (extr = 0; extr < num_extr; extr++) { + for (lt = 0; lt < NPC_MAX_LT; lt++) { + val = mkex_extr->intf_extr_lt[intf][extr][lt]; + CN20K_SET_EXTR_LT(intf, extr, lt, val); + } + } +} + +static void npc_program_mkex_profile(struct rvu *rvu, int blkaddr, + struct npc_mcam_kex_extr *mkex_extr) +{ + struct rvu_hwinfo *hw = rvu->hw; + u8 intf; + + for (intf = 0; intf < hw->npc_intfs; intf++) { + npc_program_mkex_rx(rvu, blkaddr, mkex_extr, intf); + npc_program_mkex_tx(rvu, blkaddr, mkex_extr, intf); + } + + /* Programme mkex hash profile */ + npc_program_mkex_hash(rvu, blkaddr); +} + +void npc_cn20k_load_mkex_profile(struct rvu *rvu, int blkaddr, + const char *mkex_profile) +{ + struct npc_mcam_kex_extr *mcam_kex_extr; + struct device *dev = &rvu->pdev->dev; + void __iomem *mkex_prfl_addr = NULL; + u64 prfl_sz; + int ret; + + /* If user not selected mkex profile */ + if (rvu->kpu_fwdata_sz || + !strncmp(mkex_profile, cn20k_def_pfl_name, MKEX_NAME_LEN)) + goto program_mkex_extr; + + /* Setting up the mapping for mkex profile image */ + ret = npc_fwdb_prfl_img_map(rvu, &mkex_prfl_addr, &prfl_sz); + if (ret < 0) + goto program_mkex_extr; + + mcam_kex_extr = (struct npc_mcam_kex_extr __force *)mkex_prfl_addr; + + while (((s64)prfl_sz > 0) && + (mcam_kex_extr->mkex_sign != MKEX_END_SIGN)) { + /* Compare with mkex mod_param name string */ + if (mcam_kex_extr->mkex_sign == MKEX_CN20K_SIGN && + !strncmp(mcam_kex_extr->name, mkex_profile, + MKEX_NAME_LEN)) { + rvu->kpu.mcam_kex_prfl.mkex_extr = mcam_kex_extr; + goto program_mkex_extr; + } + + mcam_kex_extr++; + prfl_sz -= sizeof(struct npc_mcam_kex_extr); + } + dev_warn(dev, "Failed to load requested profile: %s\n", mkex_profile); + rvu->kpu.mcam_kex_prfl.mkex_extr = npc_mkex_extr_default_get(); + +program_mkex_extr: + dev_info(rvu->dev, "Using %s mkex profile\n", + rvu->kpu.mcam_kex_prfl.mkex_extr->name); + /* Program selected mkex profile */ + npc_program_mkex_profile(rvu, blkaddr, + rvu->kpu.mcam_kex_prfl.mkex_extr); + if (mkex_prfl_addr) + iounmap(mkex_prfl_addr); +} + +void +npc_cn20k_enable_mcam_entry(struct rvu *rvu, int blkaddr, + int index, bool enable) +{ + struct npc_mcam *mcam = &rvu->hw->mcam; + int mcam_idx = index % mcam->banksize; + int bank = index / mcam->banksize; + u64 cfg, hw_prio; + u8 kw_type; + + npc_mcam_idx_2_key_type(rvu, index, &kw_type); + if (kw_type == NPC_MCAM_KEY_X2) { + cfg = rvu_read64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_CFG_EXT(mcam_idx, + bank)); + hw_prio = cfg & GENMASK_ULL(30, 24); + cfg = enable ? 1 : 0; + cfg |= hw_prio; + rvu_write64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_CFG_EXT(mcam_idx, bank), + cfg); + return; + } + + /* For NPC_CN20K_MCAM_KEY_X4 keys, both the banks + * need to be programmed with the same value. + */ + for (bank = 0; bank < mcam->banks_per_entry; bank++) { + cfg = rvu_read64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_CFG_EXT(mcam_idx, + bank)); + hw_prio = cfg & GENMASK_ULL(30, 24); + cfg = enable ? 1 : 0; + cfg |= hw_prio; + rvu_write64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_CFG_EXT(mcam_idx, bank), + cfg); + } +} + +void +npc_cn20k_clear_mcam_entry(struct rvu *rvu, int blkaddr, int bank, int index) +{ + rvu_write64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_CAMX_INTF_EXT(index, bank, 1), + 0); + rvu_write64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_CAMX_INTF_EXT(index, bank, 0), + 0); + + rvu_write64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W0_EXT(index, bank, 1), 0); + rvu_write64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W0_EXT(index, bank, 0), 0); + + rvu_write64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W1_EXT(index, bank, 1), 0); + rvu_write64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W1_EXT(index, bank, 0), 0); + + rvu_write64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W2_EXT(index, bank, 1), 0); + rvu_write64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W2_EXT(index, bank, 0), 0); + + rvu_write64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W3_EXT(index, bank, 1), 0); + rvu_write64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W3_EXT(index, bank, 0), 0); + + /* Clear corresponding stats register */ + rvu_write64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_STAT_EXT(index, bank), 0); +} + +static void npc_cn20k_get_keyword(struct cn20k_mcam_entry *entry, int idx, + u64 *cam0, u64 *cam1) +{ + u64 kw_mask; + + /* The two banks of every MCAM entry are used as a single double-wide + * entry that is compared with the search key as follows: + * + * NPC_AF_MCAME()_BANK(0)_CAM(0..1)_W0_EXT[MD] ->NPC_MCAM_KEY_X4_S[KW0] + * NPC_AF_MCAME()_BANK(0)_CAM(0..1)_W1_EXT[MD] ->NPC_MCAM_KEY_X4_S[KW1] + * NPC_AF_MCAME()_BANK(0)_CAM(0..1)_W2_EXT[MD] ->NPC_MCAM_KEY_X4_S[KW2] + * NPC_AF_MCAME()_BANK(0)_CAM(0..1)_W3_EXT[MD] ->NPC_MCAM_KEY_X4_S[KW3] + * NPC_AF_MCAME()_BANK(1)_CAM(0..1)_W0_EXT[MD] ->NPC_MCAM_KEY_X4_S[KW4] + * NPC_AF_MCAME()_BANK(1)_CAM(0..1)_W1_EXT[MD] ->NPC_MCAM_KEY_X4_S[KW5] + * NPC_AF_MCAME()_BANK(1)_CAM(0..1)_W2_EXT[MD] ->NPC_MCAM_KEY_X4_S[KW6] + * NPC_AF_MCAME()_BANK(1)_CAM(0..1)_W3_EXT[MD] ->NPC_MCAM_KEY_X4_S[KW7] + */ + *cam1 = entry->kw[idx]; + kw_mask = entry->kw_mask[idx]; + *cam1 &= kw_mask; + *cam0 = ~*cam1 & kw_mask; +} + +/*------------------------------------------------------------------------- + *Kex type| mcam | cam1 |cam0 | req_kwtype||<----- output > | + * in | | | | || | | + * profile| len | | | ||len | type | + *------------------------------------------------------------------------- + *X2 | 256 (X2) | 001b |110b | 0 ||X2 | X2 | + *------------------------------------------------------------------------| + *X4 | 256 (X2) | 000b |000b | 0 ||X2 | DYN | + *------------------------------------------------------------------------| + *X4 | 512 (X4) | 010b |101b | 0 ||X4 | X4 | + *------------------------------------------------------------------------| + *DYN | 256 (X2) | 000b |000b | 0 ||X2 | DYN | + *------------------------------------------------------------------------| + *DYN | 512 (X4) | 010b |101b | 0 ||X4 | X4 | + *------------------------------------------------------------------------| + *X4 | 256 (X2) | 000b |000b | X2 ||DYN | DYN | + *------------------------------------------------------------------------| + *DYNC | 256 (X2) | 000b |000b | X2 ||DYN | DYN | + *------------------------------------------------------------------------| + * X2 | 512 (X4) | xxxb |xxxb | X4 ||INVAL | INVAL | + *------------------------------------------------------------------------| + */ +static void npc_cn20k_config_kw_x2(struct rvu *rvu, struct npc_mcam *mcam, + int blkaddr, int index, u8 intf, + struct cn20k_mcam_entry *entry, + int bank, u8 kw_type, int kw, + u8 req_kw_type) +{ + u64 intf_ext = 0, intf_ext_mask = 0; + u8 tx_intf_mask = ~intf & 0x3; + u8 tx_intf = intf, kex_type; + u8 kw_type_mask = ~kw_type; + u64 cam0, cam1, kex_cfg; + + if (is_npc_intf_tx(intf)) { + /* Last bit must be set and rest don't care + * for TX interfaces + */ + tx_intf_mask = 0x1; + tx_intf = intf & tx_intf_mask; + tx_intf_mask = ~tx_intf & tx_intf_mask; + } + + kex_cfg = rvu_read64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(intf)); + kex_type = (kex_cfg & GENMASK_ULL(34, 32)) >> 32; + if ((kex_type == NPC_MCAM_KEY_DYN || kex_type == NPC_MCAM_KEY_X4) && + kw_type == NPC_MCAM_KEY_X2) { + kw_type = 0; + kw_type_mask = 0; + } + + /* Say, we need to write x2 keyword in an x4 subbank. + * req_kw_type will be x2, and kw_type will be x4. + * So in the case ignore kw bits in mcam. + */ + if (kw_type == NPC_MCAM_KEY_X4 && req_kw_type == NPC_MCAM_KEY_X2) { + kw_type = 0; + kw_type_mask = 0; + } + + intf_ext = ((u64)kw_type << 16) | tx_intf; + intf_ext_mask = (((u64)kw_type_mask << 16) & GENMASK_ULL(18, 16)) | + tx_intf_mask; + rvu_write64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_CAMX_INTF_EXT(index, bank, 1), + intf_ext); + rvu_write64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_CAMX_INTF_EXT(index, bank, 0), + intf_ext_mask); + + /* Set the match key */ + npc_cn20k_get_keyword(entry, kw, &cam0, &cam1); + rvu_write64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W0_EXT(index, bank, 1), + cam1); + rvu_write64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W0_EXT(index, bank, 0), + cam0); + + npc_cn20k_get_keyword(entry, kw + 1, &cam0, &cam1); + rvu_write64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W1_EXT(index, bank, 1), + cam1); + rvu_write64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W1_EXT(index, bank, 0), + cam0); + + npc_cn20k_get_keyword(entry, kw + 2, &cam0, &cam1); + rvu_write64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W2_EXT(index, bank, 1), + cam1); + rvu_write64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W2_EXT(index, bank, 0), + cam0); + + npc_cn20k_get_keyword(entry, kw + 3, &cam0, &cam1); + rvu_write64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W3_EXT(index, bank, 1), + cam1); + rvu_write64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W3_EXT(index, bank, 0), + cam0); +} + +static void npc_cn20k_config_kw_x4(struct rvu *rvu, struct npc_mcam *mcam, + int blkaddr, int index, u8 intf, + struct cn20k_mcam_entry *entry, + u8 kw_type, u8 req_kw_type) +{ + int kw = 0, bank; + + for (bank = 0; bank < mcam->banks_per_entry; bank++, kw = kw + 4) + npc_cn20k_config_kw_x2(rvu, mcam, blkaddr, + index, intf, + entry, bank, kw_type, + kw, req_kw_type); +} + +static void +npc_cn20k_set_mcam_bank_cfg(struct rvu *rvu, int blkaddr, int mcam_idx, + int bank, u8 kw_type, bool enable, u8 hw_prio) +{ + struct npc_mcam *mcam = &rvu->hw->mcam; + u64 bank_cfg; + + bank_cfg = (u64)hw_prio << 24; + if (enable) + bank_cfg |= 0x1; + + if (kw_type == NPC_MCAM_KEY_X2) { + rvu_write64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_CFG_EXT(mcam_idx, bank), + bank_cfg); + return; + } + + /* For NPC_MCAM_KEY_X4 keys, both the banks + * need to be programmed with the same value. + */ + for (bank = 0; bank < mcam->banks_per_entry; bank++) { + rvu_write64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_CFG_EXT(mcam_idx, bank), + bank_cfg); + } +} + +void npc_cn20k_config_mcam_entry(struct rvu *rvu, int blkaddr, int index, + u8 intf, struct cn20k_mcam_entry *entry, + bool enable, u8 hw_prio, u8 req_kw_type) +{ + struct npc_mcam *mcam = &rvu->hw->mcam; + int mcam_idx = index % mcam->banksize; + int bank = index / mcam->banksize; + int kw = 0; + u8 kw_type; + + /* Disable before mcam entry update */ + npc_cn20k_enable_mcam_entry(rvu, blkaddr, index, false); + + npc_mcam_idx_2_key_type(rvu, index, &kw_type); + /* CAM1 takes the comparison value and + * CAM0 specifies match for a bit in key being '0' or '1' or 'dontcare'. + * CAM1<n> = 0 & CAM0<n> = 1 => match if key<n> = 0 + * CAM1<n> = 1 & CAM0<n> = 0 => match if key<n> = 1 + * CAM1<n> = 0 & CAM0<n> = 0 => always match i.e dontcare. + */ + if (kw_type == NPC_MCAM_KEY_X2) { + /* Clear mcam entry to avoid writes being suppressed by NPC */ + npc_cn20k_clear_mcam_entry(rvu, blkaddr, bank, mcam_idx); + npc_cn20k_config_kw_x2(rvu, mcam, blkaddr, + mcam_idx, intf, entry, + bank, kw_type, kw, req_kw_type); + /* Set 'action' */ + rvu_write64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(mcam_idx, + bank, 0), + entry->action); + + /* Set 'action2' for inline receive */ + rvu_write64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(mcam_idx, + bank, 2), + entry->action2); + + /* Set TAG 'action' */ + rvu_write64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(mcam_idx, + bank, 1), + entry->vtag_action); + goto set_cfg; + } + + /* Clear mcam entry to avoid writes being suppressed by NPC */ + npc_cn20k_clear_mcam_entry(rvu, blkaddr, 0, mcam_idx); + npc_cn20k_clear_mcam_entry(rvu, blkaddr, 1, mcam_idx); + + npc_cn20k_config_kw_x4(rvu, mcam, blkaddr, + mcam_idx, intf, entry, + kw_type, req_kw_type); + for (bank = 0; bank < mcam->banks_per_entry; bank++) { + /* Set 'action' */ + rvu_write64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(mcam_idx, + bank, 0), + entry->action); + + /* Set TAG 'action' */ + rvu_write64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(mcam_idx, + bank, 1), + entry->vtag_action); + + /* Set 'action2' for inline receive */ + rvu_write64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(mcam_idx, + bank, 2), + entry->action2); + } + +set_cfg: + /* TODO: */ + /* PF installing VF rule */ + npc_cn20k_set_mcam_bank_cfg(rvu, blkaddr, mcam_idx, bank, + kw_type, enable, hw_prio); +} + +void npc_cn20k_copy_mcam_entry(struct rvu *rvu, int blkaddr, u16 src, u16 dest) +{ + struct npc_mcam *mcam = &rvu->hw->mcam; + u64 cfg, sreg, dreg, soff, doff; + u8 src_kwtype, dest_kwtype; + int bank, i, sb, db; + int dbank, sbank; + + dbank = npc_get_bank(mcam, dest); + sbank = npc_get_bank(mcam, src); + npc_mcam_idx_2_key_type(rvu, src, &src_kwtype); + npc_mcam_idx_2_key_type(rvu, dest, &dest_kwtype); + if (src_kwtype != dest_kwtype) + return; + + src &= (mcam->banksize - 1); + dest &= (mcam->banksize - 1); + + /* Copy INTF's, W0's, W1's, W2's, W3s CAM0 and CAM1 configuration */ + for (bank = 0; bank < mcam->banks_per_entry; bank++) { + sb = sbank + bank; + sreg = NPC_AF_CN20K_MCAMEX_BANKX_CAMX_INTF_EXT(src, sb, 0); + db = dbank + bank; + dreg = NPC_AF_CN20K_MCAMEX_BANKX_CAMX_INTF_EXT(dest, db, 0); + for (i = 0; i < 10; i++) { + cfg = rvu_read64(rvu, blkaddr, sreg + (i * 8)); + rvu_write64(rvu, blkaddr, dreg + (i * 8), cfg); + } + + /* Copy action */ + for (i = 0; i < 3; i++) { + soff = NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(src, + sb, i); + cfg = rvu_read64(rvu, blkaddr, soff); + + doff = NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(dest, db, + i); + rvu_write64(rvu, blkaddr, doff, cfg); + } + + /* Copy bank configuration */ + cfg = rvu_read64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_CFG_EXT(src, sb)); + rvu_write64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_CFG_EXT(dest, db), cfg); + if (src_kwtype == NPC_MCAM_KEY_X2) + break; + } +} + +static void npc_cn20k_fill_entryword(struct cn20k_mcam_entry *entry, int idx, + u64 cam0, u64 cam1) +{ + entry->kw[idx] = cam1; + entry->kw_mask[idx] = cam1 ^ cam0; +} + +void npc_cn20k_read_mcam_entry(struct rvu *rvu, int blkaddr, u16 index, + struct cn20k_mcam_entry *entry, + u8 *intf, u8 *ena, u8 *hw_prio) +{ + struct npc_mcam *mcam = &rvu->hw->mcam; + u64 cam0, cam1, bank_cfg, cfg; + int kw = 0, bank; + u8 kw_type; + + npc_mcam_idx_2_key_type(rvu, index, &kw_type); + + bank = npc_get_bank(mcam, index); + index &= (mcam->banksize - 1); + + cfg = rvu_read64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_CAMX_INTF_EXT(index, + bank, 1)) & 3; + *intf = cfg; + + bank_cfg = rvu_read64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_CFG_EXT(index, bank)); + *ena = bank_cfg & 0x1; + *hw_prio = (bank_cfg & GENMASK_ULL(30, 24)) >> 24; + if (kw_type == NPC_MCAM_KEY_X2) { + cam1 = rvu_read64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W0_EXT(index, + bank, + 1)); + cam0 = rvu_read64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W0_EXT(index, + bank, + 0)); + npc_cn20k_fill_entryword(entry, kw, cam0, cam1); + + cam1 = rvu_read64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W1_EXT(index, + bank, + 1)); + cam0 = rvu_read64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W1_EXT(index, + bank, + 0)); + npc_cn20k_fill_entryword(entry, kw + 1, cam0, cam1); + + cam1 = rvu_read64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W2_EXT(index, + bank, + 1)); + cam0 = rvu_read64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W2_EXT(index, + bank, + 0)); + npc_cn20k_fill_entryword(entry, kw + 2, cam0, cam1); + + cam1 = rvu_read64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W3_EXT(index, + bank, + 1)); + cam0 = rvu_read64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W3_EXT(index, + bank, + 0)); + npc_cn20k_fill_entryword(entry, kw + 3, cam0, cam1); + goto read_action; + } + + for (bank = 0; bank < mcam->banks_per_entry; bank++, kw = kw + 4) { + cam1 = rvu_read64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W0_EXT(index, + bank, + 1)); + cam0 = rvu_read64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W0_EXT(index, + bank, + 0)); + npc_cn20k_fill_entryword(entry, kw, cam0, cam1); + + cam1 = rvu_read64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W1_EXT(index, + bank, + 1)); + cam0 = rvu_read64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W1_EXT(index, + bank, + 0)); + npc_cn20k_fill_entryword(entry, kw + 1, cam0, cam1); + + cam1 = rvu_read64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W2_EXT(index, + bank, + 1)); + cam0 = rvu_read64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W2_EXT(index, + bank, + 0)); + npc_cn20k_fill_entryword(entry, kw + 2, cam0, cam1); + + cam1 = rvu_read64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W3_EXT(index, + bank, + 1)); + cam0 = rvu_read64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W3_EXT(index, + bank, + 0)); + npc_cn20k_fill_entryword(entry, kw + 3, cam0, cam1); + } + +read_action: + /* 'action' is set to same value for both bank '0' and '1'. + * Hence, reading bank '0' should be enough. + */ + cfg = rvu_read64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(index, 0, 0)); + entry->action = cfg; + + cfg = rvu_read64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(index, 0, 1)); + entry->vtag_action = cfg; +} + +int rvu_mbox_handler_npc_cn20k_mcam_write_entry(struct rvu *rvu, + struct npc_cn20k_mcam_write_entry_req *req, + struct msg_rsp *rsp) +{ + struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, req->hdr.pcifunc); + struct npc_mcam *mcam = &rvu->hw->mcam; + u16 pcifunc = req->hdr.pcifunc; + int blkaddr, rc; + u8 nix_intf; + + req->entry = npc_cn20k_vidx2idx(req->entry); + + blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); + if (blkaddr < 0) + return NPC_MCAM_INVALID_REQ; + + mutex_lock(&mcam->lock); + rc = npc_mcam_verify_entry(mcam, pcifunc, req->entry); + if (rc) + goto exit; + + if (!is_npc_interface_valid(rvu, req->intf)) { + rc = NPC_MCAM_INVALID_REQ; + goto exit; + } + + if (is_npc_intf_tx(req->intf)) + nix_intf = pfvf->nix_tx_intf; + else + nix_intf = pfvf->nix_rx_intf; + + /* For AF installed rules, the nix_intf should be set to target NIX */ + if (is_pffunc_af(req->hdr.pcifunc)) + nix_intf = req->intf; + + npc_cn20k_config_mcam_entry(rvu, blkaddr, req->entry, nix_intf, + &req->entry_data, req->enable_entry, + req->hw_prio, req->req_kw_type); + + rc = 0; +exit: + mutex_unlock(&mcam->lock); + return rc; +} + +int rvu_mbox_handler_npc_cn20k_mcam_read_entry(struct rvu *rvu, + struct npc_mcam_read_entry_req *req, + struct npc_cn20k_mcam_read_entry_rsp *rsp) +{ + struct npc_mcam *mcam = &rvu->hw->mcam; + u16 pcifunc = req->hdr.pcifunc; + int blkaddr, rc; + + req->entry = npc_cn20k_vidx2idx(req->entry); + + blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); + if (blkaddr < 0) + return NPC_MCAM_INVALID_REQ; + + mutex_lock(&mcam->lock); + rc = npc_mcam_verify_entry(mcam, pcifunc, req->entry); + if (!rc) + npc_cn20k_read_mcam_entry(rvu, blkaddr, req->entry, + &rsp->entry_data, &rsp->intf, + &rsp->enable, &rsp->hw_prio); + + mutex_unlock(&mcam->lock); + return rc; +} + +int rvu_mbox_handler_npc_cn20k_mcam_alloc_and_write_entry(struct rvu *rvu, + struct npc_cn20k_mcam_alloc_and_write_entry_req *req, + struct npc_mcam_alloc_and_write_entry_rsp *rsp) +{ + struct rvu_pfvf *pfvf = rvu_get_pfvf(rvu, req->hdr.pcifunc); + struct npc_mcam_alloc_entry_req entry_req; + struct npc_mcam_alloc_entry_rsp entry_rsp; + struct npc_mcam *mcam = &rvu->hw->mcam; + u16 entry = NPC_MCAM_ENTRY_INVALID; + int blkaddr, rc; + u8 nix_intf; + + blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); + if (blkaddr < 0) + return NPC_MCAM_INVALID_REQ; + + if (!is_npc_interface_valid(rvu, req->intf)) + return NPC_MCAM_INVALID_REQ; + + /* Try to allocate a MCAM entry */ + entry_req.hdr.pcifunc = req->hdr.pcifunc; + entry_req.contig = true; + entry_req.ref_prio = req->ref_prio; + entry_req.ref_entry = req->ref_entry; + entry_req.count = 1; + entry_req.virt = req->virt; + + rc = rvu_mbox_handler_npc_mcam_alloc_entry(rvu, + &entry_req, &entry_rsp); + if (rc) + return rc; + + if (!entry_rsp.count) + return NPC_MCAM_ALLOC_FAILED; + + /* entry_req.count is 1, so single entry is allocated */ + entry = npc_cn20k_vidx2idx(entry_rsp.entry); + + mutex_lock(&mcam->lock); + + if (is_npc_intf_tx(req->intf)) + nix_intf = pfvf->nix_tx_intf; + else + nix_intf = pfvf->nix_rx_intf; + + npc_cn20k_config_mcam_entry(rvu, blkaddr, entry, nix_intf, + &req->entry_data, req->enable_entry, + req->hw_prio, req->req_kw_type); + + mutex_unlock(&mcam->lock); + + rsp->entry = entry_rsp.entry; + return 0; +} + +static int rvu_npc_get_base_steer_rule_type(struct rvu *rvu, u16 pcifunc) +{ + if (is_lbk_vf(rvu, pcifunc)) + return NIXLF_PROMISC_ENTRY; + + return NIXLF_UCAST_ENTRY; +} + +int rvu_mbox_handler_npc_cn20k_read_base_steer_rule(struct rvu *rvu, + struct msg_req *req, + struct npc_cn20k_mcam_read_base_rule_rsp *rsp) +{ + struct npc_mcam *mcam = &rvu->hw->mcam; + int index, blkaddr, nixlf, rc = 0; + u16 pcifunc = req->hdr.pcifunc; + u8 intf, enable, hw_prio; + struct rvu_pfvf *pfvf; + int rl_type; + + blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); + if (blkaddr < 0) + return NPC_MCAM_INVALID_REQ; + + /* Return the channel number in case of PF */ + if (!(pcifunc & RVU_PFVF_FUNC_MASK)) { + pfvf = rvu_get_pfvf(rvu, pcifunc); + rsp->entry.kw[0] = pfvf->rx_chan_base; + rsp->entry.kw_mask[0] = 0xFFFULL; + goto out; + } + + /* Find the pkt steering rule installed by PF to this VF */ + mutex_lock(&mcam->lock); + for (index = 0; index < mcam->bmap_entries; index++) { + if (mcam->entry2target_pffunc[index] == pcifunc) + goto read_entry; + } + + rc = nix_get_nixlf(rvu, pcifunc, &nixlf, NULL); + if (rc < 0) { + mutex_unlock(&mcam->lock); + goto out; + } + + rl_type = rvu_npc_get_base_steer_rule_type(rvu, pcifunc); + + /* Read the default ucast entry if there is no pkt steering rule */ + index = npc_get_nixlf_mcam_index(mcam, pcifunc, nixlf, rl_type); + if (index < 0) { + mutex_unlock(&mcam->lock); + goto out; + } + +read_entry: + /* Read the mcam entry */ + npc_cn20k_read_mcam_entry(rvu, blkaddr, index, + &rsp->entry, &intf, + &enable, &hw_prio); + mutex_unlock(&mcam->lock); +out: + return rc; +} + +static u8 npc_map2cn20k_flag(u8 flag) +{ + switch (flag) { + case NPC_F_LC_U_IP_FRAG: + return NPC_CN20K_F_LC_L_IP_FRAG; + + case NPC_F_LC_U_IP6_FRAG: + return NPC_CN20K_F_LC_L_IP6_FRAG; + + case NPC_F_LC_L_6TO4: + return NPC_CN20K_F_LC_L_6TO4; + + case NPC_F_LC_L_MPLS_IN_IP: + return NPC_CN20K_F_LC_U_MPLS_IN_IP; + + case NPC_F_LC_L_IP6_TUN_IP6: + return NPC_CN20K_F_LC_U_IP6_TUN_IP6; + + case NPC_F_LC_L_IP6_MPLS_IN_IP: + return NPC_CN20K_F_LC_U_IP6_MPLS_IN_IP; + + default: + break; + } + + WARN(1, "%s: Invalid flag=%u\n", __func__, flag); + return 0xff; +} + +void +npc_cn20k_update_action_entries_n_flags(struct rvu *rvu, + struct npc_kpu_profile_adapter *pfl) +{ + struct npc_kpu_profile_action *action; + int entries, ltype; + u8 flags, val; + + for (int i = 0; i < pfl->kpus; i++) { + action = pfl->kpu[i].action; + entries = pfl->kpu[i].action_entries; + + for (int j = 0; j < entries; j++) { + if (action[j].lid != NPC_LID_LC) + continue; + + ltype = action[j].ltype; + + if (ltype != NPC_LT_LC_IP && + ltype != NPC_LT_LC_IP6 && + ltype != NPC_LT_LC_IP_OPT && + ltype != NPC_LT_LC_IP6_EXT) + continue; + + flags = action[j].flags; + + switch (flags) { + case NPC_F_LC_U_IP_FRAG: + case NPC_F_LC_U_IP6_FRAG: + case NPC_F_LC_L_6TO4: + case NPC_F_LC_L_MPLS_IN_IP: + case NPC_F_LC_L_IP6_TUN_IP6: + case NPC_F_LC_L_IP6_MPLS_IN_IP: + val = npc_map2cn20k_flag(flags); + if (val == 0xFF) { + dev_err(rvu->dev, + "%s: Error to get flag value\n", + __func__); + return; + } + + action[j].flags = val; + break; + default: + break; + } + } + } +} + +int npc_cn20k_apply_custom_kpu(struct rvu *rvu, + struct npc_kpu_profile_adapter *profile) +{ + size_t hdr_sz = sizeof(struct npc_cn20k_kpu_profile_fwdata); + struct npc_cn20k_kpu_profile_fwdata *fw = rvu->kpu_fwdata; + struct npc_kpu_profile_action *action; + struct npc_kpu_profile_cam *cam; + struct npc_kpu_fwdata *fw_kpu; + size_t offset = 0; + u16 kpu, entry; + int entries; + + hdr_sz = sizeof(struct npc_cn20k_kpu_profile_fwdata); + + if (rvu->kpu_fwdata_sz < hdr_sz) { + dev_warn(rvu->dev, "Invalid KPU profile size\n"); + return -EINVAL; + } + + if (le64_to_cpu(fw->signature) != KPU_SIGN) { + dev_warn(rvu->dev, "Invalid KPU profile signature %llx\n", + fw->signature); + return -EINVAL; + } + + /* Verify if the using known profile structure */ + if (NPC_KPU_VER_MAJ(profile->version) > + NPC_KPU_VER_MAJ(NPC_KPU_PROFILE_VER)) { + dev_warn(rvu->dev, "Not supported Major version: %d > %d\n", + NPC_KPU_VER_MAJ(profile->version), + NPC_KPU_VER_MAJ(NPC_KPU_PROFILE_VER)); + return -EINVAL; + } + + /* Verify if profile is aligned with the required kernel changes */ + if (NPC_KPU_VER_MIN(profile->version) < + NPC_KPU_VER_MIN(NPC_KPU_PROFILE_VER)) { + dev_warn(rvu->dev, + "Invalid KPU profile version: %d.%d.%d expected version <= %d.%d.%d\n", + NPC_KPU_VER_MAJ(profile->version), + NPC_KPU_VER_MIN(profile->version), + NPC_KPU_VER_PATCH(profile->version), + NPC_KPU_VER_MAJ(NPC_KPU_PROFILE_VER), + NPC_KPU_VER_MIN(NPC_KPU_PROFILE_VER), + NPC_KPU_VER_PATCH(NPC_KPU_PROFILE_VER)); + return -EINVAL; + } + + /* Verify if profile fits the HW */ + if (fw->kpus > profile->kpus) { + dev_warn(rvu->dev, "Not enough KPUs: %d > %ld\n", fw->kpus, + profile->kpus); + return -EINVAL; + } + + profile->mcam_kex_prfl.mkex_extr = &fw->mkex; + if (profile->mcam_kex_prfl.mkex_extr->mkex_sign != MKEX_CN20K_SIGN) { + dev_warn(rvu->dev, "Invalid MKEX profile signature:%llx\n", + profile->mcam_kex_prfl.mkex_extr->mkex_sign); + return -EINVAL; + } + + profile->custom = 1; + profile->name = fw->name; + profile->version = le64_to_cpu(fw->version); + profile->lt_def = &fw->lt_def; + + for (kpu = 0; kpu < fw->kpus; kpu++) { + fw_kpu = (struct npc_kpu_fwdata *)(fw->data + offset); + if (fw_kpu->entries > KPU_CN20K_MAX_CST_ENT) + dev_warn(rvu->dev, + "Too many custom entries on KPU%d: %d > %d\n", + kpu, fw_kpu->entries, KPU_CN20K_MAX_CST_ENT); + entries = min(fw_kpu->entries, KPU_CN20K_MAX_CST_ENT); + cam = (struct npc_kpu_profile_cam *)fw_kpu->data; + offset += sizeof(*fw_kpu) + fw_kpu->entries * sizeof(*cam); + action = (struct npc_kpu_profile_action *)(fw->data + offset); + offset += fw_kpu->entries * sizeof(*action); + if (rvu->kpu_fwdata_sz < hdr_sz + offset) { + dev_warn(rvu->dev, + "Profile size mismatch on KPU%i parsing.\n", + kpu + 1); + return -EINVAL; + } + + for (entry = 0; entry < entries; entry++) { + profile->kpu[kpu].cam[entry] = cam[entry]; + profile->kpu[kpu].action[entry] = action[entry]; + } + } + npc_cn20k_update_action_entries_n_flags(rvu, profile); + + return 0; +} + +int npc_mcam_idx_2_key_type(struct rvu *rvu, u16 mcam_idx, u8 *key_type) +{ + struct npc_subbank *sb; + int bank_off, sb_id; + + /* mcam_idx should be less than (2 * bank depth) */ + if (mcam_idx >= npc_priv.bank_depth * 2) { + dev_err(rvu->dev, "%s: bad params\n", + __func__); + return -EINVAL; + } + + /* find mcam offset per bank */ + bank_off = mcam_idx & (npc_priv.bank_depth - 1); + + /* Find subbank id */ + sb_id = bank_off / npc_priv.subbank_depth; + + /* Check if subbank id is more than maximum + * number of subbanks available + */ + if (sb_id >= npc_priv.num_subbanks) { + dev_err(rvu->dev, "%s: invalid subbank %d\n", + __func__, sb_id); + return -EINVAL; + } + + sb = &npc_priv.sb[sb_id]; + + *key_type = sb->key_type; + + return 0; +} + +static int npc_subbank_idx_2_mcam_idx(struct rvu *rvu, struct npc_subbank *sb, + u16 sub_off, u16 *mcam_idx) +{ + int off, bot; + + /* for x4 section, maximum allowed subbank index = + * subsection depth - 1 + */ + if (sb->key_type == NPC_MCAM_KEY_X4 && + sub_off >= npc_priv.subbank_depth) { + dev_err(rvu->dev, + "%s: Failed to get mcam idx (x4) sb->idx=%u sub_off=%u", + __func__, sb->idx, sub_off); + return -EINVAL; + } + + /* for x2 section, maximum allowed subbank index = + * 2 * subsection depth - 1 + */ + if (sb->key_type == NPC_MCAM_KEY_X2 && + sub_off >= npc_priv.subbank_depth * 2) { + dev_err(rvu->dev, + "%s: Failed to get mcam idx (x2) sb->idx=%u sub_off=%u", + __func__, sb->idx, sub_off); + return -EINVAL; + } + + /* Find subbank offset from respective subbank (w.r.t bank) */ + off = sub_off & (npc_priv.subbank_depth - 1); + + /* if subsection idx is in bank1, add bank depth, + * which is part of sb->b1b + */ + bot = sub_off >= npc_priv.subbank_depth ? sb->b1b : sb->b0b; + + *mcam_idx = bot + off; + return 0; +} + +static int npc_mcam_idx_2_subbank_idx(struct rvu *rvu, u16 mcam_idx, + struct npc_subbank **sb, + int *sb_off) +{ + int bank_off, sb_id; + + /* mcam_idx should be less than (2 * bank depth) */ + if (mcam_idx >= npc_priv.bank_depth * 2) { + dev_err(rvu->dev, "%s: Invalid mcam idx %u\n", + __func__, mcam_idx); + return -EINVAL; + } + + /* find mcam offset per bank */ + bank_off = mcam_idx & (npc_priv.bank_depth - 1); + + /* Find subbank id */ + sb_id = bank_off / npc_priv.subbank_depth; + + /* Check if subbank id is more than maximum + * number of subbanks available + */ + if (sb_id >= npc_priv.num_subbanks) { + dev_err(rvu->dev, "%s: invalid subbank %d\n", + __func__, sb_id); + return -EINVAL; + } + + *sb = &npc_priv.sb[sb_id]; + + /* Subbank offset per bank */ + *sb_off = bank_off % npc_priv.subbank_depth; + + /* Index in a subbank should add subbank depth + * if it is in bank1 + */ + if (mcam_idx >= npc_priv.bank_depth) + *sb_off += npc_priv.subbank_depth; + + return 0; +} + +static int __npc_subbank_contig_alloc(struct rvu *rvu, + struct npc_subbank *sb, + int key_type, int sidx, + int eidx, int prio, + int count, int t, int b, + unsigned long *bmap, + u16 *save) +{ + int k, offset, delta = 0; + int cnt = 0, sbd; + + sbd = npc_priv.subbank_depth; + + if (sidx >= npc_priv.bank_depth) + delta = sbd; + + switch (prio) { + case NPC_MCAM_LOWER_PRIO: + case NPC_MCAM_ANY_PRIO: + /* Find an area of size 'count' from sidx to eidx */ + offset = bitmap_find_next_zero_area(bmap, sbd, sidx - b, + count, 0); + + if (offset >= sbd) { + dev_err(rvu->dev, + "%s: Could not find contiguous(%d) entries\n", + __func__, count); + return -EFAULT; + } + + dev_dbg(rvu->dev, + "%s: sidx=%d eidx=%d t=%d b=%d offset=%d count=%d delta=%d\n", + __func__, sidx, eidx, t, b, offset, + count, delta); + + for (cnt = 0; cnt < count; cnt++) + save[cnt] = offset + cnt + delta; + + break; + + case NPC_MCAM_HIGHER_PRIO: + /* Find an area of 'count' from eidx to sidx */ + for (k = eidx - b; cnt < count && k >= (sidx - b); k--) { + /* If an intermediate slot is not free, + * reset the counter (cnt) to zero as + * request is for contiguous. + */ + if (test_bit(k, bmap)) { + cnt = 0; + continue; + } + + save[cnt++] = k + delta; + } + break; + } + + /* Found 'count' number of free slots */ + if (cnt == count) + return 0; + + dev_dbg(rvu->dev, + "%s: Could not find contiguous(%d) entries in subbank=%u\n", + __func__, count, sb->idx); + return -EFAULT; +} + +static int __npc_subbank_non_contig_alloc(struct rvu *rvu, + struct npc_subbank *sb, + int key_type, int sidx, + int eidx, int prio, + int t, int b, + unsigned long *bmap, + int count, u16 *save, + bool max_alloc, int *alloc_cnt) +{ + unsigned long index; + int cnt = 0, delta; + int k, sbd; + + sbd = npc_priv.subbank_depth; + delta = sidx >= npc_priv.bank_depth ? sbd : 0; + + switch (prio) { + /* Find an area of size 'count' from sidx to eidx */ + case NPC_MCAM_LOWER_PRIO: + case NPC_MCAM_ANY_PRIO: + index = find_next_zero_bit(bmap, sbd, sidx - b); + if (index >= sbd) { + dev_err(rvu->dev, + "%s: Error happened to alloc %u, bitmap_weight=%u, sb->idx=%u\n", + __func__, count, + bitmap_weight(bmap, sbd), + sb->idx); + break; + } + + for (k = index; cnt < count && k <= (eidx - b); k++) { + /* Skip used slots */ + if (test_bit(k, bmap)) + continue; + + save[cnt++] = k + delta; + } + break; + + /* Find an area of 'count' from eidx to sidx */ + case NPC_MCAM_HIGHER_PRIO: + for (k = eidx - b; cnt < count && k >= (sidx - b); k--) { + /* Skip used slots */ + if (test_bit(k, bmap)) + continue; + + save[cnt++] = k + delta; + } + break; + } + + /* Update allocated 'cnt' to alloc_cnt */ + *alloc_cnt = cnt; + + /* Successfully allocated requested count slots */ + if (cnt == count) + return 0; + + /* Allocation successful for cnt < count */ + if (max_alloc && cnt > 0) + return 0; + + dev_dbg(rvu->dev, + "%s: Could not find non contiguous entries(%u) in subbank(%u) cnt=%d max_alloc=%d\n", + __func__, count, sb->idx, cnt, max_alloc); + + return -EFAULT; +} + +static void __npc_subbank_sboff_2_off(struct rvu *rvu, struct npc_subbank *sb, + int sb_off, unsigned long **bmap, + int *off) +{ + int sbd; + + sbd = npc_priv.subbank_depth; + + *off = sb_off & (sbd - 1); + *bmap = (sb_off >= sbd) ? sb->b1map : sb->b0map; +} + +/* set/clear bitmap */ +static bool __npc_subbank_mark_slot(struct rvu *rvu, + struct npc_subbank *sb, + int sb_off, bool set) +{ + unsigned long *bmap; + int off; + + /* if sb_off >= subbank.depth, then slots are in + * bank1 + */ + __npc_subbank_sboff_2_off(rvu, sb, sb_off, &bmap, &off); + + dev_dbg(rvu->dev, + "%s: Marking set=%d sb_off=%d sb->idx=%d off=%d\n", + __func__, set, sb_off, sb->idx, off); + + if (set) { + /* Slot is already used */ + if (test_bit(off, bmap)) + return false; + + sb->free_cnt--; + set_bit(off, bmap); + return true; + } + + /* Slot is already free */ + if (!test_bit(off, bmap)) + return false; + + sb->free_cnt++; + clear_bit(off, bmap); + return true; +} + +static int __npc_subbank_mark_free(struct rvu *rvu, struct npc_subbank *sb) +{ + int rc, blkaddr; + + sb->flags = NPC_SUBBANK_FLAG_FREE; + sb->key_type = 0; + + bitmap_clear(sb->b0map, 0, npc_priv.subbank_depth); + bitmap_clear(sb->b1map, 0, npc_priv.subbank_depth); + + if (!xa_erase(&npc_priv.xa_sb_used, sb->arr_idx)) { + dev_err(rvu->dev, + "%s: Error to delete from xa_sb_used array\n", + __func__); + return -EFAULT; + } + + rc = xa_insert(&npc_priv.xa_sb_free, sb->arr_idx, + xa_mk_value(sb->idx), GFP_KERNEL); + if (rc) { + rc = xa_insert(&npc_priv.xa_sb_used, sb->arr_idx, + xa_mk_value(sb->idx), GFP_KERNEL); + if (rc) + dev_err(rvu->dev, + "%s: Failed to roll back sb(%u) arr_idx=%d\n", + __func__, sb->idx, sb->arr_idx); + + dev_err(rvu->dev, + "%s: Error to add sb(%u) to xa_sb_free array at arr_idx=%d\n", + __func__, sb->idx, sb->arr_idx); + return rc; + } + + blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); + rvu_write64(rvu, blkaddr, + NPC_AF_MCAM_SECTIONX_CFG_EXT(sb->idx), + NPC_MCAM_KEY_X2); + + return rc; +} + +static int __npc_subbank_mark_used(struct rvu *rvu, struct npc_subbank *sb, + int key_type) +{ + int rc; + + sb->flags = NPC_SUBBANK_FLAG_USED; + sb->key_type = key_type; + if (key_type == NPC_MCAM_KEY_X4) + sb->free_cnt = npc_priv.subbank_depth; + else + sb->free_cnt = 2 * npc_priv.subbank_depth; + + bitmap_clear(sb->b0map, 0, npc_priv.subbank_depth); + bitmap_clear(sb->b1map, 0, npc_priv.subbank_depth); + + if (!xa_erase(&npc_priv.xa_sb_free, sb->arr_idx)) { + dev_err(rvu->dev, + "%s: Error to delete from xa_sb_free array\n", + __func__); + return -EFAULT; + } + + rc = xa_insert(&npc_priv.xa_sb_used, sb->arr_idx, + xa_mk_value(sb->idx), GFP_KERNEL); + if (rc) + dev_err(rvu->dev, + "%s: Error to add to xa_sb_used array\n", __func__); + + return rc; +} + +static bool __npc_subbank_free(struct rvu *rvu, struct npc_subbank *sb, + u16 sb_off) +{ + bool deleted = false; + unsigned long *bmap; + int rc, off; + + deleted = __npc_subbank_mark_slot(rvu, sb, sb_off, false); + if (!deleted) + goto done; + + __npc_subbank_sboff_2_off(rvu, sb, sb_off, &bmap, &off); + + /* Check whether we can mark whole subbank as free */ + if (sb->key_type == NPC_MCAM_KEY_X4) { + if (sb->free_cnt < npc_priv.subbank_depth) + goto done; + } else { + if (sb->free_cnt < 2 * npc_priv.subbank_depth) + goto done; + } + + /* All slots in subbank are unused. Mark the subbank as free + * and add to free pool + */ + rc = __npc_subbank_mark_free(rvu, sb); + if (rc) + dev_err(rvu->dev, "%s: Error to free subbank\n", __func__); + +done: + return deleted; +} + +static int +npc_subbank_free(struct rvu *rvu, struct npc_subbank *sb, u16 sb_off) +{ + bool deleted; + + mutex_lock(&sb->lock); + deleted = __npc_subbank_free(rvu, sb, sb_off); + mutex_unlock(&sb->lock); + + return deleted ? 0 : -EFAULT; +} + +static int __npc_subbank_alloc(struct rvu *rvu, struct npc_subbank *sb, + int key_type, int ref, int limit, int prio, + bool contig, int count, u16 *mcam_idx, + int idx_sz, bool max_alloc, int *alloc_cnt) +{ + int cnt, t, b, i, blkaddr; + bool new_sub_bank = false; + unsigned long *bmap; + u16 *save = NULL; + int sidx, eidx; + bool diffbank; + int bw, bfree; + int rc = 0; + bool ret; + + /* Check if enough space is there to return requested number of + * mcam indexes in case of contiguous allocation + */ + if (!max_alloc && count > idx_sz) { + dev_err(rvu->dev, + "%s: Less space, count=%d idx_sz=%d sb_id=%d\n", + __func__, count, idx_sz, sb->idx); + return -ENOSPC; + } + + /* Allocation on multiple subbank is not supported by this function. + * it means that ref and limit should be on same subbank. + * + * ref and limit values should be validated w.r.t prio as below. + * say ref = 100, limit = 200, + * if NPC_MCAM_LOWER_PRIO, allocate index 100 + * if NPC_MCAM_HIGHER_PRIO, below sanity test returns error. + * if NPC_MCAM_ANY_PRIO, allocate index 100 + * + * say ref = 200, limit = 100 + * if NPC_MCAM_LOWER_PRIO, below sanity test returns error. + * if NPC_MCAM_HIGHER_PRIO, allocate index 200 + * if NPC_MCAM_ANY_PRIO, allocate index 100 + * + * Please note that NPC_MCAM_ANY_PRIO does not have any restriction + * on "ref" and "limit" values. ie, ref > limit and limit > ref + * are valid cases. + */ + if ((prio == NPC_MCAM_LOWER_PRIO && ref > limit) || + (prio == NPC_MCAM_HIGHER_PRIO && ref < limit)) { + dev_err(rvu->dev, "%s: Wrong ref_enty(%d) or limit(%d)\n", + __func__, ref, limit); + return -EINVAL; + } + + /* x4 indexes are from 0 to bank size as it combines two x2 banks */ + if (key_type == NPC_MCAM_KEY_X4 && + (ref >= npc_priv.bank_depth || limit >= npc_priv.bank_depth)) { + dev_err(rvu->dev, + "%s: Wrong ref_enty(%d) or limit(%d) for x4\n", + __func__, ref, limit); + return -EINVAL; + } + + /* This function is called either bank0 or bank1 portion of a subbank. + * so ref and limit should be on same bank. + */ + diffbank = !!((ref & npc_priv.bank_depth) ^ + (limit & npc_priv.bank_depth)); + if (diffbank) { + dev_err(rvu->dev, + "%s: request ref and limit should be from same bank\n", + __func__); + return -EINVAL; + } + + sidx = min_t(int, limit, ref); + eidx = max_t(int, limit, ref); + + /* Find total number of slots available; both used and free */ + cnt = eidx - sidx + 1; + if (contig && cnt < count) { + dev_err(rvu->dev, + "%s: Wrong ref_enty(%d) or limit(%d) for count(%d)\n", + __func__, ref, limit, count); + return -EINVAL; + } + + /* If subbank is free, check if requested number of indexes is less than + * or equal to mcam entries available in the subbank if contig. + */ + if (sb->flags & NPC_SUBBANK_FLAG_FREE) { + if (contig && count > npc_priv.subbank_depth) { + dev_err(rvu->dev, "%s: Less number of entries\n", + __func__); + return -ENOSPC; + } + + new_sub_bank = true; + goto process; + } + + /* Flag should be set for all used subbanks */ + WARN_ONCE(!(sb->flags & NPC_SUBBANK_FLAG_USED), + "Used flag is not set(%#x)\n", sb->flags); + + /* If subbank key type does not match with requested key_type, + * return error + */ + if (sb->key_type != key_type) { + dev_dbg(rvu->dev, "%s: subbank key_type mismatch\n", __func__); + return -EINVAL; + } + +process: + /* if ref or limit >= npc_priv.bank_depth, index are in bank1. + * else bank0. + */ + if (ref >= npc_priv.bank_depth) { + bmap = sb->b1map; + t = sb->b1t; + b = sb->b1b; + } else { + bmap = sb->b0map; + t = sb->b0t; + b = sb->b0b; + } + + /* Calculate free slots */ + bw = bitmap_weight(bmap, npc_priv.subbank_depth); + bfree = npc_priv.subbank_depth - bw; + + if (!bfree) { + dev_dbg(rvu->dev, "%s: subbank is full\n", __func__); + return -ENOSPC; + } + + /* If request is for contiguous , then max we can allocate is + * equal to subbank_depth + */ + if (contig && bfree < count) { + dev_dbg(rvu->dev, "%s: no space for entry\n", __func__); + return -ENOSPC; + } + + /* 'save' array stores available indexes temporarily before + * marking it as allocated + */ + save = kcalloc(count, sizeof(u16), GFP_KERNEL); + if (!save) { + rc = -ENOMEM; + goto err1; + } + + if (contig) { + rc = __npc_subbank_contig_alloc(rvu, sb, key_type, + sidx, eidx, prio, + count, t, b, + bmap, save); + /* contiguous allocation success means that + * requested number of free slots got + * allocated + */ + if (!rc) + *alloc_cnt = count; + + } else { + rc = __npc_subbank_non_contig_alloc(rvu, sb, key_type, + sidx, eidx, prio, + t, b, bmap, + count, save, + max_alloc, alloc_cnt); + } + + if (rc) + goto err1; + + /* Mark new subbank bank as used */ + if (new_sub_bank) { + blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); + if (blkaddr < 0) { + dev_err(rvu->dev, + "%s: NPC block not implemented\n", __func__); + rc = -EFAULT; + goto err1; + } + + rc = __npc_subbank_mark_used(rvu, sb, key_type); + if (rc) { + dev_err(rvu->dev, + "%s: Error to mark subbank as used\n", + __func__); + goto err2; + } + + /* Configure section type to key_type */ + rvu_write64(rvu, blkaddr, + NPC_AF_MCAM_SECTIONX_CFG_EXT(sb->idx), + key_type); + } + + for (i = 0; i < *alloc_cnt; i++) { + rc = npc_subbank_idx_2_mcam_idx(rvu, sb, save[i], + &mcam_idx[i]); + if (rc) { + dev_err(rvu->dev, + "%s: Error to find mcam idx for %u\n", + __func__, save[i]); + /* TODO: handle err case gracefully */ + goto err3; + } + + /* Mark all slots as used */ + ret = __npc_subbank_mark_slot(rvu, sb, save[i], true); + if (!ret) { + dev_err(rvu->dev, "%s: Error to mark mcam_idx %u\n", + __func__, mcam_idx[i]); + rc = -EFAULT; + goto err3; + } + } + kfree(save); + return 0; + +err3: + for (int j = 0; j < i; j++) + __npc_subbank_mark_slot(rvu, sb, save[j], false); +err2: + if (new_sub_bank) + __npc_subbank_mark_free(rvu, sb); +err1: + kfree(save); + return rc; +} + +static int +npc_subbank_alloc(struct rvu *rvu, struct npc_subbank *sb, + int key_type, int ref, int limit, int prio, + bool contig, int count, u16 *mcam_idx, + int idx_sz, bool max_alloc, int *alloc_cnt) +{ + int rc; + + mutex_lock(&sb->lock); + rc = __npc_subbank_alloc(rvu, sb, key_type, ref, limit, prio, + contig, count, mcam_idx, idx_sz, + max_alloc, alloc_cnt); + mutex_unlock(&sb->lock); + + return rc; +} + +static int +npc_del_from_pf_maps(struct rvu *rvu, u16 mcam_idx) +{ + int pcifunc, idx; + void *map; + + map = xa_erase(&npc_priv.xa_idx2pf_map, mcam_idx); + if (!map) { + dev_err(rvu->dev, + "%s: failed to erase mcam_idx(%u) from xa_idx2pf map\n", + __func__, mcam_idx); + return -EFAULT; + } + + pcifunc = xa_to_value(map); + map = xa_load(&npc_priv.xa_pf_map, pcifunc); + if (!map) { + dev_err(rvu->dev, + "%s: failed to find entry for (%u) from xa_pf_map, mcam=%u\n", + __func__, pcifunc, mcam_idx); + return -ESRCH; + } + + idx = xa_to_value(map); + + map = xa_erase(&npc_priv.xa_pf2idx_map[idx], mcam_idx); + if (!map) { + dev_err(rvu->dev, + "%s: failed to erase mcam_idx(%u) from xa_pf2idx_map map\n", + __func__, mcam_idx); + return -EFAULT; + } + return 0; +} + +static int +npc_add_to_pf_maps(struct rvu *rvu, u16 mcam_idx, int pcifunc) +{ + int rc, idx; + void *map; + + dev_dbg(rvu->dev, + "%s: add2maps mcam_idx(%u) to xa_idx2pf map pcifunc=%#x\n", + __func__, mcam_idx, pcifunc); + + rc = xa_insert(&npc_priv.xa_idx2pf_map, mcam_idx, + xa_mk_value(pcifunc), GFP_KERNEL); + + if (rc) { + map = xa_load(&npc_priv.xa_idx2pf_map, mcam_idx); + dev_err(rvu->dev, + "%s: failed to insert mcam_idx(%u) to xa_idx2pf map, existing value=%lu\n", + __func__, mcam_idx, xa_to_value(map)); + return -EFAULT; + } + + map = xa_load(&npc_priv.xa_pf_map, pcifunc); + if (!map) { + dev_err(rvu->dev, + "%s: failed to find pf map entry for pcifunc=%#x, mcam=%u\n", + __func__, pcifunc, mcam_idx); + return -ESRCH; + } + + idx = xa_to_value(map); + + rc = xa_insert(&npc_priv.xa_pf2idx_map[idx], mcam_idx, + xa_mk_value(pcifunc), GFP_KERNEL); + + if (rc) { + map = xa_load(&npc_priv.xa_pf2idx_map[idx], mcam_idx); + xa_erase(&npc_priv.xa_idx2pf_map, mcam_idx); + dev_err(rvu->dev, + "%s: failed to insert mcam_idx(%u) to xa_pf2idx_map map, earlier value=%lu idx=%u\n", + __func__, mcam_idx, xa_to_value(map), idx); + + return -EFAULT; + } + + return 0; +} + +static bool +npc_subbank_suits(struct npc_subbank *sb, int key_type) +{ + mutex_lock(&sb->lock); + + if (!sb->key_type) { + mutex_unlock(&sb->lock); + return true; + } + + if (sb->key_type == key_type) { + mutex_unlock(&sb->lock); + return true; + } + + mutex_unlock(&sb->lock); + return false; +} + +#define SB_ALIGN_UP(val) (((val) + npc_priv.subbank_depth) & \ + ~((npc_priv.subbank_depth) - 1)) +#define SB_ALIGN_DOWN(val) ALIGN_DOWN((val), npc_priv.subbank_depth) + +static void npc_subbank_iter_down(struct rvu *rvu, + int ref, int limit, + int *cur_ref, int *cur_limit, + bool *start, bool *stop) +{ + int align; + + *stop = false; + + /* ALIGN_DOWN the limit to current subbank boundary bottom index */ + if (*start) { + *start = false; + *cur_ref = ref; + align = SB_ALIGN_DOWN(ref); + if (align < limit) { + *stop = true; + *cur_limit = limit; + return; + } + *cur_limit = align; + return; + } + + *cur_ref = *cur_limit - 1; + align = *cur_ref - npc_priv.subbank_depth + 1; + if (align <= limit) { + *stop = true; + *cur_limit = limit; + return; + } + + *cur_limit = align; +} + +static void npc_subbank_iter_up(struct rvu *rvu, + int ref, int limit, + int *cur_ref, int *cur_limit, + bool *start, bool *stop) +{ + int align; + + *stop = false; + + /* ALIGN_UP the limit to current subbank boundary top index */ + if (*start) { + *start = false; + *cur_ref = ref; + + /* Find next lower prio subbank's bottom index */ + align = SB_ALIGN_UP(ref); + + /* Crosses limit ? */ + if (align - 1 > limit) { + *stop = true; + *cur_limit = limit; + return; + } + + /* Current subbank's top index */ + *cur_limit = align - 1; + return; + } + + *cur_ref = *cur_limit + 1; + align = *cur_ref + npc_priv.subbank_depth - 1; + + if (align >= limit) { + *stop = true; + *cur_limit = limit; + return; + } + + *cur_limit = align; +} + +static int +npc_subbank_iter(struct rvu *rvu, int key_type, + int ref, int limit, int prio, + int *cur_ref, int *cur_limit, + bool *start, bool *stop) +{ + if (prio != NPC_MCAM_HIGHER_PRIO) + npc_subbank_iter_up(rvu, ref, limit, + cur_ref, cur_limit, + start, stop); + else + npc_subbank_iter_down(rvu, ref, limit, + cur_ref, cur_limit, + start, stop); + + /* limit and ref should < bank_depth for x4 */ + if (key_type == NPC_MCAM_KEY_X4) { + if (*cur_ref >= npc_priv.bank_depth) + return -EINVAL; + + if (*cur_limit >= npc_priv.bank_depth) + return -EINVAL; + } + /* limit and ref should < 2 * bank_depth, for x2 */ + if (*cur_ref >= 2 * npc_priv.bank_depth) + return -EINVAL; + + if (*cur_limit >= 2 * npc_priv.bank_depth) + return -EINVAL; + + return 0; +} + +static int npc_idx_free(struct rvu *rvu, u16 *mcam_idx, int count, + bool maps_del) +{ + struct npc_subbank *sb; + u16 vidx, midx; + int sb_off, i; + bool ret; + int rc; + + /* Check if we can dealloc indexes properly ? */ + for (i = 0; i < count; i++) { + rc = npc_mcam_idx_2_subbank_idx(rvu, npc_vidx2idx(mcam_idx[i]), + &sb, &sb_off); + if (rc) { + dev_err(rvu->dev, + "Failed to free mcam idx=%u\n", mcam_idx[i]); + return rc; + } + } + + for (i = 0; i < count; i++) { + if (npc_is_vidx(mcam_idx[i])) { + vidx = mcam_idx[i]; + midx = npc_vidx2idx(vidx); + } else { + midx = mcam_idx[i]; + vidx = npc_idx2vidx(midx); + } + + if (midx >= npc_priv.bank_depth * npc_priv.num_banks) { + dev_err(rvu->dev, + "%s: Invalid mcam_idx=%u cannot be deleted\n", + __func__, mcam_idx[i]); + return -EINVAL; + } + + rc = npc_mcam_idx_2_subbank_idx(rvu, midx, + &sb, &sb_off); + if (rc) { + dev_err(rvu->dev, + "%s: Failed to find subbank info for vidx=%u\n", + __func__, vidx); + return rc; + } + + ret = npc_subbank_free(rvu, sb, sb_off); + if (ret) { + dev_err(rvu->dev, + "%s: Failed to find subbank info for vidx=%u\n", + __func__, vidx); + return -EINVAL; + } + + if (!maps_del) + continue; + + rc = npc_del_from_pf_maps(rvu, midx); + if (rc) + return rc; + + /* If there is no vidx mapping; continue */ + if (vidx == midx) + continue; + + rc = npc_vidx_maps_del_entry(rvu, vidx, NULL); + if (rc) + return rc; + } + + return 0; +} + +static int npc_multi_subbank_ref_alloc(struct rvu *rvu, int key_type, + int ref, int limit, int prio, + bool contig, int count, + u16 *mcam_idx) +{ + struct npc_subbank *sb; + unsigned long *bmap; + int sb_off, off, rc; + int cnt = 0; + bool bitset; + + if (prio != NPC_MCAM_HIGHER_PRIO) { + while (ref <= limit) { + /* Calculate subbank and subbank index */ + rc = npc_mcam_idx_2_subbank_idx(rvu, ref, + &sb, &sb_off); + if (rc) + goto err; + + /* If subbank is not suitable for requested key type + * restart search from next subbank + */ + if (!npc_subbank_suits(sb, key_type)) { + ref = SB_ALIGN_UP(ref); + if (contig) { + rc = npc_idx_free(rvu, mcam_idx, + cnt, false); + if (rc) + return rc; + cnt = 0; + } + continue; + } + + mutex_lock(&sb->lock); + + /* If subbank is free; mark it as used */ + if (sb->flags & NPC_SUBBANK_FLAG_FREE) { + rc = __npc_subbank_mark_used(rvu, sb, + key_type); + if (rc) { + mutex_unlock(&sb->lock); + dev_err(rvu->dev, + "%s:Error to add to use array\n", + __func__); + goto err; + } + } + + /* Find correct bmap */ + __npc_subbank_sboff_2_off(rvu, sb, sb_off, &bmap, &off); + + /* if bit is already set, reset 'cnt' */ + bitset = test_bit(off, bmap); + if (bitset) { + mutex_unlock(&sb->lock); + if (contig) { + rc = npc_idx_free(rvu, mcam_idx, + cnt, false); + if (rc) + return rc; + cnt = 0; + } + + ref++; + continue; + } + + set_bit(off, bmap); + sb->free_cnt--; + mcam_idx[cnt++] = ref; + mutex_unlock(&sb->lock); + + if (cnt == count) + return 0; + ref++; + } + + /* Could not allocate request count slots */ + goto err; + } + while (ref >= limit) { + rc = npc_mcam_idx_2_subbank_idx(rvu, ref, + &sb, &sb_off); + if (rc) + goto err; + + if (!npc_subbank_suits(sb, key_type)) { + ref = SB_ALIGN_DOWN(ref) - 1; + if (contig) { + rc = npc_idx_free(rvu, mcam_idx, cnt, false); + if (rc) + return rc; + + cnt = 0; + } + continue; + } + + mutex_lock(&sb->lock); + + if (sb->flags & NPC_SUBBANK_FLAG_FREE) { + rc = __npc_subbank_mark_used(rvu, sb, key_type); + if (rc) { + mutex_unlock(&sb->lock); + dev_err(rvu->dev, + "%s:Error to add to use array\n", + __func__); + goto err; + } + } + + __npc_subbank_sboff_2_off(rvu, sb, sb_off, &bmap, &off); + bitset = test_bit(off, bmap); + if (bitset) { + mutex_unlock(&sb->lock); + if (contig) { + rc = npc_idx_free(rvu, mcam_idx, cnt, false); + if (rc) + return rc; + + cnt = 0; + } + ref--; + continue; + } + + mcam_idx[cnt++] = ref; + sb->free_cnt--; + set_bit(off, bmap); + mutex_unlock(&sb->lock); + + if (cnt == count) + return 0; + ref--; + } + +err: + rc = npc_idx_free(rvu, mcam_idx, cnt, false); + if (rc) + dev_err(rvu->dev, + "%s: Error happened while freeing cnt=%u indexes\n", + __func__, cnt); + + return -ENOSPC; +} + +static int npc_subbank_free_cnt(struct rvu *rvu, struct npc_subbank *sb, + int key_type) +{ + int cnt, spd; + + spd = npc_priv.subbank_depth; + mutex_lock(&sb->lock); + + if (sb->flags & NPC_SUBBANK_FLAG_FREE) + cnt = key_type == NPC_MCAM_KEY_X4 ? spd : 2 * spd; + else + cnt = sb->free_cnt; + + mutex_unlock(&sb->lock); + return cnt; +} + +static int npc_subbank_ref_alloc(struct rvu *rvu, int key_type, + int ref, int limit, int prio, + bool contig, int count, + u16 *mcam_idx) +{ + struct npc_subbank *sb1, *sb2; + bool max_alloc, start, stop; + int r, l, sb_idx1, sb_idx2; + int tot = 0, rc; + int alloc_cnt; + + max_alloc = !contig; + + start = true; + stop = false; + + /* Loop until we cross the ref/limit boundary */ + while (!stop) { + rc = npc_subbank_iter(rvu, key_type, ref, limit, prio, + &r, &l, &start, &stop); + + dev_dbg(rvu->dev, + "%s: ref=%d limit=%d r=%d l=%d start=%d stop=%d tot=%d count=%d rc=%d\n", + __func__, ref, limit, r, l, + start, stop, tot, count, rc); + + if (rc) + goto err; + + /* Find subbank and subbank index for ref */ + rc = npc_mcam_idx_2_subbank_idx(rvu, r, &sb1, + &sb_idx1); + if (rc) + goto err; + + dev_dbg(rvu->dev, + "%s: ref subbank=%d off=%d\n", + __func__, sb1->idx, sb_idx1); + + /* Skip subbank if it is not available for the keytype */ + if (!npc_subbank_suits(sb1, key_type)) { + dev_dbg(rvu->dev, + "%s: not suitable sb=%d key_type=%d\n", + __func__, sb1->idx, key_type); + continue; + } + + /* Find subbank and subbank index for limit */ + rc = npc_mcam_idx_2_subbank_idx(rvu, l, &sb2, + &sb_idx2); + if (rc) + goto err; + + dev_dbg(rvu->dev, + "%s: limit subbank=%d off=%d\n", + __func__, sb_idx1, sb_idx2); + + /* subbank of ref and limit should be same */ + if (sb1 != sb2) { + dev_err(rvu->dev, + "%s: l(%d) and r(%d) are not in same subbank\n", + __func__, r, l); + goto err; + } + + if (contig && + npc_subbank_free_cnt(rvu, sb1, key_type) < count) { + dev_dbg(rvu->dev, "%s: less count =%d\n", + __func__, + npc_subbank_free_cnt(rvu, sb1, key_type)); + continue; + } + + /* Try in one bank of a subbank */ + alloc_cnt = 0; + rc = npc_subbank_alloc(rvu, sb1, key_type, + r, l, prio, contig, + count - tot, mcam_idx + tot, + count - tot, max_alloc, + &alloc_cnt); + + tot += alloc_cnt; + + dev_dbg(rvu->dev, "%s: Allocated tot=%d alloc_cnt=%d\n", + __func__, tot, alloc_cnt); + + if (!rc && count == tot) + return 0; + } +err: + dev_dbg(rvu->dev, "%s: Error to allocate\n", + __func__); + + /* non contiguous allocation fails. We need to do clean up */ + if (max_alloc) { + rc = npc_idx_free(rvu, mcam_idx, tot, false); + if (rc) + dev_err(rvu->dev, + "%s: failed to free %u indexes\n", + __func__, tot); + } + + return -EFAULT; +} + +/* Minimize allocation from bottom and top subbanks for noref allocations. + * Default allocations are ref based, and will be allocated from top + * subbanks (least priority subbanks). Since default allocation is at very + * early stage of kernel netdev probes, this subbanks will be moved to + * used subbanks list. This will pave a way for noref allocation from these + * used subbanks. Skip allocation for these top and bottom, and try free + * bank next. If none slot is available, come back and search in these + * subbanks. + */ + +static int npc_subbank_restricted_idxs[2]; +static bool restrict_valid = true; + +static bool npc_subbank_restrict_usage(struct rvu *rvu, int index) +{ + int i; + + if (!restrict_valid) + return false; + + for (i = 0; i < ARRAY_SIZE(npc_subbank_restricted_idxs); i++) { + if (index == npc_subbank_restricted_idxs[i]) + return true; + } + + return false; +} + +static int npc_subbank_noref_alloc(struct rvu *rvu, int key_type, bool contig, + int count, u16 *mcam_idx) +{ + struct npc_subbank *sb; + unsigned long index; + int tot = 0, rc; + bool max_alloc; + int alloc_cnt; + int idx, i; + void *val; + + max_alloc = !contig; + + /* Check used subbanks for free slots */ + xa_for_each(&npc_priv.xa_sb_used, index, val) { + idx = xa_to_value(val); + + /* Minimize allocation from restricted subbanks + * in noref allocations. + */ + if (npc_subbank_restrict_usage(rvu, idx)) + continue; + + sb = &npc_priv.sb[idx]; + + /* Skip if not suitable subbank */ + if (!npc_subbank_suits(sb, key_type)) + continue; + + if (contig && npc_subbank_free_cnt(rvu, sb, key_type) < count) + continue; + + /* try in bank 0. Try passing ref and limit equal to + * subbank boundaries + */ + alloc_cnt = 0; + rc = npc_subbank_alloc(rvu, sb, key_type, + sb->b0b, sb->b0t, 0, + contig, count - tot, + mcam_idx + tot, + count - tot, + max_alloc, &alloc_cnt); + + /* Non contiguous allocation may allocate less than + * requested 'count'. + */ + tot += alloc_cnt; + + dev_dbg(rvu->dev, + "%s: Allocated %d from subbank %d, tot=%d count=%d\n", + __func__, alloc_cnt, sb->idx, tot, count); + + /* Successfully allocated */ + if (!rc && count == tot) + return 0; + + /* x4 entries can be allocated from bank 0 only */ + if (key_type == NPC_MCAM_KEY_X4) + continue; + + /* try in bank 1 for x2 */ + alloc_cnt = 0; + rc = npc_subbank_alloc(rvu, sb, key_type, + sb->b1b, sb->b1t, 0, + contig, count - tot, + mcam_idx + tot, + count - tot, max_alloc, + &alloc_cnt); + + tot += alloc_cnt; + + dev_dbg(rvu->dev, + "%s: Allocated %d from subbank %d, tot=%d count=%d\n", + __func__, alloc_cnt, sb->idx, tot, count); + + if (!rc && count == tot) + return 0; + } + + /* Allocate in free subbanks */ + xa_for_each(&npc_priv.xa_sb_free, index, val) { + idx = xa_to_value(val); + sb = &npc_priv.sb[idx]; + + /* Minimize allocation from restricted subbanks + * in noref allocations. + */ + if (npc_subbank_restrict_usage(rvu, idx)) + continue; + + if (!npc_subbank_suits(sb, key_type)) + continue; + + /* try in bank 0 */ + alloc_cnt = 0; + rc = npc_subbank_alloc(rvu, sb, key_type, + sb->b0b, sb->b0t, 0, + contig, count - tot, + mcam_idx + tot, + count - tot, + max_alloc, &alloc_cnt); + + tot += alloc_cnt; + + dev_dbg(rvu->dev, + "%s: Allocated %d from subbank %d, tot=%d count=%d\n", + __func__, alloc_cnt, sb->idx, tot, count); + + /* Successfully allocated */ + if (!rc && count == tot) + return 0; + + /* x4 entries can be allocated from bank 0 only */ + if (key_type == NPC_MCAM_KEY_X4) + continue; + + /* try in bank 1 for x2 */ + alloc_cnt = 0; + rc = npc_subbank_alloc(rvu, sb, + key_type, sb->b1b, sb->b1t, 0, + contig, count - tot, + mcam_idx + tot, count - tot, + max_alloc, &alloc_cnt); + + tot += alloc_cnt; + + dev_dbg(rvu->dev, + "%s: Allocated %d from subbank %d, tot=%d count=%d\n", + __func__, alloc_cnt, sb->idx, tot, count); + + if (!rc && count == tot) + return 0; + } + + /* Allocate from restricted subbanks */ + for (i = 0; restrict_valid && + (i < ARRAY_SIZE(npc_subbank_restricted_idxs)); i++) { + idx = npc_subbank_restricted_idxs[i]; + sb = &npc_priv.sb[idx]; + + /* Skip if not suitable subbank */ + if (!npc_subbank_suits(sb, key_type)) + continue; + + if (contig && npc_subbank_free_cnt(rvu, sb, key_type) < count) + continue; + + /* try in bank 0. Try passing ref and limit equal to + * subbank boundaries + */ + alloc_cnt = 0; + rc = npc_subbank_alloc(rvu, sb, key_type, + sb->b0b, sb->b0t, 0, + contig, count - tot, + mcam_idx + tot, + count - tot, + max_alloc, &alloc_cnt); + + /* Non contiguous allocation may allocate less than + * requested 'count'. + */ + tot += alloc_cnt; + + dev_dbg(rvu->dev, + "%s: Allocated %d from subbank %d, tot=%d count=%d\n", + __func__, alloc_cnt, sb->idx, tot, count); + + /* Successfully allocated */ + if (!rc && count == tot) + return 0; + + /* x4 entries can be allocated from bank 0 only */ + if (key_type == NPC_MCAM_KEY_X4) + continue; + + /* try in bank 1 for x2 */ + alloc_cnt = 0; + rc = npc_subbank_alloc(rvu, sb, key_type, + sb->b1b, sb->b1t, 0, + contig, count - tot, + mcam_idx + tot, + count - tot, max_alloc, + &alloc_cnt); + + tot += alloc_cnt; + + dev_dbg(rvu->dev, + "%s: Allocated %d from subbank %d, tot=%d count=%d\n", + __func__, alloc_cnt, sb->idx, tot, count); + + if (!rc && count == tot) + return 0; + } + + /* non contiguous allocation fails. We need to do clean up */ + if (max_alloc) + npc_idx_free(rvu, mcam_idx, tot, false); + + dev_dbg(rvu->dev, "%s: non-contig allocation fails\n", + __func__); + + return -EFAULT; +} + +int npc_cn20k_idx_free(struct rvu *rvu, u16 *mcam_idx, int count) +{ + return npc_idx_free(rvu, mcam_idx, count, true); +} + +int npc_cn20k_ref_idx_alloc(struct rvu *rvu, int pcifunc, int key_type, + int prio, u16 *mcam_idx, int ref, int limit, + bool contig, int count, bool virt) +{ + bool defrag_candidate = false; + int i, eidx, rc, bd; + bool ref_valid; + u16 vidx; + + bd = npc_priv.bank_depth; + + /* Special case: ref == 0 && limit= 0 && prio == HIGH && count == 1 + * Here user wants to allocate 0th entry + */ + if (!ref && !limit && prio == NPC_MCAM_HIGHER_PRIO && + count == 1) { + rc = npc_subbank_ref_alloc(rvu, key_type, ref, limit, + prio, contig, count, mcam_idx); + + if (rc) + return rc; + goto add2map; + } + + ref_valid = !!(limit || ref); + defrag_candidate = !ref_valid && !contig && virt; + if (!ref_valid) { + if (contig && count > npc_priv.subbank_depth) + goto try_noref_multi_subbank; + + rc = npc_subbank_noref_alloc(rvu, key_type, contig, + count, mcam_idx); + if (!rc) + goto add2map; + +try_noref_multi_subbank: + eidx = (key_type == NPC_MCAM_KEY_X4) ? bd - 1 : 2 * bd - 1; + + if (prio == NPC_MCAM_HIGHER_PRIO) + rc = npc_multi_subbank_ref_alloc(rvu, key_type, + eidx, 0, + NPC_MCAM_HIGHER_PRIO, + contig, count, + mcam_idx); + else + rc = npc_multi_subbank_ref_alloc(rvu, key_type, + 0, eidx, + NPC_MCAM_LOWER_PRIO, + contig, count, + mcam_idx); + + if (!rc) + goto add2map; + + return rc; + } + + if ((prio == NPC_MCAM_LOWER_PRIO && ref > limit) || + (prio == NPC_MCAM_HIGHER_PRIO && ref < limit)) { + dev_err(rvu->dev, "%s: Wrong ref_enty(%d) or limit(%d)\n", + __func__, ref, limit); + return -EINVAL; + } + + if ((key_type == NPC_MCAM_KEY_X4 && (ref >= bd || limit >= bd)) || + (key_type == NPC_MCAM_KEY_X2 && + (ref >= 2 * bd || limit >= 2 * bd))) { + dev_err(rvu->dev, "%s: Wrong ref_enty(%d) or limit(%d)\n", + __func__, ref, limit); + return -EINVAL; + } + + if (contig && count > npc_priv.subbank_depth) + goto try_ref_multi_subbank; + + rc = npc_subbank_ref_alloc(rvu, key_type, ref, limit, + prio, contig, count, mcam_idx); + if (!rc) + goto add2map; + +try_ref_multi_subbank: + rc = npc_multi_subbank_ref_alloc(rvu, key_type, + ref, limit, prio, + contig, count, mcam_idx); + if (!rc) + goto add2map; + + return rc; + +add2map: + for (i = 0; i < count; i++) { + rc = npc_add_to_pf_maps(rvu, mcam_idx[i], pcifunc); + if (rc) + goto err; + + if (!defrag_candidate) + continue; + + rc = npc_vidx_maps_add_entry(rvu, mcam_idx[i], pcifunc, &vidx); + if (rc) { + npc_del_from_pf_maps(rvu, mcam_idx[i]); + goto err; + } + + /* Return vidx to caller */ + mcam_idx[i] = vidx; + } + + return 0; +err: + for (int j = 0; j < i; j++) { + npc_del_from_pf_maps(rvu, npc_vidx2idx(mcam_idx[j])); + + if (!defrag_candidate) + continue; + + npc_vidx_maps_del_entry(rvu, mcam_idx[j], NULL); + } + + return rc; + +} + +void npc_cn20k_subbank_calc_free(struct rvu *rvu, int *x2_free, + int *x4_free, int *sb_free) +{ + struct npc_subbank *sb; + int i; + + /* Reset all stats to zero */ + *x2_free = 0; + *x4_free = 0; + *sb_free = 0; + + for (i = 0; i < npc_priv.num_subbanks; i++) { + sb = &npc_priv.sb[i]; + mutex_lock(&sb->lock); + + /* Count number of free subbanks */ + if (sb->flags & NPC_SUBBANK_FLAG_FREE) { + (*sb_free)++; + goto next; + } + + /* Sumup x4 free count */ + if (sb->key_type == NPC_MCAM_KEY_X4) { + (*x4_free) += sb->free_cnt; + goto next; + } + + /* Sumup x2 free counts */ + (*x2_free) += sb->free_cnt; +next: + mutex_unlock(&sb->lock); + } +} + +int +rvu_mbox_handler_npc_cn20k_get_fcnt(struct rvu *rvu, + struct msg_req *req, + struct npc_cn20k_get_fcnt_rsp *rsp) +{ + npc_cn20k_subbank_calc_free(rvu, &rsp->free_x2, + &rsp->free_x4, &rsp->free_subbanks); + return 0; +} + +int +rvu_mbox_handler_npc_cn20k_get_kex_cfg(struct rvu *rvu, + struct msg_req *req, + struct npc_cn20k_get_kex_cfg_rsp *rsp) +{ + int extr, lt; + + rsp->rx_keyx_cfg = CN20K_GET_KEX_CFG(NIX_INTF_RX); + rsp->tx_keyx_cfg = CN20K_GET_KEX_CFG(NIX_INTF_TX); + + /* Get EXTRACTOR LID */ + for (extr = 0; extr < NPC_MAX_EXTRACTOR; extr++) { + rsp->intf_extr_lid[NIX_INTF_RX][extr] = + CN20K_GET_EXTR_LID(NIX_INTF_RX, extr); + rsp->intf_extr_lid[NIX_INTF_TX][extr] = + CN20K_GET_EXTR_LID(NIX_INTF_TX, extr); + } + + /* Get EXTRACTOR LTYPE */ + for (extr = 0; extr < NPC_MAX_EXTRACTOR; extr++) { + for (lt = 0; lt < NPC_MAX_LT; lt++) { + rsp->intf_extr_lt[NIX_INTF_RX][extr][lt] = + CN20K_GET_EXTR_LT(NIX_INTF_RX, extr, lt); + rsp->intf_extr_lt[NIX_INTF_TX][extr][lt] = + CN20K_GET_EXTR_LT(NIX_INTF_TX, extr, lt); + } + } + + memcpy(rsp->mkex_pfl_name, rvu->mkex_pfl_name, MKEX_NAME_LEN); + return 0; +} + +static int *subbank_srch_order; + +static void npc_populate_restricted_idxs(int num_subbanks) +{ + npc_subbank_restricted_idxs[0] = num_subbanks - 1; + npc_subbank_restricted_idxs[1] = 0; +} + +static int npc_create_srch_order(int cnt) +{ + int val = 0; + + subbank_srch_order = kcalloc(cnt, sizeof(int), + GFP_KERNEL); + if (!subbank_srch_order) + return -ENOMEM; + + /* cnt(subbank depth) is always a power of 2. There is a check in + * npc_priv_init() to check the same. + */ + for (int i = 0; i < cnt; i += 2) { + subbank_srch_order[i] = cnt / 2 - val - 1; + subbank_srch_order[i + 1] = cnt / 2 + 1 + val; + val++; + } + + subbank_srch_order[cnt - 1] = cnt / 2; + return 0; +} + +static void npc_subbank_init(struct rvu *rvu, struct npc_subbank *sb, int idx) +{ + mutex_init(&sb->lock); + + sb->b0b = idx * npc_priv.subbank_depth; + sb->b0t = sb->b0b + npc_priv.subbank_depth - 1; + + sb->b1b = npc_priv.bank_depth + idx * npc_priv.subbank_depth; + sb->b1t = sb->b1b + npc_priv.subbank_depth - 1; + + sb->flags = NPC_SUBBANK_FLAG_FREE; + sb->idx = idx; + sb->arr_idx = subbank_srch_order[idx]; + + dev_dbg(rvu->dev, "%s: sb->idx=%u sb->arr_idx=%u\n", + __func__, sb->idx, sb->arr_idx); + + /* Keep first and last subbank at end of free array; so that + * it will be used at last + */ + xa_store(&npc_priv.xa_sb_free, sb->arr_idx, + xa_mk_value(sb->idx), GFP_KERNEL); +} + +static int npc_pcifunc_map_create(struct rvu *rvu) +{ + int pf, vf, numvfs; + int cnt = 0; + u16 pcifunc; + u64 cfg; + + for (pf = 0; pf < rvu->hw->total_pfs; pf++) { + cfg = rvu_read64(rvu, BLKADDR_RVUM, RVU_PRIV_PFX_CFG(pf)); + numvfs = (cfg >> 12) & 0xFF; + + /* Skip not enabled PFs */ + if (!(cfg & BIT_ULL(20))) + goto chk_vfs; + + /* If Admin function, check on VFs */ + if (cfg & BIT_ULL(21)) + goto chk_vfs; + + pcifunc = pf << 9; + + xa_store(&npc_priv.xa_pf_map, (unsigned long)pcifunc, + xa_mk_value(cnt), GFP_KERNEL); + + cnt++; + +chk_vfs: + for (vf = 0; vf < numvfs; vf++) { + pcifunc = (pf << 9) | (vf + 1); + + xa_store(&npc_priv.xa_pf_map, (unsigned long)pcifunc, + xa_mk_value(cnt), GFP_KERNEL); + cnt++; + } + } + + return cnt; +} + +struct npc_defrag_node { + u8 idx; + u8 key_type; + bool valid; + bool refs; + u16 free_cnt; + u16 vidx_cnt; + u16 *vidx; + struct list_head list; +}; + +static bool npc_defrag_skip_restricted_sb(int sb_id) +{ + int i; + + if (!restrict_valid) + return false; + + for (i = 0; i < ARRAY_SIZE(npc_subbank_restricted_idxs); i++) + if (sb_id == npc_subbank_restricted_idxs[i]) + return true; + return false; +} + +/* Find subbank with minimum number of virtual indexes */ +static struct npc_defrag_node *npc_subbank_min_vidx(struct list_head *lh) +{ + struct npc_defrag_node *node, *tnode = NULL; + int min = INT_MAX; + + list_for_each_entry(node, lh, list) { + if (!node->valid) + continue; + + /* if subbank has ref allocated mcam indexes, that subbank + * is not a good candidate to move out indexes. + */ + if (node->refs) + continue; + + if (min > node->vidx_cnt) { + min = node->vidx_cnt; + tnode = node; + } + } + + return tnode; +} + +/* Find subbank with maximum number of free spaces */ +static struct npc_defrag_node *npc_subbank_max_free(struct list_head *lh) +{ + struct npc_defrag_node *node, *tnode = NULL; + int max = INT_MIN; + + list_for_each_entry(node, lh, list) { + if (!node->valid) + continue; + + if (max < node->free_cnt) { + max = node->free_cnt; + tnode = node; + } + } + + return tnode; +} + +static int npc_defrag_alloc_free_slots(struct rvu *rvu, + struct npc_defrag_node *f, + int cnt, u16 *save) +{ + int alloc_cnt1, alloc_cnt2; + struct npc_subbank *sb; + int rc, sb_off, i; + bool deleted; + + sb = &npc_priv.sb[f->idx]; + + alloc_cnt1 = 0; + alloc_cnt2 = 0; + + rc = __npc_subbank_alloc(rvu, sb, + NPC_MCAM_KEY_X2, sb->b0b, + sb->b0t, + NPC_MCAM_LOWER_PRIO, + false, cnt, save, cnt, true, + &alloc_cnt1); + if (alloc_cnt1 < cnt) { + rc = __npc_subbank_alloc(rvu, sb, + NPC_MCAM_KEY_X2, sb->b1b, + sb->b1t, + NPC_MCAM_LOWER_PRIO, + false, cnt - alloc_cnt1, + save + alloc_cnt1, + cnt - alloc_cnt1, + true, &alloc_cnt2); + } + + if (alloc_cnt1 + alloc_cnt2 != cnt) { + dev_err(rvu->dev, + "%s: Failed to alloc cnt=%u alloc_cnt1=%u alloc_cnt2=%u\n", + __func__, cnt, alloc_cnt1, alloc_cnt2); + goto fail_free_alloc; + } + return 0; + +fail_free_alloc: + for (i = 0; i < alloc_cnt1 + alloc_cnt2; i++) { + rc = npc_mcam_idx_2_subbank_idx(rvu, save[i], + &sb, &sb_off); + if (rc) { + dev_err(rvu->dev, + "%s: Error to find subbank for mcam idx=%u\n", + __func__, save[i]); + break; + } + + deleted = __npc_subbank_free(rvu, sb, sb_off); + if (!deleted) { + dev_err(rvu->dev, + "%s: Error to free mcam idx=%u\n", + __func__, save[i]); + break; + } + } + + return rc; +} + +static int npc_defrag_add_2_show_list(struct rvu *rvu, u16 old_midx, + u16 new_midx, u16 vidx) +{ + struct npc_defrag_show_node *node; + + node = kcalloc(1, sizeof(*node), GFP_KERNEL); + if (!node) + return -ENOMEM; + + node->old_midx = old_midx; + node->new_midx = new_midx; + node->vidx = vidx; + INIT_LIST_HEAD(&node->list); + + mutex_lock(&npc_priv.lock); + list_add_tail(&node->list, &npc_priv.defrag_lh); + mutex_unlock(&npc_priv.lock); + + return 0; +} + +static +int npc_defrag_move_vdx_to_free(struct rvu *rvu, + struct npc_defrag_node *f, + struct npc_defrag_node *v, + int cnt, u16 *save) +{ + struct npc_mcam *mcam = &rvu->hw->mcam; + int i, vidx_cnt, rc, sb_off; + u16 new_midx, old_midx, vidx; + struct npc_subbank *sb; + bool deleted; + u16 pcifunc; + int blkaddr; + void *map; + u8 bank; + u16 midx; + u64 stats; + + blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); + + vidx_cnt = v->vidx_cnt; + for (i = 0; i < cnt; i++) { + vidx = v->vidx[vidx_cnt - i - 1]; + old_midx = npc_vidx2idx(vidx); + new_midx = save[cnt - i - 1]; + + dev_dbg(rvu->dev, + "%s: Moving %u ---> %u (vidx=%u)\n", + __func__, + old_midx, new_midx, vidx); + + rc = npc_defrag_add_2_show_list(rvu, old_midx, new_midx, vidx); + if (rc) + dev_err(rvu->dev, + "%s: Error happened to add to show list vidx=%u\n", + __func__, vidx); + + /* Modify vidx to point to new mcam idx */ + rc = npc_vidx_maps_modify(rvu, vidx, new_midx); + if (rc) + return rc; + + midx = old_midx % mcam->banksize; + bank = old_midx / mcam->banksize; + stats = rvu_read64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_STAT_EXT(midx, + bank)); + + npc_cn20k_enable_mcam_entry(rvu, blkaddr, old_midx, false); + npc_cn20k_copy_mcam_entry(rvu, blkaddr, old_midx, new_midx); + npc_cn20k_enable_mcam_entry(rvu, blkaddr, new_midx, true); + + midx = new_midx % mcam->banksize; + bank = new_midx / mcam->banksize; + rvu_write64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_STAT_EXT(midx, bank), + stats); + + /* Free the old mcam idx */ + rc = npc_mcam_idx_2_subbank_idx(rvu, old_midx, + &sb, &sb_off); + if (rc) { + dev_err(rvu->dev, + "%s: Unable to calculate subbank off for mcamidx=%u\n", + __func__, old_midx); + return rc; + } + + deleted = __npc_subbank_free(rvu, sb, sb_off); + if (!deleted) { + dev_err(rvu->dev, + "%s: Failed to free mcamidx=%u sb=%u sb_off=%u\n", + __func__, old_midx, sb->idx, sb_off); + return -EFAULT; + } + + /* save pcifunc */ + map = xa_load(&npc_priv.xa_idx2pf_map, old_midx); + pcifunc = xa_to_value(map); + + /* delete from pf maps */ + rc = npc_del_from_pf_maps(rvu, old_midx); + if (rc) { + dev_err(rvu->dev, + "%s: Failed to delete pf maps for mcamidx=%u\n", + __func__, old_midx); + return rc; + } + + /* add new mcam_idx to pf map */ + rc = npc_add_to_pf_maps(rvu, new_midx, pcifunc); + if (rc) { + dev_err(rvu->dev, + "%s: Failed to add pf maps for mcamidx=%u\n", + __func__, new_midx); + return rc; + } + + /* Remove from mcam maps */ + mcam->entry2pfvf_map[old_midx] = NPC_MCAM_INVALID_MAP; + mcam->entry2cntr_map[old_midx] = NPC_MCAM_INVALID_MAP; + npc_mcam_clear_bit(mcam, old_midx); + + mcam->entry2pfvf_map[new_midx] = pcifunc; + /* Counter is not preserved */ + mcam->entry2cntr_map[new_midx] = new_midx; + npc_mcam_set_bit(mcam, new_midx); + + /* Mark as invalid */ + v->vidx[vidx_cnt - i - 1] = -1; + save[cnt - i - 1] = -1; + + f->free_cnt--; + v->vidx_cnt--; + } + + return 0; +} + +static int npc_defrag_process(struct rvu *rvu, struct list_head *lh) +{ + struct npc_defrag_node *v = NULL; + struct npc_defrag_node *f = NULL; + int rc = 0, cnt; + u16 *save; + + while (1) { + /* Find subbank with minimum vidx */ + if (!v) { + v = npc_subbank_min_vidx(lh); + if (!v) + break; + } + + /* Find subbank with maximum free slots */ + if (!f) { + f = npc_subbank_max_free(lh); + if (!f) + break; + } + + if (!v->vidx_cnt) { + list_del_init(&v->list); + v = NULL; + continue; + } + + if (!f->free_cnt) { + list_del_init(&f->list); + f = NULL; + continue; + } + + /* If both subbanks are same, choose vidx and + * search for free list again + */ + if (f == v) { + list_del_init(&f->list); + f = NULL; + continue; + } + + /* Calculate minimum free slots needs to be allocated */ + cnt = f->free_cnt > v->vidx_cnt ? v->vidx_cnt : + f->free_cnt; + + dev_dbg(rvu->dev, + "%s: cnt=%u free_cnt=%u(sb=%u) vidx_cnt=%u(sb=%u)\n", + __func__, cnt, f->free_cnt, f->idx, + v->vidx_cnt, v->idx); + + /* Allocate an array to store newly allocated + * free slots (mcam indexes) + */ + save = kcalloc(cnt, sizeof(*save), GFP_KERNEL); + if (!save) { + rc = -ENOMEM; + goto err; + } + + /* Alloc free slots for existing vidx */ + rc = npc_defrag_alloc_free_slots(rvu, f, cnt, save); + if (rc) { + kfree(save); + goto err; + } + + /* Move vidx to free slots; update pf_map and vidx maps, + * and free existing vidx mcam slots + */ + rc = npc_defrag_move_vdx_to_free(rvu, f, v, cnt, save); + if (rc) { + kfree(save); + goto err; + } + + kfree(save); + + if (!f->free_cnt) { + list_del_init(&f->list); + f = NULL; + } + + if (!v->vidx_cnt) { + list_del_init(&v->list); + v = NULL; + } + } + +err: + /* Whole defragmentation process is done within locks. if there + * is an error, it would be hard to roll back as index remove/add + * can fail again if it failed before. This would mean that there + * is bug in the index management algorithm. + * Return from here than rolling back. + */ + return rc; +} + +static void npc_defrag_list_clear(void) +{ + struct npc_defrag_show_node *node, *next; + + mutex_lock(&npc_priv.lock); + list_for_each_entry_safe(node, next, &npc_priv.defrag_lh, list) { + list_del_init(&node->list); + kfree(node); + } + + mutex_unlock(&npc_priv.lock); +} + +static void npc_lock_all_subbank(void) +{ + int i; + + for (i = 0; i < npc_priv.num_subbanks; i++) + mutex_lock(&npc_priv.sb[i].lock); +} + +static void npc_unlock_all_subbank(void) +{ + int i; + + for (i = npc_priv.num_subbanks - 1; i >= 0; i--) + mutex_unlock(&npc_priv.sb[i].lock); +} + +/* Only non-ref non-contigous mcam indexes + * are picked for defrag process + */ +int npc_cn20k_defrag(struct rvu *rvu) +{ + struct npc_mcam *mcam = &rvu->hw->mcam; + struct npc_defrag_node *node, *tnode; + struct list_head x4lh, x2lh, *lh; + int rc = 0, i, sb_off, tot; + struct npc_subbank *sb; + unsigned long index; + void *map; + u16 midx; + + /* Free previous show list */ + npc_defrag_list_clear(); + + INIT_LIST_HEAD(&x4lh); + INIT_LIST_HEAD(&x2lh); + + node = kcalloc(npc_priv.num_subbanks, sizeof(*node), GFP_KERNEL); + if (!node) + return -ENOMEM; + + /* Lock mcam */ + mutex_lock(&mcam->lock); + npc_lock_all_subbank(); + + /* Fill in node with subbank properties */ + for (i = 0; i < npc_priv.num_subbanks; i++) { + sb = &npc_priv.sb[i]; + + node[i].idx = i; + node[i].key_type = sb->key_type; + node[i].free_cnt = sb->free_cnt; + node[i].vidx = kcalloc(npc_priv.subbank_depth * 2, + sizeof(*node[i].vidx), + GFP_KERNEL); + if (!node[i].vidx) { + rc = -ENOMEM; + goto free_vidx; + } + + /* If subbank is empty, dont include it in defrag + * process + */ + if (sb->flags & NPC_SUBBANK_FLAG_FREE) { + node[i].valid = false; + continue; + } + + if (npc_defrag_skip_restricted_sb(i)) { + node[i].valid = false; + continue; + } + + node[i].valid = true; + INIT_LIST_HEAD(&node[i].list); + + /* Add node to x2 or x4 list */ + lh = sb->key_type == NPC_MCAM_KEY_X2 ? &x2lh : &x4lh; + list_add_tail(&node[i].list, lh); + } + + /* Filling vidx[] array with all vidx in that subbank */ + xa_for_each_start(&npc_priv.xa_vidx2idx_map, index, map, + npc_priv.bank_depth * 2) { + midx = xa_to_value(map); + rc = npc_mcam_idx_2_subbank_idx(rvu, midx, + &sb, &sb_off); + if (rc) { + dev_err(rvu->dev, + "%s: Error to get mcam_idx for vidx=%lu\n", + __func__, index); + goto free_vidx; + } + + tnode = &node[sb->idx]; + tnode->vidx[tnode->vidx_cnt] = index; + tnode->vidx_cnt++; + } + + /* Mark all subbank which has ref allocation */ + for (i = 0; i < npc_priv.num_subbanks; i++) { + tnode = &node[i]; + + if (!tnode->valid) + continue; + + tot = (tnode->key_type == NPC_MCAM_KEY_X2) ? + npc_priv.subbank_depth * 2 : npc_priv.subbank_depth; + + if (node[i].vidx_cnt != tot - tnode->free_cnt) + tnode->refs = true; + } + + rc = npc_defrag_process(rvu, &x2lh); + if (rc) + goto free_vidx; + + rc = npc_defrag_process(rvu, &x4lh); + if (rc) + goto free_vidx; + +free_vidx: + npc_unlock_all_subbank(); + mutex_unlock(&mcam->lock); + for (i = 0; i < npc_priv.num_subbanks; i++) + kfree(node[i].vidx); + kfree(node); + return rc; +} + +int rvu_mbox_handler_npc_defrag(struct rvu *rvu, struct msg_req *req, + struct msg_rsp *rsp) +{ + return npc_cn20k_defrag(rvu); +} + +int npc_cn20k_dft_rules_idx_get(struct rvu *rvu, u16 pcifunc, u16 *bcast, + u16 *mcast, u16 *promisc, u16 *ucast) +{ + u16 *ptr[4] = {promisc, mcast, bcast, ucast}; + unsigned long idx; + bool set = false; + void *val; + int i, j; + + if (!npc_priv.init_done) + return 0; + + if (is_lbk_vf(rvu, pcifunc)) { + if (!ptr[0]) + return -EINVAL; + + idx = NPC_DFT_RULE_ID_MK(pcifunc, NPC_DFT_RULE_PROMISC_ID); + val = xa_load(&npc_priv.xa_pf2dfl_rmap, idx); + if (!val) { + pr_debug("%s: Failed to find %s index for pcifunc=%#x\n", + __func__, + npc_dft_rule_name[NPC_DFT_RULE_PROMISC_ID], + pcifunc); + + *ptr[0] = USHRT_MAX; + return -ESRCH; + } + + *ptr[0] = xa_to_value(val); + return 0; + } + + if (is_vf(pcifunc)) { + if (!ptr[3]) + return -EINVAL; + + idx = NPC_DFT_RULE_ID_MK(pcifunc, NPC_DFT_RULE_UCAST_ID); + val = xa_load(&npc_priv.xa_pf2dfl_rmap, idx); + if (!val) { + pr_debug("%s: Failed to find %s index for pcifunc=%#x\n", + __func__, + npc_dft_rule_name[NPC_DFT_RULE_UCAST_ID], + pcifunc); + + *ptr[3] = USHRT_MAX; + return -ESRCH; + } + + *ptr[3] = xa_to_value(val); + return 0; + } + + for (i = NPC_DFT_RULE_START_ID, j = 0; i < NPC_DFT_RULE_MAX_ID; i++, + j++) { + if (!ptr[j]) + continue; + + idx = NPC_DFT_RULE_ID_MK(pcifunc, i); + val = xa_load(&npc_priv.xa_pf2dfl_rmap, idx); + if (!val) { + pr_debug("%s: Failed to find %s index for pcifunc=%#x\n", + __func__, + npc_dft_rule_name[i], pcifunc); + + *ptr[j] = USHRT_MAX; + continue; + } + + *ptr[j] = xa_to_value(val); + set = true; + } + + return set ? 0 : -ESRCH; +} + +int rvu_mbox_handler_npc_get_pfl_info(struct rvu *rvu, struct msg_req *req, + struct npc_get_pfl_info_rsp *rsp) +{ + if (!is_cn20k(rvu->pdev)) { + dev_err(rvu->dev, "Mbox support is only for cn20k\n"); + return -EOPNOTSUPP; + } + + rsp->kw_type = npc_priv.kw; + rsp->x4_slots = npc_priv.bank_depth; + return 0; +} + +int rvu_mbox_handler_npc_get_num_kws(struct rvu *rvu, + struct npc_get_num_kws_req *req, + struct npc_get_num_kws_rsp *rsp) +{ + u64 kw_mask[NPC_KWS_IN_KEY_SZ_MAX] = { 0 }; + u64 kw[NPC_KWS_IN_KEY_SZ_MAX] = { 0 }; + struct rvu_npc_mcam_rule dummy = { 0 }; + struct mcam_entry_mdata mdata = { }; + struct npc_install_flow_req *fl; + int i, cnt = 0, blkaddr; + + if (!is_cn20k(rvu->pdev)) { + dev_err(rvu->dev, "Mbox support is only for cn20k\n"); + return -EOPNOTSUPP; + } + + fl = &req->fl; + + blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); + if (blkaddr < 0) { + dev_err(rvu->dev, "%s: NPC block not implemented\n", __func__); + return NPC_MCAM_INVALID_REQ; + } + + mdata.kw = kw; + mdata.kw_mask = kw_mask; + + npc_update_flow(rvu, &mdata, fl->features, &fl->packet, + &fl->mask, &dummy, fl->intf, blkaddr); + + /* Find the most significant word valid. Traverse from + * MSB to LSB, check if cam0 or cam1 is set + */ + for (i = NPC_KWS_IN_KEY_SZ_MAX - 1; i >= 0; i--) { + if (kw[i] || kw_mask[i]) { + cnt = i + 1; + break; + } + } + + rsp->kws = cnt; + + return 0; +} + +int rvu_mbox_handler_npc_get_dft_rl_idxs(struct rvu *rvu, struct msg_req *req, + struct npc_get_dft_rl_idxs_rsp *rsp) +{ + u16 bcast, mcast, promisc, ucast; + u16 pcifunc; + int rc; + + if (!is_cn20k(rvu->pdev)) { + dev_err(rvu->dev, "Mbox support is only for cn20k\n"); + return -EOPNOTSUPP; + } + + pcifunc = req->hdr.pcifunc; + + rc = npc_cn20k_dft_rules_idx_get(rvu, pcifunc, &bcast, &mcast, + &promisc, &ucast); + if (rc) + return rc; + + rsp->bcast = bcast; + rsp->mcast = mcast; + rsp->promisc = promisc; + rsp->ucast = ucast; + return 0; +} + +static bool npc_is_cgx_or_lbk(struct rvu *rvu, u16 pcifunc) +{ + return is_pf_cgxmapped(rvu, rvu_get_pf(rvu->pdev, pcifunc)) || + is_lbk_vf(rvu, pcifunc); +} + +void npc_cn20k_dft_rules_free(struct rvu *rvu, u16 pcifunc) +{ + struct npc_mcam_free_entry_req free_req = { 0 }; + unsigned long index; + struct msg_rsp rsp; + u16 ptr[4]; + int rc, i; + void *map; + + if (!npc_priv.init_done) + return; + + if (!npc_is_cgx_or_lbk(rvu, pcifunc)) { + dev_dbg(rvu->dev, + "%s: dft rule allocation is only for cgx mapped device, pcifunc=%#x\n", + __func__, pcifunc); + return; + } + + rc = npc_cn20k_dft_rules_idx_get(rvu, pcifunc, &ptr[0], &ptr[1], + &ptr[2], &ptr[3]); + if (rc) + return; + + /* LBK */ + if (is_lbk_vf(rvu, pcifunc)) { + index = NPC_DFT_RULE_ID_MK(pcifunc, NPC_DFT_RULE_PROMISC_ID); + map = xa_erase(&npc_priv.xa_pf2dfl_rmap, index); + if (!map) + dev_dbg(rvu->dev, + "%s: Err from delete %s mcam idx from xarray (pcifunc=%#x\n", + __func__, + npc_dft_rule_name[NPC_DFT_RULE_PROMISC_ID], + pcifunc); + + goto free_rules; + } + + /* VF */ + if (is_vf(pcifunc)) { + index = NPC_DFT_RULE_ID_MK(pcifunc, NPC_DFT_RULE_UCAST_ID); + map = xa_erase(&npc_priv.xa_pf2dfl_rmap, index); + if (!map) + dev_dbg(rvu->dev, + "%s: Err from delete %s mcam idx from xarray (pcifunc=%#x\n", + __func__, + npc_dft_rule_name[NPC_DFT_RULE_UCAST_ID], + pcifunc); + + goto free_rules; + } + + /* PF */ + for (i = NPC_DFT_RULE_START_ID; i < NPC_DFT_RULE_MAX_ID; i++) { + index = NPC_DFT_RULE_ID_MK(pcifunc, i); + map = xa_erase(&npc_priv.xa_pf2dfl_rmap, index); + if (!map) + dev_dbg(rvu->dev, + "%s: Err from delete %s mcam idx from xarray (pcifunc=%#x\n", + __func__, npc_dft_rule_name[i], + pcifunc); + } + +free_rules: + + free_req.hdr.pcifunc = pcifunc; + free_req.all = 1; + rc = rvu_mbox_handler_npc_mcam_free_entry(rvu, &free_req, &rsp); + if (rc) + dev_err(rvu->dev, + "%s: Error deleting default entries (pcifunc=%#x\n", + __func__, pcifunc); +} + +int npc_cn20k_dft_rules_alloc(struct rvu *rvu, u16 pcifunc) +{ + struct npc_mcam_free_entry_req free_req = { 0 }; + u16 mcam_idx[4] = { 0 }, pf_ucast, pf_pcifunc; + struct npc_mcam_alloc_entry_req req = { 0 }; + struct npc_mcam_alloc_entry_rsp rsp = { 0 }; + int ret, eidx, i, k, pf, cnt; + struct rvu_pfvf *pfvf; + unsigned long index; + struct msg_rsp free_rsp; + u16 b, m, p, u; + + if (!npc_priv.init_done) + return 0; + + if (!npc_is_cgx_or_lbk(rvu, pcifunc)) { + dev_dbg(rvu->dev, + "%s: dft rule allocation is only for cgx mapped device, pcifunc=%#x\n", + __func__, pcifunc); + return 0; + } + + /* Check if default rules are already alloced for this pcifunc */ + ret = npc_cn20k_dft_rules_idx_get(rvu, pcifunc, &b, &m, &p, &u); + if (!ret) { + dev_dbg(rvu->dev, + "%s: default rules are already installed (pcifunc=%#x)\n", + __func__, pcifunc); + dev_dbg(rvu->dev, + "%s: bcast(%u) mcast(%u) promisc(%u) ucast(%u)\n", + __func__, b, m, p, u); + return 0; + } + + /* Set ref index as lowest priority index */ + eidx = 2 * npc_priv.bank_depth - 1; + + /* Install only UCAST for VF */ + cnt = is_vf(pcifunc) ? 1 : ARRAY_SIZE(mcam_idx); + + /* For VF pcifunc, allocate default mcam indexes by taking + * ref as PF's ucast index. + */ + if (is_vf(pcifunc)) { + pf = rvu_get_pf(rvu->pdev, pcifunc); + pf_pcifunc = pf << RVU_CN20K_PFVF_PF_SHIFT; + + /* Get PF's ucast entry index */ + ret = npc_cn20k_dft_rules_idx_get(rvu, pf_pcifunc, NULL, + NULL, NULL, &pf_ucast); + + /* There is no PF rules installed; and VF installation comes + * first. PF may come later. + * TODO: Install PF rules before installing VF rules. + */ + + /* Set PF's ucast as ref entry */ + if (!ret) + eidx = pf_ucast; + } + + pfvf = rvu_get_pfvf(rvu, pcifunc); + pfvf->hw_prio = NPC_DFT_RULE_PRIO; + + req.contig = false; + req.ref_prio = NPC_MCAM_HIGHER_PRIO; + req.ref_entry = eidx; + req.kw_type = NPC_MCAM_KEY_X2; + req.count = cnt; + req.hdr.pcifunc = pcifunc; + + ret = rvu_mbox_handler_npc_mcam_alloc_entry(rvu, &req, &rsp); + + /* successfully allocated index */ + if (!ret) { + /* Copy indexes to local array */ + for (i = 0; i < cnt; i++) + mcam_idx[i] = rsp.entry_list[i]; + + goto chk_sanity; + } + + /* If there is no slots available and request is for PF, + * return error. + */ + if (!is_vf(pcifunc)) { + dev_err(rvu->dev, + "%s: Default index allocation failed for pcifunc=%#x\n", + __func__, pcifunc); + return ret; + } + + /* We could not find an index with higher priority index for VF. + * Find rule with lower priority index and set hardware priority + * as NPC_DFT_RULE_PRIO - 1 (higher hw priority) + */ + req.contig = false; + req.kw_type = NPC_MCAM_KEY_X2; + req.count = cnt; + req.hdr.pcifunc = pcifunc; + req.ref_prio = NPC_MCAM_LOWER_PRIO; + req.ref_entry = eidx + 1; + ret = rvu_mbox_handler_npc_mcam_alloc_entry(rvu, &req, &rsp); + if (ret) { + dev_err(rvu->dev, + "%s: Default index allocation failed for pcifunc=%#x\n", + __func__, pcifunc); + return ret; + } + + /* Copy indexes to local array */ + for (i = 0; i < cnt; i++) + mcam_idx[i] = rsp.entry_list[i]; + + pfvf->hw_prio = NPC_DFT_RULE_PRIO - 1; + +chk_sanity: + /* LBK */ + if (is_lbk_vf(rvu, pcifunc)) { + index = NPC_DFT_RULE_ID_MK(pcifunc, NPC_DFT_RULE_PROMISC_ID); + ret = xa_insert(&npc_priv.xa_pf2dfl_rmap, index, + xa_mk_value(mcam_idx[0]), GFP_KERNEL); + if (ret) { + dev_err(rvu->dev, + "%s: Err to insert %s mcam idx to xarray pcifunc=%#x\n", + __func__, + npc_dft_rule_name[NPC_DFT_RULE_PROMISC_ID], + pcifunc); + goto err; + } + + goto done; + } + + /* VF */ + if (is_vf(pcifunc)) { + index = NPC_DFT_RULE_ID_MK(pcifunc, NPC_DFT_RULE_UCAST_ID); + ret = xa_insert(&npc_priv.xa_pf2dfl_rmap, index, + xa_mk_value(mcam_idx[0]), GFP_KERNEL); + if (ret) { + dev_err(rvu->dev, + "%s: Err to insert %s mcam idx to xarray pcifunc=%#x\n", + __func__, + npc_dft_rule_name[NPC_DFT_RULE_UCAST_ID], + pcifunc); + goto err; + } + + goto done; + } + + /* PF */ + for (i = NPC_DFT_RULE_START_ID, k = 0; i < NPC_DFT_RULE_MAX_ID && + k < cnt; i++, k++) { + index = NPC_DFT_RULE_ID_MK(pcifunc, i); + ret = xa_insert(&npc_priv.xa_pf2dfl_rmap, index, + xa_mk_value(mcam_idx[k]), GFP_KERNEL); + if (ret) { + dev_err(rvu->dev, + "%s: Err to insert %s mcam idx to xarray pcifunc=%#x\n", + __func__, npc_dft_rule_name[i], + pcifunc); + for (int p = NPC_DFT_RULE_START_ID; p < i; p++) { + index = NPC_DFT_RULE_ID_MK(pcifunc, p); + xa_erase(&npc_priv.xa_pf2dfl_rmap, index); + } + goto err; + } + } + +done: + return 0; + +err: + free_req.hdr.pcifunc = pcifunc; + free_req.all = 1; + ret = rvu_mbox_handler_npc_mcam_free_entry(rvu, &free_req, &free_rsp); + if (ret) + dev_err(rvu->dev, + "%s: Error deleting default entries (pcifunc=%#x\n", + __func__, pcifunc); + + return -EFAULT; +} + +static int npc_priv_init(struct rvu *rvu) +{ + struct npc_mcam *mcam = &rvu->hw->mcam; + int blkaddr, num_banks, bank_depth; + int num_subbanks, subbank_depth; + u64 npc_const1, npc_const2 = 0; + struct npc_subbank *sb; + u64 cfg; + int i; + + blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); + if (blkaddr < 0) { + dev_err(rvu->dev, "%s: NPC block not implemented\n", + __func__); + return -ENODEV; + } + + npc_const1 = rvu_read64(rvu, blkaddr, NPC_AF_CONST1); + if (npc_const1 & BIT_ULL(63)) + npc_const2 = rvu_read64(rvu, blkaddr, NPC_AF_CONST2); + + num_banks = mcam->banks; + bank_depth = mcam->banksize; + + num_subbanks = FIELD_GET(GENMASK_ULL(39, 32), npc_const2); + if (!num_subbanks) { + dev_err(rvu->dev, "Number of subbanks is zero\n"); + return -EFAULT; + } + + if (num_subbanks & (num_subbanks - 1)) { + dev_err(rvu->dev, + "subbanks cnt(%u) should be a power of 2\n", + num_subbanks); + return -EINVAL; + } + + npc_priv.num_subbanks = num_subbanks; + + subbank_depth = bank_depth / num_subbanks; + + npc_priv.bank_depth = bank_depth; + npc_priv.subbank_depth = subbank_depth; + + /* Get kex configured key size */ + cfg = rvu_read64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(0)); + npc_priv.kw = FIELD_GET(GENMASK_ULL(34, 32), cfg); + + dev_info(rvu->dev, + "banks=%u depth=%u, subbanks=%u depth=%u, key type=%s\n", + num_banks, bank_depth, num_subbanks, subbank_depth, + npc_kw_name[npc_priv.kw]); + + npc_priv.sb = kcalloc(num_subbanks, sizeof(struct npc_subbank), + GFP_KERNEL); + if (!npc_priv.sb) + return -ENOMEM; + + xa_init_flags(&npc_priv.xa_sb_used, XA_FLAGS_ALLOC); + xa_init_flags(&npc_priv.xa_sb_free, XA_FLAGS_ALLOC); + xa_init_flags(&npc_priv.xa_idx2pf_map, XA_FLAGS_ALLOC); + xa_init_flags(&npc_priv.xa_pf_map, XA_FLAGS_ALLOC); + xa_init_flags(&npc_priv.xa_pf2dfl_rmap, XA_FLAGS_ALLOC); + xa_init_flags(&npc_priv.xa_idx2vidx_map, XA_FLAGS_ALLOC); + xa_init_flags(&npc_priv.xa_vidx2idx_map, XA_FLAGS_ALLOC); + + if (npc_create_srch_order(num_subbanks)) + goto fail1; + + npc_populate_restricted_idxs(num_subbanks); + + /* Initialize subbanks */ + for (i = 0, sb = npc_priv.sb; i < num_subbanks; i++, sb++) + npc_subbank_init(rvu, sb, i); + + /* Get number of pcifuncs in the system */ + npc_priv.pf_cnt = npc_pcifunc_map_create(rvu); + npc_priv.xa_pf2idx_map = kcalloc(npc_priv.pf_cnt, + sizeof(struct xarray), + GFP_KERNEL); + if (!npc_priv.xa_pf2idx_map) + goto fail2; + + for (i = 0; i < npc_priv.pf_cnt; i++) + xa_init_flags(&npc_priv.xa_pf2idx_map[i], XA_FLAGS_ALLOC); + + INIT_LIST_HEAD(&npc_priv.defrag_lh); + mutex_init(&npc_priv.lock); + + return 0; + +fail2: + kfree(subbank_srch_order); + subbank_srch_order = NULL; + +fail1: + xa_destroy(&npc_priv.xa_sb_used); + xa_destroy(&npc_priv.xa_sb_free); + xa_destroy(&npc_priv.xa_idx2pf_map); + xa_destroy(&npc_priv.xa_pf_map); + xa_destroy(&npc_priv.xa_pf2dfl_rmap); + xa_destroy(&npc_priv.xa_idx2vidx_map); + xa_destroy(&npc_priv.xa_vidx2idx_map); + kfree(npc_priv.sb); + npc_priv.sb = NULL; + return -ENOMEM; +} + +void npc_cn20k_deinit(struct rvu *rvu) +{ + int i; + + xa_destroy(&npc_priv.xa_sb_used); + xa_destroy(&npc_priv.xa_sb_free); + xa_destroy(&npc_priv.xa_idx2pf_map); + xa_destroy(&npc_priv.xa_pf_map); + xa_destroy(&npc_priv.xa_pf2dfl_rmap); + xa_destroy(&npc_priv.xa_idx2vidx_map); + xa_destroy(&npc_priv.xa_vidx2idx_map); + + for (i = 0; i < npc_priv.pf_cnt; i++) + xa_destroy(&npc_priv.xa_pf2idx_map[i]); + + kfree(npc_priv.xa_pf2idx_map); + /* No need to destroy mutex lock as it is + * part of subbank structure + */ + kfree(npc_priv.sb); + kfree(subbank_srch_order); +} + +static int npc_setup_mcam_section(struct rvu *rvu, int key_type) +{ + int blkaddr, sec; + + blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); + if (blkaddr < 0) { + dev_err(rvu->dev, "%s: NPC block not implemented\n", __func__); + return -ENODEV; + } + + for (sec = 0; sec < npc_priv.num_subbanks; sec++) + rvu_write64(rvu, blkaddr, + NPC_AF_MCAM_SECTIONX_CFG_EXT(sec), key_type); + + return 0; +} + +int npc_cn20k_init(struct rvu *rvu) +{ + int err; + + err = npc_priv_init(rvu); + if (err) { + dev_err(rvu->dev, "%s: Error to init\n", + __func__); + return err; + } + + err = npc_setup_mcam_section(rvu, NPC_MCAM_KEY_X2); + if (err) { + dev_err(rvu->dev, "%s: mcam section configuration failure\n", + __func__); + return err; + } + + npc_priv.init_done = true; + + return 0; +} diff --git a/drivers/net/ethernet/marvell/octeontx2/af/cn20k/npc.h b/drivers/net/ethernet/marvell/octeontx2/af/cn20k/npc.h new file mode 100644 index 000000000000..815d0b257a7e --- /dev/null +++ b/drivers/net/ethernet/marvell/octeontx2/af/cn20k/npc.h @@ -0,0 +1,340 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Marvell RVU Admin Function driver + * + * Copyright (C) 2026 Marvell. + * + */ + +#ifndef NPC_CN20K_H +#define NPC_CN20K_H + +#define MKEX_CN20K_SIGN 0x19bbfdbd160 + +#define MAX_NUM_BANKS 2 +#define MAX_NUM_SUB_BANKS 32 +#define MAX_SUBBANK_DEPTH 256 + +/* strtoull of "mkexprof" with base:36 */ +#define MKEX_END_SIGN 0xdeadbeef + +#define NPC_CN20K_BYTESM GENMASK_ULL(18, 16) +#define NPC_CN20K_PARSE_NIBBLE GENMASK_ULL(22, 0) +#define NPC_CN20K_TOTAL_NIBBLE 23 + +#define CN20K_SET_EXTR_LT(intf, extr, ltype, cfg) \ + rvu_write64(rvu, BLKADDR_NPC, \ + NPC_AF_INTFX_EXTRACTORX_LTX_CFG(intf, extr, ltype), cfg) + +#define CN20K_GET_KEX_CFG(intf) \ + rvu_read64(rvu, BLKADDR_NPC, NPC_AF_INTFX_KEX_CFG(intf)) + +#define CN20K_GET_EXTR_LID(intf, extr) \ + rvu_read64(rvu, BLKADDR_NPC, \ + NPC_AF_INTFX_EXTRACTORX_CFG(intf, extr)) + +#define CN20K_SET_EXTR_LT(intf, extr, ltype, cfg) \ + rvu_write64(rvu, BLKADDR_NPC, \ + NPC_AF_INTFX_EXTRACTORX_LTX_CFG(intf, extr, ltype), cfg) + +#define CN20K_GET_EXTR_LT(intf, extr, ltype) \ + rvu_read64(rvu, BLKADDR_NPC, \ + NPC_AF_INTFX_EXTRACTORX_LTX_CFG(intf, extr, ltype)) + +/* NPC_PARSE_KEX_S nibble definitions for each field */ +#define NPC_CN20K_PARSE_NIBBLE_CHAN GENMASK_ULL(2, 0) +#define NPC_CN20K_PARSE_NIBBLE_ERRLEV BIT_ULL(3) +#define NPC_CN20K_PARSE_NIBBLE_ERRCODE GENMASK_ULL(5, 4) +#define NPC_CN20K_PARSE_NIBBLE_L2L3_BCAST BIT_ULL(6) +#define NPC_CN20K_PARSE_NIBBLE_LA_FLAGS BIT_ULL(7) +#define NPC_CN20K_PARSE_NIBBLE_LA_LTYPE BIT_ULL(8) +#define NPC_CN20K_PARSE_NIBBLE_LB_FLAGS BIT_ULL(9) +#define NPC_CN20K_PARSE_NIBBLE_LB_LTYPE BIT_ULL(10) +#define NPC_CN20K_PARSE_NIBBLE_LC_FLAGS BIT_ULL(11) +#define NPC_CN20K_PARSE_NIBBLE_LC_LTYPE BIT_ULL(12) +#define NPC_CN20K_PARSE_NIBBLE_LD_FLAGS BIT_ULL(13) +#define NPC_CN20K_PARSE_NIBBLE_LD_LTYPE BIT_ULL(14) +#define NPC_CN20K_PARSE_NIBBLE_LE_FLAGS BIT_ULL(15) +#define NPC_CN20K_PARSE_NIBBLE_LE_LTYPE BIT_ULL(16) +#define NPC_CN20K_PARSE_NIBBLE_LF_FLAGS BIT_ULL(17) +#define NPC_CN20K_PARSE_NIBBLE_LF_LTYPE BIT_ULL(18) +#define NPC_CN20K_PARSE_NIBBLE_LG_FLAGS BIT_ULL(19) +#define NPC_CN20K_PARSE_NIBBLE_LG_LTYPE BIT_ULL(20) +#define NPC_CN20K_PARSE_NIBBLE_LH_FLAGS BIT_ULL(21) +#define NPC_CN20K_PARSE_NIBBLE_LH_LTYPE BIT_ULL(22) + +/* Rx parse key extract nibble enable */ +#define NPC_CN20K_PARSE_NIBBLE_INTF_RX (NPC_CN20K_PARSE_NIBBLE_CHAN | \ + NPC_CN20K_PARSE_NIBBLE_L2L3_BCAST | \ + NPC_CN20K_PARSE_NIBBLE_LA_LTYPE | \ + NPC_CN20K_PARSE_NIBBLE_LB_LTYPE | \ + NPC_CN20K_PARSE_NIBBLE_LC_FLAGS | \ + NPC_CN20K_PARSE_NIBBLE_LC_LTYPE | \ + NPC_CN20K_PARSE_NIBBLE_LD_LTYPE | \ + NPC_CN20K_PARSE_NIBBLE_LE_LTYPE) + +/* Tx parse key extract nibble enable */ +#define NPC_CN20K_PARSE_NIBBLE_INTF_TX (NPC_CN20K_PARSE_NIBBLE_LA_LTYPE | \ + NPC_CN20K_PARSE_NIBBLE_LB_LTYPE | \ + NPC_CN20K_PARSE_NIBBLE_LC_LTYPE | \ + NPC_CN20K_PARSE_NIBBLE_LD_LTYPE | \ + NPC_CN20K_PARSE_NIBBLE_LE_LTYPE) + +/** + * enum npc_subbank_flag - NPC subbank status + * + * subbank flag indicates whether the subbank is free + * or used. + * + * @NPC_SUBBANK_FLAG_UNINIT: Subbank is not initialized. + * @NPC_SUBBANK_FLAG_FREE: Subbank is free. + * @NPC_SUBBANK_FLAG_USED: Subbank is used. + */ +enum npc_subbank_flag { + NPC_SUBBANK_FLAG_UNINIT, + NPC_SUBBANK_FLAG_FREE = BIT(0), + NPC_SUBBANK_FLAG_USED = BIT(1), +}; + +/** + * enum npc_dft_rule_id - Default rule type + * + * Mcam default rule type. + * + * @NPC_DFT_RULE_START_ID: Not used + * @NPC_DFT_RULE_PROMISC_ID: promiscuous rule + * @NPC_DFT_RULE_MCAST_ID: multicast rule + * @NPC_DFT_RULE_BCAST_ID: broadcast rule + * @NPC_DFT_RULE_UCAST_ID: unicast rule + * @NPC_DFT_RULE_MAX_ID: Maximum index. + */ +enum npc_dft_rule_id { + NPC_DFT_RULE_START_ID = 1, + NPC_DFT_RULE_PROMISC_ID = NPC_DFT_RULE_START_ID, + NPC_DFT_RULE_MCAST_ID, + NPC_DFT_RULE_BCAST_ID, + NPC_DFT_RULE_UCAST_ID, + NPC_DFT_RULE_MAX_ID, +}; + +/** + * struct npc_subbank - Subbank fields. + * @b0b: Subbanks bottom index for bank0 + * @b1b: Subbanks bottom index for bank1 + * @b0t: Subbanks top index for bank0 + * @b1t: Subbanks top index for bank1 + * @flags: Subbank flags + * @lock: Mutex lock for flags and rsrc mofiication + * @b0map: Bitmap map for bank0 indexes + * @b1map: Bitmap map for bank1 indexes + * @idx: Subbank index + * @arr_idx: Index to the free array or used array + * @free_cnt: Number of free slots in the subbank. + * @key_type: X4 or X2 subbank. + * + * MCAM resource is divided horizontally into multiple subbanks and + * Resource allocation from each subbank is managed by this data + * structure. + */ +struct npc_subbank { + u16 b0t, b0b, b1t, b1b; + enum npc_subbank_flag flags; + struct mutex lock; /* Protect subbank resources */ + DECLARE_BITMAP(b0map, MAX_SUBBANK_DEPTH); + DECLARE_BITMAP(b1map, MAX_SUBBANK_DEPTH); + u16 idx; + u16 arr_idx; + u16 free_cnt; + u8 key_type; +}; + +/** + * struct npc_defrag_show_node - Defragmentation show node + * @old_midx: Old mcam index. + * @new_midx: New mcam index. + * @vidx: Virtual index + * @list: Linked list of these nodes + * + * This structure holds information on last defragmentation + * executed on mcam resource. + */ +struct npc_defrag_show_node { + u16 old_midx; + u16 new_midx; + u16 vidx; + struct list_head list; +}; + +/** + * struct npc_priv_t - NPC private structure. + * @bank_depth: Total entries in each bank. + * @num_banks: Number of banks. + * @num_subbanks: Number of subbanks. + * @subbank_depth: Depth of subbank. + * @kw: Kex configured key type. + * @sb: Subbank array. + * @xa_sb_used: Array of used subbanks. + * @xa_sb_free: Array of free subbanks. + * @xa_pf2idx_map: PF to mcam index map. + * @xa_idx2pf_map: Mcam index to PF map. + * @xa_pf_map: Pcifunc to index map. + * @pf_cnt: Number of PFs. + * @init_done: Indicates MCAM initialization is done. + * @xa_pf2dfl_rmap: PF to default rule index map. + * @xa_idx2vidx_map: Mcam index to virtual index map. + * @xa_vidx2idx_map: virtual index to mcam index map. + * @defrag_lh: Defrag list head. + * @lock: Lock for defrag list + * + * This structure is populated during probing time by reading + * HW csr registers. + */ +struct npc_priv_t { + int bank_depth; + const int num_banks; + int num_subbanks; + int subbank_depth; + u8 kw; + struct npc_subbank *sb; + struct xarray xa_sb_used; + struct xarray xa_sb_free; + struct xarray *xa_pf2idx_map; + struct xarray xa_idx2pf_map; + struct xarray xa_pf_map; + struct xarray xa_pf2dfl_rmap; + struct xarray xa_idx2vidx_map; + struct xarray xa_vidx2idx_map; + struct list_head defrag_lh; + struct mutex lock; /* protect defrag nodes */ + int pf_cnt; + bool init_done; +}; + +struct npc_kpm_action0 { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 rsvd_63_57 : 7; + u64 byp_count : 3; + u64 capture_ena : 1; + u64 parse_done : 1; + u64 next_state : 8; + u64 rsvd_43 : 1; + u64 capture_lid : 3; + u64 capture_ltype : 4; + u64 rsvd_32_35 : 4; + u64 capture_flags : 4; + u64 ptr_advance : 8; + u64 var_len_offset : 8; + u64 var_len_mask : 8; + u64 var_len_right : 1; + u64 var_len_shift : 3; +#else + u64 var_len_shift : 3; + u64 var_len_right : 1; + u64 var_len_mask : 8; + u64 var_len_offset : 8; + u64 ptr_advance : 8; + u64 capture_flags : 4; + u64 rsvd_32_35 : 4; + u64 capture_ltype : 4; + u64 capture_lid : 3; + u64 rsvd_43 : 1; + u64 next_state : 8; + u64 parse_done : 1; + u64 capture_ena : 1; + u64 byp_count : 3; + u64 rsvd_63_57 : 7; +#endif +}; + +struct npc_mcam_kex_extr { + /* MKEX Profle Header */ + u64 mkex_sign; /* "mcam-kex-profile" (8 bytes/ASCII characters) */ + u8 name[MKEX_NAME_LEN]; /* MKEX Profile name */ + u64 cpu_model; /* Format as profiled by CPU hardware */ + u64 kpu_version; /* KPU firmware/profile version */ + u64 reserved; /* Reserved for extension */ + + /* MKEX Profle Data */ + u64 keyx_cfg[NPC_MAX_INTF]; /* NPC_AF_INTF(0..1)_KEX_CFG */ +#define NPC_MAX_EXTRACTOR 24 + /* MKEX Extractor data */ + u64 intf_extr_lid[NPC_MAX_INTF][NPC_MAX_EXTRACTOR]; + /* KEX configuration per extractor */ + u64 intf_extr_lt[NPC_MAX_INTF][NPC_MAX_EXTRACTOR][NPC_MAX_LT]; +} __packed; + +struct npc_cn20k_kpu_profile_fwdata { +#define KPU_SIGN 0x00666f727075706b +#define KPU_NAME_LEN 32 + /* Maximum number of custom KPU entries supported by + * the built-in profile. + */ +#define KPU_CN20K_MAX_CST_ENT 6 + /* KPU Profle Header */ + __le64 signature; /* "kpuprof\0" (8 bytes/ASCII characters) */ + u8 name[KPU_NAME_LEN]; /* KPU Profile name */ + __le64 version; /* KPU profile version */ + u8 kpus; + u8 reserved[7]; + + /* Default MKEX profile to be used with this KPU profile. May be + * overridden with mkex_profile module parameter. + * Format is same as for the MKEX profile to streamline processing. + */ + struct npc_mcam_kex_extr mkex; + /* LTYPE values for specific HW offloaded protocols. */ + struct npc_lt_def_cfg lt_def; + /* Dynamically sized data: + * Custom KPU CAM and ACTION configuration entries. + * struct npc_kpu_fwdata kpu[kpus]; + */ + u8 data[]; +} __packed; + +struct rvu; + +struct npc_priv_t *npc_priv_get(void); +int npc_cn20k_init(struct rvu *rvu); +void npc_cn20k_deinit(struct rvu *rvu); + +void npc_cn20k_subbank_calc_free(struct rvu *rvu, int *x2_free, + int *x4_free, int *sb_free); + +int npc_cn20k_ref_idx_alloc(struct rvu *rvu, int pcifunc, int key_type, + int prio, u16 *mcam_idx, int ref, int limit, + bool contig, int count, bool virt); +int npc_cn20k_idx_free(struct rvu *rvu, u16 *mcam_idx, int count); +void npc_cn20k_parser_profile_init(struct rvu *rvu, int blkaddr); +struct npc_mcam_kex_extr *npc_mkex_extr_default_get(void); +void npc_cn20k_load_mkex_profile(struct rvu *rvu, int blkaddr, + const char *mkex_profile); +int npc_cn20k_apply_custom_kpu(struct rvu *rvu, + struct npc_kpu_profile_adapter *profile); + +void +npc_cn20k_update_action_entries_n_flags(struct rvu *rvu, + struct npc_kpu_profile_adapter *pfl); + +int npc_cn20k_dft_rules_alloc(struct rvu *rvu, u16 pcifunc); +void npc_cn20k_dft_rules_free(struct rvu *rvu, u16 pcifunc); + +int npc_cn20k_dft_rules_idx_get(struct rvu *rvu, u16 pcifunc, u16 *bcast, + u16 *mcast, u16 *promisc, u16 *ucast); + +void npc_cn20k_config_mcam_entry(struct rvu *rvu, int blkaddr, int index, + u8 intf, struct cn20k_mcam_entry *entry, + bool enable, u8 hw_prio, u8 req_kw_type); +void npc_cn20k_enable_mcam_entry(struct rvu *rvu, int blkaddr, + int index, bool enable); +void npc_cn20k_copy_mcam_entry(struct rvu *rvu, int blkaddr, + u16 src, u16 dest); +void npc_cn20k_read_mcam_entry(struct rvu *rvu, int blkaddr, u16 index, + struct cn20k_mcam_entry *entry, u8 *intf, + u8 *ena, u8 *hw_prio); +void npc_cn20k_clear_mcam_entry(struct rvu *rvu, int blkaddr, + int bank, int index); +int npc_mcam_idx_2_key_type(struct rvu *rvu, u16 mcam_idx, u8 *key_type); +u16 npc_cn20k_vidx2idx(u16 index); +u16 npc_cn20k_idx2vidx(u16 idx); +int npc_cn20k_defrag(struct rvu *rvu); + +#endif /* NPC_CN20K_H */ diff --git a/drivers/net/ethernet/marvell/octeontx2/af/cn20k/reg.h b/drivers/net/ethernet/marvell/octeontx2/af/cn20k/reg.h index affb39803120..8bfaa507ee50 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/cn20k/reg.h +++ b/drivers/net/ethernet/marvell/octeontx2/af/cn20k/reg.h @@ -78,4 +78,69 @@ #define RVU_MBOX_VF_INT_ENA_W1C (0x38) #define RVU_MBOX_VF_VFAF_TRIGX(a) (0x2000 | (a) << 3) + +#define NIX_GINT_INT (0x200) +#define NIX_GINT_INT_W1S (0x208) + +#define ALTAF_FLR BIT_ULL(0) +#define ALTAF_RDY BIT_ULL(1) + +/* NPC registers */ +#define NPC_AF_INTFX_EXTRACTORX_CFG(a, b) \ + (0x20c000ull | (a) << 16 | (b) << 8) +#define NPC_AF_INTFX_EXTRACTORX_LTX_CFG(a, b, c) \ + (0x204000ull | (a) << 16 | (b) << 8 | (c) << 3) +#define NPC_AF_KPMX_ENTRYX_CAMX(a, b, c) \ + (0x20000ull | (a) << 12 | (b) << 3 | (c) << 16) +#define NPC_AF_KPMX_ENTRYX_ACTION0(a, b) \ + (0x40000ull | (a) << 12 | (b) << 3) +#define NPC_AF_KPMX_ENTRYX_ACTION1(a, b) \ + (0x50000ull | (a) << 12 | (b) << 3) +#define NPC_AF_KPMX_ENTRY_DISX(a, b) (0x60000ull | (a) << 12 | (b) << 3) +#define NPC_AF_KPM_PASS2_CFG 0x10210 +#define NPC_AF_KPMX_PASS2_OFFSET(a) (0x60040ull | (a) << 12) +#define NPC_AF_MCAM_SECTIONX_CFG_EXT(a) (0xf000000ull | (a) << 3) + +#define NPC_AF_CN20K_MCAMEX_BANKX_CAMX_INTF_EXT(a, b, c) ({ \ + u64 offset; \ + offset = (0x8000000ull | (a) << 4 | (b) << 20 | (c) << 3); \ + offset; }) + +#define NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W0_EXT(a, b, c) ({ \ + u64 offset; \ + offset = (0x9000000ull | (a) << 4 | (b) << 20 | (c) << 3); \ + offset; }) + +#define NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W1_EXT(a, b, c) ({ \ + u64 offset; \ + offset = (0x9400000ull | (a) << 4 | (b) << 20 | (c) << 3); \ + offset; }) + +#define NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W2_EXT(a, b, c) ({ \ + u64 offset; \ + offset = (0x9800000ull | (a) << 4 | (b) << 20 | (c) << 3); \ + offset; }) + +#define NPC_AF_CN20K_MCAMEX_BANKX_CAMX_W3_EXT(a, b, c) ({ \ + u64 offset; \ + offset = (0x9c00000ull | (a) << 4 | (b) << 20 | (c) << 3); \ + offset; }) + +#define NPC_AF_CN20K_MCAMEX_BANKX_CFG_EXT(a, b) ({ \ + u64 offset; \ + offset = (0xa000000ull | (a) << 4 | (b) << 20); \ + offset; }) + +#define NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(a, b, c) ({ \ + u64 offset; \ + offset = (0xc000000ull | (a) << 4 | (b) << 20 | (c) << 22); \ + offset; }) + +#define NPC_AF_INTFX_MISS_ACTX(a, b) (0xf003000 | (a) << 6 | (b) << 4) + +#define NPC_AF_CN20K_MCAMEX_BANKX_STAT_EXT(a, b) ({ \ + u64 offset; \ + offset = (0xb000000ull | (a) << 4 | (b) << 20); \ + offset; }) + #endif /* RVU_MBOX_REG_H */ diff --git a/drivers/net/ethernet/marvell/octeontx2/af/common.h b/drivers/net/ethernet/marvell/octeontx2/af/common.h index 8a08bebf08c2..779413a383b7 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/common.h +++ b/drivers/net/ethernet/marvell/octeontx2/af/common.h @@ -177,10 +177,6 @@ enum nix_scheduler { #define NIX_TX_ACTIONOP_MCAST (0x3ull) #define NIX_TX_ACTIONOP_DROP_VIOL (0x5ull) -#define NPC_MCAM_KEY_X1 0 -#define NPC_MCAM_KEY_X2 1 -#define NPC_MCAM_KEY_X4 2 - #define NIX_INTFX_RX(a) (0x0ull | (a) << 1) #define NIX_INTFX_TX(a) (0x1ull | (a) << 1) diff --git a/drivers/net/ethernet/marvell/octeontx2/af/mbox.h b/drivers/net/ethernet/marvell/octeontx2/af/mbox.h index a3e273126e4e..dc42c81c0942 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/mbox.h +++ b/drivers/net/ethernet/marvell/octeontx2/af/mbox.h @@ -53,6 +53,14 @@ #define MBOX_DIR_VFPF_UP 7 /* VF replies to PF */ enum { + NPC_MCAM_KEY_X1 = 0, + NPC_MCAM_KEY_DYN = NPC_MCAM_KEY_X1, + NPC_MCAM_KEY_X2, + NPC_MCAM_KEY_X4, + NPC_MCAM_KEY_MAX, +}; + +enum { TYPE_AFVF, TYPE_AFPF, }; @@ -275,6 +283,33 @@ M(NPC_GET_FIELD_HASH_INFO, 0x6013, npc_get_field_hash_info, M(NPC_GET_FIELD_STATUS, 0x6014, npc_get_field_status, \ npc_get_field_status_req, \ npc_get_field_status_rsp) \ +M(NPC_CN20K_MCAM_GET_FREE_COUNT, 0x6015, npc_cn20k_get_fcnt, \ + msg_req, npc_cn20k_get_fcnt_rsp) \ +M(NPC_CN20K_GET_KEX_CFG, 0x6016, npc_cn20k_get_kex_cfg, \ + msg_req, npc_cn20k_get_kex_cfg_rsp) \ +M(NPC_CN20K_MCAM_WRITE_ENTRY, 0x6017, npc_cn20k_mcam_write_entry, \ + npc_cn20k_mcam_write_entry_req, msg_rsp) \ +M(NPC_CN20K_MCAM_ALLOC_AND_WRITE_ENTRY, 0x6018, \ +npc_cn20k_mcam_alloc_and_write_entry, \ + npc_cn20k_mcam_alloc_and_write_entry_req, \ + npc_mcam_alloc_and_write_entry_rsp) \ +M(NPC_CN20K_MCAM_READ_ENTRY, 0x6019, npc_cn20k_mcam_read_entry, \ + npc_mcam_read_entry_req, \ + npc_cn20k_mcam_read_entry_rsp) \ +M(NPC_CN20K_MCAM_READ_BASE_RULE, 0x601a, npc_cn20k_read_base_steer_rule, \ + msg_req, npc_cn20k_mcam_read_base_rule_rsp) \ +M(NPC_MCAM_DEFRAG, 0x601b, npc_defrag, \ + msg_req, \ + msg_rsp) \ +M(NPC_MCAM_GET_NUM_KWS, 0x601c, npc_get_num_kws, \ + npc_get_num_kws_req, \ + npc_get_num_kws_rsp) \ +M(NPC_MCAM_GET_DFT_RL_IDXS, 0x601d, npc_get_dft_rl_idxs, \ + msg_req, \ + npc_get_dft_rl_idxs_rsp)\ +M(NPC_MCAM_GET_NPC_PFL_INFO, 0x601e, npc_get_pfl_info, \ + msg_req, \ + npc_get_pfl_info_rsp) \ /* NIX mbox IDs (range 0x8000 - 0xFFFF) */ \ M(NIX_LF_ALLOC, 0x8000, nix_lf_alloc, \ nix_lf_alloc_req, nix_lf_alloc_rsp) \ @@ -1527,9 +1562,11 @@ struct npc_mcam_alloc_entry_req { #define NPC_MCAM_ANY_PRIO 0 #define NPC_MCAM_LOWER_PRIO 1 #define NPC_MCAM_HIGHER_PRIO 2 - u8 priority; /* Lower or higher w.r.t ref_entry */ + u8 ref_prio; /* Lower or higher w.r.t ref_entry */ u16 ref_entry; u16 count; /* Number of entries requested */ + u8 kw_type; /* entry key type, valid for cn20k */ + u8 virt; /* Request virtual index */ }; struct npc_mcam_alloc_entry_rsp { @@ -1548,14 +1585,47 @@ struct npc_mcam_free_entry_req { u8 all; /* If all entries allocated to this PFVF to be freed */ }; +struct mcam_entry_mdata { + u64 *kw; + u64 *kw_mask; + u64 *action; + u64 *vtag_action; + u8 max_kw; +}; + +enum npc_kws_in_key_sz { + NPC_KWS_IN_KEY_SZ_7 = 7, + NPC_KWS_IN_KEY_SZ_8 = 8, + NPC_KWS_IN_KEY_SZ_MAX, +}; + struct mcam_entry { -#define NPC_MAX_KWS_IN_KEY 7 /* Number of keywords in max keywidth */ - u64 kw[NPC_MAX_KWS_IN_KEY]; - u64 kw_mask[NPC_MAX_KWS_IN_KEY]; + u64 kw[NPC_KWS_IN_KEY_SZ_7]; + u64 kw_mask[NPC_KWS_IN_KEY_SZ_7]; u64 action; u64 vtag_action; }; +struct cn20k_mcam_entry { + u64 kw[NPC_KWS_IN_KEY_SZ_8]; + u64 kw_mask[NPC_KWS_IN_KEY_SZ_8]; + u64 action; + u64 vtag_action; + u64 action2; +}; + +struct npc_cn20k_mcam_write_entry_req { + struct mbox_msghdr hdr; + struct cn20k_mcam_entry entry_data; + u16 entry; /* MCAM entry to write this match key */ + u16 cntr; /* Counter for this MCAM entry */ + u8 intf; /* Rx or Tx interface */ + u8 enable_entry;/* Enable this MCAM entry ? */ + u8 hw_prio; /* hardware priority, valid for cn20k */ + u8 req_kw_type; /* Type of kw which should be written */ + u64 reserved; /* reserved for future use */ +}; + struct npc_mcam_write_entry_req { struct mbox_msghdr hdr; struct mcam_entry entry_data; @@ -1564,6 +1634,8 @@ struct npc_mcam_write_entry_req { u8 intf; /* Rx or Tx interface */ u8 enable_entry;/* Enable this MCAM entry ? */ u8 set_cntr; /* Set counter for this entry ? */ + u8 hw_prio; /* hardware priority, valid for cn20k */ + u64 reserved; /* reserved for future use */ }; /* Enable/Disable a given entry */ @@ -1622,12 +1694,38 @@ struct npc_mcam_alloc_and_write_entry_req { struct mbox_msghdr hdr; struct mcam_entry entry_data; u16 ref_entry; - u8 priority; /* Lower or higher w.r.t ref_entry */ + u8 ref_prio; /* Lower or higher w.r.t ref_entry */ u8 intf; /* Rx or Tx interface */ u8 enable_entry;/* Enable this MCAM entry ? */ u8 alloc_cntr; /* Allocate counter and map ? */ }; +struct npc_cn20k_mcam_alloc_and_write_entry_req { + struct mbox_msghdr hdr; + struct cn20k_mcam_entry entry_data; + u16 ref_entry; + u8 ref_prio; /* Lower or higher w.r.t ref_entry */ + u8 intf; /* Rx or Tx interface */ + u8 enable_entry;/* Enable this MCAM entry ? */ + u8 hw_prio; /* hardware priority, valid for cn20k */ + u8 virt; /* Allocate virtual index */ + u8 req_kw_type; /* Key type to be written */ + u16 reserved[4]; /* reserved for future use */ +}; + +struct npc_cn20k_mcam_read_entry_rsp { + struct mbox_msghdr hdr; + struct cn20k_mcam_entry entry_data; + u8 intf; + u8 enable; + u8 hw_prio; /* valid for cn20k */ +}; + +struct npc_cn20k_mcam_read_base_rule_rsp { + struct mbox_msghdr hdr; + struct cn20k_mcam_entry entry; +}; + struct npc_mcam_alloc_and_write_entry_rsp { struct mbox_msghdr hdr; u16 entry; @@ -1653,6 +1751,19 @@ struct npc_get_kex_cfg_rsp { u8 mkex_pfl_name[MKEX_NAME_LEN]; }; +struct npc_cn20k_get_kex_cfg_rsp { + struct mbox_msghdr hdr; + u64 rx_keyx_cfg; /* NPC_AF_INTF(0)_KEX_CFG */ + u64 tx_keyx_cfg; /* NPC_AF_INTF(1)_KEX_CFG */ +#define NPC_MAX_EXTRACTOR 24 + /* MKEX Extractor data */ + u64 intf_extr_lid[NPC_MAX_INTF][NPC_MAX_EXTRACTOR]; + /* KEX configuration per extractor */ + u64 intf_extr_lt[NPC_MAX_INTF][NPC_MAX_EXTRACTOR][NPC_MAX_LT]; +#define MKEX_NAME_LEN 128 + u8 mkex_pfl_name[MKEX_NAME_LEN]; +}; + struct ptp_get_cap_rsp { struct mbox_msghdr hdr; #define PTP_CAP_HW_ATOMIC_UPDATE BIT_ULL(0) @@ -1765,11 +1876,53 @@ struct npc_install_flow_req { u8 vtag1_op; /* old counter value */ u16 cntr_val; + u8 hw_prio; + u8 req_kw_type; /* Key type to be written */ + u8 alloc_entry; /* only for cn20k */ +/* For now use any priority, once AF driver is changed to + * allocate least priority entry instead of mid zone then make + * NPC_MCAM_LEAST_PRIO as 3 + */ +#define NPC_MCAM_LEAST_PRIO NPC_MCAM_ANY_PRIO + u16 ref_prio; + u16 ref_entry; }; struct npc_install_flow_rsp { struct mbox_msghdr hdr; int counter; /* negative if no counter else counter number */ + u16 entry; + u8 kw_type; +}; + +struct npc_get_num_kws_req { + struct mbox_msghdr hdr; + struct npc_install_flow_req fl; + u32 rsvd[4]; +}; + +struct npc_get_num_kws_rsp { + struct mbox_msghdr hdr; + int kws; + u32 rsvd[4]; +}; + +struct npc_get_dft_rl_idxs_rsp { + struct mbox_msghdr hdr; + u16 bcast; + u16 mcast; + u16 promisc; + u16 ucast; + u16 vf_ucast; + u16 rsvd[7]; +}; + +struct npc_get_pfl_info_rsp { + struct mbox_msghdr hdr; + u16 x4_slots; + u8 kw_type; + u8 rsvd1[3]; + u32 rsvd2[4]; }; struct npc_delete_flow_req { @@ -1795,6 +1948,15 @@ struct npc_mcam_read_entry_rsp { struct mcam_entry entry_data; u8 intf; u8 enable; + u8 hw_prio; /* valid for cn20k */ +}; + +/* Available entries to use */ +struct npc_cn20k_get_fcnt_rsp { + struct mbox_msghdr hdr; + int free_x2; + int free_x4; + int free_subbanks; }; struct npc_mcam_read_base_rule_rsp { diff --git a/drivers/net/ethernet/marvell/octeontx2/af/npc.h b/drivers/net/ethernet/marvell/octeontx2/af/npc.h index 6c3aca6f278d..cefc5d70f3e4 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/npc.h +++ b/drivers/net/ethernet/marvell/octeontx2/af/npc.h @@ -429,6 +429,7 @@ struct nix_rx_action { /* NPC_AF_INTFX_KEX_CFG field masks */ #define NPC_PARSE_NIBBLE GENMASK_ULL(30, 0) +#define NPC_TOTAL_NIBBLE 31 /* NPC_PARSE_KEX_S nibble definitions for each field */ #define NPC_PARSE_NIBBLE_CHAN GENMASK_ULL(2, 0) @@ -643,6 +644,7 @@ struct rvu_npc_mcam_rule { u16 chan; u16 chan_mask; u8 lxmb; + u8 hw_prio; }; #endif /* NPC_H */ diff --git a/drivers/net/ethernet/marvell/octeontx2/af/npc_profile.h b/drivers/net/ethernet/marvell/octeontx2/af/npc_profile.h index 41de72c8607f..db74f7fdf028 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/npc_profile.h +++ b/drivers/net/ethernet/marvell/octeontx2/af/npc_profile.h @@ -321,6 +321,18 @@ enum npc_kpu_lb_lflag { NPC_F_LB_L_FDSA, }; +enum npc_cn20k_kpu_lc_uflag { + NPC_CN20K_F_LC_U_MPLS_IN_IP = 0x20, + NPC_CN20K_F_LC_U_IP6_TUN_IP6 = 0x40, + NPC_CN20K_F_LC_U_IP6_MPLS_IN_IP = 0x80, +}; + +enum npc_cn20k_kpu_lc_lflag { + NPC_CN20K_F_LC_L_IP_FRAG = 2, + NPC_CN20K_F_LC_L_IP6_FRAG, + NPC_CN20K_F_LC_L_6TO4, +}; + enum npc_kpu_lc_uflag { NPC_F_LC_U_UNK_PROTO = 0x10, NPC_F_LC_U_IP_FRAG = 0x20, @@ -489,7 +501,7 @@ enum NPC_ERRLEV_E { 0, 0, 0, 0, \ } -static struct npc_kpu_profile_action ikpu_action_entries[] = { +static struct npc_kpu_profile_action ikpu_action_entries[] __maybe_unused = { { NPC_ERRLEV_RE, NPC_EC_NOERR, 12, 16, 20, 0, 0, @@ -1068,7 +1080,7 @@ static struct npc_kpu_profile_action ikpu_action_entries[] = { }, }; -static struct npc_kpu_profile_cam kpu1_cam_entries[] = { +static struct npc_kpu_profile_cam kpu1_cam_entries[] __maybe_unused = { NPC_KPU_NOP_CAM, NPC_KPU_NOP_CAM, NPC_KPU_NOP_CAM, @@ -1878,7 +1890,7 @@ static struct npc_kpu_profile_cam kpu1_cam_entries[] = { }, }; -static struct npc_kpu_profile_cam kpu2_cam_entries[] = { +static struct npc_kpu_profile_cam kpu2_cam_entries[] __maybe_unused = { NPC_KPU_NOP_CAM, NPC_KPU_NOP_CAM, NPC_KPU_NOP_CAM, @@ -2823,7 +2835,7 @@ static struct npc_kpu_profile_cam kpu2_cam_entries[] = { }, }; -static struct npc_kpu_profile_cam kpu3_cam_entries[] = { +static struct npc_kpu_profile_cam kpu3_cam_entries[] __maybe_unused = { NPC_KPU_NOP_CAM, NPC_KPU_NOP_CAM, NPC_KPU_NOP_CAM, @@ -3804,7 +3816,7 @@ static struct npc_kpu_profile_cam kpu3_cam_entries[] = { }, }; -static struct npc_kpu_profile_cam kpu4_cam_entries[] = { +static struct npc_kpu_profile_cam kpu4_cam_entries[] __maybe_unused = { NPC_KPU_NOP_CAM, NPC_KPU_NOP_CAM, NPC_KPU_NOP_CAM, @@ -4119,7 +4131,7 @@ static struct npc_kpu_profile_cam kpu4_cam_entries[] = { }, }; -static struct npc_kpu_profile_cam kpu5_cam_entries[] = { +static struct npc_kpu_profile_cam kpu5_cam_entries[] __maybe_unused = { NPC_KPU_NOP_CAM, NPC_KPU_NOP_CAM, NPC_KPU_NOP_CAM, @@ -5172,7 +5184,7 @@ static struct npc_kpu_profile_cam kpu5_cam_entries[] = { }, }; -static struct npc_kpu_profile_cam kpu6_cam_entries[] = { +static struct npc_kpu_profile_cam kpu6_cam_entries[] __maybe_unused = { NPC_KPU_NOP_CAM, NPC_KPU_NOP_CAM, NPC_KPU_NOP_CAM, @@ -5901,7 +5913,7 @@ static struct npc_kpu_profile_cam kpu6_cam_entries[] = { }, }; -static struct npc_kpu_profile_cam kpu7_cam_entries[] = { +static struct npc_kpu_profile_cam kpu7_cam_entries[] __maybe_unused = { NPC_KPU_NOP_CAM, NPC_KPU_NOP_CAM, NPC_KPU_NOP_CAM, @@ -6252,7 +6264,7 @@ static struct npc_kpu_profile_cam kpu7_cam_entries[] = { }, }; -static struct npc_kpu_profile_cam kpu8_cam_entries[] = { +static struct npc_kpu_profile_cam kpu8_cam_entries[] __maybe_unused = { NPC_KPU_NOP_CAM, NPC_KPU_NOP_CAM, NPC_KPU_NOP_CAM, @@ -7089,7 +7101,7 @@ static struct npc_kpu_profile_cam kpu8_cam_entries[] = { }, }; -static struct npc_kpu_profile_cam kpu9_cam_entries[] = { +static struct npc_kpu_profile_cam kpu9_cam_entries[] __maybe_unused = { NPC_KPU_NOP_CAM, NPC_KPU_NOP_CAM, NPC_KPU_NOP_CAM, @@ -7575,7 +7587,7 @@ static struct npc_kpu_profile_cam kpu9_cam_entries[] = { }, }; -static struct npc_kpu_profile_cam kpu10_cam_entries[] = { +static struct npc_kpu_profile_cam kpu10_cam_entries[] __maybe_unused = { NPC_KPU_NOP_CAM, NPC_KPU_NOP_CAM, NPC_KPU_NOP_CAM, @@ -7746,7 +7758,7 @@ static struct npc_kpu_profile_cam kpu10_cam_entries[] = { }, }; -static struct npc_kpu_profile_cam kpu11_cam_entries[] = { +static struct npc_kpu_profile_cam kpu11_cam_entries[] __maybe_unused = { NPC_KPU_NOP_CAM, NPC_KPU_NOP_CAM, NPC_KPU_NOP_CAM, @@ -8061,7 +8073,7 @@ static struct npc_kpu_profile_cam kpu11_cam_entries[] = { }, }; -static struct npc_kpu_profile_cam kpu12_cam_entries[] = { +static struct npc_kpu_profile_cam kpu12_cam_entries[] __maybe_unused = { NPC_KPU_NOP_CAM, NPC_KPU_NOP_CAM, NPC_KPU_NOP_CAM, @@ -8322,7 +8334,7 @@ static struct npc_kpu_profile_cam kpu12_cam_entries[] = { }, }; -static struct npc_kpu_profile_cam kpu13_cam_entries[] = { +static struct npc_kpu_profile_cam kpu13_cam_entries[] __maybe_unused = { NPC_KPU_NOP_CAM, NPC_KPU_NOP_CAM, NPC_KPU_NOP_CAM, @@ -8340,7 +8352,7 @@ static struct npc_kpu_profile_cam kpu13_cam_entries[] = { }, }; -static struct npc_kpu_profile_cam kpu14_cam_entries[] = { +static struct npc_kpu_profile_cam kpu14_cam_entries[] __maybe_unused = { NPC_KPU_NOP_CAM, NPC_KPU_NOP_CAM, NPC_KPU_NOP_CAM, @@ -8358,7 +8370,7 @@ static struct npc_kpu_profile_cam kpu14_cam_entries[] = { }, }; -static struct npc_kpu_profile_cam kpu15_cam_entries[] = { +static struct npc_kpu_profile_cam kpu15_cam_entries[] __maybe_unused = { NPC_KPU_NOP_CAM, NPC_KPU_NOP_CAM, NPC_KPU_NOP_CAM, @@ -8565,7 +8577,7 @@ static struct npc_kpu_profile_cam kpu15_cam_entries[] = { }, }; -static struct npc_kpu_profile_cam kpu16_cam_entries[] = { +static struct npc_kpu_profile_cam kpu16_cam_entries[] __maybe_unused = { NPC_KPU_NOP_CAM, NPC_KPU_NOP_CAM, NPC_KPU_NOP_CAM, @@ -8628,7 +8640,7 @@ static struct npc_kpu_profile_cam kpu16_cam_entries[] = { }, }; -static struct npc_kpu_profile_action kpu1_action_entries[] = { +static struct npc_kpu_profile_action kpu1_action_entries[] __maybe_unused = { NPC_KPU_NOP_ACTION, NPC_KPU_NOP_ACTION, NPC_KPU_NOP_ACTION, @@ -9368,7 +9380,7 @@ static struct npc_kpu_profile_action kpu1_action_entries[] = { }, }; -static struct npc_kpu_profile_action kpu2_action_entries[] = { +static struct npc_kpu_profile_action kpu2_action_entries[] __maybe_unused = { NPC_KPU_NOP_ACTION, NPC_KPU_NOP_ACTION, NPC_KPU_NOP_ACTION, @@ -10209,7 +10221,7 @@ static struct npc_kpu_profile_action kpu2_action_entries[] = { }, }; -static struct npc_kpu_profile_action kpu3_action_entries[] = { +static struct npc_kpu_profile_action kpu3_action_entries[] __maybe_unused = { NPC_KPU_NOP_ACTION, NPC_KPU_NOP_ACTION, NPC_KPU_NOP_ACTION, @@ -11082,7 +11094,7 @@ static struct npc_kpu_profile_action kpu3_action_entries[] = { }, }; -static struct npc_kpu_profile_action kpu4_action_entries[] = { +static struct npc_kpu_profile_action kpu4_action_entries[] __maybe_unused = { NPC_KPU_NOP_ACTION, NPC_KPU_NOP_ACTION, NPC_KPU_NOP_ACTION, @@ -11363,7 +11375,7 @@ static struct npc_kpu_profile_action kpu4_action_entries[] = { }, }; -static struct npc_kpu_profile_action kpu5_action_entries[] = { +static struct npc_kpu_profile_action kpu5_action_entries[] __maybe_unused = { NPC_KPU_NOP_ACTION, NPC_KPU_NOP_ACTION, NPC_KPU_NOP_ACTION, @@ -12300,7 +12312,7 @@ static struct npc_kpu_profile_action kpu5_action_entries[] = { }, }; -static struct npc_kpu_profile_action kpu6_action_entries[] = { +static struct npc_kpu_profile_action kpu6_action_entries[] __maybe_unused = { NPC_KPU_NOP_ACTION, NPC_KPU_NOP_ACTION, NPC_KPU_NOP_ACTION, @@ -12949,7 +12961,7 @@ static struct npc_kpu_profile_action kpu6_action_entries[] = { }, }; -static struct npc_kpu_profile_action kpu7_action_entries[] = { +static struct npc_kpu_profile_action kpu7_action_entries[] __maybe_unused = { NPC_KPU_NOP_ACTION, NPC_KPU_NOP_ACTION, NPC_KPU_NOP_ACTION, @@ -13262,7 +13274,7 @@ static struct npc_kpu_profile_action kpu7_action_entries[] = { }, }; -static struct npc_kpu_profile_action kpu8_action_entries[] = { +static struct npc_kpu_profile_action kpu8_action_entries[] __maybe_unused = { NPC_KPU_NOP_ACTION, NPC_KPU_NOP_ACTION, NPC_KPU_NOP_ACTION, @@ -14007,7 +14019,7 @@ static struct npc_kpu_profile_action kpu8_action_entries[] = { }, }; -static struct npc_kpu_profile_action kpu9_action_entries[] = { +static struct npc_kpu_profile_action kpu9_action_entries[] __maybe_unused = { NPC_KPU_NOP_ACTION, NPC_KPU_NOP_ACTION, NPC_KPU_NOP_ACTION, @@ -14440,7 +14452,7 @@ static struct npc_kpu_profile_action kpu9_action_entries[] = { }, }; -static struct npc_kpu_profile_action kpu10_action_entries[] = { +static struct npc_kpu_profile_action kpu10_action_entries[] __maybe_unused = { NPC_KPU_NOP_ACTION, NPC_KPU_NOP_ACTION, NPC_KPU_NOP_ACTION, @@ -14593,7 +14605,7 @@ static struct npc_kpu_profile_action kpu10_action_entries[] = { }, }; -static struct npc_kpu_profile_action kpu11_action_entries[] = { +static struct npc_kpu_profile_action kpu11_action_entries[] __maybe_unused = { NPC_KPU_NOP_ACTION, NPC_KPU_NOP_ACTION, NPC_KPU_NOP_ACTION, @@ -14874,7 +14886,7 @@ static struct npc_kpu_profile_action kpu11_action_entries[] = { }, }; -static struct npc_kpu_profile_action kpu12_action_entries[] = { +static struct npc_kpu_profile_action kpu12_action_entries[] __maybe_unused = { NPC_KPU_NOP_ACTION, NPC_KPU_NOP_ACTION, NPC_KPU_NOP_ACTION, @@ -15107,7 +15119,7 @@ static struct npc_kpu_profile_action kpu12_action_entries[] = { }, }; -static struct npc_kpu_profile_action kpu13_action_entries[] = { +static struct npc_kpu_profile_action kpu13_action_entries[] __maybe_unused = { NPC_KPU_NOP_ACTION, NPC_KPU_NOP_ACTION, NPC_KPU_NOP_ACTION, @@ -15124,7 +15136,7 @@ static struct npc_kpu_profile_action kpu13_action_entries[] = { }, }; -static struct npc_kpu_profile_action kpu14_action_entries[] = { +static struct npc_kpu_profile_action kpu14_action_entries[] __maybe_unused = { NPC_KPU_NOP_ACTION, NPC_KPU_NOP_ACTION, NPC_KPU_NOP_ACTION, @@ -15141,7 +15153,7 @@ static struct npc_kpu_profile_action kpu14_action_entries[] = { }, }; -static struct npc_kpu_profile_action kpu15_action_entries[] = { +static struct npc_kpu_profile_action kpu15_action_entries[] __maybe_unused = { NPC_KPU_NOP_ACTION, NPC_KPU_NOP_ACTION, NPC_KPU_NOP_ACTION, @@ -15326,7 +15338,7 @@ static struct npc_kpu_profile_action kpu15_action_entries[] = { }, }; -static struct npc_kpu_profile_action kpu16_action_entries[] = { +static struct npc_kpu_profile_action kpu16_action_entries[] __maybe_unused = { NPC_KPU_NOP_ACTION, NPC_KPU_NOP_ACTION, NPC_KPU_NOP_ACTION, @@ -15383,7 +15395,7 @@ static struct npc_kpu_profile_action kpu16_action_entries[] = { }, }; -static struct npc_kpu_profile npc_kpu_profiles[] = { +static struct npc_kpu_profile npc_kpu_profiles[] __maybe_unused = { { ARRAY_SIZE(kpu1_cam_entries), ARRAY_SIZE(kpu1_action_entries), @@ -15482,7 +15494,7 @@ static struct npc_kpu_profile npc_kpu_profiles[] = { }, }; -static struct npc_lt_def_cfg npc_lt_defaults = { +static struct npc_lt_def_cfg npc_lt_defaults __maybe_unused = { .rx_ol2 = { .lid = NPC_LID_LA, .ltype_match = NPC_LT_LA_ETHER, @@ -15604,7 +15616,7 @@ static struct npc_lt_def_cfg npc_lt_defaults = { }, }; -static struct npc_mcam_kex npc_mkex_default = { +static struct npc_mcam_kex npc_mkex_default __maybe_unused = { .mkex_sign = MKEX_SIGN, .name = "default", .kpu_version = NPC_KPU_PROFILE_VER, diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu.c index 5b1129558e8b..e40b79076358 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu.c @@ -22,6 +22,7 @@ #include "rvu_npc_hash.h" #include "cn20k/reg.h" #include "cn20k/api.h" +#include "cn20k/npc.h" #define DRV_NAME "rvu_af" #define DRV_STRING "Marvell OcteonTX2 RVU Admin Function Driver" @@ -599,6 +600,7 @@ check_pf: static int rvu_setup_msix_resources(struct rvu *rvu) { + struct altaf_intr_notify *altaf_intr_data; struct rvu_hwinfo *hw = rvu->hw; int pf, vf, numvfs, hwvf, err; int nvecs, offset, max_msix; @@ -705,7 +707,30 @@ setup_vfmsix: rvu->msix_base_iova = iova; rvu->msixtr_base_phy = phy_addr; + if (is_rvu_otx2(rvu) || is_cn20k(rvu->pdev)) + return 0; + + if (!rvu->fwdata) + goto fail; + + altaf_intr_data = &rvu->fwdata->altaf_intr_info; + if (altaf_intr_data->gint_paddr) { + iova = dma_map_resource(rvu->dev, altaf_intr_data->gint_paddr, + PCI_MSIX_ENTRY_SIZE, + DMA_BIDIRECTIONAL, 0); + + if (dma_mapping_error(rvu->dev, iova)) + goto fail; + + altaf_intr_data->gint_iova_addr = iova; + } + return 0; + +fail: + dma_unmap_resource(rvu->dev, phy_addr, max_msix * PCI_MSIX_ENTRY_SIZE, + DMA_BIDIRECTIONAL, 0); + return -EFAULT; } static void rvu_reset_msix(struct rvu *rvu) @@ -1396,7 +1421,6 @@ static void rvu_detach_block(struct rvu *rvu, int pcifunc, int blktype) if (blkaddr < 0) return; - block = &hw->block[blkaddr]; num_lfs = rvu_get_rsrc_mapcount(pfvf, block->addr); @@ -1467,6 +1491,13 @@ static int rvu_detach_rsrcs(struct rvu *rvu, struct rsrc_detach *detach, else if ((blkid == BLKADDR_CPT1) && !detach->cptlfs) continue; } + + if (detach_all || + (detach && (blkid == BLKADDR_NIX0 || + blkid == BLKADDR_NIX1) && + detach->nixlf)) + npc_cn20k_dft_rules_free(rvu, pcifunc); + rvu_detach_block(rvu, pcifunc, block->type); } @@ -1750,6 +1781,12 @@ int rvu_mbox_handler_attach_resources(struct rvu *rvu, err = rvu_attach_block(rvu, pcifunc, BLKTYPE_NIX, 1, attach); if (err) goto fail2; + + if (is_cn20k(rvu->pdev)) { + err = npc_cn20k_dft_rules_alloc(rvu, pcifunc); + if (err) + goto fail3; + } } if (attach->sso) { @@ -1763,7 +1800,7 @@ int rvu_mbox_handler_attach_resources(struct rvu *rvu, err = rvu_attach_block(rvu, pcifunc, BLKTYPE_SSO, attach->sso, attach); if (err) - goto fail3; + goto fail4; } if (attach->ssow) { @@ -1772,7 +1809,7 @@ int rvu_mbox_handler_attach_resources(struct rvu *rvu, err = rvu_attach_block(rvu, pcifunc, BLKTYPE_SSOW, attach->ssow, attach); if (err) - goto fail4; + goto fail5; } if (attach->timlfs) { @@ -1781,7 +1818,7 @@ int rvu_mbox_handler_attach_resources(struct rvu *rvu, err = rvu_attach_block(rvu, pcifunc, BLKTYPE_TIM, attach->timlfs, attach); if (err) - goto fail5; + goto fail6; } if (attach->cptlfs) { @@ -1791,24 +1828,28 @@ int rvu_mbox_handler_attach_resources(struct rvu *rvu, err = rvu_attach_block(rvu, pcifunc, BLKTYPE_CPT, attach->cptlfs, attach); if (err) - goto fail6; + goto fail7; } mutex_unlock(&rvu->rsrc_lock); return 0; -fail6: +fail7: if (attach->timlfs) rvu_detach_block(rvu, pcifunc, BLKTYPE_TIM); -fail5: +fail6: if (attach->ssow) rvu_detach_block(rvu, pcifunc, BLKTYPE_SSOW); -fail4: +fail5: if (attach->sso) rvu_detach_block(rvu, pcifunc, BLKTYPE_SSO); +fail4: + if (is_cn20k(rvu->pdev)) + npc_cn20k_dft_rules_free(rvu, pcifunc); + fail3: if (attach->nixlf) rvu_detach_block(rvu, pcifunc, BLKTYPE_NIX); @@ -2185,6 +2226,33 @@ int rvu_mbox_handler_ndc_sync_op(struct rvu *rvu, return 0; } +static void rvu_notify_altaf(struct rvu *rvu, u16 pcifunc, u64 op) +{ + int pf, vf; + + if (!rvu->fwdata) + return; + + if (op == ALTAF_FLR) { + pf = rvu_get_pf(rvu->pdev, pcifunc); + set_bit(pf, rvu->fwdata->altaf_intr_info.flr_pf_bmap); + if (pcifunc & RVU_PFVF_FUNC_MASK) { + vf = pcifunc & RVU_PFVF_FUNC_MASK; + if (vf >= 128) { + WARN(1, + "flr_vf_bmap size is 128 bits, vf=%u\n", + vf); + return; + } + + set_bit(vf, rvu->fwdata->altaf_intr_info.flr_vf_bmap); + } + } + + rvu_write64(rvu, BLKADDR_NIX0, AF_BAR2_ALIASX(0, NIX_GINT_INT_W1S), op); + usleep_range(5000, 6000); +} + static int rvu_process_mbox_msg(struct otx2_mbox *mbox, int devid, struct mbox_msghdr *req) { @@ -2268,7 +2336,8 @@ static void __rvu_mbox_handler(struct rvu_work *mwork, int type, bool poll) offset = mbox->rx_start + ALIGN(sizeof(*req_hdr), MBOX_MSG_ALIGN); - if (req_hdr->sig && !(is_rvu_otx2(rvu) || is_cn20k(rvu->pdev))) { + if (req_hdr->sig && rvu->altaf_ready && + !(is_rvu_otx2(rvu) || is_cn20k(rvu->pdev))) { req_hdr->opt_msg = mw->mbox_wrk[devid].num_msgs; rvu_write64(rvu, BLKADDR_NIX0, RVU_AF_BAR2_SEL, RVU_AF_BAR2_PFID); @@ -2576,7 +2645,7 @@ static int rvu_mbox_init(struct rvu *rvu, struct mbox_wq_info *mw, } mw->mbox_wq = alloc_workqueue("%s", - WQ_HIGHPRI | WQ_MEM_RECLAIM, + WQ_HIGHPRI | WQ_MEM_RECLAIM | WQ_PERCPU, num, name); if (!mw->mbox_wq) { err = -ENOMEM; @@ -2777,6 +2846,16 @@ static void rvu_blklf_teardown(struct rvu *rvu, u16 pcifunc, u8 blkaddr) block = &rvu->hw->block[blkaddr]; num_lfs = rvu_get_rsrc_mapcount(rvu_get_pfvf(rvu, pcifunc), block->addr); + + if (block->addr == BLKADDR_TIM && rvu->altaf_ready) { + rvu_notify_altaf(rvu, pcifunc, ALTAF_FLR); + return; + } + + if ((block->addr == BLKADDR_SSO || block->addr == BLKADDR_SSOW) && + rvu->altaf_ready) + return; + if (!num_lfs) return; for (slot = 0; slot < num_lfs; slot++) { @@ -3060,12 +3139,12 @@ static int rvu_afvf_msix_vectors_num_ok(struct rvu *rvu) static int rvu_register_interrupts(struct rvu *rvu) { - int ret, offset, pf_vec_start; + int i, ret, offset, pf_vec_start; rvu->num_vec = pci_msix_vec_count(rvu->pdev); - rvu->irq_name = devm_kmalloc_array(rvu->dev, rvu->num_vec, - NAME_SIZE, GFP_KERNEL); + rvu->irq_name = devm_kcalloc(rvu->dev, rvu->num_vec, + NAME_SIZE, GFP_KERNEL); if (!rvu->irq_name) return -ENOMEM; @@ -3251,6 +3330,13 @@ static int rvu_register_interrupts(struct rvu *rvu) if (ret) goto fail; + for (i = 0; i < rvu->num_vec; i++) { + if (strstr(&rvu->irq_name[i * NAME_SIZE], "Mbox") || + strstr(&rvu->irq_name[i * NAME_SIZE], "FLR")) + irq_set_affinity(pci_irq_vector(rvu->pdev, i), + cpumask_of(0)); + } + return 0; fail: @@ -3279,8 +3365,8 @@ static int rvu_flr_init(struct rvu *rvu) cfg | BIT_ULL(22)); } - rvu->flr_wq = alloc_ordered_workqueue("rvu_afpf_flr", - WQ_HIGHPRI | WQ_MEM_RECLAIM); + rvu->flr_wq = alloc_workqueue("rvu_afpf_flr", + WQ_HIGHPRI | WQ_MEM_RECLAIM, 0); if (!rvu->flr_wq) return -ENOMEM; diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu.h b/drivers/net/ethernet/marvell/octeontx2/af/rvu.h index e85dac2c806d..a466181cf908 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu.h +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu.h @@ -197,7 +197,7 @@ struct npc_key_field { /* Masks where all set bits indicate position * of a field in the key */ - u64 kw_mask[NPC_MAX_KWS_IN_KEY]; + u64 kw_mask[NPC_KWS_IN_KEY_SZ_MAX]; /* Number of words in the key a field spans. If a field is * of 16 bytes and key offset is 4 then the field will use * 4 bytes in KW0, 8 bytes in KW1 and 4 bytes in KW2 and @@ -308,6 +308,7 @@ struct rvu_pfvf { u64 lmt_map_ent_w1; /* Preseving the word1 of lmtst map table entry*/ unsigned long flags; struct sdp_node_info *sdp_info; + u8 hw_prio; /* Hw priority of default rules */ }; enum rvu_pfvf_flags { @@ -447,9 +448,11 @@ struct rvu_hwinfo { u8 sdp_links; u8 cpt_links; /* Number of CPT links */ u8 npc_kpus; /* No of parser units */ + u8 npc_kpms; /* Number of enhanced parser units */ + u8 npc_kex_extr; /* Number of LDATA extractors per KEX */ u8 npc_pkinds; /* No of port kinds */ u8 npc_intfs; /* No of interfaces */ - u8 npc_kpu_entries; /* No of KPU entries */ + u16 npc_kpu_entries; /* No of KPU entries */ u16 npc_counters; /* No of match stats counters */ u32 lbk_bufsize; /* FIFO size supported by LBK */ bool npc_ext_set; /* Extended register set */ @@ -552,7 +555,11 @@ struct npc_kpu_profile_adapter { const struct npc_lt_def_cfg *lt_def; const struct npc_kpu_profile_action *ikpu; /* array[pkinds] */ const struct npc_kpu_profile *kpu; /* array[kpus] */ - struct npc_mcam_kex *mkex; + union npc_mcam_key_prfl { + struct npc_mcam_kex *mkex; + /* used for cn9k and cn10k */ + struct npc_mcam_kex_extr *mkex_extr; /* used for cn20k */ + } mcam_kex_prfl; struct npc_mcam_kex_hash *mkex_hash; bool custom; size_t pkinds; @@ -1117,8 +1124,8 @@ int rvu_cgx_cfg_pause_frm(struct rvu *rvu, u16 pcifunc, u8 tx_pause, u8 rx_pause void rvu_mac_reset(struct rvu *rvu, u16 pcifunc); u32 rvu_cgx_get_lmac_fifolen(struct rvu *rvu, int cgx, int lmac); void cgx_start_linkup(struct rvu *rvu); -int npc_get_nixlf_mcam_index(struct npc_mcam *mcam, u16 pcifunc, int nixlf, - int type); +int npc_get_nixlf_mcam_index(struct npc_mcam *mcam, + u16 pcifunc, int nixlf, int type); bool is_mcam_entry_enabled(struct rvu *rvu, struct npc_mcam *mcam, int blkaddr, int index); int rvu_npc_init(struct rvu *rvu); @@ -1184,4 +1191,5 @@ int rvu_rep_pf_init(struct rvu *rvu); int rvu_rep_install_mcam_rules(struct rvu *rvu); void rvu_rep_update_rules(struct rvu *rvu, u16 pcifunc, bool ena); int rvu_rep_notify_pfvf_state(struct rvu *rvu, u16 pcifunc, bool enable); +int npc_mcam_verify_entry(struct npc_mcam *mcam, u16 pcifunc, int entry); #endif /* RVU_H */ diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_debugfs.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_debugfs.c index 15d3cb0b9da6..fa461489acdd 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_debugfs.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_debugfs.c @@ -21,7 +21,9 @@ #include "rvu_npc_hash.h" #include "mcs.h" +#include "cn20k/reg.h" #include "cn20k/debugfs.h" +#include "cn20k/npc.h" #define DEBUGFS_DIR_NAME "octeontx2" @@ -960,30 +962,21 @@ static bool rvu_dbg_is_valid_lf(struct rvu *rvu, int blkaddr, int lf, static void print_npa_qsize(struct seq_file *m, struct rvu_pfvf *pfvf) { - char *buf; - - buf = kmalloc(PAGE_SIZE, GFP_KERNEL); - if (!buf) - return; - if (!pfvf->aura_ctx) { seq_puts(m, "Aura context is not initialized\n"); } else { - bitmap_print_to_pagebuf(false, buf, pfvf->aura_bmap, - pfvf->aura_ctx->qsize); seq_printf(m, "Aura count : %d\n", pfvf->aura_ctx->qsize); - seq_printf(m, "Aura context ena/dis bitmap : %s\n", buf); + seq_printf(m, "Aura context ena/dis bitmap : %*pb\n", + pfvf->aura_ctx->qsize, pfvf->aura_bmap); } if (!pfvf->pool_ctx) { seq_puts(m, "Pool context is not initialized\n"); } else { - bitmap_print_to_pagebuf(false, buf, pfvf->pool_bmap, - pfvf->pool_ctx->qsize); seq_printf(m, "Pool count : %d\n", pfvf->pool_ctx->qsize); - seq_printf(m, "Pool context ena/dis bitmap : %s\n", buf); + seq_printf(m, "Pool context ena/dis bitmap : %*pb\n", + pfvf->pool_ctx->qsize, pfvf->pool_bmap); } - kfree(buf); } /* The 'qsize' entry dumps current Aura/Pool context Qsize @@ -2545,17 +2538,8 @@ RVU_DEBUG_SEQ_FOPS(nix_cq_ctx, nix_cq_ctx_display, nix_cq_ctx_write); static void print_nix_qctx_qsize(struct seq_file *filp, int qsize, unsigned long *bmap, char *qtype) { - char *buf; - - buf = kmalloc(PAGE_SIZE, GFP_KERNEL); - if (!buf) - return; - - bitmap_print_to_pagebuf(false, buf, bmap, qsize); seq_printf(filp, "%s context count : %d\n", qtype, qsize); - seq_printf(filp, "%s context ena/dis bitmap : %s\n", - qtype, buf); - kfree(buf); + seq_printf(filp, "%s context ena/dis bitmap : %*pb\n", qtype, qsize, bmap); } static void print_nix_qsize(struct seq_file *filp, struct rvu_pfvf *pfvf) @@ -3196,7 +3180,9 @@ static void rvu_print_npc_mcam_info(struct seq_file *s, static int rvu_dbg_npc_mcam_info_display(struct seq_file *filp, void *unsued) { struct rvu *rvu = filp->private; + int x4_free, x2_free, sb_free; int pf, vf, numvfs, blkaddr; + struct npc_priv_t *npc_priv; struct npc_mcam *mcam; u16 pcifunc, counters; u64 cfg; @@ -3210,16 +3196,34 @@ static int rvu_dbg_npc_mcam_info_display(struct seq_file *filp, void *unsued) seq_puts(filp, "\nNPC MCAM info:\n"); /* MCAM keywidth on receive and transmit sides */ - cfg = rvu_read64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(NIX_INTF_RX)); - cfg = (cfg >> 32) & 0x07; - seq_printf(filp, "\t\t RX keywidth \t: %s\n", (cfg == NPC_MCAM_KEY_X1) ? - "112bits" : ((cfg == NPC_MCAM_KEY_X2) ? - "224bits" : "448bits")); - cfg = rvu_read64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(NIX_INTF_TX)); - cfg = (cfg >> 32) & 0x07; - seq_printf(filp, "\t\t TX keywidth \t: %s\n", (cfg == NPC_MCAM_KEY_X1) ? - "112bits" : ((cfg == NPC_MCAM_KEY_X2) ? - "224bits" : "448bits")); + if (is_cn20k(rvu->pdev)) { + npc_priv = npc_priv_get(); + seq_printf(filp, "\t\t RX keywidth \t: %s\n", + (npc_priv->kw == NPC_MCAM_KEY_X1) ? + "256bits" : "512bits"); + + npc_cn20k_subbank_calc_free(rvu, &x2_free, &x4_free, &sb_free); + seq_printf(filp, "\t\t free x4 slots\t: %d\n", x4_free); + + seq_printf(filp, "\t\t free x2 slots\t: %d\n", x2_free); + + seq_printf(filp, "\t\t free subbanks\t: %d\n", sb_free); + } else { + cfg = rvu_read64(rvu, blkaddr, + NPC_AF_INTFX_KEX_CFG(NIX_INTF_RX)); + cfg = (cfg >> 32) & 0x07; + seq_printf(filp, "\t\t RX keywidth \t: %s\n", + (cfg == NPC_MCAM_KEY_X1) ? + "112bits" : ((cfg == NPC_MCAM_KEY_X2) ? + "224bits" : "448bits")); + cfg = rvu_read64(rvu, blkaddr, + NPC_AF_INTFX_KEX_CFG(NIX_INTF_TX)); + cfg = (cfg >> 32) & 0x07; + seq_printf(filp, "\t\t TX keywidth \t: %s\n", + (cfg == NPC_MCAM_KEY_X1) ? + "112bits" : ((cfg == NPC_MCAM_KEY_X2) ? + "224bits" : "448bits")); + } mutex_lock(&mcam->lock); /* MCAM entries */ @@ -3506,11 +3510,11 @@ static int rvu_dbg_npc_mcam_show_rules(struct seq_file *s, void *unused) struct rvu_npc_mcam_rule *iter; struct rvu *rvu = s->private; struct npc_mcam *mcam; - int pf, vf = -1; + int pf, vf = -1, bank; + u16 target, index; bool enabled; + u64 hits, off; int blkaddr; - u16 target; - u64 hits; blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); if (blkaddr < 0) @@ -3554,6 +3558,15 @@ static int rvu_dbg_npc_mcam_show_rules(struct seq_file *s, void *unused) enabled = is_mcam_entry_enabled(rvu, mcam, blkaddr, iter->entry); seq_printf(s, "\tenabled: %s\n", enabled ? "yes" : "no"); + if (is_cn20k(rvu->pdev)) { + seq_printf(s, "\tpriority: %u\n", iter->hw_prio); + index = iter->entry & (mcam->banksize - 1); + bank = npc_get_bank(mcam, iter->entry); + off = NPC_AF_CN20K_MCAMEX_BANKX_STAT_EXT(index, bank); + hits = rvu_read64(rvu, blkaddr, off); + seq_printf(s, "\thits: %lld\n", hits); + continue; + } if (!iter->has_cntr) continue; @@ -3698,9 +3711,9 @@ static int rvu_dbg_npc_exact_drop_cnt(struct seq_file *s, void *unused) struct npc_exact_table *table; struct rvu *rvu = s->private; struct npc_key_field *field; + u64 cfg, cam1, off; u16 chan, pcifunc; int blkaddr, i; - u64 cfg, cam1; char *str; blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); @@ -3721,11 +3734,17 @@ static int rvu_dbg_npc_exact_drop_cnt(struct seq_file *s, void *unused) chan = field->kw_mask[0] & cam1; str = (cfg & 1) ? "enabled" : "disabled"; + if (is_cn20k(rvu->pdev)) { + off = NPC_AF_CN20K_MCAMEX_BANKX_STAT_EXT(i, 0); + seq_printf(s, "0x%x\t%d\t\t%llu\t0x%x\t%s\n", pcifunc, + i, rvu_read64(rvu, blkaddr, off), chan, str); + } else { + off = NPC_AF_MATCH_STATX(table->counter_idx[i]); + seq_printf(s, "0x%x\t%d\t\t%llu\t0x%x\t%s\n", pcifunc, + i, rvu_read64(rvu, blkaddr, off), + chan, str); + } - seq_printf(s, "0x%x\t%d\t\t%llu\t0x%x\t%s\n", pcifunc, i, - rvu_read64(rvu, blkaddr, - NPC_AF_MATCH_STATX(table->counter_idx[i])), - chan, str); } return 0; @@ -3745,6 +3764,9 @@ static void rvu_dbg_npc_init(struct rvu *rvu) debugfs_create_file("rx_miss_act_stats", 0444, rvu->rvu_dbg.npc, rvu, &rvu_dbg_npc_rx_miss_act_fops); + if (is_cn20k(rvu->pdev)) + npc_cn20k_debugfs_init(rvu); + if (!rvu->hw->cap.npc_exact_match_enabled) return; diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_devlink.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_devlink.c index a29f1ea04c7d..6494a9ee2f0d 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_devlink.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_devlink.c @@ -11,6 +11,7 @@ #include "rvu_reg.h" #include "rvu_struct.h" #include "rvu_npc_hash.h" +#include "cn20k/npc.h" #define DRV_NAME "octeontx2-af" @@ -1256,9 +1257,71 @@ enum rvu_af_dl_param_id { RVU_AF_DEVLINK_PARAM_ID_NPC_MCAM_ZONE_PERCENT, RVU_AF_DEVLINK_PARAM_ID_NPC_EXACT_FEATURE_DISABLE, RVU_AF_DEVLINK_PARAM_ID_NPC_DEF_RULE_CNTR_ENABLE, + RVU_AF_DEVLINK_PARAM_ID_NPC_DEFRAG, RVU_AF_DEVLINK_PARAM_ID_NIX_MAXLF, }; +static int rvu_af_npc_defrag_feature_get(struct devlink *devlink, u32 id, + struct devlink_param_gset_ctx *ctx, + struct netlink_ext_ack *extack) +{ + struct rvu_devlink *rvu_dl = devlink_priv(devlink); + struct rvu *rvu = rvu_dl->rvu; + bool enabled; + + enabled = is_cn20k(rvu->pdev); + + snprintf(ctx->val.vstr, sizeof(ctx->val.vstr), "%s", + enabled ? "enabled" : "disabled"); + + return 0; +} + +static int rvu_af_npc_defrag(struct devlink *devlink, u32 id, + struct devlink_param_gset_ctx *ctx, + struct netlink_ext_ack *extack) +{ + struct rvu_devlink *rvu_dl = devlink_priv(devlink); + struct rvu *rvu = rvu_dl->rvu; + + /* It is hard to roll back if defrag process fails. + * print a error message and return fault. + */ + if (npc_cn20k_defrag(rvu)) { + dev_err(rvu->dev, "Defrag process failed\n"); + return -EFAULT; + } + return 0; +} + +static int rvu_af_npc_defrag_feature_validate(struct devlink *devlink, u32 id, + union devlink_param_value val, + struct netlink_ext_ack *extack) +{ + struct rvu_devlink *rvu_dl = devlink_priv(devlink); + struct rvu *rvu = rvu_dl->rvu; + u64 enable; + + if (kstrtoull(val.vstr, 10, &enable)) { + NL_SET_ERR_MSG_MOD(extack, + "Only 1 value is supported"); + return -EINVAL; + } + + if (enable != 1) { + NL_SET_ERR_MSG_MOD(extack, + "Only initiating defrag is supported"); + return -EINVAL; + } + + if (is_cn20k(rvu->pdev)) + return 0; + + NL_SET_ERR_MSG_MOD(extack, + "Can defrag NPC only in cn20k silicon"); + return -EFAULT; +} + static int rvu_af_npc_exact_feature_get(struct devlink *devlink, u32 id, struct devlink_param_gset_ctx *ctx, struct netlink_ext_ack *extack) @@ -1561,6 +1624,15 @@ static const struct devlink_ops rvu_devlink_ops = { .eswitch_mode_set = rvu_devlink_eswitch_mode_set, }; +static const struct devlink_param rvu_af_dl_param_defrag[] = { + DEVLINK_PARAM_DRIVER(RVU_AF_DEVLINK_PARAM_ID_NPC_DEFRAG, + "npc_defrag", DEVLINK_PARAM_TYPE_STRING, + BIT(DEVLINK_PARAM_CMODE_RUNTIME), + rvu_af_npc_defrag_feature_get, + rvu_af_npc_defrag, + rvu_af_npc_defrag_feature_validate), +}; + int rvu_register_dl(struct rvu *rvu) { struct rvu_devlink *rvu_dl; @@ -1593,6 +1665,17 @@ int rvu_register_dl(struct rvu *rvu) goto err_dl_health; } + if (is_cn20k(rvu->pdev)) { + err = devlink_params_register(dl, rvu_af_dl_param_defrag, + ARRAY_SIZE(rvu_af_dl_param_defrag)); + if (err) { + dev_err(rvu->dev, + "devlink defrag params register failed with error %d", + err); + goto err_dl_defrag; + } + } + /* Register exact match devlink only for CN10K-B */ if (!rvu_npc_exact_has_match_table(rvu)) goto done; @@ -1601,7 +1684,8 @@ int rvu_register_dl(struct rvu *rvu) ARRAY_SIZE(rvu_af_dl_param_exact_match)); if (err) { dev_err(rvu->dev, - "devlink exact match params register failed with error %d", err); + "devlink exact match params register failed with error %d", + err); goto err_dl_exact_match; } @@ -1610,6 +1694,11 @@ done: return 0; err_dl_exact_match: + if (is_cn20k(rvu->pdev)) + devlink_params_unregister(dl, rvu_af_dl_param_defrag, + ARRAY_SIZE(rvu_af_dl_param_defrag)); + +err_dl_defrag: devlink_params_unregister(dl, rvu_af_dl_params, ARRAY_SIZE(rvu_af_dl_params)); err_dl_health: @@ -1627,6 +1716,10 @@ void rvu_unregister_dl(struct rvu *rvu) devlink_params_unregister(dl, rvu_af_dl_params, ARRAY_SIZE(rvu_af_dl_params)); + if (is_cn20k(rvu->pdev)) + devlink_params_unregister(dl, rvu_af_dl_param_defrag, + ARRAY_SIZE(rvu_af_dl_param_defrag)); + /* Unregister exact match devlink only for CN10K-B */ if (rvu_npc_exact_has_match_table(rvu)) devlink_params_unregister(dl, rvu_af_dl_param_exact_match, diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c index 98dd68137a09..ef5b081162eb 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_nix.c @@ -2391,8 +2391,8 @@ static void nix_smq_flush_enadis_xoff(struct rvu *rvu, int blkaddr, continue; /* skip if PF_FUNC doesn't match */ if ((TXSCH_MAP_FUNC(txsch->pfvf_map[tl2]) & ~RVU_PFVF_FUNC_MASK) != - (TXSCH_MAP_FUNC(txsch->pfvf_map[tl2_schq] & - ~RVU_PFVF_FUNC_MASK))) + (TXSCH_MAP_FUNC(txsch->pfvf_map[tl2_schq]) & + ~RVU_PFVF_FUNC_MASK)) continue; /* enable/disable XOFF */ regoff = NIX_AF_TL2X_SW_XOFF(tl2); @@ -5289,7 +5289,6 @@ int rvu_mbox_handler_nix_lf_stop_rx(struct rvu *rvu, struct msg_req *req, /* Disable the interface if it is in any multicast list */ nix_mcast_update_mce_entry(rvu, pcifunc, 0); - pfvf = rvu_get_pfvf(rvu, pcifunc); clear_bit(NIXLF_INITIALIZED, &pfvf->flags); diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc.c index 8658cb2143df..c2ca5ed1d028 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc.c @@ -16,6 +16,9 @@ #include "cgx.h" #include "npc_profile.h" #include "rvu_npc_hash.h" +#include "cn20k/npc.h" +#include "rvu_npc.h" +#include "cn20k/reg.h" #define RSVD_MCAM_ENTRIES_PER_PF 3 /* Broadcast, Promisc and AllMulticast */ #define RSVD_MCAM_ENTRIES_PER_NIXLF 1 /* Ucast for LFs */ @@ -149,10 +152,33 @@ int npc_get_nixlf_mcam_index(struct npc_mcam *mcam, { struct rvu_hwinfo *hw = container_of(mcam, struct rvu_hwinfo, mcam); struct rvu *rvu = hw->rvu; - int pf = rvu_get_pf(rvu->pdev, pcifunc); + u16 bcast, mcast, promisc, ucast; int index; + int rc; + int pf; + + if (is_cn20k(rvu->pdev)) { + rc = npc_cn20k_dft_rules_idx_get(rvu, pcifunc, &bcast, &mcast, + &promisc, &ucast); + if (rc) + return -EFAULT; + + switch (type) { + case NIXLF_BCAST_ENTRY: + return bcast; + case NIXLF_ALLMULTI_ENTRY: + return mcast; + case NIXLF_PROMISC_ENTRY: + return promisc; + case NIXLF_UCAST_ENTRY: + return ucast; + default: + return -EINVAL; + } + } /* Check if this is for a PF */ + pf = rvu_get_pf(rvu->pdev, pcifunc); if (pf && !(pcifunc & RVU_PFVF_FUNC_MASK)) { /* Reserved entries exclude PF0 */ pf--; @@ -173,7 +199,12 @@ int npc_get_nixlf_mcam_index(struct npc_mcam *mcam, int npc_get_bank(struct npc_mcam *mcam, int index) { + struct rvu_hwinfo *hw = container_of(mcam, struct rvu_hwinfo, mcam); int bank = index / mcam->banksize; + struct rvu *rvu = hw->rvu; + + if (is_cn20k(rvu->pdev)) + return bank; /* 0,1 & 2,3 banks are combined for this keysize */ if (mcam->keysize == NPC_MCAM_KEY_X2) @@ -189,7 +220,14 @@ bool is_mcam_entry_enabled(struct rvu *rvu, struct npc_mcam *mcam, u64 cfg; index &= (mcam->banksize - 1); - cfg = rvu_read64(rvu, blkaddr, NPC_AF_MCAMEX_BANKX_CFG(index, bank)); + if (is_cn20k(rvu->pdev)) + cfg = rvu_read64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_CFG_EXT(index, + bank)); + else + cfg = rvu_read64(rvu, blkaddr, + NPC_AF_MCAMEX_BANKX_CFG(index, bank)); + return (cfg & 1); } @@ -199,6 +237,13 @@ void npc_enable_mcam_entry(struct rvu *rvu, struct npc_mcam *mcam, int bank = npc_get_bank(mcam, index); int actbank = bank; + if (is_cn20k(rvu->pdev)) { + if (index < 0 || index >= mcam->banksize * mcam->banks) + return; + + return npc_cn20k_enable_mcam_entry(rvu, blkaddr, index, enable); + } + index &= (mcam->banksize - 1); for (; bank < (actbank + mcam->banks_per_entry); bank++) { rvu_write64(rvu, blkaddr, @@ -367,6 +412,7 @@ static u64 npc_get_default_entry_action(struct rvu *rvu, struct npc_mcam *mcam, int blkaddr, u16 pf_func) { int bank, nixlf, index; + u64 reg; /* get ucast entry rule entry index */ if (nix_get_nixlf(rvu, pf_func, &nixlf, NULL)) { @@ -381,8 +427,12 @@ static u64 npc_get_default_entry_action(struct rvu *rvu, struct npc_mcam *mcam, bank = npc_get_bank(mcam, index); index &= (mcam->banksize - 1); - return rvu_read64(rvu, blkaddr, - NPC_AF_MCAMEX_BANKX_ACTION(index, bank)); + if (is_cn20k(rvu->pdev)) + reg = NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(index, bank, 0); + else + reg = NPC_AF_MCAMEX_BANKX_ACTION(index, bank); + + return rvu_read64(rvu, blkaddr, reg); } static void npc_fixup_vf_rule(struct rvu *rvu, struct npc_mcam *mcam, @@ -547,6 +597,9 @@ static void npc_copy_mcam_entry(struct rvu *rvu, struct npc_mcam *mcam, u64 cfg, sreg, dreg; int bank, i; + if (is_cn20k(rvu->pdev)) + return npc_cn20k_copy_mcam_entry(rvu, blkaddr, src, dest); + src &= (mcam->banksize - 1); dest &= (mcam->banksize - 1); @@ -583,20 +636,31 @@ u64 npc_get_mcam_action(struct rvu *rvu, struct npc_mcam *mcam, int blkaddr, int index) { int bank = npc_get_bank(mcam, index); + u64 reg; index &= (mcam->banksize - 1); - return rvu_read64(rvu, blkaddr, - NPC_AF_MCAMEX_BANKX_ACTION(index, bank)); + + if (is_cn20k(rvu->pdev)) + reg = NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(index, bank, 0); + else + reg = NPC_AF_MCAMEX_BANKX_ACTION(index, bank); + return rvu_read64(rvu, blkaddr, reg); } void npc_set_mcam_action(struct rvu *rvu, struct npc_mcam *mcam, int blkaddr, int index, u64 cfg) { int bank = npc_get_bank(mcam, index); + u64 reg; index &= (mcam->banksize - 1); - return rvu_write64(rvu, blkaddr, - NPC_AF_MCAMEX_BANKX_ACTION(index, bank), cfg); + + if (is_cn20k(rvu->pdev)) + reg = NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(index, bank, 0); + else + reg = NPC_AF_MCAMEX_BANKX_ACTION(index, bank); + + return rvu_write64(rvu, blkaddr, reg, cfg); } void rvu_npc_install_ucast_entry(struct rvu *rvu, u16 pcifunc, @@ -651,6 +715,9 @@ void rvu_npc_install_ucast_entry(struct rvu *rvu, u16 pcifunc, req.match_id = action.match_id; req.flow_key_alg = action.flow_key_alg; + if (is_cn20k(rvu->pdev)) + req.hw_prio = pfvf->hw_prio; + rvu_mbox_handler_npc_install_flow(rvu, &req, &rsp); } @@ -739,6 +806,9 @@ void rvu_npc_install_promisc_entry(struct rvu *rvu, u16 pcifunc, req.match_id = action.match_id; req.flow_key_alg = flow_key_alg; + if (is_cn20k(rvu->pdev)) + req.hw_prio = pfvf->hw_prio; + rvu_mbox_handler_npc_install_flow(rvu, &req, &rsp); } @@ -819,6 +889,9 @@ void rvu_npc_install_bcast_match_entry(struct rvu *rvu, u16 pcifunc, req.hdr.pcifunc = 0; /* AF is requester */ req.vf = pcifunc; + if (is_cn20k(rvu->pdev)) + req.hw_prio = pfvf->hw_prio; + rvu_mbox_handler_npc_install_flow(rvu, &req, &rsp); } @@ -907,6 +980,9 @@ void rvu_npc_install_allmulti_entry(struct rvu *rvu, u16 pcifunc, int nixlf, req.match_id = action.match_id; req.flow_key_alg = flow_key_alg; + if (is_cn20k(rvu->pdev)) + req.hw_prio = pfvf->hw_prio; + rvu_mbox_handler_npc_install_flow(rvu, &req, &rsp); } @@ -940,33 +1016,42 @@ static void npc_update_vf_flow_entry(struct rvu *rvu, struct npc_mcam *mcam, mutex_lock(&mcam->lock); for (index = 0; index < mcam->bmap_entries; index++) { - if (mcam->entry2target_pffunc[index] == pcifunc) { - update = true; - /* update not needed for the rules added via ntuple filters */ - list_for_each_entry(rule, &mcam->mcam_rules, list) { - if (rule->entry == index) - update = false; - } - if (!update) - continue; - bank = npc_get_bank(mcam, index); - actindex = index; - entry = index & (mcam->banksize - 1); - - /* read vf flow entry enable status */ - enable = is_mcam_entry_enabled(rvu, mcam, blkaddr, - actindex); - /* disable before mcam entry update */ - npc_enable_mcam_entry(rvu, mcam, blkaddr, actindex, - false); - /* update 'action' */ + if (mcam->entry2target_pffunc[index] != pcifunc) + continue; + update = true; + /* update not needed for the rules added via ntuple filters */ + list_for_each_entry(rule, &mcam->mcam_rules, list) { + if (rule->entry == index) + update = false; + } + if (!update) + continue; + bank = npc_get_bank(mcam, index); + actindex = index; + entry = index & (mcam->banksize - 1); + + /* read vf flow entry enable status */ + enable = is_mcam_entry_enabled(rvu, mcam, blkaddr, + actindex); + /* disable before mcam entry update */ + npc_enable_mcam_entry(rvu, mcam, blkaddr, actindex, + false); + + /* update 'action' */ + if (is_cn20k(rvu->pdev)) + rvu_write64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(entry, + bank, + 0), + rx_action); + else rvu_write64(rvu, blkaddr, NPC_AF_MCAMEX_BANKX_ACTION(entry, bank), rx_action); - if (enable) - npc_enable_mcam_entry(rvu, mcam, blkaddr, - actindex, true); - } + + if (enable) + npc_enable_mcam_entry(rvu, mcam, blkaddr, + actindex, true); } mutex_unlock(&mcam->lock); } @@ -979,6 +1064,7 @@ static void npc_update_rx_action_with_alg_idx(struct rvu *rvu, struct nix_rx_act struct npc_mcam *mcam = &rvu->hw->mcam; struct rvu_hwinfo *hw = rvu->hw; int bank, op_rss; + u64 reg; if (!is_mcam_entry_enabled(rvu, mcam, blkaddr, mcam_index)) return; @@ -988,15 +1074,19 @@ static void npc_update_rx_action_with_alg_idx(struct rvu *rvu, struct nix_rx_act bank = npc_get_bank(mcam, mcam_index); mcam_index &= (mcam->banksize - 1); + if (is_cn20k(rvu->pdev)) + reg = NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(mcam_index, + bank, 0); + else + reg = NPC_AF_MCAMEX_BANKX_ACTION(mcam_index, bank); + /* If Rx action is MCAST update only RSS algorithm index */ if (!op_rss) { - *(u64 *)&action = rvu_read64(rvu, blkaddr, - NPC_AF_MCAMEX_BANKX_ACTION(mcam_index, bank)); + *(u64 *)&action = rvu_read64(rvu, blkaddr, reg); action.flow_key_alg = alg_idx; } - rvu_write64(rvu, blkaddr, - NPC_AF_MCAMEX_BANKX_ACTION(mcam_index, bank), *(u64 *)&action); + rvu_write64(rvu, blkaddr, reg, *(u64 *)&action); } void rvu_npc_update_flowkey_alg_idx(struct rvu *rvu, u16 pcifunc, int nixlf, @@ -1006,6 +1096,7 @@ void rvu_npc_update_flowkey_alg_idx(struct rvu *rvu, u16 pcifunc, int nixlf, struct nix_rx_action action; int blkaddr, index, bank; struct rvu_pfvf *pfvf; + u64 reg; blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); if (blkaddr < 0) @@ -1028,8 +1119,12 @@ void rvu_npc_update_flowkey_alg_idx(struct rvu *rvu, u16 pcifunc, int nixlf, bank = npc_get_bank(mcam, index); index &= (mcam->banksize - 1); - *(u64 *)&action = rvu_read64(rvu, blkaddr, - NPC_AF_MCAMEX_BANKX_ACTION(index, bank)); + if (is_cn20k(rvu->pdev)) + reg = NPC_AF_CN20K_MCAMEX_BANKX_ACTIONX_EXT(index, bank, 0); + else + reg = NPC_AF_MCAMEX_BANKX_ACTION(index, bank); + + *(u64 *)&action = rvu_read64(rvu, blkaddr, reg); /* Ignore if no action was set earlier */ if (!*(u64 *)&action) return; @@ -1039,8 +1134,11 @@ void rvu_npc_update_flowkey_alg_idx(struct rvu *rvu, u16 pcifunc, int nixlf, action.index = group; action.flow_key_alg = alg_idx; - rvu_write64(rvu, blkaddr, - NPC_AF_MCAMEX_BANKX_ACTION(index, bank), *(u64 *)&action); + rvu_write64(rvu, blkaddr, reg, *(u64 *)&action); + /* update the VF flow rule action with the VF default entry action */ + if (mcam_index < 0) + npc_update_vf_flow_entry(rvu, mcam, blkaddr, pcifunc, + *(u64 *)&action); /* update the action change in default rule */ pfvf = rvu_get_pfvf(rvu, pcifunc); @@ -1338,8 +1436,8 @@ static void npc_program_mkex_profile(struct rvu *rvu, int blkaddr, npc_program_mkex_hash(rvu, blkaddr); } -static int npc_fwdb_prfl_img_map(struct rvu *rvu, void __iomem **prfl_img_addr, - u64 *size) +int npc_fwdb_prfl_img_map(struct rvu *rvu, void __iomem **prfl_img_addr, + u64 *size) { u64 prfl_addr, prfl_sz; @@ -1395,7 +1493,7 @@ static void npc_load_mkex_profile(struct rvu *rvu, int blkaddr, */ if (!is_rvu_96xx_B0(rvu) || mcam_kex->keyx_cfg[NIX_INTF_RX] == mcam_kex->keyx_cfg[NIX_INTF_TX]) - rvu->kpu.mkex = mcam_kex; + rvu->kpu.mcam_kex_prfl.mkex = mcam_kex; goto program_mkex; } @@ -1405,16 +1503,17 @@ static void npc_load_mkex_profile(struct rvu *rvu, int blkaddr, dev_warn(dev, "Failed to load requested profile: %s\n", mkex_profile); program_mkex: - dev_info(rvu->dev, "Using %s mkex profile\n", rvu->kpu.mkex->name); + dev_info(rvu->dev, "Using %s mkex profile\n", + rvu->kpu.mcam_kex_prfl.mkex->name); /* Program selected mkex profile */ - npc_program_mkex_profile(rvu, blkaddr, rvu->kpu.mkex); + npc_program_mkex_profile(rvu, blkaddr, rvu->kpu.mcam_kex_prfl.mkex); if (mkex_prfl_addr) iounmap(mkex_prfl_addr); } -static void npc_config_kpuaction(struct rvu *rvu, int blkaddr, - const struct npc_kpu_profile_action *kpuaction, - int kpu, int entry, bool pkind) +void npc_config_kpuaction(struct rvu *rvu, int blkaddr, + const struct npc_kpu_profile_action *kpuaction, + int kpu, int entry, bool pkind) { struct npc_kpu_action0 action0 = {0}; struct npc_kpu_action1 action1 = {0}; @@ -1477,7 +1576,7 @@ static void npc_config_kpucam(struct rvu *rvu, int blkaddr, NPC_AF_KPUX_ENTRYX_CAMX(kpu, entry, 1), *(u64 *)&cam1); } -static inline u64 enable_mask(int count) +u64 npc_enable_mask(int count) { return (((count) < 64) ? ~(BIT_ULL(count) - 1) : (0x00ULL)); } @@ -1510,7 +1609,7 @@ static void npc_program_kpu_profile(struct rvu *rvu, int blkaddr, int kpu, /* Enable all programmed entries */ num_entries = min_t(int, profile->action_entries, profile->cam_entries); - entry_mask = enable_mask(num_entries); + entry_mask = npc_enable_mask(num_entries); /* Disable first KPU_MAX_CST_ENT entries for built-in profile */ if (!rvu->kpu.custom) entry_mask |= GENMASK_ULL(KPU_MAX_CST_ENT - 1, 0); @@ -1519,14 +1618,15 @@ static void npc_program_kpu_profile(struct rvu *rvu, int blkaddr, int kpu, if (num_entries > 64) { rvu_write64(rvu, blkaddr, NPC_AF_KPUX_ENTRY_DISX(kpu, 1), - enable_mask(num_entries - 64)); + npc_enable_mask(num_entries - 64)); } /* Enable this KPU */ rvu_write64(rvu, blkaddr, NPC_AF_KPUX_CFG(kpu), 0x01); } -static int npc_prepare_default_kpu(struct npc_kpu_profile_adapter *profile) +static void npc_prepare_default_kpu(struct rvu *rvu, + struct npc_kpu_profile_adapter *profile) { profile->custom = 0; profile->name = def_pfl_name; @@ -1536,23 +1636,38 @@ static int npc_prepare_default_kpu(struct npc_kpu_profile_adapter *profile) profile->kpu = npc_kpu_profiles; profile->kpus = ARRAY_SIZE(npc_kpu_profiles); profile->lt_def = &npc_lt_defaults; - profile->mkex = &npc_mkex_default; profile->mkex_hash = &npc_mkex_hash_default; - return 0; + if (!is_cn20k(rvu->pdev)) { + profile->mcam_kex_prfl.mkex = &npc_mkex_default; + return; + } + + profile->mcam_kex_prfl.mkex_extr = npc_mkex_extr_default_get(); + ikpu_action_entries[NPC_RX_CPT_HDR_PKIND].offset = 6; + ikpu_action_entries[NPC_RX_CPT_HDR_PKIND].mask = 0xe0; + ikpu_action_entries[NPC_RX_CPT_HDR_PKIND].shift = 0x5; + ikpu_action_entries[NPC_RX_CPT_HDR_PKIND].right = 0x1; + + npc_cn20k_update_action_entries_n_flags(rvu, profile); } static int npc_apply_custom_kpu(struct rvu *rvu, struct npc_kpu_profile_adapter *profile) { size_t hdr_sz = sizeof(struct npc_kpu_profile_fwdata), offset = 0; - struct npc_kpu_profile_fwdata *fw = rvu->kpu_fwdata; struct npc_kpu_profile_action *action; + struct npc_kpu_profile_fwdata *fw; struct npc_kpu_profile_cam *cam; struct npc_kpu_fwdata *fw_kpu; int entries; u16 kpu, entry; + if (is_cn20k(rvu->pdev)) + return npc_cn20k_apply_custom_kpu(rvu, profile); + + fw = rvu->kpu_fwdata; + if (rvu->kpu_fwdata_sz < hdr_sz) { dev_warn(rvu->dev, "Invalid KPU profile size\n"); return -EINVAL; @@ -1593,7 +1708,7 @@ static int npc_apply_custom_kpu(struct rvu *rvu, profile->custom = 1; profile->name = fw->name; profile->version = le64_to_cpu(fw->version); - profile->mkex = &fw->mkex; + profile->mcam_kex_prfl.mkex = &fw->mkex; profile->lt_def = &fw->lt_def; for (kpu = 0; kpu < fw->kpus; kpu++) { @@ -1707,7 +1822,7 @@ done: return ret; } -static void npc_load_kpu_profile(struct rvu *rvu) +void npc_load_kpu_profile(struct rvu *rvu) { struct npc_kpu_profile_adapter *profile = &rvu->kpu; const char *kpu_profile = rvu->kpu_pfl_name; @@ -1718,7 +1833,7 @@ static void npc_load_kpu_profile(struct rvu *rvu) if (!strncmp(kpu_profile, def_pfl_name, KPU_NAME_LEN)) goto revert_to_default; /* First prepare default KPU, then we'll customize top entries. */ - npc_prepare_default_kpu(profile); + npc_prepare_default_kpu(rvu, profile); /* Order of preceedence for load loading NPC profile (high to low) * Firmware binary in filesystem. @@ -1781,7 +1896,7 @@ program_kpu: return; revert_to_default: - npc_prepare_default_kpu(profile); + npc_prepare_default_kpu(rvu, profile); } static void npc_parser_profile_init(struct rvu *rvu, int blkaddr) @@ -1842,39 +1957,56 @@ int npc_mcam_rsrcs_init(struct rvu *rvu, int blkaddr) int cntr; u64 cfg; - /* Actual number of MCAM entries vary by entry size */ cfg = (rvu_read64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(0)) >> 32) & 0x07; - mcam->total_entries = (mcam->banks / BIT_ULL(cfg)) * mcam->banksize; mcam->keysize = cfg; /* Number of banks combined per MCAM entry */ - if (cfg == NPC_MCAM_KEY_X4) - mcam->banks_per_entry = 4; - else if (cfg == NPC_MCAM_KEY_X2) - mcam->banks_per_entry = 2; - else - mcam->banks_per_entry = 1; - - /* Reserve one MCAM entry for each of the NIX LF to - * guarantee space to install default matching DMAC rule. - * Also reserve 2 MCAM entries for each PF for default - * channel based matching or 'bcast & promisc' matching to - * support BCAST and PROMISC modes of operation for PFs. - * PF0 is excluded. - */ - rsvd = (nixlf_count * RSVD_MCAM_ENTRIES_PER_NIXLF) + - ((rvu->hw->total_pfs - 1) * RSVD_MCAM_ENTRIES_PER_PF); - if (mcam->total_entries <= rsvd) { - dev_warn(rvu->dev, - "Insufficient NPC MCAM size %d for pkt I/O, exiting\n", - mcam->total_entries); - return -ENOMEM; + if (is_cn20k(rvu->pdev)) { + /* In cn20k, x2 entries is allowed for x4 profile. + * set total_entries as 8192 * 2 and key size as x2. + */ + mcam->total_entries = mcam->banks * mcam->banksize; + if (cfg == NPC_MCAM_KEY_X2) + mcam->banks_per_entry = 1; + else + mcam->banks_per_entry = 2; + + rsvd = 0; + } else { + mcam->total_entries = (mcam->banks / BIT_ULL(cfg)) * + mcam->banksize; + + if (cfg == NPC_MCAM_KEY_X4) + mcam->banks_per_entry = 4; + else if (cfg == NPC_MCAM_KEY_X2) + mcam->banks_per_entry = 2; + else + mcam->banks_per_entry = 1; + + /* Reserve one MCAM entry for each of the NIX LF to + * guarantee space to install default matching DMAC rule. + * Also reserve 2 MCAM entries for each PF for default + * channel based matching or 'bcast & promisc' matching to + * support BCAST and PROMISC modes of operation for PFs. + * PF0 is excluded. + */ + rsvd = (nixlf_count * RSVD_MCAM_ENTRIES_PER_NIXLF) + + ((rvu->hw->total_pfs - 1) * RSVD_MCAM_ENTRIES_PER_PF); + if (mcam->total_entries <= rsvd) { + dev_warn(rvu->dev, + "Insufficient NPC MCAM size %d for pkt I/O, exiting\n", + mcam->total_entries); + return -ENOMEM; + } } mcam->bmap_entries = mcam->total_entries - rsvd; - mcam->nixlf_offset = mcam->bmap_entries; - mcam->pf_offset = mcam->nixlf_offset + nixlf_count; + /* cn20k does not need offsets to alloc mcam entries */ + if (!is_cn20k(rvu->pdev)) { + mcam->nixlf_offset = mcam->bmap_entries; + mcam->pf_offset = mcam->nixlf_offset + nixlf_count; + } /* Allocate bitmaps for managing MCAM entries */ mcam->bmap = bitmap_zalloc(mcam->bmap_entries, GFP_KERNEL); @@ -1898,13 +2030,15 @@ int npc_mcam_rsrcs_init(struct rvu *rvu, int blkaddr) * allocations and another 1/8th at the top for high priority * allocations. */ - mcam->lprio_count = mcam->bmap_entries / 8; - if (mcam->lprio_count > BITS_PER_LONG) - mcam->lprio_count = round_down(mcam->lprio_count, - BITS_PER_LONG); - mcam->lprio_start = mcam->bmap_entries - mcam->lprio_count; - mcam->hprio_count = mcam->lprio_count; - mcam->hprio_end = mcam->hprio_count; + if (!is_cn20k(rvu->pdev)) { + mcam->lprio_count = mcam->bmap_entries / 8; + if (mcam->lprio_count > BITS_PER_LONG) + mcam->lprio_count = round_down(mcam->lprio_count, + BITS_PER_LONG); + mcam->lprio_start = mcam->bmap_entries - mcam->lprio_count; + mcam->hprio_count = mcam->lprio_count; + mcam->hprio_end = mcam->hprio_count; + } /* Allocate bitmap for managing MCAM counters and memory * for saving counter to RVU PFFUNC allocation mapping. @@ -1984,6 +2118,19 @@ static void rvu_npc_hw_init(struct rvu *rvu, int blkaddr) hw->npc_pkinds = (npc_const1 >> 12) & 0xFFULL; hw->npc_kpu_entries = npc_const1 & 0xFFFULL; hw->npc_kpus = (npc_const >> 8) & 0x1FULL; + /* For Cn20k silicon, check if enhanced parser + * is present, then set the NUM_KPMS = NUM_KPUS / 2 and + * number of LDATA extractors per KEX. + */ + if (is_cn20k(rvu->pdev)) { + if (!(npc_const1 & BIT_ULL(62))) { + WARN(1, "Enhanced parser is not supported\n"); + return; + } + hw->npc_kpms = hw->npc_kpus / 2; + hw->npc_kex_extr = (npc_const1 >> 36) & 0x3FULL; + } + hw->npc_intfs = npc_const & 0xFULL; hw->npc_counters = (npc_const >> 48) & 0xFFFFULL; @@ -2014,28 +2161,36 @@ static void rvu_npc_hw_init(struct rvu *rvu, int blkaddr) static void rvu_npc_setup_interfaces(struct rvu *rvu, int blkaddr) { - struct npc_mcam_kex *mkex = rvu->kpu.mkex; + struct npc_mcam_kex_extr *mkex_extr = rvu->kpu.mcam_kex_prfl.mkex_extr; + struct npc_mcam_kex *mkex = rvu->kpu.mcam_kex_prfl.mkex; struct npc_mcam *mcam = &rvu->hw->mcam; struct rvu_hwinfo *hw = rvu->hw; u64 nibble_ena, rx_kex, tx_kex; + u64 *keyx_cfg, reg; u8 intf; - /* Reserve last counter for MCAM RX miss action which is set to - * drop packet. This way we will know how many pkts didn't match - * any MCAM entry. - */ - mcam->counters.max--; - mcam->rx_miss_act_cntr = mcam->counters.max; + if (is_cn20k(rvu->pdev)) { + keyx_cfg = mkex_extr->keyx_cfg; + } else { + keyx_cfg = mkex->keyx_cfg; + /* Reserve last counter for MCAM RX miss action which is set to + * drop packet. This way we will know how many pkts didn't + * match any MCAM entry. + */ + mcam->counters.max--; + mcam->rx_miss_act_cntr = mcam->counters.max; + } + + rx_kex = keyx_cfg[NIX_INTF_RX]; + tx_kex = keyx_cfg[NIX_INTF_TX]; - rx_kex = mkex->keyx_cfg[NIX_INTF_RX]; - tx_kex = mkex->keyx_cfg[NIX_INTF_TX]; nibble_ena = FIELD_GET(NPC_PARSE_NIBBLE, rx_kex); nibble_ena = rvu_npc_get_tx_nibble_cfg(rvu, nibble_ena); if (nibble_ena) { tx_kex &= ~NPC_PARSE_NIBBLE; tx_kex |= FIELD_PREP(NPC_PARSE_NIBBLE, nibble_ena); - mkex->keyx_cfg[NIX_INTF_TX] = tx_kex; + keyx_cfg[NIX_INTF_TX] = tx_kex; } /* Configure RX interfaces */ @@ -2047,12 +2202,19 @@ static void rvu_npc_setup_interfaces(struct rvu *rvu, int blkaddr) rvu_write64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(intf), rx_kex); + if (is_cn20k(rvu->pdev)) + reg = NPC_AF_INTFX_MISS_ACTX(intf, 0); + else + reg = NPC_AF_INTFX_MISS_ACT(intf); + /* If MCAM lookup doesn't result in a match, drop the received * packet. And map this action to a counter to count dropped * packets. */ - rvu_write64(rvu, blkaddr, - NPC_AF_INTFX_MISS_ACT(intf), NIX_RX_ACTIONOP_DROP); + rvu_write64(rvu, blkaddr, reg, NIX_RX_ACTIONOP_DROP); + + if (is_cn20k(rvu->pdev)) + continue; /* NPC_AF_INTFX_MISS_STAT_ACT[14:12] - counter[11:9] * NPC_AF_INTFX_MISS_STAT_ACT[8:0] - counter[8:0] @@ -2072,12 +2234,15 @@ static void rvu_npc_setup_interfaces(struct rvu *rvu, int blkaddr) rvu_write64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(intf), tx_kex); + if (is_cn20k(rvu->pdev)) + reg = NPC_AF_INTFX_MISS_ACTX(intf, 0); + else + reg = NPC_AF_INTFX_MISS_ACT(intf); + /* Set TX miss action to UCAST_DEFAULT i.e * transmit the packet on NIX LF SQ's default channel. */ - rvu_write64(rvu, blkaddr, - NPC_AF_INTFX_MISS_ACT(intf), - NIX_TX_ACTIONOP_UCAST_DEFAULT); + rvu_write64(rvu, blkaddr, reg, NIX_TX_ACTIONOP_UCAST_DEFAULT); } } @@ -2118,7 +2283,10 @@ int rvu_npc_init(struct rvu *rvu) return -ENOMEM; /* Configure KPU profile */ - npc_parser_profile_init(rvu, blkaddr); + if (is_cn20k(rvu->pdev)) + npc_cn20k_parser_profile_init(rvu, blkaddr); + else + npc_parser_profile_init(rvu, blkaddr); /* Config Outer L2, IPv4's NPC layer info */ rvu_write64(rvu, blkaddr, NPC_AF_PCK_DEF_OL2, @@ -2149,7 +2317,10 @@ int rvu_npc_init(struct rvu *rvu) npc_config_secret_key(rvu, blkaddr); /* Configure MKEX profile */ - npc_load_mkex_profile(rvu, blkaddr, rvu->mkex_pfl_name); + if (is_cn20k(rvu->pdev)) + npc_cn20k_load_mkex_profile(rvu, blkaddr, rvu->mkex_pfl_name); + else + npc_load_mkex_profile(rvu, blkaddr, rvu->mkex_pfl_name); err = npc_mcam_rsrcs_init(rvu, blkaddr); if (err) @@ -2159,9 +2330,15 @@ int rvu_npc_init(struct rvu *rvu) if (err) { dev_err(rvu->dev, "Incorrect mkex profile loaded using default mkex\n"); - npc_load_mkex_profile(rvu, blkaddr, def_pfl_name); + if (is_cn20k(rvu->pdev)) + npc_cn20k_load_mkex_profile(rvu, blkaddr, def_pfl_name); + else + npc_load_mkex_profile(rvu, blkaddr, def_pfl_name); } + if (is_cn20k(rvu->pdev)) + return npc_cn20k_init(rvu); + return 0; } @@ -2177,6 +2354,9 @@ void rvu_npc_freemem(struct rvu *rvu) else kfree(rvu->kpu_fwdata); mutex_destroy(&mcam->lock); + + if (is_cn20k(rvu->pdev)) + npc_cn20k_deinit(rvu); } void rvu_npc_get_mcam_entry_alloc_info(struct rvu *rvu, u16 pcifunc, @@ -2217,8 +2397,8 @@ void rvu_npc_get_mcam_counter_alloc_info(struct rvu *rvu, u16 pcifunc, } } -static int npc_mcam_verify_entry(struct npc_mcam *mcam, - u16 pcifunc, int entry) +int npc_mcam_verify_entry(struct npc_mcam *mcam, + u16 pcifunc, int entry) { /* verify AF installed entries */ if (is_pffunc_af(pcifunc)) @@ -2260,6 +2440,10 @@ static void npc_map_mcam_entry_and_cntr(struct rvu *rvu, struct npc_mcam *mcam, /* Set mapping and increment counter's refcnt */ mcam->entry2cntr_map[entry] = cntr; mcam->cntr_refcnt[cntr]++; + + if (is_cn20k(rvu->pdev)) + return; + /* Enable stats */ rvu_write64(rvu, blkaddr, NPC_AF_MCAMEX_BANKX_STAT_ACT(index, bank), @@ -2276,6 +2460,10 @@ static void npc_unmap_mcam_entry_and_cntr(struct rvu *rvu, /* Remove mapping and reduce counter's refcnt */ mcam->entry2cntr_map[entry] = NPC_MCAM_INVALID_MAP; mcam->cntr_refcnt[cntr]--; + + if (is_cn20k(rvu->pdev)) + return; + /* Disable stats */ rvu_write64(rvu, blkaddr, NPC_AF_MCAMEX_BANKX_STAT_ACT(index, bank), 0x00); @@ -2285,7 +2473,7 @@ static void npc_unmap_mcam_entry_and_cntr(struct rvu *rvu, * reverse bitmap too. Should be called with * 'mcam->lock' held. */ -static void npc_mcam_set_bit(struct npc_mcam *mcam, u16 index) +void npc_mcam_set_bit(struct npc_mcam *mcam, u16 index) { u16 entry, rentry; @@ -2301,7 +2489,7 @@ static void npc_mcam_set_bit(struct npc_mcam *mcam, u16 index) * reverse bitmap too. Should be called with * 'mcam->lock' held. */ -static void npc_mcam_clear_bit(struct npc_mcam *mcam, u16 index) +void npc_mcam_clear_bit(struct npc_mcam *mcam, u16 index) { u16 entry, rentry; @@ -2317,6 +2505,7 @@ static void npc_mcam_free_all_entries(struct rvu *rvu, struct npc_mcam *mcam, int blkaddr, u16 pcifunc) { u16 index, cntr; + int rc; /* Scan all MCAM entries and free the ones mapped to 'pcifunc' */ for (index = 0; index < mcam->bmap_entries; index++) { @@ -2334,6 +2523,13 @@ static void npc_mcam_free_all_entries(struct rvu *rvu, struct npc_mcam *mcam, blkaddr, index, cntr); mcam->entry2target_pffunc[index] = 0x0; + if (is_cn20k(rvu->pdev)) { + rc = npc_cn20k_idx_free(rvu, &index, 1); + if (rc) + dev_err(rvu->dev, + "Failed to free mcam idx=%u pcifunc=%#x\n", + index, pcifunc); + } } } } @@ -2423,7 +2619,7 @@ npc_get_mcam_search_range_priority(struct npc_mcam *mcam, { u16 fcnt; - if (req->priority == NPC_MCAM_HIGHER_PRIO) + if (req->ref_prio == NPC_MCAM_HIGHER_PRIO) goto hprio; /* For a low priority entry allocation @@ -2480,14 +2676,80 @@ static int npc_mcam_alloc_entries(struct npc_mcam *mcam, u16 pcifunc, struct npc_mcam_alloc_entry_req *req, struct npc_mcam_alloc_entry_rsp *rsp) { + struct rvu_hwinfo *hw = container_of(mcam, struct rvu_hwinfo, mcam); u16 entry_list[NPC_MAX_NONCONTIG_ENTRIES]; u16 fcnt, hp_fcnt, lp_fcnt; + struct rvu *rvu = hw->rvu; u16 start, end, index; int entry, next_start; bool reverse = false; unsigned long *bmap; + int ret, limit = 0; u16 max_contig; + if (!is_cn20k(rvu->pdev)) + goto not_cn20k; + + /* Only x2 or x4 key types are accepted */ + if (req->kw_type != NPC_MCAM_KEY_X2 && req->kw_type != NPC_MCAM_KEY_X4) + return NPC_MCAM_INVALID_REQ; + + /* The below table is being followed during allocation, + * + * 1. ref_entry == 0 && prio == HIGH && count == 1 : user wants to + * allocate 0th index + * 2. ref_entry == 0 && prio == HIGH && count > 1 : Invalid request + * 3. ref_entry == 0 && prio == LOW && count >= 1 : limit = 0 + * 4. ref_entry != 0 && prio == HIGH && count >= 1 : limit = 0 + * 5. ref_entry != 0 && prio == LOW && count >=1 : limit = Max + * (X2 2*8192, X4 8192) + */ + if (req->ref_entry && req->ref_prio == NPC_MCAM_LOWER_PRIO) { + if (req->kw_type == NPC_MCAM_KEY_X2) + limit = 2 * mcam->bmap_entries; + else + limit = mcam->bmap_entries; + } + + ret = npc_cn20k_ref_idx_alloc(rvu, pcifunc, req->kw_type, + req->ref_prio, rsp->entry_list, + req->ref_entry, limit, + req->contig, req->count, !!req->virt); + + if (ret) { + rsp->count = 0; + return NPC_MCAM_ALLOC_FAILED; + } + + rsp->count = req->count; + if (req->contig) + rsp->entry = rsp->entry_list[0]; + + /* cn20k, entries allocation algorithm is different. + * This common API updates some bitmap on usage etc, which + * will be used by other functions. So update those for + * cn20k as well. + */ + + mutex_lock(&mcam->lock); + /* Mark the allocated entries as used and set nixlf mapping */ + for (entry = 0; entry < rsp->count; entry++) { + index = npc_cn20k_vidx2idx(rsp->entry_list[entry]); + npc_mcam_set_bit(mcam, index); + mcam->entry2pfvf_map[index] = pcifunc; + mcam->entry2cntr_map[index] = NPC_MCAM_INVALID_MAP; + } + + /* cn20k, free count is provided thru different mbox message. + * one counter to indicate free x2 slots and free x4 slots + * does not provide any useful information to the user. + */ + rsp->free_count = -1; + mutex_unlock(&mcam->lock); + + return 0; + +not_cn20k: mutex_lock(&mcam->lock); /* Check if there are any free entries */ @@ -2523,7 +2785,7 @@ static int npc_mcam_alloc_entries(struct npc_mcam *mcam, u16 pcifunc, goto lprio_alloc; /* Get the search range for priority allocation request */ - if (req->priority) { + if (req->ref_prio) { npc_get_mcam_search_range_priority(mcam, req, &start, &end, &reverse); goto alloc; @@ -2564,11 +2826,11 @@ lprio_alloc: * and not in mid zone. */ if (!(pcifunc & RVU_PFVF_FUNC_MASK) && - req->priority == NPC_MCAM_HIGHER_PRIO) + req->ref_prio == NPC_MCAM_HIGHER_PRIO) end = req->ref_entry; if (!(pcifunc & RVU_PFVF_FUNC_MASK) && - req->priority == NPC_MCAM_LOWER_PRIO) + req->ref_prio == NPC_MCAM_LOWER_PRIO) start = req->ref_entry; } @@ -2617,7 +2879,7 @@ alloc: /* If allocating requested no of entries is unsucessful, * expand the search range to full bitmap length and retry. */ - if (!req->priority && (rsp->count < req->count) && + if (!req->ref_prio && rsp->count < req->count && ((end - start) != mcam->bmap_entries)) { reverse = true; start = 0; @@ -2628,14 +2890,14 @@ alloc: /* For priority entry allocation requests, if allocation is * failed then expand search to max possible range and retry. */ - if (req->priority && rsp->count < req->count) { - if (req->priority == NPC_MCAM_LOWER_PRIO && + if (req->ref_prio && rsp->count < req->count) { + if (req->ref_prio == NPC_MCAM_LOWER_PRIO && (start != (req->ref_entry + 1))) { start = req->ref_entry + 1; end = mcam->bmap_entries; reverse = false; goto alloc; - } else if ((req->priority == NPC_MCAM_HIGHER_PRIO) && + } else if ((req->ref_prio == NPC_MCAM_HIGHER_PRIO) && ((end - start) != req->ref_entry)) { start = 0; end = req->ref_entry; @@ -2689,6 +2951,10 @@ int npc_config_cntr_default_entries(struct rvu *rvu, bool enable) struct rvu_npc_mcam_rule *rule; int blkaddr; + /* Counter is set for each rule by default */ + if (is_cn20k(rvu->pdev)) + return -EINVAL; + blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); if (blkaddr < 0) return -EINVAL; @@ -2749,9 +3015,9 @@ int rvu_mbox_handler_npc_mcam_alloc_entry(struct rvu *rvu, /* ref_entry can't be '0' if requested priority is high. * Can't be last entry if requested priority is low. */ - if ((!req->ref_entry && req->priority == NPC_MCAM_HIGHER_PRIO) || - ((req->ref_entry == mcam->bmap_entries) && - req->priority == NPC_MCAM_LOWER_PRIO)) + if ((!req->ref_entry && req->ref_prio == NPC_MCAM_HIGHER_PRIO) || + (req->ref_entry == mcam->bmap_entries && + req->ref_prio == NPC_MCAM_LOWER_PRIO)) return NPC_MCAM_INVALID_REQ; /* Since list of allocated indices needs to be sent to requester, @@ -2780,6 +3046,8 @@ int rvu_mbox_handler_npc_mcam_free_entry(struct rvu *rvu, int blkaddr, rc = 0; u16 cntr; + req->entry = npc_cn20k_vidx2idx(req->entry); + blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); if (blkaddr < 0) return NPC_MCAM_INVALID_REQ; @@ -2808,6 +3076,14 @@ int rvu_mbox_handler_npc_mcam_free_entry(struct rvu *rvu, npc_unmap_mcam_entry_and_cntr(rvu, mcam, blkaddr, req->entry, cntr); + if (is_cn20k(rvu->pdev)) { + rc = npc_cn20k_idx_free(rvu, &req->entry, 1); + if (rc) + dev_err(rvu->dev, + "Failed to free index=%u\n", + req->entry); + } + goto exit; free_all: @@ -2861,10 +3137,13 @@ int rvu_mbox_handler_npc_mcam_write_entry(struct rvu *rvu, if (rc) goto exit; - if (req->set_cntr && - npc_mcam_verify_counter(mcam, pcifunc, req->cntr)) { - rc = NPC_MCAM_INVALID_REQ; - goto exit; + if (!is_cn20k(rvu->pdev)) { + /* Verify counter in SoCs other than cn20k */ + if (req->set_cntr && + npc_mcam_verify_counter(mcam, pcifunc, req->cntr)) { + rc = NPC_MCAM_INVALID_REQ; + goto exit; + } } if (!is_npc_interface_valid(rvu, req->intf)) { @@ -2902,6 +3181,8 @@ int rvu_mbox_handler_npc_mcam_ena_entry(struct rvu *rvu, u16 pcifunc = req->hdr.pcifunc; int blkaddr, rc; + req->entry = npc_cn20k_vidx2idx(req->entry); + blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); if (blkaddr < 0) return NPC_MCAM_INVALID_REQ; @@ -2925,6 +3206,8 @@ int rvu_mbox_handler_npc_mcam_dis_entry(struct rvu *rvu, u16 pcifunc = req->hdr.pcifunc; int blkaddr, rc; + req->entry = npc_cn20k_vidx2idx(req->entry); + blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); if (blkaddr < 0) return NPC_MCAM_INVALID_REQ; @@ -2959,8 +3242,8 @@ int rvu_mbox_handler_npc_mcam_shift_entry(struct rvu *rvu, mutex_lock(&mcam->lock); for (index = 0; index < req->shift_count; index++) { - old_entry = req->curr_entry[index]; - new_entry = req->new_entry[index]; + old_entry = npc_cn20k_vidx2idx(req->curr_entry[index]); + new_entry = npc_cn20k_vidx2idx(req->new_entry[index]); /* Check if both old and new entries are valid and * does belong to this PFFUNC or not. @@ -3032,7 +3315,6 @@ static int __npc_mcam_alloc_counter(struct rvu *rvu, if (!req->contig && req->count > NPC_MAX_NONCONTIG_COUNTERS) return NPC_MCAM_INVALID_REQ; - /* Check if unused counters are available or not */ if (!rvu_rsrc_free_count(&mcam->counters)) { return NPC_MCAM_ALLOC_FAILED; @@ -3077,6 +3359,10 @@ int rvu_mbox_handler_npc_mcam_alloc_counter(struct rvu *rvu, struct npc_mcam *mcam = &rvu->hw->mcam; int err; + /* Counter is not supported for CN20K */ + if (is_cn20k(rvu->pdev)) + return NPC_MCAM_INVALID_REQ; + mutex_lock(&mcam->lock); err = __npc_mcam_alloc_counter(rvu, req, rsp); @@ -3131,6 +3417,10 @@ int rvu_mbox_handler_npc_mcam_free_counter(struct rvu *rvu, struct npc_mcam *mcam = &rvu->hw->mcam; int err; + /* Counter is not supported for CN20K */ + if (is_cn20k(rvu->pdev)) + return NPC_MCAM_INVALID_REQ; + mutex_lock(&mcam->lock); err = __npc_mcam_free_counter(rvu, req, rsp); @@ -3189,6 +3479,10 @@ int rvu_mbox_handler_npc_mcam_unmap_counter(struct rvu *rvu, u16 index, entry = 0; int blkaddr, rc; + /* Counter is not supported for CN20K */ + if (is_cn20k(rvu->pdev)) + return NPC_MCAM_INVALID_REQ; + blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); if (blkaddr < 0) return NPC_MCAM_INVALID_REQ; @@ -3233,12 +3527,23 @@ int rvu_mbox_handler_npc_mcam_clear_counter(struct rvu *rvu, struct npc_mcam_oper_counter_req *req, struct msg_rsp *rsp) { struct npc_mcam *mcam = &rvu->hw->mcam; - int blkaddr, err; + int blkaddr, err, index, bank; blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); if (blkaddr < 0) return NPC_MCAM_INVALID_REQ; + /* For cn20k, npc mcam index is passed as cntr, as each + * mcam entry has corresponding counter. + */ + if (is_cn20k(rvu->pdev)) { + index = req->cntr & (mcam->banksize - 1); + bank = npc_get_bank(mcam, req->cntr); + rvu_write64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_STAT_EXT(index, bank), 0); + return 0; + } + mutex_lock(&mcam->lock); err = npc_mcam_verify_counter(mcam, req->hdr.pcifunc, req->cntr); mutex_unlock(&mcam->lock); @@ -3255,12 +3560,26 @@ int rvu_mbox_handler_npc_mcam_counter_stats(struct rvu *rvu, struct npc_mcam_oper_counter_rsp *rsp) { struct npc_mcam *mcam = &rvu->hw->mcam; - int blkaddr, err; + int blkaddr, err, index, bank; + u64 regval; blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); if (blkaddr < 0) return NPC_MCAM_INVALID_REQ; + /* In CN20K, mcam index is passed cntr. Each cn20k mcam entry + * has its own counter. No need to verify the counter index. + */ + if (is_cn20k(rvu->pdev)) { + index = req->cntr & (mcam->banksize - 1); + bank = npc_get_bank(mcam, req->cntr); + regval = rvu_read64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_STAT_EXT(index, + bank)); + rsp->stat = regval; + return 0; + } + mutex_lock(&mcam->lock); err = npc_mcam_verify_counter(mcam, req->hdr.pcifunc, req->cntr); mutex_unlock(&mcam->lock); @@ -3298,7 +3617,7 @@ int rvu_mbox_handler_npc_mcam_alloc_and_write_entry(struct rvu *rvu, /* Try to allocate a MCAM entry */ entry_req.hdr.pcifunc = req->hdr.pcifunc; entry_req.contig = true; - entry_req.priority = req->priority; + entry_req.ref_prio = req->ref_prio; entry_req.ref_entry = req->ref_entry; entry_req.count = 1; @@ -3555,13 +3874,26 @@ int rvu_mbox_handler_npc_mcam_entry_stats(struct rvu *rvu, if (blkaddr < 0) return NPC_MCAM_INVALID_REQ; - mutex_lock(&mcam->lock); + req->entry = npc_cn20k_vidx2idx(req->entry); index = req->entry & (mcam->banksize - 1); bank = npc_get_bank(mcam, req->entry); + mutex_lock(&mcam->lock); + + if (is_cn20k(rvu->pdev)) { + regval = rvu_read64(rvu, blkaddr, + NPC_AF_CN20K_MCAMEX_BANKX_STAT_EXT(index, + bank)); + rsp->stat_ena = 1; + rsp->stat = regval; + mutex_unlock(&mcam->lock); + return 0; + } + /* read MCAM entry STAT_ACT register */ - regval = rvu_read64(rvu, blkaddr, NPC_AF_MCAMEX_BANKX_STAT_ACT(index, bank)); + regval = rvu_read64(rvu, blkaddr, NPC_AF_MCAMEX_BANKX_STAT_ACT(index, + bank)); if (!(regval & rvu->hw->npc_stat_ena)) { rsp->stat_ena = 0; diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc.h b/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc.h new file mode 100644 index 000000000000..83c5e32e2afc --- /dev/null +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Marvell RVU Admin Function driver + * + * Copyright (C) 2026 Marvell. + * + */ + +#ifndef RVU_NPC_H +#define RVU_NPC_H + +u64 npc_enable_mask(int count); +void npc_load_kpu_profile(struct rvu *rvu); +void npc_config_kpuaction(struct rvu *rvu, int blkaddr, + const struct npc_kpu_profile_action *kpuaction, + int kpu, int entry, bool pkind); +int npc_fwdb_prfl_img_map(struct rvu *rvu, void __iomem **prfl_img_addr, + u64 *size); + +void npc_mcam_clear_bit(struct npc_mcam *mcam, u16 index); +void npc_mcam_set_bit(struct npc_mcam *mcam, u16 index); +#endif /* RVU_NPC_H */ diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc_fs.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc_fs.c index 1930b54e72f2..b45798d9fdab 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc_fs.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc_fs.c @@ -12,6 +12,8 @@ #include "npc.h" #include "rvu_npc_fs.h" #include "rvu_npc_hash.h" +#include "cn20k/reg.h" +#include "cn20k/npc.h" static const char * const npc_flow_names[] = { [NPC_DMAC] = "dmac", @@ -81,19 +83,26 @@ const char *npc_get_field_name(u8 hdr) /* Compute keyword masks and figure out the number of keywords a field * spans in the key. */ -static void npc_set_kw_masks(struct npc_mcam *mcam, u8 type, +static void npc_set_kw_masks(struct rvu *rvu, struct npc_mcam *mcam, u8 type, u8 nr_bits, int start_kwi, int offset, u8 intf) { struct npc_key_field *field = &mcam->rx_key_fields[type]; u8 bits_in_kw; int max_kwi; - if (mcam->banks_per_entry == 1) - max_kwi = 1; /* NPC_MCAM_KEY_X1 */ - else if (mcam->banks_per_entry == 2) - max_kwi = 3; /* NPC_MCAM_KEY_X2 */ - else - max_kwi = 6; /* NPC_MCAM_KEY_X4 */ + if (is_cn20k(rvu->pdev)) { + if (mcam->banks_per_entry == 1) + max_kwi = 3; /* NPC_MCAM_KEY_X2 */ + else + max_kwi = 7; /* NPC_MCAM_KEY_X4 */ + } else { + if (mcam->banks_per_entry == 1) + max_kwi = 1; /* NPC_MCAM_KEY_X1 */ + else if (mcam->banks_per_entry == 2) + max_kwi = 3; /* NPC_MCAM_KEY_X2 */ + else + max_kwi = 6; /* NPC_MCAM_KEY_X4 */ + } if (is_npc_intf_tx(intf)) field = &mcam->tx_key_fields[type]; @@ -155,7 +164,8 @@ static bool npc_is_same(struct npc_key_field *input, sizeof(struct npc_layer_mdata)) == 0; } -static void npc_set_layer_mdata(struct npc_mcam *mcam, enum key_fields type, +static void npc_set_layer_mdata(struct rvu *rvu, + struct npc_mcam *mcam, enum key_fields type, u64 cfg, u8 lid, u8 lt, u8 intf) { struct npc_key_field *input = &mcam->rx_key_fields[type]; @@ -165,13 +175,17 @@ static void npc_set_layer_mdata(struct npc_mcam *mcam, enum key_fields type, input->layer_mdata.hdr = FIELD_GET(NPC_HDR_OFFSET, cfg); input->layer_mdata.key = FIELD_GET(NPC_KEY_OFFSET, cfg); - input->layer_mdata.len = FIELD_GET(NPC_BYTESM, cfg) + 1; + if (is_cn20k(rvu->pdev)) + input->layer_mdata.len = FIELD_GET(NPC_CN20K_BYTESM, cfg) + 1; + else + input->layer_mdata.len = FIELD_GET(NPC_BYTESM, cfg) + 1; input->layer_mdata.ltype = lt; input->layer_mdata.lid = lid; } static bool npc_check_overlap_fields(struct npc_key_field *input1, - struct npc_key_field *input2) + struct npc_key_field *input2, + int max_kw) { int kwi; @@ -182,7 +196,7 @@ static bool npc_check_overlap_fields(struct npc_key_field *input1, input1->layer_mdata.ltype != input2->layer_mdata.ltype) return false; - for (kwi = 0; kwi < NPC_MAX_KWS_IN_KEY; kwi++) { + for (kwi = 0; kwi < max_kw; kwi++) { if (input1->kw_mask[kwi] & input2->kw_mask[kwi]) return true; } @@ -202,6 +216,7 @@ static bool npc_check_overlap(struct rvu *rvu, int blkaddr, struct npc_key_field *dummy, *input; int start_kwi, offset; u8 nr_bits, lid, lt, ld; + int extr, kws; u64 cfg; dummy = &mcam->rx_key_fields[NPC_UNKNOWN]; @@ -212,6 +227,11 @@ static bool npc_check_overlap(struct rvu *rvu, int blkaddr, input = &mcam->tx_key_fields[type]; } + kws = NPC_KWS_IN_KEY_SZ_7; + + if (is_cn20k(rvu->pdev)) + goto skip_cn10k_config; + for (lid = start_lid; lid < NPC_MAX_LID; lid++) { for (lt = 0; lt < NPC_MAX_LT; lt++) { for (ld = 0; ld < NPC_MAX_LD; ld++) { @@ -221,8 +241,8 @@ static bool npc_check_overlap(struct rvu *rvu, int blkaddr, if (!FIELD_GET(NPC_LDATA_EN, cfg)) continue; memset(dummy, 0, sizeof(struct npc_key_field)); - npc_set_layer_mdata(mcam, NPC_UNKNOWN, cfg, - lid, lt, intf); + npc_set_layer_mdata(rvu, mcam, NPC_UNKNOWN, + cfg, lid, lt, intf); /* exclude input */ if (npc_is_same(input, dummy)) continue; @@ -230,16 +250,49 @@ static bool npc_check_overlap(struct rvu *rvu, int blkaddr, offset = (dummy->layer_mdata.key * 8) % 64; nr_bits = dummy->layer_mdata.len * 8; /* form KW masks */ - npc_set_kw_masks(mcam, NPC_UNKNOWN, nr_bits, - start_kwi, offset, intf); + npc_set_kw_masks(rvu, mcam, NPC_UNKNOWN, + nr_bits, start_kwi, + offset, intf); /* check any input field bits falls in any * other field bits. */ - if (npc_check_overlap_fields(dummy, input)) + if (npc_check_overlap_fields(dummy, input, kws)) return true; } } } + return false; + +skip_cn10k_config: + for (extr = 0 ; extr < rvu->hw->npc_kex_extr; extr++) { + lid = CN20K_GET_EXTR_LID(intf, extr); + if (lid < start_lid) + continue; + for (lt = 0; lt < NPC_MAX_LT; lt++) { + cfg = CN20K_GET_EXTR_LT(intf, extr, lt); + if (!FIELD_GET(NPC_LDATA_EN, cfg)) + continue; + + memset(dummy, 0, sizeof(struct npc_key_field)); + npc_set_layer_mdata(rvu, mcam, NPC_UNKNOWN, cfg, + lid, lt, intf); + /* exclude input */ + if (npc_is_same(input, dummy)) + continue; + start_kwi = dummy->layer_mdata.key / 8; + offset = (dummy->layer_mdata.key * 8) % 64; + nr_bits = dummy->layer_mdata.len * 8; + /* form KW masks */ + npc_set_kw_masks(rvu, mcam, NPC_UNKNOWN, nr_bits, + start_kwi, offset, intf); + /* check any input field bits falls in any other + * field bits + */ + if (npc_check_overlap_fields(dummy, input, + NPC_KWS_IN_KEY_SZ_8)) + return true; + } + } return false; } @@ -253,7 +306,8 @@ static bool npc_check_field(struct rvu *rvu, int blkaddr, enum key_fields type, return true; } -static void npc_scan_exact_result(struct npc_mcam *mcam, u8 bit_number, +static void npc_scan_exact_result(struct rvu *rvu, + struct npc_mcam *mcam, u8 bit_number, u8 key_nibble, u8 intf) { u8 offset = (key_nibble * 4) % 64; /* offset within key word */ @@ -269,10 +323,63 @@ static void npc_scan_exact_result(struct npc_mcam *mcam, u8 bit_number, default: return; } - npc_set_kw_masks(mcam, type, nr_bits, kwi, offset, intf); + npc_set_kw_masks(rvu, mcam, type, nr_bits, kwi, offset, intf); +} + +static void npc_cn20k_scan_parse_result(struct rvu *rvu, struct npc_mcam *mcam, + u8 bit_number, u8 key_nibble, u8 intf) +{ + u8 offset = (key_nibble * 4) % 64; /* offset within key word */ + u8 kwi = (key_nibble * 4) / 64; /* which word in key */ + u8 nr_bits = 4; /* bits in a nibble */ + u8 type; + + switch (bit_number) { + case 0 ... 2: + type = NPC_CHAN; + break; + case 3: + type = NPC_ERRLEV; + break; + case 4 ... 5: + type = NPC_ERRCODE; + break; + case 6: + type = NPC_LXMB; + break; + case 8: + type = NPC_LA; + break; + case 10: + type = NPC_LB; + break; + case 12: + type = NPC_LC; + break; + case 14: + type = NPC_LD; + break; + case 16: + type = NPC_LE; + break; + case 18: + type = NPC_LF; + break; + case 20: + type = NPC_LG; + break; + case 22: + type = NPC_LH; + break; + default: + return; + } + + npc_set_kw_masks(rvu, mcam, type, nr_bits, kwi, offset, intf); } -static void npc_scan_parse_result(struct npc_mcam *mcam, u8 bit_number, +static void npc_scan_parse_result(struct rvu *rvu, + struct npc_mcam *mcam, u8 bit_number, u8 key_nibble, u8 intf) { u8 offset = (key_nibble * 4) % 64; /* offset within key word */ @@ -280,6 +387,12 @@ static void npc_scan_parse_result(struct npc_mcam *mcam, u8 bit_number, u8 nr_bits = 4; /* bits in a nibble */ u8 type; + if (is_cn20k(rvu->pdev)) { + npc_cn20k_scan_parse_result(rvu, mcam, bit_number, + key_nibble, intf); + return; + } + switch (bit_number) { case 0 ... 2: type = NPC_CHAN; @@ -322,7 +435,7 @@ static void npc_scan_parse_result(struct npc_mcam *mcam, u8 bit_number, return; } - npc_set_kw_masks(mcam, type, nr_bits, kwi, offset, intf); + npc_set_kw_masks(rvu, mcam, type, nr_bits, kwi, offset, intf); } static void npc_handle_multi_layer_fields(struct rvu *rvu, int blkaddr, u8 intf) @@ -343,8 +456,13 @@ static void npc_handle_multi_layer_fields(struct rvu *rvu, int blkaddr, u8 intf) /* Inner VLAN TCI for double tagged frames */ struct npc_key_field *vlan_tag3; u64 *features; + int i, max_kw; u8 start_lid; - int i; + + if (is_cn20k(rvu->pdev)) + max_kw = NPC_KWS_IN_KEY_SZ_8; + else + max_kw = NPC_KWS_IN_KEY_SZ_7; key_fields = mcam->rx_key_fields; features = &mcam->rx_features; @@ -382,7 +500,7 @@ static void npc_handle_multi_layer_fields(struct rvu *rvu, int blkaddr, u8 intf) /* if key profile programmed extracts Ethertype from multiple layers */ if (etype_ether->nr_kws && etype_tag1->nr_kws) { - for (i = 0; i < NPC_MAX_KWS_IN_KEY; i++) { + for (i = 0; i < max_kw; i++) { if (etype_ether->kw_mask[i] != etype_tag1->kw_mask[i]) { dev_err(rvu->dev, "mkex: Etype pos is different for untagged and tagged pkts.\n"); goto vlan_tci; @@ -391,7 +509,7 @@ static void npc_handle_multi_layer_fields(struct rvu *rvu, int blkaddr, u8 intf) key_fields[NPC_ETYPE] = *etype_tag1; } if (etype_ether->nr_kws && etype_tag2->nr_kws) { - for (i = 0; i < NPC_MAX_KWS_IN_KEY; i++) { + for (i = 0; i < max_kw; i++) { if (etype_ether->kw_mask[i] != etype_tag2->kw_mask[i]) { dev_err(rvu->dev, "mkex: Etype pos is different for untagged and double tagged pkts.\n"); goto vlan_tci; @@ -400,7 +518,7 @@ static void npc_handle_multi_layer_fields(struct rvu *rvu, int blkaddr, u8 intf) key_fields[NPC_ETYPE] = *etype_tag2; } if (etype_tag1->nr_kws && etype_tag2->nr_kws) { - for (i = 0; i < NPC_MAX_KWS_IN_KEY; i++) { + for (i = 0; i < max_kw; i++) { if (etype_tag1->kw_mask[i] != etype_tag2->kw_mask[i]) { dev_err(rvu->dev, "mkex: Etype pos is different for tagged and double tagged pkts.\n"); goto vlan_tci; @@ -431,7 +549,7 @@ vlan_tci: /* if key profile extracts outer vlan tci from multiple layers */ if (vlan_tag1->nr_kws && vlan_tag2->nr_kws) { - for (i = 0; i < NPC_MAX_KWS_IN_KEY; i++) { + for (i = 0; i < max_kw; i++) { if (vlan_tag1->kw_mask[i] != vlan_tag2->kw_mask[i]) { dev_err(rvu->dev, "mkex: Out vlan tci pos is different for tagged and double tagged pkts.\n"); goto done; @@ -466,7 +584,11 @@ static void npc_scan_ldata(struct rvu *rvu, int blkaddr, u8 lid, /* starting KW index and starting bit position */ int start_kwi, offset; - nr_bytes = FIELD_GET(NPC_BYTESM, cfg) + 1; + if (is_cn20k(rvu->pdev)) + nr_bytes = FIELD_GET(NPC_CN20K_BYTESM, cfg) + 1; + else + nr_bytes = FIELD_GET(NPC_BYTESM, cfg) + 1; + hdr = FIELD_GET(NPC_HDR_OFFSET, cfg); key = FIELD_GET(NPC_KEY_OFFSET, cfg); @@ -489,11 +611,12 @@ do { \ if ((hstart) >= hdr && \ ((hstart) + (hlen)) <= (hdr + nr_bytes)) { \ bit_offset = (hdr + nr_bytes - (hstart) - (hlen)) * 8; \ - npc_set_layer_mdata(mcam, (name), cfg, lid, lt, intf); \ + npc_set_layer_mdata(rvu, mcam, (name), cfg, lid, lt, \ + intf); \ offset += bit_offset; \ start_kwi += offset / 64; \ offset %= 64; \ - npc_set_kw_masks(mcam, (name), (hlen) * 8, \ + npc_set_kw_masks(rvu, mcam, (name), (hlen) * 8, \ start_kwi, offset, intf); \ } \ } \ @@ -636,6 +759,7 @@ static int npc_scan_kex(struct rvu *rvu, int blkaddr, u8 intf) u8 lid, lt, ld, bitnr; u64 cfg, masked_cfg; u8 key_nibble = 0; + int extr; /* Scan and note how parse result is going to be in key. * A bit set in PARSE_NIBBLE_ENA corresponds to a nibble from @@ -643,10 +767,22 @@ static int npc_scan_kex(struct rvu *rvu, int blkaddr, u8 intf) * will be concatenated in key. */ cfg = rvu_read64(rvu, blkaddr, NPC_AF_INTFX_KEX_CFG(intf)); - masked_cfg = cfg & NPC_PARSE_NIBBLE; - for_each_set_bit(bitnr, (unsigned long *)&masked_cfg, 31) { - npc_scan_parse_result(mcam, bitnr, key_nibble, intf); - key_nibble++; + if (is_cn20k(rvu->pdev)) { + masked_cfg = cfg & NPC_CN20K_PARSE_NIBBLE; + for_each_set_bit(bitnr, (unsigned long *)&masked_cfg, + NPC_CN20K_TOTAL_NIBBLE) { + npc_scan_parse_result(rvu, mcam, bitnr, + key_nibble, intf); + key_nibble++; + } + } else { + masked_cfg = cfg & NPC_PARSE_NIBBLE; + for_each_set_bit(bitnr, (unsigned long *)&masked_cfg, + NPC_TOTAL_NIBBLE) { + npc_scan_parse_result(rvu, mcam, bitnr, + key_nibble, intf); + key_nibble++; + } } /* Ignore exact match bits for mcam entries except the first rule @@ -656,10 +792,13 @@ static int npc_scan_kex(struct rvu *rvu, int blkaddr, u8 intf) masked_cfg = cfg & NPC_EXACT_NIBBLE; bitnr = NPC_EXACT_NIBBLE_START; for_each_set_bit_from(bitnr, (unsigned long *)&masked_cfg, NPC_EXACT_NIBBLE_END + 1) { - npc_scan_exact_result(mcam, bitnr, key_nibble, intf); + npc_scan_exact_result(rvu, mcam, bitnr, key_nibble, intf); key_nibble++; } + if (is_cn20k(rvu->pdev)) + goto skip_cn10k_config; + /* Scan and note how layer data is going to be in key */ for (lid = 0; lid < NPC_MAX_LID; lid++) { for (lt = 0; lt < NPC_MAX_LT; lt++) { @@ -676,6 +815,19 @@ static int npc_scan_kex(struct rvu *rvu, int blkaddr, u8 intf) } return 0; + +skip_cn10k_config: + for (extr = 0 ; extr < rvu->hw->npc_kex_extr; extr++) { + lid = CN20K_GET_EXTR_LID(intf, extr); + for (lt = 0; lt < NPC_MAX_LT; lt++) { + cfg = CN20K_GET_EXTR_LT(intf, extr, lt); + if (!FIELD_GET(NPC_LDATA_EN, cfg)) + continue; + npc_scan_ldata(rvu, blkaddr, lid, lt, cfg, + intf); + } + } + return 0; } static int npc_scan_verify_kex(struct rvu *rvu, int blkaddr) @@ -751,15 +903,17 @@ static int npc_check_unsupported_flows(struct rvu *rvu, u64 features, u8 intf) * dont care. */ void npc_update_entry(struct rvu *rvu, enum key_fields type, - struct mcam_entry *entry, u64 val_lo, + struct mcam_entry_mdata *mdata, u64 val_lo, u64 val_hi, u64 mask_lo, u64 mask_hi, u8 intf) { + u64 kw_mask[NPC_KWS_IN_KEY_SZ_MAX] = { 0 }; + u64 kw[NPC_KWS_IN_KEY_SZ_MAX] = { 0 }; struct npc_mcam *mcam = &rvu->hw->mcam; - struct mcam_entry dummy = { {0} }; struct npc_key_field *field; u64 kw1, kw2, kw3; + u64 *val, *mask; + int i, max_kw; u8 shift; - int i; field = &mcam->rx_key_fields[type]; if (is_npc_intf_tx(intf)) @@ -768,17 +922,22 @@ void npc_update_entry(struct rvu *rvu, enum key_fields type, if (!field->nr_kws) return; - for (i = 0; i < NPC_MAX_KWS_IN_KEY; i++) { + if (is_cn20k(rvu->pdev)) + max_kw = NPC_KWS_IN_KEY_SZ_8; + else + max_kw = NPC_KWS_IN_KEY_SZ_7; + + for (i = 0; i < max_kw; i++) { if (!field->kw_mask[i]) continue; /* place key value in kw[x] */ shift = __ffs64(field->kw_mask[i]); /* update entry value */ kw1 = (val_lo << shift) & field->kw_mask[i]; - dummy.kw[i] = kw1; + kw[i] = kw1; /* update entry mask */ kw1 = (mask_lo << shift) & field->kw_mask[i]; - dummy.kw_mask[i] = kw1; + kw_mask[i] = kw1; if (field->nr_kws == 1) break; @@ -788,12 +947,12 @@ void npc_update_entry(struct rvu *rvu, enum key_fields type, kw2 = shift ? val_lo >> (64 - shift) : 0; kw2 |= (val_hi << shift); kw2 &= field->kw_mask[i + 1]; - dummy.kw[i + 1] = kw2; + kw[i + 1] = kw2; /* update entry mask */ kw2 = shift ? mask_lo >> (64 - shift) : 0; kw2 |= (mask_hi << shift); kw2 &= field->kw_mask[i + 1]; - dummy.kw_mask[i + 1] = kw2; + kw_mask[i + 1] = kw2; break; } /* place remaining bits of key value in kw[x + 1], kw[x + 2] */ @@ -804,34 +963,40 @@ void npc_update_entry(struct rvu *rvu, enum key_fields type, kw2 &= field->kw_mask[i + 1]; kw3 = shift ? val_hi >> (64 - shift) : 0; kw3 &= field->kw_mask[i + 2]; - dummy.kw[i + 1] = kw2; - dummy.kw[i + 2] = kw3; + kw[i + 1] = kw2; + kw[i + 2] = kw3; /* update entry mask */ kw2 = shift ? mask_lo >> (64 - shift) : 0; kw2 |= (mask_hi << shift); kw2 &= field->kw_mask[i + 1]; kw3 = shift ? mask_hi >> (64 - shift) : 0; kw3 &= field->kw_mask[i + 2]; - dummy.kw_mask[i + 1] = kw2; - dummy.kw_mask[i + 2] = kw3; + kw_mask[i + 1] = kw2; + kw_mask[i + 2] = kw3; break; } } /* dummy is ready with values and masks for given key * field now clear and update input entry with those */ - for (i = 0; i < NPC_MAX_KWS_IN_KEY; i++) { + + val = mdata->kw; + mask = mdata->kw_mask; + + for (i = 0; i < max_kw; i++, val++, mask++) { if (!field->kw_mask[i]) continue; - entry->kw[i] &= ~field->kw_mask[i]; - entry->kw_mask[i] &= ~field->kw_mask[i]; - entry->kw[i] |= dummy.kw[i]; - entry->kw_mask[i] |= dummy.kw_mask[i]; + *val &= ~field->kw_mask[i]; + *mask &= ~field->kw_mask[i]; + + *val |= kw[i]; + *mask |= kw_mask[i]; } } -static void npc_update_ipv6_flow(struct rvu *rvu, struct mcam_entry *entry, +static void npc_update_ipv6_flow(struct rvu *rvu, + struct mcam_entry_mdata *mdata, u64 features, struct flow_msg *pkt, struct flow_msg *mask, struct rvu_npc_mcam_rule *output, u8 intf) @@ -857,7 +1022,7 @@ static void npc_update_ipv6_flow(struct rvu *rvu, struct mcam_entry *entry, val_hi = (u64)src_ip[0] << 32 | src_ip[1]; val_lo = (u64)src_ip[2] << 32 | src_ip[3]; - npc_update_entry(rvu, NPC_SIP_IPV6, entry, val_lo, val_hi, + npc_update_entry(rvu, NPC_SIP_IPV6, mdata, val_lo, val_hi, mask_lo, mask_hi, intf); memcpy(opkt->ip6src, pkt->ip6src, sizeof(opkt->ip6src)); memcpy(omask->ip6src, mask->ip6src, sizeof(omask->ip6src)); @@ -871,14 +1036,15 @@ static void npc_update_ipv6_flow(struct rvu *rvu, struct mcam_entry *entry, val_hi = (u64)dst_ip[0] << 32 | dst_ip[1]; val_lo = (u64)dst_ip[2] << 32 | dst_ip[3]; - npc_update_entry(rvu, NPC_DIP_IPV6, entry, val_lo, val_hi, + npc_update_entry(rvu, NPC_DIP_IPV6, mdata, val_lo, val_hi, mask_lo, mask_hi, intf); memcpy(opkt->ip6dst, pkt->ip6dst, sizeof(opkt->ip6dst)); memcpy(omask->ip6dst, mask->ip6dst, sizeof(omask->ip6dst)); } } -static void npc_update_vlan_features(struct rvu *rvu, struct mcam_entry *entry, +static void npc_update_vlan_features(struct rvu *rvu, + struct mcam_entry_mdata *mdata, u64 features, u8 intf) { bool ctag = !!(features & BIT_ULL(NPC_VLAN_ETYPE_CTAG)); @@ -887,24 +1053,24 @@ static void npc_update_vlan_features(struct rvu *rvu, struct mcam_entry *entry, /* If only VLAN id is given then always match outer VLAN id */ if (vid && !ctag && !stag) { - npc_update_entry(rvu, NPC_LB, entry, + npc_update_entry(rvu, NPC_LB, mdata, NPC_LT_LB_STAG_QINQ | NPC_LT_LB_CTAG, 0, NPC_LT_LB_STAG_QINQ & NPC_LT_LB_CTAG, 0, intf); return; } if (ctag) - npc_update_entry(rvu, NPC_LB, entry, NPC_LT_LB_CTAG, 0, + npc_update_entry(rvu, NPC_LB, mdata, NPC_LT_LB_CTAG, 0, ~0ULL, 0, intf); if (stag) - npc_update_entry(rvu, NPC_LB, entry, NPC_LT_LB_STAG_QINQ, 0, + npc_update_entry(rvu, NPC_LB, mdata, NPC_LT_LB_STAG_QINQ, 0, ~0ULL, 0, intf); } -static void npc_update_flow(struct rvu *rvu, struct mcam_entry *entry, - u64 features, struct flow_msg *pkt, - struct flow_msg *mask, - struct rvu_npc_mcam_rule *output, u8 intf, - int blkaddr) +void npc_update_flow(struct rvu *rvu, struct mcam_entry_mdata *mdata, + u64 features, struct flow_msg *pkt, + struct flow_msg *mask, + struct rvu_npc_mcam_rule *output, u8 intf, + int blkaddr) { u64 dmac_mask = ether_addr_to_u64(mask->dmac); u64 smac_mask = ether_addr_to_u64(mask->smac); @@ -918,39 +1084,39 @@ static void npc_update_flow(struct rvu *rvu, struct mcam_entry *entry, /* For tcp/udp/sctp LTYPE should be present in entry */ if (features & BIT_ULL(NPC_IPPROTO_TCP)) - npc_update_entry(rvu, NPC_LD, entry, NPC_LT_LD_TCP, + npc_update_entry(rvu, NPC_LD, mdata, NPC_LT_LD_TCP, 0, ~0ULL, 0, intf); if (features & BIT_ULL(NPC_IPPROTO_UDP)) - npc_update_entry(rvu, NPC_LD, entry, NPC_LT_LD_UDP, + npc_update_entry(rvu, NPC_LD, mdata, NPC_LT_LD_UDP, 0, ~0ULL, 0, intf); if (features & BIT_ULL(NPC_IPPROTO_SCTP)) - npc_update_entry(rvu, NPC_LD, entry, NPC_LT_LD_SCTP, + npc_update_entry(rvu, NPC_LD, mdata, NPC_LT_LD_SCTP, 0, ~0ULL, 0, intf); if (features & BIT_ULL(NPC_IPPROTO_ICMP)) - npc_update_entry(rvu, NPC_LD, entry, NPC_LT_LD_ICMP, + npc_update_entry(rvu, NPC_LD, mdata, NPC_LT_LD_ICMP, 0, ~0ULL, 0, intf); if (features & BIT_ULL(NPC_IPPROTO_ICMP6)) - npc_update_entry(rvu, NPC_LD, entry, NPC_LT_LD_ICMP6, + npc_update_entry(rvu, NPC_LD, mdata, NPC_LT_LD_ICMP6, 0, ~0ULL, 0, intf); /* For AH, LTYPE should be present in entry */ if (features & BIT_ULL(NPC_IPPROTO_AH)) - npc_update_entry(rvu, NPC_LD, entry, NPC_LT_LD_AH, + npc_update_entry(rvu, NPC_LD, mdata, NPC_LT_LD_AH, 0, ~0ULL, 0, intf); /* For ESP, LTYPE should be present in entry */ if (features & BIT_ULL(NPC_IPPROTO_ESP)) - npc_update_entry(rvu, NPC_LE, entry, NPC_LT_LE_ESP, + npc_update_entry(rvu, NPC_LE, mdata, NPC_LT_LE_ESP, 0, ~0ULL, 0, intf); if (features & BIT_ULL(NPC_LXMB)) { output->lxmb = is_broadcast_ether_addr(pkt->dmac) ? 2 : 1; - npc_update_entry(rvu, NPC_LXMB, entry, output->lxmb, 0, + npc_update_entry(rvu, NPC_LXMB, mdata, output->lxmb, 0, output->lxmb, 0, intf); } #define NPC_WRITE_FLOW(field, member, val_lo, val_hi, mask_lo, mask_hi) \ do { \ if (features & BIT_ULL((field))) { \ - npc_update_entry(rvu, (field), entry, (val_lo), (val_hi), \ + npc_update_entry(rvu, (field), mdata, (val_lo), (val_hi), \ (mask_lo), (mask_hi), intf); \ memcpy(&opkt->member, &pkt->member, sizeof(pkt->member)); \ memcpy(&omask->member, &mask->member, sizeof(mask->member)); \ @@ -1038,10 +1204,10 @@ do { \ NPC_WRITE_FLOW(NPC_IPFRAG_IPV6, next_header, pkt->next_header, 0, mask->next_header, 0); - npc_update_ipv6_flow(rvu, entry, features, pkt, mask, output, intf); - npc_update_vlan_features(rvu, entry, features, intf); + npc_update_ipv6_flow(rvu, mdata, features, pkt, mask, output, intf); + npc_update_vlan_features(rvu, mdata, features, intf); - npc_update_field_hash(rvu, intf, entry, blkaddr, features, + npc_update_field_hash(rvu, intf, mdata, blkaddr, features, pkt, mask, opkt, omask); } @@ -1083,6 +1249,10 @@ static void rvu_mcam_remove_counter_from_rule(struct rvu *rvu, u16 pcifunc, { struct npc_mcam *mcam = &rvu->hw->mcam; + /* There is no counter allotted for cn20k */ + if (is_cn20k(rvu->pdev)) + return; + mutex_lock(&mcam->lock); __rvu_mcam_remove_counter_from_rule(rvu, pcifunc, rule); @@ -1129,8 +1299,29 @@ static int npc_mcast_update_action_index(struct rvu *rvu, struct npc_install_flo return 0; } +void +npc_populate_mcam_mdata(struct rvu *rvu, + struct mcam_entry_mdata *mdata, + struct cn20k_mcam_entry *cn20k_entry, + struct mcam_entry *entry) +{ + if (is_cn20k(rvu->pdev)) { + mdata->kw = cn20k_entry->kw; + mdata->kw_mask = cn20k_entry->kw_mask; + mdata->action = &cn20k_entry->action; + mdata->vtag_action = &cn20k_entry->vtag_action; + mdata->max_kw = NPC_KWS_IN_KEY_SZ_8; + return; + } + mdata->kw = entry->kw; + mdata->kw_mask = entry->kw_mask; + mdata->action = &entry->action; + mdata->vtag_action = &entry->vtag_action; + mdata->max_kw = NPC_KWS_IN_KEY_SZ_7; +} + static int npc_update_rx_entry(struct rvu *rvu, struct rvu_pfvf *pfvf, - struct mcam_entry *entry, + struct mcam_entry_mdata *mdata, struct npc_install_flow_req *req, u16 target, bool pf_set_vfs_mac) { @@ -1141,7 +1332,7 @@ static int npc_update_rx_entry(struct rvu *rvu, struct rvu_pfvf *pfvf, if (rswitch->mode == DEVLINK_ESWITCH_MODE_SWITCHDEV && pf_set_vfs_mac) req->chan_mask = 0x0; /* Do not care channel */ - npc_update_entry(rvu, NPC_CHAN, entry, req->channel, 0, req->chan_mask, + npc_update_entry(rvu, NPC_CHAN, mdata, req->channel, 0, req->chan_mask, 0, NIX_INTF_RX); *(u64 *)&action = 0x00; @@ -1173,12 +1364,12 @@ static int npc_update_rx_entry(struct rvu *rvu, struct rvu_pfvf *pfvf, action.match_id = req->match_id; } - entry->action = *(u64 *)&action; + *mdata->action = *(u64 *)&action; /* VTAG0 starts at 0th byte of LID_B. * VTAG1 starts at 4th byte of LID_B. */ - entry->vtag_action = FIELD_PREP(RX_VTAG0_VALID_BIT, req->vtag0_valid) | + *mdata->vtag_action = FIELD_PREP(RX_VTAG0_VALID_BIT, req->vtag0_valid) | FIELD_PREP(RX_VTAG0_TYPE_MASK, req->vtag0_type) | FIELD_PREP(RX_VTAG0_LID_MASK, NPC_LID_LB) | FIELD_PREP(RX_VTAG0_RELPTR_MASK, 0) | @@ -1191,7 +1382,7 @@ static int npc_update_rx_entry(struct rvu *rvu, struct rvu_pfvf *pfvf, } static int npc_update_tx_entry(struct rvu *rvu, struct rvu_pfvf *pfvf, - struct mcam_entry *entry, + struct mcam_entry_mdata *mdata, struct npc_install_flow_req *req, u16 target) { struct nix_tx_action action; @@ -1204,7 +1395,7 @@ static int npc_update_tx_entry(struct rvu *rvu, struct rvu_pfvf *pfvf, if (is_pffunc_af(req->hdr.pcifunc)) mask = 0; - npc_update_entry(rvu, NPC_PF_FUNC, entry, (__force u16)htons(target), + npc_update_entry(rvu, NPC_PF_FUNC, mdata, (__force u16)htons(target), 0, mask, 0, NIX_INTF_TX); *(u64 *)&action = 0x00; @@ -1217,12 +1408,12 @@ static int npc_update_tx_entry(struct rvu *rvu, struct rvu_pfvf *pfvf, action.match_id = req->match_id; - entry->action = *(u64 *)&action; + *mdata->action = *(u64 *)&action; /* VTAG0 starts at 0th byte of LID_B. * VTAG1 starts at 4th byte of LID_B. */ - entry->vtag_action = FIELD_PREP(TX_VTAG0_DEF_MASK, req->vtag0_def) | + *mdata->vtag_action = FIELD_PREP(TX_VTAG0_DEF_MASK, req->vtag0_def) | FIELD_PREP(TX_VTAG0_OP_MASK, req->vtag0_op) | FIELD_PREP(TX_VTAG0_LID_MASK, NPC_LID_LA) | FIELD_PREP(TX_VTAG0_RELPTR_MASK, 20) | @@ -1241,9 +1432,12 @@ static int npc_install_flow(struct rvu *rvu, int blkaddr, u16 target, bool pf_set_vfs_mac) { struct rvu_npc_mcam_rule *def_ucast_rule = pfvf->def_ucast_rule; + struct npc_cn20k_mcam_write_entry_req cn20k_wreq = { 0 }; u64 features, installed_features, missing_features = 0; struct npc_mcam_write_entry_req write_req = { 0 }; struct npc_mcam *mcam = &rvu->hw->mcam; + struct cn20k_mcam_entry *cn20k_entry; + struct mcam_entry_mdata mdata = { }; struct rvu_npc_mcam_rule dummy = { 0 }; struct rvu_npc_mcam_rule *rule; u16 owner = req->hdr.pcifunc; @@ -1255,18 +1449,23 @@ static int npc_install_flow(struct rvu *rvu, int blkaddr, u16 target, installed_features = req->features; features = req->features; - entry = &write_req.entry_data; entry_index = req->entry; - npc_update_flow(rvu, entry, features, &req->packet, &req->mask, &dummy, + cn20k_entry = &cn20k_wreq.entry_data; + entry = &write_req.entry_data; + + npc_populate_mcam_mdata(rvu, &mdata, cn20k_entry, entry); + + npc_update_flow(rvu, &mdata, features, &req->packet, &req->mask, &dummy, req->intf, blkaddr); if (is_npc_intf_rx(req->intf)) { - err = npc_update_rx_entry(rvu, pfvf, entry, req, target, pf_set_vfs_mac); + err = npc_update_rx_entry(rvu, pfvf, &mdata, req, target, + pf_set_vfs_mac); if (err) return err; } else { - err = npc_update_tx_entry(rvu, pfvf, entry, req, target); + err = npc_update_tx_entry(rvu, pfvf, &mdata, req, target); if (err) return err; } @@ -1286,7 +1485,7 @@ static int npc_install_flow(struct rvu *rvu, int blkaddr, u16 target, missing_features = (def_ucast_rule->features ^ features) & def_ucast_rule->features; if (missing_features) - npc_update_flow(rvu, entry, missing_features, + npc_update_flow(rvu, &mdata, missing_features, &def_ucast_rule->packet, &def_ucast_rule->mask, &dummy, req->intf, @@ -1303,51 +1502,91 @@ find_rule: new = true; } - /* allocate new counter if rule has no counter */ - if (!req->default_rule && req->set_cntr && !rule->has_cntr) - rvu_mcam_add_counter_to_rule(rvu, owner, rule, rsp); - - /* if user wants to delete an existing counter for a rule then - * free the counter - */ - if (!req->set_cntr && rule->has_cntr) - rvu_mcam_remove_counter_from_rule(rvu, owner, rule); + if (!is_cn20k(rvu->pdev)) { + write_req.hdr.pcifunc = owner; + + /* allocate new counter if rule has no counter */ + if (!req->default_rule && req->set_cntr && !rule->has_cntr) + rvu_mcam_add_counter_to_rule(rvu, owner, rule, rsp); + + /* if user wants to delete an existing counter for a rule then + * free the counter + */ + if (!req->set_cntr && rule->has_cntr) + rvu_mcam_remove_counter_from_rule(rvu, owner, rule); + + /* AF owns the default rules so change the owner just to relax + * the checks in rvu_mbox_handler_npc_mcam_write_entry + */ + if (req->default_rule) + write_req.hdr.pcifunc = 0; + + write_req.entry = entry_index; + write_req.intf = req->intf; + write_req.enable_entry = (u8)enable; + /* if counter is available then clear and use it */ + if (req->set_cntr && rule->has_cntr) { + rvu_write64(rvu, blkaddr, + NPC_AF_MATCH_STATX(rule->cntr), + req->cntr_val); + write_req.set_cntr = 1; + write_req.cntr = rule->cntr; + } + goto update_rule; + } - write_req.hdr.pcifunc = owner; + cn20k_wreq.hdr.pcifunc = owner; - /* AF owns the default rules so change the owner just to relax - * the checks in rvu_mbox_handler_npc_mcam_write_entry - */ if (req->default_rule) - write_req.hdr.pcifunc = 0; + cn20k_wreq.hdr.pcifunc = 0; - write_req.entry = entry_index; - write_req.intf = req->intf; - write_req.enable_entry = (u8)enable; - /* if counter is available then clear and use it */ - if (req->set_cntr && rule->has_cntr) { - rvu_write64(rvu, blkaddr, NPC_AF_MATCH_STATX(rule->cntr), req->cntr_val); - write_req.set_cntr = 1; - write_req.cntr = rule->cntr; - } + cn20k_wreq.entry = entry_index; + cn20k_wreq.intf = req->intf; + cn20k_wreq.enable_entry = (u8)enable; + cn20k_wreq.hw_prio = req->hw_prio; + cn20k_wreq.req_kw_type = req->req_kw_type; + +update_rule: /* update rule */ memcpy(&rule->packet, &dummy.packet, sizeof(rule->packet)); memcpy(&rule->mask, &dummy.mask, sizeof(rule->mask)); rule->entry = entry_index; - memcpy(&rule->rx_action, &entry->action, sizeof(struct nix_rx_action)); - if (is_npc_intf_tx(req->intf)) - memcpy(&rule->tx_action, &entry->action, - sizeof(struct nix_tx_action)); - rule->vtag_action = entry->vtag_action; + if (is_cn20k(rvu->pdev)) { + memcpy(&rule->rx_action, &cn20k_entry->action, + sizeof(struct nix_rx_action)); + if (is_npc_intf_tx(req->intf)) + memcpy(&rule->tx_action, &cn20k_entry->action, + sizeof(struct nix_tx_action)); + rule->vtag_action = cn20k_entry->vtag_action; + } else { + memcpy(&rule->rx_action, &entry->action, + sizeof(struct nix_rx_action)); + if (is_npc_intf_tx(req->intf)) + memcpy(&rule->tx_action, &entry->action, + sizeof(struct nix_tx_action)); + rule->vtag_action = entry->vtag_action; + } + rule->features = installed_features; rule->default_rule = req->default_rule; rule->owner = owner; rule->enable = enable; - rule->chan_mask = write_req.entry_data.kw_mask[0] & NPC_KEX_CHAN_MASK; - rule->chan = write_req.entry_data.kw[0] & NPC_KEX_CHAN_MASK; + + if (is_cn20k(rvu->pdev)) { + rule->chan_mask = cn20k_wreq.entry_data.kw_mask[0] & + NPC_KEX_CHAN_MASK; + rule->chan = cn20k_wreq.entry_data.kw[0] & + NPC_KEX_CHAN_MASK; + } else { + rule->chan_mask = write_req.entry_data.kw_mask[0] & + NPC_KEX_CHAN_MASK; + rule->chan = write_req.entry_data.kw[0] & NPC_KEX_CHAN_MASK; + } + rule->chan &= rule->chan_mask; rule->lxmb = dummy.lxmb; + rule->hw_prio = req->hw_prio; if (is_npc_intf_tx(req->intf)) rule->intf = pfvf->nix_tx_intf; else @@ -1359,8 +1598,14 @@ find_rule: pfvf->def_ucast_rule = rule; /* write to mcam entry registers */ - err = rvu_mbox_handler_npc_mcam_write_entry(rvu, &write_req, - &write_rsp); + if (is_cn20k(rvu->pdev)) + err = rvu_mbox_handler_npc_cn20k_mcam_write_entry(rvu, + &cn20k_wreq, + &write_rsp); + else + err = rvu_mbox_handler_npc_mcam_write_entry(rvu, &write_req, + &write_rsp); + if (err) { rvu_mcam_remove_counter_from_rule(rvu, owner, rule); if (new) { @@ -1393,6 +1638,79 @@ find_rule: return 0; } +static int +rvu_npc_free_entry_for_flow_install(struct rvu *rvu, u16 pcifunc, + bool free_entry, int mcam_idx) +{ + struct npc_mcam_free_entry_req free_req = { 0 }; + struct msg_rsp rsp; + int rc; + + if (!free_entry) + return 0; + + free_req.hdr.pcifunc = pcifunc; + free_req.entry = mcam_idx; + rc = rvu_mbox_handler_npc_mcam_free_entry(rvu, &free_req, &rsp); + return rc; +} + +static int +rvu_npc_alloc_entry_for_flow_install(struct rvu *rvu, + struct npc_install_flow_req *fl_req, + u16 *mcam_idx, u8 *kw_type, + bool *allocated) +{ + struct npc_mcam_alloc_entry_req entry_req; + struct npc_mcam_alloc_entry_rsp entry_rsp; + struct npc_get_num_kws_req kws_req; + struct npc_get_num_kws_rsp kws_rsp; + int off, kw_bits, rc; + u8 *src, *dst; + + if (!is_cn20k(rvu->pdev)) { + *kw_type = -1; + return 0; + } + + if (!fl_req->alloc_entry) { + *kw_type = -1; + return 0; + } + + off = offsetof(struct npc_install_flow_req, packet); + dst = (u8 *)&kws_req.fl + off; + src = (u8 *)fl_req + off; + memcpy(dst, src, sizeof(struct npc_install_flow_req) - off); + rc = rvu_mbox_handler_npc_get_num_kws(rvu, &kws_req, &kws_rsp); + if (rc) + return rc; + + kw_bits = kws_rsp.kws * 64; + + *kw_type = NPC_MCAM_KEY_X2; + if (kw_bits > 256) + *kw_type = NPC_MCAM_KEY_X4; + + memset(&entry_req, 0, sizeof(entry_req)); + memset(&entry_rsp, 0, sizeof(entry_rsp)); + + entry_req.hdr.pcifunc = fl_req->hdr.pcifunc; + entry_req.ref_prio = fl_req->ref_prio; + entry_req.ref_entry = fl_req->ref_entry; + entry_req.kw_type = *kw_type; + entry_req.count = 1; + rc = rvu_mbox_handler_npc_mcam_alloc_entry(rvu, + &entry_req, + &entry_rsp); + if (rc) + return rc; + + *mcam_idx = entry_rsp.entry_list[0]; + *allocated = true; + return 0; +} + int rvu_mbox_handler_npc_install_flow(struct rvu *rvu, struct npc_install_flow_req *req, struct npc_install_flow_rsp *rsp) @@ -1403,7 +1721,9 @@ int rvu_mbox_handler_npc_install_flow(struct rvu *rvu, int blkaddr, nixlf, err; struct rvu_pfvf *pfvf; bool pf_set_vfs_mac = false; + bool allocated = false; bool enable = true; + u8 kw_type; u16 target; blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NPC, 0); @@ -1415,6 +1735,17 @@ int rvu_mbox_handler_npc_install_flow(struct rvu *rvu, if (!is_npc_interface_valid(rvu, req->intf)) return NPC_FLOW_INTF_INVALID; + err = rvu_npc_alloc_entry_for_flow_install(rvu, req, &req->entry, + &kw_type, &allocated); + if (err) { + dev_err(rvu->dev, + "%s: Error to alloc mcam entry for pcifunc=%#x\n", + __func__, req->hdr.pcifunc); + return err; + } + + req->entry = npc_cn20k_vidx2idx(req->entry); + /* If DMAC is not extracted in MKEX, rules installed by AF * can rely on L2MB bit set by hardware protocol checker for * broadcast and multicast addresses. @@ -1428,6 +1759,10 @@ int rvu_mbox_handler_npc_install_flow(struct rvu *rvu, dev_warn(rvu->dev, "%s: mkex profile does not support ucast flow\n", __func__); + rvu_npc_free_entry_for_flow_install(rvu, + req->hdr.pcifunc, + allocated, + req->entry); return NPC_FLOW_NOT_SUPPORTED; } @@ -1435,6 +1770,10 @@ int rvu_mbox_handler_npc_install_flow(struct rvu *rvu, dev_warn(rvu->dev, "%s: mkex profile does not support bcast/mcast flow", __func__); + rvu_npc_free_entry_for_flow_install(rvu, + req->hdr.pcifunc, + allocated, + req->entry); return NPC_FLOW_NOT_SUPPORTED; } @@ -1444,8 +1783,11 @@ int rvu_mbox_handler_npc_install_flow(struct rvu *rvu, } process_flow: - if (from_vf && req->default_rule) + if (from_vf && req->default_rule) { + rvu_npc_free_entry_for_flow_install(rvu, req->hdr.pcifunc, + allocated, req->entry); return NPC_FLOW_VF_PERM_DENIED; + } /* Each PF/VF info is maintained in struct rvu_pfvf. * rvu_pfvf for the target PF/VF needs to be retrieved @@ -1473,8 +1815,11 @@ process_flow: req->chan_mask = 0xFFF; err = npc_check_unsupported_flows(rvu, req->features, req->intf); - if (err) + if (err) { + rvu_npc_free_entry_for_flow_install(rvu, req->hdr.pcifunc, + allocated, req->entry); return NPC_FLOW_NOT_SUPPORTED; + } pfvf = rvu_get_pfvf(rvu, target); @@ -1493,8 +1838,11 @@ process_flow: /* Proceed if NIXLF is attached or not for TX rules */ err = nix_get_nixlf(rvu, target, &nixlf, NULL); - if (err && is_npc_intf_rx(req->intf) && !pf_set_vfs_mac) + if (err && is_npc_intf_rx(req->intf) && !pf_set_vfs_mac) { + rvu_npc_free_entry_for_flow_install(rvu, req->hdr.pcifunc, + allocated, req->entry); return NPC_FLOW_NO_NIXLF; + } /* don't enable rule when nixlf not attached or initialized */ if (!(is_nixlf_attached(rvu, target) && @@ -1509,20 +1857,31 @@ process_flow: enable = true; /* Do not allow requests from uninitialized VFs */ - if (from_vf && !enable) + if (from_vf && !enable) { + rvu_npc_free_entry_for_flow_install(rvu, req->hdr.pcifunc, + allocated, req->entry); return NPC_FLOW_VF_NOT_INIT; + } /* PF sets VF mac & VF NIXLF is not attached, update the mac addr */ if (pf_set_vfs_mac && !enable) { ether_addr_copy(pfvf->default_mac, req->packet.dmac); ether_addr_copy(pfvf->mac_addr, req->packet.dmac); set_bit(PF_SET_VF_MAC, &pfvf->flags); + rvu_npc_free_entry_for_flow_install(rvu, req->hdr.pcifunc, + allocated, req->entry); return 0; } mutex_lock(&rswitch->switch_lock); err = npc_install_flow(rvu, blkaddr, target, nixlf, pfvf, req, rsp, enable, pf_set_vfs_mac); + if (err) + rvu_npc_free_entry_for_flow_install(rvu, req->hdr.pcifunc, + allocated, req->entry); + + rsp->kw_type = kw_type; + rsp->entry = req->entry; mutex_unlock(&rswitch->switch_lock); return err; @@ -1559,6 +1918,10 @@ int rvu_mbox_handler_npc_delete_flow(struct rvu *rvu, struct list_head del_list; int blkaddr; + req->entry = npc_cn20k_vidx2idx(req->entry); + req->start = npc_cn20k_vidx2idx(req->start); + req->end = npc_cn20k_vidx2idx(req->end); + INIT_LIST_HEAD(&del_list); mutex_lock(&mcam->lock); @@ -1601,28 +1964,49 @@ static int npc_update_dmac_value(struct rvu *rvu, int npcblkaddr, struct rvu_npc_mcam_rule *rule, struct rvu_pfvf *pfvf) { + struct npc_cn20k_mcam_write_entry_req cn20k_wreq = { 0 }; struct npc_mcam_write_entry_req write_req = { 0 }; - struct mcam_entry *entry = &write_req.entry_data; + struct mcam_entry_mdata mdata = { }; struct npc_mcam *mcam = &rvu->hw->mcam; + struct cn20k_mcam_entry *cn20k_entry; + struct mcam_entry *entry; + u8 intf, enable, hw_prio; struct msg_rsp rsp; - u8 intf, enable; int err; + cn20k_entry = &cn20k_wreq.entry_data; + entry = &write_req.entry_data; + npc_populate_mcam_mdata(rvu, &mdata, cn20k_entry, entry); + ether_addr_copy(rule->packet.dmac, pfvf->mac_addr); - npc_read_mcam_entry(rvu, mcam, npcblkaddr, rule->entry, - entry, &intf, &enable); + if (is_cn20k(rvu->pdev)) + npc_cn20k_read_mcam_entry(rvu, npcblkaddr, rule->entry, + cn20k_entry, &intf, + &enable, &hw_prio); + else + npc_read_mcam_entry(rvu, mcam, npcblkaddr, rule->entry, + entry, &intf, &enable); - npc_update_entry(rvu, NPC_DMAC, entry, + npc_update_entry(rvu, NPC_DMAC, &mdata, ether_addr_to_u64(pfvf->mac_addr), 0, 0xffffffffffffull, 0, intf); - write_req.hdr.pcifunc = rule->owner; - write_req.entry = rule->entry; - write_req.intf = pfvf->nix_rx_intf; - mutex_unlock(&mcam->lock); - err = rvu_mbox_handler_npc_mcam_write_entry(rvu, &write_req, &rsp); + if (is_cn20k(rvu->pdev)) { + cn20k_wreq.hdr.pcifunc = rule->owner; + cn20k_wreq.entry = rule->entry; + cn20k_wreq.intf = pfvf->nix_rx_intf; + err = rvu_mbox_handler_npc_cn20k_mcam_write_entry(rvu, + &cn20k_wreq, + &rsp); + } else { + write_req.hdr.pcifunc = rule->owner; + write_req.entry = rule->entry; + write_req.intf = pfvf->nix_rx_intf; + err = rvu_mbox_handler_npc_mcam_write_entry(rvu, &write_req, + &rsp); + } mutex_lock(&mcam->lock); return err; @@ -1710,10 +2094,12 @@ int npc_install_mcam_drop_rule(struct rvu *rvu, int mcam_idx, u16 *counter_idx, u64 chan_val, u64 chan_mask, u64 exact_val, u64 exact_mask, u64 bcast_mcast_val, u64 bcast_mcast_mask) { + struct npc_cn20k_mcam_write_entry_req cn20k_req = { 0 }; struct npc_mcam_alloc_counter_req cntr_req = { 0 }; struct npc_mcam_alloc_counter_rsp cntr_rsp = { 0 }; struct npc_mcam_write_entry_req req = { 0 }; struct npc_mcam *mcam = &rvu->hw->mcam; + struct mcam_entry_mdata mdata = { }; struct rvu_npc_mcam_rule *rule; struct msg_rsp rsp; bool enabled; @@ -1757,26 +2143,51 @@ int npc_install_mcam_drop_rule(struct rvu *rvu, int mcam_idx, u16 *counter_idx, /* Reserve slot 0 */ npc_mcam_rsrcs_reserve(rvu, blkaddr, mcam_idx); - /* Allocate counter for this single drop on non hit rule */ - cntr_req.hdr.pcifunc = 0; /* AF request */ - cntr_req.contig = true; - cntr_req.count = 1; - err = rvu_mbox_handler_npc_mcam_alloc_counter(rvu, &cntr_req, &cntr_rsp); - if (err) { - dev_err(rvu->dev, "%s: Err to allocate cntr for drop rule (err=%d)\n", - __func__, err); - return -EFAULT; + if (!is_cn20k(rvu->pdev)) { + /* Allocate counter for this single drop on non hit rule */ + cntr_req.hdr.pcifunc = 0; /* AF request */ + cntr_req.contig = true; + cntr_req.count = 1; + err = rvu_mbox_handler_npc_mcam_alloc_counter(rvu, &cntr_req, + &cntr_rsp); + if (err) { + dev_err(rvu->dev, + "%s: Err to allocate cntr for drop rule (err=%d)\n", + __func__, err); + return -EFAULT; + } + *counter_idx = cntr_rsp.cntr; } - *counter_idx = cntr_rsp.cntr; + + npc_populate_mcam_mdata(rvu, &mdata, + &cn20k_req.entry_data, + &req.entry_data); /* Fill in fields for this mcam entry */ - npc_update_entry(rvu, NPC_EXACT_RESULT, &req.entry_data, exact_val, 0, + npc_update_entry(rvu, NPC_EXACT_RESULT, &mdata, exact_val, 0, exact_mask, 0, NIX_INTF_RX); - npc_update_entry(rvu, NPC_CHAN, &req.entry_data, chan_val, 0, + npc_update_entry(rvu, NPC_CHAN, &mdata, chan_val, 0, chan_mask, 0, NIX_INTF_RX); - npc_update_entry(rvu, NPC_LXMB, &req.entry_data, bcast_mcast_val, 0, + npc_update_entry(rvu, NPC_LXMB, &mdata, bcast_mcast_val, 0, bcast_mcast_mask, 0, NIX_INTF_RX); + if (is_cn20k(rvu->pdev)) { + cn20k_req.intf = NIX_INTF_RX; + cn20k_req.entry = mcam_idx; + + err = rvu_mbox_handler_npc_cn20k_mcam_write_entry(rvu, + &cn20k_req, + &rsp); + if (err) { + dev_err(rvu->dev, + "%s: Installation of single drop on non hit rule at %d failed\n", + __func__, mcam_idx); + return err; + } + + goto enable_entry; + } + req.intf = NIX_INTF_RX; req.set_cntr = true; req.cntr = cntr_rsp.cntr; @@ -1784,14 +2195,17 @@ int npc_install_mcam_drop_rule(struct rvu *rvu, int mcam_idx, u16 *counter_idx, err = rvu_mbox_handler_npc_mcam_write_entry(rvu, &req, &rsp); if (err) { - dev_err(rvu->dev, "%s: Installation of single drop on non hit rule at %d failed\n", + dev_err(rvu->dev, + "%s: Installation of single drop on non hit rule at %d failed\n", __func__, mcam_idx); return err; } - dev_err(rvu->dev, "%s: Installed single drop on non hit rule at %d, cntr=%d\n", + dev_err(rvu->dev, + "%s: Installed single drop on non hit rule at %d, cntr=%d\n", __func__, mcam_idx, req.cntr); +enable_entry: /* disable entry at Bank 0, index 0 */ npc_enable_mcam_entry(rvu, mcam, blkaddr, mcam_idx, false); diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc_fs.h b/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc_fs.h index 3f5c9042d10e..d3ba86c23959 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc_fs.h +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc_fs.h @@ -15,7 +15,19 @@ #define NPC_LDATA_EN BIT_ULL(7) void npc_update_entry(struct rvu *rvu, enum key_fields type, - struct mcam_entry *entry, u64 val_lo, + struct mcam_entry_mdata *mdata, u64 val_lo, u64 val_hi, u64 mask_lo, u64 mask_hi, u8 intf); +void npc_update_flow(struct rvu *rvu, struct mcam_entry_mdata *mdata, + u64 features, struct flow_msg *pkt, + struct flow_msg *mask, + struct rvu_npc_mcam_rule *output, u8 intf, + int blkaddr); + +void +npc_populate_mcam_mdata(struct rvu *rvu, + struct mcam_entry_mdata *mdata, + struct cn20k_mcam_entry *cn20k_entry, + struct mcam_entry *entry); + #endif /* RVU_NPC_FS_H */ diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc_hash.c b/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc_hash.c index 906d712cef19..03bb485a1aca 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc_hash.c +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc_hash.c @@ -125,6 +125,9 @@ static void npc_program_mkex_hash_rx(struct rvu *rvu, int blkaddr, struct npc_mcam_kex_hash *mkex_hash = rvu->kpu.mkex_hash; int lid, lt, ld, hash_cnt = 0; + if (is_cn20k(rvu->pdev)) + return; + if (is_npc_intf_tx(intf)) return; @@ -165,6 +168,9 @@ static void npc_program_mkex_hash_tx(struct rvu *rvu, int blkaddr, struct npc_mcam_kex_hash *mkex_hash = rvu->kpu.mkex_hash; int lid, lt, ld, hash_cnt = 0; + if (is_cn20k(rvu->pdev)) + return; + if (is_npc_intf_rx(intf)) return; @@ -224,6 +230,9 @@ void npc_program_mkex_hash(struct rvu *rvu, int blkaddr) struct rvu_hwinfo *hw = rvu->hw; u64 cfg; + if (is_cn20k(rvu->pdev)) + return; + /* Check if hardware supports hash extraction */ if (!hwcap->npc_hash_extract) return; @@ -273,7 +282,7 @@ void npc_program_mkex_hash(struct rvu *rvu, int blkaddr) } void npc_update_field_hash(struct rvu *rvu, u8 intf, - struct mcam_entry *entry, + struct mcam_entry_mdata *mdata, int blkaddr, u64 features, struct flow_msg *pkt, @@ -284,9 +293,13 @@ void npc_update_field_hash(struct rvu *rvu, u8 intf, struct npc_mcam_kex_hash *mkex_hash = rvu->kpu.mkex_hash; struct npc_get_field_hash_info_req req; struct npc_get_field_hash_info_rsp rsp; + u8 hash_idx, lid, ltype, ltype_mask; u64 ldata[2], cfg; u32 field_hash; - u8 hash_idx; + bool en; + + if (is_cn20k(rvu->pdev)) + return; if (!rvu->hw->cap.npc_hash_extract) { dev_dbg(rvu->dev, "%s: Field hash extract feature is not supported\n", __func__); @@ -298,60 +311,60 @@ void npc_update_field_hash(struct rvu *rvu, u8 intf, for (hash_idx = 0; hash_idx < NPC_MAX_HASH; hash_idx++) { cfg = rvu_read64(rvu, blkaddr, NPC_AF_INTFX_HASHX_CFG(intf, hash_idx)); - if ((cfg & BIT_ULL(11)) && (cfg & BIT_ULL(12))) { - u8 lid = (cfg & GENMASK_ULL(10, 8)) >> 8; - u8 ltype = (cfg & GENMASK_ULL(7, 4)) >> 4; - u8 ltype_mask = cfg & GENMASK_ULL(3, 0); - - if (mkex_hash->lid_lt_ld_hash_en[intf][lid][ltype][hash_idx]) { - switch (ltype & ltype_mask) { - /* If hash extract enabled is supported for IPv6 then - * 128 bit IPv6 source and destination addressed - * is hashed to 32 bit value. - */ - case NPC_LT_LC_IP6: - /* ld[0] == hash_idx[0] == Source IPv6 - * ld[1] == hash_idx[1] == Destination IPv6 - */ - if ((features & BIT_ULL(NPC_SIP_IPV6)) && !hash_idx) { - u32 src_ip[IPV6_WORDS]; - - be32_to_cpu_array(src_ip, pkt->ip6src, IPV6_WORDS); - ldata[1] = (u64)src_ip[0] << 32 | src_ip[1]; - ldata[0] = (u64)src_ip[2] << 32 | src_ip[3]; - field_hash = npc_field_hash_calc(ldata, - rsp, - intf, - hash_idx); - npc_update_entry(rvu, NPC_SIP_IPV6, entry, - field_hash, 0, - GENMASK(31, 0), 0, intf); - memcpy(&opkt->ip6src, &pkt->ip6src, - sizeof(pkt->ip6src)); - memcpy(&omask->ip6src, &mask->ip6src, - sizeof(mask->ip6src)); - } else if ((features & BIT_ULL(NPC_DIP_IPV6)) && hash_idx) { - u32 dst_ip[IPV6_WORDS]; - - be32_to_cpu_array(dst_ip, pkt->ip6dst, IPV6_WORDS); - ldata[1] = (u64)dst_ip[0] << 32 | dst_ip[1]; - ldata[0] = (u64)dst_ip[2] << 32 | dst_ip[3]; - field_hash = npc_field_hash_calc(ldata, - rsp, - intf, - hash_idx); - npc_update_entry(rvu, NPC_DIP_IPV6, entry, - field_hash, 0, - GENMASK(31, 0), 0, intf); - memcpy(&opkt->ip6dst, &pkt->ip6dst, - sizeof(pkt->ip6dst)); - memcpy(&omask->ip6dst, &mask->ip6dst, - sizeof(mask->ip6dst)); - } - - break; - } - } + en = !!(cfg & BIT_ULL(11)) && (cfg & BIT_ULL(12)); + if (!en) + continue; + + lid = (cfg & GENMASK_ULL(10, 8)) >> 8; + ltype = (cfg & GENMASK_ULL(7, 4)) >> 4; + ltype_mask = cfg & GENMASK_ULL(3, 0); + + if (!mkex_hash->lid_lt_ld_hash_en[intf][lid][ltype][hash_idx]) + continue; + + /* If hash extract enabled is supported for IPv6 then + * 128 bit IPv6 source and destination addressed + * is hashed to 32 bit value. + */ + if ((ltype & ltype_mask) != NPC_LT_LC_IP6) + continue; + + /* ld[0] == hash_idx[0] == Source IPv6 + * ld[1] == hash_idx[1] == Destination IPv6 + */ + if ((features & BIT_ULL(NPC_SIP_IPV6)) && !hash_idx) { + u32 src_ip[IPV6_WORDS]; + + be32_to_cpu_array(src_ip, pkt->ip6src, IPV6_WORDS); + ldata[1] = (u64)src_ip[0] << 32 | src_ip[1]; + ldata[0] = (u64)src_ip[2] << 32 | src_ip[3]; + field_hash = npc_field_hash_calc(ldata, rsp, intf, + hash_idx); + npc_update_entry(rvu, NPC_SIP_IPV6, mdata, field_hash, + 0, GENMASK(31, 0), 0, intf); + memcpy(&opkt->ip6src, &pkt->ip6src, + sizeof(pkt->ip6src)); + memcpy(&omask->ip6src, &mask->ip6src, + sizeof(mask->ip6src)); + continue; + } + + if ((features & BIT_ULL(NPC_DIP_IPV6)) && hash_idx) { + u32 dst_ip[IPV6_WORDS]; + + be32_to_cpu_array(dst_ip, pkt->ip6dst, IPV6_WORDS); + ldata[1] = (u64)dst_ip[0] << 32 | dst_ip[1]; + ldata[0] = (u64)dst_ip[2] << 32 | dst_ip[3]; + field_hash = npc_field_hash_calc(ldata, rsp, intf, + hash_idx); + npc_update_entry(rvu, NPC_DIP_IPV6, mdata, + field_hash, 0, GENMASK(31, 0), + 0, intf); + memcpy(&opkt->ip6dst, &pkt->ip6dst, + sizeof(pkt->ip6dst)); + memcpy(&omask->ip6dst, &mask->ip6dst, + sizeof(mask->ip6dst)); + continue; } } } @@ -1874,6 +1887,9 @@ int rvu_npc_exact_init(struct rvu *rvu) u64 cfg; bool rc; + if (is_cn20k(rvu->pdev)) + return 0; + /* Read NPC_AF_CONST3 and check for have exact * match functionality is present */ diff --git a/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc_hash.h b/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc_hash.h index cb25cf478f1f..4cbcae69b6d3 100644 --- a/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc_hash.h +++ b/drivers/net/ethernet/marvell/octeontx2/af/rvu_npc_hash.h @@ -53,7 +53,7 @@ struct npc_mcam_kex_hash { } __packed; void npc_update_field_hash(struct rvu *rvu, u8 intf, - struct mcam_entry *entry, + struct mcam_entry_mdata *mdata, int blkaddr, u64 features, struct flow_msg *pkt, diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/cn10k_macsec.c b/drivers/net/ethernet/marvell/octeontx2/nic/cn10k_macsec.c index 4649996dc7da..2cc1bdfd9b2e 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/cn10k_macsec.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/cn10k_macsec.c @@ -4,7 +4,7 @@ * Copyright (C) 2022 Marvell. */ -#include <crypto/skcipher.h> +#include <crypto/aes.h> #include <linux/rtnetlink.h> #include <linux/bitfield.h> #include "otx2_common.h" @@ -46,51 +46,22 @@ #define CN10K_MAX_HASH_LEN 16 #define CN10K_MAX_SAK_LEN 32 -static int cn10k_ecb_aes_encrypt(struct otx2_nic *pfvf, u8 *sak, - u16 sak_len, u8 *hash) +static int cn10k_ecb_aes_encrypt(struct otx2_nic *pfvf, const u8 *sak, + u16 sak_len, u8 hash[CN10K_MAX_HASH_LEN]) { - u8 data[CN10K_MAX_HASH_LEN] = { 0 }; - struct skcipher_request *req = NULL; - struct scatterlist sg_src, sg_dst; - struct crypto_skcipher *tfm; - DECLARE_CRYPTO_WAIT(wait); - int err; - - tfm = crypto_alloc_skcipher("ecb(aes)", 0, 0); - if (IS_ERR(tfm)) { - dev_err(pfvf->dev, "failed to allocate transform for ecb-aes\n"); - return PTR_ERR(tfm); - } - - req = skcipher_request_alloc(tfm, GFP_KERNEL); - if (!req) { - dev_err(pfvf->dev, "failed to allocate request for skcipher\n"); - err = -ENOMEM; - goto free_tfm; - } + static const u8 zeroes[CN10K_MAX_HASH_LEN]; + struct aes_enckey aes; - err = crypto_skcipher_setkey(tfm, sak, sak_len); - if (err) { - dev_err(pfvf->dev, "failed to set key for skcipher\n"); - goto free_req; + if (aes_prepareenckey(&aes, sak, sak_len) != 0) { + dev_err(pfvf->dev, "invalid AES key length: %d\n", sak_len); + return -EINVAL; } - /* build sg list */ - sg_init_one(&sg_src, data, CN10K_MAX_HASH_LEN); - sg_init_one(&sg_dst, hash, CN10K_MAX_HASH_LEN); - - skcipher_request_set_callback(req, 0, crypto_req_done, &wait); - skcipher_request_set_crypt(req, &sg_src, &sg_dst, - CN10K_MAX_HASH_LEN, NULL); + static_assert(CN10K_MAX_HASH_LEN == AES_BLOCK_SIZE); + aes_encrypt(&aes, hash, zeroes); - err = crypto_skcipher_encrypt(req); - err = crypto_wait_req(err, &wait); - -free_req: - skcipher_request_free(req); -free_tfm: - crypto_free_skcipher(tfm); - return err; + memzero_explicit(&aes, sizeof(aes)); + return 0; } static struct cn10k_mcs_txsc *cn10k_mcs_get_txsc(struct cn10k_mcs_cfg *cfg, diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/cn20k.c b/drivers/net/ethernet/marvell/octeontx2/nic/cn20k.c index a60f8cf53feb..a5a8f4558717 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/cn20k.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/cn20k.c @@ -251,6 +251,272 @@ static u8 cn20k_aura_bpid_idx(struct otx2_nic *pfvf, int aura_id) #endif } +static int cn20k_tc_get_entry_index(struct otx2_flow_config *flow_cfg, + struct otx2_tc_flow *node) +{ + struct otx2_tc_flow *tmp; + int index = 0; + + list_for_each_entry(tmp, &flow_cfg->flow_list_tc, list) { + if (tmp == node) + return index; + + index++; + } + + return -1; +} + +int cn20k_tc_free_mcam_entry(struct otx2_nic *nic, u16 entry) +{ + struct npc_mcam_free_entry_req *req; + int err; + + mutex_lock(&nic->mbox.lock); + req = otx2_mbox_alloc_msg_npc_mcam_free_entry(&nic->mbox); + if (!req) { + mutex_unlock(&nic->mbox.lock); + return -ENOMEM; + } + + req->entry = entry; + /* Send message to AF to free MCAM entries */ + err = otx2_sync_mbox_msg(&nic->mbox); + if (err) { + mutex_unlock(&nic->mbox.lock); + return err; + } + + mutex_unlock(&nic->mbox.lock); + + return 0; +} + +static bool cn20k_tc_check_entry_shiftable(struct otx2_nic *nic, + struct otx2_flow_config *flow_cfg, + struct otx2_tc_flow *node, int index, + bool error) +{ + struct otx2_tc_flow *first, *tmp, *n; + u32 prio = 0; + int i = 0; + u8 type; + + first = list_first_entry(&flow_cfg->flow_list_tc, struct otx2_tc_flow, + list); + type = first->kw_type; + + /* Check all the nodes from start to given index (including index) has + * same type i.e, either X2 or X4 + */ + list_for_each_entry_safe(tmp, n, &flow_cfg->flow_list_tc, list) { + if (i > index) + break; + + if (type != tmp->kw_type) { + /* List has both X2 and X4 entries so entries cannot be + * shifted to save MCAM space. + */ + if (error) + dev_err(nic->dev, "Rule %d cannot be shifted to %d\n", + tmp->prio, prio); + return false; + } + + type = tmp->kw_type; + prio = tmp->prio; + i++; + } + + return true; +} + +void cn20k_tc_update_mcam_table_del_req(struct otx2_nic *nic, + struct otx2_flow_config *flow_cfg, + struct otx2_tc_flow *node) +{ + struct otx2_tc_flow *first, *tmp, *n; + int i = 0, index; + u16 cntr_val = 0; + u16 entry; + + index = cn20k_tc_get_entry_index(flow_cfg, node); + if (index < 0) { + netdev_dbg(nic->netdev, "Could not find node\n"); + return; + } + + first = list_first_entry(&flow_cfg->flow_list_tc, struct otx2_tc_flow, + list); + entry = first->entry; + + /* If entries cannot be shifted then delete given entry + * and free it to AF too. + */ + if (!cn20k_tc_check_entry_shiftable(nic, flow_cfg, node, + index, false)) { + list_del(&node->list); + entry = node->entry; + goto free_mcam_entry; + } + + /* Find and delete the entry from the list and re-install + * all the entries from beginning to the index of the + * deleted entry to higher mcam indexes. + */ + list_for_each_entry_safe(tmp, n, &flow_cfg->flow_list_tc, list) { + if (node == tmp) { + list_del(&tmp->list); + break; + } + + otx2_del_mcam_flow_entry(nic, tmp->entry, &cntr_val); + tmp->entry = (list_next_entry(tmp, list))->entry; + tmp->req.entry = tmp->entry; + tmp->req.cntr_val = cntr_val; + } + + list_for_each_entry_safe(tmp, n, &flow_cfg->flow_list_tc, list) { + if (i == index) + break; + + otx2_add_mcam_flow_entry(nic, &tmp->req); + i++; + } + +free_mcam_entry: + if (cn20k_tc_free_mcam_entry(nic, entry)) + netdev_err(nic->netdev, "Freeing entry %d to AF failed\n", + entry); +} + +int cn20k_tc_update_mcam_table_add_req(struct otx2_nic *nic, + struct otx2_flow_config *flow_cfg, + struct otx2_tc_flow *node) +{ + struct otx2_tc_flow *tmp; + u16 cntr_val = 0; + int list_idx, i; + int entry, prev; + + /* Find the index of the entry(list_idx) whose priority + * is greater than the new entry and re-install all + * the entries from beginning to list_idx to higher + * mcam indexes. + */ + list_idx = otx2_tc_add_to_flow_list(flow_cfg, node); + entry = node->entry; + if (!cn20k_tc_check_entry_shiftable(nic, flow_cfg, node, + list_idx, true)) + /* Due to mix of X2 and X4, entries cannot be shifted. + * In this case free the entry allocated for this rule. + */ + return -EINVAL; + + for (i = 0; i < list_idx; i++) { + tmp = otx2_tc_get_entry_by_index(flow_cfg, i); + if (!tmp) + return -ENOMEM; + + otx2_del_mcam_flow_entry(nic, tmp->entry, &cntr_val); + prev = tmp->entry; + tmp->entry = entry; + tmp->req.entry = tmp->entry; + tmp->req.cntr_val = cntr_val; + otx2_add_mcam_flow_entry(nic, &tmp->req); + entry = prev; + } + + return entry; +} + +#define MAX_TC_HW_PRIORITY 125 +#define MAX_TC_VF_PRIORITY 126 +#define MAX_TC_PF_PRIORITY 127 + +static int __cn20k_tc_alloc_entry(struct otx2_nic *nic, + struct npc_install_flow_req *flow_req, + u16 *entry, u8 *type, + u32 tc_priority, bool hw_priority) +{ + struct otx2_flow_config *flow_cfg = nic->flow_cfg; + struct npc_install_flow_req *req; + struct npc_install_flow_rsp *rsp; + struct otx2_tc_flow *tmp; + int ret = 0; + + req = otx2_mbox_alloc_msg_npc_install_flow(&nic->mbox); + if (!req) + return -ENOMEM; + + memcpy(&flow_req->hdr, &req->hdr, sizeof(struct mbox_msghdr)); + memcpy(req, flow_req, sizeof(struct npc_install_flow_req)); + req->alloc_entry = 1; + + /* Allocate very least priority for first rule */ + if (hw_priority || list_empty(&flow_cfg->flow_list_tc)) { + req->ref_prio = NPC_MCAM_LEAST_PRIO; + } else { + req->ref_prio = NPC_MCAM_HIGHER_PRIO; + tmp = list_first_entry(&flow_cfg->flow_list_tc, + struct otx2_tc_flow, list); + req->ref_entry = tmp->entry; + } + + ret = otx2_sync_mbox_msg(&nic->mbox); + if (ret) + return ret; + + rsp = (struct npc_install_flow_rsp *)otx2_mbox_get_rsp(&nic->mbox.mbox, + 0, &req->hdr); + if (IS_ERR(rsp)) + return -EFAULT; + + if (entry) + *entry = rsp->entry; + if (type) + *type = rsp->kw_type; + + return ret; +} + +int cn20k_tc_alloc_entry(struct otx2_nic *nic, + struct flow_cls_offload *tc_flow_cmd, + struct otx2_tc_flow *new_node, + struct npc_install_flow_req *flow_req) +{ + bool hw_priority = false; + u16 entry_from_af; + u8 entry_type; + int ret; + + if (is_otx2_vf(nic->pcifunc)) + flow_req->hw_prio = MAX_TC_VF_PRIORITY; + else + flow_req->hw_prio = MAX_TC_PF_PRIORITY; + + if (new_node->prio <= MAX_TC_HW_PRIORITY) { + flow_req->hw_prio = new_node->prio; + hw_priority = true; + } + + mutex_lock(&nic->mbox.lock); + + ret = __cn20k_tc_alloc_entry(nic, flow_req, &entry_from_af, &entry_type, + new_node->prio, hw_priority); + if (ret) { + mutex_unlock(&nic->mbox.lock); + return ret; + } + + new_node->kw_type = entry_type; + new_node->entry = entry_from_af; + + mutex_unlock(&nic->mbox.lock); + + return 0; +} + static int cn20k_aura_aq_init(struct otx2_nic *pfvf, int aura_id, int pool_id, int numptrs) { diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/cn20k.h b/drivers/net/ethernet/marvell/octeontx2/nic/cn20k.h index 832adaf8c57f..b5e527f6d7eb 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/cn20k.h +++ b/drivers/net/ethernet/marvell/octeontx2/nic/cn20k.h @@ -10,8 +10,22 @@ #include "otx2_common.h" +struct otx2_flow_config; +struct otx2_tc_flow; + void cn20k_init(struct otx2_nic *pfvf); int cn20k_register_pfvf_mbox_intr(struct otx2_nic *pf, int numvfs); void cn20k_disable_pfvf_mbox_intr(struct otx2_nic *pf, int numvfs); void cn20k_enable_pfvf_mbox_intr(struct otx2_nic *pf, int numvfs); +void cn20k_tc_update_mcam_table_del_req(struct otx2_nic *nic, + struct otx2_flow_config *flow_cfg, + struct otx2_tc_flow *node); +int cn20k_tc_update_mcam_table_add_req(struct otx2_nic *nic, + struct otx2_flow_config *flow_cfg, + struct otx2_tc_flow *node); +int cn20k_tc_alloc_entry(struct otx2_nic *nic, + struct flow_cls_offload *tc_flow_cmd, + struct otx2_tc_flow *new_node, + struct npc_install_flow_req *dummy); +int cn20k_tc_free_mcam_entry(struct otx2_nic *nic, u16 entry); #endif /* CN20K_H */ diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h index 255c7e2633bb..eecee612b7b2 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_common.h @@ -366,6 +366,31 @@ struct otx2_flow_config { u16 ntuple_cnt; }; +struct otx2_tc_flow_stats { + u64 bytes; + u64 pkts; + u64 used; +}; + +struct otx2_tc_flow { + struct list_head list; + unsigned long cookie; + struct rcu_head rcu; + struct otx2_tc_flow_stats stats; + spinlock_t lock; /* lock for stats */ + u16 rq; + u16 entry; + u16 leaf_profile; + bool is_act_police; + u32 prio; + struct npc_install_flow_req req; + u64 rate; + u32 burst; + u32 mcast_grp_idx; + bool is_pps; + u8 kw_type; /* X2/X4 */ +}; + struct dev_hw_ops { int (*sq_aq_init)(void *dev, u16 qidx, u8 chan_offset, u16 sqb_aura); @@ -1223,4 +1248,14 @@ void otx2_dma_unmap_skb_frags(struct otx2_nic *pfvf, struct sg_list *sg); int otx2_read_free_sqe(struct otx2_nic *pfvf, u16 qidx); void otx2_queue_vf_work(struct mbox *mw, struct workqueue_struct *mbox_wq, int first, int mdevs, u64 intr); +int otx2_del_mcam_flow_entry(struct otx2_nic *nic, u16 entry, + u16 *cntr_val); +int otx2_add_mcam_flow_entry(struct otx2_nic *nic, + struct npc_install_flow_req *req); +int otx2_tc_add_to_flow_list(struct otx2_flow_config *flow_cfg, + struct otx2_tc_flow *node); + +struct otx2_tc_flow * +otx2_tc_get_entry_by_index(struct otx2_flow_config *flow_cfg, + int index); #endif /* OTX2_COMMON_H */ diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_flows.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_flows.c index 12c001ee34e2..38cc539d724d 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_flows.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_flows.c @@ -37,6 +37,98 @@ static void otx2_clear_ntuple_flow_info(struct otx2_nic *pfvf, struct otx2_flow_ flow_cfg->max_flows = 0; } +static int otx2_mcam_pfl_info_get(struct otx2_nic *pfvf, bool *is_x2, + u16 *x4_slots) +{ + struct npc_get_pfl_info_rsp *rsp; + struct msg_req *req; + static struct { + bool is_set; + bool is_x2; + u16 x4_slots; + } pfl_info; + + /* Avoid sending mboxes for constant information + * like x4_slots + */ + mutex_lock(&pfvf->mbox.lock); + if (pfl_info.is_set) { + *is_x2 = pfl_info.is_x2; + *x4_slots = pfl_info.x4_slots; + mutex_unlock(&pfvf->mbox.lock); + return 0; + } + + req = otx2_mbox_alloc_msg_npc_get_pfl_info(&pfvf->mbox); + if (!req) { + mutex_unlock(&pfvf->mbox.lock); + return -ENOMEM; + } + + /* Send message to AF */ + if (otx2_sync_mbox_msg(&pfvf->mbox)) { + mutex_unlock(&pfvf->mbox.lock); + return -EFAULT; + } + + rsp = (struct npc_get_pfl_info_rsp *)otx2_mbox_get_rsp + (&pfvf->mbox.mbox, 0, &req->hdr); + + if (IS_ERR(rsp)) { + mutex_unlock(&pfvf->mbox.lock); + return -EFAULT; + } + + *is_x2 = (rsp->kw_type == NPC_MCAM_KEY_X2); + if (*is_x2) + *x4_slots = 0; + else + *x4_slots = rsp->x4_slots; + + pfl_info.is_x2 = *is_x2; + pfl_info.x4_slots = *x4_slots; + pfl_info.is_set = true; + + mutex_unlock(&pfvf->mbox.lock); + return 0; +} + +static int otx2_get_dft_rl_idx(struct otx2_nic *pfvf, u16 *mcam_idx) +{ + struct npc_get_dft_rl_idxs_rsp *rsp; + struct msg_req *req; + + mutex_lock(&pfvf->mbox.lock); + + req = otx2_mbox_alloc_msg_npc_get_dft_rl_idxs(&pfvf->mbox); + if (!req) { + mutex_unlock(&pfvf->mbox.lock); + return -ENOMEM; + } + + /* Send message to AF */ + if (otx2_sync_mbox_msg(&pfvf->mbox)) { + mutex_unlock(&pfvf->mbox.lock); + return -EINVAL; + } + + rsp = (struct npc_get_dft_rl_idxs_rsp *)otx2_mbox_get_rsp + (&pfvf->mbox.mbox, 0, &req->hdr); + + if (IS_ERR(rsp)) { + mutex_unlock(&pfvf->mbox.lock); + return -EFAULT; + } + + if (is_otx2_lbkvf(pfvf->pdev)) + *mcam_idx = rsp->promisc; + else + *mcam_idx = rsp->ucast; + + mutex_unlock(&pfvf->mbox.lock); + return 0; +} + static int otx2_free_ntuple_mcam_entries(struct otx2_nic *pfvf) { struct otx2_flow_config *flow_cfg = pfvf->flow_cfg; @@ -69,7 +161,10 @@ int otx2_alloc_mcam_entries(struct otx2_nic *pfvf, u16 count) struct otx2_flow_config *flow_cfg = pfvf->flow_cfg; struct npc_mcam_alloc_entry_req *req; struct npc_mcam_alloc_entry_rsp *rsp; - int ent, allocated = 0; + u16 dft_idx = 0, x4_slots = 0; + int ent, allocated = 0, ref; + bool is_x2 = false; + int rc; /* Free current ones and allocate new ones with requested count */ otx2_free_ntuple_mcam_entries(pfvf); @@ -86,6 +181,22 @@ int otx2_alloc_mcam_entries(struct otx2_nic *pfvf, u16 count) return -ENOMEM; } + if (is_cn20k(pfvf->pdev)) { + rc = otx2_mcam_pfl_info_get(pfvf, &is_x2, &x4_slots); + if (rc) { + netdev_err(pfvf->netdev, "Error to retrieve profile info\n"); + return rc; + } + + rc = otx2_get_dft_rl_idx(pfvf, &dft_idx); + if (rc) { + netdev_err(pfvf->netdev, + "Error to retrieve ucast mcam idx for pcifunc %#x\n", + pfvf->pcifunc); + return rc; + } + } + mutex_lock(&pfvf->mbox.lock); /* In a single request a max of NPC_MAX_NONCONTIG_ENTRIES MCAM entries @@ -96,18 +207,31 @@ int otx2_alloc_mcam_entries(struct otx2_nic *pfvf, u16 count) if (!req) goto exit; + req->kw_type = is_x2 ? NPC_MCAM_KEY_X2 : NPC_MCAM_KEY_X4; req->contig = false; req->count = (count - allocated) > NPC_MAX_NONCONTIG_ENTRIES ? NPC_MAX_NONCONTIG_ENTRIES : count - allocated; + ref = 0; + + if (is_cn20k(pfvf->pdev)) { + req->ref_prio = NPC_MCAM_HIGHER_PRIO; + ref = dft_idx; + } + /* Allocate higher priority entries for PFs, so that VF's entries * will be on top of PF. */ if (!is_otx2_vf(pfvf->pcifunc)) { - req->priority = NPC_MCAM_HIGHER_PRIO; - req->ref_entry = flow_cfg->def_ent[0]; + req->ref_prio = NPC_MCAM_HIGHER_PRIO; + ref = flow_cfg->def_ent[0]; } + if (is_cn20k(pfvf->pdev)) + ref = is_x2 ? ref : ref & (x4_slots - 1); + + req->ref_entry = ref; + /* Send message to AF */ if (otx2_sync_mbox_msg(&pfvf->mbox)) goto exit; @@ -163,8 +287,24 @@ int otx2_mcam_entry_init(struct otx2_nic *pfvf) struct npc_get_field_status_rsp *frsp; struct npc_mcam_alloc_entry_req *req; struct npc_mcam_alloc_entry_rsp *rsp; - int vf_vlan_max_flows; - int ent, count; + int vf_vlan_max_flows, count; + int rc, ref, prio, ent; + u16 dft_idx; + + ref = 0; + prio = 0; + if (is_cn20k(pfvf->pdev)) { + rc = otx2_get_dft_rl_idx(pfvf, &dft_idx); + if (rc) { + netdev_err(pfvf->netdev, + "Error to retrieve ucast mcam idx for pcifunc %#x\n", + pfvf->pcifunc); + return rc; + } + + ref = dft_idx; + prio = NPC_MCAM_HIGHER_PRIO; + } vf_vlan_max_flows = pfvf->total_vfs * OTX2_PER_VF_VLAN_FLOWS; count = flow_cfg->ucast_flt_cnt + @@ -183,8 +323,11 @@ int otx2_mcam_entry_init(struct otx2_nic *pfvf) return -ENOMEM; } + req->kw_type = NPC_MCAM_KEY_X2; req->contig = false; req->count = count; + req->ref_prio = prio; + req->ref_entry = ref; /* Send message to AF */ if (otx2_sync_mbox_msg(&pfvf->mbox)) { @@ -819,7 +962,7 @@ static int otx2_prepare_ipv6_flow(struct ethtool_rx_flow_spec *fsp, } static int otx2_prepare_flow_request(struct ethtool_rx_flow_spec *fsp, - struct npc_install_flow_req *req) + struct npc_install_flow_req *req) { struct ethhdr *eth_mask = &fsp->m_u.ether_spec; struct ethhdr *eth_hdr = &fsp->h_u.ether_spec; @@ -945,6 +1088,58 @@ static int otx2_prepare_flow_request(struct ethtool_rx_flow_spec *fsp, return 0; } +static int otx2_get_kw_type(struct otx2_nic *pfvf, + struct npc_install_flow_req *fl_req, + u8 *kw_type) +{ + struct npc_get_num_kws_req *req; + struct npc_get_num_kws_rsp *rsp; + u8 *src, *dst; + int off, err; + int kw_bits; + + off = offsetof(struct npc_install_flow_req, packet); + + mutex_lock(&pfvf->mbox.lock); + + req = otx2_mbox_alloc_msg_npc_get_num_kws(&pfvf->mbox); + if (!req) { + mutex_unlock(&pfvf->mbox.lock); + return -ENOMEM; + } + + dst = (u8 *)&req->fl + off; + src = (u8 *)fl_req + off; + + memcpy(dst, src, sizeof(struct npc_install_flow_req) - off); + + err = otx2_sync_mbox_msg(&pfvf->mbox); + if (err) { + mutex_unlock(&pfvf->mbox.lock); + netdev_err(pfvf->netdev, + "Error to get default number of keywords\n"); + return err; + } + + rsp = (struct npc_get_num_kws_rsp *)otx2_mbox_get_rsp + (&pfvf->mbox.mbox, 0, &req->hdr); + if (IS_ERR(rsp)) { + mutex_unlock(&pfvf->mbox.lock); + return -EFAULT; + } + + kw_bits = rsp->kws * 64; + + if (kw_bits <= 256) + *kw_type = NPC_MCAM_KEY_X2; + else + *kw_type = NPC_MCAM_KEY_X4; + + mutex_unlock(&pfvf->mbox.lock); + + return 0; +} + static int otx2_is_flow_rule_dmacfilter(struct otx2_nic *pfvf, struct ethtool_rx_flow_spec *fsp) { @@ -973,12 +1168,41 @@ static int otx2_is_flow_rule_dmacfilter(struct otx2_nic *pfvf, static int otx2_add_flow_msg(struct otx2_nic *pfvf, struct otx2_flow *flow) { + struct otx2_flow_config *flow_cfg = pfvf->flow_cfg; + struct npc_install_flow_req *req, treq = { 0 }; u64 ring_cookie = flow->flow_spec.ring_cookie; #ifdef CONFIG_DCB int vlan_prio, qidx, pfc_rule = 0; #endif - struct npc_install_flow_req *req; - int err, vf = 0; + int err, vf = 0, off, sz; + bool modify = false; + u8 kw_type = 0; + u8 *src, *dst; + u16 x4_slots; + bool is_x2; + + if (is_cn20k(pfvf->pdev)) { + err = otx2_mcam_pfl_info_get(pfvf, &is_x2, &x4_slots); + if (err) { + netdev_err(pfvf->netdev, + "Error to retrieve NPC profile info, pcifunc=%#x\n", + pfvf->pcifunc); + return -EFAULT; + } + + if (!is_x2) { + err = otx2_prepare_flow_request(&flow->flow_spec, + &treq); + if (err) + return err; + + err = otx2_get_kw_type(pfvf, &treq, &kw_type); + if (err) + return err; + + modify = true; + } + } mutex_lock(&pfvf->mbox.lock); req = otx2_mbox_alloc_msg_npc_install_flow(&pfvf->mbox); @@ -987,14 +1211,29 @@ static int otx2_add_flow_msg(struct otx2_nic *pfvf, struct otx2_flow *flow) return -ENOMEM; } - err = otx2_prepare_flow_request(&flow->flow_spec, req); - if (err) { - /* free the allocated msg above */ - otx2_mbox_reset(&pfvf->mbox.mbox, 0); - mutex_unlock(&pfvf->mbox.lock); - return err; + if (modify) { + off = offsetof(struct npc_install_flow_req, packet); + sz = sizeof(struct npc_install_flow_req) - off; + dst = (u8 *)req + off; + src = (u8 *)&treq + off; + + memcpy(dst, src, sz); + req->req_kw_type = kw_type; + } else { + err = otx2_prepare_flow_request(&flow->flow_spec, req); + if (err) { + /* free the allocated msg above */ + otx2_mbox_reset(&pfvf->mbox.mbox, 0); + mutex_unlock(&pfvf->mbox.lock); + return err; + } } + netdev_dbg(pfvf->netdev, + "flow entry (%u) installed at loc:%u kw_type=%u\n", + flow_cfg->flow_ent[flow->location], + flow->location, kw_type); + req->entry = flow->entry; req->intf = NIX_INTF_RX; req->set_cntr = 1; diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_tc.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_tc.c index 04fcfbdcf69b..40162b08014d 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_tc.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_tc.c @@ -31,30 +31,6 @@ #define MCAST_INVALID_GRP (-1U) -struct otx2_tc_flow_stats { - u64 bytes; - u64 pkts; - u64 used; -}; - -struct otx2_tc_flow { - struct list_head list; - unsigned long cookie; - struct rcu_head rcu; - struct otx2_tc_flow_stats stats; - spinlock_t lock; /* lock for stats */ - u16 rq; - u16 entry; - u16 leaf_profile; - bool is_act_police; - u32 prio; - struct npc_install_flow_req req; - u32 mcast_grp_idx; - u64 rate; - u32 burst; - bool is_pps; -}; - static void otx2_get_egress_burst_cfg(struct otx2_nic *nic, u32 burst, u32 *burst_exp, u32 *burst_mantissa) { @@ -971,8 +947,9 @@ static void otx2_destroy_tc_flow_list(struct otx2_nic *pfvf) } } -static struct otx2_tc_flow *otx2_tc_get_entry_by_cookie(struct otx2_flow_config *flow_cfg, - unsigned long cookie) +static struct otx2_tc_flow * +otx2_tc_get_entry_by_cookie(struct otx2_flow_config *flow_cfg, + unsigned long cookie) { struct otx2_tc_flow *tmp; @@ -984,8 +961,8 @@ static struct otx2_tc_flow *otx2_tc_get_entry_by_cookie(struct otx2_flow_config return NULL; } -static struct otx2_tc_flow *otx2_tc_get_entry_by_index(struct otx2_flow_config *flow_cfg, - int index) +struct otx2_tc_flow * +otx2_tc_get_entry_by_index(struct otx2_flow_config *flow_cfg, int index) { struct otx2_tc_flow *tmp; int i = 0; @@ -1014,8 +991,8 @@ static void otx2_tc_del_from_flow_list(struct otx2_flow_config *flow_cfg, } } -static int otx2_tc_add_to_flow_list(struct otx2_flow_config *flow_cfg, - struct otx2_tc_flow *node) +int otx2_tc_add_to_flow_list(struct otx2_flow_config *flow_cfg, + struct otx2_tc_flow *node) { struct list_head *pos, *n; struct otx2_tc_flow *tmp; @@ -1038,7 +1015,8 @@ static int otx2_tc_add_to_flow_list(struct otx2_flow_config *flow_cfg, return index; } -static int otx2_add_mcam_flow_entry(struct otx2_nic *nic, struct npc_install_flow_req *req) +int otx2_add_mcam_flow_entry(struct otx2_nic *nic, + struct npc_install_flow_req *req) { struct npc_install_flow_req *tmp_req; int err; @@ -1064,7 +1042,7 @@ static int otx2_add_mcam_flow_entry(struct otx2_nic *nic, struct npc_install_flo return 0; } -static int otx2_del_mcam_flow_entry(struct otx2_nic *nic, u16 entry, u16 *cntr_val) +int otx2_del_mcam_flow_entry(struct otx2_nic *nic, u16 entry, u16 *cntr_val) { struct npc_delete_flow_rsp *rsp; struct npc_delete_flow_req *req; @@ -1114,6 +1092,11 @@ static int otx2_tc_update_mcam_table_del_req(struct otx2_nic *nic, int i = 0, index = 0; u16 cntr_val = 0; + if (is_cn20k(nic->pdev)) { + cn20k_tc_update_mcam_table_del_req(nic, flow_cfg, node); + return 0; + } + /* Find and delete the entry from the list and re-install * all the entries from beginning to the index of the * deleted entry to higher mcam indexes. @@ -1153,6 +1136,9 @@ static int otx2_tc_update_mcam_table_add_req(struct otx2_nic *nic, int list_idx, i; u16 cntr_val = 0; + if (is_cn20k(nic->pdev)) + return cn20k_tc_update_mcam_table_add_req(nic, flow_cfg, node); + /* Find the index of the entry(list_idx) whose priority * is greater than the new entry and re-install all * the entries from beginning to list_idx to higher @@ -1172,7 +1158,7 @@ static int otx2_tc_update_mcam_table_add_req(struct otx2_nic *nic, mcam_idx++; } - return mcam_idx; + return flow_cfg->flow_ent[mcam_idx]; } static int otx2_tc_update_mcam_table(struct otx2_nic *nic, @@ -1238,7 +1224,6 @@ static int otx2_tc_del_flow(struct otx2_nic *nic, mutex_unlock(&nic->mbox.lock); } - free_mcam_flow: otx2_del_mcam_flow_entry(nic, flow_node->entry, NULL); otx2_tc_update_mcam_table(nic, flow_cfg, flow_node, false); @@ -1254,7 +1239,7 @@ static int otx2_tc_add_flow(struct otx2_nic *nic, struct otx2_flow_config *flow_cfg = nic->flow_cfg; struct otx2_tc_flow *new_node, *old_node; struct npc_install_flow_req *req, dummy; - int rc, err, mcam_idx; + int rc, err, entry; if (!(nic->flags & OTX2_FLAG_TC_FLOWER_SUPPORT)) return -ENOMEM; @@ -1264,7 +1249,7 @@ static int otx2_tc_add_flow(struct otx2_nic *nic, return -EINVAL; } - if (flow_cfg->nr_flows == flow_cfg->max_flows) { + if (!is_cn20k(nic->pdev) && flow_cfg->nr_flows == flow_cfg->max_flows) { NL_SET_ERR_MSG_MOD(extack, "Free MCAM entry not available to add the flow"); return -ENOMEM; @@ -1292,7 +1277,23 @@ static int otx2_tc_add_flow(struct otx2_nic *nic, if (old_node) otx2_tc_del_flow(nic, tc_flow_cmd); - mcam_idx = otx2_tc_update_mcam_table(nic, flow_cfg, new_node, true); + if (is_cn20k(nic->pdev)) { + rc = cn20k_tc_alloc_entry(nic, tc_flow_cmd, new_node, &dummy); + if (rc) { + NL_SET_ERR_MSG_MOD(extack, + "MCAM rule allocation failed"); + kfree_rcu(new_node, rcu); + return rc; + } + } + + entry = otx2_tc_update_mcam_table(nic, flow_cfg, new_node, true); + if (entry < 0) { + NL_SET_ERR_MSG_MOD(extack, "Adding rule failed"); + rc = entry; + goto free_leaf; + } + mutex_lock(&nic->mbox.lock); req = otx2_mbox_alloc_msg_npc_install_flow(&nic->mbox); if (!req) { @@ -1304,7 +1305,7 @@ static int otx2_tc_add_flow(struct otx2_nic *nic, memcpy(&dummy.hdr, &req->hdr, sizeof(struct mbox_msghdr)); memcpy(req, &dummy, sizeof(struct npc_install_flow_req)); req->channel = nic->hw.rx_chan_base; - req->entry = flow_cfg->flow_ent[mcam_idx]; + req->entry = (u16)entry; req->intf = NIX_INTF_RX; req->vf = nic->pcifunc; req->set_cntr = 1; @@ -1325,6 +1326,8 @@ static int otx2_tc_add_flow(struct otx2_nic *nic, return 0; free_leaf: + if (is_cn20k(nic->pdev)) + cn20k_tc_free_mcam_entry(nic, new_node->entry); otx2_tc_del_from_flow_list(flow_cfg, new_node); if (new_node->is_act_police) { mutex_lock(&nic->mbox.lock); diff --git a/drivers/net/ethernet/mediatek/mtk_eth_soc.c b/drivers/net/ethernet/mediatek/mtk_eth_soc.c index ddc321a02fda..8d225bc9f063 100644 --- a/drivers/net/ethernet/mediatek/mtk_eth_soc.c +++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.c @@ -562,9 +562,7 @@ static void mtk_mac_config(struct phylink_config *config, unsigned int mode, int val, ge_mode, err = 0; u32 i; - /* MT76x8 has no hardware settings between for the MAC */ - if (!MTK_HAS_CAPS(eth->soc->caps, MTK_SOC_MT7628) && - mac->interface != state->interface) { + if (mac->interface != state->interface) { /* Setup soc pin functions */ switch (state->interface) { case PHY_INTERFACE_MODE_TRGMII: @@ -956,6 +954,30 @@ static const struct phylink_mac_ops mtk_phylink_ops = { .mac_enable_tx_lpi = mtk_mac_enable_tx_lpi, }; +static void rt5350_mac_config(struct phylink_config *config, unsigned int mode, + const struct phylink_link_state *state) +{ +} + +static void rt5350_mac_link_down(struct phylink_config *config, unsigned int mode, + phy_interface_t interface) +{ +} + +static void rt5350_mac_link_up(struct phylink_config *config, + struct phy_device *phy, + unsigned int mode, phy_interface_t interface, + int speed, int duplex, bool tx_pause, bool rx_pause) +{ +} + +/* MT76x8 (rt5350-eth) does not expose any MAC control registers */ +static const struct phylink_mac_ops rt5350_phylink_ops = { + .mac_config = rt5350_mac_config, + .mac_link_down = rt5350_mac_link_down, + .mac_link_up = rt5350_mac_link_up, +}; + static void mtk_mdio_config(struct mtk_eth *eth) { u32 val; @@ -3566,12 +3588,23 @@ found: return NOTIFY_DONE; } +static int mtk_max_gmac_mtu(struct mtk_eth *eth) +{ + int i, max_mtu = ETH_DATA_LEN; + + for (i = 0; i < ARRAY_SIZE(eth->netdev); i++) + if (eth->netdev[i] && eth->netdev[i]->mtu > max_mtu) + max_mtu = eth->netdev[i]->mtu; + + return max_mtu; +} + static int mtk_open(struct net_device *dev) { struct mtk_mac *mac = netdev_priv(dev); struct mtk_eth *eth = mac->hw; struct mtk_mac *target_mac; - int i, err, ppe_num; + int i, err, ppe_num, mtu; ppe_num = eth->soc->ppe_num; @@ -3618,6 +3651,10 @@ static int mtk_open(struct net_device *dev) mtk_gdm_config(eth, target_mac->id, gdm_config); } + mtu = mtk_max_gmac_mtu(eth); + for (i = 0; i < ARRAY_SIZE(eth->ppe); i++) + mtk_ppe_update_mtu(eth->ppe[i], mtu); + napi_enable(ð->tx_napi); napi_enable(ð->rx_napi); mtk_tx_irq_enable(eth, MTK_TX_DONE_INT); @@ -4311,6 +4348,7 @@ static int mtk_change_mtu(struct net_device *dev, int new_mtu) int length = new_mtu + MTK_RX_ETH_HLEN; struct mtk_mac *mac = netdev_priv(dev); struct mtk_eth *eth = mac->hw; + int max_mtu, i; if (rcu_access_pointer(eth->prog) && length > MTK_PP_MAX_BUF_SIZE) { @@ -4321,6 +4359,10 @@ static int mtk_change_mtu(struct net_device *dev, int new_mtu) mtk_set_mcr_max_rx(mac, length); WRITE_ONCE(dev->mtu, new_mtu); + max_mtu = mtk_max_gmac_mtu(eth); + for (i = 0; i < ARRAY_SIZE(eth->ppe); i++) + mtk_ppe_update_mtu(eth->ppe[i], max_mtu); + return 0; } @@ -4780,6 +4822,7 @@ static const struct net_device_ops mtk_netdev_ops = { static int mtk_add_mac(struct mtk_eth *eth, struct device_node *np) { + const struct phylink_mac_ops *mac_ops = &mtk_phylink_ops; const __be32 *_id = of_get_property(np, "reg", NULL); phy_interface_t phy_mode; struct phylink *phylink; @@ -4914,9 +4957,12 @@ static int mtk_add_mac(struct mtk_eth *eth, struct device_node *np) mac->phylink_config.supported_interfaces); } + if (MTK_HAS_CAPS(eth->soc->caps, MTK_SOC_MT7628)) + mac_ops = &rt5350_phylink_ops; + phylink = phylink_create(&mac->phylink_config, of_fwnode_handle(mac->of_node), - phy_mode, &mtk_phylink_ops); + phy_mode, mac_ops); if (IS_ERR(phylink)) { err = PTR_ERR(phylink); goto free_netdev; diff --git a/drivers/net/ethernet/mediatek/mtk_ppe.c b/drivers/net/ethernet/mediatek/mtk_ppe.c index 75f7728fc796..18279e2a7022 100644 --- a/drivers/net/ethernet/mediatek/mtk_ppe.c +++ b/drivers/net/ethernet/mediatek/mtk_ppe.c @@ -973,6 +973,36 @@ static void mtk_ppe_init_foe_table(struct mtk_ppe *ppe) } } +void mtk_ppe_update_mtu(struct mtk_ppe *ppe, int mtu) +{ + int base; + u32 val; + + if (!ppe) + return; + + /* The PPE checks output frame size against per-tag-layer MTU limits, + * treating PPPoE and DSA tags just like 802.1Q VLAN tags. The Linux + * device MTU already accounts for PPPoE (PPPOE_SES_HLEN) and DSA tag + * overhead, but 802.1Q VLAN tags are handled transparently without + * being reflected by the lower device MTU being increased by 4. + * Use the maximum MTU across all GMAC interfaces so that PPE output + * frame limits are sufficiently high regardless of which port a flow + * egresses through. + */ + base = ETH_HLEN + mtu; + + val = FIELD_PREP(MTK_PPE_VLAN_MTU0_NONE, base) | + FIELD_PREP(MTK_PPE_VLAN_MTU0_1TAG, base + VLAN_HLEN); + ppe_w32(ppe, MTK_PPE_VLAN_MTU0, val); + + val = FIELD_PREP(MTK_PPE_VLAN_MTU1_2TAG, + base + 2 * VLAN_HLEN) | + FIELD_PREP(MTK_PPE_VLAN_MTU1_3TAG, + base + 3 * VLAN_HLEN); + ppe_w32(ppe, MTK_PPE_VLAN_MTU1, val); +} + void mtk_ppe_start(struct mtk_ppe *ppe) { u32 val; diff --git a/drivers/net/ethernet/mediatek/mtk_ppe.h b/drivers/net/ethernet/mediatek/mtk_ppe.h index 223f709e2704..ba85e39a155b 100644 --- a/drivers/net/ethernet/mediatek/mtk_ppe.h +++ b/drivers/net/ethernet/mediatek/mtk_ppe.h @@ -346,6 +346,7 @@ struct mtk_ppe { struct mtk_ppe *mtk_ppe_init(struct mtk_eth *eth, void __iomem *base, int index); void mtk_ppe_deinit(struct mtk_eth *eth); +void mtk_ppe_update_mtu(struct mtk_ppe *ppe, int mtu); void mtk_ppe_start(struct mtk_ppe *ppe); int mtk_ppe_stop(struct mtk_ppe *ppe); int mtk_ppe_prepare_reset(struct mtk_ppe *ppe); diff --git a/drivers/net/ethernet/mellanox/mlx4/cmd.c b/drivers/net/ethernet/mellanox/mlx4/cmd.c index de0193d82ec1..bdaf152e6712 100644 --- a/drivers/net/ethernet/mellanox/mlx4/cmd.c +++ b/drivers/net/ethernet/mellanox/mlx4/cmd.c @@ -1782,6 +1782,7 @@ static int mlx4_master_process_vhcr(struct mlx4_dev *dev, int slave, } if (err) { + vhcr_cmd->status = mlx4_errno_to_status(err); if (!(dev->persist->state & MLX4_DEVICE_STATE_INTERNAL_ERROR)) { if (vhcr->op == MLX4_CMD_ALLOC_RES && (vhcr->in_modifier & 0xff) == RES_COUNTER && @@ -1791,9 +1792,8 @@ static int mlx4_master_process_vhcr(struct mlx4_dev *dev, int slave, slave, err); else mlx4_warn(dev, "vhcr command:0x%x slave:%d failed with error:%d, status %d\n", - vhcr->op, slave, vhcr->errno, err); + vhcr->op, slave, err, vhcr_cmd->status); } - vhcr_cmd->status = mlx4_errno_to_status(err); goto out_status; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/Makefile b/drivers/net/ethernet/mellanox/mlx5/core/Makefile index 8ffa286a18f5..d39fe9c4a87c 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/Makefile +++ b/drivers/net/ethernet/mellanox/mlx5/core/Makefile @@ -16,8 +16,9 @@ mlx5_core-y := main.o cmd.o debugfs.o fw.o eq.o uar.o pagealloc.o \ transobj.o vport.o sriov.o fs_cmd.o fs_core.o pci_irq.o \ fs_counters.o fs_ft_pool.o rl.o lag/debugfs.o lag/lag.o dev.o events.o wq.o lib/gid.o \ lib/devcom.o lib/pci_vsc.o lib/dm.o lib/fs_ttc.o diag/fs_tracepoint.o \ - diag/fw_tracer.o diag/crdump.o devlink.o diag/rsc_dump.o diag/reporter_vnic.o \ - fw_reset.o qos.o lib/tout.o lib/aso.o wc.o fs_pool.o lib/nv_param.o + diag/fw_tracer.o diag/crdump.o devlink.o sh_devlink.o diag/rsc_dump.o \ + diag/reporter_vnic.o fw_reset.o qos.o lib/tout.o lib/aso.o wc.o fs_pool.o \ + lib/nv_param.o # # Netdev basic diff --git a/drivers/net/ethernet/mellanox/mlx5/core/cmd.c b/drivers/net/ethernet/mellanox/mlx5/core/cmd.c index 6c99c7f36163..c89417c1a1f9 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/cmd.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/cmd.c @@ -196,17 +196,18 @@ static void cmd_ent_put(struct mlx5_cmd_work_ent *ent) unsigned long flags; spin_lock_irqsave(&cmd->alloc_lock, flags); - if (!refcount_dec_and_test(&ent->refcnt)) - goto out; + if (!refcount_dec_and_test(&ent->refcnt)) { + spin_unlock_irqrestore(&cmd->alloc_lock, flags); + return; + } if (ent->idx >= 0) { cmd_free_index(cmd, ent->idx); up(ent->page_queue ? &cmd->vars.pages_sem : &cmd->vars.sem); } + spin_unlock_irqrestore(&cmd->alloc_lock, flags); cmd_free_ent(ent); -out: - spin_unlock_irqrestore(&cmd->alloc_lock, flags); } static struct mlx5_cmd_layout *get_inst(struct mlx5_cmd *cmd, int idx) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/devlink.h b/drivers/net/ethernet/mellanox/mlx5/core/devlink.h index 43b9bf8829cf..4fbb3926a3e5 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/devlink.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/devlink.h @@ -14,6 +14,10 @@ enum mlx5_devlink_resource_id { MLX5_ID_RES_MAX = __MLX5_ID_RES_MAX - 1, }; +enum mlx5_devlink_port_resource_id { + MLX5_DL_PORT_RES_MAX_SFS = 1, +}; + enum mlx5_devlink_param_id { MLX5_DEVLINK_PARAM_ID_BASE = DEVLINK_PARAM_GENERIC_ID_MAX, MLX5_DEVLINK_PARAM_ID_FLOW_STEERING_MODE, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/diag/fs_tracepoint.c b/drivers/net/ethernet/mellanox/mlx5/core/diag/fs_tracepoint.c index 6d73127b7217..2cf1d3825def 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/diag/fs_tracepoint.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/diag/fs_tracepoint.c @@ -282,6 +282,9 @@ const char *parse_fs_dst(struct trace_seq *p, case MLX5_FLOW_DESTINATION_TYPE_NONE: trace_seq_printf(p, "none\n"); break; + case MLX5_FLOW_DESTINATION_TYPE_VHCA_RX: + trace_seq_printf(p, "rx_vhca_id=%u\n", dst->vhca.id); + break; } trace_seq_putc(p, 0); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en.h b/drivers/net/ethernet/mellanox/mlx5/core/en.h index ea2cd1f5d1d0..2270e2e550dd 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en.h @@ -80,7 +80,11 @@ struct page_pool; #define MLX5_SKB_FRAG_SZ(len) (SKB_DATA_ALIGN(len) + \ SKB_DATA_ALIGN(sizeof(struct skb_shared_info))) +#define MLX5E_PAGECNT_BIAS_MAX U16_MAX #define MLX5E_RX_MAX_HEAD (256) +#define MLX5E_XDP_LOG_MAX_LINEAR_SZ \ + order_base_2(MLX5_SKB_FRAG_SZ(XDP_PACKET_HEADROOM + MLX5E_RX_MAX_HEAD)) + #define MLX5E_SHAMPO_LOG_HEADER_ENTRY_SIZE (8) #define MLX5E_SHAMPO_WQ_HEADER_PER_PAGE \ (PAGE_SIZE >> MLX5E_SHAMPO_LOG_HEADER_ENTRY_SIZE) @@ -590,10 +594,14 @@ union mlx5e_alloc_units { struct mlx5e_mpw_info { u16 consumed_strides; DECLARE_BITMAP(skip_release_bitmap, MLX5_MPWRQ_MAX_PAGES_PER_WQE); - struct mlx5e_frag_page linear_page; union mlx5e_alloc_units alloc_units; }; +struct mlx5e_mpw_linear_info { + struct mlx5e_frag_page frag_page; + u16 max_frags; +}; + #define MLX5E_MAX_RX_FRAGS 4 struct mlx5e_rq; @@ -688,6 +696,7 @@ struct mlx5e_rq { u8 umr_wqebbs; u8 mtts_per_wqe; u8 umr_mode; + struct mlx5e_mpw_linear_info *linear_info; struct mlx5e_shampo_hd *shampo; } mpwqe; }; @@ -1060,13 +1069,15 @@ void mlx5e_timestamp_init(struct mlx5e_priv *priv); struct mlx5e_xsk_param; struct mlx5e_rq_param; -int mlx5e_open_rq(struct mlx5e_params *params, struct mlx5e_rq_param *param, - struct mlx5e_xsk_param *xsk, int node, u16 q_counter, +struct mlx5e_rq_opt_param; +int mlx5e_open_rq(struct mlx5e_params *params, struct mlx5e_rq_param *rq_param, + struct mlx5e_rq_opt_param *rqo, int node, u16 q_counter, struct mlx5e_rq *rq); #define MLX5E_RQ_WQES_TIMEOUT 20000 /* msecs */ int mlx5e_wait_for_min_rx_wqes(struct mlx5e_rq *rq, int wait_time); void mlx5e_close_rq(struct mlx5e_rq *rq); -int mlx5e_create_rq(struct mlx5e_rq *rq, struct mlx5e_rq_param *param, u16 q_counter); +int mlx5e_create_rq(struct mlx5e_rq *rq, struct mlx5e_rq_param *rq_param, + u16 q_counter); void mlx5e_destroy_rq(struct mlx5e_rq *rq); bool mlx5e_reset_rx_moderation(struct dim_cq_moder *cq_moder, u8 cq_period_mode, @@ -1074,6 +1085,8 @@ bool mlx5e_reset_rx_moderation(struct dim_cq_moder *cq_moder, u8 cq_period_mode, bool mlx5e_reset_rx_channels_moderation(struct mlx5e_channels *chs, u8 cq_period_mode, bool dim_enabled, bool keep_dim_state); +void mlx5e_mpwqe_dealloc_linear_page(struct mlx5e_rq *rq); + struct mlx5e_sq_param; int mlx5e_open_xdpsq(struct mlx5e_channel *c, struct mlx5e_params *params, struct mlx5e_sq_param *param, struct xsk_buff_pool *xsk_pool, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/params.c b/drivers/net/ethernet/mellanox/mlx5/core/en/params.c index 8e99d07586fa..1f4a547917ba 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/params.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/params.c @@ -10,6 +10,7 @@ #include <linux/dim.h> #include <net/page_pool/types.h> #include <net/xdp_sock_drv.h> +#include <net/netdev_queues.h> #define MLX5_MPWRQ_MAX_LOG_WQE_SZ 18 #define MLX5_REP_MPWRQ_MAX_LOG_WQE_SZ 17 @@ -21,10 +22,20 @@ static u8 mlx5e_mpwrq_min_page_shift(struct mlx5_core_dev *mdev) return min_page_shift ? : 12; } -u8 mlx5e_mpwrq_page_shift(struct mlx5_core_dev *mdev, struct mlx5e_xsk_param *xsk) +u8 mlx5e_mpwrq_page_shift(struct mlx5_core_dev *mdev, + struct mlx5e_rq_opt_param *rqo) { - u8 req_page_shift = xsk ? order_base_2(xsk->chunk_size) : PAGE_SHIFT; + struct netdev_queue_config *qcfg = rqo ? rqo->qcfg : NULL; + struct mlx5e_xsk_param *xsk = mlx5e_rqo_xsk_param(rqo); u8 min_page_shift = mlx5e_mpwrq_min_page_shift(mdev); + u8 req_page_shift; + + if (xsk) + req_page_shift = order_base_2(xsk->chunk_size); + else if (qcfg && qcfg->rx_page_size) + req_page_shift = order_base_2(qcfg->rx_page_size); + else + req_page_shift = PAGE_SHIFT; /* Regular RQ uses order-0 pages, the NIC must be able to map them. */ if (WARN_ON_ONCE(!xsk && req_page_shift < min_page_shift)) @@ -34,7 +45,8 @@ u8 mlx5e_mpwrq_page_shift(struct mlx5_core_dev *mdev, struct mlx5e_xsk_param *xs } enum mlx5e_mpwrq_umr_mode -mlx5e_mpwrq_umr_mode(struct mlx5_core_dev *mdev, struct mlx5e_xsk_param *xsk) +mlx5e_mpwrq_umr_mode(struct mlx5_core_dev *mdev, + struct mlx5e_rq_opt_param *rqo) { /* Different memory management schemes use different mechanisms to map * user-mode memory. The stricter guarantees we have, the faster @@ -45,7 +57,8 @@ mlx5e_mpwrq_umr_mode(struct mlx5_core_dev *mdev, struct mlx5e_xsk_param *xsk) * 3. KLM - indirect mapping to another MKey to arbitrary addresses, and * mappings can have different sizes. */ - u8 page_shift = mlx5e_mpwrq_page_shift(mdev, xsk); + struct mlx5e_xsk_param *xsk = mlx5e_rqo_xsk_param(rqo); + u8 page_shift = mlx5e_mpwrq_page_shift(mdev, rqo); bool unaligned = xsk ? xsk->unaligned : false; bool oversized = false; @@ -225,12 +238,12 @@ u8 mlx5e_mpwrq_max_log_rq_pkts(struct mlx5_core_dev *mdev, u8 page_shift, } u16 mlx5e_get_linear_rq_headroom(struct mlx5e_params *params, - struct mlx5e_xsk_param *xsk) + struct mlx5e_rq_opt_param *rqo) { u16 headroom; - if (xsk) - return xsk->headroom; + if (mlx5e_rqo_xsk_param(rqo)) + return rqo->xsk->headroom; headroom = NET_IP_ALIGN; if (params->xdp_prog) @@ -263,39 +276,41 @@ static u32 mlx5e_rx_get_linear_sz_skb(struct mlx5e_params *params, bool no_head_ static u32 mlx5e_rx_get_linear_stride_sz(struct mlx5_core_dev *mdev, struct mlx5e_params *params, - struct mlx5e_xsk_param *xsk, + struct mlx5e_rq_opt_param *rqo, bool mpwqe) { + struct mlx5e_xsk_param *xsk = mlx5e_rqo_xsk_param(rqo); bool no_head_tail_room; u32 sz; /* XSK frames are mapped as individual pages, because frames may come in * an arbitrary order from random locations in the UMEM. */ - if (xsk) - return mpwqe ? 1 << mlx5e_mpwrq_page_shift(mdev, xsk) : PAGE_SIZE; + if (xsk) { + return mpwqe ? + BIT(mlx5e_mpwrq_page_shift(mdev, rqo)) : PAGE_SIZE; + } - no_head_tail_room = params->xdp_prog && mpwqe && !mlx5e_rx_is_linear_skb(mdev, params, xsk); + no_head_tail_room = params->xdp_prog && mpwqe && + !mlx5e_rx_is_linear_skb(mdev, params, rqo); /* When no_head_tail_room is set, headroom and tailroom are excluded from skb calculations. * no_head_tail_room should be set in the case of XDP with Striding RQ * when SKB is not linear. This is because another page is allocated for the linear part. */ - sz = roundup_pow_of_two(mlx5e_rx_get_linear_sz_skb(params, no_head_tail_room)); + sz = mlx5e_rx_get_linear_sz_skb(params, no_head_tail_room); - /* XDP in mlx5e doesn't support multiple packets per page. - * Do not assume sz <= PAGE_SIZE if params->xdp_prog is set. - */ - return params->xdp_prog && sz < PAGE_SIZE ? PAGE_SIZE : sz; + return roundup_pow_of_two(sz); } static u8 mlx5e_mpwqe_log_pkts_per_wqe(struct mlx5_core_dev *mdev, struct mlx5e_params *params, - struct mlx5e_xsk_param *xsk) + struct mlx5e_rq_opt_param *rqo) { - u32 linear_stride_sz = mlx5e_rx_get_linear_stride_sz(mdev, params, xsk, true); - enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, xsk); - u8 page_shift = mlx5e_mpwrq_page_shift(mdev, xsk); + enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, rqo); + u32 linear_stride_sz = + mlx5e_rx_get_linear_stride_sz(mdev, params, rqo, true); + u8 page_shift = mlx5e_mpwrq_page_shift(mdev, rqo); return mlx5e_mpwrq_log_wqe_sz(mdev, page_shift, umr_mode) - order_base_2(linear_stride_sz); @@ -303,8 +318,10 @@ static u8 mlx5e_mpwqe_log_pkts_per_wqe(struct mlx5_core_dev *mdev, bool mlx5e_rx_is_linear_skb(struct mlx5_core_dev *mdev, struct mlx5e_params *params, - struct mlx5e_xsk_param *xsk) + struct mlx5e_rq_opt_param *rqo) { + struct mlx5e_xsk_param *xsk = mlx5e_rqo_xsk_param(rqo); + if (params->packet_merge.type != MLX5E_PACKET_MERGE_NONE) return false; @@ -315,7 +332,7 @@ bool mlx5e_rx_is_linear_skb(struct mlx5_core_dev *mdev, * Both XSK and non-XSK cases allocate an SKB on XDP_PASS. Packet data * must fit into a CPU page. */ - if (mlx5e_rx_get_linear_sz_skb(params, xsk) > PAGE_SIZE) + if (mlx5e_rx_get_linear_sz_skb(params, !!xsk) > PAGE_SIZE) return false; /* XSK frames must be big enough to hold the packet data. */ @@ -349,12 +366,14 @@ static bool mlx5e_verify_rx_mpwqe_strides(struct mlx5_core_dev *mdev, bool mlx5e_verify_params_rx_mpwqe_strides(struct mlx5_core_dev *mdev, struct mlx5e_params *params, - struct mlx5e_xsk_param *xsk) + struct mlx5e_rq_opt_param *rqo) { - u8 log_wqe_num_of_strides = mlx5e_mpwqe_get_log_num_strides(mdev, params, xsk); - u8 log_wqe_stride_size = mlx5e_mpwqe_get_log_stride_size(mdev, params, xsk); - enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, xsk); - u8 page_shift = mlx5e_mpwrq_page_shift(mdev, xsk); + enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, rqo); + u8 log_wqe_num_of_strides = + mlx5e_mpwqe_get_log_num_strides(mdev, params, rqo); + u8 log_wqe_stride_size = + mlx5e_mpwqe_get_log_stride_size(mdev, params, rqo); + u8 page_shift = mlx5e_mpwrq_page_shift(mdev, rqo); return mlx5e_verify_rx_mpwqe_strides(mdev, log_wqe_stride_size, log_wqe_num_of_strides, @@ -363,18 +382,20 @@ bool mlx5e_verify_params_rx_mpwqe_strides(struct mlx5_core_dev *mdev, bool mlx5e_rx_mpwqe_is_linear_skb(struct mlx5_core_dev *mdev, struct mlx5e_params *params, - struct mlx5e_xsk_param *xsk) + struct mlx5e_rq_opt_param *rqo) { - enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, xsk); - u8 page_shift = mlx5e_mpwrq_page_shift(mdev, xsk); + enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, rqo); + u32 linear_stride_sz = + mlx5e_rx_get_linear_stride_sz(mdev, params, rqo, true); + u8 page_shift = mlx5e_mpwrq_page_shift(mdev, rqo); u8 log_num_strides; u8 log_stride_sz; u8 log_wqe_sz; - if (!mlx5e_rx_is_linear_skb(mdev, params, xsk)) + if (!mlx5e_rx_is_linear_skb(mdev, params, rqo)) return false; - log_stride_sz = order_base_2(mlx5e_rx_get_linear_stride_sz(mdev, params, xsk, true)); + log_stride_sz = order_base_2(linear_stride_sz); log_wqe_sz = mlx5e_mpwrq_log_wqe_sz(mdev, page_shift, umr_mode); if (log_wqe_sz < log_stride_sz) @@ -389,13 +410,13 @@ bool mlx5e_rx_mpwqe_is_linear_skb(struct mlx5_core_dev *mdev, u8 mlx5e_mpwqe_get_log_rq_size(struct mlx5_core_dev *mdev, struct mlx5e_params *params, - struct mlx5e_xsk_param *xsk) + struct mlx5e_rq_opt_param *rqo) { - enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, xsk); + enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, rqo); u8 log_pkts_per_wqe, page_shift, max_log_rq_size; - log_pkts_per_wqe = mlx5e_mpwqe_log_pkts_per_wqe(mdev, params, xsk); - page_shift = mlx5e_mpwrq_page_shift(mdev, xsk); + log_pkts_per_wqe = mlx5e_mpwqe_log_pkts_per_wqe(mdev, params, rqo); + page_shift = mlx5e_mpwrq_page_shift(mdev, rqo); max_log_rq_size = mlx5e_mpwrq_max_log_rq_size(mdev, page_shift, umr_mode); /* Numbers are unsigned, don't subtract to avoid underflow. */ @@ -423,31 +444,29 @@ static u8 mlx5e_shampo_get_log_pkt_per_rsrv(struct mlx5e_params *params) u8 mlx5e_mpwqe_get_log_stride_size(struct mlx5_core_dev *mdev, struct mlx5e_params *params, - struct mlx5e_xsk_param *xsk) + struct mlx5e_rq_opt_param *rqo) { - if (mlx5e_rx_mpwqe_is_linear_skb(mdev, params, xsk)) - return order_base_2(mlx5e_rx_get_linear_stride_sz(mdev, params, xsk, true)); - - /* XDP in mlx5e doesn't support multiple packets per page. */ - if (params->xdp_prog) - return PAGE_SHIFT; + if (mlx5e_rx_mpwqe_is_linear_skb(mdev, params, rqo)) + return order_base_2(mlx5e_rx_get_linear_stride_sz(mdev, params, + rqo, true)); return MLX5_MPWRQ_DEF_LOG_STRIDE_SZ(mdev); } u8 mlx5e_mpwqe_get_log_num_strides(struct mlx5_core_dev *mdev, struct mlx5e_params *params, - struct mlx5e_xsk_param *xsk) + struct mlx5e_rq_opt_param *rqo) { - enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, xsk); - u8 page_shift = mlx5e_mpwrq_page_shift(mdev, xsk); + enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, rqo); + u8 page_shift = mlx5e_mpwrq_page_shift(mdev, rqo); u8 log_wqe_size, log_stride_size; log_wqe_size = mlx5e_mpwrq_log_wqe_sz(mdev, page_shift, umr_mode); - log_stride_size = mlx5e_mpwqe_get_log_stride_size(mdev, params, xsk); + log_stride_size = mlx5e_mpwqe_get_log_stride_size(mdev, params, rqo); WARN(log_wqe_size < log_stride_size, "Log WQE size %u < log stride size %u (page shift %u, umr mode %d, xsk on? %d)\n", - log_wqe_size, log_stride_size, page_shift, umr_mode, !!xsk); + log_wqe_size, log_stride_size, page_shift, umr_mode, + rqo && rqo->xsk); return log_wqe_size - log_stride_size; } @@ -459,14 +478,14 @@ u8 mlx5e_mpwqe_get_min_wqe_bulk(unsigned int wq_sz) u16 mlx5e_get_rq_headroom(struct mlx5_core_dev *mdev, struct mlx5e_params *params, - struct mlx5e_xsk_param *xsk) + struct mlx5e_rq_opt_param *rqo) { - u16 linear_headroom = mlx5e_get_linear_rq_headroom(params, xsk); + u16 linear_headroom = mlx5e_get_linear_rq_headroom(params, rqo); if (params->rq_wq_type == MLX5_WQ_TYPE_CYCLIC) return linear_headroom; - if (mlx5e_rx_mpwqe_is_linear_skb(mdev, params, xsk)) + if (mlx5e_rx_mpwqe_is_linear_skb(mdev, params, rqo)) return linear_headroom; if (params->packet_merge.type == MLX5E_PACKET_MERGE_SHAMPO) @@ -475,6 +494,18 @@ u16 mlx5e_get_rq_headroom(struct mlx5_core_dev *mdev, return 0; } +u32 mlx5e_mpwrq_max_page_size(struct mlx5_core_dev *mdev) +{ + if (mlx5_core_is_ecpf(mdev)) + return PAGE_SIZE; + + /* Two MTTs are needed to form an octword. Driver is using a + * single page per MTT for simplicity. Hence the limit of having + * at least 2 pages per WQE. + */ + return BIT(MLX5_MPWRQ_MAX_LOG_WQE_SZ - 1); +} + u16 mlx5e_calc_sq_stop_room(struct mlx5_core_dev *mdev, struct mlx5e_params *params) { bool is_mpwqe = MLX5E_GET_PFLAG(params, MLX5E_PFLAG_SKB_TX_MPWQE); @@ -535,10 +566,11 @@ int mlx5e_mpwrq_validate_regular(struct mlx5_core_dev *mdev, struct mlx5e_params } int mlx5e_mpwrq_validate_xsk(struct mlx5_core_dev *mdev, struct mlx5e_params *params, - struct mlx5e_xsk_param *xsk) + struct mlx5e_rq_opt_param *rqo) { - enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, xsk); - u8 page_shift = mlx5e_mpwrq_page_shift(mdev, xsk); + enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, rqo); + struct mlx5e_xsk_param *xsk = mlx5e_rqo_xsk_param(rqo); + u8 page_shift = mlx5e_mpwrq_page_shift(mdev, rqo); u16 max_mtu_pkts; if (!mlx5e_check_fragmented_striding_rq_cap(mdev, page_shift, umr_mode)) { @@ -547,7 +579,7 @@ int mlx5e_mpwrq_validate_xsk(struct mlx5_core_dev *mdev, struct mlx5e_params *pa return -EOPNOTSUPP; } - if (!mlx5e_rx_mpwqe_is_linear_skb(mdev, params, xsk)) { + if (!mlx5e_rx_mpwqe_is_linear_skb(mdev, params, rqo)) { mlx5_core_err(mdev, "Striding RQ linear mode for XSK can't be activated with current params\n"); return -EINVAL; } @@ -559,7 +591,8 @@ int mlx5e_mpwrq_validate_xsk(struct mlx5_core_dev *mdev, struct mlx5e_params *pa mlx5e_mpwrq_max_log_rq_pkts(mdev, page_shift, xsk->unaligned)); if (params->log_rq_mtu_frames > max_mtu_pkts) { mlx5_core_err(mdev, "Current RQ length %d is too big for XSK with given frame size %u\n", - 1 << params->log_rq_mtu_frames, xsk->chunk_size); + 1 << params->log_rq_mtu_frames, + xsk->chunk_size); return -EINVAL; } @@ -672,7 +705,7 @@ static void mlx5e_rx_compute_wqe_bulk_params(struct mlx5e_params *params, static int mlx5e_build_rq_frags_info(struct mlx5_core_dev *mdev, struct mlx5e_params *params, - struct mlx5e_xsk_param *xsk, + struct mlx5e_rq_opt_param *rqo, struct mlx5e_rq_frags_info *info, u32 *xdp_frag_size) { @@ -684,10 +717,11 @@ static int mlx5e_build_rq_frags_info(struct mlx5_core_dev *mdev, int max_mtu; int i; - if (mlx5e_rx_is_linear_skb(mdev, params, xsk)) { + if (mlx5e_rx_is_linear_skb(mdev, params, rqo)) { int frag_stride; - frag_stride = mlx5e_rx_get_linear_stride_sz(mdev, params, xsk, false); + frag_stride = mlx5e_rx_get_linear_stride_sz(mdev, params, rqo, + false); info->arr[0].frag_size = byte_count; info->arr[0].frag_stride = frag_stride; @@ -703,7 +737,7 @@ static int mlx5e_build_rq_frags_info(struct mlx5_core_dev *mdev, goto out; } - headroom = mlx5e_get_linear_rq_headroom(params, xsk); + headroom = mlx5e_get_linear_rq_headroom(params, rqo); first_frag_size_max = SKB_WITH_OVERHEAD(frag_size_max - headroom); max_mtu = mlx5e_max_nonlinear_mtu(first_frag_size_max, frag_size_max, @@ -819,12 +853,13 @@ static void mlx5e_build_common_cq_param(struct mlx5_core_dev *mdev, static u32 mlx5e_shampo_get_log_cq_size(struct mlx5_core_dev *mdev, struct mlx5e_params *params, - struct mlx5e_xsk_param *xsk) + struct mlx5e_rq_opt_param *rqo) { - u16 num_strides = BIT(mlx5e_mpwqe_get_log_num_strides(mdev, params, xsk)); - u8 log_stride_sz = mlx5e_mpwqe_get_log_stride_size(mdev, params, xsk); + u8 log_stride_sz = mlx5e_mpwqe_get_log_stride_size(mdev, params, rqo); + u16 num_strides = BIT(mlx5e_mpwqe_get_log_num_strides(mdev, params, + rqo)); int pkt_per_rsrv = BIT(mlx5e_shampo_get_log_pkt_per_rsrv(params)); - int wq_size = BIT(mlx5e_mpwqe_get_log_rq_size(mdev, params, xsk)); + int wq_size = BIT(mlx5e_mpwqe_get_log_rq_size(mdev, params, rqo)); int wqe_size = BIT(log_stride_sz) * num_strides; int rsrv_size = MLX5E_SHAMPO_WQ_RESRV_SIZE; @@ -836,7 +871,7 @@ static u32 mlx5e_shampo_get_log_cq_size(struct mlx5_core_dev *mdev, static void mlx5e_build_rx_cq_param(struct mlx5_core_dev *mdev, struct mlx5e_params *params, - struct mlx5e_xsk_param *xsk, + struct mlx5e_rq_opt_param *rqo, struct mlx5e_cq_param *param) { bool hw_stridx = false; @@ -847,10 +882,13 @@ static void mlx5e_build_rx_cq_param(struct mlx5_core_dev *mdev, case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ: hw_stridx = MLX5_CAP_GEN(mdev, mini_cqe_resp_stride_index); if (params->packet_merge.type == MLX5E_PACKET_MERGE_SHAMPO) - log_cq_size = mlx5e_shampo_get_log_cq_size(mdev, params, xsk); + log_cq_size = + mlx5e_shampo_get_log_cq_size(mdev, params, rqo); else - log_cq_size = mlx5e_mpwqe_get_log_rq_size(mdev, params, xsk) + - mlx5e_mpwqe_get_log_num_strides(mdev, params, xsk); + log_cq_size = + mlx5e_mpwqe_get_log_rq_size(mdev, params, rqo) + + mlx5e_mpwqe_get_log_num_strides(mdev, params, + rqo); break; default: /* MLX5_WQ_TYPE_CYCLIC */ log_cq_size = params->log_rq_mtu_frames; @@ -880,69 +918,90 @@ static u8 rq_end_pad_mode(struct mlx5_core_dev *mdev, struct mlx5e_params *param MLX5_WQ_END_PAD_MODE_NONE : MLX5_WQ_END_PAD_MODE_ALIGN; } +static int mlx5e_mpwqe_build_rq_param(struct mlx5_core_dev *mdev, + struct mlx5e_params *params, + struct mlx5e_rq_opt_param *rqo, + struct mlx5e_rq_param *rq_param) +{ + u8 log_rq_sz = mlx5e_mpwqe_get_log_rq_size(mdev, params, rqo); + u8 page_shift = mlx5e_mpwrq_page_shift(mdev, rqo); + u8 log_wqe_num_of_strides, log_wqe_stride_size; + enum mlx5e_mpwrq_umr_mode umr_mode; + void *rqc = rq_param->rqc; + u32 lro_timeout; + void *wq; + + log_wqe_num_of_strides = + mlx5e_mpwqe_get_log_num_strides(mdev, params, rqo); + log_wqe_stride_size = + mlx5e_mpwqe_get_log_stride_size(mdev, params, rqo); + umr_mode = mlx5e_mpwrq_umr_mode(mdev, rqo); + + wq = MLX5_ADDR_OF(rqc, rqc, wq); + if (!mlx5e_verify_rx_mpwqe_strides(mdev, log_wqe_stride_size, + log_wqe_num_of_strides, + page_shift, umr_mode)) { + mlx5_core_err(mdev, + "Bad RX MPWQE params: log_stride_size %u, log_num_strides %u, umr_mode %d\n", + log_wqe_stride_size, log_wqe_num_of_strides, + umr_mode); + return -EINVAL; + } + + MLX5_SET(wq, wq, log_wqe_num_of_strides, + log_wqe_num_of_strides - MLX5_MPWQE_LOG_NUM_STRIDES_BASE); + MLX5_SET(wq, wq, log_wqe_stride_size, + log_wqe_stride_size - MLX5_MPWQE_LOG_STRIDE_SZ_BASE); + MLX5_SET(wq, wq, log_wq_sz, log_rq_sz); + if (params->packet_merge.type != MLX5E_PACKET_MERGE_SHAMPO) + return 0; + + MLX5_SET(wq, wq, shampo_enable, true); + MLX5_SET(wq, wq, log_reservation_size, + MLX5E_SHAMPO_WQ_LOG_RESRV_SIZE - + MLX5E_SHAMPO_WQ_RESRV_SIZE_BASE_SHIFT); + MLX5_SET(wq, wq, log_max_num_of_packets_per_reservation, + mlx5e_shampo_get_log_pkt_per_rsrv(params)); + MLX5_SET(wq, wq, log_headers_entry_size, + MLX5E_SHAMPO_LOG_HEADER_ENTRY_SIZE - + MLX5E_SHAMPO_WQ_BASE_HEAD_ENTRY_SIZE_SHIFT); + lro_timeout = mlx5e_choose_lro_timeout(mdev, + MLX5E_DEFAULT_SHAMPO_TIMEOUT); + MLX5_SET(rqc, rqc, reservation_timeout, lro_timeout); + MLX5_SET(rqc, rqc, shampo_match_criteria_type, + MLX5_RQC_SHAMPO_MATCH_CRITERIA_TYPE_EXTENDED); + MLX5_SET(rqc, rqc, shampo_no_match_alignment_granularity, + MLX5_RQC_SHAMPO_NO_MATCH_ALIGNMENT_GRANULARITY_STRIDE); + + return 0; +} + int mlx5e_build_rq_param(struct mlx5_core_dev *mdev, struct mlx5e_params *params, - struct mlx5e_xsk_param *xsk, - struct mlx5e_rq_param *param) + struct mlx5e_rq_opt_param *rqo, + struct mlx5e_rq_param *rq_param) { - void *rqc = param->rqc; - void *wq = MLX5_ADDR_OF(rqc, rqc, wq); - u32 lro_timeout; + void *rqc = rq_param->rqc; int ndsegs = 1; + void *wq; int err; - switch (params->rq_wq_type) { - case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ: { - u8 log_wqe_num_of_strides = mlx5e_mpwqe_get_log_num_strides(mdev, params, xsk); - u8 log_wqe_stride_size = mlx5e_mpwqe_get_log_stride_size(mdev, params, xsk); - enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, xsk); - u8 page_shift = mlx5e_mpwrq_page_shift(mdev, xsk); - - if (!mlx5e_verify_rx_mpwqe_strides(mdev, log_wqe_stride_size, - log_wqe_num_of_strides, - page_shift, umr_mode)) { - mlx5_core_err(mdev, - "Bad RX MPWQE params: log_stride_size %u, log_num_strides %u, umr_mode %d\n", - log_wqe_stride_size, log_wqe_num_of_strides, - umr_mode); - return -EINVAL; - } - - MLX5_SET(wq, wq, log_wqe_num_of_strides, - log_wqe_num_of_strides - MLX5_MPWQE_LOG_NUM_STRIDES_BASE); - MLX5_SET(wq, wq, log_wqe_stride_size, - log_wqe_stride_size - MLX5_MPWQE_LOG_STRIDE_SZ_BASE); - MLX5_SET(wq, wq, log_wq_sz, mlx5e_mpwqe_get_log_rq_size(mdev, params, xsk)); - if (params->packet_merge.type != MLX5E_PACKET_MERGE_SHAMPO) - break; + wq = MLX5_ADDR_OF(rqc, rqc, wq); - MLX5_SET(wq, wq, shampo_enable, true); - MLX5_SET(wq, wq, log_reservation_size, - MLX5E_SHAMPO_WQ_LOG_RESRV_SIZE - - MLX5E_SHAMPO_WQ_RESRV_SIZE_BASE_SHIFT); - MLX5_SET(wq, wq, - log_max_num_of_packets_per_reservation, - mlx5e_shampo_get_log_pkt_per_rsrv(params)); - MLX5_SET(wq, wq, log_headers_entry_size, - MLX5E_SHAMPO_LOG_HEADER_ENTRY_SIZE - - MLX5E_SHAMPO_WQ_BASE_HEAD_ENTRY_SIZE_SHIFT); - lro_timeout = - mlx5e_choose_lro_timeout(mdev, - MLX5E_DEFAULT_SHAMPO_TIMEOUT); - MLX5_SET(rqc, rqc, reservation_timeout, lro_timeout); - MLX5_SET(rqc, rqc, shampo_match_criteria_type, - MLX5_RQC_SHAMPO_MATCH_CRITERIA_TYPE_EXTENDED); - MLX5_SET(rqc, rqc, shampo_no_match_alignment_granularity, - MLX5_RQC_SHAMPO_NO_MATCH_ALIGNMENT_GRANULARITY_STRIDE); + switch (params->rq_wq_type) { + case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ: + err = mlx5e_mpwqe_build_rq_param(mdev, params, rqo, rq_param); + if (err) + return err; break; - } default: /* MLX5_WQ_TYPE_CYCLIC */ MLX5_SET(wq, wq, log_wq_sz, params->log_rq_mtu_frames); - err = mlx5e_build_rq_frags_info(mdev, params, xsk, ¶m->frags_info, - ¶m->xdp_frag_size); + err = mlx5e_build_rq_frags_info(mdev, params, rqo, + &rq_param->frags_info, + &rq_param->xdp_frag_size); if (err) return err; - ndsegs = param->frags_info.num_frags; + ndsegs = rq_param->frags_info.num_frags; } MLX5_SET(wq, wq, wq_type, params->rq_wq_type); @@ -953,23 +1012,23 @@ int mlx5e_build_rq_param(struct mlx5_core_dev *mdev, MLX5_SET(rqc, rqc, vsd, params->vlan_strip_disable); MLX5_SET(rqc, rqc, scatter_fcs, params->scatter_fcs_en); - param->wq.buf_numa_node = dev_to_node(mlx5_core_dma_dev(mdev)); - mlx5e_build_rx_cq_param(mdev, params, xsk, ¶m->cqp); + rq_param->wq.buf_numa_node = dev_to_node(mlx5_core_dma_dev(mdev)); + mlx5e_build_rx_cq_param(mdev, params, rqo, &rq_param->cqp); return 0; } void mlx5e_build_drop_rq_param(struct mlx5_core_dev *mdev, - struct mlx5e_rq_param *param) + struct mlx5e_rq_param *rq_param) { - void *rqc = param->rqc; + void *rqc = rq_param->rqc; void *wq = MLX5_ADDR_OF(rqc, rqc, wq); MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_CYCLIC); MLX5_SET(wq, wq, log_wq_stride, mlx5e_get_rqwq_log_stride(MLX5_WQ_TYPE_CYCLIC, 1)); - param->wq.buf_numa_node = dev_to_node(mlx5_core_dma_dev(mdev)); + rq_param->wq.buf_numa_node = dev_to_node(mlx5_core_dma_dev(mdev)); } void mlx5e_build_tx_cq_param(struct mlx5_core_dev *mdev, @@ -1084,29 +1143,30 @@ u32 mlx5e_choose_lro_timeout(struct mlx5_core_dev *mdev, u32 wanted_timeout) static u32 mlx5e_mpwrq_total_umr_wqebbs(struct mlx5_core_dev *mdev, struct mlx5e_params *params, - struct mlx5e_xsk_param *xsk) + struct mlx5e_rq_opt_param *rqo) { - enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, xsk); - u8 page_shift = mlx5e_mpwrq_page_shift(mdev, xsk); + enum mlx5e_mpwrq_umr_mode umr_mode = mlx5e_mpwrq_umr_mode(mdev, rqo); + u8 page_shift = mlx5e_mpwrq_page_shift(mdev, rqo); u8 umr_wqebbs; umr_wqebbs = mlx5e_mpwrq_umr_wqebbs(mdev, page_shift, umr_mode); - return umr_wqebbs * (1 << mlx5e_mpwqe_get_log_rq_size(mdev, params, xsk)); + return umr_wqebbs * + (1 << mlx5e_mpwqe_get_log_rq_size(mdev, params, rqo)); } -static u8 mlx5e_build_icosq_log_wq_sz(struct mlx5_core_dev *mdev, - struct mlx5e_params *params, - struct mlx5e_rq_param *rqp) +static u32 mlx5e_max_xsk_wqebbs(struct mlx5_core_dev *mdev, + struct mlx5e_params *params) { - u32 wqebbs, total_pages, useful_space; + struct mlx5e_rq_opt_param rqo = {0}; + struct mlx5e_xsk_param xsk = {0}; + u32 max_xsk_wqebbs = 0; + u8 frame_shift; - /* MLX5_WQ_TYPE_CYCLIC */ - if (params->rq_wq_type != MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) - return MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE; + if (!params->xdp_prog) + return 0; - /* UMR WQEs for the regular RQ. */ - wqebbs = mlx5e_mpwrq_total_umr_wqebbs(mdev, params, NULL); + rqo.xsk = &xsk; /* If XDP program is attached, XSK may be turned on at any time without * restarting the channel. ICOSQ must be big enough to fit UMR WQEs of @@ -1118,41 +1178,54 @@ static u8 mlx5e_build_icosq_log_wq_sz(struct mlx5_core_dev *mdev, * from capabilities. Hence, we have to try all valid values of XSK * frame size (and page_shift) to find the maximum. */ - if (params->xdp_prog) { - u32 max_xsk_wqebbs = 0; - u8 frame_shift; - - for (frame_shift = XDP_UMEM_MIN_CHUNK_SHIFT; - frame_shift <= PAGE_SHIFT; frame_shift++) { - /* The headroom doesn't affect the calculation. */ - struct mlx5e_xsk_param xsk = { - .chunk_size = 1 << frame_shift, - .unaligned = false, - }; - - /* XSK aligned mode. */ - max_xsk_wqebbs = max(max_xsk_wqebbs, - mlx5e_mpwrq_total_umr_wqebbs(mdev, params, &xsk)); - - /* XSK unaligned mode, frame size is a power of two. */ - xsk.unaligned = true; - max_xsk_wqebbs = max(max_xsk_wqebbs, - mlx5e_mpwrq_total_umr_wqebbs(mdev, params, &xsk)); - - /* XSK unaligned mode, frame size is not equal to stride size. */ - xsk.chunk_size -= 1; - max_xsk_wqebbs = max(max_xsk_wqebbs, - mlx5e_mpwrq_total_umr_wqebbs(mdev, params, &xsk)); - - /* XSK unaligned mode, frame size is a triple power of two. */ - xsk.chunk_size = (1 << frame_shift) / 4 * 3; - max_xsk_wqebbs = max(max_xsk_wqebbs, - mlx5e_mpwrq_total_umr_wqebbs(mdev, params, &xsk)); - } + for (frame_shift = XDP_UMEM_MIN_CHUNK_SHIFT; + frame_shift <= PAGE_SHIFT; frame_shift++) { + u32 total_wqebbs; + + /* The headroom doesn't affect the calculations below. */ + + /* XSK aligned mode. */ + xsk.chunk_size = 1 << frame_shift; + xsk.unaligned = false; + total_wqebbs = mlx5e_mpwrq_total_umr_wqebbs(mdev, params, &rqo); + max_xsk_wqebbs = max(max_xsk_wqebbs, total_wqebbs); + + /* XSK unaligned mode, frame size is a power of two. */ + xsk.unaligned = true; + total_wqebbs = mlx5e_mpwrq_total_umr_wqebbs(mdev, params, &rqo); + max_xsk_wqebbs = max(max_xsk_wqebbs, total_wqebbs); - wqebbs += max_xsk_wqebbs; + /* XSK unaligned mode, frame size is not equal to stride + * size. + */ + xsk.chunk_size -= 1; + total_wqebbs = mlx5e_mpwrq_total_umr_wqebbs(mdev, params, &rqo); + max_xsk_wqebbs = max(max_xsk_wqebbs, total_wqebbs); + + /* XSK unaligned mode, frame size is a triple power of two. */ + xsk.chunk_size = (1 << frame_shift) / 4 * 3; + total_wqebbs = mlx5e_mpwrq_total_umr_wqebbs(mdev, params, &rqo); + max_xsk_wqebbs = max(max_xsk_wqebbs, total_wqebbs); } + return max_xsk_wqebbs; +} + +static u8 mlx5e_build_icosq_log_wq_sz(struct mlx5_core_dev *mdev, + struct mlx5e_params *params, + struct mlx5e_rq_param *rq_param) +{ + u32 wqebbs, total_pages, useful_space; + + /* MLX5_WQ_TYPE_CYCLIC */ + if (params->rq_wq_type != MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) + return MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE; + + /* UMR WQEs for the regular RQ. */ + wqebbs = mlx5e_mpwrq_total_umr_wqebbs(mdev, params, NULL); + + wqebbs += mlx5e_max_xsk_wqebbs(mdev, params); + /* UMR WQEs don't cross the page boundary, they are padded with NOPs. * This padding is always smaller than the max WQE size. That gives us * at least (PAGE_SIZE - (max WQE size - MLX5_SEND_WQE_BB)) useful bytes @@ -1222,12 +1295,15 @@ void mlx5e_build_xdpsq_param(struct mlx5_core_dev *mdev, int mlx5e_build_channel_param(struct mlx5_core_dev *mdev, struct mlx5e_params *params, + struct netdev_queue_config *qcfg, struct mlx5e_channel_param *cparam) { u8 icosq_log_wq_sz, async_icosq_log_wq_sz; int err; - err = mlx5e_build_rq_param(mdev, params, NULL, &cparam->rq); + cparam->rq_opt.qcfg = qcfg; + + err = mlx5e_build_rq_param(mdev, params, &cparam->rq_opt, &cparam->rq); if (err) return err; @@ -1241,3 +1317,13 @@ int mlx5e_build_channel_param(struct mlx5_core_dev *mdev, return 0; } + +void mlx5e_build_xsk_channel_param(struct mlx5_core_dev *mdev, + struct mlx5e_params *params, + struct mlx5e_xsk_param *xsk, + struct mlx5e_channel_param *cparam) +{ + cparam->rq_opt.xsk = xsk; + mlx5e_build_rq_param(mdev, params, &cparam->rq_opt, &cparam->rq); + mlx5e_build_xdpsq_param(mdev, params, &cparam->xdp_sq); +} diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/params.h b/drivers/net/ethernet/mellanox/mlx5/core/en/params.h index 00617c65fe3c..275f9be53a34 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/params.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/params.h @@ -8,10 +8,15 @@ struct mlx5e_xsk_param { u16 headroom; - u16 chunk_size; + u32 chunk_size; bool unaligned; }; +struct mlx5e_rq_opt_param { + struct mlx5e_xsk_param *xsk; + struct netdev_queue_config *qcfg; +}; + struct mlx5e_cq_param { u32 cqc[MLX5_ST_SZ_DW(cqc)]; struct mlx5_wq_param wq; @@ -38,6 +43,7 @@ struct mlx5e_sq_param { struct mlx5e_channel_param { struct mlx5e_rq_param rq; + struct mlx5e_rq_opt_param rq_opt; struct mlx5e_sq_param txq_sq; struct mlx5e_sq_param xdp_sq; struct mlx5e_sq_param icosq; @@ -56,9 +62,11 @@ struct mlx5e_create_sq_param { /* Striding RQ dynamic parameters */ -u8 mlx5e_mpwrq_page_shift(struct mlx5_core_dev *mdev, struct mlx5e_xsk_param *xsk); +u8 mlx5e_mpwrq_page_shift(struct mlx5_core_dev *mdev, + struct mlx5e_rq_opt_param *rqo); enum mlx5e_mpwrq_umr_mode -mlx5e_mpwrq_umr_mode(struct mlx5_core_dev *mdev, struct mlx5e_xsk_param *xsk); +mlx5e_mpwrq_umr_mode(struct mlx5_core_dev *mdev, + struct mlx5e_rq_opt_param *rqo); u8 mlx5e_mpwrq_umr_entry_size(enum mlx5e_mpwrq_umr_mode mode); u8 mlx5e_mpwrq_log_wqe_sz(struct mlx5_core_dev *mdev, u8 page_shift, enum mlx5e_mpwrq_umr_mode umr_mode); @@ -80,22 +88,22 @@ u8 mlx5e_mpwrq_max_log_rq_pkts(struct mlx5_core_dev *mdev, u8 page_shift, bool slow_pci_heuristic(struct mlx5_core_dev *mdev); int mlx5e_mpwrq_validate_regular(struct mlx5_core_dev *mdev, struct mlx5e_params *params); int mlx5e_mpwrq_validate_xsk(struct mlx5_core_dev *mdev, struct mlx5e_params *params, - struct mlx5e_xsk_param *xsk); + struct mlx5e_rq_opt_param *rqo); void mlx5e_build_rq_params(struct mlx5_core_dev *mdev, struct mlx5e_params *params); void mlx5e_set_rq_type(struct mlx5_core_dev *mdev, struct mlx5e_params *params); void mlx5e_init_rq_type_params(struct mlx5_core_dev *mdev, struct mlx5e_params *params); u16 mlx5e_get_linear_rq_headroom(struct mlx5e_params *params, - struct mlx5e_xsk_param *xsk); + struct mlx5e_rq_opt_param *rqo); bool mlx5e_rx_is_linear_skb(struct mlx5_core_dev *mdev, struct mlx5e_params *params, - struct mlx5e_xsk_param *xsk); + struct mlx5e_rq_opt_param *rqo); bool mlx5e_rx_mpwqe_is_linear_skb(struct mlx5_core_dev *mdev, struct mlx5e_params *params, - struct mlx5e_xsk_param *xsk); + struct mlx5e_rq_opt_param *rqo); u8 mlx5e_mpwqe_get_log_rq_size(struct mlx5_core_dev *mdev, struct mlx5e_params *params, - struct mlx5e_xsk_param *xsk); + struct mlx5e_rq_opt_param *rqo); u32 mlx5e_shampo_hd_per_wqe(struct mlx5_core_dev *mdev, struct mlx5e_params *params, struct mlx5e_rq_param *rq_param); @@ -105,21 +113,22 @@ u32 mlx5e_shampo_hd_per_wq(struct mlx5_core_dev *mdev, u32 mlx5e_choose_lro_timeout(struct mlx5_core_dev *mdev, u32 wanted_timeout); u8 mlx5e_mpwqe_get_log_stride_size(struct mlx5_core_dev *mdev, struct mlx5e_params *params, - struct mlx5e_xsk_param *xsk); + struct mlx5e_rq_opt_param *rqo); u8 mlx5e_mpwqe_get_log_num_strides(struct mlx5_core_dev *mdev, struct mlx5e_params *params, - struct mlx5e_xsk_param *xsk); + struct mlx5e_rq_opt_param *rqo); u8 mlx5e_mpwqe_get_min_wqe_bulk(unsigned int wq_sz); u16 mlx5e_get_rq_headroom(struct mlx5_core_dev *mdev, struct mlx5e_params *params, - struct mlx5e_xsk_param *xsk); + struct mlx5e_rq_opt_param *rqo); +u32 mlx5e_mpwrq_max_page_size(struct mlx5_core_dev *mdev); /* Build queue parameters */ void mlx5e_build_create_cq_param(struct mlx5e_create_cq_param *ccp, struct mlx5e_channel *c); int mlx5e_build_rq_param(struct mlx5_core_dev *mdev, struct mlx5e_params *params, - struct mlx5e_xsk_param *xsk, + struct mlx5e_rq_opt_param *rqo, struct mlx5e_rq_param *param); void mlx5e_build_drop_rq_param(struct mlx5_core_dev *mdev, struct mlx5e_rq_param *param); @@ -136,13 +145,19 @@ void mlx5e_build_xdpsq_param(struct mlx5_core_dev *mdev, struct mlx5e_sq_param *param); int mlx5e_build_channel_param(struct mlx5_core_dev *mdev, struct mlx5e_params *params, + struct netdev_queue_config *qcfg, struct mlx5e_channel_param *cparam); +void mlx5e_build_xsk_channel_param(struct mlx5_core_dev *mdev, + struct mlx5e_params *params, + struct mlx5e_xsk_param *xsk, + struct mlx5e_channel_param *cparam); + u16 mlx5e_calc_sq_stop_room(struct mlx5_core_dev *mdev, struct mlx5e_params *params); int mlx5e_validate_params(struct mlx5_core_dev *mdev, struct mlx5e_params *params); bool mlx5e_verify_params_rx_mpwqe_strides(struct mlx5_core_dev *mdev, struct mlx5e_params *params, - struct mlx5e_xsk_param *xsk); + struct mlx5e_rq_opt_param *rqo); static inline void mlx5e_params_print_info(struct mlx5_core_dev *mdev, struct mlx5e_params *params) @@ -158,4 +173,10 @@ static inline void mlx5e_params_print_info(struct mlx5_core_dev *mdev, "enhanced" : "basic"); }; +static inline struct mlx5e_xsk_param * +mlx5e_rqo_xsk_param(struct mlx5e_rq_opt_param *rqo) +{ + return rqo ? rqo->xsk : NULL; +} + #endif /* __MLX5_EN_PARAMS_H__ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c b/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c index 1b76647f3194..723f66a6bd63 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c @@ -660,13 +660,13 @@ static void mlx5e_ptp_build_rq_param(struct mlx5_core_dev *mdev, struct net_device *netdev, struct mlx5e_ptp_params *ptp_params) { - struct mlx5e_rq_param *rq_params = &ptp_params->rq_param; + struct mlx5e_rq_param *rq_param = &ptp_params->rq_param; struct mlx5e_params *params = &ptp_params->params; params->rq_wq_type = MLX5_WQ_TYPE_CYCLIC; mlx5e_init_rq_type_params(mdev, params); params->sw_mtu = netdev->max_mtu; - mlx5e_build_rq_param(mdev, params, NULL, rq_params); + mlx5e_build_rq_param(mdev, params, NULL, rq_param); } static void mlx5e_ptp_build_params(struct mlx5e_ptp *c, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/rep/neigh.c b/drivers/net/ethernet/mellanox/mlx5/core/en/rep/neigh.c index d220b045b331..648f4521c096 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/rep/neigh.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/rep/neigh.c @@ -10,6 +10,7 @@ #include <linux/notifier.h> #include <net/netevent.h> #include <net/arp.h> +#include <net/ndisc.h> #include "neigh.h" #include "tc.h" #include "en_rep.h" @@ -18,8 +19,8 @@ static unsigned long mlx5e_rep_ipv6_interval(void) { - if (IS_ENABLED(CONFIG_IPV6) && ipv6_stub->nd_tbl) - return NEIGH_VAR(&ipv6_stub->nd_tbl->parms, DELAY_PROBE_TIME); + if (IS_ENABLED(CONFIG_IPV6) && ipv6_mod_enabled()) + return NEIGH_VAR(&nd_tbl.parms, DELAY_PROBE_TIME); return ~0UL; } @@ -217,7 +218,7 @@ static int mlx5e_rep_netevent_event(struct notifier_block *nb, case NETEVENT_NEIGH_UPDATE: n = ptr; #if IS_ENABLED(CONFIG_IPV6) - if (n->tbl != ipv6_stub->nd_tbl && n->tbl != &arp_tbl) + if (n->tbl != &nd_tbl && n->tbl != &arp_tbl) #else if (n->tbl != &arp_tbl) #endif @@ -238,7 +239,7 @@ static int mlx5e_rep_netevent_event(struct notifier_block *nb, * done per device delay prob time parameter. */ #if IS_ENABLED(CONFIG_IPV6) - if (!p->dev || (p->tbl != ipv6_stub->nd_tbl && p->tbl != &arp_tbl)) + if (!p->dev || (p->tbl != &nd_tbl && p->tbl != &arp_tbl)) #else if (!p->dev || p->tbl != &arp_tbl) #endif diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_tun.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_tun.c index a14f216048cd..de74dbfe7b20 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_tun.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_tun.c @@ -453,8 +453,7 @@ static int mlx5e_route_lookup_ipv6_get(struct mlx5e_priv *priv, if (tunnel && tunnel->get_remote_ifindex) attr->fl.fl6.flowi6_oif = tunnel->get_remote_ifindex(dev); - dst = ipv6_stub->ipv6_dst_lookup_flow(dev_net(dev), NULL, &attr->fl.fl6, - NULL); + dst = ip6_dst_lookup_flow(dev_net(dev), NULL, &attr->fl.fl6, NULL); if (IS_ERR(dst)) return PTR_ERR(dst); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_tun_encap.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_tun_encap.c index bfd401bee9e8..8b827201935e 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_tun_encap.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_tun_encap.c @@ -402,7 +402,7 @@ void mlx5e_tc_update_neigh_used_value(struct mlx5e_neigh_hash_entry *nhe) tbl = &arp_tbl; #if IS_ENABLED(CONFIG_IPV6) else if (m_neigh->family == AF_INET6) - tbl = ipv6_stub->nd_tbl; + tbl = &nd_tbl; #endif else return; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c b/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c index 80f9fc10877a..d3bab198c99c 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c @@ -37,9 +37,10 @@ #include <linux/bitfield.h> #include <net/page_pool/helpers.h> -int mlx5e_xdp_max_mtu(struct mlx5e_params *params, struct mlx5e_xsk_param *xsk) +int mlx5e_xdp_max_mtu(struct mlx5e_params *params, + struct mlx5e_rq_opt_param *rqo) { - int hr = mlx5e_get_linear_rq_headroom(params, xsk); + int hr = mlx5e_get_linear_rq_headroom(params, rqo); /* Let S := SKB_DATA_ALIGN(sizeof(struct skb_shared_info)). * The condition checked in mlx5e_rx_is_linear_skb is: @@ -122,7 +123,7 @@ mlx5e_xmit_xdp_buff(struct mlx5e_xdpsq *sq, struct mlx5e_rq *rq, * mode. */ - dma_addr = page_pool_get_dma_addr(page) + (xdpf->data - (void *)xdpf); + dma_addr = page_pool_get_dma_addr(page) + offset_in_page(xdpf->data); dma_sync_single_for_device(sq->pdev, dma_addr, xdptxd->len, DMA_BIDIRECTIONAL); if (xdptxd->has_frags) { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.h b/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.h index 46ab0a9e8cdd..3c54f8962664 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.h @@ -96,7 +96,8 @@ union mlx5e_xdp_info { }; struct mlx5e_xsk_param; -int mlx5e_xdp_max_mtu(struct mlx5e_params *params, struct mlx5e_xsk_param *xsk); +int mlx5e_xdp_max_mtu(struct mlx5e_params *params, + struct mlx5e_rq_opt_param *rqo); bool mlx5e_xdp_handle(struct mlx5e_rq *rq, struct bpf_prog *prog, struct mlx5e_xdp_buff *mlctx); void mlx5e_xdp_mpwqe_complete(struct mlx5e_xdpsq *sq); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/pool.c b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/pool.c index c5ba22eed8f1..96a259914cbd 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/pool.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/pool.c @@ -78,6 +78,8 @@ static int mlx5e_xsk_enable_locked(struct mlx5e_priv *priv, struct xsk_buff_pool *pool, u16 ix) { struct mlx5e_params *params = &priv->channels.params; + struct mlx5e_channel_param *cparam; + enum mlx5e_mpwrq_umr_mode umr_mode; struct mlx5e_xsk_param xsk; struct mlx5e_channel *c; int err; @@ -88,18 +90,24 @@ static int mlx5e_xsk_enable_locked(struct mlx5e_priv *priv, if (unlikely(!mlx5e_xsk_is_pool_sane(pool))) return -EINVAL; + cparam = kvzalloc_obj(*cparam); + if (!cparam) + return -ENOMEM; + err = mlx5e_xsk_map_pool(mlx5_sd_ch_ix_get_dev(priv->mdev, ix), pool); if (unlikely(err)) - return err; + goto err_free_cparam; err = mlx5e_xsk_add_pool(&priv->xsk, pool, ix); if (unlikely(err)) goto err_unmap_pool; mlx5e_build_xsk_param(pool, &xsk); + mlx5e_build_xsk_channel_param(priv->mdev, params, &xsk, cparam); + umr_mode = mlx5e_mpwrq_umr_mode(priv->mdev, &cparam->rq_opt); if (priv->channels.params.rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ && - mlx5e_mpwrq_umr_mode(priv->mdev, &xsk) == MLX5E_MPWRQ_UMR_MODE_OVERSIZED) { + umr_mode == MLX5E_MPWRQ_UMR_MODE_OVERSIZED) { const char *recommendation = is_power_of_2(xsk.chunk_size) ? "Upgrade firmware" : "Disable striding RQ"; @@ -121,7 +129,7 @@ static int mlx5e_xsk_enable_locked(struct mlx5e_priv *priv, c = priv->channels.c[ix]; - err = mlx5e_open_xsk(priv, params, &xsk, pool, c); + err = mlx5e_open_xsk(priv, params, cparam, pool, c); if (unlikely(err)) goto err_remove_pool; @@ -137,6 +145,8 @@ static int mlx5e_xsk_enable_locked(struct mlx5e_priv *priv, mlx5e_deactivate_rq(&c->rq); mlx5e_flush_rq(&c->rq, MLX5_RQC_STATE_RDY); + kvfree(cparam); + return 0; err_remove_pool: @@ -145,17 +155,22 @@ err_remove_pool: err_unmap_pool: mlx5e_xsk_unmap_pool(priv, pool); +err_free_cparam: + kvfree(cparam); + return err; validate_closed: /* Check the configuration in advance, rather than fail at a later stage * (in mlx5e_xdp_set or on open) and end up with no channels. */ - if (!mlx5e_validate_xsk_param(params, &xsk, priv->mdev)) { + if (!mlx5e_validate_xsk_param(params, &cparam->rq_opt, priv->mdev)) { err = -EINVAL; goto err_remove_pool; } + kvfree(cparam); + return 0; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.c b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.c index d39f778ecd85..11500fd213a5 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.c @@ -9,9 +9,9 @@ static int mlx5e_legacy_rq_validate_xsk(struct mlx5_core_dev *mdev, struct mlx5e_params *params, - struct mlx5e_xsk_param *xsk) + struct mlx5e_rq_opt_param *rqo) { - if (!mlx5e_rx_is_linear_skb(mdev, params, xsk)) { + if (!mlx5e_rx_is_linear_skb(mdev, params, rqo)) { mlx5_core_err(mdev, "Legacy RQ linear mode for XSK can't be activated with current params\n"); return -EINVAL; } @@ -25,9 +25,14 @@ static int mlx5e_legacy_rq_validate_xsk(struct mlx5_core_dev *mdev, #define MLX5E_MIN_XSK_CHUNK_SIZE max(2048, XDP_UMEM_MIN_CHUNK_SIZE) bool mlx5e_validate_xsk_param(struct mlx5e_params *params, - struct mlx5e_xsk_param *xsk, + struct mlx5e_rq_opt_param *rqo, struct mlx5_core_dev *mdev) { + struct mlx5e_xsk_param *xsk = mlx5e_rqo_xsk_param(rqo); + + if (WARN_ON(!xsk)) + return false; + /* AF_XDP doesn't support frames larger than PAGE_SIZE, * and xsk->chunk_size is limited to 65535 bytes. */ @@ -42,21 +47,12 @@ bool mlx5e_validate_xsk_param(struct mlx5e_params *params, */ switch (params->rq_wq_type) { case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ: - return !mlx5e_mpwrq_validate_xsk(mdev, params, xsk); + return !mlx5e_mpwrq_validate_xsk(mdev, params, rqo); default: /* MLX5_WQ_TYPE_CYCLIC */ - return !mlx5e_legacy_rq_validate_xsk(mdev, params, xsk); + return !mlx5e_legacy_rq_validate_xsk(mdev, params, rqo); } } -static void mlx5e_build_xsk_cparam(struct mlx5_core_dev *mdev, - struct mlx5e_params *params, - struct mlx5e_xsk_param *xsk, - struct mlx5e_channel_param *cparam) -{ - mlx5e_build_rq_param(mdev, params, xsk, &cparam->rq); - mlx5e_build_xdpsq_param(mdev, params, &cparam->xdp_sq); -} - static int mlx5e_init_xsk_rq(struct mlx5e_channel *c, struct mlx5e_params *params, struct xsk_buff_pool *pool, @@ -90,19 +86,23 @@ static int mlx5e_init_xsk_rq(struct mlx5e_channel *c, return xdp_rxq_info_reg(&rq->xdp_rxq, rq->netdev, rq_xdp_ix, c->napi.napi_id); } -static int mlx5e_open_xsk_rq(struct mlx5e_channel *c, struct mlx5e_params *params, - struct mlx5e_rq_param *rq_params, struct xsk_buff_pool *pool, - struct mlx5e_xsk_param *xsk) +static int mlx5e_open_xsk_rq(struct mlx5e_channel *c, + struct mlx5e_params *params, + struct mlx5e_channel_param *cparam, + struct xsk_buff_pool *pool) { + struct mlx5e_rq_param *rq_param = &cparam->rq; + struct mlx5e_rq_opt_param *rqo = &cparam->rq_opt; u16 q_counter = c->priv->q_counter[c->sd_ix]; struct mlx5e_rq *xskrq = &c->xskrq; int err; - err = mlx5e_init_xsk_rq(c, params, pool, xsk, xskrq); + err = mlx5e_init_xsk_rq(c, params, pool, rqo->xsk, xskrq); if (err) return err; - err = mlx5e_open_rq(params, rq_params, xsk, cpu_to_node(c->cpu), q_counter, xskrq); + err = mlx5e_open_rq(params, rq_param, rqo, cpu_to_node(c->cpu), + q_counter, xskrq); if (err) return err; @@ -111,30 +111,24 @@ static int mlx5e_open_xsk_rq(struct mlx5e_channel *c, struct mlx5e_params *param } int mlx5e_open_xsk(struct mlx5e_priv *priv, struct mlx5e_params *params, - struct mlx5e_xsk_param *xsk, struct xsk_buff_pool *pool, + struct mlx5e_channel_param *cparam, + struct xsk_buff_pool *pool, struct mlx5e_channel *c) { - struct mlx5e_channel_param *cparam; struct mlx5e_create_cq_param ccp; int err; mlx5e_build_create_cq_param(&ccp, c); - if (!mlx5e_validate_xsk_param(params, xsk, priv->mdev)) + if (!mlx5e_validate_xsk_param(params, &cparam->rq_opt, priv->mdev)) return -EINVAL; - cparam = kvzalloc_obj(*cparam); - if (!cparam) - return -ENOMEM; - - mlx5e_build_xsk_cparam(priv->mdev, params, xsk, cparam); - err = mlx5e_open_cq(c->mdev, params->rx_cq_moderation, &cparam->rq.cqp, &ccp, &c->xskrq.cq); if (unlikely(err)) - goto err_free_cparam; + return err; - err = mlx5e_open_xsk_rq(c, params, &cparam->rq, pool, xsk); + err = mlx5e_open_xsk_rq(c, params, cparam, pool); if (unlikely(err)) goto err_close_rx_cq; @@ -153,8 +147,6 @@ int mlx5e_open_xsk(struct mlx5e_priv *priv, struct mlx5e_params *params, if (unlikely(err)) goto err_close_tx_cq; - kvfree(cparam); - set_bit(MLX5E_CHANNEL_STATE_XSK, c->state); return 0; @@ -168,9 +160,6 @@ err_close_rq: err_close_rx_cq: mlx5e_close_cq(&c->xskrq.cq); -err_free_cparam: - kvfree(cparam); - return err; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.h b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.h index 50e111b85efd..664ec78192c3 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xsk/setup.h @@ -9,10 +9,12 @@ struct mlx5e_xsk_param; bool mlx5e_validate_xsk_param(struct mlx5e_params *params, - struct mlx5e_xsk_param *xsk, + struct mlx5e_rq_opt_param *rqo, struct mlx5_core_dev *mdev); +struct mlx5e_channel_param; int mlx5e_open_xsk(struct mlx5e_priv *priv, struct mlx5e_params *params, - struct mlx5e_xsk_param *xsk, struct xsk_buff_pool *pool, + struct mlx5e_channel_param *cparam, + struct xsk_buff_pool *pool, struct mlx5e_channel *c); void mlx5e_close_xsk(struct mlx5e_channel *c); void mlx5e_activate_xsk(struct mlx5e_channel *c); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c index 64e13747084e..a52e12c3c95a 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec.c @@ -36,7 +36,6 @@ #include <linux/inetdevice.h> #include <linux/netdevice.h> #include <net/netevent.h> -#include <net/ipv6_stubs.h> #include "en.h" #include "eswitch.h" diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_offload.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_offload.c index 05faad5083d9..145677ce9640 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_offload.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_offload.c @@ -1,6 +1,8 @@ // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB /* Copyright (c) 2017, Mellanox Technologies inc. All rights reserved. */ +#include <linux/iopoll.h> + #include "mlx5_core.h" #include "en.h" #include "ipsec.h" @@ -592,7 +594,6 @@ int mlx5e_ipsec_aso_query(struct mlx5e_ipsec_sa_entry *sa_entry, struct mlx5_wqe_aso_ctrl_seg *ctrl; struct mlx5e_hw_objs *res; struct mlx5_aso_wqe *wqe; - unsigned long expires; u8 ds_cnt; int ret; @@ -614,13 +615,8 @@ int mlx5e_ipsec_aso_query(struct mlx5e_ipsec_sa_entry *sa_entry, mlx5e_ipsec_aso_copy(ctrl, data); mlx5_aso_post_wqe(aso->aso, false, &wqe->ctrl); - expires = jiffies + msecs_to_jiffies(10); - do { - ret = mlx5_aso_poll_cq(aso->aso, false); - if (ret) - /* We are in atomic context */ - udelay(10); - } while (ret && time_is_after_jiffies(expires)); + read_poll_timeout_atomic(mlx5_aso_poll_cq, ret, !ret, 10, + 10 * USEC_PER_MSEC, false, aso->aso, false); if (!ret) memcpy(sa_entry->ctx, aso->ctx, MLX5_ST_SZ_BYTES(ipsec_aso)); spin_unlock_bh(&aso->lock); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c b/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c index 4a8dc85d5924..bb61e2179078 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c @@ -371,6 +371,9 @@ void mlx5e_ethtool_get_ringparam(struct mlx5e_priv *priv, param->tx_max_pending = 1 << MLX5E_PARAMS_MAXIMUM_LOG_SQ_SIZE; param->rx_pending = 1 << priv->channels.params.log_rq_mtu_frames; param->tx_pending = 1 << priv->channels.params.log_sq_size; + + kernel_param->hds_thresh = 0; + kernel_param->hds_thresh_max = 0; } static void mlx5e_get_ringparam(struct net_device *dev, @@ -2735,7 +2738,8 @@ const struct ethtool_ops mlx5e_ethtool_ops = { ETHTOOL_COALESCE_USE_ADAPTIVE | ETHTOOL_COALESCE_USE_CQE, .supported_input_xfrm = RXH_XFRM_SYM_OR_XOR, - .supported_ring_params = ETHTOOL_RING_USE_TCP_DATA_SPLIT, + .supported_ring_params = ETHTOOL_RING_USE_TCP_DATA_SPLIT | + ETHTOOL_RING_USE_HDS_THRS, .get_drvinfo = mlx5e_get_drvinfo, .get_link = ethtool_op_get_link, .get_link_ext_state = mlx5e_get_link_ext_state, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_fs.c b/drivers/net/ethernet/mellanox/mlx5/core/en_fs.c index 9352e2183312..fdfe9d1cfe21 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_fs.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_fs.c @@ -47,7 +47,6 @@ struct mlx5e_flow_steering { bool state_destroy; bool vlan_strip_disable; struct mlx5_core_dev *mdev; - struct net_device *netdev; struct mlx5_flow_namespace *ns; struct mlx5_flow_namespace *egress_ns; #ifdef CONFIG_MLX5_EN_RXNFC @@ -823,8 +822,14 @@ static void mlx5e_destroy_promisc_table(struct mlx5e_flow_steering *fs) void mlx5e_fs_set_rx_mode_work(struct mlx5e_flow_steering *fs, struct net_device *netdev) { + struct mlx5e_priv *priv = netdev_priv(netdev); struct mlx5e_l2_table *ea = &fs->l2; + if (mlx5e_is_uplink_rep(priv)) { + mlx5e_handle_netdev_addr(fs, netdev); + goto update_vport_context; + } + bool rx_mode_enable = fs->state_destroy; bool promisc_enabled = rx_mode_enable && (netdev->flags & IFF_PROMISC); bool allmulti_enabled = rx_mode_enable && (netdev->flags & IFF_ALLMULTI); @@ -864,6 +869,7 @@ void mlx5e_fs_set_rx_mode_work(struct mlx5e_flow_steering *fs, ea->allmulti_enabled = allmulti_enabled; ea->broadcast_enabled = broadcast_enabled; +update_vport_context: mlx5e_vport_context_update(fs, netdev); } @@ -984,6 +990,9 @@ static int mlx5e_add_l2_flow_rule(struct mlx5e_flow_steering *fs, u8 *mc_dmac; u8 *mv_dmac; + if (!ft) + return -EINVAL; + spec = kvzalloc_obj(*spec); if (!spec) return -ENOMEM; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c index b6c12460b54a..6c4eeb88588c 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c @@ -369,6 +369,41 @@ static int mlx5e_rq_alloc_mpwqe_info(struct mlx5e_rq *rq, int node) return 0; } +static int mlx5e_rq_alloc_mpwqe_linear_info(struct mlx5e_rq *rq, int node, + struct mlx5e_params *params, + struct mlx5e_rq_opt_param *rqo) +{ + struct mlx5_core_dev *mdev = rq->mdev; + struct mlx5e_mpw_linear_info *li; + u32 linear_frag_count; + + if (mlx5e_rx_mpwqe_is_linear_skb(mdev, params, rqo) || + !params->xdp_prog) + return 0; + + li = kvzalloc_node(sizeof(*li), GFP_KERNEL, node); + if (!li) + return -ENOMEM; + + linear_frag_count = + BIT(rq->mpwqe.page_shift - MLX5E_XDP_LOG_MAX_LINEAR_SZ); + if (linear_frag_count > U16_MAX) { + netdev_warn(rq->netdev, + "rq %d: linear_frag_count (%u) larger than expected (%u), page_shift: %u, log_max_linear_sz: %u\n", + rq->ix, linear_frag_count, U16_MAX, + rq->mpwqe.page_shift, MLX5E_XDP_LOG_MAX_LINEAR_SZ); + kvfree(li); + return -EINVAL; + } + + li->max_frags = linear_frag_count; + rq->mpwqe.linear_info = li; + + /* Set to max to force allocation on first run. */ + li->frag_page.frags = li->max_frags; + + return 0; +} static u8 mlx5e_mpwrq_access_mode(enum mlx5e_mpwrq_umr_mode umr_mode) { @@ -636,14 +671,18 @@ static void mlx5e_rq_timeout_work(struct work_struct *timeout_work) static int mlx5e_alloc_mpwqe_rq_drop_page(struct mlx5e_rq *rq) { - rq->wqe_overflow.page = alloc_page(GFP_KERNEL); + /* xsk can have page_shift < PAGE_SHIFT */ + u16 page_order = max_t(s16, rq->mpwqe.page_shift - PAGE_SHIFT, 0); + u32 page_size = BIT(PAGE_SHIFT + page_order); + + rq->wqe_overflow.page = alloc_pages(GFP_KERNEL, page_order); if (!rq->wqe_overflow.page) return -ENOMEM; rq->wqe_overflow.addr = dma_map_page(rq->pdev, rq->wqe_overflow.page, 0, - PAGE_SIZE, rq->buff.map_dir); + page_size, rq->buff.map_dir); if (dma_mapping_error(rq->pdev, rq->wqe_overflow.addr)) { - __free_page(rq->wqe_overflow.page); + __free_pages(rq->wqe_overflow.page, page_order); return -ENOMEM; } return 0; @@ -651,9 +690,12 @@ static int mlx5e_alloc_mpwqe_rq_drop_page(struct mlx5e_rq *rq) static void mlx5e_free_mpwqe_rq_drop_page(struct mlx5e_rq *rq) { - dma_unmap_page(rq->pdev, rq->wqe_overflow.addr, PAGE_SIZE, - rq->buff.map_dir); - __free_page(rq->wqe_overflow.page); + u16 page_order = max_t(s16, rq->mpwqe.page_shift - PAGE_SHIFT, 0); + u32 page_size = BIT(PAGE_SHIFT + page_order); + + dma_unmap_page(rq->pdev, rq->wqe_overflow.addr, page_size, + rq->buff.map_dir); + __free_pages(rq->wqe_overflow.page, page_order); } static int mlx5e_init_rxq_rq(struct mlx5e_channel *c, struct mlx5e_params *params, @@ -780,7 +822,7 @@ static int mlx5e_create_rq_hd_mkey(struct mlx5_core_dev *mdev, static int mlx5_rq_shampo_alloc(struct mlx5_core_dev *mdev, struct mlx5e_params *params, - struct mlx5e_rq_param *rqp, + struct mlx5e_rq_param *rq_param, struct mlx5e_rq *rq, int node) { @@ -791,7 +833,7 @@ static int mlx5_rq_shampo_alloc(struct mlx5_core_dev *mdev, if (!test_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state)) return 0; - hd_per_wq = mlx5e_shampo_hd_per_wq(mdev, params, rqp); + hd_per_wq = mlx5e_shampo_hd_per_wq(mdev, params, rq_param); hd_buf_size = hd_per_wq * BIT(MLX5E_SHAMPO_LOG_HEADER_ENTRY_SIZE); nentries = hd_buf_size / PAGE_SIZE; if (!nentries) { @@ -851,19 +893,19 @@ static void mlx5e_rq_free_shampo(struct mlx5e_rq *rq) } static int mlx5e_alloc_rq(struct mlx5e_params *params, - struct mlx5e_xsk_param *xsk, - struct mlx5e_rq_param *rqp, + struct mlx5e_rq_param *rq_param, + struct mlx5e_rq_opt_param *rqo, int node, struct mlx5e_rq *rq) { + void *rqc_wq = MLX5_ADDR_OF(rqc, rq_param->rqc, wq); struct mlx5_core_dev *mdev = rq->mdev; - void *rqc = rqp->rqc; - void *rqc_wq = MLX5_ADDR_OF(rqc, rqc, wq); + u32 pool_order = 0; u32 pool_size; int wq_sz; int err; int i; - rqp->wq.db_numa_node = node; + rq_param->wq.db_numa_node = node; INIT_WORK(&rq->recover_work, mlx5e_rq_err_cqe_work); INIT_WORK(&rq->rx_timeout_work, mlx5e_rq_timeout_work); @@ -872,28 +914,28 @@ static int mlx5e_alloc_rq(struct mlx5e_params *params, RCU_INIT_POINTER(rq->xdp_prog, params->xdp_prog); rq->buff.map_dir = params->xdp_prog ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE; - rq->buff.headroom = mlx5e_get_rq_headroom(mdev, params, xsk); + rq->buff.headroom = mlx5e_get_rq_headroom(mdev, params, rqo); pool_size = 1 << params->log_rq_mtu_frames; rq->mkey_be = cpu_to_be32(mdev->mlx5e_res.hw_objs.mkey); switch (rq->wq_type) { case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ: - err = mlx5_wq_ll_create(mdev, &rqp->wq, rqc_wq, &rq->mpwqe.wq, - &rq->wq_ctrl); + err = mlx5_wq_ll_create(mdev, &rq_param->wq, rqc_wq, + &rq->mpwqe.wq, &rq->wq_ctrl); if (err) goto err_rq_xdp_prog; - err = mlx5e_alloc_mpwqe_rq_drop_page(rq); - if (err) - goto err_rq_wq_destroy; - rq->mpwqe.wq.db = &rq->mpwqe.wq.db[MLX5_RCV_DBR]; wq_sz = mlx5_wq_ll_get_size(&rq->mpwqe.wq); - rq->mpwqe.page_shift = mlx5e_mpwrq_page_shift(mdev, xsk); - rq->mpwqe.umr_mode = mlx5e_mpwrq_umr_mode(mdev, xsk); + rq->mpwqe.page_shift = mlx5e_mpwrq_page_shift(mdev, rqo); + err = mlx5e_alloc_mpwqe_rq_drop_page(rq); + if (err) + goto err_rq_wq_destroy; + + rq->mpwqe.umr_mode = mlx5e_mpwrq_umr_mode(mdev, rqo); rq->mpwqe.pages_per_wqe = mlx5e_mpwrq_pages_per_wqe(mdev, rq->mpwqe.page_shift, rq->mpwqe.umr_mode); @@ -905,14 +947,14 @@ static int mlx5e_alloc_rq(struct mlx5e_params *params, rq->mpwqe.umr_mode); pool_size = rq->mpwqe.pages_per_wqe << - mlx5e_mpwqe_get_log_rq_size(mdev, params, xsk); - - if (!mlx5e_rx_mpwqe_is_linear_skb(mdev, params, xsk) && params->xdp_prog) - pool_size *= 2; /* additional page per packet for the linear part */ + mlx5e_mpwqe_get_log_rq_size(mdev, params, rqo); + pool_order = rq->mpwqe.page_shift - PAGE_SHIFT; - rq->mpwqe.log_stride_sz = mlx5e_mpwqe_get_log_stride_size(mdev, params, xsk); + rq->mpwqe.log_stride_sz = + mlx5e_mpwqe_get_log_stride_size(mdev, params, + rqo); rq->mpwqe.num_strides = - BIT(mlx5e_mpwqe_get_log_num_strides(mdev, params, xsk)); + BIT(mlx5e_mpwqe_get_log_num_strides(mdev, params, rqo)); rq->mpwqe.min_wqe_bulk = mlx5e_mpwqe_get_min_wqe_bulk(wq_sz); rq->buff.frame0_sz = (1 << rq->mpwqe.log_stride_sz); @@ -925,14 +967,18 @@ static int mlx5e_alloc_rq(struct mlx5e_params *params, if (err) goto err_rq_mkey; - err = mlx5_rq_shampo_alloc(mdev, params, rqp, rq, node); + err = mlx5e_rq_alloc_mpwqe_linear_info(rq, node, params, rqo); if (err) goto err_free_mpwqe_info; + err = mlx5_rq_shampo_alloc(mdev, params, rq_param, rq, node); + if (err) + goto err_free_mpwqe_linear_info; + break; default: /* MLX5_WQ_TYPE_CYCLIC */ - err = mlx5_wq_cyc_create(mdev, &rqp->wq, rqc_wq, &rq->wqe.wq, - &rq->wq_ctrl); + err = mlx5_wq_cyc_create(mdev, &rq_param->wq, rqc_wq, + &rq->wqe.wq, &rq->wq_ctrl); if (err) goto err_rq_xdp_prog; @@ -940,7 +986,7 @@ static int mlx5e_alloc_rq(struct mlx5e_params *params, wq_sz = mlx5_wq_cyc_get_size(&rq->wqe.wq); - rq->wqe.info = rqp->frags_info; + rq->wqe.info = rq_param->frags_info; rq->buff.frame0_sz = rq->wqe.info.arr[0].frag_stride; err = mlx5e_init_wqe_alloc_info(rq, node); @@ -948,7 +994,7 @@ static int mlx5e_alloc_rq(struct mlx5e_params *params, goto err_rq_wq_destroy; } - if (xsk) { + if (mlx5e_rqo_xsk_param(rqo)) { err = xdp_rxq_info_reg_mem_model(&rq->xdp_rxq, MEM_TYPE_XSK_BUFF_POOL, NULL); if (err) @@ -958,7 +1004,13 @@ static int mlx5e_alloc_rq(struct mlx5e_params *params, /* Create a page_pool and register it with rxq */ struct page_pool_params pp_params = { 0 }; - pp_params.order = 0; + if (WARN_ON(BIT(PAGE_SHIFT + pool_order) / 64 > + MLX5E_PAGECNT_BIAS_MAX)) { + err = -E2BIG; + goto err_free_by_rq_type; + } + + pp_params.order = pool_order; pp_params.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV; pp_params.pool_size = pool_size; pp_params.nid = node; @@ -966,7 +1018,7 @@ static int mlx5e_alloc_rq(struct mlx5e_params *params, pp_params.napi = rq->cq.napi; pp_params.netdev = rq->netdev; pp_params.dma_dir = rq->buff.map_dir; - pp_params.max_len = PAGE_SIZE; + pp_params.max_len = BIT(PAGE_SHIFT + pool_order); pp_params.queue_idx = rq->ix; /* Shampo header data split allow for unreadable netmem */ @@ -1037,6 +1089,8 @@ err_free_by_rq_type: switch (rq->wq_type) { case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ: mlx5e_rq_free_shampo(rq); +err_free_mpwqe_linear_info: + kvfree(rq->mpwqe.linear_info); err_free_mpwqe_info: kvfree(rq->mpwqe.info); err_rq_mkey: @@ -1064,6 +1118,7 @@ static void mlx5e_free_rq(struct mlx5e_rq *rq) switch (rq->wq_type) { case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ: mlx5e_rq_free_shampo(rq); + kvfree(rq->mpwqe.linear_info); kvfree(rq->mpwqe.info); mlx5_core_destroy_mkey(rq->mdev, be32_to_cpu(rq->mpwqe.umr_mkey_be)); mlx5e_free_mpwqe_rq_drop_page(rq); @@ -1085,7 +1140,8 @@ static void mlx5e_free_rq(struct mlx5e_rq *rq) xdp_rxq_info_unreg(&rq->xdp_rxq); } -int mlx5e_create_rq(struct mlx5e_rq *rq, struct mlx5e_rq_param *param, u16 q_counter) +int mlx5e_create_rq(struct mlx5e_rq *rq, struct mlx5e_rq_param *rq_param, + u16 q_counter) { struct mlx5_core_dev *mdev = rq->mdev; u8 ts_format; @@ -1107,7 +1163,7 @@ int mlx5e_create_rq(struct mlx5e_rq *rq, struct mlx5e_rq_param *param, u16 q_cou rqc = MLX5_ADDR_OF(create_rq_in, in, ctx); wq = MLX5_ADDR_OF(rqc, rqc, wq); - memcpy(rqc, param->rqc, sizeof(param->rqc)); + memcpy(rqc, rq_param->rqc, sizeof(rq_param->rqc)); MLX5_SET(rqc, rqc, cqn, rq->cq.mcq.cqn); MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RST); @@ -1302,6 +1358,8 @@ void mlx5e_free_rx_descs(struct mlx5e_rq *rq) mlx5_wq_ll_pop(wq, wqe_ix_be, &wqe->next.next_wqe_index); } + + mlx5e_mpwqe_dealloc_linear_page(rq); } else { struct mlx5_wq_cyc *wq = &rq->wqe.wq; u16 missing = mlx5_wq_cyc_missing(wq); @@ -1323,8 +1381,8 @@ void mlx5e_free_rx_descs(struct mlx5e_rq *rq) } -int mlx5e_open_rq(struct mlx5e_params *params, struct mlx5e_rq_param *param, - struct mlx5e_xsk_param *xsk, int node, u16 q_counter, +int mlx5e_open_rq(struct mlx5e_params *params, struct mlx5e_rq_param *rq_param, + struct mlx5e_rq_opt_param *rqo, int node, u16 q_counter, struct mlx5e_rq *rq) { struct mlx5_core_dev *mdev = rq->mdev; @@ -1333,11 +1391,11 @@ int mlx5e_open_rq(struct mlx5e_params *params, struct mlx5e_rq_param *param, if (params->packet_merge.type == MLX5E_PACKET_MERGE_SHAMPO) __set_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state); - err = mlx5e_alloc_rq(params, xsk, param, node, rq); + err = mlx5e_alloc_rq(params, rq_param, rqo, node, rq); if (err) return err; - err = mlx5e_create_rq(rq, param, q_counter); + err = mlx5e_create_rq(rq, rq_param, q_counter); if (err) goto err_free_rq; @@ -2506,17 +2564,20 @@ static int mlx5e_set_tx_maxrate(struct net_device *dev, int index, u32 rate) return err; } -static int mlx5e_open_rxq_rq(struct mlx5e_channel *c, struct mlx5e_params *params, - struct mlx5e_rq_param *rq_params) +static int mlx5e_open_rxq_rq(struct mlx5e_channel *c, + struct mlx5e_params *params, + struct mlx5e_rq_param *rq_param, + struct mlx5e_rq_opt_param *rqo) { u16 q_counter = c->priv->q_counter[c->sd_ix]; int err; - err = mlx5e_init_rxq_rq(c, params, rq_params->xdp_frag_size, &c->rq); + err = mlx5e_init_rxq_rq(c, params, rq_param->xdp_frag_size, &c->rq); if (err) return err; - return mlx5e_open_rq(params, rq_params, NULL, cpu_to_node(c->cpu), q_counter, &c->rq); + return mlx5e_open_rq(params, rq_param, rqo, cpu_to_node(c->cpu), + q_counter, &c->rq); } static struct mlx5e_icosq * @@ -2619,7 +2680,7 @@ static int mlx5e_open_queues(struct mlx5e_channel *c, if (err) goto err_close_icosq; - err = mlx5e_open_rxq_rq(c, params, &cparam->rq); + err = mlx5e_open_rxq_rq(c, params, &cparam->rq, &cparam->rq_opt); if (err) goto err_close_sqs; @@ -2764,6 +2825,7 @@ static void mlx5e_channel_pick_doorbell(struct mlx5e_channel *c) static int mlx5e_open_channel(struct mlx5e_priv *priv, int ix, struct mlx5e_params *params, + struct netdev_queue_config *qcfg, struct xsk_buff_pool *xsk_pool, struct mlx5e_channel **cp) { @@ -2797,7 +2859,7 @@ static int mlx5e_open_channel(struct mlx5e_priv *priv, int ix, goto err_free; } - err = mlx5e_build_channel_param(mdev, params, cparam); + err = mlx5e_build_channel_param(mdev, params, qcfg, cparam); if (err) goto err_free; @@ -2828,7 +2890,8 @@ static int mlx5e_open_channel(struct mlx5e_priv *priv, int ix, if (xsk_pool) { mlx5e_build_xsk_param(xsk_pool, &xsk); - err = mlx5e_open_xsk(priv, params, &xsk, xsk_pool, c); + mlx5e_build_xsk_channel_param(priv->mdev, params, &xsk, cparam); + err = mlx5e_open_xsk(priv, params, cparam, xsk_pool, c); if (unlikely(err)) goto err_close_queues; } @@ -2921,7 +2984,8 @@ int mlx5e_open_channels(struct mlx5e_priv *priv, if (chs->params.xdp_prog) xsk_pool = mlx5e_xsk_get_pool(&chs->params, chs->params.xsk, i); - err = mlx5e_open_channel(priv, i, &chs->params, xsk_pool, &chs->c[i]); + err = mlx5e_open_channel(priv, i, &chs->params, NULL, + xsk_pool, &chs->c[i]); if (err) goto err_close_channels; } @@ -3577,15 +3641,14 @@ static void mlx5e_free_drop_rq(struct mlx5e_rq *rq) static int mlx5e_alloc_drop_rq(struct mlx5_core_dev *mdev, struct mlx5e_rq *rq, - struct mlx5e_rq_param *param) + struct mlx5e_rq_param *rq_param) { - void *rqc = param->rqc; - void *rqc_wq = MLX5_ADDR_OF(rqc, rqc, wq); + void *rqc_wq = MLX5_ADDR_OF(rqc, rq_param->rqc, wq); int err; - param->wq.db_numa_node = param->wq.buf_numa_node; + rq_param->wq.db_numa_node = rq_param->wq.buf_numa_node; - err = mlx5_wq_cyc_create(mdev, ¶m->wq, rqc_wq, &rq->wqe.wq, + err = mlx5_wq_cyc_create(mdev, &rq_param->wq, rqc_wq, &rq->wqe.wq, &rq->wq_ctrl); if (err) return err; @@ -4079,9 +4142,6 @@ mlx5e_get_stats(struct net_device *dev, struct rtnl_link_stats64 *stats) static void mlx5e_nic_set_rx_mode(struct mlx5e_priv *priv) { - if (mlx5e_is_uplink_rep(priv)) - return; /* no rx mode for uplink rep */ - queue_work(priv->wq, &priv->set_rx_mode_work); } @@ -4586,6 +4646,7 @@ static bool mlx5e_xsk_validate_mtu(struct net_device *netdev, for (ix = 0; ix < chs->params.num_channels; ix++) { struct xsk_buff_pool *xsk_pool = mlx5e_xsk_get_pool(&chs->params, chs->params.xsk, ix); + struct mlx5e_rq_opt_param rqo = {0}; struct mlx5e_xsk_param xsk; int max_xdp_mtu; @@ -4593,12 +4654,13 @@ static bool mlx5e_xsk_validate_mtu(struct net_device *netdev, continue; mlx5e_build_xsk_param(xsk_pool, &xsk); - max_xdp_mtu = mlx5e_xdp_max_mtu(new_params, &xsk); + rqo.xsk = &xsk; + max_xdp_mtu = mlx5e_xdp_max_mtu(new_params, &rqo); /* Validate XSK params and XDP MTU in advance */ - if (!mlx5e_validate_xsk_param(new_params, &xsk, mdev) || + if (!mlx5e_validate_xsk_param(new_params, &rqo, mdev) || new_params->sw_mtu > max_xdp_mtu) { - u32 hr = mlx5e_get_linear_rq_headroom(new_params, &xsk); + u32 hr = mlx5e_get_linear_rq_headroom(new_params, &rqo); int max_mtu_frame, max_mtu_page, max_mtu; /* Two criteria must be met: @@ -5549,9 +5611,52 @@ static const struct netdev_stat_ops mlx5e_stat_ops = { struct mlx5_qmgmt_data { struct mlx5e_channel *c; - struct mlx5e_channel_param cparam; }; +static void mlx5e_queue_default_qcfg(struct net_device *dev, + struct netdev_queue_config *qcfg) +{ + qcfg->rx_page_size = PAGE_SIZE; +} + +static int mlx5e_queue_validate_qcfg(struct net_device *dev, + struct netdev_queue_config *qcfg, + struct netlink_ext_ack *extack) +{ + struct mlx5e_priv *priv = netdev_priv(dev); + struct mlx5_core_dev *mdev = priv->mdev; + u32 max; + + if (!is_power_of_2(qcfg->rx_page_size)) { + netdev_err(priv->netdev, "rx_page_size not power of 2: %u", + qcfg->rx_page_size); + return -EINVAL; + } + + max = mlx5e_mpwrq_max_page_size(mdev); + if (qcfg->rx_page_size < PAGE_SIZE || qcfg->rx_page_size > max) { + netdev_err(priv->netdev, + "Selected rx_page_size %u not in supported range [%lu, %u]\n", + qcfg->rx_page_size, PAGE_SIZE, max); + return -ERANGE; + } + + return 0; +} + +static bool mlx5e_queue_validate_page_size(struct net_device *dev, + struct netdev_queue_config *qcfg, + int queue_index) +{ + if (qcfg->rx_page_size == PAGE_SIZE) + return true; + + if (!netif_rxq_has_unreadable_mp(dev, queue_index)) + return false; + + return true; +} + static int mlx5e_queue_mem_alloc(struct net_device *dev, struct netdev_queue_config *qcfg, void *newq, int queue_index) @@ -5560,7 +5665,6 @@ static int mlx5e_queue_mem_alloc(struct net_device *dev, struct mlx5e_priv *priv = netdev_priv(dev); struct mlx5e_channels *chs = &priv->channels; struct mlx5e_params params = chs->params; - struct mlx5_core_dev *mdev; int err; mutex_lock(&priv->state_lock); @@ -5584,12 +5688,14 @@ static int mlx5e_queue_mem_alloc(struct net_device *dev, goto unlock; } - mdev = mlx5_sd_ch_ix_get_dev(priv->mdev, queue_index); - err = mlx5e_build_channel_param(mdev, ¶ms, &new->cparam); - if (err) + if (!mlx5e_queue_validate_page_size(dev, qcfg, queue_index)) { + netdev_err(priv->netdev, "High order pages are supported only in Zero-Copy mode\n"); + err = -EINVAL; goto unlock; + } - err = mlx5e_open_channel(priv, queue_index, ¶ms, NULL, &new->c); + err = mlx5e_open_channel(priv, queue_index, ¶ms, qcfg, NULL, + &new->c); unlock: mutex_unlock(&priv->state_lock); return err; @@ -5668,6 +5774,9 @@ static const struct netdev_queue_mgmt_ops mlx5e_queue_mgmt_ops = { .ndo_queue_start = mlx5e_queue_start, .ndo_queue_stop = mlx5e_queue_stop, .ndo_queue_get_dma_dev = mlx5e_queue_get_dma_dev, + .ndo_default_qcfg = mlx5e_queue_default_qcfg, + .ndo_validate_qcfg = mlx5e_queue_validate_qcfg, + .supported_params = QCFG_RX_PAGE_SIZE, }; static void mlx5e_build_nic_netdev(struct net_device *netdev) @@ -6411,12 +6520,23 @@ int mlx5e_attach_netdev(struct mlx5e_priv *priv) /* max number of channels may have changed */ max_nch = mlx5e_calc_max_nch(priv->mdev, priv->netdev, profile); + + /* Locking is required by ethtool_rxfh_indir_lost() (sends + * ETHTOOL_MSG_RSS_NTF) and by netif_set_real_num_*_queues in case + * the netdev has been registered by this point (if this function + * was called in the reload or resume flow). + */ + if (need_lock) { + rtnl_lock(); + netdev_lock(priv->netdev); + } + if (priv->channels.params.num_channels > max_nch) { mlx5_core_warn(priv->mdev, "MLX5E: Reducing number of channels to %d\n", max_nch); /* Reducing the number of channels - RXFH has to be reset, and * mlx5e_num_channels_changed below will build the RQT. */ - priv->netdev->priv_flags &= ~IFF_RXFH_CONFIGURED; + ethtool_rxfh_indir_lost(priv->netdev); priv->channels.params.num_channels = max_nch; if (priv->channels.params.mqprio.mode == TC_MQPRIO_MODE_CHANNEL) { mlx5_core_warn(priv->mdev, "MLX5E: Disabling MQPRIO channel mode\n"); @@ -6433,15 +6553,7 @@ int mlx5e_attach_netdev(struct mlx5e_priv *priv) /* 1. Set the real number of queues in the kernel the first time. * 2. Set our default XPS cpumask. * 3. Build the RQT. - * - * Locking is required by netif_set_real_num_*_queues in case the - * netdev has been registered by this point (if this function was called - * in the reload or resume flow). */ - if (need_lock) { - rtnl_lock(); - netdev_lock(priv->netdev); - } err = mlx5e_num_channels_changed(priv); if (need_lock) { netdev_unlock(priv->netdev); @@ -6756,6 +6868,14 @@ static int _mlx5e_probe(struct auxiliary_device *adev) goto err_resume; } + /* mlx5e_fix_features() returns early when the device is not present + * to avoid dereferencing cleared priv during profile changes. + * This also causes it to be a no-op during register_netdev(), where + * the device is not yet present. + * Trigger an additional features update that will actually work. + */ + mlx5e_update_features(netdev); + mlx5e_dcbnl_init_app(priv); mlx5_core_uplink_netdev_set(mdev, netdev); mlx5e_params_print_info(mdev, &priv->channels.params); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c index 1db4ecb2356f..ba6c0f38cc73 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c @@ -38,7 +38,6 @@ #include <net/pkt_cls.h> #include <net/act_api.h> #include <net/devlink.h> -#include <net/ipv6_stubs.h> #include "eswitch.h" #include "en.h" @@ -1369,6 +1368,8 @@ static void mlx5e_uplink_rep_disable(struct mlx5e_priv *priv) netdev_unlock(priv->netdev); rtnl_unlock(); + /* clean-up uplink's mpfs mac table */ + queue_work(priv->wq, &priv->set_rx_mode_work); mlx5e_rep_bridge_cleanup(priv); mlx5e_dcbnl_delete_app(priv); mlx5_notifier_unregister(mdev, &priv->events_nb); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c index 268e20884757..5b60aa47c75b 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c @@ -272,8 +272,6 @@ static inline u32 mlx5e_decompress_cqes_start(struct mlx5e_rq *rq, return mlx5e_decompress_cqes_cont(rq, wq, 1, budget_rem); } -#define MLX5E_PAGECNT_BIAS_MAX (PAGE_SIZE / 64) - static int mlx5e_page_alloc_fragmented(struct page_pool *pp, struct mlx5e_frag_page *frag_page) { @@ -302,6 +300,35 @@ static void mlx5e_page_release_fragmented(struct page_pool *pp, page_pool_put_unrefed_netmem(pp, netmem, -1, true); } +static int mlx5e_mpwqe_linear_page_refill(struct mlx5e_rq *rq) +{ + struct mlx5e_mpw_linear_info *li = rq->mpwqe.linear_info; + + if (likely(li->frag_page.frags < li->max_frags)) + return 0; + + if (likely(li->frag_page.netmem)) { + mlx5e_page_release_fragmented(rq->page_pool, &li->frag_page); + li->frag_page.netmem = 0; + } + + return mlx5e_page_alloc_fragmented(rq->page_pool, &li->frag_page); +} + +static void *mlx5e_mpwqe_get_linear_page_frag(struct mlx5e_rq *rq) +{ + struct mlx5e_mpw_linear_info *li = rq->mpwqe.linear_info; + u32 frag_offset; + + if (unlikely(mlx5e_mpwqe_linear_page_refill(rq))) + return NULL; + + frag_offset = li->frag_page.frags << MLX5E_XDP_LOG_MAX_LINEAR_SZ; + WARN_ON(frag_offset >= BIT(rq->mpwqe.page_shift)); + + return netmem_address(li->frag_page.netmem) + frag_offset; +} + static inline int mlx5e_get_rx_frag(struct mlx5e_rq *rq, struct mlx5e_wqe_frag_info *frag) { @@ -704,6 +731,22 @@ static void mlx5e_dealloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix) bitmap_fill(wi->skip_release_bitmap, rq->mpwqe.pages_per_wqe); } +void mlx5e_mpwqe_dealloc_linear_page(struct mlx5e_rq *rq) +{ + struct mlx5e_mpw_linear_info *li = rq->mpwqe.linear_info; + + if (!li || !li->frag_page.netmem) + return; + + mlx5e_page_release_fragmented(rq->page_pool, &li->frag_page); + + /* Recovery flow can call this function and then alloc again, so leave + * things in a good state for re-allocation. + */ + li->frag_page.netmem = 0; + li->frag_page.frags = li->max_frags; +} + INDIRECT_CALLABLE_SCOPE bool mlx5e_post_rx_wqes(struct mlx5e_rq *rq) { struct mlx5_wq_cyc *wq = &rq->wqe.wq; @@ -1844,11 +1887,14 @@ mlx5e_shampo_fill_skb_data(struct sk_buff *skb, struct mlx5e_rq *rq, struct mlx5e_frag_page *frag_page, u32 data_bcnt, u32 data_offset) { + u32 page_size = BIT(rq->mpwqe.page_shift); + net_prefetchw(skb->data); do { /* Non-linear mode, hence non-XSK, which always uses PAGE_SIZE. */ - u32 pg_consumed_bytes = min_t(u32, PAGE_SIZE - data_offset, data_bcnt); + u32 pg_consumed_bytes = min_t(u32, page_size - data_offset, + data_bcnt); unsigned int truesize = pg_consumed_bytes; mlx5e_add_skb_frag(rq, skb, frag_page, data_offset, @@ -1868,7 +1914,9 @@ mlx5e_skb_from_cqe_mpwrq_nonlinear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *w struct mlx5e_frag_page *frag_page = &wi->alloc_units.frag_pages[page_idx]; u16 headlen = min_t(u16, MLX5E_RX_MAX_HEAD, cqe_bcnt); struct mlx5e_frag_page *head_page = frag_page; + struct mlx5e_frag_page *linear_page = NULL; struct mlx5e_xdp_buff *mxbuf = &rq->mxbuf; + u32 page_size = BIT(rq->mpwqe.page_shift); u32 frag_offset = head_offset; u32 byte_cnt = cqe_bcnt; struct skb_shared_info *sinfo; @@ -1895,17 +1943,18 @@ mlx5e_skb_from_cqe_mpwrq_nonlinear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *w if (prog) { /* area for bpf_xdp_[store|load]_bytes */ net_prefetchw(netmem_address(frag_page->netmem) + frag_offset); - if (unlikely(mlx5e_page_alloc_fragmented(rq->page_pool, - &wi->linear_page))) { + + va = mlx5e_mpwqe_get_linear_page_frag(rq); + if (!va) { rq->stats->buff_alloc_err++; return NULL; } - va = netmem_address(wi->linear_page.netmem); net_prefetchw(va); /* xdp_frame data area */ linear_hr = XDP_PACKET_HEADROOM; linear_data_len = 0; linear_frame_sz = MLX5_SKB_FRAG_SZ(linear_hr + MLX5E_RX_MAX_HEAD); + linear_page = &rq->mpwqe.linear_info->frag_page; } else { skb = napi_alloc_skb(rq->cq.napi, ALIGN(MLX5E_RX_MAX_HEAD, sizeof(long))); @@ -1923,9 +1972,9 @@ mlx5e_skb_from_cqe_mpwrq_nonlinear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *w linear_hr = skb_headroom(skb); linear_data_len = headlen; linear_frame_sz = MLX5_SKB_FRAG_SZ(skb_end_offset(skb)); - if (unlikely(frag_offset >= PAGE_SIZE)) { + if (unlikely(frag_offset >= page_size)) { frag_page++; - frag_offset -= PAGE_SIZE; + frag_offset -= page_size; } } @@ -1937,7 +1986,7 @@ mlx5e_skb_from_cqe_mpwrq_nonlinear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *w while (byte_cnt) { /* Non-linear mode, hence non-XSK, which always uses PAGE_SIZE. */ pg_consumed_bytes = - min_t(u32, PAGE_SIZE - frag_offset, byte_cnt); + min_t(u32, page_size - frag_offset, byte_cnt); if (test_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state)) truesize += pg_consumed_bytes; @@ -1964,17 +2013,15 @@ mlx5e_skb_from_cqe_mpwrq_nonlinear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *w for (pfp = head_page; pfp < frag_page; pfp++) pfp->frags++; - wi->linear_page.frags++; + linear_page->frags++; } - mlx5e_page_release_fragmented(rq->page_pool, - &wi->linear_page); return NULL; /* page/packet was consumed by XDP */ } new_nr_frags = sinfo->nr_frags; nr_frags_free = old_nr_frags - new_nr_frags; if (unlikely(nr_frags_free)) - truesize -= (nr_frags_free - 1) * PAGE_SIZE + + truesize -= (nr_frags_free - 1) * page_size + ALIGN(pg_consumed_bytes, BIT(rq->mpwqe.log_stride_sz)); @@ -1984,15 +2031,11 @@ mlx5e_skb_from_cqe_mpwrq_nonlinear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *w rq, mxbuf->xdp.data_hard_start, linear_frame_sz, mxbuf->xdp.data - mxbuf->xdp.data_hard_start, len, mxbuf->xdp.data - mxbuf->xdp.data_meta); - if (unlikely(!skb)) { - mlx5e_page_release_fragmented(rq->page_pool, - &wi->linear_page); + if (unlikely(!skb)) return NULL; - } skb_mark_for_recycle(skb); - wi->linear_page.frags++; - mlx5e_page_release_fragmented(rq->page_pool, &wi->linear_page); + linear_page->frags++; if (xdp_buff_has_frags(&mxbuf->xdp)) { struct mlx5e_frag_page *pagep; @@ -2161,15 +2204,16 @@ mlx5e_shampo_flush_skb(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe, bool match) rq->hw_gro_data->skb = NULL; } -static bool -mlx5e_hw_gro_skb_has_enough_space(struct sk_buff *skb, u16 data_bcnt) +static bool mlx5e_hw_gro_skb_has_enough_space(struct sk_buff *skb, + u16 data_bcnt, + u32 page_size) { int nr_frags = skb_shinfo(skb)->nr_frags; - if (PAGE_SIZE >= GRO_LEGACY_MAX_SIZE) + if (page_size >= GRO_LEGACY_MAX_SIZE) return skb->len + data_bcnt <= GRO_LEGACY_MAX_SIZE; else - return PAGE_SIZE * nr_frags + data_bcnt <= GRO_LEGACY_MAX_SIZE; + return page_size * nr_frags + data_bcnt <= GRO_LEGACY_MAX_SIZE; } static void mlx5e_handle_rx_cqe_mpwrq_shampo(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe) @@ -2178,18 +2222,19 @@ static void mlx5e_handle_rx_cqe_mpwrq_shampo(struct mlx5e_rq *rq, struct mlx5_cq u16 header_index = mlx5e_shampo_get_cqe_header_index(rq, cqe); u32 wqe_offset = be32_to_cpu(cqe->shampo.data_offset); u16 cstrides = mpwrq_get_cqe_consumed_strides(cqe); - u32 data_offset = wqe_offset & (PAGE_SIZE - 1); u32 cqe_bcnt = mpwrq_get_cqe_byte_cnt(cqe); u16 wqe_id = be16_to_cpu(cqe->wqe_id); - u32 page_idx = wqe_offset >> PAGE_SHIFT; u16 head_size = cqe->shampo.header_size; struct sk_buff **skb = &rq->hw_gro_data->skb; bool flush = cqe->shampo.flush; bool match = cqe->shampo.match; + u32 page_size = BIT(rq->mpwqe.page_shift); struct mlx5e_rq_stats *stats = rq->stats; struct mlx5e_rx_wqe_ll *wqe; struct mlx5e_mpw_info *wi; struct mlx5_wq_ll *wq; + u32 data_offset; + u32 page_idx; wi = mlx5e_get_mpw_info(rq, wqe_id); wi->consumed_strides += cstrides; @@ -2205,7 +2250,11 @@ static void mlx5e_handle_rx_cqe_mpwrq_shampo(struct mlx5e_rq *rq, struct mlx5_cq goto mpwrq_cqe_out; } - if (*skb && (!match || !(mlx5e_hw_gro_skb_has_enough_space(*skb, data_bcnt)))) { + data_offset = wqe_offset & (page_size - 1); + page_idx = wqe_offset >> rq->mpwqe.page_shift; + if (*skb && + !(match && mlx5e_hw_gro_skb_has_enough_space(*skb, data_bcnt, + page_size))) { match = false; mlx5e_shampo_flush_skb(rq, cqe, match); } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c index a8af84fc9763..1a3ecf073913 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c @@ -916,11 +916,30 @@ static int mlx5e_stats_get_ieee(struct mlx5_core_dev *mdev, sz, MLX5_REG_PPCNT, 0, 0); } +static int mlx5e_stats_get_per_prio(struct mlx5_core_dev *mdev, + u32 *ppcnt_per_prio, int prio) +{ + u32 in[MLX5_ST_SZ_DW(ppcnt_reg)] = {}; + int sz = MLX5_ST_SZ_BYTES(ppcnt_reg); + + if (!(MLX5_CAP_PCAM_FEATURE(mdev, pfcc_mask) && + MLX5_CAP_DEBUG(mdev, stall_detect))) + return -EOPNOTSUPP; + + MLX5_SET(ppcnt_reg, in, local_port, 1); + MLX5_SET(ppcnt_reg, in, grp, MLX5_PER_PRIORITY_COUNTERS_GROUP); + MLX5_SET(ppcnt_reg, in, prio_tc, prio); + return mlx5_core_access_reg(mdev, in, sz, ppcnt_per_prio, sz, + MLX5_REG_PPCNT, 0, 0); +} + void mlx5e_stats_pause_get(struct mlx5e_priv *priv, struct ethtool_pause_stats *pause_stats) { u32 ppcnt_ieee_802_3[MLX5_ST_SZ_DW(ppcnt_reg)]; struct mlx5_core_dev *mdev = priv->mdev; + u64 ps_stats = 0; + int prio; if (mlx5e_stats_get_ieee(mdev, ppcnt_ieee_802_3)) return; @@ -933,6 +952,17 @@ void mlx5e_stats_pause_get(struct mlx5e_priv *priv, MLX5E_READ_CTR64_BE_F(ppcnt_ieee_802_3, eth_802_3_cntrs_grp_data_layout, a_pause_mac_ctrl_frames_received); + + for (prio = 0; prio < NUM_PPORT_PRIO; prio++) { + if (mlx5e_stats_get_per_prio(mdev, ppcnt_ieee_802_3, prio)) + return; + + ps_stats += MLX5E_READ_CTR64_BE_F(ppcnt_ieee_802_3, + eth_per_prio_grp_data_layout, + device_stall_critical_watermark_cnt); + } + + pause_stats->tx_pause_storm_events = ps_stats; } void mlx5e_stats_eth_phy_get(struct mlx5e_priv *priv, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c index 1434b65d4746..a9001d1c902f 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c @@ -35,12 +35,12 @@ #include <net/sch_generic.h> #include <net/pkt_cls.h> #include <linux/mlx5/fs.h> +#include <linux/mlx5/lag.h> #include <linux/mlx5/device.h> #include <linux/rhashtable.h> #include <linux/refcount.h> #include <linux/completion.h> #include <net/arp.h> -#include <net/ipv6_stubs.h> #include <net/bareudp.h> #include <net/bonding.h> #include <net/dst_metadata.h> @@ -2131,7 +2131,7 @@ static void mlx5e_tc_del_fdb_peer_flow(struct mlx5e_tc_flow *flow, mutex_unlock(&esw->offloads.peer_mutex); list_for_each_entry_safe(peer_flow, tmp, &flow->peer_flows, peer_flows) { - if (peer_index != mlx5_get_dev_index(peer_flow->priv->mdev)) + if (peer_index != mlx5_lag_get_dev_seq(peer_flow->priv->mdev)) continue; list_del(&peer_flow->peer_flows); @@ -2154,7 +2154,7 @@ static void mlx5e_tc_del_fdb_peers_flow(struct mlx5e_tc_flow *flow) devcom = flow->priv->mdev->priv.eswitch->devcom; mlx5_devcom_for_each_peer_entry(devcom, peer_esw, pos) { - i = mlx5_get_dev_index(peer_esw->dev); + i = mlx5_lag_get_dev_seq(peer_esw->dev); mlx5e_tc_del_fdb_peer_flow(flow, i); } } @@ -4584,7 +4584,7 @@ static int mlx5e_tc_add_fdb_peer_flow(struct flow_cls_offload *f, struct mlx5_eswitch *esw = priv->mdev->priv.eswitch; struct mlx5_esw_flow_attr *attr = flow->attr->esw_attr; struct mlx5e_tc_flow_parse_attr *parse_attr; - int i = mlx5_get_dev_index(peer_esw->dev); + int i = mlx5_lag_get_dev_seq(peer_esw->dev); struct mlx5e_rep_priv *peer_urpriv; struct mlx5e_tc_flow *peer_flow; struct mlx5_core_dev *in_mdev; @@ -5525,7 +5525,7 @@ void mlx5e_tc_clean_fdb_peer_flows(struct mlx5_eswitch *esw) devcom = esw->devcom; mlx5_devcom_for_each_peer_entry(devcom, peer_esw, pos) { - i = mlx5_get_dev_index(peer_esw->dev); + i = mlx5_lag_get_dev_seq(peer_esw->dev); list_for_each_entry_safe(flow, tmp, &esw->offloads.peer_flows[i], peer[i]) mlx5e_tc_del_fdb_peers_flow(flow); } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/esw/devlink_port.c b/drivers/net/ethernet/mellanox/mlx5/core/esw/devlink_port.c index 8bffba85f21f..e1d11326af1b 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/esw/devlink_port.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/esw/devlink_port.c @@ -3,6 +3,7 @@ #include <linux/mlx5/driver.h> #include "eswitch.h" +#include "devlink.h" static void mlx5_esw_get_port_parent_id(struct mlx5_core_dev *dev, struct netdev_phys_item_id *ppid) @@ -158,6 +159,32 @@ static const struct devlink_port_ops mlx5_esw_dl_sf_port_ops = { .port_fn_max_io_eqs_set = mlx5_devlink_port_fn_max_io_eqs_set, }; +static int mlx5_esw_devlink_port_res_register(struct mlx5_eswitch *esw, + struct devlink_port *dl_port) +{ + struct devlink_resource_size_params size_params; + struct mlx5_core_dev *dev = esw->dev; + u16 max_sfs, sf_base_id; + int err; + + err = mlx5_esw_sf_max_hpf_functions(dev, &max_sfs, &sf_base_id); + if (err) + return err; + + devlink_resource_size_params_init(&size_params, max_sfs, max_sfs, 1, + DEVLINK_RESOURCE_UNIT_ENTRY); + + return devl_port_resource_register(dl_port, "max_SFs", max_sfs, + MLX5_DL_PORT_RES_MAX_SFS, + DEVLINK_RESOURCE_ID_PARENT_TOP, + &size_params); +} + +static void mlx5_esw_devlink_port_res_unregister(struct devlink_port *dl_port) +{ + devl_port_resources_unregister(dl_port); +} + int mlx5_esw_offloads_devlink_port_register(struct mlx5_eswitch *esw, struct mlx5_vport *vport) { struct mlx5_core_dev *dev = esw->dev; @@ -189,6 +216,15 @@ int mlx5_esw_offloads_devlink_port_register(struct mlx5_eswitch *esw, struct mlx if (err) goto rate_err; + if (vport_num == MLX5_VPORT_PF) { + err = mlx5_esw_devlink_port_res_register(esw, + &dl_port->dl_port); + if (err) + mlx5_core_dbg(dev, + "Failed to register port resources: %d\n", + err); + } + return 0; rate_err: @@ -203,6 +239,7 @@ void mlx5_esw_offloads_devlink_port_unregister(struct mlx5_vport *vport) if (!vport->dl_port) return; dl_port = vport->dl_port; + mlx5_esw_devlink_port_res_unregister(&dl_port->dl_port); mlx5_esw_qos_vport_update_parent(vport, NULL, NULL); devl_rate_leaf_destroy(&dl_port->dl_port); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h index c2563bee74df..5128f5020dae 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h @@ -273,7 +273,7 @@ struct mlx5_eswitch_fdb { struct mlx5_flow_group *send_to_vport_grp; struct mlx5_flow_group *send_to_vport_meta_grp; struct mlx5_flow_group *peer_miss_grp; - struct mlx5_flow_handle **peer_miss_rules[MLX5_MAX_PORTS]; + struct xarray peer_miss_rules; struct mlx5_flow_group *miss_grp; struct mlx5_flow_handle **send_to_vport_meta_rules; struct mlx5_flow_handle *miss_rule_uni; @@ -942,6 +942,12 @@ int mlx5_esw_ipsec_vf_packet_offload_supported(struct mlx5_core_dev *dev, u16 vport_num); bool mlx5_esw_host_functions_enabled(const struct mlx5_core_dev *dev); void mlx5_eswitch_safe_aux_devs_remove(struct mlx5_core_dev *dev); +struct mlx5_flow_group * +mlx5_esw_lag_demux_fg_create(struct mlx5_eswitch *esw, + struct mlx5_flow_table *ft); +struct mlx5_flow_handle * +mlx5_esw_lag_demux_rule_create(struct mlx5_eswitch *esw, u16 vport_num, + struct mlx5_flow_table *lag_ft); #else /* CONFIG_MLX5_ESWITCH */ /* eswitch API stubs */ static inline int mlx5_eswitch_init(struct mlx5_core_dev *dev) { return 0; } @@ -1027,6 +1033,12 @@ mlx5_esw_vport_vhca_id(struct mlx5_eswitch *esw, u16 vportn, u16 *vhca_id) static inline void mlx5_eswitch_safe_aux_devs_remove(struct mlx5_core_dev *dev) {} +static inline struct mlx5_flow_handle * +mlx5_esw_lag_demux_rule_create(struct mlx5_eswitch *esw, u16 vport_num, + struct mlx5_flow_table *lag_ft) +{ + return ERR_PTR(-EOPNOTSUPP); +} #endif /* CONFIG_MLX5_ESWITCH */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c index 01f6aecc4fcc..a078d06f4567 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c @@ -1190,7 +1190,7 @@ static int esw_add_fdb_peer_miss_rules(struct mlx5_eswitch *esw, struct mlx5_flow_handle *flow; struct mlx5_vport *peer_vport; struct mlx5_flow_spec *spec; - int err, pfindex; + int err; unsigned long i; void *misc; @@ -1270,14 +1270,10 @@ static int esw_add_fdb_peer_miss_rules(struct mlx5_eswitch *esw, } } - pfindex = mlx5_get_dev_index(peer_dev); - if (pfindex >= MLX5_MAX_PORTS) { - esw_warn(esw->dev, "Peer dev index(%d) is over the max num defined(%d)\n", - pfindex, MLX5_MAX_PORTS); - err = -EINVAL; + err = xa_insert(&esw->fdb_table.offloads.peer_miss_rules, + MLX5_CAP_GEN(peer_dev, vhca_id), flows, GFP_KERNEL); + if (err) goto add_ec_vf_flow_err; - } - esw->fdb_table.offloads.peer_miss_rules[pfindex] = flows; kvfree(spec); return 0; @@ -1319,12 +1315,13 @@ static void esw_del_fdb_peer_miss_rules(struct mlx5_eswitch *esw, struct mlx5_core_dev *peer_dev) { struct mlx5_eswitch *peer_esw = peer_dev->priv.eswitch; - u16 peer_index = mlx5_get_dev_index(peer_dev); + u16 peer_vhca_id = MLX5_CAP_GEN(peer_dev, vhca_id); struct mlx5_flow_handle **flows; struct mlx5_vport *peer_vport; unsigned long i; - flows = esw->fdb_table.offloads.peer_miss_rules[peer_index]; + flows = xa_erase(&esw->fdb_table.offloads.peer_miss_rules, + peer_vhca_id); if (!flows) return; @@ -1350,7 +1347,6 @@ static void esw_del_fdb_peer_miss_rules(struct mlx5_eswitch *esw, } kvfree(flows); - esw->fdb_table.offloads.peer_miss_rules[peer_index] = NULL; } static int esw_add_fdb_miss_rule(struct mlx5_eswitch *esw) @@ -1460,6 +1456,83 @@ esw_add_restore_rule(struct mlx5_eswitch *esw, u32 tag) return flow_rule; } +struct mlx5_flow_group * +mlx5_esw_lag_demux_fg_create(struct mlx5_eswitch *esw, + struct mlx5_flow_table *ft) +{ + int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); + struct mlx5_flow_group *fg; + void *match_criteria; + void *flow_group_in; + + if (!mlx5_eswitch_vport_match_metadata_enabled(esw)) + return ERR_PTR(-EOPNOTSUPP); + + if (IS_ERR(ft)) + return ERR_CAST(ft); + + flow_group_in = kvzalloc(inlen, GFP_KERNEL); + if (!flow_group_in) + return ERR_PTR(-ENOMEM); + + match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in, + match_criteria); + MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable, + MLX5_MATCH_MISC_PARAMETERS_2); + MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0); + MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, + ft->max_fte - 1); + + MLX5_SET(fte_match_param, match_criteria, + misc_parameters_2.metadata_reg_c_0, + mlx5_eswitch_get_vport_metadata_mask()); + + fg = mlx5_create_flow_group(ft, flow_group_in); + kvfree(flow_group_in); + if (IS_ERR(fg)) + esw_warn(esw->dev, "Can't create LAG demux flow group\n"); + + return fg; +} + +struct mlx5_flow_handle * +mlx5_esw_lag_demux_rule_create(struct mlx5_eswitch *esw, u16 vport_num, + struct mlx5_flow_table *lag_ft) +{ + struct mlx5_flow_spec *spec = kvzalloc(sizeof(*spec), GFP_KERNEL); + struct mlx5_flow_destination dest = {}; + struct mlx5_flow_act flow_act = {}; + struct mlx5_flow_handle *ret; + void *misc; + + if (!spec) + return ERR_PTR(-ENOMEM); + + if (!mlx5_eswitch_vport_match_metadata_enabled(esw)) { + kvfree(spec); + return ERR_PTR(-EOPNOTSUPP); + } + + misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, + misc_parameters_2); + MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, + mlx5_eswitch_get_vport_metadata_mask()); + spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2; + + misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, + misc_parameters_2); + MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, + mlx5_eswitch_get_vport_metadata_for_match(esw, vport_num)); + + flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; + dest.type = MLX5_FLOW_DESTINATION_TYPE_VHCA_RX; + dest.vhca.id = MLX5_CAP_GEN(esw->dev, vhca_id); + + ret = mlx5_add_flow_rules(lag_ft, spec, &flow_act, &dest, 1); + kvfree(spec); + return ret; +} + #define MAX_PF_SQ 256 #define MAX_SQ_NVPORTS 32 @@ -2048,7 +2121,8 @@ static int esw_create_vport_rx_group(struct mlx5_eswitch *esw) if (IS_ERR(g)) { err = PTR_ERR(g); - mlx5_core_warn(esw->dev, "Failed to create vport rx group err %d\n", err); + esw_warn(esw->dev, "Failed to create vport rx group err %d\n", + err); goto out; } @@ -2093,7 +2167,8 @@ static int esw_create_vport_rx_drop_group(struct mlx5_eswitch *esw) if (IS_ERR(g)) { err = PTR_ERR(g); - mlx5_core_warn(esw->dev, "Failed to create vport rx drop group err %d\n", err); + esw_warn(esw->dev, + "Failed to create vport rx drop group err %d\n", err); goto out; } @@ -3247,6 +3322,7 @@ void mlx5_esw_offloads_devcom_init(struct mlx5_eswitch *esw, return; xa_init(&esw->paired); + xa_init(&esw->fdb_table.offloads.peer_miss_rules); esw->num_peers = 0; esw->devcom = mlx5_devcom_register_component(esw->dev->priv.devc, MLX5_DEVCOM_ESW_OFFLOADS, @@ -3274,6 +3350,7 @@ void mlx5_esw_offloads_devcom_cleanup(struct mlx5_eswitch *esw) mlx5_devcom_unregister_component(esw->devcom); xa_destroy(&esw->paired); + xa_destroy(&esw->fdb_table.offloads.peer_miss_rules); esw->devcom = NULL; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.c b/drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.c index c348ee62cd3a..1cd4cd898ec2 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.c @@ -716,6 +716,10 @@ static int mlx5_cmd_set_fte(struct mlx5_core_dev *dev, id = dst->dest_attr.ft->id; ifc_type = MLX5_IFC_FLOW_DESTINATION_TYPE_TABLE_TYPE; break; + case MLX5_FLOW_DESTINATION_TYPE_VHCA_RX: + id = dst->dest_attr.vhca.id; + ifc_type = MLX5_IFC_FLOW_DESTINATION_TYPE_VHCA_RX; + break; default: id = dst->dest_attr.tir_num; ifc_type = MLX5_IFC_FLOW_DESTINATION_TYPE_TIR; @@ -1183,7 +1187,7 @@ int mlx5_fs_cmd_set_l2table_entry_silent(struct mlx5_core_dev *dev, u8 silent_mo { u32 in[MLX5_ST_SZ_DW(set_l2_table_entry_in)] = {}; - if (silent_mode && !MLX5_CAP_GEN(dev, silent_mode)) + if (silent_mode && !MLX5_CAP_GEN(dev, silent_mode_set)) return -EOPNOTSUPP; MLX5_SET(set_l2_table_entry_in, in, opcode, MLX5_CMD_OP_SET_L2_TABLE_ENTRY); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c b/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c index 2c3544880a30..61a6ba1e49dd 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c @@ -503,7 +503,8 @@ static bool is_fwd_dest_type(enum mlx5_flow_destination_type type) type == MLX5_FLOW_DESTINATION_TYPE_FLOW_SAMPLER || type == MLX5_FLOW_DESTINATION_TYPE_TIR || type == MLX5_FLOW_DESTINATION_TYPE_RANGE || - type == MLX5_FLOW_DESTINATION_TYPE_TABLE_TYPE; + type == MLX5_FLOW_DESTINATION_TYPE_TABLE_TYPE || + type == MLX5_FLOW_DESTINATION_TYPE_VHCA_RX; } static bool check_valid_spec(const struct mlx5_flow_spec *spec) @@ -1437,15 +1438,9 @@ mlx5_create_vport_flow_table(struct mlx5_flow_namespace *ns, struct mlx5_flow_table* mlx5_create_lag_demux_flow_table(struct mlx5_flow_namespace *ns, - int prio, u32 level) + struct mlx5_flow_table_attr *ft_attr) { - struct mlx5_flow_table_attr ft_attr = {}; - - ft_attr.level = level; - ft_attr.prio = prio; - ft_attr.max_fte = 1; - - return __mlx5_create_flow_table(ns, &ft_attr, FS_FT_OP_MOD_LAG_DEMUX, 0); + return __mlx5_create_flow_table(ns, ft_attr, FS_FT_OP_MOD_LAG_DEMUX, 0); } EXPORT_SYMBOL(mlx5_create_lag_demux_flow_table); @@ -1890,7 +1885,9 @@ static bool mlx5_flow_dests_cmp(struct mlx5_flow_destination *d1, d1->range.hit_ft == d2->range.hit_ft && d1->range.miss_ft == d2->range.miss_ft && d1->range.min == d2->range.min && - d1->range.max == d2->range.max)) + d1->range.max == d2->range.max) || + (d1->type == MLX5_FLOW_DESTINATION_TYPE_VHCA_RX && + d1->vhca.id == d2->vhca.id)) return true; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fw.c b/drivers/net/ethernet/mellanox/mlx5/core/fw.c index c1f220e5fe18..7eab7225dcc0 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/fw.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/fw.c @@ -255,6 +255,12 @@ int mlx5_query_hca_caps(struct mlx5_core_dev *dev) return err; } + if (MLX5_CAP_GEN(dev, tlp_device_emulation_manager)) { + err = mlx5_core_get_caps_mode(dev, MLX5_CAP_TLP_EMULATION, HCA_CAP_OPMOD_GET_CUR); + if (err) + return err; + } + if (MLX5_CAP_GEN(dev, ipsec_offload)) { err = mlx5_core_get_caps_mode(dev, MLX5_CAP_IPSEC, HCA_CAP_OPMOD_GET_CUR); if (err) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lag/debugfs.c b/drivers/net/ethernet/mellanox/mlx5/core/lag/debugfs.c index b8d5f6a44d26..e9b8d79d2d21 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lag/debugfs.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lag/debugfs.c @@ -145,7 +145,8 @@ static int members_show(struct seq_file *file, void *priv) ldev = mlx5_lag_dev(dev); mutex_lock(&ldev->lock); mlx5_ldev_for_each(i, 0, ldev) - seq_printf(file, "%s\n", dev_name(ldev->pf[i].dev->device)); + seq_printf(file, "%s\n", + dev_name(mlx5_lag_pf(ldev, i)->dev->device)); mutex_unlock(&ldev->lock); return 0; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c b/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c index 044adfdf9aa2..449e4bd86c06 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c @@ -35,6 +35,7 @@ #include <linux/mlx5/driver.h> #include <linux/mlx5/eswitch.h> #include <linux/mlx5/vport.h> +#include <linux/mlx5/lag.h> #include "lib/mlx5.h" #include "lib/devcom.h" #include "mlx5_core.h" @@ -232,6 +233,7 @@ static void mlx5_do_bond_work(struct work_struct *work); static void mlx5_ldev_free(struct kref *ref) { struct mlx5_lag *ldev = container_of(ref, struct mlx5_lag, ref); + struct lag_func *pf; struct net *net; int i; @@ -241,13 +243,16 @@ static void mlx5_ldev_free(struct kref *ref) } mlx5_ldev_for_each(i, 0, ldev) { - if (ldev->pf[i].dev && - ldev->pf[i].port_change_nb.nb.notifier_call) { - struct mlx5_nb *nb = &ldev->pf[i].port_change_nb; + pf = mlx5_lag_pf(ldev, i); + if (pf->port_change_nb.nb.notifier_call) { + struct mlx5_nb *nb = &pf->port_change_nb; - mlx5_eq_notifier_unregister(ldev->pf[i].dev, nb); + mlx5_eq_notifier_unregister(pf->dev, nb); } + xa_erase(&ldev->pfs, i); + kfree(pf); } + xa_destroy(&ldev->pfs); mlx5_lag_mp_cleanup(ldev); cancel_delayed_work_sync(&ldev->bond_work); @@ -284,6 +289,7 @@ static struct mlx5_lag *mlx5_lag_dev_alloc(struct mlx5_core_dev *dev) kref_init(&ldev->ref); mutex_init(&ldev->lock); + xa_init_flags(&ldev->pfs, XA_FLAGS_ALLOC); INIT_DELAYED_WORK(&ldev->bond_work, mlx5_do_bond_work); INIT_WORK(&ldev->speed_update_work, mlx5_mpesw_speed_update_work); @@ -309,23 +315,54 @@ static struct mlx5_lag *mlx5_lag_dev_alloc(struct mlx5_core_dev *dev) int mlx5_lag_dev_get_netdev_idx(struct mlx5_lag *ldev, struct net_device *ndev) { + struct lag_func *pf; int i; - mlx5_ldev_for_each(i, 0, ldev) - if (ldev->pf[i].netdev == ndev) + mlx5_ldev_for_each(i, 0, ldev) { + pf = mlx5_lag_pf(ldev, i); + if (pf->netdev == ndev) return i; + } return -ENOENT; } +static int mlx5_lag_get_master_idx(struct mlx5_lag *ldev) +{ + unsigned long idx = 0; + void *entry; + + if (!ldev) + return -ENOENT; + + entry = xa_find(&ldev->pfs, &idx, U8_MAX, MLX5_LAG_XA_MARK_MASTER); + if (!entry) + return -ENOENT; + + return (int)idx; +} + int mlx5_lag_get_dev_index_by_seq(struct mlx5_lag *ldev, int seq) { - int i, num = 0; + int master_idx, i, num = 0; if (!ldev) return -ENOENT; + master_idx = mlx5_lag_get_master_idx(ldev); + + /* If seq 0 is requested and there's a primary PF, return it */ + if (master_idx >= 0) { + if (seq == 0) + return master_idx; + num++; + } + mlx5_ldev_for_each(i, 0, ldev) { + /* Skip the primary PF in the loop */ + if (i == master_idx) + continue; + if (num == seq) return i; num++; @@ -333,6 +370,108 @@ int mlx5_lag_get_dev_index_by_seq(struct mlx5_lag *ldev, int seq) return -ENOENT; } +/* Reverse of mlx5_lag_get_dev_index_by_seq: given a device, return its + * sequence number in the LAG. Master is always 0, others numbered + * sequentially starting from 1. + */ +int mlx5_lag_get_dev_seq(struct mlx5_core_dev *dev) +{ + struct mlx5_lag *ldev = mlx5_lag_dev(dev); + int master_idx, i, num = 1; + struct lag_func *pf; + + if (!ldev) + return -ENOENT; + + master_idx = mlx5_lag_get_master_idx(ldev); + if (master_idx < 0) + return -ENOENT; + + pf = mlx5_lag_pf(ldev, master_idx); + if (pf && pf->dev == dev) + return 0; + + mlx5_ldev_for_each(i, 0, ldev) { + if (i == master_idx) + continue; + pf = mlx5_lag_pf(ldev, i); + if (pf->dev == dev) + return num; + num++; + } + return -ENOENT; +} +EXPORT_SYMBOL(mlx5_lag_get_dev_seq); + +/* Devcom events for LAG master marking */ +#define LAG_DEVCOM_PAIR (0) +#define LAG_DEVCOM_UNPAIR (1) + +static void mlx5_lag_mark_master(struct mlx5_lag *ldev) +{ + int lowest_dev_idx = INT_MAX; + struct lag_func *pf; + int master_xa_idx = -1; + int dev_idx; + int i; + + mlx5_ldev_for_each(i, 0, ldev) { + pf = mlx5_lag_pf(ldev, i); + dev_idx = mlx5_get_dev_index(pf->dev); + if (dev_idx < lowest_dev_idx) { + lowest_dev_idx = dev_idx; + master_xa_idx = i; + } + } + + if (master_xa_idx >= 0) + xa_set_mark(&ldev->pfs, master_xa_idx, MLX5_LAG_XA_MARK_MASTER); +} + +static void mlx5_lag_clear_master(struct mlx5_lag *ldev) +{ + unsigned long idx = 0; + void *entry; + + entry = xa_find(&ldev->pfs, &idx, U8_MAX, MLX5_LAG_XA_MARK_MASTER); + if (!entry) + return; + + xa_clear_mark(&ldev->pfs, idx, MLX5_LAG_XA_MARK_MASTER); +} + +/* Devcom event handler to manage LAG master marking */ +static int mlx5_lag_devcom_event(int event, void *my_data, void *event_data) +{ + struct mlx5_core_dev *dev = my_data; + struct mlx5_lag *ldev; + int idx; + + ldev = mlx5_lag_dev(dev); + if (!ldev) + return 0; + + mutex_lock(&ldev->lock); + switch (event) { + case LAG_DEVCOM_PAIR: + /* No need to mark more than once */ + idx = mlx5_lag_get_master_idx(ldev); + if (idx >= 0) + break; + /* Check if all LAG ports are now registered */ + if (mlx5_lag_num_devs(ldev) == ldev->ports) + mlx5_lag_mark_master(ldev); + break; + + case LAG_DEVCOM_UNPAIR: + /* Clear master mark when a device is removed */ + mlx5_lag_clear_master(ldev); + break; + } + mutex_unlock(&ldev->lock); + return 0; +} + int mlx5_lag_num_devs(struct mlx5_lag *ldev) { int i, num = 0; @@ -349,14 +488,17 @@ int mlx5_lag_num_devs(struct mlx5_lag *ldev) int mlx5_lag_num_netdevs(struct mlx5_lag *ldev) { + struct lag_func *pf; int i, num = 0; if (!ldev) return 0; - mlx5_ldev_for_each(i, 0, ldev) - if (ldev->pf[i].netdev) + mlx5_ldev_for_each(i, 0, ldev) { + pf = mlx5_lag_pf(ldev, i); + if (pf->netdev) num++; + } return num; } @@ -400,11 +542,12 @@ static void mlx5_infer_tx_affinity_mapping(struct lag_tracker *tracker, /* Use native mapping by default where each port's buckets * point the native port: 1 1 1 .. 1 2 2 2 ... 2 3 3 3 ... 3 etc + * ports[] values are 1-indexed device indices for FW. */ mlx5_ldev_for_each(i, 0, ldev) { for (j = 0; j < buckets; j++) { idx = i * buckets + j; - ports[idx] = i + 1; + ports[idx] = mlx5_lag_xa_to_dev_idx(ldev, i) + 1; } } @@ -416,33 +559,42 @@ static void mlx5_infer_tx_affinity_mapping(struct lag_tracker *tracker, /* Go over the disabled ports and for each assign a random active port */ for (i = 0; i < disabled_ports_num; i++) { for (j = 0; j < buckets; j++) { + int rand_xa_idx; + get_random_bytes(&rand, 4); - ports[disabled[i] * buckets + j] = enabled[rand % enabled_ports_num] + 1; + rand_xa_idx = enabled[rand % enabled_ports_num]; + ports[disabled[i] * buckets + j] = + mlx5_lag_xa_to_dev_idx(ldev, rand_xa_idx) + 1; } } } static bool mlx5_lag_has_drop_rule(struct mlx5_lag *ldev) { + struct lag_func *pf; int i; - mlx5_ldev_for_each(i, 0, ldev) - if (ldev->pf[i].has_drop) + mlx5_ldev_for_each(i, 0, ldev) { + pf = mlx5_lag_pf(ldev, i); + if (pf->has_drop) return true; + } return false; } static void mlx5_lag_drop_rule_cleanup(struct mlx5_lag *ldev) { + struct lag_func *pf; int i; mlx5_ldev_for_each(i, 0, ldev) { - if (!ldev->pf[i].has_drop) + pf = mlx5_lag_pf(ldev, i); + if (!pf->has_drop) continue; - mlx5_esw_acl_ingress_vport_drop_rule_destroy(ldev->pf[i].dev->priv.eswitch, + mlx5_esw_acl_ingress_vport_drop_rule_destroy(pf->dev->priv.eswitch, MLX5_VPORT_UPLINK); - ldev->pf[i].has_drop = false; + pf->has_drop = false; } } @@ -451,6 +603,7 @@ static void mlx5_lag_drop_rule_setup(struct mlx5_lag *ldev, { u8 disabled_ports[MLX5_MAX_PORTS] = {}; struct mlx5_core_dev *dev; + struct lag_func *pf; int disabled_index; int num_disabled; int err; @@ -468,11 +621,12 @@ static void mlx5_lag_drop_rule_setup(struct mlx5_lag *ldev, for (i = 0; i < num_disabled; i++) { disabled_index = disabled_ports[i]; - dev = ldev->pf[disabled_index].dev; + pf = mlx5_lag_pf(ldev, disabled_index); + dev = pf->dev; err = mlx5_esw_acl_ingress_vport_drop_rule_create(dev->priv.eswitch, MLX5_VPORT_UPLINK); if (!err) - ldev->pf[disabled_index].has_drop = true; + pf->has_drop = true; else mlx5_core_err(dev, "Failed to create lag drop rule, error: %d", err); @@ -504,7 +658,7 @@ static int _mlx5_modify_lag(struct mlx5_lag *ldev, u8 *ports) if (idx < 0) return -EINVAL; - dev0 = ldev->pf[idx].dev; + dev0 = mlx5_lag_pf(ldev, idx)->dev; if (test_bit(MLX5_LAG_MODE_FLAG_HASH_BASED, &ldev->mode_flags)) { ret = mlx5_lag_port_sel_modify(ldev, ports); if (ret || @@ -521,6 +675,7 @@ static int _mlx5_modify_lag(struct mlx5_lag *ldev, u8 *ports) static struct net_device *mlx5_lag_active_backup_get_netdev(struct mlx5_core_dev *dev) { struct net_device *ndev = NULL; + struct lag_func *pf; struct mlx5_lag *ldev; unsigned long flags; int i, last_idx; @@ -531,14 +686,17 @@ static struct net_device *mlx5_lag_active_backup_get_netdev(struct mlx5_core_dev if (!ldev) goto unlock; - mlx5_ldev_for_each(i, 0, ldev) + mlx5_ldev_for_each(i, 0, ldev) { + pf = mlx5_lag_pf(ldev, i); if (ldev->tracker.netdev_state[i].tx_enabled) - ndev = ldev->pf[i].netdev; + ndev = pf->netdev; + } if (!ndev) { last_idx = mlx5_lag_get_dev_index_by_seq(ldev, ldev->ports - 1); if (last_idx < 0) goto unlock; - ndev = ldev->pf[last_idx].netdev; + pf = mlx5_lag_pf(ldev, last_idx); + ndev = pf->netdev; } dev_hold(ndev); @@ -563,7 +721,7 @@ void mlx5_modify_lag(struct mlx5_lag *ldev, if (first_idx < 0) return; - dev0 = ldev->pf[first_idx].dev; + dev0 = mlx5_lag_pf(ldev, first_idx)->dev; mlx5_infer_tx_affinity_mapping(tracker, ldev, ldev->buckets, ports); mlx5_ldev_for_each(i, 0, ldev) { @@ -615,7 +773,7 @@ static int mlx5_lag_set_port_sel_mode(struct mlx5_lag *ldev, mode == MLX5_LAG_MODE_MULTIPATH) return 0; - dev0 = ldev->pf[first_idx].dev; + dev0 = mlx5_lag_pf(ldev, first_idx)->dev; if (!MLX5_CAP_PORT_SELECTION(dev0, port_select_flow_table)) { if (ldev->ports > 2) @@ -661,19 +819,24 @@ char *mlx5_get_str_port_sel_mode(enum mlx5_lag_mode mode, unsigned long flags) static int mlx5_lag_create_single_fdb(struct mlx5_lag *ldev) { - int first_idx = mlx5_lag_get_dev_index_by_seq(ldev, MLX5_LAG_P1); + int master_idx = mlx5_lag_get_dev_index_by_seq(ldev, MLX5_LAG_P1); struct mlx5_eswitch *master_esw; struct mlx5_core_dev *dev0; int i, j; int err; - if (first_idx < 0) + if (master_idx < 0) return -EINVAL; - dev0 = ldev->pf[first_idx].dev; + dev0 = mlx5_lag_pf(ldev, master_idx)->dev; master_esw = dev0->priv.eswitch; - mlx5_ldev_for_each(i, first_idx + 1, ldev) { - struct mlx5_eswitch *slave_esw = ldev->pf[i].dev->priv.eswitch; + mlx5_ldev_for_each(i, 0, ldev) { + struct mlx5_eswitch *slave_esw; + + if (i == master_idx) + continue; + + slave_esw = mlx5_lag_pf(ldev, i)->dev->priv.eswitch; err = mlx5_eswitch_offloads_single_fdb_add_one(master_esw, slave_esw, ldev->ports); @@ -682,9 +845,12 @@ static int mlx5_lag_create_single_fdb(struct mlx5_lag *ldev) } return 0; err: - mlx5_ldev_for_each_reverse(j, i, first_idx + 1, ldev) + mlx5_ldev_for_each_reverse(j, i, 0, ldev) { + if (j == master_idx) + continue; mlx5_eswitch_offloads_single_fdb_del_one(master_esw, - ldev->pf[j].dev->priv.eswitch); + mlx5_lag_pf(ldev, j)->dev->priv.eswitch); + } return err; } @@ -693,8 +859,8 @@ static int mlx5_create_lag(struct mlx5_lag *ldev, enum mlx5_lag_mode mode, unsigned long flags) { - bool shared_fdb = test_bit(MLX5_LAG_MODE_FLAG_SHARED_FDB, &flags); int first_idx = mlx5_lag_get_dev_index_by_seq(ldev, MLX5_LAG_P1); + bool shared_fdb = test_bit(MLX5_LAG_MODE_FLAG_SHARED_FDB, &flags); u32 in[MLX5_ST_SZ_DW(destroy_lag_in)] = {}; struct mlx5_core_dev *dev0; int err; @@ -702,7 +868,7 @@ static int mlx5_create_lag(struct mlx5_lag *ldev, if (first_idx < 0) return -EINVAL; - dev0 = ldev->pf[first_idx].dev; + dev0 = mlx5_lag_pf(ldev, first_idx)->dev; if (tracker) mlx5_lag_print_mapping(dev0, ldev, tracker, flags); mlx5_core_info(dev0, "shared_fdb:%d mode:%s\n", @@ -740,16 +906,17 @@ int mlx5_activate_lag(struct mlx5_lag *ldev, enum mlx5_lag_mode mode, bool shared_fdb) { - int first_idx = mlx5_lag_get_dev_index_by_seq(ldev, MLX5_LAG_P1); bool roce_lag = mode == MLX5_LAG_MODE_ROCE; struct mlx5_core_dev *dev0; unsigned long flags = 0; + int master_idx; int err; - if (first_idx < 0) + master_idx = mlx5_lag_get_dev_index_by_seq(ldev, MLX5_LAG_P1); + if (master_idx < 0) return -EINVAL; - dev0 = ldev->pf[first_idx].dev; + dev0 = mlx5_lag_pf(ldev, master_idx)->dev; err = mlx5_lag_set_flags(ldev, mode, tracker, shared_fdb, &flags); if (err) return err; @@ -793,7 +960,7 @@ int mlx5_activate_lag(struct mlx5_lag *ldev, int mlx5_deactivate_lag(struct mlx5_lag *ldev) { - int first_idx = mlx5_lag_get_dev_index_by_seq(ldev, MLX5_LAG_P1); + int master_idx = mlx5_lag_get_dev_index_by_seq(ldev, MLX5_LAG_P1); u32 in[MLX5_ST_SZ_DW(destroy_lag_in)] = {}; bool roce_lag = __mlx5_lag_is_roce(ldev); unsigned long flags = ldev->mode_flags; @@ -802,19 +969,22 @@ int mlx5_deactivate_lag(struct mlx5_lag *ldev) int err; int i; - if (first_idx < 0) + if (master_idx < 0) return -EINVAL; - dev0 = ldev->pf[first_idx].dev; + dev0 = mlx5_lag_pf(ldev, master_idx)->dev; master_esw = dev0->priv.eswitch; ldev->mode = MLX5_LAG_MODE_NONE; ldev->mode_flags = 0; mlx5_lag_mp_reset(ldev); if (test_bit(MLX5_LAG_MODE_FLAG_SHARED_FDB, &flags)) { - mlx5_ldev_for_each(i, first_idx + 1, ldev) + mlx5_ldev_for_each(i, 0, ldev) { + if (i == master_idx) + continue; mlx5_eswitch_offloads_single_fdb_del_one(master_esw, - ldev->pf[i].dev->priv.eswitch); + mlx5_lag_pf(ldev, i)->dev->priv.eswitch); + } clear_bit(MLX5_LAG_MODE_FLAG_SHARED_FDB, &flags); } @@ -844,68 +1014,82 @@ int mlx5_deactivate_lag(struct mlx5_lag *ldev) bool mlx5_lag_check_prereq(struct mlx5_lag *ldev) { - int first_idx = mlx5_lag_get_dev_index_by_seq(ldev, MLX5_LAG_P1); + int master_idx = mlx5_lag_get_dev_index_by_seq(ldev, MLX5_LAG_P1); #ifdef CONFIG_MLX5_ESWITCH struct mlx5_core_dev *dev; u8 mode; #endif + struct lag_func *pf; bool roce_support; int i; - if (first_idx < 0 || mlx5_lag_num_devs(ldev) != ldev->ports) + if (master_idx < 0 || mlx5_lag_num_devs(ldev) != ldev->ports) return false; #ifdef CONFIG_MLX5_ESWITCH mlx5_ldev_for_each(i, 0, ldev) { - dev = ldev->pf[i].dev; + pf = mlx5_lag_pf(ldev, i); + dev = pf->dev; if (mlx5_eswitch_num_vfs(dev->priv.eswitch) && !is_mdev_switchdev_mode(dev)) return false; } - dev = ldev->pf[first_idx].dev; + pf = mlx5_lag_pf(ldev, master_idx); + dev = pf->dev; mode = mlx5_eswitch_mode(dev); - mlx5_ldev_for_each(i, 0, ldev) - if (mlx5_eswitch_mode(ldev->pf[i].dev) != mode) + mlx5_ldev_for_each(i, 0, ldev) { + pf = mlx5_lag_pf(ldev, i); + if (mlx5_eswitch_mode(pf->dev) != mode) return false; + } #else - mlx5_ldev_for_each(i, 0, ldev) - if (mlx5_sriov_is_enabled(ldev->pf[i].dev)) + mlx5_ldev_for_each(i, 0, ldev) { + pf = mlx5_lag_pf(ldev, i); + if (mlx5_sriov_is_enabled(pf->dev)) return false; + } #endif - roce_support = mlx5_get_roce_state(ldev->pf[first_idx].dev); - mlx5_ldev_for_each(i, first_idx + 1, ldev) - if (mlx5_get_roce_state(ldev->pf[i].dev) != roce_support) + pf = mlx5_lag_pf(ldev, master_idx); + roce_support = mlx5_get_roce_state(pf->dev); + mlx5_ldev_for_each(i, 0, ldev) { + if (i == master_idx) + continue; + pf = mlx5_lag_pf(ldev, i); + if (mlx5_get_roce_state(pf->dev) != roce_support) return false; + } return true; } void mlx5_lag_add_devices(struct mlx5_lag *ldev) { + struct lag_func *pf; int i; mlx5_ldev_for_each(i, 0, ldev) { - if (ldev->pf[i].dev->priv.flags & - MLX5_PRIV_FLAGS_DISABLE_ALL_ADEV) + pf = mlx5_lag_pf(ldev, i); + if (pf->dev->priv.flags & MLX5_PRIV_FLAGS_DISABLE_ALL_ADEV) continue; - ldev->pf[i].dev->priv.flags &= ~MLX5_PRIV_FLAGS_DISABLE_IB_ADEV; - mlx5_rescan_drivers_locked(ldev->pf[i].dev); + pf->dev->priv.flags &= ~MLX5_PRIV_FLAGS_DISABLE_IB_ADEV; + mlx5_rescan_drivers_locked(pf->dev); } } void mlx5_lag_remove_devices(struct mlx5_lag *ldev) { + struct lag_func *pf; int i; mlx5_ldev_for_each(i, 0, ldev) { - if (ldev->pf[i].dev->priv.flags & - MLX5_PRIV_FLAGS_DISABLE_ALL_ADEV) + pf = mlx5_lag_pf(ldev, i); + if (pf->dev->priv.flags & MLX5_PRIV_FLAGS_DISABLE_ALL_ADEV) continue; - ldev->pf[i].dev->priv.flags |= MLX5_PRIV_FLAGS_DISABLE_IB_ADEV; - mlx5_rescan_drivers_locked(ldev->pf[i].dev); + pf->dev->priv.flags |= MLX5_PRIV_FLAGS_DISABLE_IB_ADEV; + mlx5_rescan_drivers_locked(pf->dev); } } @@ -921,7 +1105,7 @@ void mlx5_disable_lag(struct mlx5_lag *ldev) if (idx < 0) return; - dev0 = ldev->pf[idx].dev; + dev0 = mlx5_lag_pf(ldev, idx)->dev; roce_lag = __mlx5_lag_is_roce(ldev); if (shared_fdb) { @@ -931,8 +1115,11 @@ void mlx5_disable_lag(struct mlx5_lag *ldev) dev0->priv.flags |= MLX5_PRIV_FLAGS_DISABLE_IB_ADEV; mlx5_rescan_drivers_locked(dev0); } - mlx5_ldev_for_each(i, idx + 1, ldev) - mlx5_nic_vport_disable_roce(ldev->pf[i].dev); + mlx5_ldev_for_each(i, 0, ldev) { + if (i == idx) + continue; + mlx5_nic_vport_disable_roce(mlx5_lag_pf(ldev, i)->dev); + } } err = mlx5_deactivate_lag(ldev); @@ -944,21 +1131,25 @@ void mlx5_disable_lag(struct mlx5_lag *ldev) if (shared_fdb) mlx5_ldev_for_each(i, 0, ldev) - if (!(ldev->pf[i].dev->priv.flags & MLX5_PRIV_FLAGS_DISABLE_ALL_ADEV)) - mlx5_eswitch_reload_ib_reps(ldev->pf[i].dev->priv.eswitch); + if (!(mlx5_lag_pf(ldev, i)->dev->priv.flags & MLX5_PRIV_FLAGS_DISABLE_ALL_ADEV)) + mlx5_eswitch_reload_ib_reps(mlx5_lag_pf(ldev, i)->dev->priv.eswitch); } bool mlx5_lag_shared_fdb_supported(struct mlx5_lag *ldev) { - int idx = mlx5_lag_get_dev_index_by_seq(ldev, MLX5_LAG_P1); struct mlx5_core_dev *dev; + bool ret = false; + int idx; int i; + idx = mlx5_lag_get_dev_index_by_seq(ldev, MLX5_LAG_P1); if (idx < 0) return false; - mlx5_ldev_for_each(i, idx + 1, ldev) { - dev = ldev->pf[i].dev; + mlx5_ldev_for_each(i, 0, ldev) { + if (i == idx) + continue; + dev = mlx5_lag_pf(ldev, i)->dev; if (is_mdev_switchdev_mode(dev) && mlx5_eswitch_vport_match_metadata_enabled(dev->priv.eswitch) && MLX5_CAP_GEN(dev, lag_native_fdb_selection) && @@ -969,28 +1160,33 @@ bool mlx5_lag_shared_fdb_supported(struct mlx5_lag *ldev) return false; } - dev = ldev->pf[idx].dev; + dev = mlx5_lag_pf(ldev, idx)->dev; if (is_mdev_switchdev_mode(dev) && mlx5_eswitch_vport_match_metadata_enabled(dev->priv.eswitch) && mlx5_esw_offloads_devcom_is_ready(dev->priv.eswitch) && MLX5_CAP_ESW(dev, esw_shared_ingress_acl) && mlx5_eswitch_get_npeers(dev->priv.eswitch) == MLX5_CAP_GEN(dev, num_lag_ports) - 1) - return true; + ret = true; - return false; + return ret; } static bool mlx5_lag_is_roce_lag(struct mlx5_lag *ldev) { bool roce_lag = true; + struct lag_func *pf; int i; - mlx5_ldev_for_each(i, 0, ldev) - roce_lag = roce_lag && !mlx5_sriov_is_enabled(ldev->pf[i].dev); + mlx5_ldev_for_each(i, 0, ldev) { + pf = mlx5_lag_pf(ldev, i); + roce_lag = roce_lag && !mlx5_sriov_is_enabled(pf->dev); + } #ifdef CONFIG_MLX5_ESWITCH - mlx5_ldev_for_each(i, 0, ldev) - roce_lag = roce_lag && is_mdev_legacy_mode(ldev->pf[i].dev); + mlx5_ldev_for_each(i, 0, ldev) { + pf = mlx5_lag_pf(ldev, i); + roce_lag = roce_lag && is_mdev_legacy_mode(pf->dev); + } #endif return roce_lag; @@ -1014,13 +1210,17 @@ mlx5_lag_sum_devices_speed(struct mlx5_lag *ldev, u32 *sum_speed, int (*get_speed)(struct mlx5_core_dev *, u32 *)) { struct mlx5_core_dev *pf_mdev; + struct lag_func *pf; int pf_idx; u32 speed; int ret; *sum_speed = 0; mlx5_ldev_for_each(pf_idx, 0, ldev) { - pf_mdev = ldev->pf[pf_idx].dev; + pf = mlx5_lag_pf(ldev, pf_idx); + if (!pf) + continue; + pf_mdev = pf->dev; if (!pf_mdev) continue; @@ -1086,6 +1286,7 @@ static void mlx5_lag_modify_device_vports_speed(struct mlx5_core_dev *mdev, void mlx5_lag_set_vports_agg_speed(struct mlx5_lag *ldev) { struct mlx5_core_dev *mdev; + struct lag_func *pf; u32 speed; int pf_idx; @@ -1105,7 +1306,10 @@ void mlx5_lag_set_vports_agg_speed(struct mlx5_lag *ldev) speed = speed / MLX5_MAX_TX_SPEED_UNIT; mlx5_ldev_for_each(pf_idx, 0, ldev) { - mdev = ldev->pf[pf_idx].dev; + pf = mlx5_lag_pf(ldev, pf_idx); + if (!pf) + continue; + mdev = pf->dev; if (!mdev) continue; @@ -1116,12 +1320,16 @@ void mlx5_lag_set_vports_agg_speed(struct mlx5_lag *ldev) void mlx5_lag_reset_vports_speed(struct mlx5_lag *ldev) { struct mlx5_core_dev *mdev; + struct lag_func *pf; u32 speed; int pf_idx; int ret; mlx5_ldev_for_each(pf_idx, 0, ldev) { - mdev = ldev->pf[pf_idx].dev; + pf = mlx5_lag_pf(ldev, pf_idx); + if (!pf) + continue; + mdev = pf->dev; if (!mdev) continue; @@ -1152,7 +1360,7 @@ static void mlx5_do_bond(struct mlx5_lag *ldev) if (idx < 0) return; - dev0 = ldev->pf[idx].dev; + dev0 = mlx5_lag_pf(ldev, idx)->dev; if (!mlx5_lag_is_ready(ldev)) { do_bond = false; } else { @@ -1182,16 +1390,23 @@ static void mlx5_do_bond(struct mlx5_lag *ldev) mlx5_lag_add_devices(ldev); if (shared_fdb) { mlx5_ldev_for_each(i, 0, ldev) - mlx5_eswitch_reload_ib_reps(ldev->pf[i].dev->priv.eswitch); + mlx5_eswitch_reload_ib_reps(mlx5_lag_pf(ldev, i)->dev->priv.eswitch); } return; - } else if (roce_lag) { + } + + if (roce_lag) { + struct mlx5_core_dev *dev; + dev0->priv.flags &= ~MLX5_PRIV_FLAGS_DISABLE_IB_ADEV; mlx5_rescan_drivers_locked(dev0); - mlx5_ldev_for_each(i, idx + 1, ldev) { - if (mlx5_get_roce_state(ldev->pf[i].dev)) - mlx5_nic_vport_enable_roce(ldev->pf[i].dev); + mlx5_ldev_for_each(i, 0, ldev) { + if (i == idx) + continue; + dev = mlx5_lag_pf(ldev, i)->dev; + if (mlx5_get_roce_state(dev)) + mlx5_nic_vport_enable_roce(dev); } } else if (shared_fdb) { int i; @@ -1200,7 +1415,7 @@ static void mlx5_do_bond(struct mlx5_lag *ldev) mlx5_rescan_drivers_locked(dev0); mlx5_ldev_for_each(i, 0, ldev) { - err = mlx5_eswitch_reload_ib_reps(ldev->pf[i].dev->priv.eswitch); + err = mlx5_eswitch_reload_ib_reps(mlx5_lag_pf(ldev, i)->dev->priv.eswitch); if (err) break; } @@ -1211,7 +1426,7 @@ static void mlx5_do_bond(struct mlx5_lag *ldev) mlx5_deactivate_lag(ldev); mlx5_lag_add_devices(ldev); mlx5_ldev_for_each(i, 0, ldev) - mlx5_eswitch_reload_ib_reps(ldev->pf[i].dev->priv.eswitch); + mlx5_eswitch_reload_ib_reps(mlx5_lag_pf(ldev, i)->dev->priv.eswitch); mlx5_core_err(dev0, "Failed to enable lag\n"); return; } @@ -1243,16 +1458,171 @@ static void mlx5_do_bond(struct mlx5_lag *ldev) struct mlx5_devcom_comp_dev *mlx5_lag_get_devcom_comp(struct mlx5_lag *ldev) { struct mlx5_devcom_comp_dev *devcom = NULL; + struct lag_func *pf; int i; mutex_lock(&ldev->lock); i = mlx5_get_next_ldev_func(ldev, 0); - if (i < MLX5_MAX_PORTS) - devcom = ldev->pf[i].dev->priv.hca_devcom_comp; + if (i < MLX5_MAX_PORTS) { + pf = mlx5_lag_pf(ldev, i); + devcom = pf->dev->priv.hca_devcom_comp; + } mutex_unlock(&ldev->lock); return devcom; } +static int mlx5_lag_demux_ft_fg_init(struct mlx5_core_dev *dev, + struct mlx5_flow_table_attr *ft_attr, + struct mlx5_lag *ldev) +{ +#ifdef CONFIG_MLX5_ESWITCH + struct mlx5_flow_namespace *ns; + struct mlx5_flow_group *fg; + int err; + + ns = mlx5_get_flow_namespace(dev, MLX5_FLOW_NAMESPACE_LAG); + if (!ns) + return 0; + + ldev->lag_demux_ft = mlx5_create_flow_table(ns, ft_attr); + if (IS_ERR(ldev->lag_demux_ft)) + return PTR_ERR(ldev->lag_demux_ft); + + fg = mlx5_esw_lag_demux_fg_create(dev->priv.eswitch, + ldev->lag_demux_ft); + if (IS_ERR(fg)) { + err = PTR_ERR(fg); + mlx5_destroy_flow_table(ldev->lag_demux_ft); + ldev->lag_demux_ft = NULL; + return err; + } + + ldev->lag_demux_fg = fg; + return 0; +#else + return -EOPNOTSUPP; +#endif +} + +static int mlx5_lag_demux_fw_init(struct mlx5_core_dev *dev, + struct mlx5_flow_table_attr *ft_attr, + struct mlx5_lag *ldev) +{ + struct mlx5_flow_namespace *ns; + int err; + + ns = mlx5_get_flow_namespace(dev, MLX5_FLOW_NAMESPACE_LAG); + if (!ns) + return 0; + + ldev->lag_demux_fg = NULL; + ft_attr->max_fte = 1; + ldev->lag_demux_ft = mlx5_create_lag_demux_flow_table(ns, ft_attr); + if (IS_ERR(ldev->lag_demux_ft)) { + err = PTR_ERR(ldev->lag_demux_ft); + ldev->lag_demux_ft = NULL; + return err; + } + + return 0; +} + +int mlx5_lag_demux_init(struct mlx5_core_dev *dev, + struct mlx5_flow_table_attr *ft_attr) +{ + struct mlx5_lag *ldev; + + if (!ft_attr) + return -EINVAL; + + ldev = mlx5_lag_dev(dev); + if (!ldev) + return -ENODEV; + + xa_init(&ldev->lag_demux_rules); + + if (mlx5_get_sd(dev)) + return mlx5_lag_demux_ft_fg_init(dev, ft_attr, ldev); + + return mlx5_lag_demux_fw_init(dev, ft_attr, ldev); +} +EXPORT_SYMBOL(mlx5_lag_demux_init); + +void mlx5_lag_demux_cleanup(struct mlx5_core_dev *dev) +{ + struct mlx5_flow_handle *rule; + struct mlx5_lag *ldev; + unsigned long vport_num; + + ldev = mlx5_lag_dev(dev); + if (!ldev) + return; + + xa_for_each(&ldev->lag_demux_rules, vport_num, rule) + mlx5_del_flow_rules(rule); + xa_destroy(&ldev->lag_demux_rules); + + if (ldev->lag_demux_fg) + mlx5_destroy_flow_group(ldev->lag_demux_fg); + if (ldev->lag_demux_ft) + mlx5_destroy_flow_table(ldev->lag_demux_ft); + ldev->lag_demux_fg = NULL; + ldev->lag_demux_ft = NULL; +} +EXPORT_SYMBOL(mlx5_lag_demux_cleanup); + +int mlx5_lag_demux_rule_add(struct mlx5_core_dev *vport_dev, u16 vport_num, + int index) +{ + struct mlx5_flow_handle *rule; + struct mlx5_lag *ldev; + int err; + + ldev = mlx5_lag_dev(vport_dev); + if (!ldev || !ldev->lag_demux_fg) + return 0; + + if (xa_load(&ldev->lag_demux_rules, index)) + return 0; + + rule = mlx5_esw_lag_demux_rule_create(vport_dev->priv.eswitch, + vport_num, ldev->lag_demux_ft); + if (IS_ERR(rule)) { + err = PTR_ERR(rule); + mlx5_core_warn(vport_dev, + "Failed to create LAG demux rule for vport %u, err %d\n", + vport_num, err); + return err; + } + + err = xa_err(xa_store(&ldev->lag_demux_rules, index, rule, + GFP_KERNEL)); + if (err) { + mlx5_del_flow_rules(rule); + mlx5_core_warn(vport_dev, + "Failed to store LAG demux rule for vport %u, err %d\n", + vport_num, err); + } + + return err; +} +EXPORT_SYMBOL(mlx5_lag_demux_rule_add); + +void mlx5_lag_demux_rule_del(struct mlx5_core_dev *dev, int index) +{ + struct mlx5_flow_handle *rule; + struct mlx5_lag *ldev; + + ldev = mlx5_lag_dev(dev); + if (!ldev || !ldev->lag_demux_fg) + return; + + rule = xa_erase(&ldev->lag_demux_rules, index); + if (rule) + mlx5_del_flow_rules(rule); +} +EXPORT_SYMBOL(mlx5_lag_demux_rule_del); + static void mlx5_queue_bond_work(struct mlx5_lag *ldev, unsigned long delay) { queue_delayed_work(ldev->wq, &ldev->bond_work, delay); @@ -1297,6 +1667,7 @@ static int mlx5_handle_changeupper_event(struct mlx5_lag *ldev, struct netdev_lag_upper_info *lag_upper_info = NULL; bool is_bonded, is_in_lag, mode_supported; bool has_inactive = 0; + struct lag_func *pf; struct slave *slave; u8 bond_status = 0; int num_slaves = 0; @@ -1317,7 +1688,8 @@ static int mlx5_handle_changeupper_event(struct mlx5_lag *ldev, rcu_read_lock(); for_each_netdev_in_bond_rcu(upper, ndev_tmp) { mlx5_ldev_for_each(i, 0, ldev) { - if (ldev->pf[i].netdev == ndev_tmp) { + pf = mlx5_lag_pf(ldev, i); + if (pf->netdev == ndev_tmp) { idx++; break; } @@ -1537,59 +1909,92 @@ static void mlx5_ldev_add_netdev(struct mlx5_lag *ldev, struct mlx5_core_dev *dev, struct net_device *netdev) { - unsigned int fn = mlx5_get_dev_index(dev); + struct lag_func *pf; unsigned long flags; + int i; spin_lock_irqsave(&lag_lock, flags); - ldev->pf[fn].netdev = netdev; - ldev->tracker.netdev_state[fn].link_up = 0; - ldev->tracker.netdev_state[fn].tx_enabled = 0; + /* Find pf entry by matching dev pointer */ + mlx5_ldev_for_each(i, 0, ldev) { + pf = mlx5_lag_pf(ldev, i); + if (pf->dev == dev) { + pf->netdev = netdev; + ldev->tracker.netdev_state[i].link_up = 0; + ldev->tracker.netdev_state[i].tx_enabled = 0; + break; + } + } spin_unlock_irqrestore(&lag_lock, flags); } static void mlx5_ldev_remove_netdev(struct mlx5_lag *ldev, struct net_device *netdev) { + struct lag_func *pf; unsigned long flags; int i; spin_lock_irqsave(&lag_lock, flags); mlx5_ldev_for_each(i, 0, ldev) { - if (ldev->pf[i].netdev == netdev) { - ldev->pf[i].netdev = NULL; + pf = mlx5_lag_pf(ldev, i); + if (pf->netdev == netdev) { + pf->netdev = NULL; break; } } spin_unlock_irqrestore(&lag_lock, flags); } -static void mlx5_ldev_add_mdev(struct mlx5_lag *ldev, +static int mlx5_ldev_add_mdev(struct mlx5_lag *ldev, struct mlx5_core_dev *dev) { - unsigned int fn = mlx5_get_dev_index(dev); + struct lag_func *pf; + u32 idx; + int err; + + pf = kzalloc_obj(*pf); + if (!pf) + return -ENOMEM; + + err = xa_alloc(&ldev->pfs, &idx, pf, XA_LIMIT(0, MLX5_MAX_PORTS - 1), + GFP_KERNEL); + if (err) { + kfree(pf); + return err; + } - ldev->pf[fn].dev = dev; + pf->idx = idx; + pf->dev = dev; dev->priv.lag = ldev; - MLX5_NB_INIT(&ldev->pf[fn].port_change_nb, + MLX5_NB_INIT(&pf->port_change_nb, mlx5_lag_mpesw_port_change_event, PORT_CHANGE); - mlx5_eq_notifier_register(dev, &ldev->pf[fn].port_change_nb); + mlx5_eq_notifier_register(dev, &pf->port_change_nb); + + return 0; } static void mlx5_ldev_remove_mdev(struct mlx5_lag *ldev, struct mlx5_core_dev *dev) { - int fn; + struct lag_func *pf; + int i; - fn = mlx5_get_dev_index(dev); - if (ldev->pf[fn].dev != dev) + mlx5_ldev_for_each(i, 0, ldev) { + pf = mlx5_lag_pf(ldev, i); + if (pf->dev == dev) + break; + } + if (i >= MLX5_MAX_PORTS) return; - if (ldev->pf[fn].port_change_nb.nb.notifier_call) - mlx5_eq_notifier_unregister(dev, &ldev->pf[fn].port_change_nb); + if (pf->port_change_nb.nb.notifier_call) + mlx5_eq_notifier_unregister(dev, &pf->port_change_nb); - ldev->pf[fn].dev = NULL; + pf->dev = NULL; dev->priv.lag = NULL; + xa_erase(&ldev->pfs, pf->idx); + kfree(pf); } /* Must be called with HCA devcom component lock held */ @@ -1598,6 +2003,7 @@ static int __mlx5_lag_dev_add_mdev(struct mlx5_core_dev *dev) struct mlx5_devcom_comp_dev *pos = NULL; struct mlx5_lag *ldev = NULL; struct mlx5_core_dev *tmp_dev; + int err; tmp_dev = mlx5_devcom_get_next_peer_data(dev->priv.hca_devcom_comp, &pos); if (tmp_dev) @@ -1609,7 +2015,12 @@ static int __mlx5_lag_dev_add_mdev(struct mlx5_core_dev *dev) mlx5_core_err(dev, "Failed to alloc lag dev\n"); return 0; } - mlx5_ldev_add_mdev(ldev, dev); + err = mlx5_ldev_add_mdev(ldev, dev); + if (err) { + mlx5_core_err(dev, "Failed to add mdev to lag dev\n"); + mlx5_ldev_put(ldev); + return 0; + } return 0; } @@ -1619,7 +2030,12 @@ static int __mlx5_lag_dev_add_mdev(struct mlx5_core_dev *dev) return -EAGAIN; } mlx5_ldev_get(ldev); - mlx5_ldev_add_mdev(ldev, dev); + err = mlx5_ldev_add_mdev(ldev, dev); + if (err) { + mlx5_ldev_put(ldev); + mutex_unlock(&ldev->lock); + return err; + } mutex_unlock(&ldev->lock); return 0; @@ -1647,7 +2063,8 @@ static int mlx5_lag_register_hca_devcom_comp(struct mlx5_core_dev *dev) dev->priv.hca_devcom_comp = mlx5_devcom_register_component(dev->priv.devc, MLX5_DEVCOM_HCA_PORTS, - &attr, NULL, dev); + &attr, mlx5_lag_devcom_event, + dev); if (!dev->priv.hca_devcom_comp) { mlx5_core_err(dev, "Failed to register devcom HCA component."); @@ -1678,6 +2095,9 @@ recheck: } mlx5_ldev_remove_mdev(ldev, dev); mutex_unlock(&ldev->lock); + /* Send devcom event to notify peers that a device is being removed */ + mlx5_devcom_send_event(dev->priv.hca_devcom_comp, + LAG_DEVCOM_UNPAIR, LAG_DEVCOM_UNPAIR, dev); mlx5_lag_unregister_hca_devcom_comp(dev); mlx5_ldev_put(ldev); } @@ -1701,6 +2121,9 @@ recheck: msleep(100); goto recheck; } + /* Send devcom event to notify peers that a device was added */ + mlx5_devcom_send_event(dev->priv.hca_devcom_comp, + LAG_DEVCOM_PAIR, LAG_DEVCOM_UNPAIR, dev); mlx5_ldev_add_debugfs(dev); } @@ -1746,21 +2169,25 @@ void mlx5_lag_add_netdev(struct mlx5_core_dev *dev, int mlx5_get_pre_ldev_func(struct mlx5_lag *ldev, int start_idx, int end_idx) { + struct lag_func *pf; int i; - for (i = start_idx; i >= end_idx; i--) - if (ldev->pf[i].dev) + for (i = start_idx; i >= end_idx; i--) { + pf = xa_load(&ldev->pfs, i); + if (pf && pf->dev) return i; + } return -1; } int mlx5_get_next_ldev_func(struct mlx5_lag *ldev, int start_idx) { - int i; + struct lag_func *pf; + unsigned long idx; - for (i = start_idx; i < MLX5_MAX_PORTS; i++) - if (ldev->pf[i].dev) - return i; + xa_for_each_start(&ldev->pfs, idx, pf, start_idx) + if (pf->dev) + return idx; return MLX5_MAX_PORTS; } @@ -1814,13 +2241,17 @@ bool mlx5_lag_is_master(struct mlx5_core_dev *dev) { struct mlx5_lag *ldev; unsigned long flags; + struct lag_func *pf; bool res = false; int idx; spin_lock_irqsave(&lag_lock, flags); ldev = mlx5_lag_dev(dev); idx = mlx5_lag_get_dev_index_by_seq(ldev, MLX5_LAG_P1); - res = ldev && __mlx5_lag_is_active(ldev) && idx >= 0 && dev == ldev->pf[idx].dev; + if (ldev && __mlx5_lag_is_active(ldev) && idx >= 0) { + pf = mlx5_lag_pf(ldev, idx); + res = pf && dev == pf->dev; + } spin_unlock_irqrestore(&lag_lock, flags); return res; @@ -1899,6 +2330,7 @@ u8 mlx5_lag_get_slave_port(struct mlx5_core_dev *dev, { struct mlx5_lag *ldev; unsigned long flags; + struct lag_func *pf; u8 port = 0; int i; @@ -1908,7 +2340,8 @@ u8 mlx5_lag_get_slave_port(struct mlx5_core_dev *dev, goto unlock; mlx5_ldev_for_each(i, 0, ldev) { - if (ldev->pf[i].netdev == slave) { + pf = mlx5_lag_pf(ldev, i); + if (pf->netdev == slave) { port = i; break; } @@ -1939,6 +2372,7 @@ struct mlx5_core_dev *mlx5_lag_get_next_peer_mdev(struct mlx5_core_dev *dev, int struct mlx5_core_dev *peer_dev = NULL; struct mlx5_lag *ldev; unsigned long flags; + struct lag_func *pf; int idx; spin_lock_irqsave(&lag_lock, flags); @@ -1948,9 +2382,11 @@ struct mlx5_core_dev *mlx5_lag_get_next_peer_mdev(struct mlx5_core_dev *dev, int if (*i == MLX5_MAX_PORTS) goto unlock; - mlx5_ldev_for_each(idx, *i, ldev) - if (ldev->pf[idx].dev != dev) + mlx5_ldev_for_each(idx, *i, ldev) { + pf = mlx5_lag_pf(ldev, idx); + if (pf->dev != dev) break; + } if (idx == MLX5_MAX_PORTS) { *i = idx; @@ -1958,7 +2394,8 @@ struct mlx5_core_dev *mlx5_lag_get_next_peer_mdev(struct mlx5_core_dev *dev, int } *i = idx + 1; - peer_dev = ldev->pf[idx].dev; + pf = mlx5_lag_pf(ldev, idx); + peer_dev = pf->dev; unlock: spin_unlock_irqrestore(&lag_lock, flags); @@ -1976,6 +2413,7 @@ int mlx5_lag_query_cong_counters(struct mlx5_core_dev *dev, int ret = 0, i, j, idx = 0; struct mlx5_lag *ldev; unsigned long flags; + struct lag_func *pf; int num_ports; void *out; @@ -1995,8 +2433,10 @@ int mlx5_lag_query_cong_counters(struct mlx5_core_dev *dev, ldev = mlx5_lag_dev(dev); if (ldev && __mlx5_lag_is_active(ldev)) { num_ports = ldev->ports; - mlx5_ldev_for_each(i, 0, ldev) - mdev[idx++] = ldev->pf[i].dev; + mlx5_ldev_for_each(i, 0, ldev) { + pf = mlx5_lag_pf(ldev, i); + mdev[idx++] = pf->dev; + } } else { num_ports = 1; mdev[MLX5_LAG_P1] = dev; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.h b/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.h index be1afece5fdc..6c911374f409 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.h @@ -5,8 +5,17 @@ #define __MLX5_LAG_H__ #include <linux/debugfs.h> +#include <linux/errno.h> +#include <linux/xarray.h> +#include <linux/mlx5/fs.h> #define MLX5_LAG_MAX_HASH_BUCKETS 16 +/* XArray mark for the LAG master device + * (device with lowest mlx5_get_dev_index). + * Note: XA_MARK_0 is reserved by XA_FLAGS_ALLOC for free-slot tracking. + */ +#define MLX5_LAG_XA_MARK_MASTER XA_MARK_1 + #include "mlx5_core.h" #include "mp.h" #include "port_sel.h" @@ -39,6 +48,7 @@ struct lag_func { struct mlx5_core_dev *dev; struct net_device *netdev; bool has_drop; + unsigned int idx; /* xarray index assigned by LAG */ struct mlx5_nb port_change_nb; }; @@ -64,7 +74,7 @@ struct mlx5_lag { int mode_changes_in_progress; u8 v2p_map[MLX5_MAX_PORTS * MLX5_LAG_MAX_HASH_BUCKETS]; struct kref ref; - struct lag_func pf[MLX5_MAX_PORTS]; + struct xarray pfs; struct lag_tracker tracker; struct workqueue_struct *wq; struct delayed_work bond_work; @@ -76,6 +86,9 @@ struct mlx5_lag { /* Protect lag fields/state changes */ struct mutex lock; struct lag_mpesw lag_mpesw; + struct mlx5_flow_table *lag_demux_ft; + struct mlx5_flow_group *lag_demux_fg; + struct xarray lag_demux_rules; }; static inline struct mlx5_lag * @@ -84,6 +97,34 @@ mlx5_lag_dev(struct mlx5_core_dev *dev) return dev->priv.lag; } +static inline struct lag_func * +mlx5_lag_pf(struct mlx5_lag *ldev, unsigned int idx) +{ + return xa_load(&ldev->pfs, idx); +} + +/* Get device index (mlx5_get_dev_index) from xarray index */ +static inline int mlx5_lag_xa_to_dev_idx(struct mlx5_lag *ldev, int xa_idx) +{ + struct lag_func *pf = mlx5_lag_pf(ldev, xa_idx); + + return pf ? mlx5_get_dev_index(pf->dev) : -ENOENT; +} + +/* Find lag_func by device index (reverse lookup from mlx5_get_dev_index) */ +static inline struct lag_func * +mlx5_lag_pf_by_dev_idx(struct mlx5_lag *ldev, int dev_idx) +{ + struct lag_func *pf; + unsigned long idx; + + xa_for_each(&ldev->pfs, idx, pf) { + if (mlx5_get_dev_index(pf->dev) == dev_idx) + return pf; + } + return NULL; +} + static inline bool __mlx5_lag_is_active(struct mlx5_lag *ldev) { @@ -98,6 +139,12 @@ mlx5_lag_is_ready(struct mlx5_lag *ldev) bool mlx5_lag_shared_fdb_supported(struct mlx5_lag *ldev); bool mlx5_lag_check_prereq(struct mlx5_lag *ldev); +int mlx5_lag_demux_init(struct mlx5_core_dev *dev, + struct mlx5_flow_table_attr *ft_attr); +void mlx5_lag_demux_cleanup(struct mlx5_core_dev *dev); +int mlx5_lag_demux_rule_add(struct mlx5_core_dev *dev, u16 vport_num, + int vport_index); +void mlx5_lag_demux_rule_del(struct mlx5_core_dev *dev, int vport_index); void mlx5_modify_lag(struct mlx5_lag *ldev, struct lag_tracker *tracker); int mlx5_activate_lag(struct mlx5_lag *ldev, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lag/mp.c b/drivers/net/ethernet/mellanox/mlx5/core/lag/mp.c index c4c2bf33ef35..f42e051fa7e7 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lag/mp.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lag/mp.c @@ -29,8 +29,8 @@ static bool mlx5_lag_multipath_check_prereq(struct mlx5_lag *ldev) if (ldev->ports > MLX5_LAG_MULTIPATH_OFFLOADS_SUPPORTED_PORTS) return false; - return mlx5_esw_multipath_prereq(ldev->pf[idx0].dev, - ldev->pf[idx1].dev); + return mlx5_esw_multipath_prereq(mlx5_lag_pf(ldev, idx0)->dev, + mlx5_lag_pf(ldev, idx1)->dev); } bool mlx5_lag_is_multipath(struct mlx5_core_dev *dev) @@ -80,18 +80,18 @@ static void mlx5_lag_set_port_affinity(struct mlx5_lag *ldev, tracker.netdev_state[idx1].link_up = true; break; default: - mlx5_core_warn(ldev->pf[idx0].dev, + mlx5_core_warn(mlx5_lag_pf(ldev, idx0)->dev, "Invalid affinity port %d", port); return; } if (tracker.netdev_state[idx0].tx_enabled) - mlx5_notifier_call_chain(ldev->pf[idx0].dev->priv.events, + mlx5_notifier_call_chain(mlx5_lag_pf(ldev, idx0)->dev->priv.events, MLX5_DEV_EVENT_PORT_AFFINITY, (void *)0); if (tracker.netdev_state[idx1].tx_enabled) - mlx5_notifier_call_chain(ldev->pf[idx1].dev->priv.events, + mlx5_notifier_call_chain(mlx5_lag_pf(ldev, idx1)->dev->priv.events, MLX5_DEV_EVENT_PORT_AFFINITY, (void *)0); @@ -146,7 +146,7 @@ mlx5_lag_get_next_fib_dev(struct mlx5_lag *ldev, fib_dev = fib_info_nh(fi, i)->fib_nh_dev; ldev_idx = mlx5_lag_dev_get_netdev_idx(ldev, fib_dev); if (ldev_idx >= 0) - return ldev->pf[ldev_idx].netdev; + return mlx5_lag_pf(ldev, ldev_idx)->netdev; } return NULL; @@ -178,7 +178,7 @@ static void mlx5_lag_fib_route_event(struct mlx5_lag *ldev, unsigned long event, mp->fib.dst_len <= fen_info->dst_len && !(mp->fib.dst_len == fen_info->dst_len && fi->fib_priority < mp->fib.priority)) { - mlx5_core_dbg(ldev->pf[idx].dev, + mlx5_core_dbg(mlx5_lag_pf(ldev, idx)->dev, "Multipath entry with lower priority was rejected\n"); return; } @@ -194,7 +194,7 @@ static void mlx5_lag_fib_route_event(struct mlx5_lag *ldev, unsigned long event, } if (nh_dev0 == nh_dev1) { - mlx5_core_warn(ldev->pf[idx].dev, + mlx5_core_warn(mlx5_lag_pf(ldev, idx)->dev, "Multipath offload doesn't support routes with multiple nexthops of the same device"); return; } @@ -203,7 +203,7 @@ static void mlx5_lag_fib_route_event(struct mlx5_lag *ldev, unsigned long event, if (__mlx5_lag_is_active(ldev)) { mlx5_ldev_for_each(i, 0, ldev) { dev_idx++; - if (ldev->pf[i].netdev == nh_dev0) + if (mlx5_lag_pf(ldev, i)->netdev == nh_dev0) break; } mlx5_lag_set_port_affinity(ldev, dev_idx); @@ -240,7 +240,7 @@ static void mlx5_lag_fib_nexthop_event(struct mlx5_lag *ldev, /* nh added/removed */ if (event == FIB_EVENT_NH_DEL) { mlx5_ldev_for_each(i, 0, ldev) { - if (ldev->pf[i].netdev == fib_nh->fib_nh_dev) + if (mlx5_lag_pf(ldev, i)->netdev == fib_nh->fib_nh_dev) break; dev_idx++; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lag/mpesw.c b/drivers/net/ethernet/mellanox/mlx5/core/lag/mpesw.c index 74d5c2ed14ff..5eea12a6887a 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lag/mpesw.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lag/mpesw.c @@ -16,7 +16,7 @@ static void mlx5_mpesw_metadata_cleanup(struct mlx5_lag *ldev) int i; mlx5_ldev_for_each(i, 0, ldev) { - dev = ldev->pf[i].dev; + dev = mlx5_lag_pf(ldev, i)->dev; esw = dev->priv.eswitch; pf_metadata = ldev->lag_mpesw.pf_metadata[i]; if (!pf_metadata) @@ -37,7 +37,7 @@ static int mlx5_mpesw_metadata_set(struct mlx5_lag *ldev) int i, err; mlx5_ldev_for_each(i, 0, ldev) { - dev = ldev->pf[i].dev; + dev = mlx5_lag_pf(ldev, i)->dev; esw = dev->priv.eswitch; pf_metadata = mlx5_esw_match_metadata_alloc(esw); if (!pf_metadata) { @@ -53,7 +53,7 @@ static int mlx5_mpesw_metadata_set(struct mlx5_lag *ldev) } mlx5_ldev_for_each(i, 0, ldev) { - dev = ldev->pf[i].dev; + dev = mlx5_lag_pf(ldev, i)->dev; mlx5_notifier_call_chain(dev->priv.events, MLX5_DEV_EVENT_MULTIPORT_ESW, (void *)0); } @@ -67,9 +67,9 @@ err_metadata: static int mlx5_lag_enable_mpesw(struct mlx5_lag *ldev) { + int idx = mlx5_lag_get_dev_index_by_seq(ldev, MLX5_LAG_P1); struct mlx5_core_dev *dev0; int err; - int idx; int i; if (ldev->mode == MLX5_LAG_MODE_MPESW) @@ -78,11 +78,10 @@ static int mlx5_lag_enable_mpesw(struct mlx5_lag *ldev) if (ldev->mode != MLX5_LAG_MODE_NONE) return -EINVAL; - idx = mlx5_lag_get_dev_index_by_seq(ldev, MLX5_LAG_P1); if (idx < 0) return -EINVAL; - dev0 = ldev->pf[idx].dev; + dev0 = mlx5_lag_pf(ldev, idx)->dev; if (mlx5_eswitch_mode(dev0) != MLX5_ESWITCH_OFFLOADS || !MLX5_CAP_PORT_SELECTION(dev0, port_select_flow_table) || !MLX5_CAP_GEN(dev0, create_lag_when_not_master_up) || @@ -105,7 +104,7 @@ static int mlx5_lag_enable_mpesw(struct mlx5_lag *ldev) dev0->priv.flags &= ~MLX5_PRIV_FLAGS_DISABLE_IB_ADEV; mlx5_rescan_drivers_locked(dev0); mlx5_ldev_for_each(i, 0, ldev) { - err = mlx5_eswitch_reload_ib_reps(ldev->pf[i].dev->priv.eswitch); + err = mlx5_eswitch_reload_ib_reps(mlx5_lag_pf(ldev, i)->dev->priv.eswitch); if (err) goto err_rescan_drivers; } @@ -121,7 +120,7 @@ err_rescan_drivers: err_add_devices: mlx5_lag_add_devices(ldev); mlx5_ldev_for_each(i, 0, ldev) - mlx5_eswitch_reload_ib_reps(ldev->pf[i].dev->priv.eswitch); + mlx5_eswitch_reload_ib_reps(mlx5_lag_pf(ldev, i)->dev->priv.eswitch); mlx5_mpesw_metadata_cleanup(ldev); return err; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lag/port_sel.c b/drivers/net/ethernet/mellanox/mlx5/core/lag/port_sel.c index 16c7d16215c4..2a034b2a3eee 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lag/port_sel.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lag/port_sel.c @@ -50,7 +50,7 @@ static int mlx5_lag_create_port_sel_table(struct mlx5_lag *ldev, if (first_idx < 0) return -EINVAL; - dev = ldev->pf[first_idx].dev; + dev = mlx5_lag_pf(ldev, first_idx)->dev; ft_attr.max_fte = ldev->ports * ldev->buckets; ft_attr.level = MLX5_LAG_FT_LEVEL_DEFINER; @@ -84,8 +84,12 @@ static int mlx5_lag_create_port_sel_table(struct mlx5_lag *ldev, idx = i * ldev->buckets + j; affinity = ports[idx]; - dest.vport.vhca_id = MLX5_CAP_GEN(ldev->pf[affinity - 1].dev, - vhca_id); + /* affinity is 1-indexed device index, + * use reverse lookup. + */ + dest.vport.vhca_id = + MLX5_CAP_GEN(mlx5_lag_pf_by_dev_idx(ldev, affinity - 1)->dev, + vhca_id); lag_definer->rules[idx] = mlx5_add_flow_rules(lag_definer->ft, NULL, &flow_act, &dest, 1); @@ -307,7 +311,7 @@ mlx5_lag_create_definer(struct mlx5_lag *ldev, enum netdev_lag_hash hash, if (first_idx < 0) return ERR_PTR(-EINVAL); - dev = ldev->pf[first_idx].dev; + dev = mlx5_lag_pf(ldev, first_idx)->dev; lag_definer = kzalloc_obj(*lag_definer); if (!lag_definer) return ERR_PTR(-ENOMEM); @@ -356,8 +360,8 @@ static void mlx5_lag_destroy_definer(struct mlx5_lag *ldev, if (first_idx < 0) return; - dev = ldev->pf[first_idx].dev; - mlx5_ldev_for_each(i, first_idx, ldev) { + dev = mlx5_lag_pf(ldev, first_idx)->dev; + mlx5_ldev_for_each(i, 0, ldev) { for (j = 0; j < ldev->buckets; j++) { idx = i * ldev->buckets + j; mlx5_del_flow_rules(lag_definer->rules[idx]); @@ -520,7 +524,7 @@ static int mlx5_lag_create_ttc_table(struct mlx5_lag *ldev) if (first_idx < 0) return -EINVAL; - dev = ldev->pf[first_idx].dev; + dev = mlx5_lag_pf(ldev, first_idx)->dev; mlx5_lag_set_outer_ttc_params(ldev, &ttc_params); port_sel->outer.ttc = mlx5_create_ttc_table(dev, &ttc_params); return PTR_ERR_OR_ZERO(port_sel->outer.ttc); @@ -536,7 +540,7 @@ static int mlx5_lag_create_inner_ttc_table(struct mlx5_lag *ldev) if (first_idx < 0) return -EINVAL; - dev = ldev->pf[first_idx].dev; + dev = mlx5_lag_pf(ldev, first_idx)->dev; mlx5_lag_set_inner_ttc_params(ldev, &ttc_params); port_sel->inner.ttc = mlx5_create_inner_ttc_table(dev, &ttc_params); return PTR_ERR_OR_ZERO(port_sel->inner.ttc); @@ -594,8 +598,12 @@ static int __mlx5_lag_modify_definers_destinations(struct mlx5_lag *ldev, if (ldev->v2p_map[idx] == ports[idx]) continue; - dest.vport.vhca_id = MLX5_CAP_GEN(ldev->pf[ports[idx] - 1].dev, - vhca_id); + /* ports[] contains 1-indexed device indices, + * use reverse lookup. + */ + dest.vport.vhca_id = + MLX5_CAP_GEN(mlx5_lag_pf_by_dev_idx(ldev, ports[idx] - 1)->dev, + vhca_id); err = mlx5_modify_rule_destination(def->rules[idx], &dest, NULL); if (err) return err; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/clock.c b/drivers/net/ethernet/mellanox/mlx5/core/lib/clock.c index bd4e042077af..d785f1b4f2e1 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/clock.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/clock.c @@ -38,10 +38,10 @@ #include "lib/eq.h" #include "en.h" #include "clock.h" -#ifdef CONFIG_X86 +#if defined(CONFIG_X86) || defined(CONFIG_ARM_ARCH_TIMER) #include <linux/timekeeping.h> #include <linux/cpufeature.h> -#endif /* CONFIG_X86 */ +#endif /* CONFIG_X86 || CONFIG_ARM_ARCH_TIMER */ #define MLX5_RT_CLOCK_IDENTITY_SIZE MLX5_FLD_SZ_BYTES(mrtcq_reg, rt_clock_identity) @@ -229,7 +229,7 @@ static int mlx5_set_mtutc(struct mlx5_core_dev *dev, u32 *mtutc, u32 size) MLX5_REG_MTUTC, 0, 1); } -#ifdef CONFIG_X86 +#if defined(CONFIG_X86) || defined(CONFIG_ARM_ARCH_TIMER) static bool mlx5_is_ptm_source_time_available(struct mlx5_core_dev *dev) { u32 out[MLX5_ST_SZ_DW(mtptm_reg)] = {0}; @@ -275,7 +275,8 @@ static int mlx5_mtctr_read(struct mlx5_core_dev *mdev, host = MLX5_GET64(mtctr_reg, out, first_clock_timestamp); *sys_counterval = (struct system_counterval_t) { .cycles = host, - .cs_id = CSID_X86_ART, + .cs_id = IS_ENABLED(CONFIG_X86) ? CSID_X86_ART : + CSID_ARM_ARCH_COUNTER, .use_nsecs = true, }; *device = MLX5_GET64(mtctr_reg, out, second_clock_timestamp); @@ -373,7 +374,7 @@ unlock: mlx5_clock_unlock(clock); return err; } -#endif /* CONFIG_X86 */ +#endif /* CONFIG_X86 || CONFIG_ARM_ARCH_TIMER */ static u64 mlx5_read_time(struct mlx5_core_dev *dev, struct ptp_system_timestamp *sts, @@ -1301,6 +1302,25 @@ static void mlx5_init_timer_max_freq_adjustment(struct mlx5_core_dev *mdev) min(S32_MAX, 1 << log_max_freq_adjustment); } +static void mlx5_init_crosststamp(struct mlx5_core_dev *mdev, + bool expose_cycles, struct mlx5_clock *clock) +{ +#if defined(CONFIG_X86) + if (!boot_cpu_has(X86_FEATURE_ART)) + return; +#endif /* CONFIG_X86 */ +#if defined(CONFIG_X86) || defined(CONFIG_ARM_ARCH_TIMER) + if (!MLX5_CAP_MCAM_REG3(mdev, mtptm) || + !MLX5_CAP_MCAM_REG3(mdev, mtctr)) + return; + + clock->ptp_info.getcrosststamp = mlx5_ptp_getcrosststamp; + if (expose_cycles) + clock->ptp_info.getcrosscycles = mlx5_ptp_getcrosscycles; + +#endif /* CONFIG_X86 || CONFIG_ARM_ARCH_TIMER */ +} + static void mlx5_init_timer_clock(struct mlx5_core_dev *mdev) { struct mlx5_clock *clock = mdev->clock; @@ -1315,15 +1335,7 @@ static void mlx5_init_timer_clock(struct mlx5_core_dev *mdev) expose_cycles = !MLX5_CAP_GEN(mdev, disciplined_fr_counter) || !mlx5_real_time_mode(mdev); -#ifdef CONFIG_X86 - if (MLX5_CAP_MCAM_REG3(mdev, mtptm) && - MLX5_CAP_MCAM_REG3(mdev, mtctr) && boot_cpu_has(X86_FEATURE_ART)) { - clock->ptp_info.getcrosststamp = mlx5_ptp_getcrosststamp; - if (expose_cycles) - clock->ptp_info.getcrosscycles = - mlx5_ptp_getcrosscycles; - } -#endif /* CONFIG_X86 */ + mlx5_init_crosststamp(mdev, expose_cycles, clock); if (expose_cycles) clock->ptp_info.getcyclesx64 = mlx5_ptp_getcyclesx; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/sd.c b/drivers/net/ethernet/mellanox/mlx5/core/lib/sd.c index 954942ad93c5..762c783156b4 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/sd.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/sd.c @@ -107,7 +107,7 @@ static bool mlx5_sd_is_supported(struct mlx5_core_dev *dev, u8 host_buses) /* Disconnect secondaries from the network */ if (!MLX5_CAP_GEN(dev, eswitch_manager)) return false; - if (!MLX5_CAP_GEN(dev, silent_mode)) + if (!MLX5_CAP_GEN(dev, silent_mode_set)) return false; /* RX steering from primary to secondaries */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/main.c b/drivers/net/ethernet/mellanox/mlx5/core/main.c index 3f73d9b1115d..b4501cdc2351 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/main.c @@ -74,6 +74,7 @@ #include "mlx5_irq.h" #include "hwmon.h" #include "lag/lag.h" +#include "sh_devlink.h" MODULE_AUTHOR("Eli Cohen <eli@mellanox.com>"); MODULE_DESCRIPTION("Mellanox 5th generation network adapters (ConnectX series) core driver"); @@ -950,8 +951,7 @@ static int mlx5_pci_init(struct mlx5_core_dev *dev, struct pci_dev *pdev, pci_enable_atomic_ops_to_root(pdev, PCI_EXP_DEVCAP2_ATOMIC_COMP128)) mlx5_core_dbg(dev, "Enabling pci atomics failed\n"); - dev->iseg_base = dev->bar_addr; - dev->iseg = ioremap(dev->iseg_base, sizeof(*dev->iseg)); + dev->iseg = ioremap(dev->bar_addr, sizeof(*dev->iseg)); if (!dev->iseg) { err = -ENOMEM; mlx5_core_err(dev, "Failed mapping initialization segment, aborting\n"); @@ -1520,10 +1520,16 @@ int mlx5_init_one(struct mlx5_core_dev *dev) int err; devl_lock(devlink); + if (dev->shd) { + err = devl_nested_devlink_set(dev->shd, devlink); + if (err) + goto unlock; + } devl_register(devlink); err = mlx5_init_one_devl_locked(dev); if (err) devl_unregister(devlink); +unlock: devl_unlock(devlink); return err; } @@ -1772,6 +1778,7 @@ static const int types[] = { MLX5_CAP_CRYPTO, MLX5_CAP_SHAMPO, MLX5_CAP_ADV_RDMA, + MLX5_CAP_TLP_EMULATION, }; static void mlx5_hca_caps_free(struct mlx5_core_dev *dev) @@ -2005,6 +2012,13 @@ static int probe_one(struct pci_dev *pdev, const struct pci_device_id *id) goto pci_init_err; } + err = mlx5_shd_init(dev); + if (err) { + mlx5_core_err(dev, "mlx5_shd_init failed with error code %d\n", + err); + goto shd_init_err; + } + err = mlx5_init_one(dev); if (err) { mlx5_core_err(dev, "mlx5_init_one failed with error code %d\n", @@ -2018,6 +2032,8 @@ static int probe_one(struct pci_dev *pdev, const struct pci_device_id *id) return 0; err_init_one: + mlx5_shd_uninit(dev); +shd_init_err: mlx5_pci_close(dev); pci_init_err: mlx5_mdev_uninit(dev); @@ -2039,6 +2055,7 @@ static void remove_one(struct pci_dev *pdev) mlx5_drain_health_wq(dev); mlx5_sriov_disable(pdev, false); mlx5_uninit_one(dev); + mlx5_shd_uninit(dev); mlx5_pci_close(dev); mlx5_mdev_uninit(dev); mlx5_adev_idx_free(dev->priv.adev_idx); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/pagealloc.c b/drivers/net/ethernet/mellanox/mlx5/core/pagealloc.c index 5ccb3ce98acb..77ffa31cc505 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/pagealloc.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/pagealloc.c @@ -77,7 +77,8 @@ static u32 get_function(u16 func_id, bool ec_function) static u16 func_id_to_type(struct mlx5_core_dev *dev, u16 func_id, bool ec_function) { if (!func_id) - return mlx5_core_is_ecpf(dev) && !ec_function ? MLX5_HOST_PF : MLX5_PF; + return mlx5_core_is_ecpf(dev) && !ec_function ? + MLX5_HOST_PF : MLX5_SELF; if (func_id <= max(mlx5_core_max_vfs(dev), mlx5_core_max_ec_vfs(dev))) { if (ec_function) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/sf/dev/driver.c b/drivers/net/ethernet/mellanox/mlx5/core/sf/dev/driver.c index c45540fe7d9d..4391ef0bab5d 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/sf/dev/driver.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/sf/dev/driver.c @@ -37,7 +37,6 @@ static int mlx5_sf_dev_probe(struct auxiliary_device *adev, const struct auxilia mdev->device = &adev->dev; mdev->pdev = sf_dev->parent_mdev->pdev; mdev->bar_addr = sf_dev->bar_base_addr; - mdev->iseg_base = sf_dev->bar_base_addr; mdev->coredev_type = MLX5_COREDEV_SF; mdev->priv.parent_mdev = sf_dev->parent_mdev; mdev->priv.adev_idx = adev->id; @@ -53,7 +52,7 @@ static int mlx5_sf_dev_probe(struct auxiliary_device *adev, const struct auxilia goto mdev_err; } - mdev->iseg = ioremap(mdev->iseg_base, sizeof(*mdev->iseg)); + mdev->iseg = ioremap(mdev->bar_addr, sizeof(*mdev->iseg)); if (!mdev->iseg) { mlx5_core_warn(mdev, "remap error\n"); err = -ENOMEM; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/sh_devlink.c b/drivers/net/ethernet/mellanox/mlx5/core/sh_devlink.c new file mode 100644 index 000000000000..b925364765ac --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/sh_devlink.c @@ -0,0 +1,61 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB +/* Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. */ + +#include <linux/mlx5/driver.h> +#include <net/devlink.h> + +#include "sh_devlink.h" + +static const struct devlink_ops mlx5_shd_ops = { +}; + +int mlx5_shd_init(struct mlx5_core_dev *dev) +{ + u8 *vpd_data __free(kfree) = NULL; + struct pci_dev *pdev = dev->pdev; + unsigned int vpd_size, kw_len; + struct devlink *devlink; + char *sn, *end; + int start; + int err; + + if (!mlx5_core_is_pf(dev)) + return 0; + + vpd_data = pci_vpd_alloc(pdev, &vpd_size); + if (IS_ERR(vpd_data)) { + err = PTR_ERR(vpd_data); + return err == -ENODEV ? 0 : err; + } + start = pci_vpd_find_ro_info_keyword(vpd_data, vpd_size, "V3", &kw_len); + if (start < 0) { + /* Fall-back to SN for older devices. */ + start = pci_vpd_find_ro_info_keyword(vpd_data, vpd_size, + PCI_VPD_RO_KEYWORD_SERIALNO, &kw_len); + if (start < 0) + return 0; /* No usable serial number found, ignore. */ + } + sn = kstrndup(vpd_data + start, kw_len, GFP_KERNEL); + if (!sn) + return -ENOMEM; + /* Firmware may return spaces at the end of the string, strip it. */ + end = strchrnul(sn, ' '); + *end = '\0'; + + /* Get or create shared devlink instance */ + devlink = devlink_shd_get(sn, &mlx5_shd_ops, 0, pdev->dev.driver); + kfree(sn); + if (!devlink) + return -ENOMEM; + + dev->shd = devlink; + return 0; +} + +void mlx5_shd_uninit(struct mlx5_core_dev *dev) +{ + if (!dev->shd) + return; + + devlink_shd_put(dev->shd); +} diff --git a/drivers/net/ethernet/mellanox/mlx5/core/sh_devlink.h b/drivers/net/ethernet/mellanox/mlx5/core/sh_devlink.h new file mode 100644 index 000000000000..8ab8d6940227 --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/sh_devlink.h @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ +/* Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. */ + +#ifndef __MLX5_SH_DEVLINK_H__ +#define __MLX5_SH_DEVLINK_H__ + +#include <linux/mlx5/driver.h> + +int mlx5_shd_init(struct mlx5_core_dev *dev); +void mlx5_shd_uninit(struct mlx5_core_dev *dev); + +#endif /* __MLX5_SH_DEVLINK_H__ */ diff --git a/drivers/net/ethernet/mellanox/mlxsw/Kconfig b/drivers/net/ethernet/mellanox/mlxsw/Kconfig index 74f7e27b490f..2229a2694aa5 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/Kconfig +++ b/drivers/net/ethernet/mellanox/mlxsw/Kconfig @@ -56,7 +56,6 @@ config MLXSW_SPECTRUM depends on MLXSW_CORE && MLXSW_PCI && NET_SWITCHDEV && VLAN_8021Q depends on PSAMPLE || PSAMPLE=n depends on BRIDGE || BRIDGE=n - depends on IPV6 || IPV6=n depends on NET_IPGRE || NET_IPGRE=n depends on IPV6_GRE || IPV6_GRE=n depends on VXLAN || VXLAN=n diff --git a/drivers/net/ethernet/meta/fbnic/fbnic.h b/drivers/net/ethernet/meta/fbnic/fbnic.h index 779a083b9215..d0715695c43e 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic.h +++ b/drivers/net/ethernet/meta/fbnic/fbnic.h @@ -57,6 +57,7 @@ struct fbnic_dev { u64 dsn; u32 mps; u32 readrq; + u8 relaxed_ord; /* Local copy of the devices TCAM */ struct fbnic_act_tcam act_tcam[FBNIC_RPC_TCAM_ACT_NUM_ENTRIES]; @@ -98,6 +99,9 @@ struct fbnic_dev { /* MDIO bus for PHYs */ struct mii_bus *mdio_bus; + + /* In units of ms since API supports values in ms */ + u16 ps_timeout; }; /* Reserve entry 0 in the MSI-X "others" array until we have filled all @@ -194,6 +198,38 @@ void fbnic_synchronize_irq(struct fbnic_dev *fbd, int nr); int fbnic_request_irq(struct fbnic_dev *dev, int nr, irq_handler_t handler, unsigned long flags, const char *name, void *data); void fbnic_free_irq(struct fbnic_dev *dev, int nr, void *data); + +/** + * enum fbnic_msix_self_test_codes - return codes from self test routines + * + * These are the codes returned from the self test routines and + * stored in the test result array indexed by the specific + * test name. + * + * @FBNIC_TEST_MSIX_SUCCESS: no errors + * @FBNIC_TEST_MSIX_NOMEM: allocation failure + * @FBNIC_TEST_MSIX_IRQ_REQ_FAIL: IRQ request failure + * @FBNIC_TEST_MSIX_MASK: masking failed to prevent IRQ + * @FBNIC_TEST_MSIX_UNMASK: unmasking failure w/ sw status set + * @FBNIC_TEST_MSIX_IRQ_CLEAR: interrupt when clearing mask + * @FBNIC_TEST_MSIX_NO_INTERRUPT: no interrupt when not masked + * @FBNIC_TEST_MSIX_NO_CLEAR_OR_MASK: status not cleared, or mask not set + * @FBNIC_TEST_MSIX_BITS_SET_AFTER_TEST: Bits are set after test + */ +enum fbnic_msix_self_test_codes { + FBNIC_TEST_MSIX_SUCCESS = 0, + FBNIC_TEST_MSIX_NOMEM = 5, + FBNIC_TEST_MSIX_IRQ_REQ_FAIL = 10, + FBNIC_TEST_MSIX_MASK = 20, + FBNIC_TEST_MSIX_UNMASK = 30, + FBNIC_TEST_MSIX_IRQ_CLEAR = 40, + FBNIC_TEST_MSIX_NO_INTERRUPT = 50, + FBNIC_TEST_MSIX_NO_CLEAR_OR_MASK = 60, + FBNIC_TEST_MSIX_BITS_SET_AFTER_TEST = 70, +}; + +enum fbnic_msix_self_test_codes fbnic_msix_test(struct fbnic_dev *fbd); + void fbnic_free_irqs(struct fbnic_dev *fbd); int fbnic_alloc_irqs(struct fbnic_dev *fbd); diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_csr.c b/drivers/net/ethernet/meta/fbnic/fbnic_csr.c index d9c0dc1c2af9..dc62d623e37c 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_csr.c +++ b/drivers/net/ethernet/meta/fbnic/fbnic_csr.c @@ -147,3 +147,131 @@ int fbnic_csr_regs_len(struct fbnic_dev *fbd) return len; } + +/* CSR register test data + * + * The register test will be used to verify hardware is behaving as expected. + * + * The test itself will have us writing to registers that should have no + * side effects due to us resetting after the test has been completed. + * While the test is being run the interface should be offline. + */ +struct fbnic_csr_reg_test_data { + int reg; + u16 reg_offset; + u8 array_len; + u32 read; + u32 write; +}; + +#define FBNIC_QUEUE_REG_TEST(_name, _read, _write) { \ + .reg = FBNIC_QUEUE(0) + FBNIC_QUEUE_##_name, \ + .reg_offset = FBNIC_QUEUE_STRIDE, \ + .array_len = 64, \ + .read = _read, \ + .write = _write \ +} + +static const struct fbnic_csr_reg_test_data pattern_test[] = { + FBNIC_QUEUE_REG_TEST(TWQ0_CTL, FBNIC_QUEUE_TWQ_CTL_RESET, + FBNIC_QUEUE_TWQ_CTL_RESET), + FBNIC_QUEUE_REG_TEST(TWQ0_PTRS, 0, ~0), + FBNIC_QUEUE_REG_TEST(TWQ0_SIZE, FBNIC_QUEUE_TWQ_SIZE_MASK, ~0), + FBNIC_QUEUE_REG_TEST(TWQ0_BAL, FBNIC_QUEUE_BAL_MASK, ~0), + FBNIC_QUEUE_REG_TEST(TWQ0_BAH, ~0, ~0), + FBNIC_QUEUE_REG_TEST(TWQ1_CTL, FBNIC_QUEUE_TWQ_CTL_RESET, + FBNIC_QUEUE_TWQ_CTL_RESET), + FBNIC_QUEUE_REG_TEST(TWQ1_PTRS, 0, ~0), + FBNIC_QUEUE_REG_TEST(TWQ1_SIZE, FBNIC_QUEUE_TWQ_SIZE_MASK, ~0), + FBNIC_QUEUE_REG_TEST(TWQ1_BAL, FBNIC_QUEUE_BAL_MASK, ~0), + FBNIC_QUEUE_REG_TEST(TWQ1_BAH, ~0, ~0), + FBNIC_QUEUE_REG_TEST(TCQ_CTL, FBNIC_QUEUE_TCQ_CTL_RESET, + FBNIC_QUEUE_TCQ_CTL_RESET), + FBNIC_QUEUE_REG_TEST(TCQ_PTRS, 0, ~0), + FBNIC_QUEUE_REG_TEST(TCQ_SIZE, FBNIC_QUEUE_TCQ_SIZE_MASK, ~0), + FBNIC_QUEUE_REG_TEST(TCQ_BAL, FBNIC_QUEUE_BAL_MASK, ~0), + FBNIC_QUEUE_REG_TEST(TCQ_BAH, ~0, ~0), + FBNIC_QUEUE_REG_TEST(RCQ_CTL, FBNIC_QUEUE_RCQ_CTL_RESET, + FBNIC_QUEUE_RCQ_CTL_RESET), + FBNIC_QUEUE_REG_TEST(RCQ_PTRS, 0, ~0), + FBNIC_QUEUE_REG_TEST(RCQ_SIZE, FBNIC_QUEUE_RCQ_SIZE_MASK, ~0), + FBNIC_QUEUE_REG_TEST(RCQ_BAL, FBNIC_QUEUE_BAL_MASK, ~0), + FBNIC_QUEUE_REG_TEST(RCQ_BAH, ~0, ~0), + FBNIC_QUEUE_REG_TEST(BDQ_CTL, FBNIC_QUEUE_BDQ_CTL_RESET, + FBNIC_QUEUE_BDQ_CTL_RESET), + FBNIC_QUEUE_REG_TEST(BDQ_HPQ_PTRS, 0, ~0), + FBNIC_QUEUE_REG_TEST(BDQ_HPQ_SIZE, FBNIC_QUEUE_BDQ_SIZE_MASK, ~0), + FBNIC_QUEUE_REG_TEST(BDQ_HPQ_BAL, FBNIC_QUEUE_BAL_MASK, ~0), + FBNIC_QUEUE_REG_TEST(BDQ_HPQ_BAH, ~0, ~0), + FBNIC_QUEUE_REG_TEST(BDQ_PPQ_PTRS, 0, ~0), + FBNIC_QUEUE_REG_TEST(BDQ_PPQ_SIZE, FBNIC_QUEUE_BDQ_SIZE_MASK, ~0), + FBNIC_QUEUE_REG_TEST(BDQ_PPQ_BAL, FBNIC_QUEUE_BAL_MASK, ~0), + FBNIC_QUEUE_REG_TEST(BDQ_PPQ_BAH, ~0, ~0), +}; + +static enum fbnic_reg_self_test_codes +fbnic_csr_reg_pattern_test(struct fbnic_dev *fbd, int index, + const struct fbnic_csr_reg_test_data *test_data) +{ + static const u32 pattern[] = { ~0, 0x5A5A5A5A, 0xA5A5A5A5, 0}; + enum fbnic_reg_self_test_codes reg; + int i; + + reg = test_data->reg + test_data->reg_offset * index; + for (i = 0; i < ARRAY_SIZE(pattern); i++) { + u32 val = pattern[i] & test_data->write; + u32 result; + + wr32(fbd, reg, val); + result = rd32(fbd, reg); + val &= test_data->read; + + if (result == val) + continue; + + dev_err(fbd->dev, + "%s: reg 0x%06X failed, expected 0x%08X received 0x%08X\n", + __func__, reg, val, result); + + /* Note that FBNIC_INTR_STATUS(0) could be tested and fail + * and the result would not be reported since the register + * offset is 0. However as that register isn't included in + * the register test that isn't an issue. + */ + return reg; + } + + return FBNIC_REG_TEST_SUCCESS; +} + +/** + * fbnic_csr_regs_test() - Verify behavior of NIC registers + * @fbd: device to test + * + * This function is meant to test the bit values of various registers in + * the NIC device. Specifically this test will verify which bits are + * writable and which ones are not. It will write varying patterns of bits + * to the registers testing for sticky bits, or bits that are writable but + * should not be. + * + * Return: FBNIC_REG_TEST_SUCCESS on success, register number on failure + **/ +enum fbnic_reg_self_test_codes fbnic_csr_regs_test(struct fbnic_dev *fbd) +{ + const struct fbnic_csr_reg_test_data *test_data; + + for (test_data = pattern_test; + test_data < pattern_test + ARRAY_SIZE(pattern_test); test_data++) { + u32 i; + + for (i = 0; i < test_data->array_len; i++) { + enum fbnic_reg_self_test_codes reg = + fbnic_csr_reg_pattern_test(fbd, i, test_data); + + if (reg) + return reg; + } + } + + return FBNIC_REG_TEST_SUCCESS; +} diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_csr.h b/drivers/net/ethernet/meta/fbnic/fbnic_csr.h index b717db879cd3..81794bd326e1 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_csr.h +++ b/drivers/net/ethernet/meta/fbnic/fbnic_csr.h @@ -6,6 +6,8 @@ #include <linux/bitops.h> +struct fbnic_dev; + #define CSR_BIT(nr) (1u << (nr)) #define CSR_GENMASK(h, l) GENMASK(h, l) @@ -230,6 +232,7 @@ enum { #define FBNIC_INTR_MSIX_CTRL_VECTOR_MASK CSR_GENMASK(7, 0) #define FBNIC_INTR_MSIX_CTRL_ENABLE CSR_BIT(31) enum { + FBNIC_INTR_MSIX_CTRL_RXB_IDX = 7, FBNIC_INTR_MSIX_CTRL_PCS_IDX = 34, }; @@ -560,6 +563,11 @@ enum { #define FBNIC_RXB_DROP_THLD_CNT 8 #define FBNIC_RXB_DROP_THLD_ON CSR_GENMASK(12, 0) #define FBNIC_RXB_DROP_THLD_OFF CSR_GENMASK(25, 13) +#define FBNIC_RXB_PAUSE_STORM(n) (0x08019 + (n)) /* 0x20064 + 4*n */ +#define FBNIC_RXB_PAUSE_STORM_CNT 4 +#define FBNIC_RXB_PAUSE_STORM_FORCE_NORMAL CSR_BIT(20) +#define FBNIC_RXB_PAUSE_STORM_THLD_TIME CSR_GENMASK(19, 0) +#define FBNIC_RXB_PAUSE_STORM_UNIT_WR 0x0801d /* 0x20074 */ #define FBNIC_RXB_ECN_THLD(n) (0x0801e + (n)) /* 0x20078 + 4*n */ #define FBNIC_RXB_ECN_THLD_CNT 8 #define FBNIC_RXB_ECN_THLD_ON CSR_GENMASK(12, 0) @@ -596,6 +604,9 @@ enum { #define FBNIC_RXB_INTF_CREDIT_MASK2 CSR_GENMASK(11, 8) #define FBNIC_RXB_INTF_CREDIT_MASK3 CSR_GENMASK(15, 12) +#define FBNIC_RXB_ERR_INTR_STS 0x08050 /* 0x20140 */ +#define FBNIC_RXB_ERR_INTR_STS_PS CSR_GENMASK(15, 12) +#define FBNIC_RXB_ERR_INTR_MASK 0x08052 /* 0x20148 */ #define FBNIC_RXB_PAUSE_EVENT_CNT(n) (0x08053 + (n)) /* 0x2014c + 4*n */ #define FBNIC_RXB_DROP_FRMS_STS(n) (0x08057 + (n)) /* 0x2015c + 4*n */ #define FBNIC_RXB_DROP_BYTES_STS_L(n) \ @@ -618,6 +629,7 @@ enum { FBNIC_RXB_ENQUEUE_INDICES = 4 }; +#define FBNIC_RXB_INTR_PS_COUNT(n) (0x080e9 + (n)) /* 0x203a4 + 4*n */ #define FBNIC_RXB_DRBO_FRM_CNT_SRC(n) (0x080f9 + (n)) /* 0x203e4 + 4*n */ #define FBNIC_RXB_DRBO_BYTE_CNT_SRC_L(n) \ (0x080fd + (n)) /* 0x203f4 + 4*n */ @@ -636,6 +648,7 @@ enum { #define FBNIC_RXB_PBUF_FIFO_LEVEL(n) (0x0811d + (n)) /* 0x20474 + 4*n */ +#define FBNIC_RXB_PAUSE_STORM_UNIT_RD 0x08125 /* 0x20494 */ #define FBNIC_RXB_INTEGRITY_ERR(n) (0x0812f + (n)) /* 0x204bc + 4*n */ #define FBNIC_RXB_MAC_ERR(n) (0x08133 + (n)) /* 0x204cc + 4*n */ #define FBNIC_RXB_PARSER_ERR(n) (0x08137 + (n)) /* 0x204dc + 4*n */ @@ -962,9 +975,21 @@ enum { #define FBNIC_PUL_OB_TLP_HDR_AW_CFG 0x3103d /* 0xc40f4 */ #define FBNIC_PUL_OB_TLP_HDR_AW_CFG_FLUSH CSR_BIT(19) #define FBNIC_PUL_OB_TLP_HDR_AW_CFG_BME CSR_BIT(18) +#define FBNIC_PUL_OB_TLP_HDR_AW_CFG_RDE_ATTR CSR_GENMASK(17, 15) +#define FBNIC_PUL_OB_TLP_HDR_AW_CFG_RQM_ATTR CSR_GENMASK(14, 12) +#define FBNIC_PUL_OB_TLP_HDR_AW_CFG_TQM_ATTR CSR_GENMASK(11, 9) +enum { + FBNIC_TLP_ATTR_NS = 1, + FBNIC_TLP_ATTR_RO = 2, + FBNIC_TLP_ATTR_IDO = 4, +}; + #define FBNIC_PUL_OB_TLP_HDR_AR_CFG 0x3103e /* 0xc40f8 */ #define FBNIC_PUL_OB_TLP_HDR_AR_CFG_FLUSH CSR_BIT(19) #define FBNIC_PUL_OB_TLP_HDR_AR_CFG_BME CSR_BIT(18) +#define FBNIC_PUL_OB_TLP_HDR_AR_CFG_TDE_ATTR CSR_GENMASK(17, 15) +#define FBNIC_PUL_OB_TLP_HDR_AR_CFG_RQM_ATTR CSR_GENMASK(14, 12) +#define FBNIC_PUL_OB_TLP_HDR_AR_CFG_TQM_ATTR CSR_GENMASK(11, 9) #define FBNIC_PUL_USER_OB_RD_TLP_CNT_31_0 \ 0x3106e /* 0xc41b8 */ #define FBNIC_PUL_USER_OB_RD_DWORD_CNT_31_0 \ @@ -1210,4 +1235,21 @@ enum{ FBNIC_CSR_VERSION_V1_0_ASIC = 1, }; +/** + * enum fbnic_reg_self_test_codes - return codes from self test routines + * + * This is the code that is returned from the register self test + * routines. + * + * The test either returns success or the register number + * that failed during the test. + * + * @FBNIC_REG_TEST_SUCCESS: no errors + */ +enum fbnic_reg_self_test_codes { + FBNIC_REG_TEST_SUCCESS = 0, +}; + +enum fbnic_reg_self_test_codes fbnic_csr_regs_test(struct fbnic_dev *fbd); + #endif /* _FBNIC_CSR_H_ */ diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_devlink.c b/drivers/net/ethernet/meta/fbnic/fbnic_devlink.c index f1c992f5fe94..546e1c12d287 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_devlink.c +++ b/drivers/net/ethernet/meta/fbnic/fbnic_devlink.c @@ -647,6 +647,7 @@ struct fbnic_dev *fbnic_devlink_alloc(struct pci_dev *pdev) fbd->dsn = pci_get_dsn(pdev); fbd->mps = pcie_get_mps(pdev); fbd->readrq = pcie_get_readrq(pdev); + fbd->relaxed_ord = pcie_relaxed_ordering_enabled(pdev); fbd->mac_addr_boundary = FBNIC_RPC_TCAM_MACDA_DEFAULT_BOUNDARY; diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_ethtool.c b/drivers/net/ethernet/meta/fbnic/fbnic_ethtool.c index 401c2196b9ff..f14de2366854 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_ethtool.c +++ b/drivers/net/ethernet/meta/fbnic/fbnic_ethtool.c @@ -126,6 +126,20 @@ static const struct fbnic_stat fbnic_gstrings_xdp_stats[] = { #define FBNIC_STATS_LEN \ (FBNIC_HW_STATS_LEN + FBNIC_XDP_STATS_LEN * FBNIC_MAX_XDPQS) +enum fbnic_self_test_results { + TEST_REG = 0, + TEST_MSIX, + TEST_MBX, +}; + +static const char fbnic_gstrings_self_test[][ETH_GSTRING_LEN] = { + [TEST_REG] = "Register test (offline)", + [TEST_MSIX] = "MSI-X Interrupt test (offline)", + [TEST_MBX] = "FW mailbox test (on/offline)", +}; + +#define FBNIC_TEST_LEN ARRAY_SIZE(fbnic_gstrings_self_test) + static void fbnic_get_drvinfo(struct net_device *netdev, struct ethtool_drvinfo *drvinfo) { @@ -475,6 +489,10 @@ static void fbnic_get_strings(struct net_device *dev, u32 sset, u8 *data) for (i = 0; i < FBNIC_MAX_XDPQS; i++) fbnic_get_xdp_queue_strings(&data, i); break; + case ETH_SS_TEST: + memcpy(data, fbnic_gstrings_self_test, + sizeof(fbnic_gstrings_self_test)); + break; } } @@ -566,6 +584,8 @@ static int fbnic_get_sset_count(struct net_device *dev, int sset) switch (sset) { case ETH_SS_STATS: return FBNIC_STATS_LEN; + case ETH_SS_TEST: + return FBNIC_TEST_LEN; default: return -EOPNOTSUPP; } @@ -1478,6 +1498,78 @@ fbnic_remove_rxfh_context(struct net_device *netdev, return 0; } +static int fbnic_ethtool_regs_test(struct net_device *netdev, u64 *data) +{ + struct fbnic_net *fbn = netdev_priv(netdev); + struct fbnic_dev *fbd = fbn->fbd; + + *data = fbnic_csr_regs_test(fbd); + + return !!*data; +} + +/** + * fbnic_ethtool_msix_test - Verify behavior of NIC interrupts + * @netdev: netdev device to test + * @data: Pointer to results storage + * + * This function is meant to test the global interrupt registers and the + * PCIe IP MSI-X functionality. It essentially goes through and tests + * test various combinations of the set, clear, and mask bits in order to + * verify the behavior is as we expect it to be from the driver. + * + * Return: non-zero on failure. + **/ +static int fbnic_ethtool_msix_test(struct net_device *netdev, u64 *data) +{ + struct fbnic_net *fbn = netdev_priv(netdev); + struct fbnic_dev *fbd = fbn->fbd; + + *data = fbnic_msix_test(fbd); + + return !!*data; +} + +static int fbnic_ethtool_mbx_self_test(struct net_device *netdev, u64 *data) +{ + struct fbnic_net *fbn = netdev_priv(netdev); + struct fbnic_dev *fbd = fbn->fbd; + + *data = fbnic_fw_mbx_self_test(fbd); + + return !!*data; +} + +static void fbnic_self_test(struct net_device *netdev, + struct ethtool_test *eth_test, u64 *data) +{ + bool if_running = netif_running(netdev); + + if (fbnic_ethtool_mbx_self_test(netdev, &data[TEST_MBX])) + eth_test->flags |= ETH_TEST_FL_FAILED; + + if (!(eth_test->flags & ETH_TEST_FL_OFFLINE)) { + data[TEST_REG] = 0; + data[TEST_MSIX] = 0; + return; + } + + if (if_running) + netif_close(netdev); + + if (fbnic_ethtool_regs_test(netdev, &data[TEST_REG])) + eth_test->flags |= ETH_TEST_FL_FAILED; + + if (fbnic_ethtool_msix_test(netdev, &data[TEST_MSIX])) + eth_test->flags |= ETH_TEST_FL_FAILED; + + if (if_running && netif_open(netdev, NULL)) { + netdev_err(netdev, + "Failed to re-initialize hardware following test\n"); + eth_test->flags |= ETH_TEST_FL_FAILED; + } +} + static void fbnic_get_channels(struct net_device *netdev, struct ethtool_channels *ch) { @@ -1641,6 +1733,47 @@ static void fbnic_get_ts_stats(struct net_device *netdev, } } +static int fbnic_get_tunable(struct net_device *netdev, + const struct ethtool_tunable *tun, + void *data) +{ + struct fbnic_net *fbn = netdev_priv(netdev); + int err = 0; + + switch (tun->id) { + case ETHTOOL_PFC_PREVENTION_TOUT: + *(u16 *)data = fbn->fbd->ps_timeout; + break; + default: + err = -EOPNOTSUPP; + break; + } + + return err; +} + +static int fbnic_set_tunable(struct net_device *netdev, + const struct ethtool_tunable *tun, + const void *data) +{ + struct fbnic_net *fbn = netdev_priv(netdev); + int err; + + switch (tun->id) { + case ETHTOOL_PFC_PREVENTION_TOUT: { + u16 ps_timeout = *(u16 *)data; + + err = fbnic_mac_ps_protect_to_config(fbn->fbd, ps_timeout); + break; + } + default: + err = -EOPNOTSUPP; + break; + } + + return err; +} + static int fbnic_get_module_eeprom_by_page(struct net_device *netdev, const struct ethtool_module_eeprom *page_data, @@ -1713,6 +1846,7 @@ fbnic_get_pause_stats(struct net_device *netdev, struct fbnic_net *fbn = netdev_priv(netdev); struct fbnic_mac_stats *mac_stats; struct fbnic_dev *fbd = fbn->fbd; + u64 tx_ps_events; mac_stats = &fbd->hw_stats.mac; @@ -1720,6 +1854,8 @@ fbnic_get_pause_stats(struct net_device *netdev, pause_stats->tx_pause_frames = mac_stats->pause.tx_pause_frames.value; pause_stats->rx_pause_frames = mac_stats->pause.rx_pause_frames.value; + tx_ps_events = mac_stats->pause.tx_pause_storm_events.value; + pause_stats->tx_pause_storm_events = tx_ps_events; } static void @@ -1896,6 +2032,7 @@ static const struct ethtool_ops fbnic_ethtool_ops = { .get_pause_stats = fbnic_get_pause_stats, .get_pauseparam = fbnic_phylink_get_pauseparam, .set_pauseparam = fbnic_phylink_set_pauseparam, + .self_test = fbnic_self_test, .get_strings = fbnic_get_strings, .get_ethtool_stats = fbnic_get_ethtool_stats, .get_sset_count = fbnic_get_sset_count, @@ -1915,6 +2052,8 @@ static const struct ethtool_ops fbnic_ethtool_ops = { .set_channels = fbnic_set_channels, .get_ts_info = fbnic_get_ts_info, .get_ts_stats = fbnic_get_ts_stats, + .get_tunable = fbnic_get_tunable, + .set_tunable = fbnic_set_tunable, .get_link_ksettings = fbnic_phylink_ethtool_ksettings_get, .get_fec_stats = fbnic_get_fec_stats, .get_fecparam = fbnic_phylink_get_fecparam, diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_fw.c b/drivers/net/ethernet/meta/fbnic/fbnic_fw.c index 1f0b6350bef4..0c6812fcf185 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_fw.c +++ b/drivers/net/ethernet/meta/fbnic/fbnic_fw.c @@ -379,6 +379,37 @@ fbnic_fw_get_cmpl_by_type(struct fbnic_dev *fbd, u32 msg_type) } /** + * fbnic_fw_xmit_test_msg - Create and transmit a test message to FW mailbox + * @fbd: FBNIC device structure + * @cmpl: fw completion struct + * + * Return: zero on success, negative value on failure + * + * Generates a single page mailbox test message and places it in the Tx + * mailbox queue. Expectation is that the FW will validate that the nested + * value matches the external values, and then will echo them back to us. + * + * Also sets a completion slot for use in the completion wait calls when + * the cmpl arg is non-NULL. + */ +int fbnic_fw_xmit_test_msg(struct fbnic_dev *fbd, + struct fbnic_fw_completion *cmpl) +{ + struct fbnic_tlv_msg *test_msg; + int err; + + test_msg = fbnic_tlv_test_create(fbd); + if (!test_msg) + return -ENOMEM; + + err = fbnic_mbx_map_req_w_cmpl(fbd, test_msg, cmpl); + if (err) + free_page((unsigned long)test_msg); + + return err; +} + +/** * fbnic_fw_xmit_simple_msg - Transmit a simple single TLV message w/o data * @fbd: FBNIC device structure * @msg_type: ENUM value indicating message type to send @@ -417,6 +448,7 @@ static int fbnic_fw_xmit_simple_msg(struct fbnic_dev *fbd, u32 msg_type) static int fbnic_mbx_init_desc_ring(struct fbnic_dev *fbd, int mbx_idx) { + u8 tlp_attr = fbd->relaxed_ord ? FBNIC_TLP_ATTR_RO : 0; struct fbnic_fw_mbx *mbx = &fbd->mbx[mbx_idx]; mbx->ready = true; @@ -425,14 +457,24 @@ static int fbnic_mbx_init_desc_ring(struct fbnic_dev *fbd, int mbx_idx) case FBNIC_IPC_MBX_RX_IDX: /* Enable DMA writes from the device */ wr32(fbd, FBNIC_PUL_OB_TLP_HDR_AW_CFG, - FBNIC_PUL_OB_TLP_HDR_AW_CFG_BME); + FBNIC_PUL_OB_TLP_HDR_AW_CFG_BME | + FIELD_PREP(FBNIC_PUL_OB_TLP_HDR_AW_CFG_RDE_ATTR, + tlp_attr) | + FIELD_PREP(FBNIC_PUL_OB_TLP_HDR_AW_CFG_TQM_ATTR, + tlp_attr)); /* Make sure we have a page for the FW to write to */ return fbnic_mbx_alloc_rx_msgs(fbd); case FBNIC_IPC_MBX_TX_IDX: /* Enable DMA reads from the device */ wr32(fbd, FBNIC_PUL_OB_TLP_HDR_AR_CFG, - FBNIC_PUL_OB_TLP_HDR_AR_CFG_BME); + FBNIC_PUL_OB_TLP_HDR_AR_CFG_BME | + FIELD_PREP(FBNIC_PUL_OB_TLP_HDR_AR_CFG_TDE_ATTR, + tlp_attr) | + FIELD_PREP(FBNIC_PUL_OB_TLP_HDR_AR_CFG_RQM_ATTR, + tlp_attr) | + FIELD_PREP(FBNIC_PUL_OB_TLP_HDR_AR_CFG_TQM_ATTR, + tlp_attr)); break; } @@ -1556,7 +1598,29 @@ free_message: return err; } +static int +fbnic_fw_parser_test(void *opaque, struct fbnic_tlv_msg **results) +{ + struct fbnic_fw_completion *cmpl; + struct fbnic_dev *fbd = opaque; + int err; + + /* find cmpl */ + cmpl = fbnic_fw_get_cmpl_by_type(fbd, FBNIC_TLV_MSG_ID_TEST); + if (!cmpl) + return -ENOSPC; + + err = fbnic_tlv_parser_test(opaque, results); + + cmpl->result = err; + complete(&cmpl->done); + fbnic_fw_put_cmpl(cmpl); + + return err; +} + static const struct fbnic_tlv_parser fbnic_fw_tlv_parser[] = { + FBNIC_TLV_PARSER(TEST, fbnic_tlv_test_index, fbnic_fw_parser_test), FBNIC_TLV_PARSER(FW_CAP_RESP, fbnic_fw_cap_resp_index, fbnic_fw_parse_cap_resp), FBNIC_TLV_PARSER(OWNERSHIP_RESP, fbnic_ownership_resp_index, @@ -1787,6 +1851,53 @@ void fbnic_mbx_flush_tx(struct fbnic_dev *fbd) } while (time_is_after_jiffies(timeout)); } +/** + * fbnic_fw_mbx_self_test() - verify firmware interface + * @fbd: device to test + * + * This function tests the interfaces to/from the firmware. + * + * Return: See enum fbnic_fw_self_test_codes + **/ +enum fbnic_fw_self_test_codes fbnic_fw_mbx_self_test(struct fbnic_dev *fbd) +{ + enum fbnic_fw_self_test_codes err; + struct fbnic_fw_completion *cmpl; + + /* Skip test if FW interface is not present */ + if (!fbnic_fw_present(fbd)) + return FBNIC_TEST_FW_NO_FIRMWARE; + + cmpl = fbnic_fw_alloc_cmpl(FBNIC_TLV_MSG_ID_TEST); + if (!cmpl) + return FBNIC_TEST_FW_NO_CMPL; + + /* Load a test message onto the FW mailbox interface + * and arm the completion. + */ + err = fbnic_fw_xmit_test_msg(fbd, cmpl); + if (err) { + err = FBNIC_TEST_FW_NO_XMIT; + goto exit_free; + } + + /* Verify we received a message back */ + if (!fbnic_mbx_wait_for_cmpl(cmpl)) { + err = FBNIC_TEST_FW_NO_MSG; + goto exit_cleanup; + } + + /* Verify there were no parsing errors */ + if (cmpl->result) + err = FBNIC_TEST_FW_PARSE; +exit_cleanup: + fbnic_mbx_clear_cmpl(fbd, cmpl); +exit_free: + fbnic_fw_put_cmpl(cmpl); + + return err; +} + int fbnic_fw_xmit_rpc_macda_sync(struct fbnic_dev *fbd) { struct fbnic_tlv_msg *mac_array; diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_fw.h b/drivers/net/ethernet/meta/fbnic/fbnic_fw.h index 8f7218900562..d84723e4cfa3 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_fw.h +++ b/drivers/net/ethernet/meta/fbnic/fbnic_fw.h @@ -104,6 +104,33 @@ void fbnic_mbx_clear_cmpl(struct fbnic_dev *fbd, void fbnic_mbx_poll(struct fbnic_dev *fbd); int fbnic_mbx_poll_tx_ready(struct fbnic_dev *fbd); void fbnic_mbx_flush_tx(struct fbnic_dev *fbd); + +/** + * enum fbnic_fw_self_test_codes - return codes from self test routines + * + * These are the codes returned from the self test routines and + * stored in the test result array indexed by the specific + * test name. + * + * @FBNIC_TEST_FW_SUCCESS: test success + * @FBNIC_TEST_FW_NO_FIRMWARE: FW interface not present + * @FBNIC_TEST_FW_NO_CMPL: No completion available + * @FBNIC_TEST_FW_NO_XMIT: Could not xmit message + * @FBNIC_TEST_FW_NO_MSG: no message returned + * @FBNIC_TEST_FW_PARSE: returned message had parsing error + */ +enum fbnic_fw_self_test_codes { + FBNIC_TEST_FW_SUCCESS = 0, + FBNIC_TEST_FW_NO_FIRMWARE = 10, + FBNIC_TEST_FW_NO_CMPL = 20, + FBNIC_TEST_FW_NO_XMIT = 30, + FBNIC_TEST_FW_NO_MSG = 40, + FBNIC_TEST_FW_PARSE = 50, +}; + +enum fbnic_fw_self_test_codes fbnic_fw_mbx_self_test(struct fbnic_dev *fbd); +int fbnic_fw_xmit_test_msg(struct fbnic_dev *fbd, + struct fbnic_fw_completion *c); int fbnic_fw_xmit_ownership_msg(struct fbnic_dev *fbd, bool take_ownership); int fbnic_fw_init_heartbeat(struct fbnic_dev *fbd, bool poll); void fbnic_fw_check_heartbeat(struct fbnic_dev *fbd); diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_hw_stats.h b/drivers/net/ethernet/meta/fbnic/fbnic_hw_stats.h index aa3f429a9aed..caea4be46762 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_hw_stats.h +++ b/drivers/net/ethernet/meta/fbnic/fbnic_hw_stats.h @@ -54,6 +54,7 @@ struct fbnic_rmon_stats { struct fbnic_pause_stats { struct fbnic_stat_counter tx_pause_frames; struct fbnic_stat_counter rx_pause_frames; + struct fbnic_stat_counter tx_pause_storm_events; }; struct fbnic_eth_mac_stats { diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_irq.c b/drivers/net/ethernet/meta/fbnic/fbnic_irq.c index 02e8b0b257fe..5e383d40abc7 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_irq.c +++ b/drivers/net/ethernet/meta/fbnic/fbnic_irq.c @@ -170,6 +170,8 @@ int fbnic_mac_request_irq(struct fbnic_dev *fbd) fbnic_wr32(fbd, FBNIC_INTR_MSIX_CTRL(FBNIC_INTR_MSIX_CTRL_PCS_IDX), FBNIC_PCS_MSIX_ENTRY | FBNIC_INTR_MSIX_CTRL_ENABLE); + fbnic_wr32(fbd, FBNIC_INTR_MSIX_CTRL(FBNIC_INTR_MSIX_CTRL_RXB_IDX), 0); + fbd->mac_msix_vector = vector; return 0; @@ -238,6 +240,160 @@ void fbnic_free_irq(struct fbnic_dev *fbd, int nr, void *data) free_irq(irq, data); } +struct fbnic_msix_test_data { + struct fbnic_dev *fbd; + unsigned long test_msix_status[BITS_TO_LONGS(FBNIC_MAX_MSIX_VECS)]; + int irq_vector[FBNIC_MAX_MSIX_VECS]; +}; + +static irqreturn_t fbnic_irq_test(int irq, void *data) +{ + struct fbnic_msix_test_data *test_data = data; + struct fbnic_dev *fbd = test_data->fbd; + int i; + + for (i = fbd->num_irqs; i--;) { + if (test_data->irq_vector[i] == irq) { + set_bit(i, test_data->test_msix_status); + break; + } + } + + return IRQ_HANDLED; +} + +/** + * fbnic_msix_test - Verify behavior of NIC interrupts + * @fbd: device to test + * + * This function is meant to test the global interrupt registers and the + * PCIe IP MSI-X functionality. It essentially goes through and tests + * various combinations of the set, clear, and mask bits in order to + * verify the behavior is as we expect it to be from the driver. + * + * Return: See enum fbnic_msix_self_test_codes + **/ +enum fbnic_msix_self_test_codes fbnic_msix_test(struct fbnic_dev *fbd) +{ + enum fbnic_msix_self_test_codes result = FBNIC_TEST_MSIX_SUCCESS; + struct pci_dev *pdev = to_pci_dev(fbd->dev); + struct fbnic_msix_test_data *test_data; + u32 mask = 0; + int i; + + /* Allocate bitmap and IRQ vector table */ + test_data = kzalloc_obj(*test_data, GFP_KERNEL); + + /* memory allocation failure */ + if (!test_data) + return FBNIC_TEST_MSIX_NOMEM; + + /* Initialize test data */ + test_data->fbd = fbd; + + for (i = FBNIC_NON_NAPI_VECTORS; i < fbd->num_irqs; i++) { + /* Add IRQ to vector table so it can be found */ + test_data->irq_vector[i] = pci_irq_vector(pdev, i); + + /* Enable the interrupt */ + if (!fbnic_request_irq(fbd, i, fbnic_irq_test, 0, + fbd->netdev->name, test_data)) + continue; + + while (i-- > FBNIC_NON_NAPI_VECTORS) + fbnic_free_irq(fbd, i, test_data); + kfree(test_data); + + /* IRQ request failure */ + return FBNIC_TEST_MSIX_IRQ_REQ_FAIL; + } + + /* Test each bit individually */ + for (i = FBNIC_NON_NAPI_VECTORS; i < fbd->num_irqs; i++) { + mask = 1U << (i % 32); + + /* Start with mask set and interrupt cleared */ + fbnic_wr32(fbd, FBNIC_INTR_MASK_SET(i / 32), mask); + fbnic_wrfl(fbd); + fbnic_wr32(fbd, FBNIC_INTR_CLEAR(i / 32), mask); + fbnic_wrfl(fbd); + + /* masking failure to prevent interrupt */ + result = FBNIC_TEST_MSIX_MASK; + + fbnic_wr32(fbd, FBNIC_INTR_SET(i / 32), mask); + fbnic_wrfl(fbd); + usleep_range(10000, 11000); + + if (test_bit(i, test_data->test_msix_status)) + break; + + /* unmasking failure w/ sw status set */ + result = FBNIC_TEST_MSIX_UNMASK; + + fbnic_wr32(fbd, FBNIC_INTR_MASK_CLEAR(i / 32), mask); + fbnic_wrfl(fbd); + usleep_range(10000, 11000); + + if (!test_bit(i, test_data->test_msix_status)) + break; + + /* interrupt when clearing mask */ + result = FBNIC_TEST_MSIX_IRQ_CLEAR; + + clear_bit(i, test_data->test_msix_status); + fbnic_wr32(fbd, FBNIC_INTR_MASK_CLEAR(i / 32), mask); + fbnic_wrfl(fbd); + usleep_range(10000, 11000); + + if (test_bit(i, test_data->test_msix_status)) + break; + + /* interrupt not triggering when not masked */ + result = FBNIC_TEST_MSIX_NO_INTERRUPT; + + fbnic_wr32(fbd, FBNIC_INTR_SET(i / 32), mask); + fbnic_wrfl(fbd); + usleep_range(10000, 11000); + + if (!test_bit(i, test_data->test_msix_status)) + break; + + /* status not cleared, or mask not set */ + result = FBNIC_TEST_MSIX_NO_CLEAR_OR_MASK; + if (mask & fbnic_rd32(fbd, FBNIC_INTR_STATUS(i / 32))) + break; + if (!(mask & fbnic_rd32(fbd, FBNIC_INTR_MASK(i / 32)))) + break; + + /* Result = 0 - Success */ + result = FBNIC_TEST_MSIX_SUCCESS; + + clear_bit(i, test_data->test_msix_status); + } + + if (i < fbd->num_irqs) { + fbnic_wr32(fbd, FBNIC_INTR_MASK_SET(i / 32), mask); + fbnic_wrfl(fbd); + fbnic_wr32(fbd, FBNIC_INTR_CLEAR(i / 32), mask); + fbnic_wrfl(fbd); + clear_bit(i, test_data->test_msix_status); + } + + for (i = FBNIC_NON_NAPI_VECTORS; i < fbd->num_irqs; i++) { + /* Test for bits set after testing */ + if (test_bit(i, test_data->test_msix_status)) + result = FBNIC_TEST_MSIX_BITS_SET_AFTER_TEST; + + /* Free IRQ */ + fbnic_free_irq(fbd, i, test_data); + } + + kfree(test_data); + + return result; +} + void fbnic_napi_name_irqs(struct fbnic_dev *fbd) { unsigned int i; diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_mac.c b/drivers/net/ethernet/meta/fbnic/fbnic_mac.c index 9d0e4b2cc9ac..53b7a938b4c2 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_mac.c +++ b/drivers/net/ethernet/meta/fbnic/fbnic_mac.c @@ -143,6 +143,7 @@ static void fbnic_mac_init_qm(struct fbnic_dev *fbd) #define FBNIC_DROP_EN_MASK 0x7d #define FBNIC_PAUSE_EN_MASK 0x14 #define FBNIC_ECN_EN_MASK 0x10 +#define FBNIC_PS_EN_MASK 0x01 struct fbnic_fifo_config { unsigned int addr; @@ -417,9 +418,29 @@ static void __fbnic_mac_stat_rd64(struct fbnic_dev *fbd, bool reset, u32 reg, stat->reported = true; } +static void fbnic_mac_stat_rd32(struct fbnic_dev *fbd, bool reset, u32 reg, + struct fbnic_stat_counter *stat) +{ + u32 new_reg_value; + + new_reg_value = rd32(fbd, reg); + if (!reset) + stat->value += new_reg_value - stat->u.old_reg_value_32; + stat->u.old_reg_value_32 = new_reg_value; + stat->reported = true; +} + #define fbnic_mac_stat_rd64(fbd, reset, __stat, __CSR) \ __fbnic_mac_stat_rd64(fbd, reset, FBNIC_##__CSR##_L, &(__stat)) +bool fbnic_mac_check_tx_pause(struct fbnic_dev *fbd) +{ + u32 command_config; + + command_config = rd32(fbd, FBNIC_MAC_COMMAND_CONFIG); + return !(command_config & FBNIC_MAC_COMMAND_CONFIG_TX_PAUSE_DIS); +} + static void fbnic_mac_tx_pause_config(struct fbnic_dev *fbd, bool tx_pause) { u32 rxb_pause_ctrl; @@ -434,6 +455,49 @@ static void fbnic_mac_tx_pause_config(struct fbnic_dev *fbd, bool tx_pause) wr32(fbd, FBNIC_RXB_PAUSE_DROP_CTRL, rxb_pause_ctrl); } +static void +fbnic_mac_ps_protect_to_reset(struct fbnic_dev *fbd, u16 timeout_ms) +{ + wr32(fbd, FBNIC_RXB_PAUSE_STORM_UNIT_WR, FBNIC_RXB_PS_CLK_DIV); + + wr32(fbd, FBNIC_RXB_PAUSE_STORM(FBNIC_RXB_INTF_NET), + FIELD_PREP(FBNIC_RXB_PAUSE_STORM_THLD_TIME, + FBNIC_MAC_RXB_PS_TO(timeout_ms)) | + FBNIC_RXB_PAUSE_STORM_FORCE_NORMAL); + wrfl(fbd); + wr32(fbd, FBNIC_RXB_PAUSE_STORM(FBNIC_RXB_INTF_NET), + FIELD_PREP(FBNIC_RXB_PAUSE_STORM_THLD_TIME, + FBNIC_MAC_RXB_PS_TO(timeout_ms))); +} + +static void +fbnic_mac_ps_protect_config(struct fbnic_dev *fbd, bool ps_protect) +{ + u16 timeout; + u32 reg; + + ps_protect = ps_protect && fbd->ps_timeout; + timeout = ps_protect ? fbd->ps_timeout : FBNIC_MAC_PS_TO_DEFAULT_MS; + + fbnic_mac_ps_protect_to_reset(fbd, timeout); + + reg = rd32(fbd, FBNIC_RXB_PAUSE_DROP_CTRL); + reg &= ~FBNIC_RXB_PAUSE_DROP_CTRL_PS_ENABLE; + reg |= FIELD_PREP(FBNIC_RXB_PAUSE_DROP_CTRL_PS_ENABLE, ps_protect); + wr32(fbd, FBNIC_RXB_PAUSE_DROP_CTRL, reg); + + /* Clear any pending interrupt status first */ + wr32(fbd, FBNIC_RXB_ERR_INTR_STS, + FIELD_PREP(FBNIC_RXB_ERR_INTR_STS_PS, FBNIC_PS_EN_MASK)); + + /* Unmask the Network to Host PS interrupt if tx_pause is on */ + reg = rd32(fbd, FBNIC_RXB_ERR_INTR_MASK); + reg |= FBNIC_RXB_ERR_INTR_STS_PS; + if (ps_protect) + reg &= ~FBNIC_RXB_ERR_INTR_STS_PS; + wr32(fbd, FBNIC_RXB_ERR_INTR_MASK, reg); +} + static int fbnic_mac_get_link_event(struct fbnic_dev *fbd) { u32 intr_mask = rd32(fbd, FBNIC_SIG_PCS_INTR_STS); @@ -658,6 +722,7 @@ static void fbnic_mac_link_up_asic(struct fbnic_dev *fbd, u32 cmd_cfg, mac_ctrl; fbnic_mac_tx_pause_config(fbd, tx_pause); + fbnic_mac_ps_protect_config(fbd, tx_pause); cmd_cfg = __fbnic_mac_cmd_config_asic(fbd, tx_pause, rx_pause); mac_ctrl = rd32(fbd, FBNIC_SIG_MAC_IN0); @@ -759,6 +824,9 @@ fbnic_mac_get_pause_stats(struct fbnic_dev *fbd, bool reset, MAC_STAT_TX_XOFF_STB); fbnic_mac_stat_rd64(fbd, reset, pause_stats->rx_pause_frames, MAC_STAT_RX_XOFF_STB); + fbnic_mac_stat_rd32(fbd, reset, + FBNIC_RXB_INTR_PS_COUNT(FBNIC_RXB_INTF_NET), + &pause_stats->tx_pause_storm_events); } static void @@ -918,3 +986,46 @@ int fbnic_mac_init(struct fbnic_dev *fbd) return 0; } + +int fbnic_mac_ps_protect_to_config(struct fbnic_dev *fbd, u16 timeout_ms) +{ + u16 old_timeout_ms = fbd->ps_timeout; + + if (timeout_ms == old_timeout_ms) + return 0; + + if (timeout_ms == PFC_STORM_PREVENTION_AUTO) + timeout_ms = FBNIC_MAC_PS_TO_DEFAULT_MS; + + if (timeout_ms > FBNIC_MAC_PS_TO_MAX_MS) + return -EINVAL; + + fbd->ps_timeout = timeout_ms; + + if (!fbnic_mac_check_tx_pause(fbd)) + return 0; + + if (timeout_ms == 0) + fbnic_mac_ps_protect_config(fbd, false); + else if (old_timeout_ms == 0) + fbnic_mac_ps_protect_config(fbd, true); + else + fbnic_mac_ps_protect_to_reset(fbd, fbd->ps_timeout); + + return 0; +} + +void fbnic_mac_ps_protect_handler(struct fbnic_dev *fbd) +{ + u32 rxb_err_sts = rd32(fbd, FBNIC_RXB_ERR_INTR_STS); + + /* Check if pause storm interrupt for network was triggered */ + if (rxb_err_sts & FIELD_PREP(FBNIC_RXB_ERR_INTR_STS_PS, + FBNIC_PS_EN_MASK)) { + /* Write 1 to clear the interrupt status first */ + wr32(fbd, FBNIC_RXB_ERR_INTR_STS, + FIELD_PREP(FBNIC_RXB_ERR_INTR_STS_PS, FBNIC_PS_EN_MASK)); + + fbnic_mac_ps_protect_to_reset(fbd, fbd->ps_timeout); + } +} diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_mac.h b/drivers/net/ethernet/meta/fbnic/fbnic_mac.h index f08fe8b7c497..10f30e0e8f69 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_mac.h +++ b/drivers/net/ethernet/meta/fbnic/fbnic_mac.h @@ -8,6 +8,30 @@ struct fbnic_dev; +/* The RXB clock runs at 600 MHZ in the ASIC and the PAUSE_STORM_UNIT_WR + * is 10us granularity, so set the clock to 6000 (0x1770) + */ +#define FBNIC_RXB_PS_CLK_DIV 0x1770 + +/* Convert milliseconds to pause storm timeout units (10us granularity) */ +#define FBNIC_MAC_RXB_PS_TO(ms) ((ms) * 100) + +/* Convert pause storm timeout units (10us granularity) to milliseconds */ +#define FBNIC_MAC_RXB_PS_TO_MS(ps) ((ps) / 100) + +/* Set the default timer to 500ms, which should be longer than any + * reasonable period of continuous pausing. The service task, which runs + * once per second, periodically resets the pause storm trigger. + * + * As a result, on a functioning system, if pause continues, we enforce + * a duty cycle determined by the configured pause storm timeout (50% + * default). A crashed system will not have the service task and therefore + * pause will remain disabled until reboot recovery. + */ +#define FBNIC_MAC_PS_TO_DEFAULT_MS 500 +#define FBNIC_MAC_PS_TO_MAX_MS \ + FBNIC_MAC_RXB_PS_TO_MS(FIELD_MAX(FBNIC_RXB_PAUSE_STORM_THLD_TIME)) + #define FBNIC_MAX_JUMBO_FRAME_SIZE 9742 /* States loosely based on section 136.8.11.7.5 of IEEE 802.3-2022 Ethernet @@ -119,4 +143,7 @@ struct fbnic_mac { int fbnic_mac_init(struct fbnic_dev *fbd); void fbnic_mac_get_fw_settings(struct fbnic_dev *fbd, u8 *aui, u8 *fec); +int fbnic_mac_ps_protect_to_config(struct fbnic_dev *fbd, u16 timeout); +void fbnic_mac_ps_protect_handler(struct fbnic_dev *fbd); +bool fbnic_mac_check_tx_pause(struct fbnic_dev *fbd); #endif /* _FBNIC_MAC_H_ */ diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_pci.c b/drivers/net/ethernet/meta/fbnic/fbnic_pci.c index 3fa9d1910daa..b7c0b7349d00 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_pci.c +++ b/drivers/net/ethernet/meta/fbnic/fbnic_pci.c @@ -139,7 +139,7 @@ void fbnic_up(struct fbnic_net *fbn) /* Enable Tx/Rx processing */ fbnic_napi_enable(fbn); - netif_tx_start_all_queues(fbn->netdev); + netif_tx_wake_all_queues(fbn->netdev); fbnic_service_task_start(fbn); @@ -220,6 +220,9 @@ static void fbnic_service_task(struct work_struct *work) fbnic_get_hw_stats32(fbd); + if (fbd->ps_timeout && fbnic_mac_check_tx_pause(fbd)) + fbnic_mac_ps_protect_handler(fbd); + fbnic_fw_check_heartbeat(fbd); fbnic_health_check(fbd); @@ -296,6 +299,8 @@ static int fbnic_probe(struct pci_dev *pdev, const struct pci_device_id *ent) /* Populate driver with hardware-specific info and handlers */ fbd->max_num_queues = info->max_num_queues; + fbd->ps_timeout = FBNIC_MAC_PS_TO_DEFAULT_MS; + pci_set_master(pdev); pci_save_state(pdev); diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_tlv.c b/drivers/net/ethernet/meta/fbnic/fbnic_tlv.c index 517ed8b2f1cb..c55d4f76a5fc 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_tlv.c +++ b/drivers/net/ethernet/meta/fbnic/fbnic_tlv.c @@ -551,6 +551,172 @@ int fbnic_tlv_parser_error(void *opaque, struct fbnic_tlv_msg **results) return -EBADMSG; } +#define FBNIC_TLV_TEST_STRING_LEN 32 + +struct fbnic_tlv_test { + u64 test_u64; + s64 test_s64; + u32 test_u32; + s32 test_s32; + u16 test_u16; + s16 test_s16; + u8 test_mac[ETH_ALEN]; + u8 test_mac_array[4][ETH_ALEN]; + u8 test_true; + u8 test_false; + char test_string[FBNIC_TLV_TEST_STRING_LEN]; +}; + +static struct fbnic_tlv_test test_struct; + +const struct fbnic_tlv_index fbnic_tlv_test_index[] = { + FBNIC_TLV_ATTR_U64(FBNIC_TLV_TEST_MSG_U64), + FBNIC_TLV_ATTR_S64(FBNIC_TLV_TEST_MSG_S64), + FBNIC_TLV_ATTR_U32(FBNIC_TLV_TEST_MSG_U32), + FBNIC_TLV_ATTR_S32(FBNIC_TLV_TEST_MSG_S32), + FBNIC_TLV_ATTR_U32(FBNIC_TLV_TEST_MSG_U16), + FBNIC_TLV_ATTR_S32(FBNIC_TLV_TEST_MSG_S16), + FBNIC_TLV_ATTR_MAC_ADDR(FBNIC_TLV_TEST_MSG_MAC_ADDR), + FBNIC_TLV_ATTR_FLAG(FBNIC_TLV_TEST_MSG_FLAG_TRUE), + FBNIC_TLV_ATTR_FLAG(FBNIC_TLV_TEST_MSG_FLAG_FALSE), + FBNIC_TLV_ATTR_STRING(FBNIC_TLV_TEST_MSG_STRING, + FBNIC_TLV_TEST_STRING_LEN), + FBNIC_TLV_ATTR_ARRAY(FBNIC_TLV_TEST_MSG_ARRAY), + FBNIC_TLV_ATTR_NESTED(FBNIC_TLV_TEST_MSG_NESTED), + FBNIC_TLV_ATTR_LAST +}; + +static void fbnic_tlv_test_struct_init(void) +{ + int i = FBNIC_TLV_TEST_STRING_LEN - 1; + + /* Populate the struct with random data */ + get_random_once(&test_struct, + offsetof(struct fbnic_tlv_test, test_string) + i); + + /* Force true/false to their expected values */ + test_struct.test_false = false; + test_struct.test_true = true; + + /* Convert test_string to a true ASCII string */ + test_struct.test_string[i] = '\0'; + while (i--) { + /* Force characters into displayable range */ + if (test_struct.test_string[i] < 64 || + test_struct.test_string[i] >= 96) { + test_struct.test_string[i] %= 32; + test_struct.test_string[i] += 64; + } + } +} + +static int fbnic_tlv_test_attr_data(struct fbnic_tlv_msg *msg) +{ + struct fbnic_tlv_msg *array; + int err, i; + + err = fbnic_tlv_attr_put_int(msg, FBNIC_TLV_TEST_MSG_U64, + test_struct.test_u64); + if (err) + return err; + + err = fbnic_tlv_attr_put_int(msg, FBNIC_TLV_TEST_MSG_S64, + test_struct.test_s64); + if (err) + return err; + + err = fbnic_tlv_attr_put_int(msg, FBNIC_TLV_TEST_MSG_U32, + test_struct.test_u32); + if (err) + return err; + + err = fbnic_tlv_attr_put_int(msg, FBNIC_TLV_TEST_MSG_S32, + test_struct.test_s32); + if (err) + return err; + + err = fbnic_tlv_attr_put_int(msg, FBNIC_TLV_TEST_MSG_U16, + test_struct.test_u16); + if (err) + return err; + + err = fbnic_tlv_attr_put_int(msg, FBNIC_TLV_TEST_MSG_S16, + test_struct.test_s16); + if (err) + return err; + + err = fbnic_tlv_attr_put_value(msg, FBNIC_TLV_TEST_MSG_MAC_ADDR, + test_struct.test_mac, ETH_ALEN); + if (err) + return err; + + /* Start MAC address array */ + array = fbnic_tlv_attr_nest_start(msg, FBNIC_TLV_TEST_MSG_ARRAY); + if (!array) + return -ENOSPC; + + for (i = 0; i < 4; i++) { + err = fbnic_tlv_attr_put_value(array, + FBNIC_TLV_TEST_MSG_MAC_ADDR, + test_struct.test_mac_array[i], + ETH_ALEN); + if (err) + return err; + } + + /* Close array */ + fbnic_tlv_attr_nest_stop(msg); + + err = fbnic_tlv_attr_put_flag(msg, FBNIC_TLV_TEST_MSG_FLAG_TRUE); + if (err) + return err; + + return fbnic_tlv_attr_put_string(msg, FBNIC_TLV_TEST_MSG_STRING, + test_struct.test_string); +} + +/** + * fbnic_tlv_test_create - Allocate a test message and fill it w/ data + * @fbd: FBNIC device structure + * + * Return: NULL on failure to allocate or pointer to new TLV test message. + **/ +struct fbnic_tlv_msg *fbnic_tlv_test_create(struct fbnic_dev *fbd) +{ + struct fbnic_tlv_msg *msg, *nest; + int err; + + msg = fbnic_tlv_msg_alloc(FBNIC_TLV_MSG_ID_TEST); + if (!msg) + return NULL; + + /* Randomize struct data */ + fbnic_tlv_test_struct_init(); + + /* Add first level of data to message */ + err = fbnic_tlv_test_attr_data(msg); + if (err) + goto free_message; + + /* Start second level nested */ + nest = fbnic_tlv_attr_nest_start(msg, FBNIC_TLV_TEST_MSG_NESTED); + if (!nest) + goto free_message; + + /* Add nested data */ + err = fbnic_tlv_test_attr_data(nest); + if (err) + goto free_message; + + /* Close nest and report full message */ + fbnic_tlv_attr_nest_stop(msg); + + return msg; +free_message: + free_page((unsigned long)msg); + return NULL; +} + void fbnic_tlv_attr_addr_copy(u8 *dest, struct fbnic_tlv_msg *src) { u8 *mac_addr; @@ -558,3 +724,113 @@ void fbnic_tlv_attr_addr_copy(u8 *dest, struct fbnic_tlv_msg *src) mac_addr = fbnic_tlv_attr_get_value_ptr(src); memcpy(dest, mac_addr, ETH_ALEN); } + +/** + * fbnic_tlv_parser_test_attr - Function loading test attributes into structure + * @str: Test structure to load + * @results: Pointer to results array + * + * Copies attributes into structure. Any attribute that doesn't exist in the + * results array is not populated. + **/ +static void fbnic_tlv_parser_test_attr(struct fbnic_tlv_test *str, + struct fbnic_tlv_msg **results) +{ + struct fbnic_tlv_msg *array_results[4]; + struct fbnic_tlv_msg *attr; + char *string = NULL; + int i, err; + + str->test_u64 = fta_get_uint(results, FBNIC_TLV_TEST_MSG_U64); + str->test_u32 = fta_get_uint(results, FBNIC_TLV_TEST_MSG_U32); + str->test_u16 = fta_get_uint(results, FBNIC_TLV_TEST_MSG_U16); + + str->test_s64 = fta_get_sint(results, FBNIC_TLV_TEST_MSG_S64); + str->test_s32 = fta_get_sint(results, FBNIC_TLV_TEST_MSG_S32); + str->test_s16 = fta_get_sint(results, FBNIC_TLV_TEST_MSG_S16); + + attr = results[FBNIC_TLV_TEST_MSG_MAC_ADDR]; + if (attr) + fbnic_tlv_attr_addr_copy(str->test_mac, attr); + + attr = results[FBNIC_TLV_TEST_MSG_ARRAY]; + if (attr) { + int len = le16_to_cpu(attr->hdr.len) / sizeof(u32) - 1; + + err = fbnic_tlv_attr_parse_array(&attr[1], len, + array_results, + fbnic_tlv_test_index, + FBNIC_TLV_TEST_MSG_MAC_ADDR, + 4); + if (!err) { + for (i = 0; i < 4 && array_results[i]; i++) + fbnic_tlv_attr_addr_copy(str->test_mac_array[i], + array_results[i]); + } + } + + str->test_true = !!results[FBNIC_TLV_TEST_MSG_FLAG_TRUE]; + str->test_false = !!results[FBNIC_TLV_TEST_MSG_FLAG_FALSE]; + + attr = results[FBNIC_TLV_TEST_MSG_STRING]; + if (attr) { + string = fbnic_tlv_attr_get_value_ptr(attr); + strscpy(str->test_string, string, FBNIC_TLV_TEST_STRING_LEN); + } +} + +static void fbnic_tlv_test_dump(struct fbnic_tlv_test *value, char *prefix) +{ + print_hex_dump(KERN_INFO, prefix, DUMP_PREFIX_OFFSET, 16, 1, + value, sizeof(*value), true); +} + +/** + * fbnic_tlv_parser_test - Function for parsing and testing test message + * @opaque: Unused value + * @results: Results of parser output + * + * Return: negative value on error, or 0 on success. + * + * Parses attributes to structures and compares the structure to the + * expected test value that should have been used to populate the message. + * + * Used to verify message generation and parser are working correctly. + **/ +int fbnic_tlv_parser_test(void *opaque, struct fbnic_tlv_msg **results) +{ + struct fbnic_tlv_msg *nest_results[FBNIC_TLV_RESULTS_MAX] = { 0 }; + struct fbnic_tlv_test result_struct; + struct fbnic_tlv_msg *attr; + int err; + + memset(&result_struct, 0, sizeof(result_struct)); + fbnic_tlv_parser_test_attr(&result_struct, results); + + if (memcmp(&test_struct, &result_struct, sizeof(test_struct))) { + fbnic_tlv_test_dump(&result_struct, "fbnic: found - "); + fbnic_tlv_test_dump(&test_struct, "fbnic: expected - "); + return -EINVAL; + } + + attr = results[FBNIC_TLV_TEST_MSG_NESTED]; + if (!attr) + return -EINVAL; + + err = fbnic_tlv_attr_parse(&attr[1], + le16_to_cpu(attr->hdr.len) / sizeof(u32) - 1, + nest_results, fbnic_tlv_test_index); + if (err) + return err; + + memset(&result_struct, 0, sizeof(result_struct)); + fbnic_tlv_parser_test_attr(&result_struct, nest_results); + + if (memcmp(&test_struct, &result_struct, sizeof(test_struct))) { + fbnic_tlv_test_dump(&result_struct, "fbnic: found - "); + fbnic_tlv_test_dump(&test_struct, "fbnic: expected - "); + return -EINVAL; + } + + return 0; +} diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_tlv.h b/drivers/net/ethernet/meta/fbnic/fbnic_tlv.h index 3508b46ebdd0..9c4e4759394a 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_tlv.h +++ b/drivers/net/ethernet/meta/fbnic/fbnic_tlv.h @@ -9,6 +9,8 @@ #include <linux/const.h> #include <linux/types.h> +struct fbnic_dev; + #define FBNIC_TLV_MSG_ALIGN(len) ALIGN(len, sizeof(u32)) #define FBNIC_TLV_MSG_SIZE(len) \ (FBNIC_TLV_MSG_ALIGN(len) / sizeof(u32)) @@ -153,6 +155,31 @@ int fbnic_tlv_parser_error(void *opaque, struct fbnic_tlv_msg **results); #define fta_get_str(_results, _id, _dst, _dstsize) \ fbnic_tlv_attr_get_string(_results[_id], _dst, _dstsize) +#define FBNIC_TLV_MSG_ID_TEST 0 + +enum fbnic_tlv_test_attr_id { + FBNIC_TLV_TEST_MSG_U64, + FBNIC_TLV_TEST_MSG_S64, + FBNIC_TLV_TEST_MSG_U32, + FBNIC_TLV_TEST_MSG_S32, + FBNIC_TLV_TEST_MSG_U16, + FBNIC_TLV_TEST_MSG_S16, + FBNIC_TLV_TEST_MSG_MAC_ADDR, + FBNIC_TLV_TEST_MSG_FLAG_TRUE, + FBNIC_TLV_TEST_MSG_FLAG_FALSE, + FBNIC_TLV_TEST_MSG_STRING, + FBNIC_TLV_TEST_MSG_NESTED, + FBNIC_TLV_TEST_MSG_ARRAY, + FBNIC_TLV_TEST_MSG_MAX +}; + +extern const struct fbnic_tlv_index fbnic_tlv_test_index[]; +struct fbnic_tlv_msg *fbnic_tlv_test_create(struct fbnic_dev *fbd); +int fbnic_tlv_parser_test(void *opaque, struct fbnic_tlv_msg **results); + +#define FBNIC_TLV_MSG_TEST \ + FBNIC_TLV_PARSER(TEST, fbnic_tlv_test_index, \ + fbnic_tlv_parser_test) #define FBNIC_TLV_MSG_ERROR \ FBNIC_TLV_PARSER(UNKNOWN, NULL, fbnic_tlv_parser_error) #endif /* _FBNIC_TLV_H_ */ diff --git a/drivers/net/ethernet/microchip/fdma/Kconfig b/drivers/net/ethernet/microchip/fdma/Kconfig index ec228c061351..57a54e7167d3 100644 --- a/drivers/net/ethernet/microchip/fdma/Kconfig +++ b/drivers/net/ethernet/microchip/fdma/Kconfig @@ -3,8 +3,6 @@ # Microchip FDMA API configuration # -if NET_VENDOR_MICROCHIP - config FDMA bool "FDMA API" if COMPILE_TEST help @@ -14,5 +12,3 @@ config FDMA Say Y here if you want to build the FDMA API that provides a common set of functions and data structures for interacting with the Frame DMA engine in multiple microchip switchcores. - -endif # NET_VENDOR_MICROCHIP diff --git a/drivers/net/ethernet/microchip/lan743x_ethtool.c b/drivers/net/ethernet/microchip/lan743x_ethtool.c index 8a3c1ecc7866..9195419ecee0 100644 --- a/drivers/net/ethernet/microchip/lan743x_ethtool.c +++ b/drivers/net/ethernet/microchip/lan743x_ethtool.c @@ -1079,6 +1079,13 @@ static int lan743x_ethtool_set_eee(struct net_device *netdev, return phylink_ethtool_set_eee(adapter->phylink, eee); } +static int lan743x_ethtool_nway_reset(struct net_device *netdev) +{ + struct lan743x_adapter *adapter = netdev_priv(netdev); + + return phylink_ethtool_nway_reset(adapter->phylink); +} + static int lan743x_ethtool_set_link_ksettings(struct net_device *netdev, const struct ethtool_link_ksettings *cmd) @@ -1369,6 +1376,7 @@ const struct ethtool_ops lan743x_ethtool_ops = { .set_rxfh = lan743x_ethtool_set_rxfh, .get_rxfh_fields = lan743x_ethtool_get_rxfh_fields, .get_ts_info = lan743x_ethtool_get_ts_info, + .nway_reset = lan743x_ethtool_nway_reset, .get_eee = lan743x_ethtool_get_eee, .set_eee = lan743x_ethtool_set_eee, .get_link_ksettings = lan743x_ethtool_get_link_ksettings, diff --git a/drivers/net/ethernet/microchip/lan743x_main.c b/drivers/net/ethernet/microchip/lan743x_main.c index f0b5dd752f08..f3332417162e 100644 --- a/drivers/net/ethernet/microchip/lan743x_main.c +++ b/drivers/net/ethernet/microchip/lan743x_main.c @@ -28,9 +28,15 @@ #define RFE_RD_FIFO_TH_3_DWORDS 0x3 +static bool pci11x1x_is_a0(struct lan743x_adapter *adapter) +{ + u32 dev_rev = adapter->csr.id_rev & ID_REV_CHIP_REV_MASK_; + return dev_rev == ID_REV_CHIP_REV_PCI11X1X_A0_; +} + static void pci11x1x_strap_get_status(struct lan743x_adapter *adapter) { - u32 chip_rev; + u32 fpga_rev; u32 cfg_load; u32 hw_cfg; u32 strap; @@ -47,18 +53,19 @@ static void pci11x1x_strap_get_status(struct lan743x_adapter *adapter) cfg_load = lan743x_csr_read(adapter, ETH_SYS_CONFIG_LOAD_STARTED_REG); lan743x_hs_syslock_release(adapter); hw_cfg = lan743x_csr_read(adapter, HW_CFG); - - if (cfg_load & GEN_SYS_LOAD_STARTED_REG_ETH_ || - hw_cfg & HW_CFG_RST_PROTECT_) { - strap = lan743x_csr_read(adapter, STRAP_READ); + strap = lan743x_csr_read(adapter, STRAP_READ); + if ((pci11x1x_is_a0(adapter) && + (cfg_load & GEN_SYS_LOAD_STARTED_REG_ETH_ || + hw_cfg & HW_CFG_RST_PROTECT_)) || + (strap & STRAP_READ_USE_SGMII_EN_)) { if (strap & STRAP_READ_SGMII_EN_) adapter->is_sgmii_en = true; else adapter->is_sgmii_en = false; } else { - chip_rev = lan743x_csr_read(adapter, FPGA_REV); - if (chip_rev) { - if (chip_rev & FPGA_SGMII_OP) + fpga_rev = lan743x_csr_read(adapter, FPGA_REV); + if (fpga_rev) { + if (fpga_rev & FPGA_SGMII_OP) adapter->is_sgmii_en = true; else adapter->is_sgmii_en = false; diff --git a/drivers/net/ethernet/microchip/lan743x_main.h b/drivers/net/ethernet/microchip/lan743x_main.h index 02a28b709163..160d94a7cee6 100644 --- a/drivers/net/ethernet/microchip/lan743x_main.h +++ b/drivers/net/ethernet/microchip/lan743x_main.h @@ -27,6 +27,7 @@ #define ID_REV_CHIP_REV_MASK_ (0x0000FFFF) #define ID_REV_CHIP_REV_A0_ (0x00000000) #define ID_REV_CHIP_REV_B0_ (0x00000010) +#define ID_REV_CHIP_REV_PCI11X1X_A0_ (0x000000A0) #define ID_REV_CHIP_REV_PCI11X1X_B0_ (0x000000B0) #define FPGA_REV (0x04) diff --git a/drivers/net/ethernet/microchip/sparx5/sparx5_calendar.c b/drivers/net/ethernet/microchip/sparx5/sparx5_calendar.c index 8c67ff1d411b..35815fb58080 100644 --- a/drivers/net/ethernet/microchip/sparx5/sparx5_calendar.c +++ b/drivers/net/ethernet/microchip/sparx5/sparx5_calendar.c @@ -151,7 +151,7 @@ enum sparx5_cal_bw sparx5_get_port_cal_speed(struct sparx5 *sparx5, u32 portno) } /* Auto configure the QSYS calendar based on port configuration */ -int sparx5_config_auto_calendar(struct sparx5 *sparx5) +static int sparx5_config_auto_calendar(struct sparx5 *sparx5) { const struct sparx5_consts *consts = sparx5->data->consts; u32 cal[7], value, idx, portno; @@ -578,7 +578,7 @@ update_err: } /* Configure the DSM calendar based on port configuration */ -int sparx5_config_dsm_calendar(struct sparx5 *sparx5) +static int sparx5_config_dsm_calendar(struct sparx5 *sparx5) { const struct sparx5_ops *ops = sparx5->data->ops; int taxi; @@ -610,3 +610,14 @@ cal_out: kfree(data); return err; } + +int sparx5_calendar_init(struct sparx5 *sparx5) +{ + int err; + + err = sparx5_config_auto_calendar(sparx5); + if (err) + return err; + + return sparx5_config_dsm_calendar(sparx5); +} diff --git a/drivers/net/ethernet/microchip/sparx5/sparx5_ethtool.c b/drivers/net/ethernet/microchip/sparx5/sparx5_ethtool.c index 049541eeaae0..d42c57bead89 100644 --- a/drivers/net/ethernet/microchip/sparx5/sparx5_ethtool.c +++ b/drivers/net/ethernet/microchip/sparx5/sparx5_ethtool.c @@ -1244,7 +1244,7 @@ const struct ethtool_ops sparx5_ethtool_ops = { .set_pauseparam = sparx5_set_pauseparam, }; -int sparx_stats_init(struct sparx5 *sparx5) +int sparx5_stats_init(struct sparx5 *sparx5) { const struct sparx5_consts *consts = sparx5->data->consts; char queue_name[32]; @@ -1278,3 +1278,10 @@ int sparx_stats_init(struct sparx5 *sparx5) return 0; } + +void sparx5_stats_deinit(struct sparx5 *sparx5) +{ + cancel_delayed_work_sync(&sparx5->stats_work); + destroy_workqueue(sparx5->stats_queue); + mutex_destroy(&sparx5->queue_stats_lock); +} diff --git a/drivers/net/ethernet/microchip/sparx5/sparx5_mactable.c b/drivers/net/ethernet/microchip/sparx5/sparx5_mactable.c index f5584244612c..2bf9c5f64151 100644 --- a/drivers/net/ethernet/microchip/sparx5/sparx5_mactable.c +++ b/drivers/net/ethernet/microchip/sparx5/sparx5_mactable.c @@ -419,7 +419,7 @@ static void sparx5_mact_handle_entry(struct sparx5 *sparx5, true); } -void sparx5_mact_pull_work(struct work_struct *work) +static void sparx5_mact_pull_work(struct work_struct *work) { struct delayed_work *del_work = to_delayed_work(work); struct sparx5 *sparx5 = container_of(del_work, struct sparx5, @@ -489,8 +489,10 @@ void sparx5_set_ageing(struct sparx5 *sparx5, int msecs) LRN_AUTOAGE_CFG(0)); } -void sparx5_mact_init(struct sparx5 *sparx5) +int sparx5_mact_init(struct sparx5 *sparx5) { + char queue_name[32]; + mutex_init(&sparx5->lock); /* Flush MAC table */ @@ -502,4 +504,32 @@ void sparx5_mact_init(struct sparx5 *sparx5) dev_warn(sparx5->dev, "MAC flush error\n"); sparx5_set_ageing(sparx5, BR_DEFAULT_AGEING_TIME / HZ * 1000); + + /* Add host mode BC address (points only to CPU) */ + sparx5_mact_learn(sparx5, sparx5_get_pgid(sparx5, PGID_CPU), + (unsigned char[]){0xff, 0xff, 0xff, 0xff, 0xff, 0xff}, + NULL_VID); + + mutex_init(&sparx5->mdb_lock); + INIT_LIST_HEAD(&sparx5->mdb_entries); + mutex_init(&sparx5->mact_lock); + INIT_LIST_HEAD(&sparx5->mact_entries); + snprintf(queue_name, sizeof(queue_name), "%s-mact", + dev_name(sparx5->dev)); + sparx5->mact_queue = create_singlethread_workqueue(queue_name); + if (!sparx5->mact_queue) + return -ENOMEM; + + INIT_DELAYED_WORK(&sparx5->mact_work, sparx5_mact_pull_work); + queue_delayed_work(sparx5->mact_queue, &sparx5->mact_work, + SPX5_MACT_PULL_DELAY); + + return 0; +} + +void sparx5_mact_deinit(struct sparx5 *sparx5) +{ + cancel_delayed_work_sync(&sparx5->mact_work); + destroy_workqueue(sparx5->mact_queue); + mutex_destroy(&sparx5->mact_lock); } diff --git a/drivers/net/ethernet/microchip/sparx5/sparx5_main.c b/drivers/net/ethernet/microchip/sparx5/sparx5_main.c index 9befd0719f74..dad713e9ddd5 100644 --- a/drivers/net/ethernet/microchip/sparx5/sparx5_main.c +++ b/drivers/net/ethernet/microchip/sparx5/sparx5_main.c @@ -658,6 +658,58 @@ static int sparx5_qlim_set(struct sparx5 *sparx5) return 0; } +static int sparx5_frame_io_init(struct sparx5 *sparx5) +{ + const struct sparx5_ops *ops = sparx5->data->ops; + int err = -ENXIO; + + /* Start Frame DMA with fallback to register based INJ/XTR */ + if (sparx5->fdma_irq >= 0) { + if (GCB_CHIP_ID_REV_ID_GET(sparx5->chip_id) > 0 || + !is_sparx5(sparx5)) + err = devm_request_irq(sparx5->dev, + sparx5->fdma_irq, + sparx5_fdma_handler, + 0, + "sparx5-fdma", sparx5); + if (!err) { + err = ops->fdma_init(sparx5); + if (!err) + sparx5_fdma_start(sparx5); + } + if (err) + sparx5->fdma_irq = -ENXIO; + } else { + sparx5->fdma_irq = -ENXIO; + } + if (err && sparx5->xtr_irq >= 0) { + err = devm_request_irq(sparx5->dev, sparx5->xtr_irq, + sparx5_xtr_handler, IRQF_SHARED, + "sparx5-xtr", sparx5); + if (!err) + err = sparx5_manual_injection_mode(sparx5); + if (err) + sparx5->xtr_irq = -ENXIO; + } else { + sparx5->xtr_irq = -ENXIO; + } + + return err; +} + +static void sparx5_frame_io_deinit(struct sparx5 *sparx5) +{ + if (sparx5->xtr_irq >= 0) { + disable_irq(sparx5->xtr_irq); + sparx5->xtr_irq = -ENXIO; + } + if (sparx5->fdma_irq >= 0) { + disable_irq(sparx5->fdma_irq); + sparx5->data->ops->fdma_deinit(sparx5); + sparx5->fdma_irq = -ENXIO; + } +} + /* Some boards needs to map the SGPIO for signal detect explicitly to the * port module */ @@ -683,14 +735,10 @@ static void sparx5_board_init(struct sparx5 *sparx5) GCB_HW_SGPIO_TO_SD_MAP_CFG(idx)); } -static int sparx5_start(struct sparx5 *sparx5) +static void sparx5_forwarding_init(struct sparx5 *sparx5) { - u8 broadcast[ETH_ALEN] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff}; const struct sparx5_consts *consts = sparx5->data->consts; - const struct sparx5_ops *ops = sparx5->data->ops; - char queue_name[32]; u32 idx; - int err; /* Setup own UPSIDs */ for (idx = 0; idx < consts->n_own_upsids; idx++) { @@ -728,117 +776,9 @@ static int sparx5_start(struct sparx5 *sparx5) ANA_CL_FILTER_CTRL_FORCE_FCS_UPDATE_ENA, sparx5, ANA_CL_FILTER_CTRL(idx)); - /* Init MAC table, ageing */ - sparx5_mact_init(sparx5); - - /* Init PGID table arbitrator */ - sparx5_pgid_init(sparx5); - - /* Setup VLANs */ - sparx5_vlan_init(sparx5); - - /* Add host mode BC address (points only to CPU) */ - sparx5_mact_learn(sparx5, sparx5_get_pgid(sparx5, PGID_CPU), broadcast, - NULL_VID); - /* Enable queue limitation watermarks */ sparx5_qlim_set(sparx5); - err = sparx5_config_auto_calendar(sparx5); - if (err) - return err; - - err = sparx5_config_dsm_calendar(sparx5); - if (err) - return err; - - /* Init stats */ - err = sparx_stats_init(sparx5); - if (err) - return err; - - /* Init mact_sw struct */ - mutex_init(&sparx5->mact_lock); - INIT_LIST_HEAD(&sparx5->mact_entries); - snprintf(queue_name, sizeof(queue_name), "%s-mact", - dev_name(sparx5->dev)); - sparx5->mact_queue = create_singlethread_workqueue(queue_name); - if (!sparx5->mact_queue) - return -ENOMEM; - - INIT_DELAYED_WORK(&sparx5->mact_work, sparx5_mact_pull_work); - queue_delayed_work(sparx5->mact_queue, &sparx5->mact_work, - SPX5_MACT_PULL_DELAY); - - mutex_init(&sparx5->mdb_lock); - INIT_LIST_HEAD(&sparx5->mdb_entries); - - err = sparx5_register_netdevs(sparx5); - if (err) - return err; - - sparx5_board_init(sparx5); - err = sparx5_register_notifier_blocks(sparx5); - if (err) - return err; - - err = sparx5_vcap_init(sparx5); - if (err) { - sparx5_unregister_notifier_blocks(sparx5); - return err; - } - - /* Start Frame DMA with fallback to register based INJ/XTR */ - err = -ENXIO; - if (sparx5->fdma_irq >= 0) { - if (GCB_CHIP_ID_REV_ID_GET(sparx5->chip_id) > 0 || - !is_sparx5(sparx5)) - err = devm_request_irq(sparx5->dev, - sparx5->fdma_irq, - sparx5_fdma_handler, - 0, - "sparx5-fdma", sparx5); - if (!err) { - err = ops->fdma_init(sparx5); - if (!err) - sparx5_fdma_start(sparx5); - } - if (err) - sparx5->fdma_irq = -ENXIO; - } else { - sparx5->fdma_irq = -ENXIO; - } - if (err && sparx5->xtr_irq >= 0) { - err = devm_request_irq(sparx5->dev, sparx5->xtr_irq, - sparx5_xtr_handler, IRQF_SHARED, - "sparx5-xtr", sparx5); - if (!err) - err = sparx5_manual_injection_mode(sparx5); - if (err) - sparx5->xtr_irq = -ENXIO; - } else { - sparx5->xtr_irq = -ENXIO; - } - - if (sparx5->ptp_irq >= 0 && - sparx5_has_feature(sparx5, SPX5_FEATURE_PTP)) { - err = devm_request_threaded_irq(sparx5->dev, sparx5->ptp_irq, - NULL, ops->ptp_irq_handler, - IRQF_ONESHOT, "sparx5-ptp", - sparx5); - if (err) - sparx5->ptp_irq = -ENXIO; - - sparx5->ptp = 1; - } - - return err; -} - -static void sparx5_cleanup_ports(struct sparx5 *sparx5) -{ - sparx5_unregister_netdevs(sparx5); - sparx5_destroy_netdevs(sparx5); } static int mchp_sparx5_probe(struct platform_device *pdev) @@ -999,9 +939,14 @@ static int mchp_sparx5_probe(struct platform_device *pdev) } } - err = sparx5_start(sparx5); + sparx5_pgid_init(sparx5); + sparx5_vlan_init(sparx5); + sparx5_board_init(sparx5); + sparx5_forwarding_init(sparx5); + + err = sparx5_calendar_init(sparx5); if (err) { - dev_err(sparx5->dev, "Start failed\n"); + dev_err(sparx5->dev, "Failed to initialize calendar\n"); goto cleanup_ports; } @@ -1011,20 +956,66 @@ static int mchp_sparx5_probe(struct platform_device *pdev) goto cleanup_ports; } - err = sparx5_ptp_init(sparx5); + err = sparx5_vcap_init(sparx5); if (err) { - dev_err(sparx5->dev, "PTP failed\n"); + dev_err(sparx5->dev, "Failed to initialize VCAP\n"); goto cleanup_ports; } + err = sparx5_mact_init(sparx5); + if (err) { + dev_err(sparx5->dev, "Failed to initialize MAC table\n"); + goto cleanup_vcap; + } + + err = sparx5_stats_init(sparx5); + if (err) { + dev_err(sparx5->dev, "Failed to initialize stats\n"); + goto cleanup_mact; + } + INIT_LIST_HEAD(&sparx5->mall_entries); + err = sparx5_frame_io_init(sparx5); + if (err) { + dev_err(sparx5->dev, "Failed to initialize frame I/O\n"); + goto cleanup_stats; + } + + err = sparx5_ptp_init(sparx5); + if (err) { + dev_err(sparx5->dev, "Failed to initialize PTP\n"); + goto cleanup_frame_io; + } + + err = sparx5_register_netdevs(sparx5); + if (err) { + dev_err(sparx5->dev, "Failed to register net devices\n"); + goto cleanup_ptp; + } + + err = sparx5_register_notifier_blocks(sparx5); + if (err) { + dev_err(sparx5->dev, "Failed to register notifier blocks\n"); + goto cleanup_netdevs; + } + goto cleanup_config; +cleanup_netdevs: + sparx5_unregister_netdevs(sparx5); +cleanup_ptp: + sparx5_ptp_deinit(sparx5); +cleanup_frame_io: + sparx5_frame_io_deinit(sparx5); +cleanup_stats: + sparx5_stats_deinit(sparx5); +cleanup_mact: + sparx5_mact_deinit(sparx5); +cleanup_vcap: + sparx5_vcap_deinit(sparx5); cleanup_ports: - sparx5_cleanup_ports(sparx5); - if (sparx5->mact_queue) - destroy_workqueue(sparx5->mact_queue); + sparx5_destroy_netdevs(sparx5); cleanup_config: kfree(configs); cleanup_pnode: @@ -1035,24 +1026,16 @@ cleanup_pnode: static void mchp_sparx5_remove(struct platform_device *pdev) { struct sparx5 *sparx5 = platform_get_drvdata(pdev); - const struct sparx5_ops *ops = sparx5->data->ops; debugfs_remove_recursive(sparx5->debugfs_root); - if (sparx5->xtr_irq) { - disable_irq(sparx5->xtr_irq); - sparx5->xtr_irq = -ENXIO; - } - if (sparx5->fdma_irq) { - disable_irq(sparx5->fdma_irq); - sparx5->fdma_irq = -ENXIO; - } - sparx5_ptp_deinit(sparx5); - ops->fdma_deinit(sparx5); - sparx5_cleanup_ports(sparx5); - sparx5_vcap_destroy(sparx5); - /* Unregister netdevs */ sparx5_unregister_notifier_blocks(sparx5); - destroy_workqueue(sparx5->mact_queue); + sparx5_unregister_netdevs(sparx5); + sparx5_ptp_deinit(sparx5); + sparx5_frame_io_deinit(sparx5); + sparx5_stats_deinit(sparx5); + sparx5_mact_deinit(sparx5); + sparx5_vcap_deinit(sparx5); + sparx5_destroy_netdevs(sparx5); } static const struct sparx5_regs sparx5_regs = { diff --git a/drivers/net/ethernet/microchip/sparx5/sparx5_main.h b/drivers/net/ethernet/microchip/sparx5/sparx5_main.h index fe7d8bcc0cd9..6a745bb71b5c 100644 --- a/drivers/net/ethernet/microchip/sparx5/sparx5_main.h +++ b/drivers/net/ethernet/microchip/sparx5/sparx5_main.h @@ -470,7 +470,6 @@ void sparx5_fdma_reload(struct sparx5 *sparx5, struct fdma *fdma); void sparx5_fdma_injection_mode(struct sparx5 *sparx5); /* sparx5_mactable.c */ -void sparx5_mact_pull_work(struct work_struct *work); int sparx5_mact_learn(struct sparx5 *sparx5, int port, const unsigned char mac[ETH_ALEN], u16 vid); bool sparx5_mact_getnext(struct sparx5 *sparx5, @@ -489,7 +488,8 @@ int sparx5_del_mact_entry(struct sparx5 *sparx5, int sparx5_mc_sync(struct net_device *dev, const unsigned char *addr); int sparx5_mc_unsync(struct net_device *dev, const unsigned char *addr); void sparx5_set_ageing(struct sparx5 *sparx5, int msecs); -void sparx5_mact_init(struct sparx5 *sparx5); +int sparx5_mact_init(struct sparx5 *sparx5); +void sparx5_mact_deinit(struct sparx5 *sparx5); /* sparx5_vlan.c */ void sparx5_pgid_update_mask(struct sparx5_port *port, int pgid, bool enable); @@ -504,8 +504,7 @@ int sparx5_vlan_vid_del(struct sparx5_port *port, u16 vid); void sparx5_vlan_port_apply(struct sparx5 *sparx5, struct sparx5_port *port); /* sparx5_calendar.c */ -int sparx5_config_auto_calendar(struct sparx5 *sparx5); -int sparx5_config_dsm_calendar(struct sparx5 *sparx5); +int sparx5_calendar_init(struct sparx5 *sparx5); int sparx5_dsm_calendar_calc(struct sparx5 *sparx5, u32 taxi, struct sparx5_calendar_data *data); u32 sparx5_cal_speed_to_value(enum sparx5_cal_bw speed); @@ -514,7 +513,8 @@ enum sparx5_cal_bw sparx5_get_port_cal_speed(struct sparx5 *sparx5, u32 portno); /* sparx5_ethtool.c */ void sparx5_get_stats64(struct net_device *ndev, struct rtnl_link_stats64 *stats); -int sparx_stats_init(struct sparx5 *sparx5); +int sparx5_stats_init(struct sparx5 *sparx5); +void sparx5_stats_deinit(struct sparx5 *sparx5); /* sparx5_dcb.c */ #ifdef CONFIG_SPARX5_DCB @@ -563,7 +563,7 @@ void sparx5_get_hwtimestamp(struct sparx5 *sparx5, /* sparx5_vcap_impl.c */ int sparx5_vcap_init(struct sparx5 *sparx5); -void sparx5_vcap_destroy(struct sparx5 *sparx5); +void sparx5_vcap_deinit(struct sparx5 *sparx5); /* sparx5_pgid.c */ enum sparx5_pgid_type { diff --git a/drivers/net/ethernet/microchip/sparx5/sparx5_ptp.c b/drivers/net/ethernet/microchip/sparx5/sparx5_ptp.c index 8b2e07821a95..a16ec8136d6d 100644 --- a/drivers/net/ethernet/microchip/sparx5/sparx5_ptp.c +++ b/drivers/net/ethernet/microchip/sparx5/sparx5_ptp.c @@ -606,9 +606,22 @@ static int sparx5_ptp_phc_init(struct sparx5 *sparx5, int sparx5_ptp_init(struct sparx5 *sparx5) { u64 tod_adj = sparx5_ptp_get_nominal_value(sparx5); + const struct sparx5_ops *ops = sparx5->data->ops; struct sparx5_port *port; int err, i; + if (sparx5->ptp_irq >= 0 && + sparx5_has_feature(sparx5, SPX5_FEATURE_PTP)) { + err = devm_request_threaded_irq(sparx5->dev, sparx5->ptp_irq, + NULL, ops->ptp_irq_handler, + IRQF_ONESHOT, "sparx5-ptp", + sparx5); + if (err) + sparx5->ptp_irq = -ENXIO; + + sparx5->ptp = 1; + } + if (!sparx5->ptp) return 0; @@ -660,6 +673,11 @@ void sparx5_ptp_deinit(struct sparx5 *sparx5) struct sparx5_port *port; int i; + if (sparx5->ptp_irq >= 0) { + disable_irq(sparx5->ptp_irq); + sparx5->ptp_irq = -ENXIO; + } + for (i = 0; i < sparx5->data->consts->n_ports; i++) { port = sparx5->ports[i]; if (!port) diff --git a/drivers/net/ethernet/microchip/sparx5/sparx5_vcap_impl.c b/drivers/net/ethernet/microchip/sparx5/sparx5_vcap_impl.c index 3d64a0448d43..95b93e46a41d 100644 --- a/drivers/net/ethernet/microchip/sparx5/sparx5_vcap_impl.c +++ b/drivers/net/ethernet/microchip/sparx5/sparx5_vcap_impl.c @@ -2083,7 +2083,7 @@ int sparx5_vcap_init(struct sparx5 *sparx5) return err; } -void sparx5_vcap_destroy(struct sparx5 *sparx5) +void sparx5_vcap_deinit(struct sparx5 *sparx5) { struct vcap_control *ctrl = sparx5->vcap_ctrl; struct vcap_admin *admin, *admin_next; diff --git a/drivers/net/ethernet/microsoft/mana/gdma_main.c b/drivers/net/ethernet/microsoft/mana/gdma_main.c index 786186c9a115..098fbda0d128 100644 --- a/drivers/net/ethernet/microsoft/mana/gdma_main.c +++ b/drivers/net/ethernet/microsoft/mana/gdma_main.c @@ -4,6 +4,7 @@ #include <linux/debugfs.h> #include <linux/module.h> #include <linux/pci.h> +#include <linux/sizes.h> #include <linux/utsname.h> #include <linux/version.h> #include <linux/msi.h> @@ -39,49 +40,90 @@ static u64 mana_gd_r64(struct gdma_context *g, u64 offset) return readq(g->bar0_va + offset); } -static void mana_gd_init_pf_regs(struct pci_dev *pdev) +static int mana_gd_init_pf_regs(struct pci_dev *pdev) { struct gdma_context *gc = pci_get_drvdata(pdev); void __iomem *sriov_base_va; u64 sriov_base_off; gc->db_page_size = mana_gd_r32(gc, GDMA_PF_REG_DB_PAGE_SIZE) & 0xFFFF; - gc->db_page_base = gc->bar0_va + - mana_gd_r64(gc, GDMA_PF_REG_DB_PAGE_OFF); - gc->phys_db_page_base = gc->bar0_pa + - mana_gd_r64(gc, GDMA_PF_REG_DB_PAGE_OFF); + /* mana_gd_ring_doorbell() accesses offsets up to DOORBELL_OFFSET_EQ + * (0xFF8) + 8 bytes = 4KB within each doorbell page, so the page + * size must be at least SZ_4K. + */ + if (gc->db_page_size < SZ_4K) { + dev_err(gc->dev, + "Doorbell page size %llu too small (min %u)\n", + gc->db_page_size, SZ_4K); + return -EPROTO; + } + + gc->db_page_off = mana_gd_r64(gc, GDMA_PF_REG_DB_PAGE_OFF); + + /* Validate doorbell offset is within BAR0 */ + if (gc->db_page_off >= gc->bar0_size) { + dev_err(gc->dev, + "Doorbell offset 0x%llx exceeds BAR0 size 0x%llx\n", + gc->db_page_off, (u64)gc->bar0_size); + return -EPROTO; + } + + gc->db_page_base = gc->bar0_va + gc->db_page_off; + gc->phys_db_page_base = gc->bar0_pa + gc->db_page_off; sriov_base_off = mana_gd_r64(gc, GDMA_SRIOV_REG_CFG_BASE_OFF); sriov_base_va = gc->bar0_va + sriov_base_off; gc->shm_base = sriov_base_va + mana_gd_r64(gc, sriov_base_off + GDMA_PF_REG_SHM_OFF); + + return 0; } -static void mana_gd_init_vf_regs(struct pci_dev *pdev) +static int mana_gd_init_vf_regs(struct pci_dev *pdev) { struct gdma_context *gc = pci_get_drvdata(pdev); gc->db_page_size = mana_gd_r32(gc, GDMA_REG_DB_PAGE_SIZE) & 0xFFFF; - gc->db_page_base = gc->bar0_va + - mana_gd_r64(gc, GDMA_REG_DB_PAGE_OFFSET); + /* mana_gd_ring_doorbell() accesses offsets up to DOORBELL_OFFSET_EQ + * (0xFF8) + 8 bytes = 4KB within each doorbell page, so the page + * size must be at least SZ_4K. + */ + if (gc->db_page_size < SZ_4K) { + dev_err(gc->dev, + "Doorbell page size %llu too small (min %u)\n", + gc->db_page_size, SZ_4K); + return -EPROTO; + } + + gc->db_page_off = mana_gd_r64(gc, GDMA_REG_DB_PAGE_OFFSET); - gc->phys_db_page_base = gc->bar0_pa + - mana_gd_r64(gc, GDMA_REG_DB_PAGE_OFFSET); + /* Validate doorbell offset is within BAR0 */ + if (gc->db_page_off >= gc->bar0_size) { + dev_err(gc->dev, + "Doorbell offset 0x%llx exceeds BAR0 size 0x%llx\n", + gc->db_page_off, (u64)gc->bar0_size); + return -EPROTO; + } + + gc->db_page_base = gc->bar0_va + gc->db_page_off; + gc->phys_db_page_base = gc->bar0_pa + gc->db_page_off; gc->shm_base = gc->bar0_va + mana_gd_r64(gc, GDMA_REG_SHM_OFFSET); + + return 0; } -static void mana_gd_init_registers(struct pci_dev *pdev) +static int mana_gd_init_registers(struct pci_dev *pdev) { struct gdma_context *gc = pci_get_drvdata(pdev); if (gc->is_pf) - mana_gd_init_pf_regs(pdev); + return mana_gd_init_pf_regs(pdev); else - mana_gd_init_vf_regs(pdev); + return mana_gd_init_vf_regs(pdev); } /* Suppress logging when we set timeout to zero */ @@ -490,15 +532,9 @@ static void mana_serv_reset(struct pci_dev *pdev) dev_info(&pdev->dev, "MANA reset cycle completed\n"); out: - gc->in_service = false; + clear_bit(GC_IN_SERVICE, &gc->flags); } -struct mana_serv_work { - struct work_struct serv_work; - struct pci_dev *pdev; - enum gdma_eqe_type type; -}; - static void mana_do_service(enum gdma_eqe_type type, struct pci_dev *pdev) { switch (type) { @@ -558,12 +594,42 @@ static void mana_serv_func(struct work_struct *w) module_put(THIS_MODULE); } +int mana_schedule_serv_work(struct gdma_context *gc, enum gdma_eqe_type type) +{ + struct mana_serv_work *mns_wk; + + if (test_and_set_bit(GC_IN_SERVICE, &gc->flags)) { + dev_info(gc->dev, "Already in service\n"); + return -EBUSY; + } + + if (!try_module_get(THIS_MODULE)) { + dev_info(gc->dev, "Module is unloading\n"); + clear_bit(GC_IN_SERVICE, &gc->flags); + return -ENODEV; + } + + mns_wk = kzalloc(sizeof(*mns_wk), GFP_ATOMIC); + if (!mns_wk) { + module_put(THIS_MODULE); + clear_bit(GC_IN_SERVICE, &gc->flags); + return -ENOMEM; + } + + dev_info(gc->dev, "Start MANA service type:%d\n", type); + mns_wk->pdev = to_pci_dev(gc->dev); + mns_wk->type = type; + pci_dev_get(mns_wk->pdev); + INIT_WORK(&mns_wk->serv_work, mana_serv_func); + schedule_work(&mns_wk->serv_work); + return 0; +} + static void mana_gd_process_eqe(struct gdma_queue *eq) { u32 head = eq->head % (eq->queue_size / GDMA_EQE_SIZE); struct gdma_context *gc = eq->gdma_dev->gdma_context; struct gdma_eqe *eq_eqe_ptr = eq->queue_mem_ptr; - struct mana_serv_work *mns_wk; union gdma_eqe_info eqe_info; enum gdma_eqe_type type; struct gdma_event event; @@ -623,30 +689,7 @@ static void mana_gd_process_eqe(struct gdma_queue *eq) "Service is to be processed in probe\n"); break; } - - if (gc->in_service) { - dev_info(gc->dev, "Already in service\n"); - break; - } - - if (!try_module_get(THIS_MODULE)) { - dev_info(gc->dev, "Module is unloading\n"); - break; - } - - mns_wk = kzalloc_obj(*mns_wk, GFP_ATOMIC); - if (!mns_wk) { - module_put(THIS_MODULE); - break; - } - - dev_info(gc->dev, "Start MANA service type:%d\n", type); - gc->in_service = true; - mns_wk->pdev = to_pci_dev(gc->dev); - mns_wk->type = type; - pci_dev_get(mns_wk->pdev); - INIT_WORK(&mns_wk->serv_work, mana_serv_func); - schedule_work(&mns_wk->serv_work); + mana_schedule_serv_work(gc, type); break; default: @@ -1255,6 +1298,17 @@ int mana_gd_register_device(struct gdma_dev *gd) return err ? err : -EPROTO; } + /* Validate that doorbell page for db_id is within the BAR0 region. + * In mana_gd_ring_doorbell(), the address is calculated as: + * addr = db_page_base + db_page_size * db_id + * = (bar0_va + db_page_off) + (db_page_size * db_id) + * So we need: db_page_off + db_page_size * (db_id + 1) <= bar0_size + */ + if (gc->db_page_off + gc->db_page_size * ((u64)resp.db_id + 1) > gc->bar0_size) { + dev_err(gc->dev, "Doorbell ID %u out of range\n", resp.db_id); + return -EPROTO; + } + gd->pdid = resp.pdid; gd->gpa_mkey = resp.gpa_mkey; gd->doorbell = resp.db_id; @@ -1889,7 +1943,10 @@ static int mana_gd_setup(struct pci_dev *pdev) struct gdma_context *gc = pci_get_drvdata(pdev); int err; - mana_gd_init_registers(pdev); + err = mana_gd_init_registers(pdev); + if (err) + return err; + mana_smc_init(&gc->shm_channel, gc->dev, gc->shm_base); gc->service_wq = alloc_ordered_workqueue("gdma_service_wq", 0); @@ -1996,6 +2053,7 @@ static int mana_gd_probe(struct pci_dev *pdev, const struct pci_device_id *ent) mutex_init(&gc->eq_test_event_mutex); pci_set_drvdata(pdev, gc); gc->bar0_pa = pci_resource_start(pdev, 0); + gc->bar0_size = pci_resource_len(pdev, 0); bar0_va = pci_iomap(pdev, bar, 0); if (!bar0_va) @@ -2007,11 +2065,8 @@ static int mana_gd_probe(struct pci_dev *pdev, const struct pci_device_id *ent) gc->dev = &pdev->dev; xa_init(&gc->irq_contexts); - if (gc->is_pf) - gc->mana_pci_debugfs = debugfs_create_dir("0", mana_debugfs_root); - else - gc->mana_pci_debugfs = debugfs_create_dir(pci_slot_name(pdev->slot), - mana_debugfs_root); + gc->mana_pci_debugfs = debugfs_create_dir(pci_name(pdev), + mana_debugfs_root); err = mana_gd_setup(pdev); if (err) diff --git a/drivers/net/ethernet/microsoft/mana/hw_channel.c b/drivers/net/ethernet/microsoft/mana/hw_channel.c index 48a9acea4ab6..dbbde0fa57e7 100644 --- a/drivers/net/ethernet/microsoft/mana/hw_channel.c +++ b/drivers/net/ethernet/microsoft/mana/hw_channel.c @@ -853,6 +853,7 @@ int mana_hwc_send_request(struct hw_channel_context *hwc, u32 req_len, struct hwc_caller_ctx *ctx; u32 dest_vrcq = 0; u32 dest_vrq = 0; + u32 command; u16 msg_id; int err; @@ -878,6 +879,7 @@ int mana_hwc_send_request(struct hw_channel_context *hwc, u32 req_len, req_msg->req.hwc_msg_id = msg_id; tx_wr->msg_size = req_len; + command = req_msg->req.msg_type; if (gc->is_pf) { dest_vrq = hwc->pf_dest_vrq_id; @@ -893,8 +895,8 @@ int mana_hwc_send_request(struct hw_channel_context *hwc, u32 req_len, if (!wait_for_completion_timeout(&ctx->comp_event, (msecs_to_jiffies(hwc->hwc_timeout)))) { if (hwc->hwc_timeout != 0) - dev_err(hwc->dev, "HWC: Request timed out: %u ms\n", - hwc->hwc_timeout); + dev_err(hwc->dev, "Command 0x%x timed out: %u ms\n", + command, hwc->hwc_timeout); /* Reduce further waiting if HWC no response */ if (hwc->hwc_timeout > 1) @@ -914,9 +916,9 @@ int mana_hwc_send_request(struct hw_channel_context *hwc, u32 req_len, err = -EOPNOTSUPP; goto out; } - if (req_msg->req.msg_type != MANA_QUERY_PHY_STAT) - dev_err(hwc->dev, "HWC: Failed hw_channel req: 0x%x\n", - ctx->status_code); + if (command != MANA_QUERY_PHY_STAT) + dev_err(hwc->dev, "Command 0x%x failed with status: 0x%x\n", + command, ctx->status_code); err = -EPROTO; goto out; } diff --git a/drivers/net/ethernet/microsoft/mana/mana_en.c b/drivers/net/ethernet/microsoft/mana/mana_en.c index 09a53c977545..6302432b9bf6 100644 --- a/drivers/net/ethernet/microsoft/mana/mana_en.c +++ b/drivers/net/ethernet/microsoft/mana/mana_en.c @@ -882,7 +882,7 @@ static void mana_tx_timeout(struct net_device *netdev, unsigned int txqueue) struct gdma_context *gc = ac->gdma_dev->gdma_context; /* Already in service, hence tx queue reset is not required.*/ - if (gc->in_service) + if (test_bit(GC_IN_SERVICE, &gc->flags)) return; /* Note: If there are pending queue reset work for this port(apc), @@ -1028,8 +1028,8 @@ static int mana_send_request(struct mana_context *ac, void *in_buf, if (req->req.msg_type != MANA_QUERY_PHY_STAT && mana_need_log(gc, err)) - dev_err(dev, "Failed to send mana message: %d, 0x%x\n", - err, resp->status); + dev_err(dev, "Command 0x%x failed with status: 0x%x, err: %d\n", + req->req.msg_type, resp->status, err); return err ? err : -EPROTO; } @@ -1214,10 +1214,16 @@ static int mana_query_device_cfg(struct mana_context *ac, u32 proto_major_ver, *max_num_vports = resp.max_num_vports; - if (resp.hdr.response.msg_version >= GDMA_MESSAGE_V2) + if (resp.hdr.response.msg_version >= GDMA_MESSAGE_V2) { + if (resp.adapter_mtu < ETH_MIN_MTU + ETH_HLEN) { + dev_err(dev, "Adapter MTU too small: %u\n", + resp.adapter_mtu); + return -EPROTO; + } gc->adapter_mtu = resp.adapter_mtu; - else + } else { gc->adapter_mtu = ETH_FRAME_LEN; + } if (resp.hdr.response.msg_version >= GDMA_MESSAGE_V3) *bm_hostmode = resp.bm_hostmode; @@ -1342,8 +1348,8 @@ int mana_cfg_vport(struct mana_port_context *apc, u32 protection_dom_id, apc->tx_shortform_allowed = resp.short_form_allowed; apc->tx_vp_offset = resp.tx_vport_offset; - netdev_info(apc->ndev, "Configured vPort %llu PD %u DB %u\n", - apc->port_handle, protection_dom_id, doorbell_pg_id); + netdev_info(apc->ndev, "Enabled vPort %llu PD %u DB %u MAC %pM\n", + apc->port_handle, protection_dom_id, doorbell_pg_id, apc->mac_addr); out: if (err) mana_uncfg_vport(apc); @@ -1372,6 +1378,7 @@ static int mana_cfg_vport_steering(struct mana_port_context *apc, sizeof(resp)); req->hdr.req.msg_version = GDMA_MESSAGE_V2; + req->hdr.resp.msg_version = GDMA_MESSAGE_V2; req->vport = apc->port_handle; req->num_indir_entries = apc->indir_table_sz; @@ -1383,7 +1390,9 @@ static int mana_cfg_vport_steering(struct mana_port_context *apc, req->update_hashkey = update_key; req->update_indir_tab = update_tab; req->default_rxobj = apc->default_rxobj; - req->cqe_coalescing_enable = 0; + + if (rx != TRI_STATE_FALSE) + req->cqe_coalescing_enable = apc->cqe_coalescing_enable; if (update_key) memcpy(&req->hashkey, apc->hashkey, MANA_HASH_KEY_SIZE); @@ -1412,8 +1421,13 @@ static int mana_cfg_vport_steering(struct mana_port_context *apc, netdev_err(ndev, "vPort RX configuration failed: 0x%x\n", resp.hdr.status); err = -EPROTO; + goto out; } + if (resp.hdr.response.msg_version >= GDMA_MESSAGE_V2) + apc->cqe_coalescing_timeout_ns = + resp.cqe_coalescing_timeout_ns; + netdev_info(ndev, "Configured steering vPort %llu entries %u\n", apc->port_handle, apc->indir_table_sz); out: @@ -1922,11 +1936,12 @@ static struct sk_buff *mana_build_skb(struct mana_rxq *rxq, void *buf_va, } static void mana_rx_skb(void *buf_va, bool from_pool, - struct mana_rxcomp_oob *cqe, struct mana_rxq *rxq) + struct mana_rxcomp_oob *cqe, struct mana_rxq *rxq, + int i) { struct mana_stats_rx *rx_stats = &rxq->stats; struct net_device *ndev = rxq->ndev; - uint pkt_len = cqe->ppi[0].pkt_len; + uint pkt_len = cqe->ppi[i].pkt_len; u16 rxq_idx = rxq->rxq_idx; struct napi_struct *napi; struct xdp_buff xdp = {}; @@ -1970,7 +1985,7 @@ static void mana_rx_skb(void *buf_va, bool from_pool, } if (cqe->rx_hashtype != 0 && (ndev->features & NETIF_F_RXHASH)) { - hash_value = cqe->ppi[0].pkt_hash; + hash_value = cqe->ppi[i].pkt_hash; if (cqe->rx_hashtype & MANA_HASH_L4) skb_set_hash(skb, hash_value, PKT_HASH_TYPE_L4); @@ -2105,9 +2120,11 @@ static void mana_process_rx_cqe(struct mana_rxq *rxq, struct mana_cq *cq, struct mana_recv_buf_oob *rxbuf_oob; struct mana_port_context *apc; struct device *dev = gc->dev; + bool coalesced = false; void *old_buf = NULL; u32 curr, pktlen; bool old_fp; + int i; apc = netdev_priv(ndev); @@ -2119,13 +2136,16 @@ static void mana_process_rx_cqe(struct mana_rxq *rxq, struct mana_cq *cq, ++ndev->stats.rx_dropped; rxbuf_oob = &rxq->rx_oobs[rxq->buf_index]; netdev_warn_once(ndev, "Dropped a truncated packet\n"); - goto drop; - case CQE_RX_COALESCED_4: - netdev_err(ndev, "RX coalescing is unsupported\n"); - apc->eth_stats.rx_coalesced_err++; + mana_move_wq_tail(rxq->gdma_rq, + rxbuf_oob->wqe_inf.wqe_size_in_bu); + mana_post_pkt_rxq(rxq); return; + case CQE_RX_COALESCED_4: + coalesced = true; + break; + case CQE_RX_OBJECT_FENCE: complete(&rxq->fence_event); return; @@ -2137,30 +2157,47 @@ static void mana_process_rx_cqe(struct mana_rxq *rxq, struct mana_cq *cq, return; } - pktlen = oob->ppi[0].pkt_len; + for (i = 0; i < MANA_RXCOMP_OOB_NUM_PPI; i++) { + old_buf = NULL; + pktlen = oob->ppi[i].pkt_len; + if (pktlen == 0) + break; - if (pktlen == 0) { - /* data packets should never have packetlength of zero */ - netdev_err(ndev, "RX pkt len=0, rq=%u, cq=%u, rxobj=0x%llx\n", - rxq->gdma_id, cq->gdma_id, rxq->rxobj); - return; - } + curr = rxq->buf_index; + rxbuf_oob = &rxq->rx_oobs[curr]; + WARN_ON_ONCE(rxbuf_oob->wqe_inf.wqe_size_in_bu != 1); - curr = rxq->buf_index; - rxbuf_oob = &rxq->rx_oobs[curr]; - WARN_ON_ONCE(rxbuf_oob->wqe_inf.wqe_size_in_bu != 1); + mana_refill_rx_oob(dev, rxq, rxbuf_oob, &old_buf, &old_fp); - mana_refill_rx_oob(dev, rxq, rxbuf_oob, &old_buf, &old_fp); + /* Unsuccessful refill will have old_buf == NULL. + * In this case, mana_rx_skb() will drop the packet. + */ + mana_rx_skb(old_buf, old_fp, oob, rxq, i); - /* Unsuccessful refill will have old_buf == NULL. - * In this case, mana_rx_skb() will drop the packet. - */ - mana_rx_skb(old_buf, old_fp, oob, rxq); + mana_move_wq_tail(rxq->gdma_rq, + rxbuf_oob->wqe_inf.wqe_size_in_bu); -drop: - mana_move_wq_tail(rxq->gdma_rq, rxbuf_oob->wqe_inf.wqe_size_in_bu); + mana_post_pkt_rxq(rxq); + + if (!coalesced) + break; + } - mana_post_pkt_rxq(rxq); + /* Collect coalesced CQE count based on packets processed. + * Coalesced CQEs have at least 2 packets, so index is i - 2. + */ + if (i > 1) { + u64_stats_update_begin(&rxq->stats.syncp); + rxq->stats.coalesced_cqe[i - 2]++; + u64_stats_update_end(&rxq->stats.syncp); + } else if (!i && !pktlen) { + u64_stats_update_begin(&rxq->stats.syncp); + rxq->stats.pkt_len0_err++; + u64_stats_update_end(&rxq->stats.syncp); + netdev_err_once(ndev, + "RX pkt len=0, rq=%u, cq=%u, rxobj=0x%llx\n", + rxq->gdma_id, cq->gdma_id, rxq->rxobj); + } } static void mana_poll_rx_cq(struct mana_cq *cq) @@ -3117,6 +3154,8 @@ static int mana_init_port(struct net_device *ndev) eth_hw_addr_set(ndev, apc->mac_addr); sprintf(vport, "vport%d", port_idx); apc->mana_port_debugfs = debugfs_create_dir(vport, gc->mana_pci_debugfs); + debugfs_create_u32("current_speed", 0400, apc->mana_port_debugfs, + &apc->speed); return 0; reset_apc: @@ -3333,12 +3372,14 @@ static int mana_probe_port(struct mana_context *ac, int port_idx, apc->ac = ac; apc->ndev = ndev; apc->max_queues = gc->max_num_queues; - apc->num_queues = gc->max_num_queues; + /* Use MANA_DEF_NUM_QUEUES as default, still honoring the HW limit */ + apc->num_queues = min(gc->max_num_queues, MANA_DEF_NUM_QUEUES); apc->tx_queue_size = DEF_TX_BUFFERS_PER_QUEUE; apc->rx_queue_size = DEF_RX_BUFFERS_PER_QUEUE; apc->port_handle = INVALID_MANA_HANDLE; apc->pf_filter_handle = INVALID_MANA_HANDLE; apc->port_idx = port_idx; + apc->cqe_coalescing_enable = 0; mutex_init(&apc->vport_mutex); apc->vport_use_count = 0; @@ -3393,8 +3434,6 @@ static int mana_probe_port(struct mana_context *ac, int port_idx, netif_carrier_on(ndev); - debugfs_create_u32("current_speed", 0400, apc->mana_port_debugfs, &apc->speed); - return 0; free_indir: @@ -3547,6 +3586,7 @@ static void mana_gf_stats_work_handler(struct work_struct *work) { struct mana_context *ac = container_of(to_delayed_work(work), struct mana_context, gf_stats_work); + struct gdma_context *gc = ac->gdma_dev->gdma_context; int err; err = mana_query_gf_stats(ac); @@ -3554,6 +3594,12 @@ static void mana_gf_stats_work_handler(struct work_struct *work) /* HWC timeout detected - reset stats and stop rescheduling */ ac->hwc_timeout_occurred = true; memset(&ac->hc_stats, 0, sizeof(ac->hc_stats)); + dev_warn(gc->dev, + "Gf stats wk handler: gf stats query timed out.\n"); + /* As HWC timed out, indicating a faulty HW state and needs a + * reset. + */ + mana_schedule_serv_work(gc, GDMA_EQE_HWC_RESET_REQUEST); return; } schedule_delayed_work(&ac->gf_stats_work, MANA_GF_STATS_PERIOD); diff --git a/drivers/net/ethernet/microsoft/mana/mana_ethtool.c b/drivers/net/ethernet/microsoft/mana/mana_ethtool.c index f2d220b371b5..6a4b42fe0944 100644 --- a/drivers/net/ethernet/microsoft/mana/mana_ethtool.c +++ b/drivers/net/ethernet/microsoft/mana/mana_ethtool.c @@ -20,8 +20,6 @@ static const struct mana_stats_desc mana_eth_stats[] = { tx_cqe_unknown_type)}, {"tx_linear_pkt_cnt", offsetof(struct mana_ethtool_stats, tx_linear_pkt_cnt)}, - {"rx_coalesced_err", offsetof(struct mana_ethtool_stats, - rx_coalesced_err)}, {"rx_cqe_unknown_type", offsetof(struct mana_ethtool_stats, rx_cqe_unknown_type)}, }; @@ -151,7 +149,7 @@ static void mana_get_strings(struct net_device *ndev, u32 stringset, u8 *data) { struct mana_port_context *apc = netdev_priv(ndev); unsigned int num_queues = apc->num_queues; - int i; + int i, j; if (stringset != ETH_SS_STATS) return; @@ -170,6 +168,9 @@ static void mana_get_strings(struct net_device *ndev, u32 stringset, u8 *data) ethtool_sprintf(&data, "rx_%d_xdp_drop", i); ethtool_sprintf(&data, "rx_%d_xdp_tx", i); ethtool_sprintf(&data, "rx_%d_xdp_redirect", i); + ethtool_sprintf(&data, "rx_%d_pkt_len0_err", i); + for (j = 0; j < MANA_RXCOMP_OOB_NUM_PPI - 1; j++) + ethtool_sprintf(&data, "rx_%d_coalesced_cqe_%d", i, j + 2); } for (i = 0; i < num_queues; i++) { @@ -203,6 +204,8 @@ static void mana_get_ethtool_stats(struct net_device *ndev, u64 xdp_xmit; u64 xdp_drop; u64 xdp_tx; + u64 pkt_len0_err; + u64 coalesced_cqe[MANA_RXCOMP_OOB_NUM_PPI - 1]; u64 tso_packets; u64 tso_bytes; u64 tso_inner_packets; @@ -211,7 +214,7 @@ static void mana_get_ethtool_stats(struct net_device *ndev, u64 short_pkt_fmt; u64 csum_partial; u64 mana_map_err; - int q, i = 0; + int q, i = 0, j; if (!apc->port_is_up) return; @@ -241,6 +244,9 @@ static void mana_get_ethtool_stats(struct net_device *ndev, xdp_drop = rx_stats->xdp_drop; xdp_tx = rx_stats->xdp_tx; xdp_redirect = rx_stats->xdp_redirect; + pkt_len0_err = rx_stats->pkt_len0_err; + for (j = 0; j < MANA_RXCOMP_OOB_NUM_PPI - 1; j++) + coalesced_cqe[j] = rx_stats->coalesced_cqe[j]; } while (u64_stats_fetch_retry(&rx_stats->syncp, start)); data[i++] = packets; @@ -248,6 +254,9 @@ static void mana_get_ethtool_stats(struct net_device *ndev, data[i++] = xdp_drop; data[i++] = xdp_tx; data[i++] = xdp_redirect; + data[i++] = pkt_len0_err; + for (j = 0; j < MANA_RXCOMP_OOB_NUM_PPI - 1; j++) + data[i++] = coalesced_cqe[j]; } for (q = 0; q < num_queues; q++) { @@ -390,6 +399,61 @@ static void mana_get_channels(struct net_device *ndev, channel->combined_count = apc->num_queues; } +#define MANA_RX_CQE_NSEC_DEF 2048 +static int mana_get_coalesce(struct net_device *ndev, + struct ethtool_coalesce *ec, + struct kernel_ethtool_coalesce *kernel_coal, + struct netlink_ext_ack *extack) +{ + struct mana_port_context *apc = netdev_priv(ndev); + + kernel_coal->rx_cqe_frames = + apc->cqe_coalescing_enable ? MANA_RXCOMP_OOB_NUM_PPI : 1; + + kernel_coal->rx_cqe_nsecs = apc->cqe_coalescing_timeout_ns; + + /* Return the default timeout value for old FW not providing + * this value. + */ + if (apc->port_is_up && apc->cqe_coalescing_enable && + !kernel_coal->rx_cqe_nsecs) + kernel_coal->rx_cqe_nsecs = MANA_RX_CQE_NSEC_DEF; + + return 0; +} + +static int mana_set_coalesce(struct net_device *ndev, + struct ethtool_coalesce *ec, + struct kernel_ethtool_coalesce *kernel_coal, + struct netlink_ext_ack *extack) +{ + struct mana_port_context *apc = netdev_priv(ndev); + u8 saved_cqe_coalescing_enable; + int err; + + if (kernel_coal->rx_cqe_frames != 1 && + kernel_coal->rx_cqe_frames != MANA_RXCOMP_OOB_NUM_PPI) { + NL_SET_ERR_MSG_FMT(extack, + "rx-frames must be 1 or %u, got %u", + MANA_RXCOMP_OOB_NUM_PPI, + kernel_coal->rx_cqe_frames); + return -EINVAL; + } + + saved_cqe_coalescing_enable = apc->cqe_coalescing_enable; + apc->cqe_coalescing_enable = + kernel_coal->rx_cqe_frames == MANA_RXCOMP_OOB_NUM_PPI; + + if (!apc->port_is_up) + return 0; + + err = mana_config_rss(apc, TRI_STATE_TRUE, false, false); + if (err) + apc->cqe_coalescing_enable = saved_cqe_coalescing_enable; + + return err; +} + static int mana_set_channels(struct net_device *ndev, struct ethtool_channels *channels) { @@ -510,6 +574,7 @@ static int mana_get_link_ksettings(struct net_device *ndev, } const struct ethtool_ops mana_ethtool_ops = { + .supported_coalesce_params = ETHTOOL_COALESCE_RX_CQE_FRAMES, .get_ethtool_stats = mana_get_ethtool_stats, .get_sset_count = mana_get_sset_count, .get_strings = mana_get_strings, @@ -520,6 +585,8 @@ const struct ethtool_ops mana_ethtool_ops = { .set_rxfh = mana_set_rxfh, .get_channels = mana_get_channels, .set_channels = mana_set_channels, + .get_coalesce = mana_get_coalesce, + .set_coalesce = mana_set_coalesce, .get_ringparam = mana_get_ringparam, .set_ringparam = mana_set_ringparam, .get_link_ksettings = mana_get_link_ksettings, diff --git a/drivers/net/ethernet/netronome/Kconfig b/drivers/net/ethernet/netronome/Kconfig index d03d6e96f730..d115d16d4649 100644 --- a/drivers/net/ethernet/netronome/Kconfig +++ b/drivers/net/ethernet/netronome/Kconfig @@ -33,7 +33,6 @@ config NFP_APP_FLOWER bool "NFP4000/NFP6000 TC Flower offload support" depends on NFP depends on NET_SWITCHDEV - depends on IPV6!=m || NFP=m default y help Enable driver support for TC Flower offload on NFP4000 and NFP6000. diff --git a/drivers/net/ethernet/netronome/nfp/flower/action.c b/drivers/net/ethernet/netronome/nfp/flower/action.c index aca2a7417af3..ae2f8b31adfb 100644 --- a/drivers/net/ethernet/netronome/nfp/flower/action.c +++ b/drivers/net/ethernet/netronome/nfp/flower/action.c @@ -470,7 +470,7 @@ nfp_fl_set_tun(struct nfp_app *app, struct nfp_fl_set_tun *set_tun, flow.daddr = ip_tun->key.u.ipv6.dst; flow.flowi4_proto = IPPROTO_UDP; - dst = ipv6_stub->ipv6_dst_lookup_flow(net, NULL, &flow, NULL); + dst = ip6_dst_lookup_flow(net, NULL, &flow, NULL); if (!IS_ERR(dst)) { set_tun->ttl = ip6_dst_hoplimit(dst); dst_release(dst); diff --git a/drivers/net/ethernet/netronome/nfp/flower/tunnel_conf.c b/drivers/net/ethernet/netronome/nfp/flower/tunnel_conf.c index 0cef0e2b85d0..ca30702f8878 100644 --- a/drivers/net/ethernet/netronome/nfp/flower/tunnel_conf.c +++ b/drivers/net/ethernet/netronome/nfp/flower/tunnel_conf.c @@ -650,7 +650,7 @@ static void nfp_tun_neigh_update(struct work_struct *work) flow6.daddr = *(struct in6_addr *)n->primary_key; if (!neigh_invalid) { struct dst_entry *dst; - /* Use ipv6_dst_lookup_flow to populate flow6->saddr + /* Use ip6_dst_lookup_flow to populate flow6->saddr * and other fields. This information is only needed * for new entries, lookup can be skipped when an entry * gets invalidated - as only the daddr is needed for @@ -730,7 +730,7 @@ nfp_tun_neigh_event_handler(struct notifier_block *nb, unsigned long event, return NOTIFY_DONE; } #if IS_ENABLED(CONFIG_IPV6) - if (n->tbl != ipv6_stub->nd_tbl && n->tbl != &arp_tbl) + if (n->tbl != &nd_tbl && n->tbl != &arp_tbl) #else if (n->tbl != &arp_tbl) #endif @@ -815,8 +815,7 @@ void nfp_tunnel_request_route_v6(struct nfp_app *app, struct sk_buff *skb) flow.flowi6_proto = IPPROTO_UDP; #if IS_ENABLED(CONFIG_INET) && IS_ENABLED(CONFIG_IPV6) - dst = ipv6_stub->ipv6_dst_lookup_flow(dev_net(netdev), NULL, &flow, - NULL); + dst = ip6_dst_lookup_flow(dev_net(netdev), NULL, &flow, NULL); if (IS_ERR(dst)) goto fail_rcu_unlock; #else diff --git a/drivers/net/ethernet/pensando/ionic/ionic_ethtool.c b/drivers/net/ethernet/pensando/ionic/ionic_ethtool.c index 1514c1019f28..78a802eb159f 100644 --- a/drivers/net/ethernet/pensando/ionic/ionic_ethtool.c +++ b/drivers/net/ethernet/pensando/ionic/ionic_ethtool.c @@ -188,10 +188,9 @@ static int ionic_get_link_ksettings(struct net_device *netdev, case IONIC_XCVR_PID_QSFP_100G_CWDM4: case IONIC_XCVR_PID_QSFP_100G_PSM4: case IONIC_XCVR_PID_QSFP_100G_LR4: - ethtool_link_ksettings_add_link_mode(ks, supported, - 100000baseLR4_ER4_Full); - break; case IONIC_XCVR_PID_QSFP_100G_ER4: + case IONIC_XCVR_PID_QSFP_100G_FR4: + case IONIC_XCVR_PID_QSFP_100G_DR4: ethtool_link_ksettings_add_link_mode(ks, supported, 100000baseLR4_ER4_Full); break; @@ -212,6 +211,7 @@ static int ionic_get_link_ksettings(struct net_device *netdev, break; case IONIC_XCVR_PID_QSFP_200G_AOC: case IONIC_XCVR_PID_QSFP_200G_SR4: + case IONIC_XCVR_PID_QSFP_200G_AEC: ethtool_link_ksettings_add_link_mode(ks, supported, 200000baseSR4_Full); break; @@ -232,6 +232,9 @@ static int ionic_get_link_ksettings(struct net_device *netdev, 400000baseDR4_Full); break; case IONIC_XCVR_PID_QSFP_400G_SR4: + case IONIC_XCVR_PID_QSFP_400G_AOC: + case IONIC_XCVR_PID_QSFP_400G_AEC: + case IONIC_XCVR_PID_QSFP_400G_LPO: ethtool_link_ksettings_add_link_mode(ks, supported, 400000baseSR4_Full); break; diff --git a/drivers/net/ethernet/pensando/ionic/ionic_if.h b/drivers/net/ethernet/pensando/ionic/ionic_if.h index 47559c909c8b..23d6e2b4791e 100644 --- a/drivers/net/ethernet/pensando/ionic/ionic_if.h +++ b/drivers/net/ethernet/pensando/ionic/ionic_if.h @@ -1341,6 +1341,12 @@ enum ionic_xcvr_pid { IONIC_XCVR_PID_QSFP_400G_DR4 = 80, IONIC_XCVR_PID_QSFP_400G_SR4 = 81, IONIC_XCVR_PID_QSFP_400G_VR4 = 82, + IONIC_XCVR_PID_QSFP_400G_AOC = 83, + IONIC_XCVR_PID_QSFP_400G_AEC = 84, + IONIC_XCVR_PID_QSFP_200G_AEC = 85, + IONIC_XCVR_PID_QSFP_400G_LPO = 86, + IONIC_XCVR_PID_QSFP_100G_FR4 = 87, + IONIC_XCVR_PID_QSFP_100G_DR4 = 88, }; /** diff --git a/drivers/net/ethernet/qlogic/qed/qed_l2.c b/drivers/net/ethernet/qlogic/qed/qed_l2.c index 0f6905e33df6..2d873a596261 100644 --- a/drivers/net/ethernet/qlogic/qed/qed_l2.c +++ b/drivers/net/ethernet/qlogic/qed/qed_l2.c @@ -7,6 +7,7 @@ #include <linux/types.h> #include <asm/byteorder.h> #include <asm/param.h> +#include <linux/crc32.h> #include <linux/delay.h> #include <linux/dma-mapping.h> #include <linux/etherdevice.h> @@ -39,7 +40,6 @@ #include "qed_sriov.h" #define QED_MAX_SGES_NUM 16 -#define CRC32_POLY 0x1edc6f41 struct qed_l2_info { u32 queues; @@ -1412,50 +1412,19 @@ int qed_sp_eth_filter_ucast(struct qed_hwfn *p_hwfn, return 0; } -/******************************************************************************* - * Description: - * Calculates crc 32 on a buffer - * Note: crc32_length MUST be aligned to 8 - * Return: - ******************************************************************************/ -static u32 qed_calc_crc32c(u8 *crc32_packet, - u32 crc32_length, u32 crc32_seed, u8 complement) -{ - u32 byte = 0, bit = 0, crc32_result = crc32_seed; - u8 msb = 0, current_byte = 0; - - if ((!crc32_packet) || - (crc32_length == 0) || - ((crc32_length % 8) != 0)) - return crc32_result; - for (byte = 0; byte < crc32_length; byte++) { - current_byte = crc32_packet[byte]; - for (bit = 0; bit < 8; bit++) { - msb = (u8)(crc32_result >> 31); - crc32_result = crc32_result << 1; - if (msb != (0x1 & (current_byte >> bit))) { - crc32_result = crc32_result ^ CRC32_POLY; - crc32_result |= 1; /*crc32_result[0] = 1;*/ - } - } - } - return crc32_result; -} - -static u32 qed_crc32c_le(u32 seed, u8 *mac, u32 len) -{ - u32 packet_buf[2] = { 0 }; - - memcpy((u8 *)(&packet_buf[0]), &mac[0], 6); - return qed_calc_crc32c((u8 *)packet_buf, 8, seed, 0); -} - u8 qed_mcast_bin_from_mac(u8 *mac) { - u32 crc = qed_crc32c_le(ETH_MULTICAST_BIN_FROM_MAC_SEED, - mac, ETH_ALEN); + u8 padded_mac[8] = {}; + u32 crc; - return crc & 0xff; + memcpy(padded_mac, mac, ETH_ALEN); + /* + * This uses the standard CRC-32C, but with the input and output CRCs + * bit-reflected and the bit-reflected output CRC truncated to 8 bits. + */ + crc = crc32c(bitrev32(ETH_MULTICAST_BIN_FROM_MAC_SEED), padded_mac, + sizeof(padded_mac)); + return bitrev8(crc >> 24); } static int diff --git a/drivers/net/ethernet/qlogic/qlcnic/qlcnic.h b/drivers/net/ethernet/qlogic/qlcnic/qlcnic.h index 3d0b5cd978cb..d7c8fadb49de 100644 --- a/drivers/net/ethernet/qlogic/qlcnic/qlcnic.h +++ b/drivers/net/ethernet/qlogic/qlcnic/qlcnic.h @@ -354,7 +354,7 @@ struct qlcnic_skb_frag { /* * There will be one qlcnic_buffer per skb packet. These will be - * used to save the dma info for pci_unmap_page() + * used to save the dma info for dma_unmap_page() */ struct qlcnic_cmd_buffer { struct sk_buff *skb; diff --git a/drivers/net/ethernet/realtek/r8169.h b/drivers/net/ethernet/realtek/r8169.h index aed4cf852091..0b9c1d4eb48b 100644 --- a/drivers/net/ethernet/realtek/r8169.h +++ b/drivers/net/ethernet/realtek/r8169.h @@ -68,6 +68,7 @@ enum mac_version { RTL_GIGA_MAC_VER_61, RTL_GIGA_MAC_VER_63, RTL_GIGA_MAC_VER_64, + RTL_GIGA_MAC_VER_65, RTL_GIGA_MAC_VER_66, RTL_GIGA_MAC_VER_70, RTL_GIGA_MAC_VER_80, diff --git a/drivers/net/ethernet/realtek/r8169_main.c b/drivers/net/ethernet/realtek/r8169_main.c index 58788d196c57..791277e750ba 100644 --- a/drivers/net/ethernet/realtek/r8169_main.c +++ b/drivers/net/ethernet/realtek/r8169_main.c @@ -60,6 +60,7 @@ #define FIRMWARE_8125D_2 "rtl_nic/rtl8125d-2.fw" #define FIRMWARE_8125K_1 "rtl_nic/rtl8125k-1.fw" #define FIRMWARE_8125BP_2 "rtl_nic/rtl8125bp-2.fw" +#define FIRMWARE_8125CP_1 "rtl_nic/rtl8125cp-1.fw" #define FIRMWARE_9151A_1 "rtl_nic/rtl9151a-1.fw" #define FIRMWARE_8126A_2 "rtl_nic/rtl8126a-2.fw" #define FIRMWARE_8126A_3 "rtl_nic/rtl8126a-3.fw" @@ -112,6 +113,9 @@ static const struct rtl_chip_info { /* 8125BP family. */ { 0x7cf, 0x681, RTL_GIGA_MAC_VER_66, "RTL8125BP", FIRMWARE_8125BP_2 }, + /* 8125CP family*/ + { 0x7cf, 0x708, RTL_GIGA_MAC_VER_65, "RTL8125CP", FIRMWARE_8125CP_1 }, + /* 8125D family. */ { 0x7cf, 0x68b, RTL_GIGA_MAC_VER_64, "RTL9151A", FIRMWARE_9151A_1 }, { 0x7cf, 0x68a, RTL_GIGA_MAC_VER_64, "RTL8125K", FIRMWARE_8125K_1 }, @@ -802,6 +806,7 @@ MODULE_FIRMWARE(FIRMWARE_8125D_1); MODULE_FIRMWARE(FIRMWARE_8125D_2); MODULE_FIRMWARE(FIRMWARE_8125K_1); MODULE_FIRMWARE(FIRMWARE_8125BP_2); +MODULE_FIRMWARE(FIRMWARE_8125CP_1); MODULE_FIRMWARE(FIRMWARE_9151A_1); MODULE_FIRMWARE(FIRMWARE_8126A_2); MODULE_FIRMWARE(FIRMWARE_8126A_3); @@ -4021,6 +4026,7 @@ static void rtl_hw_config(struct rtl8169_private *tp) [RTL_GIGA_MAC_VER_61] = rtl_hw_start_8125a_2, [RTL_GIGA_MAC_VER_63] = rtl_hw_start_8125b, [RTL_GIGA_MAC_VER_64] = rtl_hw_start_8125d, + [RTL_GIGA_MAC_VER_65] = rtl_hw_start_8125d, [RTL_GIGA_MAC_VER_66] = rtl_hw_start_8125d, [RTL_GIGA_MAC_VER_70] = rtl_hw_start_8126a, [RTL_GIGA_MAC_VER_80] = rtl_hw_start_8127a, @@ -4040,6 +4046,7 @@ static void rtl_hw_start_8125(struct rtl8169_private *tp) switch (tp->mac_version) { case RTL_GIGA_MAC_VER_61: case RTL_GIGA_MAC_VER_64: + case RTL_GIGA_MAC_VER_65: case RTL_GIGA_MAC_VER_66: case RTL_GIGA_MAC_VER_80: for (i = 0xa00; i < 0xb00; i += 4) diff --git a/drivers/net/ethernet/realtek/r8169_phy_config.c b/drivers/net/ethernet/realtek/r8169_phy_config.c index 032d9d2cfa2a..4f1d447136ec 100644 --- a/drivers/net/ethernet/realtek/r8169_phy_config.c +++ b/drivers/net/ethernet/realtek/r8169_phy_config.c @@ -14,6 +14,9 @@ #include "r8169.h" +#define LINK_SPEED_10M_PLL_OFF BIT(0) +#define ALDPS_PLL_OFF BIT(1) + typedef void (*rtl_phy_cfg_fct)(struct rtl8169_private *tp, struct phy_device *phydev); @@ -1102,6 +1105,28 @@ static void rtl8125d_hw_phy_config(struct rtl8169_private *tp, rtl8125_config_eee_phy(phydev); } +static void rtl8125cp_hw_phy_config(struct rtl8169_private *tp, + struct phy_device *phydev) +{ + r8169_apply_firmware(tp); + rtl8168g_enable_gphy_10m(phydev); + + phy_modify_paged(phydev, 0xad0, 0x17, 0x007f, 0x000b); + phy_modify_paged(phydev, 0xad7, 0x14, 0x0000, BIT(4)); + rtl8125_phy_param(phydev, 0x807f, 0xff00, 0x5300); + r8168g_phy_param(phydev, 0x81b8, 0xffff, 0x00b4); + r8168g_phy_param(phydev, 0x81ba, 0xffff, 0x00e4); + r8168g_phy_param(phydev, 0x81c5, 0xffff, 0x0104); + r8168g_phy_param(phydev, 0x81d0, 0xffff, 0x054d); + phy_modify_paged(phydev, 0x0a43, 0x10, 0x0000, + LINK_SPEED_10M_PLL_OFF | ALDPS_PLL_OFF); + phy_modify_paged(phydev, 0x0a44, 0x11, 0x0000, BIT(7)); + + rtl8125_legacy_force_mode(phydev); + rtl8168g_disable_aldps(phydev); + rtl8125_config_eee_phy(phydev); +} + static void rtl8125bp_hw_phy_config(struct rtl8169_private *tp, struct phy_device *phydev) { @@ -1344,6 +1369,7 @@ void r8169_hw_phy_config(struct rtl8169_private *tp, struct phy_device *phydev, [RTL_GIGA_MAC_VER_61] = rtl8125a_2_hw_phy_config, [RTL_GIGA_MAC_VER_63] = rtl8125b_hw_phy_config, [RTL_GIGA_MAC_VER_64] = rtl8125d_hw_phy_config, + [RTL_GIGA_MAC_VER_65] = rtl8125cp_hw_phy_config, [RTL_GIGA_MAC_VER_66] = rtl8125bp_hw_phy_config, [RTL_GIGA_MAC_VER_70] = rtl8126a_hw_phy_config, [RTL_GIGA_MAC_VER_80] = rtl8127a_1_hw_phy_config, diff --git a/drivers/net/ethernet/renesas/ravb_main.c b/drivers/net/ethernet/renesas/ravb_main.c index 84b657fc2e15..1dbfadb2a881 100644 --- a/drivers/net/ethernet/renesas/ravb_main.c +++ b/drivers/net/ethernet/renesas/ravb_main.c @@ -694,6 +694,9 @@ static int ravb_dmac_init(struct net_device *ndev) const struct ravb_hw_info *info = priv->info; int error; + /* Clear transmission suspension */ + ravb_modify(ndev, CCC, CCC_DTSR, 0); + /* Set CONFIG mode */ error = ravb_set_opmode(ndev, CCC_OPC_CONFIG); if (error) @@ -1103,6 +1106,12 @@ static int ravb_stop_dma(struct net_device *ndev) if (error) return error; + /* Request for transmission suspension */ + ravb_modify(ndev, CCC, CCC_DTSR, CCC_DTSR); + error = ravb_wait(ndev, CSR, CSR_DTS, CSR_DTS); + if (error) + netdev_err(ndev, "failed to stop AXI BUS\n"); + /* Stop AVB-DMAC process */ return ravb_set_opmode(ndev, CCC_OPC_CONFIG); } @@ -2367,6 +2376,7 @@ static int ravb_close(struct net_device *ndev) ravb_write(ndev, 0, RIC0); ravb_write(ndev, 0, RIC2); ravb_write(ndev, 0, TIC); + ravb_write(ndev, 0, ECSIPR); /* PHY disconnect */ if (ndev->phydev) { diff --git a/drivers/net/ethernet/rocker/rocker_ofdpa.c b/drivers/net/ethernet/rocker/rocker_ofdpa.c index 50ea5f9ef63a..66a8ae67c3ea 100644 --- a/drivers/net/ethernet/rocker/rocker_ofdpa.c +++ b/drivers/net/ethernet/rocker/rocker_ofdpa.c @@ -104,7 +104,6 @@ struct ofdpa_group_tbl_entry { u32 cmd; u32 group_id; /* key */ u16 group_count; - u32 *group_ids; union { struct { u8 pop_vlan; @@ -123,6 +122,8 @@ struct ofdpa_group_tbl_entry { u32 group_id; } l3_unicast; }; + + u32 group_ids[] __counted_by(group_count); }; struct ofdpa_fdb_tbl_entry { @@ -1059,19 +1060,6 @@ ofdpa_group_tbl_find(const struct ofdpa *ofdpa, return NULL; } -static void ofdpa_group_tbl_entry_free(struct ofdpa_group_tbl_entry *entry) -{ - switch (ROCKER_GROUP_TYPE_GET(entry->group_id)) { - case ROCKER_OF_DPA_GROUP_TYPE_L2_FLOOD: - case ROCKER_OF_DPA_GROUP_TYPE_L2_MCAST: - kfree(entry->group_ids); - break; - default: - break; - } - kfree(entry); -} - static int ofdpa_group_tbl_add(struct ofdpa_port *ofdpa_port, int flags, struct ofdpa_group_tbl_entry *match) { @@ -1085,7 +1073,7 @@ static int ofdpa_group_tbl_add(struct ofdpa_port *ofdpa_port, int flags, if (found) { hash_del(&found->entry); - ofdpa_group_tbl_entry_free(found); + kfree(found); found = match; found->cmd = ROCKER_TLV_CMD_TYPE_OF_DPA_GROUP_MOD; } else { @@ -1122,14 +1110,14 @@ static int ofdpa_group_tbl_del(struct ofdpa_port *ofdpa_port, int flags, spin_unlock_irqrestore(&ofdpa->group_tbl_lock, lock_flags); - ofdpa_group_tbl_entry_free(match); + kfree(match); if (found) { err = rocker_cmd_exec(ofdpa_port->rocker_port, ofdpa_flags_nowait(flags), ofdpa_cmd_group_tbl_del, found, NULL, NULL); - ofdpa_group_tbl_entry_free(found); + kfree(found); } return err; @@ -1166,18 +1154,13 @@ static int ofdpa_group_l2_fan_out(struct ofdpa_port *ofdpa_port, { struct ofdpa_group_tbl_entry *entry; - entry = kzalloc_obj(*entry); + entry = kzalloc_flex(*entry, group_ids, group_count); if (!entry) return -ENOMEM; - entry->group_id = group_id; entry->group_count = group_count; + entry->group_id = group_id; - entry->group_ids = kcalloc(group_count, sizeof(u32), GFP_KERNEL); - if (!entry->group_ids) { - kfree(entry); - return -ENOMEM; - } memcpy(entry->group_ids, group_ids, group_count * sizeof(u32)); return ofdpa_group_tbl_do(ofdpa_port, flags, entry); diff --git a/drivers/net/ethernet/sfc/ethtool.c b/drivers/net/ethernet/sfc/ethtool.c index 362388754a29..c0038b942913 100644 --- a/drivers/net/ethernet/sfc/ethtool.c +++ b/drivers/net/ethernet/sfc/ethtool.c @@ -231,6 +231,8 @@ static int efx_ethtool_get_ts_info(struct net_device *net_dev, { struct efx_nic *efx = efx_netdev_priv(net_dev); + ts_info->so_timestamping = SOF_TIMESTAMPING_TX_SOFTWARE; + efx_ptp_get_ts_info(efx, ts_info); return 0; } diff --git a/drivers/net/ethernet/sfc/tc_counters.c b/drivers/net/ethernet/sfc/tc_counters.c index d168282f30bf..b84235e93ffe 100644 --- a/drivers/net/ethernet/sfc/tc_counters.c +++ b/drivers/net/ethernet/sfc/tc_counters.c @@ -112,7 +112,7 @@ static void efx_tc_counter_work(struct work_struct *work) encap->neigh->egdev); else #if IS_ENABLED(CONFIG_IPV6) - n = neigh_lookup(ipv6_stub->nd_tbl, + n = neigh_lookup(&nd_tbl, &encap->neigh->dst_ip6, encap->neigh->egdev); #else diff --git a/drivers/net/ethernet/sfc/tc_encap_actions.c b/drivers/net/ethernet/sfc/tc_encap_actions.c index da35705cc5e1..db222abef53b 100644 --- a/drivers/net/ethernet/sfc/tc_encap_actions.c +++ b/drivers/net/ethernet/sfc/tc_encap_actions.c @@ -149,8 +149,7 @@ static int efx_bind_neigh(struct efx_nic *efx, #if IS_ENABLED(CONFIG_IPV6) struct dst_entry *dst; - dst = ipv6_stub->ipv6_dst_lookup_flow(net, NULL, &flow6, - NULL); + dst = ip6_dst_lookup_flow(net, NULL, &flow6, NULL); rc = PTR_ERR_OR_ZERO(dst); if (rc) { NL_SET_ERR_MSG_MOD(extack, "Failed to lookup route for IPv6 encap"); @@ -531,7 +530,7 @@ static int efx_neigh_event(struct efx_nic *efx, struct neighbour *n) if (n->tbl == &arp_tbl) { keysize = sizeof(keys.dst_ip); #if IS_ENABLED(CONFIG_IPV6) - } else if (n->tbl == ipv6_stub->nd_tbl) { + } else if (n->tbl == &nd_tbl) { ipv6 = true; keysize = sizeof(keys.dst_ip6); #endif diff --git a/drivers/net/ethernet/sfc/tx.c b/drivers/net/ethernet/sfc/tx.c index 4dff19b6ef17..ea0fc1000476 100644 --- a/drivers/net/ethernet/sfc/tx.c +++ b/drivers/net/ethernet/sfc/tx.c @@ -371,6 +371,8 @@ netdev_tx_t __efx_enqueue_skb(struct efx_tx_queue *tx_queue, struct sk_buff *skb if (!data_mapped && (efx_tx_map_data(tx_queue, skb, segments))) goto err; + skb_tx_timestamp(skb); + efx_tx_maybe_stop_queue(tx_queue); tx_queue->xmit_pending = true; diff --git a/drivers/net/ethernet/spacemit/k1_emac.c b/drivers/net/ethernet/spacemit/k1_emac.c index 15d43e4a748b..f7f16397a2c2 100644 --- a/drivers/net/ethernet/spacemit/k1_emac.c +++ b/drivers/net/ethernet/spacemit/k1_emac.c @@ -57,7 +57,6 @@ struct desc_buf { u64 dma_addr; - void *buff_addr; u16 dma_len; u8 map_as_page; }; @@ -70,7 +69,6 @@ struct emac_tx_desc_buffer { struct emac_rx_desc_buffer { struct sk_buff *skb; u64 dma_addr; - void *buff_addr; u16 dma_len; u8 map_as_page; }; @@ -340,7 +338,6 @@ static void emac_free_tx_buf(struct emac_priv *priv, int i) buf->dma_addr = 0; buf->map_as_page = false; - buf->buff_addr = NULL; } if (tx_buf->skb) { diff --git a/drivers/net/ethernet/stmicro/stmmac/Kconfig b/drivers/net/ethernet/stmicro/stmmac/Kconfig index 07088d03dbab..e3dd5adda5ac 100644 --- a/drivers/net/ethernet/stmicro/stmmac/Kconfig +++ b/drivers/net/ethernet/stmicro/stmmac/Kconfig @@ -20,7 +20,6 @@ if STMMAC_ETH config STMMAC_SELFTESTS bool "Support for STMMAC Selftests" depends on INET - depends on STMMAC_ETH default n help This adds support for STMMAC Selftests using ethtool. Enable this @@ -29,7 +28,6 @@ config STMMAC_SELFTESTS config STMMAC_PLATFORM tristate "STMMAC Platform bus support" - depends on STMMAC_ETH select MFD_SYSCON default y help @@ -132,6 +130,18 @@ config DWMAC_MESON the stmmac device driver. This driver is used for Meson6, Meson8, Meson8b and GXBB SoCs. +config DWMAC_NUVOTON + tristate "Nuvoton MA35 dwmac support" + default ARCH_MA35 + depends on OF && (ARCH_MA35 || COMPILE_TEST) + select MFD_SYSCON + help + Support for Ethernet controller on Nuvoton MA35 series SoC. + + This selects the Nuvoton MA35 series SoC glue layer support + for the stmmac device driver. The nuvoton-dwmac driver is + used for MA35 series SoCs. + config DWMAC_QCOM_ETHQOS tristate "Qualcomm ETHQOS support" default ARCH_QCOM @@ -216,6 +226,18 @@ config DWMAC_SOPHGO for the stmmac device driver. This driver is used for the ethernet controllers on various Sophgo SoCs. +config DWMAC_SPACEMIT + tristate "Spacemit dwmac support" + depends on OF && (ARCH_SPACEMIT || COMPILE_TEST) + select MFD_SYSCON + default m if ARCH_SPACEMIT + help + Support for ethernet controllers on Spacemit RISC-V SoCs + + This selects the Spacemit platform specific glue layer support + for the stmmac device driver. This driver is used for the + Spacemit K3 ethernet controllers. + config DWMAC_STARFIVE tristate "StarFive dwmac support" depends on OF && (ARCH_STARFIVE || COMPILE_TEST) @@ -312,7 +334,6 @@ config DWMAC_IMX8 config DWMAC_INTEL_PLAT tristate "Intel dwmac support" depends on OF && COMMON_CLK - depends on STMMAC_ETH help Support for ethernet controllers on Intel SoCs @@ -347,7 +368,7 @@ config DWMAC_VISCONTI help Support for ethernet controller on Visconti SoCs. -endif +endif # STMMAC_PLATFORM config STMMAC_LIBPCI tristate @@ -357,7 +378,7 @@ config STMMAC_LIBPCI config DWMAC_INTEL tristate "Intel GMAC support" default X86 - depends on X86 && STMMAC_ETH && PCI + depends on X86 && PCI depends on COMMON_CLK depends on ACPI help @@ -396,4 +417,4 @@ config STMMAC_PCI If you have a controller with this interface, say Y or M here. If unsure, say N. -endif +endif # STMMAC_ETH diff --git a/drivers/net/ethernet/stmicro/stmmac/Makefile b/drivers/net/ethernet/stmicro/stmmac/Makefile index c9263987ef8d..a1cea2f57252 100644 --- a/drivers/net/ethernet/stmicro/stmmac/Makefile +++ b/drivers/net/ethernet/stmicro/stmmac/Makefile @@ -20,6 +20,7 @@ obj-$(CONFIG_DWMAC_IPQ806X) += dwmac-ipq806x.o obj-$(CONFIG_DWMAC_LPC18XX) += dwmac-lpc18xx.o obj-$(CONFIG_DWMAC_MEDIATEK) += dwmac-mediatek.o obj-$(CONFIG_DWMAC_MESON) += dwmac-meson.o dwmac-meson8b.o +obj-$(CONFIG_DWMAC_NUVOTON) += dwmac-nuvoton.o obj-$(CONFIG_DWMAC_QCOM_ETHQOS) += dwmac-qcom-ethqos.o obj-$(CONFIG_DWMAC_RENESAS_GBETH) += dwmac-renesas-gbeth.o obj-$(CONFIG_DWMAC_ROCKCHIP) += dwmac-rk.o @@ -27,6 +28,7 @@ obj-$(CONFIG_DWMAC_RZN1) += dwmac-rzn1.o obj-$(CONFIG_DWMAC_S32) += dwmac-s32.o obj-$(CONFIG_DWMAC_SOCFPGA) += dwmac-altr-socfpga.o obj-$(CONFIG_DWMAC_SOPHGO) += dwmac-sophgo.o +obj-$(CONFIG_DWMAC_SPACEMIT) += dwmac-spacemit.o obj-$(CONFIG_DWMAC_STARFIVE) += dwmac-starfive.o obj-$(CONFIG_DWMAC_STI) += dwmac-sti.o obj-$(CONFIG_DWMAC_STM32) += dwmac-stm32.o diff --git a/drivers/net/ethernet/stmicro/stmmac/chain_mode.c b/drivers/net/ethernet/stmicro/stmmac/chain_mode.c index 37f9417c7c0e..fc04a23342cf 100644 --- a/drivers/net/ethernet/stmicro/stmmac/chain_mode.c +++ b/drivers/net/ethernet/stmicro/stmmac/chain_mode.c @@ -47,7 +47,7 @@ static int jumbo_frm(struct stmmac_tx_queue *tx_q, struct sk_buff *skb, while (len != 0) { tx_q->tx_skbuff[entry] = NULL; - entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_tx_size); + entry = STMMAC_NEXT_ENTRY(entry, priv->dma_conf.dma_tx_size); desc = tx_q->dma_tx + entry; if (len > bmax) { diff --git a/drivers/net/ethernet/stmicro/stmmac/common.h b/drivers/net/ethernet/stmicro/stmmac/common.h index d26e8a063022..927ea6230073 100644 --- a/drivers/net/ethernet/stmicro/stmmac/common.h +++ b/drivers/net/ethernet/stmicro/stmmac/common.h @@ -63,7 +63,7 @@ static inline bool dwmac_is_xmac(enum dwmac_core_type core_type) #define DMA_MIN_RX_SIZE 64 #define DMA_MAX_RX_SIZE 1024 #define DMA_DEFAULT_RX_SIZE 512 -#define STMMAC_GET_ENTRY(x, size) ((x + 1) & (size - 1)) +#define STMMAC_NEXT_ENTRY(x, size) ((x + 1) & (size - 1)) #undef FRAME_FILTER_DEBUG /* #define FRAME_FILTER_DEBUG */ @@ -257,6 +257,7 @@ struct stmmac_safety_stats { (sizeof(struct stmmac_safety_stats) / sizeof(unsigned long)) /* CSR Frequency Access Defines*/ +#define CSR_F_20M 20000000 #define CSR_F_35M 35000000 #define CSR_F_60M 60000000 #define CSR_F_100M 100000000 @@ -277,9 +278,6 @@ struct stmmac_safety_stats { #define FLOW_TX 2 #define FLOW_AUTO (FLOW_TX | FLOW_RX) -/* PCS defines */ -#define STMMAC_PCS_SGMII (1 << 1) - #define SF_DMA_MODE 1 /* DMA STORE-AND-FORWARD Operation Mode */ /* DMA HW feature register fields */ @@ -445,8 +443,8 @@ struct dma_features { unsigned int number_rx_channel; unsigned int number_tx_channel; /* TX and RX number of queues */ - unsigned int number_rx_queues; - unsigned int number_tx_queues; + u8 number_rx_queues; + u8 number_tx_queues; /* PPS output */ unsigned int pps_out_num; /* Number of Traffic Classes */ @@ -607,12 +605,9 @@ struct mac_link { struct mii_regs { unsigned int addr; /* MII Address */ unsigned int data; /* MII Data */ - unsigned int addr_shift; /* MII address shift */ - unsigned int reg_shift; /* MII reg shift */ - unsigned int addr_mask; /* MII address mask */ - unsigned int reg_mask; /* MII reg mask */ - unsigned int clk_csr_shift; - unsigned int clk_csr_mask; + u32 addr_mask; /* MII address mask */ + u32 reg_mask; /* MII reg mask */ + u32 clk_csr_mask; }; struct mac_device_info { @@ -634,8 +629,6 @@ struct mac_device_info { unsigned int unicast_filter_entries; unsigned int mcast_bits_log2; unsigned int rx_csum; - unsigned int pcs; - unsigned int xlgmac; unsigned int num_vlan; u32 vlan_filter[32]; bool vlan_fail_q_en; diff --git a/drivers/net/ethernet/stmicro/stmmac/descs.h b/drivers/net/ethernet/stmicro/stmmac/descs.h index e62e2ebcf273..846007cc245f 100644 --- a/drivers/net/ethernet/stmicro/stmmac/descs.h +++ b/drivers/net/ethernet/stmicro/stmmac/descs.h @@ -173,6 +173,8 @@ struct dma_edesc { struct dma_desc basic; }; +#define dma_desc_to_edesc(x) container_of(x, struct dma_edesc, basic) + /* Transmit checksum insertion control */ #define TX_CIC_FULL 3 /* Include IP header and pseudoheader */ diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-dwc-qos-eth.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-dwc-qos-eth.c index d043bad4a862..b0c5d1ecabce 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-dwc-qos-eth.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-dwc-qos-eth.c @@ -88,7 +88,7 @@ static int dwc_eth_dwmac_config_dt(struct platform_device *pdev, plat_dat->core_type = DWMAC_CORE_GMAC4; plat_dat->dma_cfg->aal = 1; plat_dat->flags |= STMMAC_FLAG_TSO_EN; - plat_dat->pmt = 1; + plat_dat->pmt = true; return 0; } @@ -112,7 +112,8 @@ static int dwc_qos_probe(struct platform_device *pdev, #define AUTO_CAL_STATUS 0x880c #define AUTO_CAL_STATUS_ACTIVE BIT(31) -static void tegra_eqos_fix_speed(void *bsp_priv, int speed, unsigned int mode) +static void tegra_eqos_fix_speed(void *bsp_priv, phy_interface_t interface, + int speed, unsigned int mode) { struct tegra_eqos *eqos = bsp_priv; bool needs_calibration = false; diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-imx.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-imx.c index c4e85197629d..01260dbbb698 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-imx.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-imx.c @@ -42,13 +42,14 @@ struct imx_priv_data; struct imx_dwmac_ops { - u32 addr_width; u32 flags; + u8 addr_width; bool mac_rgmii_txclk_auto_adj; int (*fix_soc_reset)(struct stmmac_priv *priv); int (*set_intf_mode)(struct imx_priv_data *dwmac, u8 phy_intf_sel); - void (*fix_mac_speed)(void *priv, int speed, unsigned int mode); + void (*fix_mac_speed)(void *priv, phy_interface_t interface, + int speed, unsigned int mode); }; struct imx_priv_data { @@ -160,7 +161,8 @@ static int imx_dwmac_set_clk_tx_rate(void *bsp_priv, struct clk *clk_tx_i, return stmmac_set_clk_tx_rate(bsp_priv, clk_tx_i, interface, speed); } -static void imx_dwmac_fix_speed(void *priv, int speed, unsigned int mode) +static void imx_dwmac_fix_speed(void *priv, phy_interface_t interface, + int speed, unsigned int mode) { struct plat_stmmacenet_data *plat_dat; struct imx_priv_data *dwmac = priv; @@ -185,13 +187,14 @@ static void imx_dwmac_fix_speed(void *priv, int speed, unsigned int mode) dev_err(dwmac->dev, "failed to set tx rate %lu\n", rate); } -static void imx93_dwmac_fix_speed(void *priv, int speed, unsigned int mode) +static void imx93_dwmac_fix_speed(void *priv, phy_interface_t interface, + int speed, unsigned int mode) { struct imx_priv_data *dwmac = priv; unsigned int iface; int ctrl, old_ctrl; - imx_dwmac_fix_speed(priv, speed, mode); + imx_dwmac_fix_speed(priv, interface, speed, mode); if (!dwmac || mode != MLO_AN_FIXED) return; @@ -322,11 +325,7 @@ static int imx_dwmac_probe(struct platform_device *pdev) return ret; } - if (data->flags & STMMAC_FLAG_HWTSTAMP_CORRECT_LATENCY) - plat_dat->flags |= STMMAC_FLAG_HWTSTAMP_CORRECT_LATENCY; - - if (data->flags & STMMAC_FLAG_KEEP_PREAMBLE_BEFORE_SFD) - plat_dat->flags |= STMMAC_FLAG_KEEP_PREAMBLE_BEFORE_SFD; + plat_dat->flags |= data->flags; /* Default TX Q0 to use TSO and rest TXQ for TBS */ for (int i = 1; i < plat_dat->tx_queues_to_use; i++) @@ -363,7 +362,8 @@ static struct imx_dwmac_ops imx8mp_dwmac_data = { .addr_width = 34, .mac_rgmii_txclk_auto_adj = false, .set_intf_mode = imx8mp_set_intf_mode, - .flags = STMMAC_FLAG_HWTSTAMP_CORRECT_LATENCY | + .flags = STMMAC_FLAG_EEE_DISABLE | + STMMAC_FLAG_HWTSTAMP_CORRECT_LATENCY | STMMAC_FLAG_KEEP_PREAMBLE_BEFORE_SFD, }; diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-intel.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-intel.c index 92d77b0c2f54..7898b5075a8b 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-intel.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-intel.c @@ -298,7 +298,7 @@ static void tgl_get_interfaces(struct stmmac_priv *priv, void *bsp_priv, if (FIELD_GET(SERDES_LINK_MODE_MASK, data) == SERDES_LINK_MODE_2G5) { dev_info(priv->device, "Link Speed Mode: 2.5Gbps\n"); - priv->plat->mdio_bus_data->default_an_inband = false; + priv->plat->default_an_inband = false; interface = PHY_INTERFACE_MODE_2500BASEX; } else { interface = PHY_INTERFACE_MODE_SGMII; @@ -566,7 +566,7 @@ static void common_default_data(struct plat_stmmacenet_data *plat) /* clk_csr_i = 20-35MHz & MDC = clk_csr_i/16 */ plat->clk_csr = STMMAC_CSR_20_35M; plat->core_type = DWMAC_CORE_GMAC; - plat->force_sf_dma_mode = 1; + plat->force_sf_dma_mode = true; plat->mdio_bus_data->needs_reset = true; } @@ -589,7 +589,7 @@ static int intel_mgbe_common_data(struct pci_dev *pdev, int ret; int i; - plat->pdev = pdev; + plat->provide_bus_info = true; plat->phy_addr = -1; plat->clk_csr = STMMAC_CSR_250_300M; plat->core_type = DWMAC_CORE_GMAC4; @@ -636,8 +636,6 @@ static int intel_mgbe_common_data(struct pci_dev *pdev, plat->dma_cfg->pbl = 32; plat->dma_cfg->pblx8 = true; - plat->dma_cfg->fixed_burst = 0; - plat->dma_cfg->mixed_burst = 0; plat->dma_cfg->aal = 0; plat->dma_cfg->dche = true; @@ -701,8 +699,8 @@ static int intel_mgbe_common_data(struct pci_dev *pdev, /* Intel mgbe SGMII interface uses pcs-xcps */ if (plat->phy_interface == PHY_INTERFACE_MODE_SGMII || plat->phy_interface == PHY_INTERFACE_MODE_1000BASEX) { - plat->mdio_bus_data->pcs_mask = BIT(INTEL_MGBE_XPCS_ADDR); - plat->mdio_bus_data->default_an_inband = true; + plat->mdio_bus_data->pcs_mask = BIT_U32(INTEL_MGBE_XPCS_ADDR); + plat->default_an_inband = true; plat->select_pcs = intel_mgbe_select_pcs; } @@ -1106,7 +1104,7 @@ static int quark_default_data(struct pci_dev *pdev, plat->dma_cfg->pbl = 16; plat->dma_cfg->pblx8 = true; - plat->dma_cfg->fixed_burst = 1; + plat->dma_cfg->fixed_burst = true; /* AXI (TODO) */ return 0; @@ -1253,11 +1251,6 @@ static int intel_eth_pci_probe(struct pci_dev *pdev, if (!plat->mdio_bus_data) return -ENOMEM; - plat->dma_cfg = devm_kzalloc(&pdev->dev, sizeof(*plat->dma_cfg), - GFP_KERNEL); - if (!plat->dma_cfg) - return -ENOMEM; - plat->safety_feat_cfg = devm_kzalloc(&pdev->dev, sizeof(*plat->safety_feat_cfg), GFP_KERNEL); diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-loongson.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-loongson.c index 815213223583..eb14c197d6ae 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-loongson.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-loongson.c @@ -94,7 +94,7 @@ static void loongson_default_data(struct pci_dev *pdev, /* clk_csr_i = 100-150MHz & MDC = clk_csr_i/62 */ plat->clk_csr = STMMAC_CSR_100_150M; plat->core_type = DWMAC_CORE_GMAC; - plat->force_sf_dma_mode = 1; + plat->force_sf_dma_mode = true; /* Increase the default value for multicast hash bins */ plat->multicast_filter_bins = 256; @@ -143,7 +143,8 @@ static struct stmmac_pci_info loongson_gmac_pci_info = { .setup = loongson_gmac_data, }; -static void loongson_gnet_fix_speed(void *priv, int speed, unsigned int mode) +static void loongson_gnet_fix_speed(void *priv, phy_interface_t interface, + int speed, unsigned int mode) { struct loongson_data *ld = (struct loongson_data *)priv; struct net_device *ndev = dev_get_drvdata(ld->dev); @@ -167,7 +168,7 @@ static int loongson_gnet_data(struct pci_dev *pdev, loongson_default_data(pdev, plat); plat->phy_interface = PHY_INTERFACE_MODE_GMII; - plat->mdio_bus_data->phy_mask = ~(u32)BIT(2); + plat->mdio_bus_data->phy_mask = ~BIT_U32(2); plat->fix_mac_speed = loongson_gnet_fix_speed; return 0; @@ -366,12 +367,9 @@ static int loongson_dwmac_setup(void *apriv, struct mac_device_info *mac) mac->link.speed_mask = GMAC_CONTROL_PS | GMAC_CONTROL_FES; mac->mii.addr = GMAC_MII_ADDR; mac->mii.data = GMAC_MII_DATA; - mac->mii.addr_shift = 11; - mac->mii.addr_mask = 0x0000F800; - mac->mii.reg_shift = 6; - mac->mii.reg_mask = 0x000007C0; - mac->mii.clk_csr_shift = 2; - mac->mii.clk_csr_mask = GENMASK(5, 2); + mac->mii.addr_mask = GENMASK_U32(15, 11); + mac->mii.reg_mask = GENMASK_U32(10, 6); + mac->mii.clk_csr_mask = GENMASK_U32(5, 2); return 0; } @@ -515,10 +513,6 @@ static int loongson_dwmac_probe(struct pci_dev *pdev, const struct pci_device_id if (!plat->mdio_bus_data) return -ENOMEM; - plat->dma_cfg = devm_kzalloc(&pdev->dev, sizeof(*plat->dma_cfg), GFP_KERNEL); - if (!plat->dma_cfg) - return -ENOMEM; - ld = devm_kzalloc(&pdev->dev, sizeof(*ld), GFP_KERNEL); if (!ld) return -ENOMEM; diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-mediatek.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-mediatek.c index 1f2d7d19ca56..30ae0dba7fff 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-mediatek.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-mediatek.c @@ -93,9 +93,9 @@ struct mediatek_dwmac_variant { const char * const *clk_list; int num_clks; - u32 dma_bit_mask; u32 rx_delay_max; u32 tx_delay_max; + u8 dma_bit_mask; }; /* list of clocks required for mac */ @@ -268,9 +268,9 @@ static const struct mediatek_dwmac_variant mt2712_gmac_variant = { .dwmac_set_delay = mt2712_set_delay, .clk_list = mt2712_dwmac_clk_l, .num_clks = ARRAY_SIZE(mt2712_dwmac_clk_l), - .dma_bit_mask = 33, .rx_delay_max = 17600, .tx_delay_max = 17600, + .dma_bit_mask = 33, }; static int mt8195_set_interface(struct mediatek_dwmac_plat_data *plat, @@ -418,9 +418,9 @@ static const struct mediatek_dwmac_variant mt8195_gmac_variant = { .dwmac_set_delay = mt8195_set_delay, .clk_list = mt8195_dwmac_clk_l, .num_clks = ARRAY_SIZE(mt8195_dwmac_clk_l), - .dma_bit_mask = 35, .rx_delay_max = 9280, .tx_delay_max = 9280, + .dma_bit_mask = 35, }; static int mediatek_dwmac_config_dt(struct mediatek_dwmac_plat_data *plat) @@ -564,7 +564,7 @@ static int mediatek_dwmac_common_data(struct platform_device *pdev, plat->flags &= ~STMMAC_FLAG_USE_PHY_WOL; else plat->flags |= STMMAC_FLAG_USE_PHY_WOL; - plat->riwt_off = 1; + plat->riwt_off = true; plat->maxmtu = ETH_DATA_LEN; plat->host_dma_width = priv_plat->variant->dma_bit_mask; plat->bsp_priv = priv_plat; diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-motorcomm.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-motorcomm.c index 663d87ccfa0f..124fcd55a429 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-motorcomm.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-motorcomm.c @@ -219,10 +219,6 @@ motorcomm_default_plat_data(struct pci_dev *pdev) if (!plat->mdio_bus_data) return NULL; - plat->dma_cfg = devm_kzalloc(dev, sizeof(*plat->dma_cfg), GFP_KERNEL); - if (!plat->dma_cfg) - return NULL; - plat->axi = devm_kzalloc(dev, sizeof(*plat->axi), GFP_KERNEL); if (!plat->axi) return NULL; @@ -236,7 +232,6 @@ motorcomm_default_plat_data(struct pci_dev *pdev) plat->axi->axi_wr_osr_lmt = 1; plat->axi->axi_rd_osr_lmt = 1; - plat->axi->axi_mb = true; plat->axi->axi_blen_regval = DMA_AXI_BLEN4 | DMA_AXI_BLEN8 | DMA_AXI_BLEN16 | DMA_AXI_BLEN32; diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-nuvoton.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-nuvoton.c new file mode 100644 index 000000000000..e2240b68ad98 --- /dev/null +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-nuvoton.c @@ -0,0 +1,136 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Nuvoton DWMAC specific glue layer + * + * Copyright (C) 2025 Nuvoton Technology Corp. + * + * Author: Joey Lu <a0987203069@gmail.com> + */ + +#include <linux/mfd/syscon.h> +#include <linux/mod_devicetable.h> +#include <linux/of.h> +#include <linux/of_net.h> +#include <linux/platform_device.h> +#include <linux/regmap.h> +#include <linux/stmmac.h> + +#include "stmmac_platform.h" + +#define NVT_REG_SYS_GMAC0MISCR 0x108 +#define NVT_REG_SYS_GMAC1MISCR 0x10C + +#define NVT_MISCR_RMII BIT(0) + +/* Two thousand picoseconds are evenly mapped to a 4-bit field, + * resulting in each step being 2000/15 picoseconds. + */ +#define NVT_PATH_DELAY_STEP 134 +#define NVT_TX_DELAY_MASK GENMASK(19, 16) +#define NVT_RX_DELAY_MASK GENMASK(23, 20) + +struct nvt_priv_data { + struct device *dev; + struct regmap *regmap; + u32 macid; +}; + +static int nvt_gmac_get_delay(struct device *dev, const char *property) +{ + u32 arg; + + if (of_property_read_u32(dev->of_node, property, &arg)) + return 0; + + if (arg > 2000) + return -EINVAL; + + if (arg == 2000) + return 15; + + return arg / NVT_PATH_DELAY_STEP; +} + +static int nvt_set_phy_intf_sel(void *bsp_priv, u8 phy_intf_sel) +{ + struct nvt_priv_data *priv = bsp_priv; + u32 reg, val; + int ret; + + if (phy_intf_sel == PHY_INTF_SEL_RGMII) { + ret = nvt_gmac_get_delay(priv->dev, "rx-internal-delay-ps"); + if (ret < 0) + return ret; + val = FIELD_PREP(NVT_RX_DELAY_MASK, ret); + + ret = nvt_gmac_get_delay(priv->dev, "tx-internal-delay-ps"); + if (ret < 0) + return ret; + val |= FIELD_PREP(NVT_TX_DELAY_MASK, ret); + } else if (phy_intf_sel == PHY_INTF_SEL_RMII) { + val = NVT_MISCR_RMII; + } else { + return -EINVAL; + } + + reg = (priv->macid == 0) ? NVT_REG_SYS_GMAC0MISCR : NVT_REG_SYS_GMAC1MISCR; + regmap_update_bits(priv->regmap, reg, + NVT_RX_DELAY_MASK | NVT_TX_DELAY_MASK | NVT_MISCR_RMII, val); + + return 0; +} + +static int nvt_gmac_probe(struct platform_device *pdev) +{ + struct plat_stmmacenet_data *plat_dat; + struct stmmac_resources stmmac_res; + struct device *dev = &pdev->dev; + struct nvt_priv_data *priv; + int ret; + + ret = stmmac_get_platform_resources(pdev, &stmmac_res); + if (ret) + return dev_err_probe(dev, ret, "Failed to get platform resources\n"); + + plat_dat = devm_stmmac_probe_config_dt(pdev, stmmac_res.mac); + if (IS_ERR(plat_dat)) + return dev_err_probe(dev, PTR_ERR(plat_dat), "Failed to get platform data\n"); + + priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); + if (!priv) + return dev_err_probe(dev, -ENOMEM, "Failed to allocate private data\n"); + + priv->regmap = syscon_regmap_lookup_by_phandle_args(dev->of_node, "nuvoton,sys", + 1, &priv->macid); + if (IS_ERR(priv->regmap)) + return dev_err_probe(dev, PTR_ERR(priv->regmap), "Failed to get sys register\n"); + + if (priv->macid > 1) + return dev_err_probe(dev, -EINVAL, "Invalid sys arguments\n"); + + plat_dat->bsp_priv = priv; + plat_dat->set_phy_intf_sel = nvt_set_phy_intf_sel; + + return stmmac_pltfr_probe(pdev, plat_dat, &stmmac_res); +} + +static const struct of_device_id nvt_dwmac_match[] = { + { .compatible = "nuvoton,ma35d1-dwmac"}, + { } +}; +MODULE_DEVICE_TABLE(of, nvt_dwmac_match); + +static struct platform_driver nvt_dwmac_driver = { + .probe = nvt_gmac_probe, + .remove = stmmac_pltfr_remove, + .driver = { + .name = "nuvoton-dwmac", + .pm = &stmmac_pltfr_pm_ops, + .of_match_table = nvt_dwmac_match, + }, +}; +module_platform_driver(nvt_dwmac_driver); + +MODULE_AUTHOR("Joey Lu <a0987203069@gmail.com>"); +MODULE_DESCRIPTION("Nuvoton DWMAC specific glue layer"); +MODULE_LICENSE("GPL"); diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-qcom-ethqos.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-qcom-ethqos.c index af8204c0e188..ac7d6d3e205a 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-qcom-ethqos.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-qcom-ethqos.c @@ -87,12 +87,12 @@ struct ethqos_emac_por { }; struct ethqos_emac_driver_data { - const struct ethqos_emac_por *por; - unsigned int num_por; + const struct ethqos_emac_por *rgmii_por; + unsigned int num_rgmii_por; bool rgmii_config_loopback_en; bool has_emac_ge_3; + u8 dma_addr_width; const char *link_clk_name; - u32 dma_addr_width; struct dwmac4_addrs dwmac4_addrs; bool needs_sgmii_loopback; }; @@ -100,16 +100,12 @@ struct ethqos_emac_driver_data { struct qcom_ethqos { struct platform_device *pdev; void __iomem *rgmii_base; - int (*configure_func)(struct qcom_ethqos *ethqos, int speed); - - unsigned int link_clk_rate; struct clk *link_clk; struct phy *serdes_phy; - int serdes_speed; phy_interface_t phy_mode; - const struct ethqos_emac_por *por; - unsigned int num_por; + const struct ethqos_emac_por *rgmii_por; + unsigned int num_rgmii_por; bool rgmii_config_loopback_en; bool has_emac_ge_3; bool needs_sgmii_loopback; @@ -174,19 +170,20 @@ static void rgmii_dump(void *priv) rgmii_readl(ethqos, EMAC_SYSTEM_LOW_POWER_DEBUG)); } -static void -ethqos_update_link_clk(struct qcom_ethqos *ethqos, int speed) +static int ethqos_set_clk_tx_rate(void *bsp_priv, struct clk *clk_tx_i, + phy_interface_t interface, int speed) { + struct qcom_ethqos *ethqos = bsp_priv; long rate; - if (!phy_interface_mode_is_rgmii(ethqos->phy_mode)) - return; + if (!phy_interface_mode_is_rgmii(interface)) + return 0; rate = rgmii_clock(speed); - if (rate > 0) - ethqos->link_clk_rate = rate * 2; + if (rate < 0) + return rate; - clk_set_rate(ethqos->link_clk, ethqos->link_clk_rate); + return clk_set_rate(ethqos->link_clk, rate * 2); } static void @@ -204,7 +201,6 @@ qcom_ethqos_set_sgmii_loopback(struct qcom_ethqos *ethqos, bool enable) static void ethqos_set_func_clk_en(struct qcom_ethqos *ethqos) { - qcom_ethqos_set_sgmii_loopback(ethqos, true); rgmii_setmask(ethqos, RGMII_CONFIG_FUNC_CLK_EN, RGMII_IO_MACRO_CONFIG); } @@ -218,8 +214,8 @@ static const struct ethqos_emac_por emac_v2_3_0_por[] = { }; static const struct ethqos_emac_driver_data emac_v2_3_0_data = { - .por = emac_v2_3_0_por, - .num_por = ARRAY_SIZE(emac_v2_3_0_por), + .rgmii_por = emac_v2_3_0_por, + .num_rgmii_por = ARRAY_SIZE(emac_v2_3_0_por), .rgmii_config_loopback_en = true, .has_emac_ge_3 = false, }; @@ -234,8 +230,8 @@ static const struct ethqos_emac_por emac_v2_1_0_por[] = { }; static const struct ethqos_emac_driver_data emac_v2_1_0_data = { - .por = emac_v2_1_0_por, - .num_por = ARRAY_SIZE(emac_v2_1_0_por), + .rgmii_por = emac_v2_1_0_por, + .num_rgmii_por = ARRAY_SIZE(emac_v2_1_0_por), .rgmii_config_loopback_en = false, .has_emac_ge_3 = false, }; @@ -250,8 +246,8 @@ static const struct ethqos_emac_por emac_v3_0_0_por[] = { }; static const struct ethqos_emac_driver_data emac_v3_0_0_data = { - .por = emac_v3_0_0_por, - .num_por = ARRAY_SIZE(emac_v3_0_0_por), + .rgmii_por = emac_v3_0_0_por, + .num_rgmii_por = ARRAY_SIZE(emac_v3_0_0_por), .rgmii_config_loopback_en = false, .has_emac_ge_3 = true, .dwmac4_addrs = { @@ -282,8 +278,8 @@ static const struct ethqos_emac_por emac_v4_0_0_por[] = { }; static const struct ethqos_emac_driver_data emac_v4_0_0_data = { - .por = emac_v4_0_0_por, - .num_por = ARRAY_SIZE(emac_v4_0_0_por), + .rgmii_por = emac_v4_0_0_por, + .num_rgmii_por = ARRAY_SIZE(emac_v4_0_0_por), .rgmii_config_loopback_en = false, .has_emac_ge_3 = true, .link_clk_name = "phyaux", @@ -361,10 +357,12 @@ static int ethqos_dll_configure(struct qcom_ethqos *ethqos) SDCC_HC_REG_DLL_CONFIG2); rgmii_updatel(ethqos, SDCC_DLL_CONFIG2_MCLK_FREQ_CALC, - 0x1A << 10, SDCC_HC_REG_DLL_CONFIG2); + FIELD_PREP(SDCC_DLL_CONFIG2_MCLK_FREQ_CALC, 26), + SDCC_HC_REG_DLL_CONFIG2); rgmii_updatel(ethqos, SDCC_DLL_CONFIG2_DDR_TRAFFIC_INIT_SEL, - BIT(2), SDCC_HC_REG_DLL_CONFIG2); + FIELD_PREP(SDCC_DLL_CONFIG2_DDR_TRAFFIC_INIT_SEL, + 1), SDCC_HC_REG_DLL_CONFIG2); rgmii_setmask(ethqos, SDCC_DLL_CONFIG2_DDR_TRAFFIC_INIT_SW, SDCC_HC_REG_DLL_CONFIG2); @@ -376,155 +374,131 @@ static int ethqos_dll_configure(struct qcom_ethqos *ethqos) static int ethqos_rgmii_macro_init(struct qcom_ethqos *ethqos, int speed) { struct device *dev = ðqos->pdev->dev; - int phase_shift; - int loopback; - - /* Determine if the PHY adds a 2 ns TX delay or the MAC handles it */ - if (ethqos->phy_mode == PHY_INTERFACE_MODE_RGMII_ID || - ethqos->phy_mode == PHY_INTERFACE_MODE_RGMII_TXID) - phase_shift = 0; - else - phase_shift = RGMII_CONFIG2_TX_CLK_PHASE_SHIFT_EN; + unsigned int prg_rclk_dly, loopback; + unsigned int phase_shift; /* Disable loopback mode */ rgmii_clrmask(ethqos, RGMII_CONFIG2_TX_TO_RX_LOOPBACK_EN, RGMII_IO_MACRO_CONFIG2); - /* Determine if this platform wants loopback enabled after programming */ - if (ethqos->rgmii_config_loopback_en) - loopback = RGMII_CONFIG_LOOPBACK_EN; - else - loopback = 0; - /* Select RGMII, write 0 to interface select */ rgmii_clrmask(ethqos, RGMII_CONFIG_INTF_SEL, RGMII_IO_MACRO_CONFIG); - switch (speed) { - case SPEED_1000: - rgmii_setmask(ethqos, RGMII_CONFIG_DDR_MODE, - RGMII_IO_MACRO_CONFIG); + if (speed != SPEED_1000 && speed != SPEED_100 && speed != SPEED_10) { + dev_err(dev, "Invalid speed %d\n", speed); + return -EINVAL; + } + + rgmii_setmask(ethqos, RGMII_CONFIG_DDR_MODE, RGMII_IO_MACRO_CONFIG); + + if (speed == SPEED_1000) { rgmii_clrmask(ethqos, RGMII_CONFIG_BYPASS_TX_ID_EN, RGMII_IO_MACRO_CONFIG); rgmii_setmask(ethqos, RGMII_CONFIG_POS_NEG_DATA_SEL, RGMII_IO_MACRO_CONFIG); rgmii_setmask(ethqos, RGMII_CONFIG_PROG_SWAP, RGMII_IO_MACRO_CONFIG); - rgmii_clrmask(ethqos, RGMII_CONFIG2_DATA_DIVIDE_CLK_SEL, - RGMII_IO_MACRO_CONFIG2); - - rgmii_updatel(ethqos, RGMII_CONFIG2_TX_CLK_PHASE_SHIFT_EN, - phase_shift, RGMII_IO_MACRO_CONFIG2); - rgmii_clrmask(ethqos, RGMII_CONFIG2_RSVD_CONFIG15, - RGMII_IO_MACRO_CONFIG2); - rgmii_setmask(ethqos, RGMII_CONFIG2_RX_PROG_SWAP, - RGMII_IO_MACRO_CONFIG2); - - /* PRG_RCLK_DLY = TCXO period * TCXO_CYCLES_CNT / 2 * RX delay ns, - * in practice this becomes PRG_RCLK_DLY = 52 * 4 / 2 * RX delay ns - */ - if (ethqos->has_emac_ge_3) { - /* 0.9 ns */ - rgmii_updatel(ethqos, SDCC_DDR_CONFIG_PRG_RCLK_DLY, - 115, SDCC_HC_REG_DDR_CONFIG); - } else { - /* 1.8 ns */ - rgmii_updatel(ethqos, SDCC_DDR_CONFIG_PRG_RCLK_DLY, - 57, SDCC_HC_REG_DDR_CONFIG); - } - rgmii_setmask(ethqos, SDCC_DDR_CONFIG_PRG_DLY_EN, - SDCC_HC_REG_DDR_CONFIG); - rgmii_updatel(ethqos, RGMII_CONFIG_LOOPBACK_EN, - loopback, RGMII_IO_MACRO_CONFIG); - break; - - case SPEED_100: - rgmii_setmask(ethqos, RGMII_CONFIG_DDR_MODE, - RGMII_IO_MACRO_CONFIG); + } else { rgmii_setmask(ethqos, RGMII_CONFIG_BYPASS_TX_ID_EN, RGMII_IO_MACRO_CONFIG); rgmii_clrmask(ethqos, RGMII_CONFIG_POS_NEG_DATA_SEL, RGMII_IO_MACRO_CONFIG); rgmii_clrmask(ethqos, RGMII_CONFIG_PROG_SWAP, RGMII_IO_MACRO_CONFIG); - rgmii_clrmask(ethqos, RGMII_CONFIG2_DATA_DIVIDE_CLK_SEL, - RGMII_IO_MACRO_CONFIG2); - rgmii_updatel(ethqos, RGMII_CONFIG2_TX_CLK_PHASE_SHIFT_EN, - phase_shift, RGMII_IO_MACRO_CONFIG2); - rgmii_updatel(ethqos, RGMII_CONFIG_MAX_SPD_PRG_2, - BIT(6), RGMII_IO_MACRO_CONFIG); - rgmii_clrmask(ethqos, RGMII_CONFIG2_RSVD_CONFIG15, - RGMII_IO_MACRO_CONFIG2); + } - if (ethqos->has_emac_ge_3) - rgmii_setmask(ethqos, RGMII_CONFIG2_RX_PROG_SWAP, - RGMII_IO_MACRO_CONFIG2); - else - rgmii_clrmask(ethqos, RGMII_CONFIG2_RX_PROG_SWAP, - RGMII_IO_MACRO_CONFIG2); + rgmii_clrmask(ethqos, RGMII_CONFIG2_DATA_DIVIDE_CLK_SEL, + RGMII_IO_MACRO_CONFIG2); - /* Write 0x5 to PRG_RCLK_DLY_CODE */ - rgmii_updatel(ethqos, SDCC_DDR_CONFIG_EXT_PRG_RCLK_DLY_CODE, - (BIT(29) | BIT(27)), SDCC_HC_REG_DDR_CONFIG); - rgmii_setmask(ethqos, SDCC_DDR_CONFIG_EXT_PRG_RCLK_DLY, - SDCC_HC_REG_DDR_CONFIG); - rgmii_setmask(ethqos, SDCC_DDR_CONFIG_EXT_PRG_RCLK_DLY_EN, - SDCC_HC_REG_DDR_CONFIG); - rgmii_updatel(ethqos, RGMII_CONFIG_LOOPBACK_EN, - loopback, RGMII_IO_MACRO_CONFIG); - break; + /* Determine if the PHY adds a 2 ns TX delay or the MAC handles it */ + if (ethqos->phy_mode == PHY_INTERFACE_MODE_RGMII_ID || + ethqos->phy_mode == PHY_INTERFACE_MODE_RGMII_TXID) + phase_shift = 0; + else + phase_shift = RGMII_CONFIG2_TX_CLK_PHASE_SHIFT_EN; - case SPEED_10: - rgmii_setmask(ethqos, RGMII_CONFIG_DDR_MODE, - RGMII_IO_MACRO_CONFIG); - rgmii_setmask(ethqos, RGMII_CONFIG_BYPASS_TX_ID_EN, - RGMII_IO_MACRO_CONFIG); - rgmii_clrmask(ethqos, RGMII_CONFIG_POS_NEG_DATA_SEL, - RGMII_IO_MACRO_CONFIG); - rgmii_clrmask(ethqos, RGMII_CONFIG_PROG_SWAP, + rgmii_updatel(ethqos, RGMII_CONFIG2_TX_CLK_PHASE_SHIFT_EN, phase_shift, + RGMII_IO_MACRO_CONFIG2); + + if (speed == SPEED_100) + rgmii_updatel(ethqos, RGMII_CONFIG_MAX_SPD_PRG_2, + FIELD_PREP(RGMII_CONFIG_MAX_SPD_PRG_2, 1), RGMII_IO_MACRO_CONFIG); - rgmii_clrmask(ethqos, RGMII_CONFIG2_DATA_DIVIDE_CLK_SEL, - RGMII_IO_MACRO_CONFIG2); - rgmii_updatel(ethqos, RGMII_CONFIG2_TX_CLK_PHASE_SHIFT_EN, - phase_shift, RGMII_IO_MACRO_CONFIG2); + else if (speed == SPEED_10) rgmii_updatel(ethqos, RGMII_CONFIG_MAX_SPD_PRG_9, - BIT(12) | GENMASK(9, 8), + FIELD_PREP(RGMII_CONFIG_MAX_SPD_PRG_9, 19), RGMII_IO_MACRO_CONFIG); - rgmii_clrmask(ethqos, RGMII_CONFIG2_RSVD_CONFIG15, + + rgmii_clrmask(ethqos, RGMII_CONFIG2_RSVD_CONFIG15, + RGMII_IO_MACRO_CONFIG2); + + if (speed == SPEED_1000 || ethqos->has_emac_ge_3) + rgmii_setmask(ethqos, RGMII_CONFIG2_RX_PROG_SWAP, + RGMII_IO_MACRO_CONFIG2); + else + rgmii_clrmask(ethqos, RGMII_CONFIG2_RX_PROG_SWAP, RGMII_IO_MACRO_CONFIG2); - if (ethqos->has_emac_ge_3) - rgmii_setmask(ethqos, RGMII_CONFIG2_RX_PROG_SWAP, - RGMII_IO_MACRO_CONFIG2); - else - rgmii_clrmask(ethqos, RGMII_CONFIG2_RX_PROG_SWAP, - RGMII_IO_MACRO_CONFIG2); + + if (speed != SPEED_1000) { /* Write 0x5 to PRG_RCLK_DLY_CODE */ rgmii_updatel(ethqos, SDCC_DDR_CONFIG_EXT_PRG_RCLK_DLY_CODE, - (BIT(29) | BIT(27)), SDCC_HC_REG_DDR_CONFIG); + FIELD_PREP(SDCC_DDR_CONFIG_EXT_PRG_RCLK_DLY_CODE, + 5), SDCC_HC_REG_DDR_CONFIG); + rgmii_setmask(ethqos, SDCC_DDR_CONFIG_EXT_PRG_RCLK_DLY, SDCC_HC_REG_DDR_CONFIG); + rgmii_setmask(ethqos, SDCC_DDR_CONFIG_EXT_PRG_RCLK_DLY_EN, SDCC_HC_REG_DDR_CONFIG); - rgmii_updatel(ethqos, RGMII_CONFIG_LOOPBACK_EN, - loopback, RGMII_IO_MACRO_CONFIG); - break; - default: - dev_err(dev, "Invalid speed %d\n", speed); - return -EINVAL; + } else { + /* PRG_RCLK_DLY = TCXO period * TCXO_CYCLES_CNT / + * (2 * RX delay ns), + * in practice this becomes PRG_RCLK_DLY = 52 * 4 / + * (2 * RX delay ns) + */ + if (ethqos->has_emac_ge_3) { + /* 0.9 ns */ + prg_rclk_dly = 115; + } else { + /* 1.8 ns */ + prg_rclk_dly = 57; + } + + rgmii_updatel(ethqos, SDCC_DDR_CONFIG_PRG_RCLK_DLY, + FIELD_PREP(SDCC_DDR_CONFIG_PRG_RCLK_DLY, + prg_rclk_dly), SDCC_HC_REG_DDR_CONFIG); + + rgmii_setmask(ethqos, SDCC_DDR_CONFIG_PRG_DLY_EN, + SDCC_HC_REG_DDR_CONFIG); } + if (ethqos->rgmii_config_loopback_en) + loopback = RGMII_CONFIG_LOOPBACK_EN; + else + loopback = 0; + + rgmii_updatel(ethqos, RGMII_CONFIG_LOOPBACK_EN, loopback, + RGMII_IO_MACRO_CONFIG); + return 0; } -static int ethqos_configure_rgmii(struct qcom_ethqos *ethqos, int speed) +static void ethqos_fix_mac_speed_rgmii(void *bsp_priv, + phy_interface_t interface, int speed, + unsigned int mode) { - struct device *dev = ðqos->pdev->dev; + struct qcom_ethqos *ethqos = bsp_priv; + struct device *dev; unsigned int i; u32 val; + dev = ðqos->pdev->dev; + /* Reset to POR values and enable clk */ - for (i = 0; i < ethqos->num_por; i++) - rgmii_writel(ethqos, ethqos->por[i].value, - ethqos->por[i].offset); + for (i = 0; i < ethqos->num_rgmii_por; i++) + rgmii_writel(ethqos, ethqos->rgmii_por[i].value, + ethqos->rgmii_por[i].offset); + ethqos_set_func_clk_en(ethqos); /* Initialize the DLL first */ @@ -580,73 +554,42 @@ static int ethqos_configure_rgmii(struct qcom_ethqos *ethqos, int speed) ethqos_dll_configure(ethqos); ethqos_rgmii_macro_init(ethqos, speed); - - return 0; } -static void ethqos_set_serdes_speed(struct qcom_ethqos *ethqos, int speed) +static void ethqos_pcs_set_inband(struct qcom_ethqos *ethqos, bool enable) { - if (ethqos->serdes_speed != speed) { - phy_set_speed(ethqos->serdes_phy, speed); - ethqos->serdes_speed = speed; - } -} + struct net_device *dev = platform_get_drvdata(ethqos->pdev); + struct stmmac_priv *priv = netdev_priv(dev); -static void ethqos_pcs_set_inband(struct stmmac_priv *priv, bool enable) -{ stmmac_pcs_ctrl_ane(priv, enable, 0); } /* On interface toggle MAC registers gets reset. * Configure MAC block for SGMII on ethernet phy link up */ -static int ethqos_configure_sgmii(struct qcom_ethqos *ethqos, int speed) +static void ethqos_fix_mac_speed_sgmii(void *bsp_priv, + phy_interface_t interface, int speed, + unsigned int mode) { - struct net_device *dev = platform_get_drvdata(ethqos->pdev); - struct stmmac_priv *priv = netdev_priv(dev); + struct qcom_ethqos *ethqos = bsp_priv; switch (speed) { case SPEED_2500: - rgmii_setmask(ethqos, RGMII_CONFIG2_RGMII_CLK_SEL_CFG, - RGMII_IO_MACRO_CONFIG2); - ethqos_set_serdes_speed(ethqos, SPEED_2500); - ethqos_pcs_set_inband(priv, false); - break; case SPEED_1000: rgmii_setmask(ethqos, RGMII_CONFIG2_RGMII_CLK_SEL_CFG, RGMII_IO_MACRO_CONFIG2); - ethqos_set_serdes_speed(ethqos, SPEED_1000); - ethqos_pcs_set_inband(priv, true); break; case SPEED_100: - ethqos_set_serdes_speed(ethqos, SPEED_1000); - ethqos_pcs_set_inband(priv, true); break; case SPEED_10: rgmii_updatel(ethqos, RGMII_CONFIG_SGMII_CLK_DVDR, FIELD_PREP(RGMII_CONFIG_SGMII_CLK_DVDR, SGMII_10M_RX_CLK_DVDR), RGMII_IO_MACRO_CONFIG); - ethqos_set_serdes_speed(ethqos, SPEED_1000); - ethqos_pcs_set_inband(priv, true); break; } - return 0; -} - -static int ethqos_configure(struct qcom_ethqos *ethqos, int speed) -{ - return ethqos->configure_func(ethqos, speed); -} - -static void ethqos_fix_mac_speed(void *priv, int speed, unsigned int mode) -{ - struct qcom_ethqos *ethqos = priv; - - qcom_ethqos_set_sgmii_loopback(ethqos, false); - ethqos_update_link_clk(ethqos, speed); - ethqos_configure(ethqos, speed); + ethqos_pcs_set_inband(ethqos, interface == PHY_INTERFACE_MODE_SGMII); } static int qcom_ethqos_serdes_powerup(struct net_device *ndev, void *priv) @@ -659,16 +602,8 @@ static int qcom_ethqos_serdes_powerup(struct net_device *ndev, void *priv) return ret; ret = phy_power_on(ethqos->serdes_phy); - if (ret) { + if (ret) phy_exit(ethqos->serdes_phy); - return ret; - } - - ret = phy_set_speed(ethqos->serdes_phy, ethqos->serdes_speed); - if (ret) { - phy_power_off(ethqos->serdes_phy); - phy_exit(ethqos->serdes_phy); - } return ret; } @@ -681,6 +616,23 @@ static void qcom_ethqos_serdes_powerdown(struct net_device *ndev, void *priv) phy_exit(ethqos->serdes_phy); } +static int ethqos_mac_finish_serdes(struct net_device *ndev, void *priv, + unsigned int mode, + phy_interface_t interface) +{ + struct qcom_ethqos *ethqos = priv; + int ret = 0; + + qcom_ethqos_set_sgmii_loopback(ethqos, false); + + if (interface == PHY_INTERFACE_MODE_SGMII || + interface == PHY_INTERFACE_MODE_2500BASEX) + ret = phy_set_mode_ext(ethqos->serdes_phy, PHY_MODE_ETHERNET, + interface); + + return ret; +} + static int ethqos_clks_config(void *priv, bool enabled) { struct qcom_ethqos *ethqos = priv; @@ -696,8 +648,9 @@ static int ethqos_clks_config(void *priv, bool enabled) /* Enable functional clock to prevent DMA reset to timeout due * to lacking PHY clock after the hardware block has been power * cycled. The actual configuration will be adjusted once - * ethqos_fix_mac_speed() is invoked. + * ethqos' fix_mac_speed() method is invoked. */ + qcom_ethqos_set_sgmii_loopback(ethqos, true); ethqos_set_func_clk_en(ethqos); } else { clk_disable_unprepare(ethqos->link_clk); @@ -761,11 +714,12 @@ static int qcom_ethqos_probe(struct platform_device *pdev) case PHY_INTERFACE_MODE_RGMII_ID: case PHY_INTERFACE_MODE_RGMII_RXID: case PHY_INTERFACE_MODE_RGMII_TXID: - ethqos->configure_func = ethqos_configure_rgmii; + plat_dat->fix_mac_speed = ethqos_fix_mac_speed_rgmii; break; case PHY_INTERFACE_MODE_2500BASEX: case PHY_INTERFACE_MODE_SGMII: - ethqos->configure_func = ethqos_configure_sgmii; + plat_dat->fix_mac_speed = ethqos_fix_mac_speed_sgmii; + plat_dat->mac_finish = ethqos_mac_finish_serdes; break; default: dev_err(dev, "Unsupported phy mode %s\n", @@ -780,8 +734,8 @@ static int qcom_ethqos_probe(struct platform_device *pdev) "Failed to map rgmii resource\n"); data = of_device_get_match_data(dev); - ethqos->por = data->por; - ethqos->num_por = data->num_por; + ethqos->rgmii_por = data->rgmii_por; + ethqos->num_rgmii_por = data->num_rgmii_por; ethqos->rgmii_config_loopback_en = data->rgmii_config_loopback_en; ethqos->has_emac_ge_3 = data->has_emac_ge_3; ethqos->needs_sgmii_loopback = data->needs_sgmii_loopback; @@ -804,18 +758,26 @@ static int qcom_ethqos_probe(struct platform_device *pdev) return dev_err_probe(dev, PTR_ERR(ethqos->serdes_phy), "Failed to get serdes phy\n"); - ethqos->serdes_speed = SPEED_1000; - ethqos_update_link_clk(ethqos, SPEED_1000); + ethqos_set_clk_tx_rate(ethqos, NULL, plat_dat->phy_interface, + SPEED_1000); + + qcom_ethqos_set_sgmii_loopback(ethqos, true); ethqos_set_func_clk_en(ethqos); + /* The clocks are controlled by firmware, so we don't know for certain + * what clock rate is being used. Hardware documentation mentions that + * the AHB slave clock will be in the range of 50 to 100MHz, which + * equates to a MDC between 1.19 and 2.38MHz. + */ + plat_dat->clk_csr = STMMAC_CSR_60_100M; plat_dat->bsp_priv = ethqos; - plat_dat->fix_mac_speed = ethqos_fix_mac_speed; + plat_dat->set_clk_tx_rate = ethqos_set_clk_tx_rate; plat_dat->dump_debug_regs = rgmii_dump; plat_dat->ptp_clk_freq_config = ethqos_ptp_clk_freq_config; plat_dat->core_type = DWMAC_CORE_GMAC4; if (ethqos->has_emac_ge_3) plat_dat->dwmac4_addrs = &data->dwmac4_addrs; - plat_dat->pmt = 1; + plat_dat->pmt = true; if (of_property_read_bool(np, "snps,tso")) plat_dat->flags |= STMMAC_FLAG_TSO_EN; if (of_device_is_compatible(np, "qcom,qcs404-ethqos")) diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-rk.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-rk.c index b0441a368cb1..8d7042e68926 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-rk.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-rk.c @@ -214,7 +214,7 @@ static int rk_configure_io_clksel(struct rk_priv_data *bsp_priv) cru = !io; /* The io_clksel configuration can be either: - * 0=CRU, 1=IO (rk3506, rk3520, rk3576) or + * 0=CRU, 1=IO (rk3506, rk3528, rk3576) or * 0=IO, 1=CRU (rk3588) * where CRU means the transmit clock comes from the CRU and IO * means the transmit clock comes from IO. diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-s32.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-s32.c index af594a096676..024d8e10e918 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-s32.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-s32.c @@ -2,7 +2,7 @@ /* * NXP S32G/R GMAC glue layer * - * Copyright 2019-2024 NXP + * Copyright 2019-2026 NXP * */ @@ -110,6 +110,37 @@ static void s32_gmac_exit(struct device *dev, void *priv) clk_disable_unprepare(gmac->rx_clk); } +static void s32_gmac_setup_multi_irq(struct device *dev, + struct plat_stmmacenet_data *plat, + struct stmmac_resources *res) +{ + int i; + + /* RX IRQs */ + for (i = 0; i < plat->rx_queues_to_use; i++) { + if (res->rx_irq[i] <= 0) { + dev_dbg(dev, "Missing RX queue %d interrupt\n", i); + goto mac_irq_mode; + } + } + + /* TX IRQs */ + for (i = 0; i < plat->tx_queues_to_use; i++) { + if (res->tx_irq[i] <= 0) { + dev_dbg(dev, "Missing TX queue %d interrupt\n", i); + goto mac_irq_mode; + } + } + + plat->flags |= STMMAC_FLAG_MULTI_MSI_EN; + dev_info(dev, "Multi-IRQ mode (per queue IRQs) selected\n"); + return; + +mac_irq_mode: + plat->flags &= ~STMMAC_FLAG_MULTI_MSI_EN; + dev_info(dev, "MAC IRQ mode selected\n"); +} + static int s32_dwmac_probe(struct platform_device *pdev) { struct plat_stmmacenet_data *plat; @@ -163,8 +194,11 @@ static int s32_dwmac_probe(struct platform_device *pdev) /* S32CC core feature set */ plat->core_type = DWMAC_CORE_GMAC4; - plat->pmt = 1; + plat->pmt = true; plat->flags |= STMMAC_FLAG_SPH_DISABLE; + + s32_gmac_setup_multi_irq(dev, plat, &res); + plat->rx_fifo_size = 20480; plat->tx_fifo_size = 20480; diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-socfpga.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-socfpga.c index 4c8991f3b38d..1d7f0a57d288 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-socfpga.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-socfpga.c @@ -53,14 +53,14 @@ struct socfpga_dwmac; struct socfpga_dwmac_ops { - int (*set_phy_mode)(struct socfpga_dwmac *dwmac_priv); + int (*set_phy_mode)(struct socfpga_dwmac *dwmac_priv, + struct device *dev); void (*setup_plat_dat)(struct socfpga_dwmac *dwmac_priv); }; struct socfpga_dwmac { u32 reg_offset; u32 reg_shift; - struct device *dev; struct plat_stmmacenet_data *plat_dat; struct regmap *sys_mgr_base_addr; struct reset_control *stmmac_rst; @@ -72,18 +72,30 @@ struct socfpga_dwmac { const struct socfpga_dwmac_ops *ops; }; -static void socfpga_dwmac_fix_mac_speed(void *bsp_priv, int speed, +static phy_interface_t socfpga_get_plat_phymode(struct socfpga_dwmac *dwmac) +{ + return dwmac->plat_dat->phy_interface; +} + +static void socfpga_sgmii_config(struct socfpga_dwmac *dwmac, bool enable) +{ + u16 val = enable ? SGMII_ADAPTER_ENABLE : SGMII_ADAPTER_DISABLE; + + writew(val, dwmac->sgmii_adapter_base + SGMII_ADAPTER_CTRL_REG); +} + +static void socfpga_dwmac_fix_mac_speed(void *bsp_priv, + phy_interface_t interface, int speed, unsigned int mode) { struct socfpga_dwmac *dwmac = (struct socfpga_dwmac *)bsp_priv; - struct stmmac_priv *priv = netdev_priv(dev_get_drvdata(dwmac->dev)); - void __iomem *splitter_base = dwmac->splitter_base; void __iomem *sgmii_adapter_base = dwmac->sgmii_adapter_base; + phy_interface_t phymode = socfpga_get_plat_phymode(dwmac); + void __iomem *splitter_base = dwmac->splitter_base; u32 val; if (sgmii_adapter_base) - writew(SGMII_ADAPTER_DISABLE, - sgmii_adapter_base + SGMII_ADAPTER_CTRL_REG); + socfpga_sgmii_config(dwmac, false); if (splitter_base) { val = readl(splitter_base + EMAC_SPLITTER_CTRL_REG); @@ -105,11 +117,9 @@ static void socfpga_dwmac_fix_mac_speed(void *bsp_priv, int speed, writel(val, splitter_base + EMAC_SPLITTER_CTRL_REG); } - if ((priv->plat->phy_interface == PHY_INTERFACE_MODE_SGMII || - priv->plat->phy_interface == PHY_INTERFACE_MODE_1000BASEX) && - sgmii_adapter_base) - writew(SGMII_ADAPTER_ENABLE, - sgmii_adapter_base + SGMII_ADAPTER_CTRL_REG); + if ((phymode == PHY_INTERFACE_MODE_SGMII || + phymode == PHY_INTERFACE_MODE_1000BASEX) && sgmii_adapter_base) + socfpga_sgmii_config(dwmac, true); } static int socfpga_dwmac_parse_data(struct socfpga_dwmac *dwmac, struct device *dev) @@ -233,7 +243,6 @@ static int socfpga_dwmac_parse_data(struct socfpga_dwmac *dwmac, struct device * dwmac->reg_offset = reg_offset; dwmac->reg_shift = reg_shift; dwmac->sys_mgr_base_addr = sys_mgr_base_addr; - dwmac->dev = dev; of_node_put(np_sgmii_adapter); return 0; @@ -243,18 +252,6 @@ err_node_put: return ret; } -static int socfpga_get_plat_phymode(struct socfpga_dwmac *dwmac) -{ - return dwmac->plat_dat->phy_interface; -} - -static void socfpga_sgmii_config(struct socfpga_dwmac *dwmac, bool enable) -{ - u16 val = enable ? SGMII_ADAPTER_ENABLE : SGMII_ADAPTER_DISABLE; - - writew(val, dwmac->sgmii_adapter_base + SGMII_ADAPTER_CTRL_REG); -} - static int socfpga_set_phy_mode_common(int phymode, u32 *val) { switch (phymode) { @@ -384,16 +381,17 @@ static int smtg_crosststamp(ktime_t *device, struct system_counterval_t *system, return 0; } -static int socfpga_gen5_set_phy_mode(struct socfpga_dwmac *dwmac) +static int socfpga_gen5_set_phy_mode(struct socfpga_dwmac *dwmac, + struct device *dev) { struct regmap *sys_mgr_base_addr = dwmac->sys_mgr_base_addr; - int phymode = socfpga_get_plat_phymode(dwmac); + phy_interface_t phymode = socfpga_get_plat_phymode(dwmac); u32 reg_offset = dwmac->reg_offset; u32 reg_shift = dwmac->reg_shift; u32 ctrl, val, module; if (socfpga_set_phy_mode_common(phymode, &val)) { - dev_err(dwmac->dev, "bad phy mode %d\n", phymode); + dev_err(dev, "bad phy mode %d\n", phymode); return -EINVAL; } @@ -442,10 +440,11 @@ static int socfpga_gen5_set_phy_mode(struct socfpga_dwmac *dwmac) return 0; } -static int socfpga_gen10_set_phy_mode(struct socfpga_dwmac *dwmac) +static int socfpga_gen10_set_phy_mode(struct socfpga_dwmac *dwmac, + struct device *dev) { struct regmap *sys_mgr_base_addr = dwmac->sys_mgr_base_addr; - int phymode = socfpga_get_plat_phymode(dwmac); + phy_interface_t phymode = socfpga_get_plat_phymode(dwmac); u32 reg_offset = dwmac->reg_offset; u32 reg_shift = dwmac->reg_shift; u32 ctrl, val, module; @@ -554,7 +553,7 @@ static int socfpga_dwmac_init(struct device *dev, void *bsp_priv) { struct socfpga_dwmac *dwmac = bsp_priv; - return dwmac->ops->set_phy_mode(dwmac); + return dwmac->ops->set_phy_mode(dwmac, dev); } static void socfpga_gen5_setup_plat_dat(struct socfpga_dwmac *dwmac) @@ -564,7 +563,7 @@ static void socfpga_gen5_setup_plat_dat(struct socfpga_dwmac *dwmac) plat_dat->core_type = DWMAC_CORE_GMAC; /* Rx watchdog timer in dwmac is buggy in this hw */ - plat_dat->riwt_off = 1; + plat_dat->riwt_off = true; } static void socfpga_agilex5_setup_plat_dat(struct socfpga_dwmac *dwmac) diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-spacemit.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-spacemit.c new file mode 100644 index 000000000000..223754cc5c79 --- /dev/null +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-spacemit.c @@ -0,0 +1,227 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Spacemit DWMAC platform driver + * + * Copyright (C) 2026 Inochi Amaoto <inochiama@gmail.com> + */ + +#include <linux/clk.h> +#include <linux/math.h> +#include <linux/mod_devicetable.h> +#include <linux/module.h> +#include <linux/mfd/syscon.h> +#include <linux/of.h> +#include <linux/platform_device.h> +#include <linux/property.h> +#include <linux/regmap.h> + +#include "stmmac_platform.h" + +/* ctrl register bits */ +#define CTRL_PHY_INTF_RGMII BIT(3) +#define CTRL_PHY_INTF_MII BIT(4) +#define CTRL_WAKE_IRQ_EN BIT(9) +#define CTRL_PHY_IRQ_EN BIT(12) + +/* dline register bits */ +#define RGMII_RX_DLINE_EN BIT(0) +#define RGMII_RX_DLINE_STEP GENMASK(5, 4) +#define RGMII_RX_DLINE_CODE GENMASK(15, 8) +#define RGMII_TX_DLINE_EN BIT(16) +#define RGMII_TX_DLINE_STEP GENMASK(21, 20) +#define RGMII_TX_DLINE_CODE GENMASK(31, 24) + +#define MAX_DLINE_DELAY_CODE 0xff +#define MAX_WORKED_DELAY 2800 +/* Note: the delay step value is at 0.1ps */ +#define K3_DELAY_STEP 367 + +struct spacmit_dwmac { + struct regmap *apmu; + unsigned int ctrl_offset; + unsigned int dline_offset; +}; + +static int spacemit_dwmac_set_delay(struct spacmit_dwmac *dwmac, + unsigned int tx_code, unsigned int rx_code) +{ + unsigned int mask, val; + + mask = RGMII_TX_DLINE_STEP | RGMII_TX_DLINE_CODE | RGMII_TX_DLINE_EN | + RGMII_RX_DLINE_STEP | RGMII_RX_DLINE_CODE | RGMII_RX_DLINE_EN; + + /* + * Since the delay step provided by config 0 is small enough, and + * it can cover the range of the valid delay, so there is no needed + * to use other step config. + */ + val = FIELD_PREP(RGMII_TX_DLINE_STEP, 0) | + FIELD_PREP(RGMII_TX_DLINE_CODE, tx_code) | RGMII_TX_DLINE_EN | + FIELD_PREP(RGMII_RX_DLINE_STEP, 0) | + FIELD_PREP(RGMII_RX_DLINE_CODE, rx_code) | RGMII_RX_DLINE_EN; + + return regmap_update_bits(dwmac->apmu, dwmac->dline_offset, + mask, val); +} + +static int spacemit_dwmac_detected_delay_value(unsigned int delay) +{ + if (delay == 0) + return 0; + + if (delay > MAX_WORKED_DELAY) + return -EINVAL; + + /* + * Note K3 require a specific factor for calculate + * the delay, in this scenario it is 0.9. So the + * formula is code * step / 10 * 0.9 + */ + return DIV_ROUND_CLOSEST(delay * 10 * 10, K3_DELAY_STEP * 9); +} + +static int spacemit_dwmac_fix_delay(struct spacmit_dwmac *dwmac, + unsigned int tx_delay, + unsigned int rx_delay) +{ + int rx_code; + int tx_code; + + rx_code = spacemit_dwmac_detected_delay_value(rx_delay); + if (rx_code < 0) + return rx_code; + + tx_code = spacemit_dwmac_detected_delay_value(tx_delay); + if (tx_code < 0) + return tx_code; + + return spacemit_dwmac_set_delay(dwmac, tx_code, rx_code); +} + +static int spacemit_dwmac_update_irq_config(struct spacmit_dwmac *dwmac, + struct stmmac_resources *stmmac_res) +{ + unsigned int val = stmmac_res->wol_irq >= 0 ? CTRL_WAKE_IRQ_EN : 0; + unsigned int mask = CTRL_WAKE_IRQ_EN; + + return regmap_update_bits(dwmac->apmu, dwmac->ctrl_offset, + mask, val); +} + +static void spacemit_get_interfaces(struct stmmac_priv *priv, void *bsp_priv, + unsigned long *interfaces) +{ + __set_bit(PHY_INTERFACE_MODE_MII, interfaces); + __set_bit(PHY_INTERFACE_MODE_RMII, interfaces); + phy_interface_set_rgmii(interfaces); +} + +static int spacemit_set_phy_intf_sel(void *bsp_priv, u8 phy_intf_sel) +{ + unsigned int mask = CTRL_PHY_INTF_MII | CTRL_PHY_INTF_RGMII; + struct spacmit_dwmac *dwmac = bsp_priv; + unsigned int val = 0; + + switch (phy_intf_sel) { + case PHY_INTF_SEL_GMII_MII: + val = CTRL_PHY_INTF_MII; + break; + + case PHY_INTF_SEL_RMII: + break; + + case PHY_INTF_SEL_RGMII: + val = CTRL_PHY_INTF_RGMII; + break; + + default: + return -EINVAL; + } + + return regmap_update_bits(dwmac->apmu, dwmac->ctrl_offset, + mask, val); +} + +static int spacemit_dwmac_probe(struct platform_device *pdev) +{ + struct plat_stmmacenet_data *plat_dat; + struct stmmac_resources stmmac_res; + struct device *dev = &pdev->dev; + struct spacmit_dwmac *dwmac; + unsigned int offset[2]; + struct regmap *apmu; + struct clk *clk_tx; + u32 rx_delay = 0; + u32 tx_delay = 0; + int ret; + + ret = stmmac_get_platform_resources(pdev, &stmmac_res); + if (ret) + return dev_err_probe(dev, ret, + "failed to get platform resources\n"); + + dwmac = devm_kzalloc(&pdev->dev, sizeof(*dwmac), GFP_KERNEL); + if (!dwmac) + return -ENOMEM; + + plat_dat = devm_stmmac_probe_config_dt(pdev, stmmac_res.mac); + if (IS_ERR(plat_dat)) + return dev_err_probe(dev, PTR_ERR(plat_dat), + "failed to parse DT parameters\n"); + + clk_tx = devm_clk_get_enabled(&pdev->dev, "tx"); + if (IS_ERR(clk_tx)) + return dev_err_probe(&pdev->dev, PTR_ERR(clk_tx), + "failed to get tx clock\n"); + + apmu = syscon_regmap_lookup_by_phandle_args(pdev->dev.of_node, + "spacemit,apmu", 2, + offset); + if (IS_ERR(apmu)) + return dev_err_probe(dev, PTR_ERR(apmu), + "Failed to get apmu regmap\n"); + + dwmac->apmu = apmu; + dwmac->ctrl_offset = offset[0]; + dwmac->dline_offset = offset[1]; + + ret = spacemit_dwmac_update_irq_config(dwmac, &stmmac_res); + if (ret) + return dev_err_probe(dev, ret, "Failed to configure irq config\n"); + + of_property_read_u32(pdev->dev.of_node, "tx-internal-delay-ps", + &tx_delay); + of_property_read_u32(pdev->dev.of_node, "rx-internal-delay-ps", + &rx_delay); + + plat_dat->get_interfaces = spacemit_get_interfaces; + plat_dat->set_phy_intf_sel = spacemit_set_phy_intf_sel; + plat_dat->bsp_priv = dwmac; + + ret = spacemit_dwmac_fix_delay(dwmac, tx_delay, rx_delay); + if (ret) + return dev_err_probe(dev, ret, "Failed to configure delay\n"); + + return stmmac_dvr_probe(dev, plat_dat, &stmmac_res); +} + +static const struct of_device_id spacemit_dwmac_match[] = { + { .compatible = "spacemit,k3-dwmac" }, + { /* sentinel */ } +}; +MODULE_DEVICE_TABLE(of, spacemit_dwmac_match); + +static struct platform_driver spacemit_dwmac_driver = { + .probe = spacemit_dwmac_probe, + .remove = stmmac_pltfr_remove, + .driver = { + .name = "spacemit-dwmac", + .pm = &stmmac_pltfr_pm_ops, + .of_match_table = spacemit_dwmac_match, + }, +}; +module_platform_driver(spacemit_dwmac_driver); + +MODULE_AUTHOR("Inochi Amaoto <inochiama@gmail.com>"); +MODULE_DESCRIPTION("Spacemit DWMAC platform driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-sti.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-sti.c index f50547b67fbc..6ebbf95d158f 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-sti.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-sti.c @@ -91,11 +91,13 @@ struct sti_dwmac { struct regmap *regmap; bool gmac_en; int speed; - void (*fix_retime_src)(void *priv, int speed, unsigned int mode); + void (*fix_retime_src)(void *priv, phy_interface_t interface, + int speed, unsigned int mode); }; struct sti_dwmac_of_data { - void (*fix_retime_src)(void *priv, int speed, unsigned int mode); + void (*fix_retime_src)(void *priv, phy_interface_t interface, + int speed, unsigned int mode); }; enum { @@ -114,7 +116,8 @@ static u32 stih4xx_tx_retime_val[] = { | STIH4XX_ETH_SEL_INTERNAL_NOTEXT_PHYCLK, }; -static void stih4xx_fix_retime_src(void *priv, int spd, unsigned int mode) +static void stih4xx_fix_retime_src(void *priv, phy_interface_t interface, + int spd, unsigned int mode) { struct sti_dwmac *dwmac = priv; u32 src = dwmac->tx_retime_src; @@ -170,7 +173,7 @@ static int sti_set_phy_intf_sel(void *bsp_priv, u8 phy_intf_sel) val = (dwmac->interface == PHY_INTERFACE_MODE_REVMII) ? 0 : ENMII; regmap_update_bits(regmap, reg, ENMII_MASK, val); - dwmac->fix_retime_src(dwmac, dwmac->speed, 0); + dwmac->fix_retime_src(dwmac, dwmac->interface, dwmac->speed, 0); return 0; } diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-sun8i.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-sun8i.c index c01b86fd64da..48c52eb96233 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-sun8i.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-sun8i.c @@ -718,7 +718,7 @@ static void sun8i_dwmac_set_filter(struct mac_device_info *hw, static void sun8i_dwmac_flow_ctrl(struct mac_device_info *hw, unsigned int duplex, unsigned int fc, - unsigned int pause_time, u32 tx_cnt) + unsigned int pause_time, u8 tx_cnt) { void __iomem *ioaddr = hw->pcsr; u32 v; @@ -1063,12 +1063,9 @@ static int sun8i_dwmac_setup(void *ppriv, struct mac_device_info *mac) mac->link.duplex = EMAC_DUPLEX_FULL; mac->mii.addr = EMAC_MDIO_CMD; mac->mii.data = EMAC_MDIO_DATA; - mac->mii.reg_shift = 4; - mac->mii.reg_mask = GENMASK(8, 4); - mac->mii.addr_shift = 12; - mac->mii.addr_mask = GENMASK(16, 12); - mac->mii.clk_csr_shift = 20; - mac->mii.clk_csr_mask = GENMASK(22, 20); + mac->mii.reg_mask = GENMASK_U32(8, 4); + mac->mii.addr_mask = GENMASK_U32(16, 12); + mac->mii.clk_csr_mask = GENMASK_U32(22, 20); mac->unicast_filter_entries = 8; /* Synopsys Id is not available */ @@ -1182,7 +1179,7 @@ static int sun8i_dwmac_probe(struct platform_device *pdev) * hardware features were copied from Allwinner drivers. */ plat_dat->rx_coe = STMMAC_RX_COE_TYPE2; - plat_dat->tx_coe = 1; + plat_dat->tx_coe = true; plat_dat->flags |= STMMAC_FLAG_HAS_SUN8I; plat_dat->bsp_priv = gmac; plat_dat->init = sun8i_dwmac_init; diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-sunxi.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-sunxi.c index 52593ba3a3a3..74bd996d93c9 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-sunxi.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-sunxi.c @@ -135,7 +135,7 @@ static int sun7i_gmac_probe(struct platform_device *pdev) /* platform data specifying hardware features and callbacks. * hardware features were copied from Allwinner drivers. */ - plat_dat->tx_coe = 1; + plat_dat->tx_coe = true; plat_dat->core_type = DWMAC_CORE_GMAC; plat_dat->bsp_priv = gmac; plat_dat->init = sun7i_gmac_init; diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-tegra.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-tegra.c index 21a0a11fc011..4ede4420c93b 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-tegra.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-tegra.c @@ -325,7 +325,7 @@ static int tegra_mgbe_probe(struct platform_device *pdev) plat->core_type = DWMAC_CORE_XGMAC; plat->flags |= STMMAC_FLAG_TSO_EN; - plat->pmt = 1; + plat->pmt = true; plat->bsp_priv = mgbe; if (!plat->mdio_node) diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac1000.h b/drivers/net/ethernet/stmicro/stmmac/dwmac1000.h index 9fe639fb06bb..1de1f929d61c 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac1000.h +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac1000.h @@ -70,18 +70,8 @@ enum power_event { #define GMAC_RGSMIIIS 0x000000d8 /* RGMII/SMII status */ /* SGMII/RGMII status register */ -#define GMAC_RGSMIIIS_LNKMODE BIT(0) -#define GMAC_RGSMIIIS_SPEED GENMASK(2, 1) -#define GMAC_RGSMIIIS_LNKSTS BIT(3) -#define GMAC_RGSMIIIS_JABTO BIT(4) -#define GMAC_RGSMIIIS_FALSECARDET BIT(5) +#define GMAC_RSGMIIIS_MASK GENMASK(15, 0) #define GMAC_RGSMIIIS_SMIDRXS BIT(16) -/* LNKMOD */ -#define GMAC_RGSMIIIS_LNKMOD_MASK 0x1 -/* LNKSPEED */ -#define GMAC_RGSMIIIS_SPEED_125 0x2 -#define GMAC_RGSMIIIS_SPEED_25 0x1 -#define GMAC_RGSMIIIS_SPEED_2_5 0x0 /* GMAC Configuration defines */ #define GMAC_CONTROL_2K 0x08000000 /* IEEE 802.3as 2K packets */ diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac1000_core.c b/drivers/net/ethernet/stmicro/stmmac/dwmac1000_core.c index af566636fad9..caac85fc08f1 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac1000_core.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac1000_core.c @@ -22,14 +22,19 @@ #include "stmmac_ptp.h" #include "dwmac1000.h" +static const struct stmmac_pcs_info dwmac1000_pcs_info = { + .pcs_offset = GMAC_PCS_BASE, + .rgsmii_offset = GMAC_RGSMIIIS, + .rgsmii_status_mask = GMAC_RSGMIIIS_MASK, + .int_mask = GMAC_INT_DISABLE_PCSLINK | GMAC_INT_DISABLE_PCSAN, +}; + static int dwmac1000_pcs_init(struct stmmac_priv *priv) { if (!priv->dma_cap.pcs) return 0; - return stmmac_integrated_pcs_init(priv, GMAC_PCS_BASE, - GMAC_INT_DISABLE_PCSLINK | - GMAC_INT_DISABLE_PCSAN); + return stmmac_integrated_pcs_init(priv, &dwmac1000_pcs_info); } static void dwmac1000_core_init(struct mac_device_info *hw, @@ -222,7 +227,7 @@ static void dwmac1000_set_filter(struct mac_device_info *hw, static void dwmac1000_flow_ctrl(struct mac_device_info *hw, unsigned int duplex, unsigned int fc, unsigned int pause_time, - u32 tx_cnt) + u8 tx_cnt) { void __iomem *ioaddr = hw->pcsr; /* Set flow such that DZPQ in Mac Register 6 is 0, @@ -496,12 +501,9 @@ int dwmac1000_setup(struct stmmac_priv *priv) mac->link.speed_mask = GMAC_CONTROL_PS | GMAC_CONTROL_FES; mac->mii.addr = GMAC_MII_ADDR; mac->mii.data = GMAC_MII_DATA; - mac->mii.addr_shift = 11; - mac->mii.addr_mask = 0x0000F800; - mac->mii.reg_shift = 6; - mac->mii.reg_mask = 0x000007C0; - mac->mii.clk_csr_shift = 2; - mac->mii.clk_csr_mask = GENMASK(5, 2); + mac->mii.addr_mask = GENMASK_U32(15, 11); + mac->mii.reg_mask = GENMASK_U32(10, 6); + mac->mii.clk_csr_mask = GENMASK_U32(5, 2); return 0; } diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac100_core.c b/drivers/net/ethernet/stmicro/stmmac/dwmac100_core.c index db4fbe64a38a..94d24d355d95 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac100_core.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac100_core.c @@ -126,7 +126,7 @@ static void dwmac100_set_filter(struct mac_device_info *hw, static void dwmac100_flow_ctrl(struct mac_device_info *hw, unsigned int duplex, unsigned int fc, unsigned int pause_time, - u32 tx_cnt) + u8 tx_cnt) { void __iomem *ioaddr = hw->pcsr; unsigned int flow = MAC_FLOW_CTRL_ENABLE; @@ -184,12 +184,9 @@ int dwmac100_setup(struct stmmac_priv *priv) mac->link.speed_mask = MAC_CONTROL_PS; mac->mii.addr = MAC_MII_ADDR; mac->mii.data = MAC_MII_DATA; - mac->mii.addr_shift = 11; - mac->mii.addr_mask = 0x0000F800; - mac->mii.reg_shift = 6; - mac->mii.reg_mask = 0x000007C0; - mac->mii.clk_csr_shift = 2; - mac->mii.clk_csr_mask = GENMASK(5, 2); + mac->mii.addr_mask = GENMASK_U32(15, 11); + mac->mii.reg_mask = GENMASK_U32(10, 6); + mac->mii.clk_csr_mask = GENMASK_U32(5, 2); return 0; } diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac4.h b/drivers/net/ethernet/stmicro/stmmac/dwmac4.h index d797d936aee1..ffcd036d4c02 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac4.h +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac4.h @@ -470,15 +470,7 @@ static inline u32 mtl_low_credx_base_addr(const struct dwmac4_addrs *addrs, #define GMAC_PHYIF_CTRLSTATUS_TC BIT(0) #define GMAC_PHYIF_CTRLSTATUS_LUD BIT(1) #define GMAC_PHYIF_CTRLSTATUS_SMIDRXS BIT(4) -#define GMAC_PHYIF_CTRLSTATUS_LNKMOD BIT(16) -#define GMAC_PHYIF_CTRLSTATUS_SPEED GENMASK(18, 17) -#define GMAC_PHYIF_CTRLSTATUS_LNKSTS BIT(19) -#define GMAC_PHYIF_CTRLSTATUS_JABTO BIT(20) -#define GMAC_PHYIF_CTRLSTATUS_FALSECARDET BIT(21) -/* LNKSPEED */ -#define GMAC_PHYIF_CTRLSTATUS_SPEED_125 0x2 -#define GMAC_PHYIF_CTRLSTATUS_SPEED_25 0x1 -#define GMAC_PHYIF_CTRLSTATUS_SPEED_2_5 0x0 +#define GMAC_PHYIF_CTRLSTATUS_RSGMII_MASK GENMASK(31, 16) extern const struct stmmac_dma_ops dwmac4_dma_ops; extern const struct stmmac_dma_ops dwmac410_dma_ops; diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac4_core.c b/drivers/net/ethernet/stmicro/stmmac/dwmac4_core.c index 623868afe93d..c6fcfae27c3d 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac4_core.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac4_core.c @@ -22,13 +22,19 @@ #include "dwmac4.h" #include "dwmac5.h" +static const struct stmmac_pcs_info dwmac4_pcs_info = { + .pcs_offset = GMAC_PCS_BASE, + .rgsmii_offset = GMAC_PHYIF_CONTROL_STATUS, + .rgsmii_status_mask = GMAC_PHYIF_CTRLSTATUS_RSGMII_MASK, + .int_mask = GMAC_INT_PCS_LINK | GMAC_INT_PCS_ANE, +}; + static int dwmac4_pcs_init(struct stmmac_priv *priv) { if (!priv->dma_cap.pcs) return 0; - return stmmac_integrated_pcs_init(priv, GMAC_PCS_BASE, - GMAC_INT_PCS_LINK | GMAC_INT_PCS_ANE); + return stmmac_integrated_pcs_init(priv, &dwmac4_pcs_info); } static void dwmac4_core_init(struct mac_device_info *hw, @@ -373,8 +379,8 @@ static void dwmac4_get_umac_addr(struct mac_device_info *hw, { void __iomem *ioaddr = hw->pcsr; - stmmac_dwmac4_get_mac_addr(ioaddr, addr, GMAC_ADDR_HIGH(reg_n), - GMAC_ADDR_LOW(reg_n)); + stmmac_get_mac_addr(ioaddr, addr, GMAC_ADDR_HIGH(reg_n), + GMAC_ADDR_LOW(reg_n)); } static int dwmac4_set_lpi_mode(struct mac_device_info *hw, @@ -547,11 +553,11 @@ static void dwmac4_set_filter(struct mac_device_info *hw, static void dwmac4_flow_ctrl(struct mac_device_info *hw, unsigned int duplex, unsigned int fc, unsigned int pause_time, - u32 tx_cnt) + u8 tx_cnt) { void __iomem *ioaddr = hw->pcsr; unsigned int flow = 0; - u32 queue = 0; + u8 queue; pr_debug("GMAC Flow-Control:\n"); if (fc & FLOW_RX) { @@ -1030,12 +1036,9 @@ int dwmac4_setup(struct stmmac_priv *priv) mac->link.speed_mask = GMAC_CONFIG_FES | GMAC_CONFIG_PS; mac->mii.addr = GMAC_MDIO_ADDR; mac->mii.data = GMAC_MDIO_DATA; - mac->mii.addr_shift = 21; - mac->mii.addr_mask = GENMASK(25, 21); - mac->mii.reg_shift = 16; - mac->mii.reg_mask = GENMASK(20, 16); - mac->mii.clk_csr_shift = 8; - mac->mii.clk_csr_mask = GENMASK(11, 8); + mac->mii.addr_mask = GENMASK_U32(25, 21); + mac->mii.reg_mask = GENMASK_U32(20, 16); + mac->mii.clk_csr_mask = GENMASK_U32(11, 8); mac->num_vlan = stmmac_get_num_vlan(priv->ioaddr); return 0; diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac4_descs.c b/drivers/net/ethernet/stmicro/stmmac/dwmac4_descs.c index e226dc6a1b17..2994df41ec2c 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac4_descs.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac4_descs.c @@ -171,16 +171,6 @@ static int dwmac4_wrback_get_rx_status(struct stmmac_extra_stats *x, return ret; } -static int dwmac4_rd_get_tx_len(struct dma_desc *p) -{ - return (le32_to_cpu(p->des2) & TDES2_BUFFER1_SIZE_MASK); -} - -static int dwmac4_get_tx_owner(struct dma_desc *p) -{ - return (le32_to_cpu(p->des3) & TDES3_OWN) >> TDES3_OWN_SHIFT; -} - static void dwmac4_set_tx_owner(struct dma_desc *p) { p->des3 |= cpu_to_le32(TDES3_OWN); @@ -196,12 +186,6 @@ static void dwmac4_set_rx_owner(struct dma_desc *p, int disable_rx_ic) p->des3 |= cpu_to_le32(flags); } -static int dwmac4_get_tx_ls(struct dma_desc *p) -{ - return (le32_to_cpu(p->des3) & TDES3_LAST_DESCRIPTOR) - >> TDES3_LAST_DESCRIPTOR_SHIFT; -} - static u16 dwmac4_wrback_get_rx_vlan_tci(struct dma_desc *p) { return (le32_to_cpu(p->des0) & RDES0_VLAN_TAG_MASK); @@ -305,12 +289,13 @@ exit: } static void dwmac4_rd_init_rx_desc(struct dma_desc *p, int disable_rx_ic, - int mode, int end, int bfsize) + u8 descriptor_mode, int end, int bfsize) { dwmac4_set_rx_owner(p, disable_rx_ic); } -static void dwmac4_rd_init_tx_desc(struct dma_desc *p, int mode, int end) +static void dwmac4_rd_init_tx_desc(struct dma_desc *p, u8 descriptor_mode, + int end) { p->des0 = 0; p->des1 = 0; @@ -319,8 +304,9 @@ static void dwmac4_rd_init_tx_desc(struct dma_desc *p, int mode, int end) } static void dwmac4_rd_prepare_tx_desc(struct dma_desc *p, int is_fs, int len, - bool csum_flag, int mode, bool tx_own, - bool ls, unsigned int tot_pkt_len) + bool csum_flag, u8 descriptor_mode, + bool tx_own, bool ls, + unsigned int tot_pkt_len) { u32 tdes3 = le32_to_cpu(p->des3); @@ -397,7 +383,7 @@ static void dwmac4_rd_prepare_tso_tx_desc(struct dma_desc *p, int is_fs, p->des3 = cpu_to_le32(tdes3); } -static void dwmac4_release_tx_desc(struct dma_desc *p, int mode) +static void dwmac4_release_tx_desc(struct dma_desc *p, u8 descriptor_mode) { p->des0 = 0; p->des1 = 0; @@ -443,7 +429,7 @@ static void dwmac4_display_ring(void *head, unsigned int size, bool rx, extp++; } } else if (desc_size == sizeof(struct dma_edesc)) { - struct dma_edesc *ep = (struct dma_edesc *)head; + struct dma_edesc *ep = dma_desc_to_edesc(head); for (i = 0; i < size; i++) { dma_addr = dma_rx_phy + i * sizeof(*ep); @@ -551,11 +537,8 @@ static void dwmac4_set_tbs(struct dma_edesc *p, u32 sec, u32 nsec) const struct stmmac_desc_ops dwmac4_desc_ops = { .tx_status = dwmac4_wrback_get_tx_status, .rx_status = dwmac4_wrback_get_rx_status, - .get_tx_len = dwmac4_rd_get_tx_len, - .get_tx_owner = dwmac4_get_tx_owner, .set_tx_owner = dwmac4_set_tx_owner, .set_rx_owner = dwmac4_set_rx_owner, - .get_tx_ls = dwmac4_get_tx_ls, .get_rx_vlan_tci = dwmac4_wrback_get_rx_vlan_tci, .get_rx_vlan_valid = dwmac4_wrback_get_rx_vlan_valid, .get_rx_frame_len = dwmac4_wrback_get_rx_frame_len, diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac4_dma.c b/drivers/net/ethernet/stmicro/stmmac/dwmac4_dma.c index 60b880cdd9da..28728271fbc9 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac4_dma.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac4_dma.c @@ -582,7 +582,7 @@ const struct stmmac_dma_ops dwmac410_dma_ops = { .dump_regs = dwmac4_dump_dma_regs, .dma_rx_mode = dwmac4_dma_rx_chan_op_mode, .dma_tx_mode = dwmac4_dma_tx_chan_op_mode, - .enable_dma_irq = dwmac410_enable_dma_irq, + .enable_dma_irq = dwmac4_enable_dma_irq, .disable_dma_irq = dwmac4_disable_dma_irq, .start_tx = dwmac4_dma_start_tx, .stop_tx = dwmac4_dma_stop_tx, diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac4_dma.h b/drivers/net/ethernet/stmicro/stmmac/dwmac4_dma.h index 9d9077a4ac9f..43b036d4e95b 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac4_dma.h +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac4_dma.h @@ -99,6 +99,8 @@ static inline u32 dma_chanx_base_addr(const struct dwmac4_addrs *addrs, #define DMA_CHAN_INTR_ENA_NIE_4_10 BIT(15) #define DMA_CHAN_INTR_ENA_AIE_4_10 BIT(14) #define DMA_CHAN_INTR_ENA_FBE BIT(12) +#define DMA_CHAN_INTR_ENA_RPS BIT(8) +#define DMA_CHAN_INTR_ENA_RBU BIT(7) #define DMA_CHAN_INTR_ENA_RIE BIT(6) #define DMA_CHAN_INTR_ENA_TIE BIT(0) @@ -107,24 +109,24 @@ static inline u32 dma_chanx_base_addr(const struct dwmac4_addrs *addrs, DMA_CHAN_INTR_ENA_TIE) #define DMA_CHAN_INTR_ABNORMAL (DMA_CHAN_INTR_ENA_AIE | \ + DMA_CHAN_INTR_ENA_RPS | \ + DMA_CHAN_INTR_ENA_RBU | \ DMA_CHAN_INTR_ENA_FBE) /* DMA default interrupt mask for 4.00 */ #define DMA_CHAN_INTR_DEFAULT_MASK (DMA_CHAN_INTR_NORMAL | \ DMA_CHAN_INTR_ABNORMAL) -#define DMA_CHAN_INTR_DEFAULT_RX (DMA_CHAN_INTR_ENA_RIE) -#define DMA_CHAN_INTR_DEFAULT_TX (DMA_CHAN_INTR_ENA_TIE) #define DMA_CHAN_INTR_NORMAL_4_10 (DMA_CHAN_INTR_ENA_NIE_4_10 | \ DMA_CHAN_INTR_ENA_RIE | \ DMA_CHAN_INTR_ENA_TIE) #define DMA_CHAN_INTR_ABNORMAL_4_10 (DMA_CHAN_INTR_ENA_AIE_4_10 | \ + DMA_CHAN_INTR_ENA_RPS | \ + DMA_CHAN_INTR_ENA_RBU | \ DMA_CHAN_INTR_ENA_FBE) /* DMA default interrupt mask for 4.10a */ #define DMA_CHAN_INTR_DEFAULT_MASK_4_10 (DMA_CHAN_INTR_NORMAL_4_10 | \ DMA_CHAN_INTR_ABNORMAL_4_10) -#define DMA_CHAN_INTR_DEFAULT_RX_4_10 (DMA_CHAN_INTR_ENA_RIE) -#define DMA_CHAN_INTR_DEFAULT_TX_4_10 (DMA_CHAN_INTR_ENA_TIE) #define DMA_CHAN_RX_WATCHDOG(addrs, x) (dma_chanx_base_addr(addrs, x) + 0x38) #define DMA_CHAN_SLOT_CTRL_STATUS(addrs, x) (dma_chanx_base_addr(addrs, x) + 0x3c) @@ -174,12 +176,8 @@ static inline u32 dma_chanx_base_addr(const struct dwmac4_addrs *addrs, int dwmac4_dma_reset(void __iomem *ioaddr); void dwmac4_enable_dma_irq(struct stmmac_priv *priv, void __iomem *ioaddr, u32 chan, bool rx, bool tx); -void dwmac410_enable_dma_irq(struct stmmac_priv *priv, void __iomem *ioaddr, - u32 chan, bool rx, bool tx); void dwmac4_disable_dma_irq(struct stmmac_priv *priv, void __iomem *ioaddr, u32 chan, bool rx, bool tx); -void dwmac410_disable_dma_irq(struct stmmac_priv *priv, void __iomem *ioaddr, - u32 chan, bool rx, bool tx); void dwmac4_dma_start_tx(struct stmmac_priv *priv, void __iomem *ioaddr, u32 chan); void dwmac4_dma_stop_tx(struct stmmac_priv *priv, void __iomem *ioaddr, diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac4_lib.c b/drivers/net/ethernet/stmicro/stmmac/dwmac4_lib.c index c098047a3bff..a0249715fafa 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac4_lib.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac4_lib.c @@ -116,23 +116,9 @@ void dwmac4_enable_dma_irq(struct stmmac_priv *priv, void __iomem *ioaddr, u32 value = readl(ioaddr + DMA_CHAN_INTR_ENA(dwmac4_addrs, chan)); if (rx) - value |= DMA_CHAN_INTR_DEFAULT_RX; + value |= DMA_CHAN_INTR_ENA_RIE; if (tx) - value |= DMA_CHAN_INTR_DEFAULT_TX; - - writel(value, ioaddr + DMA_CHAN_INTR_ENA(dwmac4_addrs, chan)); -} - -void dwmac410_enable_dma_irq(struct stmmac_priv *priv, void __iomem *ioaddr, - u32 chan, bool rx, bool tx) -{ - const struct dwmac4_addrs *dwmac4_addrs = priv->plat->dwmac4_addrs; - u32 value = readl(ioaddr + DMA_CHAN_INTR_ENA(dwmac4_addrs, chan)); - - if (rx) - value |= DMA_CHAN_INTR_DEFAULT_RX_4_10; - if (tx) - value |= DMA_CHAN_INTR_DEFAULT_TX_4_10; + value |= DMA_CHAN_INTR_ENA_TIE; writel(value, ioaddr + DMA_CHAN_INTR_ENA(dwmac4_addrs, chan)); } @@ -144,23 +130,9 @@ void dwmac4_disable_dma_irq(struct stmmac_priv *priv, void __iomem *ioaddr, u32 value = readl(ioaddr + DMA_CHAN_INTR_ENA(dwmac4_addrs, chan)); if (rx) - value &= ~DMA_CHAN_INTR_DEFAULT_RX; - if (tx) - value &= ~DMA_CHAN_INTR_DEFAULT_TX; - - writel(value, ioaddr + DMA_CHAN_INTR_ENA(dwmac4_addrs, chan)); -} - -void dwmac410_disable_dma_irq(struct stmmac_priv *priv, void __iomem *ioaddr, - u32 chan, bool rx, bool tx) -{ - const struct dwmac4_addrs *dwmac4_addrs = priv->plat->dwmac4_addrs; - u32 value = readl(ioaddr + DMA_CHAN_INTR_ENA(dwmac4_addrs, chan)); - - if (rx) - value &= ~DMA_CHAN_INTR_DEFAULT_RX_4_10; + value &= ~DMA_CHAN_INTR_ENA_RIE; if (tx) - value &= ~DMA_CHAN_INTR_DEFAULT_TX_4_10; + value &= ~DMA_CHAN_INTR_ENA_TIE; writel(value, ioaddr + DMA_CHAN_INTR_ENA(dwmac4_addrs, chan)); } @@ -254,21 +226,3 @@ void stmmac_dwmac4_set_mac(void __iomem *ioaddr, bool enable) if (value != old_val) writel(value, ioaddr + GMAC_CONFIG); } - -void stmmac_dwmac4_get_mac_addr(void __iomem *ioaddr, unsigned char *addr, - unsigned int high, unsigned int low) -{ - unsigned int hi_addr, lo_addr; - - /* Read the MAC address from the hardware */ - hi_addr = readl(ioaddr + high); - lo_addr = readl(ioaddr + low); - - /* Extract the MAC address from the high and low words */ - addr[0] = lo_addr & 0xff; - addr[1] = (lo_addr >> 8) & 0xff; - addr[2] = (lo_addr >> 16) & 0xff; - addr[3] = (lo_addr >> 24) & 0xff; - addr[4] = hi_addr & 0xff; - addr[5] = (hi_addr >> 8) & 0xff; -} diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac5.c b/drivers/net/ethernet/stmicro/stmmac/dwmac5.c index 1c431b918719..f1bb981cab7c 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac5.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac5.c @@ -525,8 +525,6 @@ int dwmac5_flex_pps_config(void __iomem *ioaddr, int index, u32 val = readl(ioaddr + MAC_PPS_CONTROL); u64 period; - if (!cfg->available) - return -EINVAL; if (tnsec & TRGTBUSY0) return -EBUSY; if (!sub_second_inc || !systime_flags) diff --git a/drivers/net/ethernet/stmicro/stmmac/dwxgmac2_core.c b/drivers/net/ethernet/stmicro/stmmac/dwxgmac2_core.c index 49893b9fb88c..f02b434bbd50 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwxgmac2_core.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwxgmac2_core.c @@ -355,10 +355,10 @@ static int dwxgmac2_host_mtl_irq_status(struct stmmac_priv *priv, static void dwxgmac2_flow_ctrl(struct mac_device_info *hw, unsigned int duplex, unsigned int fc, unsigned int pause_time, - u32 tx_cnt) + u8 tx_cnt) { void __iomem *ioaddr = hw->pcsr; - u32 i; + u8 i; if (fc & FLOW_RX) writel(XGMAC_RFE, ioaddr + XGMAC_RX_FLOW_CTRL); @@ -1162,8 +1162,6 @@ static int dwxgmac2_flex_pps_config(void __iomem *ioaddr, int index, u32 val = readl(ioaddr + XGMAC_PPS_CONTROL); u64 period; - if (!cfg->available) - return -EINVAL; if (tnsec & XGMAC_TRGTBUSY0) return -EBUSY; if (!sub_second_inc || !systime_flags) @@ -1551,12 +1549,9 @@ int dwxgmac2_setup(struct stmmac_priv *priv) mac->mii.addr = XGMAC_MDIO_ADDR; mac->mii.data = XGMAC_MDIO_DATA; - mac->mii.addr_shift = 16; - mac->mii.addr_mask = GENMASK(20, 16); - mac->mii.reg_shift = 0; - mac->mii.reg_mask = GENMASK(15, 0); - mac->mii.clk_csr_shift = 19; - mac->mii.clk_csr_mask = GENMASK(21, 19); + mac->mii.addr_mask = GENMASK_U32(20, 16); + mac->mii.reg_mask = GENMASK_U32(15, 0); + mac->mii.clk_csr_mask = GENMASK_U32(21, 19); mac->num_vlan = stmmac_get_num_vlan(priv->ioaddr); return 0; @@ -1594,12 +1589,9 @@ int dwxlgmac2_setup(struct stmmac_priv *priv) mac->mii.addr = XGMAC_MDIO_ADDR; mac->mii.data = XGMAC_MDIO_DATA; - mac->mii.addr_shift = 16; - mac->mii.addr_mask = GENMASK(20, 16); - mac->mii.reg_shift = 0; - mac->mii.reg_mask = GENMASK(15, 0); - mac->mii.clk_csr_shift = 19; - mac->mii.clk_csr_mask = GENMASK(21, 19); + mac->mii.addr_mask = GENMASK_U32(20, 16); + mac->mii.reg_mask = GENMASK_U32(15, 0); + mac->mii.clk_csr_mask = GENMASK_U32(21, 19); return 0; } diff --git a/drivers/net/ethernet/stmicro/stmmac/dwxgmac2_descs.c b/drivers/net/ethernet/stmicro/stmmac/dwxgmac2_descs.c index 41e5b420a215..b5f200a87484 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwxgmac2_descs.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwxgmac2_descs.c @@ -40,16 +40,6 @@ static int dwxgmac2_get_rx_status(struct stmmac_extra_stats *x, return good_frame; } -static int dwxgmac2_get_tx_len(struct dma_desc *p) -{ - return (le32_to_cpu(p->des2) & XGMAC_TDES2_B1L); -} - -static int dwxgmac2_get_tx_owner(struct dma_desc *p) -{ - return (le32_to_cpu(p->des3) & XGMAC_TDES3_OWN) > 0; -} - static void dwxgmac2_set_tx_owner(struct dma_desc *p) { p->des3 |= cpu_to_le32(XGMAC_TDES3_OWN); @@ -65,11 +55,6 @@ static void dwxgmac2_set_rx_owner(struct dma_desc *p, int disable_rx_ic) p->des3 |= cpu_to_le32(flags); } -static int dwxgmac2_get_tx_ls(struct dma_desc *p) -{ - return (le32_to_cpu(p->des3) & XGMAC_RDES3_LD) > 0; -} - static u16 dwxgmac2_wrback_get_rx_vlan_tci(struct dma_desc *p) { return le32_to_cpu(p->des0) & XGMAC_RDES0_VLAN_TAG_MASK; @@ -145,12 +130,13 @@ static int dwxgmac2_get_rx_timestamp_status(void *desc, void *next_desc, } static void dwxgmac2_init_rx_desc(struct dma_desc *p, int disable_rx_ic, - int mode, int end, int bfsize) + u8 descriptor_mode, int end, int bfsize) { dwxgmac2_set_rx_owner(p, disable_rx_ic); } -static void dwxgmac2_init_tx_desc(struct dma_desc *p, int mode, int end) +static void dwxgmac2_init_tx_desc(struct dma_desc *p, u8 descriptor_mode, + int end) { p->des0 = 0; p->des1 = 0; @@ -159,8 +145,9 @@ static void dwxgmac2_init_tx_desc(struct dma_desc *p, int mode, int end) } static void dwxgmac2_prepare_tx_desc(struct dma_desc *p, int is_fs, int len, - bool csum_flag, int mode, bool tx_own, - bool ls, unsigned int tot_pkt_len) + bool csum_flag, u8 descriptor_mode, + bool tx_own, bool ls, + unsigned int tot_pkt_len) { u32 tdes3 = le32_to_cpu(p->des3); @@ -234,7 +221,7 @@ static void dwxgmac2_prepare_tso_tx_desc(struct dma_desc *p, int is_fs, p->des3 = cpu_to_le32(tdes3); } -static void dwxgmac2_release_tx_desc(struct dma_desc *p, int mode) +static void dwxgmac2_release_tx_desc(struct dma_desc *p, u8 descriptor_mode) { p->des0 = 0; p->des1 = 0; @@ -355,11 +342,8 @@ static void dwxgmac2_set_tbs(struct dma_edesc *p, u32 sec, u32 nsec) const struct stmmac_desc_ops dwxgmac210_desc_ops = { .tx_status = dwxgmac2_get_tx_status, .rx_status = dwxgmac2_get_rx_status, - .get_tx_len = dwxgmac2_get_tx_len, - .get_tx_owner = dwxgmac2_get_tx_owner, .set_tx_owner = dwxgmac2_set_tx_owner, .set_rx_owner = dwxgmac2_set_rx_owner, - .get_tx_ls = dwxgmac2_get_tx_ls, .get_rx_vlan_tci = dwxgmac2_wrback_get_rx_vlan_tci, .get_rx_vlan_valid = dwxgmac2_wrback_get_rx_vlan_valid, .get_rx_frame_len = dwxgmac2_get_rx_frame_len, diff --git a/drivers/net/ethernet/stmicro/stmmac/enh_desc.c b/drivers/net/ethernet/stmicro/stmmac/enh_desc.c index 8f6993c8bcae..051253601225 100644 --- a/drivers/net/ethernet/stmicro/stmmac/enh_desc.c +++ b/drivers/net/ethernet/stmicro/stmmac/enh_desc.c @@ -76,11 +76,6 @@ static int enh_desc_get_tx_status(struct stmmac_extra_stats *x, return ret; } -static int enh_desc_get_tx_len(struct dma_desc *p) -{ - return (le32_to_cpu(p->des1) & ETDES1_BUFFER1_SIZE_MASK); -} - static int enh_desc_coe_rdes0(int ipc_err, int type, int payload_err) { int ret = good_frame; @@ -250,7 +245,7 @@ static int enh_desc_get_rx_status(struct stmmac_extra_stats *x, } static void enh_desc_init_rx_desc(struct dma_desc *p, int disable_rx_ic, - int mode, int end, int bfsize) + u8 descriptor_mode, int end, int bfsize) { int bfsize1; @@ -259,7 +254,7 @@ static void enh_desc_init_rx_desc(struct dma_desc *p, int disable_rx_ic, bfsize1 = min(bfsize, BUF_SIZE_8KiB); p->des1 |= cpu_to_le32(bfsize1 & ERDES1_BUFFER1_SIZE_MASK); - if (mode == STMMAC_CHAIN_MODE) + if (descriptor_mode == STMMAC_CHAIN_MODE) ehn_desc_rx_set_on_chain(p); else ehn_desc_rx_set_on_ring(p, end, bfsize); @@ -268,20 +263,16 @@ static void enh_desc_init_rx_desc(struct dma_desc *p, int disable_rx_ic, p->des1 |= cpu_to_le32(ERDES1_DISABLE_IC); } -static void enh_desc_init_tx_desc(struct dma_desc *p, int mode, int end) +static void enh_desc_init_tx_desc(struct dma_desc *p, u8 descriptor_mode, + int end) { p->des0 &= cpu_to_le32(~ETDES0_OWN); - if (mode == STMMAC_CHAIN_MODE) + if (descriptor_mode == STMMAC_CHAIN_MODE) enh_desc_end_tx_desc_on_chain(p); else enh_desc_end_tx_desc_on_ring(p, end); } -static int enh_desc_get_tx_owner(struct dma_desc *p) -{ - return (le32_to_cpu(p->des0) & ETDES0_OWN) >> 31; -} - static void enh_desc_set_tx_owner(struct dma_desc *p) { p->des0 |= cpu_to_le32(ETDES0_OWN); @@ -292,29 +283,25 @@ static void enh_desc_set_rx_owner(struct dma_desc *p, int disable_rx_ic) p->des0 |= cpu_to_le32(RDES0_OWN); } -static int enh_desc_get_tx_ls(struct dma_desc *p) -{ - return (le32_to_cpu(p->des0) & ETDES0_LAST_SEGMENT) >> 29; -} - -static void enh_desc_release_tx_desc(struct dma_desc *p, int mode) +static void enh_desc_release_tx_desc(struct dma_desc *p, u8 descriptor_mode) { int ter = (le32_to_cpu(p->des0) & ETDES0_END_RING) >> 21; memset(p, 0, offsetof(struct dma_desc, des2)); - if (mode == STMMAC_CHAIN_MODE) + if (descriptor_mode == STMMAC_CHAIN_MODE) enh_desc_end_tx_desc_on_chain(p); else enh_desc_end_tx_desc_on_ring(p, ter); } static void enh_desc_prepare_tx_desc(struct dma_desc *p, int is_fs, int len, - bool csum_flag, int mode, bool tx_own, - bool ls, unsigned int tot_pkt_len) + bool csum_flag, u8 descriptor_mode, + bool tx_own, bool ls, + unsigned int tot_pkt_len) { u32 tdes0 = le32_to_cpu(p->des0); - if (mode == STMMAC_CHAIN_MODE) + if (descriptor_mode == STMMAC_CHAIN_MODE) enh_set_tx_desc_len_on_chain(p, len); else enh_set_tx_desc_len_on_ring(p, len); @@ -445,14 +432,11 @@ static void enh_desc_clear(struct dma_desc *p) const struct stmmac_desc_ops enh_desc_ops = { .tx_status = enh_desc_get_tx_status, .rx_status = enh_desc_get_rx_status, - .get_tx_len = enh_desc_get_tx_len, .init_rx_desc = enh_desc_init_rx_desc, .init_tx_desc = enh_desc_init_tx_desc, - .get_tx_owner = enh_desc_get_tx_owner, .release_tx_desc = enh_desc_release_tx_desc, .prepare_tx_desc = enh_desc_prepare_tx_desc, .set_tx_ic = enh_desc_set_tx_ic, - .get_tx_ls = enh_desc_get_tx_ls, .set_tx_owner = enh_desc_set_tx_owner, .set_rx_owner = enh_desc_set_rx_owner, .get_rx_frame_len = enh_desc_get_rx_frame_len, diff --git a/drivers/net/ethernet/stmicro/stmmac/hwif.c b/drivers/net/ethernet/stmicro/stmmac/hwif.c index 7e69ff4b9a98..511b0fd5e834 100644 --- a/drivers/net/ethernet/stmicro/stmmac/hwif.c +++ b/drivers/net/ethernet/stmicro/stmmac/hwif.c @@ -57,11 +57,11 @@ static void stmmac_dwmac_mode_quirk(struct stmmac_priv *priv) if (priv->chain_mode) { dev_info(priv->device, "Chain mode enabled\n"); - priv->mode = STMMAC_CHAIN_MODE; + priv->descriptor_mode = STMMAC_CHAIN_MODE; mac->mode = &chain_mode_ops; } else { dev_info(priv->device, "Ring mode enabled\n"); - priv->mode = STMMAC_RING_MODE; + priv->descriptor_mode = STMMAC_RING_MODE; mac->mode = &ring_mode_ops; } } @@ -76,7 +76,7 @@ static int stmmac_dwmac1_quirks(struct stmmac_priv *priv) /* GMAC older than 3.50 has no extended descriptors */ if (priv->synopsys_id >= DWMAC_CORE_3_50) { dev_info(priv->device, "Enabled extended descriptors\n"); - priv->extend_desc = 1; + priv->extend_desc = true; } else { dev_warn(priv->device, "Extended descriptors not supported\n"); } @@ -97,12 +97,6 @@ static int stmmac_dwmac4_quirks(struct stmmac_priv *priv) return 0; } -static int stmmac_dwxlgmac_quirks(struct stmmac_priv *priv) -{ - priv->hw->xlgmac = true; - return 0; -} - int stmmac_reset(struct stmmac_priv *priv) { struct plat_stmmacenet_data *plat = priv->plat; @@ -293,7 +287,6 @@ static const struct stmmac_hwif_entry { .mmc = &dwxgmac_mmc_ops, .est = &dwmac510_est_ops, .setup = dwxlgmac2_setup, - .quirks = stmmac_dwxlgmac_quirks, }, }; diff --git a/drivers/net/ethernet/stmicro/stmmac/hwif.h b/drivers/net/ethernet/stmicro/stmmac/hwif.h index 0db96a387259..e6317b94fff7 100644 --- a/drivers/net/ethernet/stmicro/stmmac/hwif.h +++ b/drivers/net/ethernet/stmicro/stmmac/hwif.h @@ -38,27 +38,24 @@ struct dma_edesc; /* Descriptors helpers */ struct stmmac_desc_ops { /* DMA RX descriptor ring initialization */ - void (*init_rx_desc)(struct dma_desc *p, int disable_rx_ic, int mode, - int end, int bfsize); + void (*init_rx_desc)(struct dma_desc *p, int disable_rx_ic, + u8 descriptor_mode, int end, int bfsize); /* DMA TX descriptor ring initialization */ - void (*init_tx_desc)(struct dma_desc *p, int mode, int end); + void (*init_tx_desc)(struct dma_desc *p, u8 descriptor_mode, int end); /* Invoked by the xmit function to prepare the tx descriptor */ void (*prepare_tx_desc)(struct dma_desc *p, int is_fs, int len, - bool csum_flag, int mode, bool tx_own, bool ls, - unsigned int tot_pkt_len); + bool csum_flag, u8 descriptor_mode, bool tx_own, + bool ls, unsigned int tot_pkt_len); void (*prepare_tso_tx_desc)(struct dma_desc *p, int is_fs, int len1, int len2, bool tx_own, bool ls, unsigned int tcphdrlen, unsigned int tcppayloadlen); /* Set/get the owner of the descriptor */ void (*set_tx_owner)(struct dma_desc *p); - int (*get_tx_owner)(struct dma_desc *p); /* Clean the tx descriptor as soon as the tx irq is received */ - void (*release_tx_desc)(struct dma_desc *p, int mode); + void (*release_tx_desc)(struct dma_desc *p, u8 descriptor_mode); /* Clear interrupt on tx frame completion. When this bit is * set an interrupt happens as soon as the frame is transmitted */ void (*set_tx_ic)(struct dma_desc *p); - /* Last tx segment reports the transmit status */ - int (*get_tx_ls)(struct dma_desc *p); /* Get the tag of the descriptor */ u16 (*get_rx_vlan_tci)(struct dma_desc *p); /* Get the valid status of descriptor */ @@ -66,8 +63,6 @@ struct stmmac_desc_ops { /* Return the transmit status looking at the TDES1 */ int (*tx_status)(struct stmmac_extra_stats *x, struct dma_desc *p, void __iomem *ioaddr); - /* Get the buffer size from the descriptor */ - int (*get_tx_len)(struct dma_desc *p); /* Handle extra events on specific interrupts hw dependent */ void (*set_rx_owner)(struct dma_desc *p, int disable_rx_ic); /* Get the receive frame size */ @@ -116,22 +111,16 @@ struct stmmac_desc_ops { stmmac_do_void_callback(__priv, desc, prepare_tso_tx_desc, __args) #define stmmac_set_tx_owner(__priv, __args...) \ stmmac_do_void_callback(__priv, desc, set_tx_owner, __args) -#define stmmac_get_tx_owner(__priv, __args...) \ - stmmac_do_callback(__priv, desc, get_tx_owner, __args) #define stmmac_release_tx_desc(__priv, __args...) \ stmmac_do_void_callback(__priv, desc, release_tx_desc, __args) #define stmmac_set_tx_ic(__priv, __args...) \ stmmac_do_void_callback(__priv, desc, set_tx_ic, __args) -#define stmmac_get_tx_ls(__priv, __args...) \ - stmmac_do_callback(__priv, desc, get_tx_ls, __args) #define stmmac_get_rx_vlan_tci(__priv, __args...) \ stmmac_do_callback(__priv, desc, get_rx_vlan_tci, __args) #define stmmac_get_rx_vlan_valid(__priv, __args...) \ stmmac_do_callback(__priv, desc, get_rx_vlan_valid, __args) #define stmmac_tx_status(__priv, __args...) \ stmmac_do_callback(__priv, desc, tx_status, __args) -#define stmmac_get_tx_len(__priv, __args...) \ - stmmac_do_callback(__priv, desc, get_tx_len, __args) #define stmmac_set_rx_owner(__priv, __args...) \ stmmac_do_void_callback(__priv, desc, set_rx_owner, __args) #define stmmac_get_rx_frame_len(__priv, __args...) \ @@ -363,7 +352,7 @@ struct stmmac_ops { void (*set_filter)(struct mac_device_info *hw, struct net_device *dev); /* Flow control setting */ void (*flow_ctrl)(struct mac_device_info *hw, unsigned int duplex, - unsigned int fc, unsigned int pause_time, u32 tx_cnt); + unsigned int fc, unsigned int pause_time, u8 tx_cnt); /* Set power management mode (e.g. magic frame) */ void (*pmt)(struct mac_device_info *hw, unsigned long mode); /* Set/Get Unicast MAC addresses */ diff --git a/drivers/net/ethernet/stmicro/stmmac/norm_desc.c b/drivers/net/ethernet/stmicro/stmmac/norm_desc.c index 859cb9242a52..c4b613564f87 100644 --- a/drivers/net/ethernet/stmicro/stmmac/norm_desc.c +++ b/drivers/net/ethernet/stmicro/stmmac/norm_desc.c @@ -55,11 +55,6 @@ static int ndesc_get_tx_status(struct stmmac_extra_stats *x, return ret; } -static int ndesc_get_tx_len(struct dma_desc *p) -{ - return (le32_to_cpu(p->des1) & RDES1_BUFFER1_SIZE_MASK); -} - /* This function verifies if each incoming frame has some errors * and, if required, updates the multicast statistics. * In case of success, it returns good_frame because the GMAC device @@ -113,8 +108,8 @@ static int ndesc_get_rx_status(struct stmmac_extra_stats *x, return ret; } -static void ndesc_init_rx_desc(struct dma_desc *p, int disable_rx_ic, int mode, - int end, int bfsize) +static void ndesc_init_rx_desc(struct dma_desc *p, int disable_rx_ic, + u8 descriptor_mode, int end, int bfsize) { int bfsize1; @@ -123,7 +118,7 @@ static void ndesc_init_rx_desc(struct dma_desc *p, int disable_rx_ic, int mode, bfsize1 = min(bfsize, BUF_SIZE_2KiB - 1); p->des1 |= cpu_to_le32(bfsize1 & RDES1_BUFFER1_SIZE_MASK); - if (mode == STMMAC_CHAIN_MODE) + if (descriptor_mode == STMMAC_CHAIN_MODE) ndesc_rx_set_on_chain(p, end); else ndesc_rx_set_on_ring(p, end, bfsize); @@ -132,20 +127,15 @@ static void ndesc_init_rx_desc(struct dma_desc *p, int disable_rx_ic, int mode, p->des1 |= cpu_to_le32(RDES1_DISABLE_IC); } -static void ndesc_init_tx_desc(struct dma_desc *p, int mode, int end) +static void ndesc_init_tx_desc(struct dma_desc *p, u8 descriptor_mode, int end) { p->des0 &= cpu_to_le32(~TDES0_OWN); - if (mode == STMMAC_CHAIN_MODE) + if (descriptor_mode == STMMAC_CHAIN_MODE) ndesc_tx_set_on_chain(p); else ndesc_end_tx_desc_on_ring(p, end); } -static int ndesc_get_tx_owner(struct dma_desc *p) -{ - return (le32_to_cpu(p->des0) & TDES0_OWN) >> 31; -} - static void ndesc_set_tx_owner(struct dma_desc *p) { p->des0 |= cpu_to_le32(TDES0_OWN); @@ -156,25 +146,21 @@ static void ndesc_set_rx_owner(struct dma_desc *p, int disable_rx_ic) p->des0 |= cpu_to_le32(RDES0_OWN); } -static int ndesc_get_tx_ls(struct dma_desc *p) -{ - return (le32_to_cpu(p->des1) & TDES1_LAST_SEGMENT) >> 30; -} - -static void ndesc_release_tx_desc(struct dma_desc *p, int mode) +static void ndesc_release_tx_desc(struct dma_desc *p, u8 descriptor_mode) { int ter = (le32_to_cpu(p->des1) & TDES1_END_RING) >> 25; memset(p, 0, offsetof(struct dma_desc, des2)); - if (mode == STMMAC_CHAIN_MODE) + if (descriptor_mode == STMMAC_CHAIN_MODE) ndesc_tx_set_on_chain(p); else ndesc_end_tx_desc_on_ring(p, ter); } static void ndesc_prepare_tx_desc(struct dma_desc *p, int is_fs, int len, - bool csum_flag, int mode, bool tx_own, - bool ls, unsigned int tot_pkt_len) + bool csum_flag, u8 descriptor_mode, + bool tx_own, bool ls, + unsigned int tot_pkt_len) { u32 tdes1 = le32_to_cpu(p->des1); @@ -191,7 +177,7 @@ static void ndesc_prepare_tx_desc(struct dma_desc *p, int is_fs, int len, p->des1 = cpu_to_le32(tdes1); - if (mode == STMMAC_CHAIN_MODE) + if (descriptor_mode == STMMAC_CHAIN_MODE) norm_set_tx_desc_len_on_chain(p, len); else norm_set_tx_desc_len_on_ring(p, len); @@ -291,14 +277,11 @@ static void ndesc_clear(struct dma_desc *p) const struct stmmac_desc_ops ndesc_ops = { .tx_status = ndesc_get_tx_status, .rx_status = ndesc_get_rx_status, - .get_tx_len = ndesc_get_tx_len, .init_rx_desc = ndesc_init_rx_desc, .init_tx_desc = ndesc_init_tx_desc, - .get_tx_owner = ndesc_get_tx_owner, .release_tx_desc = ndesc_release_tx_desc, .prepare_tx_desc = ndesc_prepare_tx_desc, .set_tx_ic = ndesc_set_tx_ic, - .get_tx_ls = ndesc_get_tx_ls, .set_tx_owner = ndesc_set_tx_owner, .set_rx_owner = ndesc_set_rx_owner, .get_rx_frame_len = ndesc_get_rx_frame_len, diff --git a/drivers/net/ethernet/stmicro/stmmac/ring_mode.c b/drivers/net/ethernet/stmicro/stmmac/ring_mode.c index 382d94a3b972..78fc6aa5bbe9 100644 --- a/drivers/net/ethernet/stmicro/stmmac/ring_mode.c +++ b/drivers/net/ethernet/stmicro/stmmac/ring_mode.c @@ -51,7 +51,7 @@ static int jumbo_frm(struct stmmac_tx_queue *tx_q, struct sk_buff *skb, stmmac_prepare_tx_desc(priv, desc, 1, bmax, csum, STMMAC_RING_MODE, 0, false, skb->len); tx_q->tx_skbuff[entry] = NULL; - entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_tx_size); + entry = STMMAC_NEXT_ENTRY(entry, priv->dma_conf.dma_tx_size); if (priv->extend_desc) desc = (struct dma_desc *)(tx_q->dma_etx + entry); diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac.h b/drivers/net/ethernet/stmicro/stmmac/stmmac.h index 33667a26708c..8ba8f03e1ce0 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac.h +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac.h @@ -47,13 +47,13 @@ enum stmmac_txbuf_type { }; struct stmmac_tx_info { + struct xsk_tx_metadata_compl xsk_meta; dma_addr_t buf; - bool map_as_page; unsigned len; + enum stmmac_txbuf_type buf_type; + bool map_as_page; bool last_segment; bool is_jumbo; - enum stmmac_txbuf_type buf_type; - struct xsk_tx_metadata_compl xsk_meta; }; #define STMMAC_TBS_AVAIL BIT(0) @@ -79,7 +79,6 @@ struct stmmac_tx_queue { unsigned int cur_tx; unsigned int dirty_tx; dma_addr_t dma_tx_phy; - dma_addr_t tx_tail_addr; u32 mss; }; @@ -131,7 +130,6 @@ struct stmmac_rx_queue { unsigned int buf_alloc_num; unsigned int napi_skb_frag_size; dma_addr_t dma_rx_phy; - u32 rx_tail_addr; unsigned int state_saved; struct { struct sk_buff *skb; @@ -181,7 +179,6 @@ struct stmmac_tc_entry { #define STMMAC_PPS_MAX 4 struct stmmac_pps_cfg { - bool available; struct timespec64 start; struct timespec64 period; }; @@ -244,6 +241,23 @@ struct stmmac_est { u32 max_sdu[MTL_MAX_TX_QUEUES]; }; +struct stmmac_msi { + int sfty_ce_irq; + int sfty_ue_irq; + int rx_irq[MTL_MAX_RX_QUEUES]; + int tx_irq[MTL_MAX_TX_QUEUES]; + + /*irq name */ + char int_name_mac[IFNAMSIZ + 9]; + char int_name_wol[IFNAMSIZ + 9]; + char int_name_lpi[IFNAMSIZ + 9]; + char int_name_sfty[IFNAMSIZ + 10]; + char int_name_sfty_ce[IFNAMSIZ + 10]; + char int_name_sfty_ue[IFNAMSIZ + 10]; + char int_name_rx_irq[MTL_MAX_RX_QUEUES][IFNAMSIZ + 14]; + char int_name_tx_irq[MTL_MAX_TX_QUEUES][IFNAMSIZ + 18]; +}; + struct stmmac_priv { /* Frequently used values are kept adjacent for cache effect */ u32 tx_coal_frames[MTL_MAX_TX_QUEUES]; @@ -251,8 +265,9 @@ struct stmmac_priv { u32 rx_coal_frames[MTL_MAX_RX_QUEUES]; int hwts_tx_en; + /* skb_shinfo(skb)->gso_type types that we handle */ + unsigned int gso_enabled_types; bool tx_path_in_lpi_mode; - bool tso; bool sph_active; bool sph_capable; u32 sarc_type; @@ -302,9 +317,17 @@ struct stmmac_priv { bool eee_active; bool eee_sw_timer_en; bool legacy_serdes_is_powered; - unsigned int mode; - unsigned int chain_mode; - int extend_desc; + /* descriptor format: + * when clear: struct dma_desc or for tx TBS struct dma_edesc + * when set, struct dma_extended_desc + */ + bool extend_desc; + /* chain_mode: requested descriptor mode */ + bool chain_mode; + /* descriptor_mode: actual descriptor mode, + * see STMMAC_CHAIN_MODE or STMMAC_RING_MODE + */ + u8 descriptor_mode; struct kernel_hwtstamp_config tstamp_config; struct ptp_clock *ptp_clock; struct ptp_clock_info ptp_clock_ops; @@ -325,19 +348,7 @@ struct stmmac_priv { unsigned long active_vlans[BITS_TO_LONGS(VLAN_N_VID)]; unsigned int num_double_vlans; int sfty_irq; - int sfty_ce_irq; - int sfty_ue_irq; - int rx_irq[MTL_MAX_RX_QUEUES]; - int tx_irq[MTL_MAX_TX_QUEUES]; - /*irq name */ - char int_name_mac[IFNAMSIZ + 9]; - char int_name_wol[IFNAMSIZ + 9]; - char int_name_lpi[IFNAMSIZ + 9]; - char int_name_sfty[IFNAMSIZ + 10]; - char int_name_sfty_ce[IFNAMSIZ + 10]; - char int_name_sfty_ue[IFNAMSIZ + 10]; - char int_name_rx_irq[MTL_MAX_RX_QUEUES][IFNAMSIZ + 14]; - char int_name_tx_irq[MTL_MAX_TX_QUEUES][IFNAMSIZ + 18]; + struct stmmac_msi *msi; #ifdef CONFIG_DEBUG_FS struct dentry *dbgfs_dir; @@ -403,7 +414,7 @@ void stmmac_dvr_remove(struct device *dev); int stmmac_dvr_probe(struct device *device, struct plat_stmmacenet_data *plat_dat, struct stmmac_resources *res); -int stmmac_reinit_queues(struct net_device *dev, u32 rx_cnt, u32 tx_cnt); +int stmmac_reinit_queues(struct net_device *dev, u8 rx_cnt, u8 tx_cnt); int stmmac_reinit_ringparam(struct net_device *dev, u32 rx_size, u32 tx_size); int stmmac_set_clk_tx_rate(void *bsp_priv, struct clk *clk_tx_i, phy_interface_t interface, int speed); diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_ethtool.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_ethtool.c index c1e26965d9b5..92585d27ab88 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_ethtool.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_ethtool.c @@ -312,10 +312,9 @@ static void stmmac_ethtool_getdrvinfo(struct net_device *dev, strscpy(info->driver, MAC100_ETHTOOL_NAME, sizeof(info->driver)); - if (priv->plat->pdev) { - strscpy(info->bus_info, pci_name(priv->plat->pdev), + if (priv->plat->provide_bus_info) + strscpy(info->bus_info, dev_name(priv->device), sizeof(info->bus_info)); - } } static int stmmac_ethtool_get_link_ksettings(struct net_device *dev, diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c index 13d3cac056be..01a983001ab4 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c @@ -14,6 +14,7 @@ https://bugzilla.stlinux.com/ *******************************************************************************/ +#include <linux/circ_buf.h> #include <linux/clk.h> #include <linux/kernel.h> #include <linux/interrupt.h> @@ -263,10 +264,10 @@ static void stmmac_verify_args(void) static void __stmmac_disable_all_queues(struct stmmac_priv *priv) { - u32 rx_queues_cnt = priv->plat->rx_queues_to_use; - u32 tx_queues_cnt = priv->plat->tx_queues_to_use; - u32 maxq = max(rx_queues_cnt, tx_queues_cnt); - u32 queue; + u8 rx_queues_cnt = priv->plat->rx_queues_to_use; + u8 tx_queues_cnt = priv->plat->tx_queues_to_use; + u8 maxq = max(rx_queues_cnt, tx_queues_cnt); + u8 queue; for (queue = 0; queue < maxq; queue++) { struct stmmac_channel *ch = &priv->channel[queue]; @@ -290,9 +291,9 @@ static void __stmmac_disable_all_queues(struct stmmac_priv *priv) */ static void stmmac_disable_all_queues(struct stmmac_priv *priv) { - u32 rx_queues_cnt = priv->plat->rx_queues_to_use; + u8 rx_queues_cnt = priv->plat->rx_queues_to_use; struct stmmac_rx_queue *rx_q; - u32 queue; + u8 queue; /* synchronize_rcu() needed for pending XDP buffers to drain */ for (queue = 0; queue < rx_queues_cnt; queue++) { @@ -312,10 +313,10 @@ static void stmmac_disable_all_queues(struct stmmac_priv *priv) */ static void stmmac_enable_all_queues(struct stmmac_priv *priv) { - u32 rx_queues_cnt = priv->plat->rx_queues_to_use; - u32 tx_queues_cnt = priv->plat->tx_queues_to_use; - u32 maxq = max(rx_queues_cnt, tx_queues_cnt); - u32 queue; + u8 rx_queues_cnt = priv->plat->rx_queues_to_use; + u8 tx_queues_cnt = priv->plat->tx_queues_to_use; + u8 maxq = max(rx_queues_cnt, tx_queues_cnt); + u8 queue; for (queue = 0; queue < maxq; queue++) { struct stmmac_channel *ch = &priv->channel[queue]; @@ -356,14 +357,90 @@ static void print_pkt(unsigned char *buf, int len) static inline u32 stmmac_tx_avail(struct stmmac_priv *priv, u32 queue) { struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; - u32 avail; - if (tx_q->dirty_tx > tx_q->cur_tx) - avail = tx_q->dirty_tx - tx_q->cur_tx - 1; + return CIRC_SPACE(tx_q->cur_tx, tx_q->dirty_tx, + priv->dma_conf.dma_tx_size); +} + +static size_t stmmac_get_tx_desc_size(struct stmmac_priv *priv, + struct stmmac_tx_queue *tx_q) +{ + if (priv->extend_desc) + return sizeof(struct dma_extended_desc); + else if (tx_q->tbs & STMMAC_TBS_AVAIL) + return sizeof(struct dma_edesc); + else + return sizeof(struct dma_desc); +} + +static struct dma_desc *stmmac_get_tx_desc(struct stmmac_priv *priv, + struct stmmac_tx_queue *tx_q, + unsigned int index) +{ + if (priv->extend_desc) + return &tx_q->dma_etx[index].basic; + else if (tx_q->tbs & STMMAC_TBS_AVAIL) + return &tx_q->dma_entx[index].basic; + else + return &tx_q->dma_tx[index]; +} + +static void stmmac_set_queue_tx_tail_ptr(struct stmmac_priv *priv, + struct stmmac_tx_queue *tx_q, + unsigned int chan, unsigned int index) +{ + size_t desc_size; + u32 tx_tail_addr; + + desc_size = stmmac_get_tx_desc_size(priv, tx_q); + + tx_tail_addr = tx_q->dma_tx_phy + index * desc_size; + stmmac_set_tx_tail_ptr(priv, priv->ioaddr, tx_tail_addr, chan); +} + +static size_t stmmac_get_rx_desc_size(struct stmmac_priv *priv) +{ + if (priv->extend_desc) + return sizeof(struct dma_extended_desc); + else + return sizeof(struct dma_desc); +} + +static struct dma_desc *stmmac_get_rx_desc(struct stmmac_priv *priv, + struct stmmac_rx_queue *rx_q, + unsigned int index) +{ + if (priv->extend_desc) + return &rx_q->dma_erx[index].basic; + else + return &rx_q->dma_rx[index]; +} + +static void stmmac_set_queue_rx_tail_ptr(struct stmmac_priv *priv, + struct stmmac_rx_queue *rx_q, + unsigned int chan, unsigned int index) +{ + /* This only needs to deal with normal descriptors as enhanced + * descriptiors are only supported with dwmac1000 (<v4.0) which + * does not implement .set_rx_tail_ptr + */ + u32 rx_tail_addr = rx_q->dma_rx_phy + index * sizeof(struct dma_desc); + + stmmac_set_rx_tail_ptr(priv, priv->ioaddr, rx_tail_addr, chan); +} + +static void stmmac_set_queue_rx_buf_size(struct stmmac_priv *priv, + struct stmmac_rx_queue *rx_q, + unsigned int chan) +{ + u32 buf_size; + + if (rx_q->xsk_pool && rx_q->buf_alloc_num) + buf_size = xsk_pool_get_rx_frame_size(rx_q->xsk_pool); else - avail = priv->dma_conf.dma_tx_size - tx_q->cur_tx + tx_q->dirty_tx - 1; + buf_size = priv->dma_conf.dma_buf_sz; - return avail; + stmmac_set_dma_bfsize(priv, priv->ioaddr, buf_size, chan); } /** @@ -374,20 +451,15 @@ static inline u32 stmmac_tx_avail(struct stmmac_priv *priv, u32 queue) static inline u32 stmmac_rx_dirty(struct stmmac_priv *priv, u32 queue) { struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; - u32 dirty; - if (rx_q->dirty_rx <= rx_q->cur_rx) - dirty = rx_q->cur_rx - rx_q->dirty_rx; - else - dirty = priv->dma_conf.dma_rx_size - rx_q->dirty_rx + rx_q->cur_rx; - - return dirty; + return CIRC_CNT(rx_q->cur_rx, rx_q->dirty_rx, + priv->dma_conf.dma_rx_size); } static bool stmmac_eee_tx_busy(struct stmmac_priv *priv) { - u32 tx_cnt = priv->plat->tx_queues_to_use; - u32 queue; + u8 tx_cnt = priv->plat->tx_queues_to_use; + u8 queue; /* check if all TX queues have the work finished */ for (queue = 0; queue < tx_cnt; queue++) { @@ -918,7 +990,7 @@ static int stmmac_legacy_serdes_power_up(struct stmmac_priv *priv) static void stmmac_mac_flow_ctrl(struct stmmac_priv *priv, u32 duplex, unsigned int flow_ctrl) { - u32 tx_cnt = priv->plat->tx_queues_to_use; + u8 tx_cnt = priv->plat->tx_queues_to_use; stmmac_flow_ctrl(priv, priv->hw, duplex, flow_ctrl, priv->pause_time, tx_cnt); @@ -955,11 +1027,8 @@ static struct phylink_pcs *stmmac_mac_select_pcs(struct phylink_config *config, return pcs; } - /* The PCS control register is only relevant for SGMII, TBI and RTBI - * modes. We no longer support TBI or RTBI, so only configure this - * register when operating in SGMII mode with the integrated PCS. - */ - if (priv->hw->pcs & STMMAC_PCS_SGMII && priv->integrated_pcs) + if (priv->integrated_pcs && + test_bit(interface, priv->integrated_pcs->pcs.supported_interfaces)) return &priv->integrated_pcs->pcs; return NULL; @@ -978,7 +1047,8 @@ static int stmmac_mac_finish(struct phylink_config *config, unsigned int mode, struct stmmac_priv *priv = netdev_priv(ndev); if (priv->plat->mac_finish) - priv->plat->mac_finish(ndev, priv->plat->bsp_priv, mode, interface); + priv->plat->mac_finish(ndev, priv->plat->bsp_priv, mode, + interface); return 0; } @@ -1073,7 +1143,8 @@ static void stmmac_mac_link_up(struct phylink_config *config, } if (priv->plat->fix_mac_speed) - priv->plat->fix_mac_speed(priv->plat->bsp_priv, speed, mode); + priv->plat->fix_mac_speed(priv->plat->bsp_priv, interface, + speed, mode); if (!duplex) ctrl &= ~priv->hw->link.duplex; @@ -1216,7 +1287,6 @@ static void stmmac_check_pcs_mode(struct stmmac_priv *priv) if (priv->dma_cap.pcs && interface == PHY_INTERFACE_MODE_SGMII) { netdev_dbg(priv->dev, "PCS SGMII support enabled\n"); - priv->hw->pcs = STMMAC_PCS_SGMII; switch (speed) { case SPEED_10: @@ -1260,10 +1330,7 @@ static int stmmac_init_phy(struct net_device *dev) xpcs_get_an_mode(priv->hw->xpcs, mode) == DW_AN_C73) return 0; - fwnode = priv->plat->port_node; - if (!fwnode) - fwnode = dev_fwnode(priv->device); - + fwnode = dev_fwnode(priv->device); if (fwnode) phy_fwnode = fwnode_get_phy_node(fwnode); else @@ -1319,9 +1386,7 @@ static int stmmac_init_phy(struct net_device *dev) static int stmmac_phylink_setup(struct stmmac_priv *priv) { - struct stmmac_mdio_bus_data *mdio_bus_data; struct phylink_config *config; - struct fwnode_handle *fwnode; struct phylink_pcs *pcs; struct phylink *phylink; @@ -1345,9 +1410,7 @@ static int stmmac_phylink_setup(struct stmmac_priv *priv) priv->tx_lpi_clk_stop = priv->plat->flags & STMMAC_FLAG_EN_TX_LPI_CLOCKGATING; - mdio_bus_data = priv->plat->mdio_bus_data; - if (mdio_bus_data) - config->default_an_inband = mdio_bus_data->default_an_inband; + config->default_an_inband = priv->plat->default_an_inband; /* Get the PHY interface modes (at the PHY end of the link) that * are supported by the platform. @@ -1375,10 +1438,27 @@ static int stmmac_phylink_setup(struct stmmac_priv *priv) config->supported_interfaces, pcs->supported_interfaces); - if (priv->dma_cap.eee) { - /* Assume all supported interfaces also support LPI */ - memcpy(config->lpi_interfaces, config->supported_interfaces, - sizeof(config->lpi_interfaces)); + /* Some platforms, e.g. iMX8MP, wire lpi_intr_o to the same interrupt + * used for stmmac's main interrupts, which leads to interrupt storms. + * STMMAC_FLAG_EEE_DISABLE allows EEE to be disabled on such platforms. + */ + if (priv->dma_cap.eee && + !(priv->plat->flags & STMMAC_FLAG_EEE_DISABLE)) { + /* The GMAC 3.74a databook states that EEE is only supported + * in MII, GMII, and RGMII interfaces. + */ + __set_bit(PHY_INTERFACE_MODE_MII, config->lpi_interfaces); + __set_bit(PHY_INTERFACE_MODE_GMII, config->lpi_interfaces); + phy_interface_set_rgmii(config->lpi_interfaces); + + /* If we have a non-integrated PCS, assume that it is connected + * to the GMAC using GMII or another EEE compatible interface, + * and thus all PCS-supported interfaces support LPI. + */ + if (pcs) + phy_interface_or(config->lpi_interfaces, + config->lpi_interfaces, + pcs->supported_interfaces); /* All full duplex speeds above 100Mbps are supported */ config->lpi_capabilities = ~(MAC_1000FD - 1) | MAC_100FD; @@ -1396,11 +1476,8 @@ static int stmmac_phylink_setup(struct stmmac_priv *priv) config->wol_mac_support |= WAKE_MAGIC; } - fwnode = priv->plat->port_node; - if (!fwnode) - fwnode = dev_fwnode(priv->device); - - phylink = phylink_create(config, fwnode, priv->plat->phy_interface, + phylink = phylink_create(config, dev_fwnode(priv->device), + priv->plat->phy_interface, &stmmac_phylink_mac_ops); if (IS_ERR(phylink)) return PTR_ERR(phylink); @@ -1412,10 +1489,10 @@ static int stmmac_phylink_setup(struct stmmac_priv *priv) static void stmmac_display_rx_rings(struct stmmac_priv *priv, struct stmmac_dma_conf *dma_conf) { - u32 rx_cnt = priv->plat->rx_queues_to_use; + u8 rx_cnt = priv->plat->rx_queues_to_use; unsigned int desc_size; void *head_rx; - u32 queue; + u8 queue; /* Display RX rings */ for (queue = 0; queue < rx_cnt; queue++) { @@ -1423,13 +1500,8 @@ static void stmmac_display_rx_rings(struct stmmac_priv *priv, pr_info("\tRX Queue %u rings\n", queue); - if (priv->extend_desc) { - head_rx = (void *)rx_q->dma_erx; - desc_size = sizeof(struct dma_extended_desc); - } else { - head_rx = (void *)rx_q->dma_rx; - desc_size = sizeof(struct dma_desc); - } + head_rx = stmmac_get_rx_desc(priv, rx_q, 0); + desc_size = stmmac_get_rx_desc_size(priv); /* Display RX ring */ stmmac_display_ring(priv, head_rx, dma_conf->dma_rx_size, true, @@ -1440,10 +1512,10 @@ static void stmmac_display_rx_rings(struct stmmac_priv *priv, static void stmmac_display_tx_rings(struct stmmac_priv *priv, struct stmmac_dma_conf *dma_conf) { - u32 tx_cnt = priv->plat->tx_queues_to_use; + u8 tx_cnt = priv->plat->tx_queues_to_use; unsigned int desc_size; void *head_tx; - u32 queue; + u8 queue; /* Display TX rings */ for (queue = 0; queue < tx_cnt; queue++) { @@ -1451,16 +1523,8 @@ static void stmmac_display_tx_rings(struct stmmac_priv *priv, pr_info("\tTX Queue %d rings\n", queue); - if (priv->extend_desc) { - head_tx = (void *)tx_q->dma_etx; - desc_size = sizeof(struct dma_extended_desc); - } else if (tx_q->tbs & STMMAC_TBS_AVAIL) { - head_tx = (void *)tx_q->dma_entx; - desc_size = sizeof(struct dma_edesc); - } else { - head_tx = (void *)tx_q->dma_tx; - desc_size = sizeof(struct dma_desc); - } + head_tx = stmmac_get_tx_desc(priv, tx_q, 0); + desc_size = stmmac_get_tx_desc_size(priv, tx_q); stmmac_display_ring(priv, head_tx, dma_conf->dma_tx_size, false, tx_q->dma_tx_phy, desc_size); @@ -1516,20 +1580,18 @@ static void stmmac_clear_rx_descriptors(struct stmmac_priv *priv, u32 queue) { struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; + struct dma_desc *desc; int i; /* Clear the RX descriptors */ - for (i = 0; i < dma_conf->dma_rx_size; i++) - if (priv->extend_desc) - stmmac_init_rx_desc(priv, &rx_q->dma_erx[i].basic, - priv->use_riwt, priv->mode, - (i == dma_conf->dma_rx_size - 1), - dma_conf->dma_buf_sz); - else - stmmac_init_rx_desc(priv, &rx_q->dma_rx[i], - priv->use_riwt, priv->mode, - (i == dma_conf->dma_rx_size - 1), - dma_conf->dma_buf_sz); + for (i = 0; i < dma_conf->dma_rx_size; i++) { + desc = stmmac_get_rx_desc(priv, rx_q, i); + + stmmac_init_rx_desc(priv, desc, priv->use_riwt, + priv->descriptor_mode, + (i == dma_conf->dma_rx_size - 1), + dma_conf->dma_buf_sz); + } } /** @@ -1552,14 +1614,8 @@ static void stmmac_clear_tx_descriptors(struct stmmac_priv *priv, int last = (i == (dma_conf->dma_tx_size - 1)); struct dma_desc *p; - if (priv->extend_desc) - p = &tx_q->dma_etx[i].basic; - else if (tx_q->tbs & STMMAC_TBS_AVAIL) - p = &tx_q->dma_entx[i].basic; - else - p = &tx_q->dma_tx[i]; - - stmmac_init_tx_desc(priv, p, priv->mode, last); + p = stmmac_get_tx_desc(priv, tx_q, i); + stmmac_init_tx_desc(priv, p, priv->descriptor_mode, last); } } @@ -1573,9 +1629,9 @@ static void stmmac_clear_tx_descriptors(struct stmmac_priv *priv, static void stmmac_clear_descriptors(struct stmmac_priv *priv, struct stmmac_dma_conf *dma_conf) { - u32 rx_queue_cnt = priv->plat->rx_queues_to_use; - u32 tx_queue_cnt = priv->plat->tx_queues_to_use; - u32 queue; + u8 rx_queue_cnt = priv->plat->rx_queues_to_use; + u8 tx_queue_cnt = priv->plat->tx_queues_to_use; + u8 queue; /* Clear the RX descriptors */ for (queue = 0; queue < rx_queue_cnt; queue++) @@ -1733,10 +1789,7 @@ static int stmmac_alloc_rx_buffers(struct stmmac_priv *priv, struct dma_desc *p; int ret; - if (priv->extend_desc) - p = &((rx_q->dma_erx + i)->basic); - else - p = rx_q->dma_rx + i; + p = stmmac_get_rx_desc(priv, rx_q, i); ret = stmmac_init_rx_buffers(priv, dma_conf, p, i, flags, queue); @@ -1791,10 +1844,7 @@ static int stmmac_alloc_rx_buffers_zc(struct stmmac_priv *priv, dma_addr_t dma_addr; struct dma_desc *p; - if (priv->extend_desc) - p = (struct dma_desc *)(rx_q->dma_erx + i); - else - p = rx_q->dma_rx + i; + p = stmmac_get_rx_desc(priv, rx_q, i); buf = &rx_q->buf_pool[i]; @@ -1833,6 +1883,7 @@ static int __init_dma_rx_desc_rings(struct stmmac_priv *priv, u32 queue, gfp_t flags) { struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; + void *des; int ret; netif_dbg(priv, probe, priv->dev, @@ -1851,7 +1902,7 @@ static int __init_dma_rx_desc_rings(struct stmmac_priv *priv, NULL)); netdev_info(priv->dev, "Register MEM_TYPE_XSK_BUFF_POOL RxQ-%d\n", - rx_q->queue_index); + queue); xsk_pool_set_rxq_info(rx_q->xsk_pool, &rx_q->xdp_rxq); } else { WARN_ON(xdp_rxq_info_reg_mem_model(&rx_q->xdp_rxq, @@ -1859,7 +1910,7 @@ static int __init_dma_rx_desc_rings(struct stmmac_priv *priv, rx_q->page_pool)); netdev_info(priv->dev, "Register MEM_TYPE_PAGE_POOL RxQ-%d\n", - rx_q->queue_index); + queue); } if (rx_q->xsk_pool) { @@ -1874,15 +1925,14 @@ static int __init_dma_rx_desc_rings(struct stmmac_priv *priv, } /* Setup the chained descriptor addresses */ - if (priv->mode == STMMAC_CHAIN_MODE) { + if (priv->descriptor_mode == STMMAC_CHAIN_MODE) { if (priv->extend_desc) - stmmac_mode_init(priv, rx_q->dma_erx, - rx_q->dma_rx_phy, - dma_conf->dma_rx_size, 1); + des = rx_q->dma_erx; else - stmmac_mode_init(priv, rx_q->dma_rx, - rx_q->dma_rx_phy, - dma_conf->dma_rx_size, 0); + des = rx_q->dma_rx; + + stmmac_mode_init(priv, des, rx_q->dma_rx_phy, + dma_conf->dma_rx_size, priv->extend_desc); } return 0; @@ -1893,7 +1943,7 @@ static int init_dma_rx_desc_rings(struct net_device *dev, gfp_t flags) { struct stmmac_priv *priv = netdev_priv(dev); - u32 rx_count = priv->plat->rx_queues_to_use; + u8 rx_count = priv->plat->rx_queues_to_use; int queue; int ret; @@ -1927,6 +1977,34 @@ err_init_rx_buffers: return ret; } +static void stmmac_set_tx_dma_entry(struct stmmac_tx_queue *tx_q, + unsigned int entry, + enum stmmac_txbuf_type type, + dma_addr_t addr, size_t len, + bool map_as_page) +{ + tx_q->tx_skbuff_dma[entry].buf = addr; + tx_q->tx_skbuff_dma[entry].len = len; + tx_q->tx_skbuff_dma[entry].buf_type = type; + tx_q->tx_skbuff_dma[entry].map_as_page = map_as_page; + tx_q->tx_skbuff_dma[entry].last_segment = false; + tx_q->tx_skbuff_dma[entry].is_jumbo = false; +} + +static void stmmac_set_tx_skb_dma_entry(struct stmmac_tx_queue *tx_q, + unsigned int entry, dma_addr_t addr, + size_t len, bool map_as_page) +{ + stmmac_set_tx_dma_entry(tx_q, entry, STMMAC_TXBUF_T_SKB, addr, len, + map_as_page); +} + +static void stmmac_set_tx_dma_last_segment(struct stmmac_tx_queue *tx_q, + unsigned int entry) +{ + tx_q->tx_skbuff_dma[entry].last_segment = true; +} + /** * __init_dma_tx_desc_rings - init the TX descriptor ring (per queue) * @priv: driver private structure @@ -1948,7 +2026,7 @@ static int __init_dma_tx_desc_rings(struct stmmac_priv *priv, (u32)tx_q->dma_tx_phy); /* Setup the chained descriptor addresses */ - if (priv->mode == STMMAC_CHAIN_MODE) { + if (priv->descriptor_mode == STMMAC_CHAIN_MODE) { if (priv->extend_desc) stmmac_mode_init(priv, tx_q->dma_etx, tx_q->dma_tx_phy, @@ -1964,19 +2042,10 @@ static int __init_dma_tx_desc_rings(struct stmmac_priv *priv, for (i = 0; i < dma_conf->dma_tx_size; i++) { struct dma_desc *p; - if (priv->extend_desc) - p = &((tx_q->dma_etx + i)->basic); - else if (tx_q->tbs & STMMAC_TBS_AVAIL) - p = &((tx_q->dma_entx + i)->basic); - else - p = tx_q->dma_tx + i; - + p = stmmac_get_tx_desc(priv, tx_q, i); stmmac_clear_desc(priv, p); + stmmac_set_tx_skb_dma_entry(tx_q, i, 0, 0, false); - tx_q->tx_skbuff_dma[i].buf = 0; - tx_q->tx_skbuff_dma[i].map_as_page = false; - tx_q->tx_skbuff_dma[i].len = 0; - tx_q->tx_skbuff_dma[i].last_segment = false; tx_q->tx_skbuff[i] = NULL; } @@ -1987,8 +2056,8 @@ static int init_dma_tx_desc_rings(struct net_device *dev, struct stmmac_dma_conf *dma_conf) { struct stmmac_priv *priv = netdev_priv(dev); - u32 tx_queue_cnt; - u32 queue; + u8 tx_queue_cnt; + u8 queue; tx_queue_cnt = priv->plat->tx_queues_to_use; @@ -2059,8 +2128,8 @@ static void dma_free_tx_skbufs(struct stmmac_priv *priv, */ static void stmmac_free_tx_skbufs(struct stmmac_priv *priv) { - u32 tx_queue_cnt = priv->plat->tx_queues_to_use; - u32 queue; + u8 tx_queue_cnt = priv->plat->tx_queues_to_use; + u8 queue; for (queue = 0; queue < tx_queue_cnt; queue++) dma_free_tx_skbufs(priv, &priv->dma_conf, queue); @@ -2077,6 +2146,8 @@ static void __free_dma_rx_desc_resources(struct stmmac_priv *priv, u32 queue) { struct stmmac_rx_queue *rx_q = &dma_conf->rx_queue[queue]; + size_t size; + void *addr; /* Release the DMA RX socket buffers */ if (rx_q->xsk_pool) @@ -2088,14 +2159,14 @@ static void __free_dma_rx_desc_resources(struct stmmac_priv *priv, rx_q->xsk_pool = NULL; /* Free DMA regions of consistent memory previously allocated */ - if (!priv->extend_desc) - dma_free_coherent(priv->device, dma_conf->dma_rx_size * - sizeof(struct dma_desc), - rx_q->dma_rx, rx_q->dma_rx_phy); + if (priv->extend_desc) + addr = rx_q->dma_erx; else - dma_free_coherent(priv->device, dma_conf->dma_rx_size * - sizeof(struct dma_extended_desc), - rx_q->dma_erx, rx_q->dma_rx_phy); + addr = rx_q->dma_rx; + + size = stmmac_get_rx_desc_size(priv) * dma_conf->dma_rx_size; + + dma_free_coherent(priv->device, size, addr, rx_q->dma_rx_phy); if (xdp_rxq_info_is_reg(&rx_q->xdp_rxq)) xdp_rxq_info_unreg(&rx_q->xdp_rxq); @@ -2108,8 +2179,8 @@ static void __free_dma_rx_desc_resources(struct stmmac_priv *priv, static void free_dma_rx_desc_resources(struct stmmac_priv *priv, struct stmmac_dma_conf *dma_conf) { - u32 rx_count = priv->plat->rx_queues_to_use; - u32 queue; + u8 rx_count = priv->plat->rx_queues_to_use; + u8 queue; /* Free RX queue resources */ for (queue = 0; queue < rx_count; queue++) @@ -2134,17 +2205,14 @@ static void __free_dma_tx_desc_resources(struct stmmac_priv *priv, dma_free_tx_skbufs(priv, dma_conf, queue); if (priv->extend_desc) { - size = sizeof(struct dma_extended_desc); addr = tx_q->dma_etx; } else if (tx_q->tbs & STMMAC_TBS_AVAIL) { - size = sizeof(struct dma_edesc); addr = tx_q->dma_entx; } else { - size = sizeof(struct dma_desc); addr = tx_q->dma_tx; } - size *= dma_conf->dma_tx_size; + size = stmmac_get_tx_desc_size(priv, tx_q) * dma_conf->dma_tx_size; dma_free_coherent(priv->device, size, addr, tx_q->dma_tx_phy); @@ -2155,8 +2223,8 @@ static void __free_dma_tx_desc_resources(struct stmmac_priv *priv, static void free_dma_tx_desc_resources(struct stmmac_priv *priv, struct stmmac_dma_conf *dma_conf) { - u32 tx_count = priv->plat->tx_queues_to_use; - u32 queue; + u8 tx_count = priv->plat->tx_queues_to_use; + u8 queue; /* Free TX queue resources */ for (queue = 0; queue < tx_count; queue++) @@ -2183,6 +2251,8 @@ static int __alloc_dma_rx_desc_resources(struct stmmac_priv *priv, struct page_pool_params pp_params = { 0 }; unsigned int dma_buf_sz_pad, num_pages; unsigned int napi_id; + size_t size; + void *addr; int ret; dma_buf_sz_pad = stmmac_rx_offset(priv) + dma_conf->dma_buf_sz + @@ -2218,24 +2288,17 @@ static int __alloc_dma_rx_desc_resources(struct stmmac_priv *priv, if (!rx_q->buf_pool) return -ENOMEM; - if (priv->extend_desc) { - rx_q->dma_erx = dma_alloc_coherent(priv->device, - dma_conf->dma_rx_size * - sizeof(struct dma_extended_desc), - &rx_q->dma_rx_phy, - GFP_KERNEL); - if (!rx_q->dma_erx) - return -ENOMEM; + size = stmmac_get_rx_desc_size(priv) * dma_conf->dma_rx_size; - } else { - rx_q->dma_rx = dma_alloc_coherent(priv->device, - dma_conf->dma_rx_size * - sizeof(struct dma_desc), - &rx_q->dma_rx_phy, - GFP_KERNEL); - if (!rx_q->dma_rx) - return -ENOMEM; - } + addr = dma_alloc_coherent(priv->device, size, &rx_q->dma_rx_phy, + GFP_KERNEL); + if (!addr) + return -ENOMEM; + + if (priv->extend_desc) + rx_q->dma_erx = addr; + else + rx_q->dma_rx = addr; if (stmmac_xdp_is_enabled(priv) && test_bit(queue, priv->af_xdp_zc_qps)) @@ -2243,9 +2306,7 @@ static int __alloc_dma_rx_desc_resources(struct stmmac_priv *priv, else napi_id = ch->rx_napi.napi_id; - ret = xdp_rxq_info_reg(&rx_q->xdp_rxq, priv->dev, - rx_q->queue_index, - napi_id); + ret = xdp_rxq_info_reg(&rx_q->xdp_rxq, priv->dev, queue, napi_id); if (ret) { netdev_err(priv->dev, "Failed to register xdp rxq info\n"); return -EINVAL; @@ -2257,8 +2318,8 @@ static int __alloc_dma_rx_desc_resources(struct stmmac_priv *priv, static int alloc_dma_rx_desc_resources(struct stmmac_priv *priv, struct stmmac_dma_conf *dma_conf) { - u32 rx_count = priv->plat->rx_queues_to_use; - u32 queue; + u8 rx_count = priv->plat->rx_queues_to_use; + u8 queue; int ret; /* RX queues buffers and DMA */ @@ -2306,14 +2367,7 @@ static int __alloc_dma_tx_desc_resources(struct stmmac_priv *priv, if (!tx_q->tx_skbuff) return -ENOMEM; - if (priv->extend_desc) - size = sizeof(struct dma_extended_desc); - else if (tx_q->tbs & STMMAC_TBS_AVAIL) - size = sizeof(struct dma_edesc); - else - size = sizeof(struct dma_desc); - - size *= dma_conf->dma_tx_size; + size = stmmac_get_tx_desc_size(priv, tx_q) * dma_conf->dma_tx_size; addr = dma_alloc_coherent(priv->device, size, &tx_q->dma_tx_phy, GFP_KERNEL); @@ -2333,8 +2387,8 @@ static int __alloc_dma_tx_desc_resources(struct stmmac_priv *priv, static int alloc_dma_tx_desc_resources(struct stmmac_priv *priv, struct stmmac_dma_conf *dma_conf) { - u32 tx_count = priv->plat->tx_queues_to_use; - u32 queue; + u8 tx_count = priv->plat->tx_queues_to_use; + u8 queue; int ret; /* TX queues buffers and DMA */ @@ -2398,8 +2452,8 @@ static void free_dma_desc_resources(struct stmmac_priv *priv, */ static void stmmac_mac_enable_rx_queues(struct stmmac_priv *priv) { - u32 rx_queues_count = priv->plat->rx_queues_to_use; - int queue; + u8 rx_queues_count = priv->plat->rx_queues_to_use; + u8 queue; u8 mode; for (queue = 0; queue < rx_queues_count; queue++) { @@ -2462,10 +2516,10 @@ static void stmmac_stop_tx_dma(struct stmmac_priv *priv, u32 chan) static void stmmac_enable_all_dma_irq(struct stmmac_priv *priv) { - u32 rx_channels_count = priv->plat->rx_queues_to_use; - u32 tx_channels_count = priv->plat->tx_queues_to_use; - u32 dma_csr_ch = max(rx_channels_count, tx_channels_count); - u32 chan; + u8 rx_channels_count = priv->plat->rx_queues_to_use; + u8 tx_channels_count = priv->plat->tx_queues_to_use; + u8 dma_csr_ch = max(rx_channels_count, tx_channels_count); + u8 chan; for (chan = 0; chan < dma_csr_ch; chan++) { struct stmmac_channel *ch = &priv->channel[chan]; @@ -2485,9 +2539,9 @@ static void stmmac_enable_all_dma_irq(struct stmmac_priv *priv) */ static void stmmac_start_all_dma(struct stmmac_priv *priv) { - u32 rx_channels_count = priv->plat->rx_queues_to_use; - u32 tx_channels_count = priv->plat->tx_queues_to_use; - u32 chan = 0; + u8 rx_channels_count = priv->plat->rx_queues_to_use; + u8 tx_channels_count = priv->plat->tx_queues_to_use; + u8 chan; for (chan = 0; chan < rx_channels_count; chan++) stmmac_start_rx_dma(priv, chan); @@ -2504,9 +2558,9 @@ static void stmmac_start_all_dma(struct stmmac_priv *priv) */ static void stmmac_stop_all_dma(struct stmmac_priv *priv) { - u32 rx_channels_count = priv->plat->rx_queues_to_use; - u32 tx_channels_count = priv->plat->tx_queues_to_use; - u32 chan = 0; + u8 rx_channels_count = priv->plat->rx_queues_to_use; + u8 tx_channels_count = priv->plat->tx_queues_to_use; + u8 chan; for (chan = 0; chan < rx_channels_count; chan++) stmmac_stop_rx_dma(priv, chan); @@ -2523,14 +2577,14 @@ static void stmmac_stop_all_dma(struct stmmac_priv *priv) */ static void stmmac_dma_operation_mode(struct stmmac_priv *priv) { - u32 rx_channels_count = priv->plat->rx_queues_to_use; - u32 tx_channels_count = priv->plat->tx_queues_to_use; + u8 rx_channels_count = priv->plat->rx_queues_to_use; + u8 tx_channels_count = priv->plat->tx_queues_to_use; int rxfifosz = priv->plat->rx_fifo_size; int txfifosz = priv->plat->tx_fifo_size; u32 txmode = 0; u32 rxmode = 0; - u32 chan = 0; u8 qmode = 0; + u8 chan; if (rxfifosz == 0) rxfifosz = priv->dma_cap.rx_fifo_size; @@ -2565,23 +2619,13 @@ static void stmmac_dma_operation_mode(struct stmmac_priv *priv) /* configure all channels */ for (chan = 0; chan < rx_channels_count; chan++) { struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[chan]; - u32 buf_size; qmode = priv->plat->rx_queues_cfg[chan].mode_to_use; stmmac_dma_rx_mode(priv, priv->ioaddr, rxmode, chan, rxfifosz, qmode); - if (rx_q->xsk_pool) { - buf_size = xsk_pool_get_rx_frame_size(rx_q->xsk_pool); - stmmac_set_dma_bfsize(priv, priv->ioaddr, - buf_size, - chan); - } else { - stmmac_set_dma_bfsize(priv, priv->ioaddr, - priv->dma_conf.dma_buf_sz, - chan); - } + stmmac_set_queue_rx_buf_size(priv, rx_q, chan); } for (chan = 0; chan < tx_channels_count; chan++) { @@ -2686,30 +2730,20 @@ static bool stmmac_xdp_xmit_zc(struct stmmac_priv *priv, u32 queue, u32 budget) continue; } - if (likely(priv->extend_desc)) - tx_desc = (struct dma_desc *)(tx_q->dma_etx + entry); - else if (tx_q->tbs & STMMAC_TBS_AVAIL) - tx_desc = &tx_q->dma_entx[entry].basic; - else - tx_desc = tx_q->dma_tx + entry; - + tx_desc = stmmac_get_tx_desc(priv, tx_q, entry); dma_addr = xsk_buff_raw_get_dma(pool, xdp_desc.addr); meta = xsk_buff_get_metadata(pool, xdp_desc.addr); xsk_buff_raw_dma_sync_for_device(pool, dma_addr, xdp_desc.len); - tx_q->tx_skbuff_dma[entry].buf_type = STMMAC_TXBUF_T_XSK_TX; - /* To return XDP buffer to XSK pool, we simple call * xsk_tx_completed(), so we don't need to fill up * 'buf' and 'xdpf'. */ - tx_q->tx_skbuff_dma[entry].buf = 0; - tx_q->xdpf[entry] = NULL; + stmmac_set_tx_dma_entry(tx_q, entry, STMMAC_TXBUF_T_XSK_TX, + 0, xdp_desc.len, false); + stmmac_set_tx_dma_last_segment(tx_q, entry); - tx_q->tx_skbuff_dma[entry].map_as_page = false; - tx_q->tx_skbuff_dma[entry].len = xdp_desc.len; - tx_q->tx_skbuff_dma[entry].last_segment = true; - tx_q->tx_skbuff_dma[entry].is_jumbo = false; + tx_q->xdpf[entry] = NULL; stmmac_set_desc_addr(priv, tx_desc, dma_addr); @@ -2736,7 +2770,7 @@ static bool stmmac_xdp_xmit_zc(struct stmmac_priv *priv, u32 queue, u32 budget) } stmmac_prepare_tx_desc(priv, tx_desc, 1, xdp_desc.len, - csum, priv->mode, true, true, + csum, priv->descriptor_mode, true, true, xdp_desc.len); stmmac_enable_dma_transmission(priv, priv->ioaddr, queue); @@ -2744,7 +2778,7 @@ static bool stmmac_xdp_xmit_zc(struct stmmac_priv *priv, u32 queue, u32 budget) xsk_tx_metadata_to_compl(meta, &tx_q->tx_skbuff_dma[entry].xsk_meta); - tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, priv->dma_conf.dma_tx_size); + tx_q->cur_tx = STMMAC_NEXT_ENTRY(tx_q->cur_tx, priv->dma_conf.dma_tx_size); entry = tx_q->cur_tx; } u64_stats_update_begin(&txq_stats->napi_syncp); @@ -2823,13 +2857,7 @@ static int stmmac_tx_clean(struct stmmac_priv *priv, int budget, u32 queue, skb = NULL; } - if (priv->extend_desc) - p = (struct dma_desc *)(tx_q->dma_etx + entry); - else if (tx_q->tbs & STMMAC_TBS_AVAIL) - p = &tx_q->dma_entx[entry].basic; - else - p = tx_q->dma_tx + entry; - + p = stmmac_get_tx_desc(priv, tx_q, entry); status = stmmac_tx_status(priv, &priv->xstats, p, priv->ioaddr); /* Check if the descriptor is owned by the DMA */ if (unlikely(status & tx_dma_own)) @@ -2884,6 +2912,9 @@ static int stmmac_tx_clean(struct stmmac_priv *priv, int budget, u32 queue, tx_q->tx_skbuff_dma[entry].map_as_page = false; } + /* This looks at tx_q->tx_skbuff_dma[tx_q->dirty_tx].is_jumbo + * and tx_q->tx_skbuff_dma[tx_q->dirty_tx].last_segment + */ stmmac_clean_desc3(priv, tx_q, p); tx_q->tx_skbuff_dma[entry].last_segment = false; @@ -2913,9 +2944,9 @@ static int stmmac_tx_clean(struct stmmac_priv *priv, int budget, u32 queue, } } - stmmac_release_tx_desc(priv, p, priv->mode); + stmmac_release_tx_desc(priv, p, priv->descriptor_mode); - entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_tx_size); + entry = STMMAC_NEXT_ENTRY(entry, priv->dma_conf.dma_tx_size); } tx_q->dirty_tx = entry; @@ -3014,8 +3045,8 @@ static void stmmac_set_dma_operation_mode(struct stmmac_priv *priv, u32 txmode, { u8 rxqmode = priv->plat->rx_queues_cfg[chan].mode_to_use; u8 txqmode = priv->plat->tx_queues_cfg[chan].mode_to_use; - u32 rx_channels_count = priv->plat->rx_queues_to_use; - u32 tx_channels_count = priv->plat->tx_queues_to_use; + u8 rx_channels_count = priv->plat->rx_queues_to_use; + u8 tx_channels_count = priv->plat->tx_queues_to_use; int rxfifosz = priv->plat->rx_fifo_size; int txfifosz = priv->plat->tx_fifo_size; @@ -3090,12 +3121,12 @@ static int stmmac_napi_check(struct stmmac_priv *priv, u32 chan, u32 dir) */ static void stmmac_dma_interrupt(struct stmmac_priv *priv) { - u32 tx_channel_count = priv->plat->tx_queues_to_use; - u32 rx_channel_count = priv->plat->rx_queues_to_use; - u32 channels_to_check = tx_channel_count > rx_channel_count ? - tx_channel_count : rx_channel_count; - u32 chan; + u8 tx_channel_count = priv->plat->tx_queues_to_use; + u8 rx_channel_count = priv->plat->rx_queues_to_use; + u8 channels_to_check = tx_channel_count > rx_channel_count ? + tx_channel_count : rx_channel_count; int status[MAX_T(u32, MTL_MAX_TX_QUEUES, MTL_MAX_RX_QUEUES)]; + u8 chan; /* Make sure we never check beyond our status buffer. */ if (WARN_ON_ONCE(channels_to_check > ARRAY_SIZE(status))) @@ -3239,21 +3270,13 @@ static int stmmac_prereset_configure(struct stmmac_priv *priv) */ static int stmmac_init_dma_engine(struct stmmac_priv *priv) { - u32 rx_channels_count = priv->plat->rx_queues_to_use; - u32 tx_channels_count = priv->plat->tx_queues_to_use; - u32 dma_csr_ch = max(rx_channels_count, tx_channels_count); + u8 rx_channels_count = priv->plat->rx_queues_to_use; + u8 tx_channels_count = priv->plat->tx_queues_to_use; + u8 dma_csr_ch = max(rx_channels_count, tx_channels_count); struct stmmac_rx_queue *rx_q; struct stmmac_tx_queue *tx_q; - u32 chan = 0; int ret = 0; - - if (!priv->plat->dma_cfg || !priv->plat->dma_cfg->pbl) { - netdev_err(priv->dev, "Invalid DMA configuration\n"); - return -EINVAL; - } - - if (priv->extend_desc && (priv->mode == STMMAC_RING_MODE)) - priv->plat->dma_cfg->atds = 1; + u8 chan; ret = stmmac_prereset_configure(priv); if (ret) @@ -3284,11 +3307,8 @@ static int stmmac_init_dma_engine(struct stmmac_priv *priv) stmmac_init_rx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, rx_q->dma_rx_phy, chan); - rx_q->rx_tail_addr = rx_q->dma_rx_phy + - (rx_q->buf_alloc_num * - sizeof(struct dma_desc)); - stmmac_set_rx_tail_ptr(priv, priv->ioaddr, - rx_q->rx_tail_addr, chan); + stmmac_set_queue_rx_tail_ptr(priv, rx_q, chan, + rx_q->buf_alloc_num); } /* DMA TX Channel Configuration */ @@ -3298,9 +3318,7 @@ static int stmmac_init_dma_engine(struct stmmac_priv *priv) stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, tx_q->dma_tx_phy, chan); - tx_q->tx_tail_addr = tx_q->dma_tx_phy; - stmmac_set_tx_tail_ptr(priv, priv->ioaddr, - tx_q->tx_tail_addr, chan); + stmmac_set_queue_tx_tail_ptr(priv, tx_q, chan, 0); } return ret; @@ -3316,7 +3334,7 @@ static void stmmac_tx_timer_arm(struct stmmac_priv *priv, u32 queue) if (!tx_coal_timer) return; - ch = &priv->channel[tx_q->queue_index]; + ch = &priv->channel[queue]; napi = tx_q->xsk_pool ? &ch->rxtx_napi : &ch->tx_napi; /* Arm timer only if napi is not already scheduled. @@ -3369,9 +3387,9 @@ static enum hrtimer_restart stmmac_tx_timer(struct hrtimer *t) */ static void stmmac_init_coalesce(struct stmmac_priv *priv) { - u32 tx_channel_count = priv->plat->tx_queues_to_use; - u32 rx_channel_count = priv->plat->rx_queues_to_use; - u32 chan; + u8 tx_channel_count = priv->plat->tx_queues_to_use; + u8 rx_channel_count = priv->plat->rx_queues_to_use; + u8 chan; for (chan = 0; chan < tx_channel_count; chan++) { struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[chan]; @@ -3388,9 +3406,9 @@ static void stmmac_init_coalesce(struct stmmac_priv *priv) static void stmmac_set_rings_length(struct stmmac_priv *priv) { - u32 rx_channels_count = priv->plat->rx_queues_to_use; - u32 tx_channels_count = priv->plat->tx_queues_to_use; - u32 chan; + u8 rx_channels_count = priv->plat->rx_queues_to_use; + u8 tx_channels_count = priv->plat->tx_queues_to_use; + u8 chan; /* set TX ring length */ for (chan = 0; chan < tx_channels_count; chan++) @@ -3410,9 +3428,9 @@ static void stmmac_set_rings_length(struct stmmac_priv *priv) */ static void stmmac_set_tx_queue_weight(struct stmmac_priv *priv) { - u32 tx_queues_count = priv->plat->tx_queues_to_use; + u8 tx_queues_count = priv->plat->tx_queues_to_use; u32 weight; - u32 queue; + u8 queue; for (queue = 0; queue < tx_queues_count; queue++) { weight = priv->plat->tx_queues_cfg[queue].weight; @@ -3427,9 +3445,9 @@ static void stmmac_set_tx_queue_weight(struct stmmac_priv *priv) */ static void stmmac_configure_cbs(struct stmmac_priv *priv) { - u32 tx_queues_count = priv->plat->tx_queues_to_use; + u8 tx_queues_count = priv->plat->tx_queues_to_use; u32 mode_to_use; - u32 queue; + u8 queue; /* queue 0 is reserved for legacy traffic */ for (queue = 1; queue < tx_queues_count; queue++) { @@ -3453,8 +3471,8 @@ static void stmmac_configure_cbs(struct stmmac_priv *priv) */ static void stmmac_rx_queue_dma_chan_map(struct stmmac_priv *priv) { - u32 rx_queues_count = priv->plat->rx_queues_to_use; - u32 queue; + u8 rx_queues_count = priv->plat->rx_queues_to_use; + u8 queue; u32 chan; for (queue = 0; queue < rx_queues_count; queue++) { @@ -3470,8 +3488,8 @@ static void stmmac_rx_queue_dma_chan_map(struct stmmac_priv *priv) */ static void stmmac_mac_config_rx_queues_prio(struct stmmac_priv *priv) { - u32 rx_queues_count = priv->plat->rx_queues_to_use; - u32 queue; + u8 rx_queues_count = priv->plat->rx_queues_to_use; + u8 queue; u32 prio; for (queue = 0; queue < rx_queues_count; queue++) { @@ -3490,8 +3508,8 @@ static void stmmac_mac_config_rx_queues_prio(struct stmmac_priv *priv) */ static void stmmac_mac_config_tx_queues_prio(struct stmmac_priv *priv) { - u32 tx_queues_count = priv->plat->tx_queues_to_use; - u32 queue; + u8 tx_queues_count = priv->plat->tx_queues_to_use; + u8 queue; u32 prio; for (queue = 0; queue < tx_queues_count; queue++) { @@ -3510,9 +3528,9 @@ static void stmmac_mac_config_tx_queues_prio(struct stmmac_priv *priv) */ static void stmmac_mac_config_rx_queues_routing(struct stmmac_priv *priv) { - u32 rx_queues_count = priv->plat->rx_queues_to_use; - u32 queue; + u8 rx_queues_count = priv->plat->rx_queues_to_use; u8 packet; + u8 queue; for (queue = 0; queue < rx_queues_count; queue++) { /* no specific packet type routing specified for the queue */ @@ -3547,8 +3565,8 @@ static void stmmac_mac_config_rss(struct stmmac_priv *priv) */ static void stmmac_mtl_configuration(struct stmmac_priv *priv) { - u32 rx_queues_count = priv->plat->rx_queues_to_use; - u32 tx_queues_count = priv->plat->tx_queues_to_use; + u8 rx_queues_count = priv->plat->rx_queues_to_use; + u8 tx_queues_count = priv->plat->tx_queues_to_use; if (tx_queues_count > 1) stmmac_set_tx_queue_weight(priv); @@ -3601,6 +3619,17 @@ static void stmmac_safety_feat_configuration(struct stmmac_priv *priv) } } +/* STM32MP25xx (dwmac v5.3) states "Do not enable time-based scheduling for + * channels on which the TSO feature is enabled." If we have a skb for a + * channel which has TBS enabled, fall back to software GSO. + */ +static bool stmmac_tso_channel_permitted(struct stmmac_priv *priv, + unsigned int chan) +{ + /* TSO and TBS cannot co-exist */ + return !(priv->dma_conf.tx_queue[chan].tbs & STMMAC_TBS_AVAIL); +} + /** * stmmac_hw_setup - setup mac in a usable state. * @dev : pointer to the device structure. @@ -3616,10 +3645,10 @@ static void stmmac_safety_feat_configuration(struct stmmac_priv *priv) static int stmmac_hw_setup(struct net_device *dev) { struct stmmac_priv *priv = netdev_priv(dev); - u32 rx_cnt = priv->plat->rx_queues_to_use; - u32 tx_cnt = priv->plat->tx_queues_to_use; + u8 rx_cnt = priv->plat->rx_queues_to_use; + u8 tx_cnt = priv->plat->tx_queues_to_use; bool sph_en; - u32 chan; + u8 chan; int ret; /* Make sure RX clock is enabled */ @@ -3687,12 +3716,9 @@ static int stmmac_hw_setup(struct net_device *dev) stmmac_set_rings_length(priv); /* Enable TSO */ - if (priv->tso) { + if (priv->dma_cap.tsoen && priv->plat->flags & STMMAC_FLAG_TSO_EN) { for (chan = 0; chan < tx_cnt; chan++) { - struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[chan]; - - /* TSO and TBS cannot co-exist */ - if (tx_q->tbs & STMMAC_TBS_AVAIL) + if (!stmmac_tso_channel_permitted(priv, chan)) continue; stmmac_enable_tso(priv, priv->ioaddr, 1, chan); @@ -3735,6 +3761,7 @@ static void stmmac_free_irq(struct net_device *dev, enum request_irq_err irq_err, int irq_idx) { struct stmmac_priv *priv = netdev_priv(dev); + struct stmmac_msi *msi = priv->msi; int j; switch (irq_err) { @@ -3742,28 +3769,30 @@ static void stmmac_free_irq(struct net_device *dev, irq_idx = priv->plat->tx_queues_to_use; fallthrough; case REQ_IRQ_ERR_TX: - for (j = irq_idx - 1; j >= 0; j--) { - if (priv->tx_irq[j] > 0) { - irq_set_affinity_hint(priv->tx_irq[j], NULL); - free_irq(priv->tx_irq[j], &priv->dma_conf.tx_queue[j]); + for (j = irq_idx - 1; msi && j >= 0; j--) { + if (msi->tx_irq[j] > 0) { + irq_set_affinity_hint(msi->tx_irq[j], NULL); + free_irq(msi->tx_irq[j], + &priv->dma_conf.tx_queue[j]); } } irq_idx = priv->plat->rx_queues_to_use; fallthrough; case REQ_IRQ_ERR_RX: - for (j = irq_idx - 1; j >= 0; j--) { - if (priv->rx_irq[j] > 0) { - irq_set_affinity_hint(priv->rx_irq[j], NULL); - free_irq(priv->rx_irq[j], &priv->dma_conf.rx_queue[j]); + for (j = irq_idx - 1; msi && j >= 0; j--) { + if (msi->rx_irq[j] > 0) { + irq_set_affinity_hint(msi->rx_irq[j], NULL); + free_irq(msi->rx_irq[j], + &priv->dma_conf.rx_queue[j]); } } - if (priv->sfty_ue_irq > 0 && priv->sfty_ue_irq != dev->irq) - free_irq(priv->sfty_ue_irq, dev); + if (msi && msi->sfty_ue_irq > 0 && msi->sfty_ue_irq != dev->irq) + free_irq(msi->sfty_ue_irq, dev); fallthrough; case REQ_IRQ_ERR_SFTY_UE: - if (priv->sfty_ce_irq > 0 && priv->sfty_ce_irq != dev->irq) - free_irq(priv->sfty_ce_irq, dev); + if (msi && msi->sfty_ce_irq > 0 && msi->sfty_ce_irq != dev->irq) + free_irq(msi->sfty_ce_irq, dev); fallthrough; case REQ_IRQ_ERR_SFTY_CE: if (priv->wol_irq > 0 && priv->wol_irq != dev->irq) @@ -3783,9 +3812,30 @@ static void stmmac_free_irq(struct net_device *dev, } } +static int stmmac_msi_init(struct stmmac_priv *priv, + struct stmmac_resources *res) +{ + int i; + + priv->msi = devm_kmalloc(priv->device, sizeof(*priv->msi), GFP_KERNEL); + if (!priv->msi) + return -ENOMEM; + + priv->msi->sfty_ce_irq = res->sfty_ce_irq; + priv->msi->sfty_ue_irq = res->sfty_ue_irq; + + for (i = 0; i < MTL_MAX_RX_QUEUES; i++) + priv->msi->rx_irq[i] = res->rx_irq[i]; + for (i = 0; i < MTL_MAX_TX_QUEUES; i++) + priv->msi->tx_irq[i] = res->tx_irq[i]; + + return 0; +} + static int stmmac_request_irq_multi_msi(struct net_device *dev) { struct stmmac_priv *priv = netdev_priv(dev); + struct stmmac_msi *msi = priv->msi; enum request_irq_err irq_err; int irq_idx = 0; char *int_name; @@ -3793,7 +3843,7 @@ static int stmmac_request_irq_multi_msi(struct net_device *dev) int i; /* For common interrupt */ - int_name = priv->int_name_mac; + int_name = msi->int_name_mac; sprintf(int_name, "%s:%s", dev->name, "mac"); ret = request_irq(dev->irq, stmmac_mac_interrupt, 0, int_name, dev); @@ -3809,7 +3859,7 @@ static int stmmac_request_irq_multi_msi(struct net_device *dev) * is used for WoL */ if (priv->wol_irq > 0 && priv->wol_irq != dev->irq) { - int_name = priv->int_name_wol; + int_name = msi->int_name_wol; sprintf(int_name, "%s:%s", dev->name, "wol"); ret = request_irq(priv->wol_irq, stmmac_mac_interrupt, @@ -3827,7 +3877,7 @@ static int stmmac_request_irq_multi_msi(struct net_device *dev) * Error line in case of another line is used */ if (priv->sfty_irq > 0 && priv->sfty_irq != dev->irq) { - int_name = priv->int_name_sfty; + int_name = msi->int_name_sfty; sprintf(int_name, "%s:%s", dev->name, "safety"); ret = request_irq(priv->sfty_irq, stmmac_safety_interrupt, 0, int_name, dev); @@ -3843,16 +3893,16 @@ static int stmmac_request_irq_multi_msi(struct net_device *dev) /* Request the Safety Feature Correctible Error line in * case of another line is used */ - if (priv->sfty_ce_irq > 0 && priv->sfty_ce_irq != dev->irq) { - int_name = priv->int_name_sfty_ce; + if (msi->sfty_ce_irq > 0 && msi->sfty_ce_irq != dev->irq) { + int_name = msi->int_name_sfty_ce; sprintf(int_name, "%s:%s", dev->name, "safety-ce"); - ret = request_irq(priv->sfty_ce_irq, + ret = request_irq(msi->sfty_ce_irq, stmmac_safety_interrupt, 0, int_name, dev); if (unlikely(ret < 0)) { netdev_err(priv->dev, "%s: alloc sfty ce MSI %d (error: %d)\n", - __func__, priv->sfty_ce_irq, ret); + __func__, msi->sfty_ce_irq, ret); irq_err = REQ_IRQ_ERR_SFTY_CE; goto irq_error; } @@ -3861,16 +3911,16 @@ static int stmmac_request_irq_multi_msi(struct net_device *dev) /* Request the Safety Feature Uncorrectible Error line in * case of another line is used */ - if (priv->sfty_ue_irq > 0 && priv->sfty_ue_irq != dev->irq) { - int_name = priv->int_name_sfty_ue; + if (msi->sfty_ue_irq > 0 && msi->sfty_ue_irq != dev->irq) { + int_name = msi->int_name_sfty_ue; sprintf(int_name, "%s:%s", dev->name, "safety-ue"); - ret = request_irq(priv->sfty_ue_irq, + ret = request_irq(msi->sfty_ue_irq, stmmac_safety_interrupt, 0, int_name, dev); if (unlikely(ret < 0)) { netdev_err(priv->dev, "%s: alloc sfty ue MSI %d (error: %d)\n", - __func__, priv->sfty_ue_irq, ret); + __func__, msi->sfty_ue_irq, ret); irq_err = REQ_IRQ_ERR_SFTY_UE; goto irq_error; } @@ -3880,23 +3930,23 @@ static int stmmac_request_irq_multi_msi(struct net_device *dev) for (i = 0; i < priv->plat->rx_queues_to_use; i++) { if (i >= MTL_MAX_RX_QUEUES) break; - if (priv->rx_irq[i] == 0) + if (msi->rx_irq[i] == 0) continue; - int_name = priv->int_name_rx_irq[i]; + int_name = msi->int_name_rx_irq[i]; sprintf(int_name, "%s:%s-%d", dev->name, "rx", i); - ret = request_irq(priv->rx_irq[i], + ret = request_irq(msi->rx_irq[i], stmmac_msi_intr_rx, 0, int_name, &priv->dma_conf.rx_queue[i]); if (unlikely(ret < 0)) { netdev_err(priv->dev, "%s: alloc rx-%d MSI %d (error: %d)\n", - __func__, i, priv->rx_irq[i], ret); + __func__, i, msi->rx_irq[i], ret); irq_err = REQ_IRQ_ERR_RX; irq_idx = i; goto irq_error; } - irq_set_affinity_hint(priv->rx_irq[i], + irq_set_affinity_hint(msi->rx_irq[i], cpumask_of(i % num_online_cpus())); } @@ -3904,23 +3954,23 @@ static int stmmac_request_irq_multi_msi(struct net_device *dev) for (i = 0; i < priv->plat->tx_queues_to_use; i++) { if (i >= MTL_MAX_TX_QUEUES) break; - if (priv->tx_irq[i] == 0) + if (msi->tx_irq[i] == 0) continue; - int_name = priv->int_name_tx_irq[i]; + int_name = msi->int_name_tx_irq[i]; sprintf(int_name, "%s:%s-%d", dev->name, "tx", i); - ret = request_irq(priv->tx_irq[i], + ret = request_irq(msi->tx_irq[i], stmmac_msi_intr_tx, 0, int_name, &priv->dma_conf.tx_queue[i]); if (unlikely(ret < 0)) { netdev_err(priv->dev, "%s: alloc tx-%d MSI %d (error: %d)\n", - __func__, i, priv->tx_irq[i], ret); + __func__, i, msi->tx_irq[i], ret); irq_err = REQ_IRQ_ERR_TX; irq_idx = i; goto irq_error; } - irq_set_affinity_hint(priv->tx_irq[i], + irq_set_affinity_hint(msi->tx_irq[i], cpumask_of(i % num_online_cpus())); } @@ -4011,7 +4061,8 @@ static struct stmmac_dma_conf * stmmac_setup_dma_desc(struct stmmac_priv *priv, unsigned int mtu) { struct stmmac_dma_conf *dma_conf; - int chan, bfsize, ret; + int bfsize, ret; + u8 chan; dma_conf = kzalloc_obj(*dma_conf); if (!dma_conf) { @@ -4086,7 +4137,7 @@ static int __stmmac_open(struct net_device *dev, struct stmmac_dma_conf *dma_conf) { struct stmmac_priv *priv = netdev_priv(dev); - u32 chan; + u8 chan; int ret; for (int i = 0; i < MTL_MAX_TX_QUEUES; i++) @@ -4185,7 +4236,7 @@ err_dma_resources: static void __stmmac_release(struct net_device *dev) { struct stmmac_priv *priv = netdev_priv(dev); - u32 chan; + u8 chan; /* Stop and disconnect the PHY */ phylink_stop(priv->phylink); @@ -4258,7 +4309,7 @@ static bool stmmac_vlan_insert(struct stmmac_priv *priv, struct sk_buff *skb, return false; stmmac_set_tx_owner(priv, p); - tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, priv->dma_conf.dma_tx_size); + tx_q->cur_tx = STMMAC_NEXT_ENTRY(tx_q->cur_tx, priv->dma_conf.dma_tx_size); return true; } @@ -4286,7 +4337,7 @@ static void stmmac_tso_allocator(struct stmmac_priv *priv, dma_addr_t des, while (tmp_len > 0) { dma_addr_t curr_addr; - tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, + tx_q->cur_tx = STMMAC_NEXT_ENTRY(tx_q->cur_tx, priv->dma_conf.dma_tx_size); WARN_ON(tx_q->tx_skbuff[tx_q->cur_tx]); @@ -4312,14 +4363,6 @@ static void stmmac_tso_allocator(struct stmmac_priv *priv, dma_addr_t des, static void stmmac_flush_tx_descriptors(struct stmmac_priv *priv, int queue) { struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; - int desc_size; - - if (likely(priv->extend_desc)) - desc_size = sizeof(struct dma_extended_desc); - else if (tx_q->tbs & STMMAC_TBS_AVAIL) - desc_size = sizeof(struct dma_edesc); - else - desc_size = sizeof(struct dma_desc); /* The own bit must be the latest setting done when prepare the * descriptor and then barrier is needed to make sure that @@ -4327,8 +4370,90 @@ static void stmmac_flush_tx_descriptors(struct stmmac_priv *priv, int queue) */ wmb(); - tx_q->tx_tail_addr = tx_q->dma_tx_phy + (tx_q->cur_tx * desc_size); - stmmac_set_tx_tail_ptr(priv, priv->ioaddr, tx_q->tx_tail_addr, queue); + stmmac_set_queue_tx_tail_ptr(priv, tx_q, queue, tx_q->cur_tx); +} + +static void stmmac_set_gso_types(struct stmmac_priv *priv, bool tso) +{ + if (!tso) { + priv->gso_enabled_types = 0; + } else { + /* Manage oversized TCP frames for GMAC4 device */ + priv->gso_enabled_types = SKB_GSO_TCPV4 | SKB_GSO_TCPV6; + if (priv->plat->core_type == DWMAC_CORE_GMAC4) + priv->gso_enabled_types |= SKB_GSO_UDP_L4; + } +} + +static void stmmac_set_gso_features(struct net_device *ndev) +{ + struct stmmac_priv *priv = netdev_priv(ndev); + const struct stmmac_dma_cfg *dma_cfg; + int txpbl; + + if (priv->dma_cap.tsoen) + dev_info(priv->device, "TSO supported\n"); + + if (!(priv->plat->flags & STMMAC_FLAG_TSO_EN)) + return; + + if (!priv->dma_cap.tsoen) { + dev_warn(priv->device, "platform requests unsupported TSO\n"); + return; + } + + /* FIXME: + * STM32MP151 (v4.2 userver v4.0) states that TxPBL must be >= 4. It + * is not clear whether PBLx8 (which multiplies the PBL value by 8) + * influences this. + */ + dma_cfg = priv->plat->dma_cfg; + txpbl = dma_cfg->txpbl ?: dma_cfg->pbl; + if (txpbl < 4) { + dev_warn(priv->device, "txpbl(%d) is too low for TSO\n", txpbl); + return; + } + + ndev->hw_features |= NETIF_F_TSO | NETIF_F_TSO6; + if (priv->plat->core_type == DWMAC_CORE_GMAC4) + ndev->hw_features |= NETIF_F_GSO_UDP_L4; + + stmmac_set_gso_types(priv, true); + + dev_info(priv->device, "TSO feature enabled\n"); +} + +static size_t stmmac_tso_header_size(struct sk_buff *skb) +{ + size_t size; + + if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) + size = skb_transport_offset(skb) + sizeof(struct udphdr); + else + size = skb_tcp_all_headers(skb); + + return size; +} + +/* STM32MP151 (dwmac v4.2) and STM32MP25xx (dwmac v5.3) states for TDES2 normal + * (read format) descriptor that the maximum header length supported for the + * TSO feature is 1023 bytes. + * + * While IPv4 is limited to MAC+VLAN+IPv4+ext+TCP+ext = 138 bytes, the IPv6 + * extension headers aren't similarly limited. + * + * Fall back to software GSO for these skbs. Also check that the MSS is >= + * the recommended 64 bytes (documented in ETH_DMACxCR register description), + * and that a the header plus MSS is not larger than 16383 (documented in + * "Building the Descriptor and the packet for the TSO feature"). + */ +static bool stmmac_tso_valid_packet(struct sk_buff *skb) +{ + size_t header_len = stmmac_tso_header_size(skb); + unsigned int gso_size = skb_shinfo(skb)->gso_size; + + return header_len <= 1023 && gso_size >= 64 && + header_len + gso_size < 16383; } /** @@ -4381,19 +4506,6 @@ static netdev_tx_t stmmac_tso_xmit(struct sk_buff *skb, struct net_device *dev) u8 proto_hdr_len, hdr; dma_addr_t des; - /* Always insert VLAN tag to SKB payload for TSO frames. - * - * Never insert VLAN tag by HW, since segments split by - * TSO engine will be un-tagged by mistake. - */ - if (skb_vlan_tag_present(skb)) { - skb = __vlan_hwaccel_push_inside(skb); - if (unlikely(!skb)) { - priv->xstats.tx_dropped++; - return NETDEV_TX_OK; - } - } - nfrags = skb_shinfo(skb)->nr_frags; queue = skb_get_queue_mapping(skb); @@ -4402,13 +4514,11 @@ static netdev_tx_t stmmac_tso_xmit(struct sk_buff *skb, struct net_device *dev) first_tx = tx_q->cur_tx; /* Compute header lengths */ - if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) { - proto_hdr_len = skb_transport_offset(skb) + sizeof(struct udphdr); + proto_hdr_len = stmmac_tso_header_size(skb); + if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) hdr = sizeof(struct udphdr); - } else { - proto_hdr_len = skb_tcp_all_headers(skb); + else hdr = tcp_hdrlen(skb); - } /* Desc availability based on threshold should be enough safe */ if (unlikely(stmmac_tx_avail(priv, queue) < @@ -4437,7 +4547,7 @@ static netdev_tx_t stmmac_tso_xmit(struct sk_buff *skb, struct net_device *dev) stmmac_set_mss(priv, mss_desc, mss); tx_q->mss = mss; - tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, + tx_q->cur_tx = STMMAC_NEXT_ENTRY(tx_q->cur_tx, priv->dma_conf.dma_tx_size); WARN_ON(tx_q->tx_skbuff[tx_q->cur_tx]); } @@ -4480,10 +4590,8 @@ static netdev_tx_t stmmac_tso_xmit(struct sk_buff *skb, struct net_device *dev) * this DMA buffer right after the DMA engine completely finishes the * full buffer transmission. */ - tx_q->tx_skbuff_dma[tx_q->cur_tx].buf = des; - tx_q->tx_skbuff_dma[tx_q->cur_tx].len = skb_headlen(skb); - tx_q->tx_skbuff_dma[tx_q->cur_tx].map_as_page = false; - tx_q->tx_skbuff_dma[tx_q->cur_tx].buf_type = STMMAC_TXBUF_T_SKB; + stmmac_set_tx_skb_dma_entry(tx_q, tx_q->cur_tx, des, skb_headlen(skb), + false); /* Prepare fragments */ for (i = 0; i < nfrags; i++) { @@ -4498,20 +4606,18 @@ static netdev_tx_t stmmac_tso_xmit(struct sk_buff *skb, struct net_device *dev) stmmac_tso_allocator(priv, des, skb_frag_size(frag), (i == nfrags - 1), queue); - tx_q->tx_skbuff_dma[tx_q->cur_tx].buf = des; - tx_q->tx_skbuff_dma[tx_q->cur_tx].len = skb_frag_size(frag); - tx_q->tx_skbuff_dma[tx_q->cur_tx].map_as_page = true; - tx_q->tx_skbuff_dma[tx_q->cur_tx].buf_type = STMMAC_TXBUF_T_SKB; + stmmac_set_tx_skb_dma_entry(tx_q, tx_q->cur_tx, des, + skb_frag_size(frag), true); } - tx_q->tx_skbuff_dma[tx_q->cur_tx].last_segment = true; + stmmac_set_tx_dma_last_segment(tx_q, tx_q->cur_tx); /* Only the last descriptor gets to point to the skb. */ tx_q->tx_skbuff[tx_q->cur_tx] = skb; - tx_q->tx_skbuff_dma[tx_q->cur_tx].buf_type = STMMAC_TXBUF_T_SKB; /* Manage tx mitigation */ - tx_packets = (tx_q->cur_tx + 1) - first_tx; + tx_packets = CIRC_CNT(tx_q->cur_tx + 1, first_tx, + priv->dma_conf.dma_tx_size); tx_q->tx_count_frames += tx_packets; if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && priv->hwts_tx_en) @@ -4541,7 +4647,7 @@ static netdev_tx_t stmmac_tso_xmit(struct sk_buff *skb, struct net_device *dev) * ndo_start_xmit will fill this descriptor the next time it's * called and stmmac_tx_clean may clean up to this descriptor. */ - tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, priv->dma_conf.dma_tx_size); + tx_q->cur_tx = STMMAC_NEXT_ENTRY(tx_q->cur_tx, priv->dma_conf.dma_tx_size); if (unlikely(stmmac_tx_avail(priv, queue) <= (MAX_SKB_FRAGS + 1))) { netif_dbg(priv, hw, priv->dev, "%s: stop transmitted packets\n", @@ -4570,8 +4676,8 @@ static netdev_tx_t stmmac_tso_xmit(struct sk_buff *skb, struct net_device *dev) /* If we only have one entry used, then the first entry is the last * segment. */ - is_last_segment = ((tx_q->cur_tx - first_entry) & - (priv->dma_conf.dma_tx_size - 1)) == 1; + is_last_segment = CIRC_CNT(tx_q->cur_tx, first_entry, + priv->dma_conf.dma_tx_size) == 1; /* Complete the first descriptor before granting the DMA */ stmmac_prepare_tso_tx_desc(priv, first, 1, proto_hdr_len, 0, 1, @@ -4650,31 +4756,20 @@ static netdev_tx_t stmmac_xmit(struct sk_buff *skb, struct net_device *dev) u32 queue = skb_get_queue_mapping(skb); int nfrags = skb_shinfo(skb)->nr_frags; unsigned int first_entry, tx_packets; - int gso = skb_shinfo(skb)->gso_type; struct stmmac_txq_stats *txq_stats; - struct dma_edesc *tbs_desc = NULL; - struct dma_desc *desc, *first; + struct dma_desc *desc, *first_desc; struct stmmac_tx_queue *tx_q; int i, csum_insertion = 0; int entry, first_tx; - dma_addr_t des; + dma_addr_t dma_addr; u32 sdu_len; - tx_q = &priv->dma_conf.tx_queue[queue]; - txq_stats = &priv->xstats.txq_stats[queue]; - first_tx = tx_q->cur_tx; - if (priv->tx_path_in_lpi_mode && priv->eee_sw_timer_en) stmmac_stop_sw_lpi(priv); - /* Manage oversized TCP frames for GMAC4 device */ - if (skb_is_gso(skb) && priv->tso) { - if (gso & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6)) - return stmmac_tso_xmit(skb, dev); - if (priv->plat->core_type == DWMAC_CORE_GMAC4 && - (gso & SKB_GSO_UDP_L4)) - return stmmac_tso_xmit(skb, dev); - } + if (skb_is_gso(skb) && + skb_shinfo(skb)->gso_type & priv->gso_enabled_types) + return stmmac_tso_xmit(skb, dev); if (priv->est && priv->est->enable && priv->est->max_sdu[queue]) { @@ -4700,6 +4795,9 @@ static netdev_tx_t stmmac_xmit(struct sk_buff *skb, struct net_device *dev) return NETDEV_TX_BUSY; } + tx_q = &priv->dma_conf.tx_queue[queue]; + first_tx = tx_q->cur_tx; + /* Check if VLAN can be inserted by HW */ has_vlan = stmmac_vlan_insert(priv, skb, tx_q); @@ -4707,89 +4805,103 @@ static netdev_tx_t stmmac_xmit(struct sk_buff *skb, struct net_device *dev) first_entry = entry; WARN_ON(tx_q->tx_skbuff[first_entry]); - csum_insertion = (skb->ip_summed == CHECKSUM_PARTIAL); - /* DWMAC IPs can be synthesized to support tx coe only for a few tx - * queues. In that case, checksum offloading for those queues that don't - * support tx coe needs to fallback to software checksum calculation. - * - * Packets that won't trigger the COE e.g. most DSA-tagged packets will - * also have to be checksummed in software. - */ - if (csum_insertion && - (priv->plat->tx_queues_cfg[queue].coe_unsupported || - !stmmac_has_ip_ethertype(skb))) { - if (unlikely(skb_checksum_help(skb))) - goto dma_map_err; - csum_insertion = !csum_insertion; - } - - if (likely(priv->extend_desc)) - desc = (struct dma_desc *)(tx_q->dma_etx + entry); - else if (tx_q->tbs & STMMAC_TBS_AVAIL) - desc = &tx_q->dma_entx[entry].basic; - else - desc = tx_q->dma_tx + entry; - - first = desc; + desc = stmmac_get_tx_desc(priv, tx_q, entry); + first_desc = desc; if (has_vlan) - stmmac_set_desc_vlan(priv, first, STMMAC_VLAN_INSERT); + stmmac_set_desc_vlan(priv, first_desc, STMMAC_VLAN_INSERT); enh_desc = priv->plat->enh_desc; /* To program the descriptors according to the size of the frame */ if (enh_desc) is_jumbo = stmmac_is_jumbo_frm(priv, skb->len, enh_desc); + csum_insertion = skb->ip_summed == CHECKSUM_PARTIAL; + if (unlikely(is_jumbo)) { entry = stmmac_jumbo_frm(priv, tx_q, skb, csum_insertion); if (unlikely(entry < 0) && (entry != -EINVAL)) goto dma_map_err; + } else { + bool last_segment = (nfrags == 0); + + dma_addr = dma_map_single(priv->device, skb->data, + nopaged_len, DMA_TO_DEVICE); + if (dma_mapping_error(priv->device, dma_addr)) + goto dma_map_err; + + stmmac_set_tx_skb_dma_entry(tx_q, first_entry, dma_addr, + nopaged_len, false); + + stmmac_set_desc_addr(priv, first_desc, dma_addr); + + if (last_segment) + stmmac_set_tx_dma_last_segment(tx_q, first_entry); + + if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && + priv->hwts_tx_en)) { + /* declare that device is doing timestamping */ + skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; + stmmac_enable_tx_timestamp(priv, first_desc); + } + + /* Prepare the first descriptor without setting the OWN bit */ + stmmac_prepare_tx_desc(priv, first_desc, 1, nopaged_len, + csum_insertion, priv->descriptor_mode, + 0, last_segment, skb->len); + } + + if (priv->sarc_type) + stmmac_set_desc_sarc(priv, first_desc, priv->sarc_type); + + /* STMMAC_TBS_EN can only be set if STMMAC_TBS_AVAIL has already + * been set, which means the underlying type of the descriptors + * will be struct stmmac_edesc. Therefore, it is safe to convert + * the basic descriptor to the enhanced descriptor here. + */ + if (tx_q->tbs & STMMAC_TBS_EN) { + struct timespec64 ts = ns_to_timespec64(skb->tstamp); + + stmmac_set_desc_tbs(priv, dma_desc_to_edesc(first_desc), + ts.tv_sec, ts.tv_nsec); } for (i = 0; i < nfrags; i++) { const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; - int len = skb_frag_size(frag); + unsigned int frag_size = skb_frag_size(frag); bool last_segment = (i == (nfrags - 1)); - entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_tx_size); + entry = STMMAC_NEXT_ENTRY(entry, priv->dma_conf.dma_tx_size); WARN_ON(tx_q->tx_skbuff[entry]); - if (likely(priv->extend_desc)) - desc = (struct dma_desc *)(tx_q->dma_etx + entry); - else if (tx_q->tbs & STMMAC_TBS_AVAIL) - desc = &tx_q->dma_entx[entry].basic; - else - desc = tx_q->dma_tx + entry; + desc = stmmac_get_tx_desc(priv, tx_q, entry); - des = skb_frag_dma_map(priv->device, frag, 0, len, - DMA_TO_DEVICE); - if (dma_mapping_error(priv->device, des)) + dma_addr = skb_frag_dma_map(priv->device, frag, 0, frag_size, + DMA_TO_DEVICE); + if (dma_mapping_error(priv->device, dma_addr)) goto dma_map_err; /* should reuse desc w/o issues */ - tx_q->tx_skbuff_dma[entry].buf = des; - - stmmac_set_desc_addr(priv, desc, des); - - tx_q->tx_skbuff_dma[entry].map_as_page = true; - tx_q->tx_skbuff_dma[entry].len = len; - tx_q->tx_skbuff_dma[entry].last_segment = last_segment; - tx_q->tx_skbuff_dma[entry].buf_type = STMMAC_TXBUF_T_SKB; + stmmac_set_tx_skb_dma_entry(tx_q, entry, dma_addr, frag_size, + true); + stmmac_set_desc_addr(priv, desc, dma_addr); /* Prepare the descriptor and set the own bit too */ - stmmac_prepare_tx_desc(priv, desc, 0, len, csum_insertion, - priv->mode, 1, last_segment, skb->len); + stmmac_prepare_tx_desc(priv, desc, 0, frag_size, csum_insertion, + priv->descriptor_mode, 1, last_segment, + skb->len); } + stmmac_set_tx_dma_last_segment(tx_q, entry); + /* Only the last descriptor gets to point to the skb. */ tx_q->tx_skbuff[entry] = skb; - tx_q->tx_skbuff_dma[entry].buf_type = STMMAC_TXBUF_T_SKB; /* According to the coalesce parameter the IC bit for the latest * segment is reset and the timer re-started to clean the tx status. * This approach takes care about the fragments: desc is the first * element in case of no SG. */ - tx_packets = (entry + 1) - first_tx; + tx_packets = CIRC_CNT(entry + 1, first_tx, priv->dma_conf.dma_tx_size); tx_q->tx_count_frames += tx_packets; if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && priv->hwts_tx_en) @@ -4805,13 +4917,7 @@ static netdev_tx_t stmmac_xmit(struct sk_buff *skb, struct net_device *dev) set_ic = false; if (set_ic) { - if (likely(priv->extend_desc)) - desc = &tx_q->dma_etx[entry].basic; - else if (tx_q->tbs & STMMAC_TBS_AVAIL) - desc = &tx_q->dma_entx[entry].basic; - else - desc = &tx_q->dma_tx[entry]; - + desc = stmmac_get_tx_desc(priv, tx_q, entry); tx_q->tx_count_frames = 0; stmmac_set_tx_ic(priv, desc); } @@ -4821,14 +4927,14 @@ static netdev_tx_t stmmac_xmit(struct sk_buff *skb, struct net_device *dev) * ndo_start_xmit will fill this descriptor the next time it's * called and stmmac_tx_clean may clean up to this descriptor. */ - entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_tx_size); + entry = STMMAC_NEXT_ENTRY(entry, priv->dma_conf.dma_tx_size); tx_q->cur_tx = entry; if (netif_msg_pktdata(priv)) { netdev_dbg(priv->dev, "%s: curr=%d dirty=%d f=%d, e=%d, first=%p, nfrags=%d", __func__, tx_q->cur_tx, tx_q->dirty_tx, first_entry, - entry, first, nfrags); + entry, first_desc, nfrags); netdev_dbg(priv->dev, ">>> frame to be transmitted: "); print_pkt(skb->data, skb->len); @@ -4840,57 +4946,17 @@ static netdev_tx_t stmmac_xmit(struct sk_buff *skb, struct net_device *dev) netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, queue)); } + txq_stats = &priv->xstats.txq_stats[queue]; u64_stats_update_begin(&txq_stats->q_syncp); u64_stats_add(&txq_stats->q.tx_bytes, skb->len); if (set_ic) u64_stats_inc(&txq_stats->q.tx_set_ic_bit); u64_stats_update_end(&txq_stats->q_syncp); - if (priv->sarc_type) - stmmac_set_desc_sarc(priv, first, priv->sarc_type); - - /* Ready to fill the first descriptor and set the OWN bit w/o any - * problems because all the descriptors are actually ready to be - * passed to the DMA engine. + /* Set the OWN bit on the first descriptor now that all descriptors + * for this skb are populated. */ - if (likely(!is_jumbo)) { - bool last_segment = (nfrags == 0); - - des = dma_map_single(priv->device, skb->data, - nopaged_len, DMA_TO_DEVICE); - if (dma_mapping_error(priv->device, des)) - goto dma_map_err; - - tx_q->tx_skbuff_dma[first_entry].buf = des; - tx_q->tx_skbuff_dma[first_entry].buf_type = STMMAC_TXBUF_T_SKB; - tx_q->tx_skbuff_dma[first_entry].map_as_page = false; - - stmmac_set_desc_addr(priv, first, des); - - tx_q->tx_skbuff_dma[first_entry].len = nopaged_len; - tx_q->tx_skbuff_dma[first_entry].last_segment = last_segment; - - if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && - priv->hwts_tx_en)) { - /* declare that device is doing timestamping */ - skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; - stmmac_enable_tx_timestamp(priv, first); - } - - /* Prepare the first descriptor setting the OWN bit too */ - stmmac_prepare_tx_desc(priv, first, 1, nopaged_len, - csum_insertion, priv->mode, 0, last_segment, - skb->len); - } - - if (tx_q->tbs & STMMAC_TBS_EN) { - struct timespec64 ts = ns_to_timespec64(skb->tstamp); - - tbs_desc = &tx_q->dma_entx[first_entry]; - stmmac_set_desc_tbs(priv, tbs_desc, ts.tv_sec, ts.tv_nsec); - } - - stmmac_set_tx_owner(priv, first); + stmmac_set_tx_owner(priv, first_desc); netdev_tx_sent_queue(netdev_get_tx_queue(dev, queue), skb->len); @@ -4909,6 +4975,46 @@ max_sdu_err: return NETDEV_TX_OK; } +static netdev_features_t stmmac_features_check(struct sk_buff *skb, + struct net_device *dev, + netdev_features_t features) +{ + struct stmmac_priv *priv = netdev_priv(dev); + u16 queue = skb_get_queue_mapping(skb); + + /* DWMAC IPs can be synthesized to support tx coe only for a few tx + * queues. In that case, checksum offloading for those queues that don't + * support tx coe needs to fallback to software checksum calculation. + * + * Packets that won't trigger the COE e.g. most DSA-tagged packets will + * also have to be checksummed in software. + * + * Note that disabling hardware checksumming also disables TSO. See + * harmonize_features() in net/core/dev.c + */ + if (priv->plat->tx_queues_cfg[queue].coe_unsupported || + !stmmac_has_ip_ethertype(skb)) + features &= ~(NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM); + + if (skb_is_gso(skb)) { + if (!stmmac_tso_channel_permitted(priv, queue) || + !stmmac_tso_valid_packet(skb)) + features &= ~NETIF_F_GSO_MASK; + + /* If we are going to be using hardware TSO, always insert + * VLAN tag to SKB payload for TSO frames. + * + * Never insert VLAN tag by HW, since segments split by + * TSO engine will be un-tagged by mistake. + */ + if (features & NETIF_F_GSO_MASK) + features &= ~(NETIF_F_HW_VLAN_STAG_TX | + NETIF_F_HW_VLAN_CTAG_TX); + } + + return vlan_features_check(skb, features); +} + static void stmmac_rx_vlan(struct net_device *dev, struct sk_buff *skb) { struct vlan_ethhdr *veth = skb_vlan_eth_hdr(skb); @@ -4949,10 +5055,7 @@ static inline void stmmac_rx_refill(struct stmmac_priv *priv, u32 queue) struct dma_desc *p; bool use_rx_wd; - if (priv->extend_desc) - p = (struct dma_desc *)(rx_q->dma_erx + entry); - else - p = rx_q->dma_rx + entry; + p = stmmac_get_rx_desc(priv, rx_q, entry); if (!buf->page) { buf->page = page_pool_alloc_pages(rx_q->page_pool, gfp); @@ -4990,12 +5093,10 @@ static inline void stmmac_rx_refill(struct stmmac_priv *priv, u32 queue) dma_wmb(); stmmac_set_rx_owner(priv, p, use_rx_wd); - entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_rx_size); + entry = STMMAC_NEXT_ENTRY(entry, priv->dma_conf.dma_rx_size); } rx_q->dirty_rx = entry; - rx_q->rx_tail_addr = rx_q->dma_rx_phy + - (rx_q->dirty_rx * sizeof(struct dma_desc)); - stmmac_set_rx_tail_ptr(priv, priv->ioaddr, rx_q->rx_tail_addr, queue); + stmmac_set_queue_rx_tail_ptr(priv, rx_q, queue, rx_q->dirty_rx); /* Wake up Rx DMA from the suspend state if required */ stmmac_enable_dma_reception(priv, priv->ioaddr, queue); } @@ -5070,6 +5171,7 @@ static int stmmac_xdp_xmit_xdpf(struct stmmac_priv *priv, int queue, struct stmmac_tx_queue *tx_q = &priv->dma_conf.tx_queue[queue]; bool csum = !priv->plat->tx_queues_cfg[queue].coe_unsupported; unsigned int entry = tx_q->cur_tx; + enum stmmac_txbuf_type buf_type; struct dma_desc *tx_desc; dma_addr_t dma_addr; bool set_ic; @@ -5084,20 +5186,14 @@ static int stmmac_xdp_xmit_xdpf(struct stmmac_priv *priv, int queue, return STMMAC_XDP_CONSUMED; } - if (likely(priv->extend_desc)) - tx_desc = (struct dma_desc *)(tx_q->dma_etx + entry); - else if (tx_q->tbs & STMMAC_TBS_AVAIL) - tx_desc = &tx_q->dma_entx[entry].basic; - else - tx_desc = tx_q->dma_tx + entry; - + tx_desc = stmmac_get_tx_desc(priv, tx_q, entry); if (dma_map) { dma_addr = dma_map_single(priv->device, xdpf->data, xdpf->len, DMA_TO_DEVICE); if (dma_mapping_error(priv->device, dma_addr)) return STMMAC_XDP_CONSUMED; - tx_q->tx_skbuff_dma[entry].buf_type = STMMAC_TXBUF_T_XDP_NDO; + buf_type = STMMAC_TXBUF_T_XDP_NDO; } else { struct page *page = virt_to_page(xdpf->data); @@ -5106,21 +5202,19 @@ static int stmmac_xdp_xmit_xdpf(struct stmmac_priv *priv, int queue, dma_sync_single_for_device(priv->device, dma_addr, xdpf->len, DMA_BIDIRECTIONAL); - tx_q->tx_skbuff_dma[entry].buf_type = STMMAC_TXBUF_T_XDP_TX; + buf_type = STMMAC_TXBUF_T_XDP_TX; } - tx_q->tx_skbuff_dma[entry].buf = dma_addr; - tx_q->tx_skbuff_dma[entry].map_as_page = false; - tx_q->tx_skbuff_dma[entry].len = xdpf->len; - tx_q->tx_skbuff_dma[entry].last_segment = true; - tx_q->tx_skbuff_dma[entry].is_jumbo = false; + stmmac_set_tx_dma_entry(tx_q, entry, buf_type, dma_addr, xdpf->len, + false); + stmmac_set_tx_dma_last_segment(tx_q, entry); tx_q->xdpf[entry] = xdpf; stmmac_set_desc_addr(priv, tx_desc, dma_addr); stmmac_prepare_tx_desc(priv, tx_desc, 1, xdpf->len, - csum, priv->mode, true, true, + csum, priv->descriptor_mode, true, true, xdpf->len); tx_q->tx_count_frames++; @@ -5140,7 +5234,7 @@ static int stmmac_xdp_xmit_xdpf(struct stmmac_priv *priv, int queue, stmmac_enable_dma_transmission(priv, priv->ioaddr, queue); - entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_tx_size); + entry = STMMAC_NEXT_ENTRY(entry, priv->dma_conf.dma_tx_size); tx_q->cur_tx = entry; return STMMAC_XDP_TX; @@ -5351,10 +5445,7 @@ static bool stmmac_rx_refill_zc(struct stmmac_priv *priv, u32 queue, u32 budget) } } - if (priv->extend_desc) - rx_desc = (struct dma_desc *)(rx_q->dma_erx + entry); - else - rx_desc = rx_q->dma_rx + entry; + rx_desc = stmmac_get_rx_desc(priv, rx_q, entry); dma_addr = xsk_buff_xdp_get_dma(buf->xdp); stmmac_set_desc_addr(priv, rx_desc, dma_addr); @@ -5374,14 +5465,12 @@ static bool stmmac_rx_refill_zc(struct stmmac_priv *priv, u32 queue, u32 budget) dma_wmb(); stmmac_set_rx_owner(priv, rx_desc, use_rx_wd); - entry = STMMAC_GET_ENTRY(entry, priv->dma_conf.dma_rx_size); + entry = STMMAC_NEXT_ENTRY(entry, priv->dma_conf.dma_rx_size); } if (rx_desc) { rx_q->dirty_rx = entry; - rx_q->rx_tail_addr = rx_q->dma_rx_phy + - (rx_q->dirty_rx * sizeof(struct dma_desc)); - stmmac_set_rx_tail_ptr(priv, priv->ioaddr, rx_q->rx_tail_addr, queue); + stmmac_set_queue_rx_tail_ptr(priv, rx_q, queue, rx_q->dirty_rx); } return ret; @@ -5412,16 +5501,10 @@ static int stmmac_rx_zc(struct stmmac_priv *priv, int limit, u32 queue) int status = 0; if (netif_msg_rx_status(priv)) { - void *rx_head; + void *rx_head = stmmac_get_rx_desc(priv, rx_q, 0); netdev_dbg(priv->dev, "%s: descriptor ring:\n", __func__); - if (priv->extend_desc) { - rx_head = (void *)rx_q->dma_erx; - desc_size = sizeof(struct dma_extended_desc); - } else { - rx_head = (void *)rx_q->dma_rx; - desc_size = sizeof(struct dma_desc); - } + desc_size = stmmac_get_rx_desc_size(priv); stmmac_display_ring(priv, rx_head, priv->dma_conf.dma_rx_size, true, rx_q->dma_rx_phy, desc_size); @@ -5457,10 +5540,7 @@ read_again: dirty = 0; } - if (priv->extend_desc) - p = (struct dma_desc *)(rx_q->dma_erx + entry); - else - p = rx_q->dma_rx + entry; + p = stmmac_get_rx_desc(priv, rx_q, entry); /* read the status of the incoming frame */ status = stmmac_rx_status(priv, &priv->xstats, p); @@ -5469,14 +5549,11 @@ read_again: break; /* Prefetch the next RX descriptor */ - rx_q->cur_rx = STMMAC_GET_ENTRY(rx_q->cur_rx, + rx_q->cur_rx = STMMAC_NEXT_ENTRY(rx_q->cur_rx, priv->dma_conf.dma_rx_size); next_entry = rx_q->cur_rx; - if (priv->extend_desc) - np = (struct dma_desc *)(rx_q->dma_erx + next_entry); - else - np = rx_q->dma_rx + next_entry; + np = stmmac_get_rx_desc(priv, rx_q, next_entry); prefetch(np); @@ -5612,16 +5689,10 @@ static int stmmac_rx(struct stmmac_priv *priv, int limit, u32 queue) limit = min(priv->dma_conf.dma_rx_size - 1, (unsigned int)limit); if (netif_msg_rx_status(priv)) { - void *rx_head; + void *rx_head = stmmac_get_rx_desc(priv, rx_q, 0); netdev_dbg(priv->dev, "%s: descriptor ring:\n", __func__); - if (priv->extend_desc) { - rx_head = (void *)rx_q->dma_erx; - desc_size = sizeof(struct dma_extended_desc); - } else { - rx_head = (void *)rx_q->dma_rx; - desc_size = sizeof(struct dma_desc); - } + desc_size = stmmac_get_rx_desc_size(priv); stmmac_display_ring(priv, rx_head, priv->dma_conf.dma_rx_size, true, rx_q->dma_rx_phy, desc_size); @@ -5654,10 +5725,7 @@ read_again: entry = next_entry; buf = &rx_q->buf_pool[entry]; - if (priv->extend_desc) - p = (struct dma_desc *)(rx_q->dma_erx + entry); - else - p = rx_q->dma_rx + entry; + p = stmmac_get_rx_desc(priv, rx_q, entry); /* read the status of the incoming frame */ status = stmmac_rx_status(priv, &priv->xstats, p); @@ -5665,14 +5733,11 @@ read_again: if (unlikely(status & dma_own)) break; - rx_q->cur_rx = STMMAC_GET_ENTRY(rx_q->cur_rx, + rx_q->cur_rx = STMMAC_NEXT_ENTRY(rx_q->cur_rx, priv->dma_conf.dma_rx_size); next_entry = rx_q->cur_rx; - if (priv->extend_desc) - np = (struct dma_desc *)(rx_q->dma_erx + next_entry); - else - np = rx_q->dma_rx + next_entry; + np = stmmac_get_rx_desc(priv, rx_q, next_entry); prefetch(np); @@ -6104,14 +6169,6 @@ static netdev_features_t stmmac_fix_features(struct net_device *dev, if (priv->plat->bugged_jumbo && (dev->mtu > ETH_DATA_LEN)) features &= ~NETIF_F_CSUM_MASK; - /* Disable tso if asked by ethtool */ - if ((priv->plat->flags & STMMAC_FLAG_TSO_EN) && (priv->dma_cap.tsoen)) { - if (features & NETIF_F_TSO) - priv->tso = true; - else - priv->tso = false; - } - return features; } @@ -6132,12 +6189,14 @@ static int stmmac_set_features(struct net_device *netdev, if (priv->sph_capable) { bool sph_en = (priv->hw->rx_csum > 0) && priv->sph_active; - u32 chan; + u8 chan; for (chan = 0; chan < priv->plat->rx_queues_to_use; chan++) stmmac_enable_sph(priv, priv->ioaddr, sph_en, chan); } + stmmac_set_gso_types(priv, features & NETIF_F_TSO); + if (features & NETIF_F_HW_VLAN_CTAG_RX) priv->hw->hw_vlan_en = true; else @@ -6152,11 +6211,11 @@ static int stmmac_set_features(struct net_device *netdev, static void stmmac_common_interrupt(struct stmmac_priv *priv) { - u32 rx_cnt = priv->plat->rx_queues_to_use; - u32 tx_cnt = priv->plat->tx_queues_to_use; - u32 queues_count; - u32 queue; + u8 rx_cnt = priv->plat->rx_queues_to_use; + u8 tx_cnt = priv->plat->tx_queues_to_use; + u8 queues_count; bool xmac; + u8 queue; xmac = dwmac_is_xmac(priv->plat->core_type); queues_count = (rx_cnt > tx_cnt) ? rx_cnt : tx_cnt; @@ -6454,9 +6513,9 @@ static int stmmac_rings_status_show(struct seq_file *seq, void *v) { struct net_device *dev = seq->private; struct stmmac_priv *priv = netdev_priv(dev); - u32 rx_count = priv->plat->rx_queues_to_use; - u32 tx_count = priv->plat->tx_queues_to_use; - u32 queue; + u8 rx_count = priv->plat->rx_queues_to_use; + u8 tx_count = priv->plat->tx_queues_to_use; + u8 queue; if ((dev->flags & IFF_UP) == 0) return 0; @@ -6581,9 +6640,9 @@ static int stmmac_dma_cap_show(struct seq_file *seq, void *v) priv->dma_cap.number_rx_channel); seq_printf(seq, "\tNumber of Additional TX channel: %d\n", priv->dma_cap.number_tx_channel); - seq_printf(seq, "\tNumber of Additional RX queues: %d\n", + seq_printf(seq, "\tNumber of Additional RX queues: %u\n", priv->dma_cap.number_rx_queues); - seq_printf(seq, "\tNumber of Additional TX queues: %d\n", + seq_printf(seq, "\tNumber of Additional TX queues: %u\n", priv->dma_cap.number_tx_queues); seq_printf(seq, "\tEnhanced descriptors: %s\n", (priv->dma_cap.enh_desc) ? "Y" : "N"); @@ -6945,7 +7004,6 @@ void stmmac_enable_rx_queue(struct stmmac_priv *priv, u32 queue) struct stmmac_rx_queue *rx_q = &priv->dma_conf.rx_queue[queue]; struct stmmac_channel *ch = &priv->channel[queue]; unsigned long flags; - u32 buf_size; int ret; ret = __alloc_dma_rx_desc_resources(priv, &priv->dma_conf, queue); @@ -6965,23 +7023,11 @@ void stmmac_enable_rx_queue(struct stmmac_priv *priv, u32 queue) stmmac_clear_rx_descriptors(priv, &priv->dma_conf, queue); stmmac_init_rx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, - rx_q->dma_rx_phy, rx_q->queue_index); + rx_q->dma_rx_phy, queue); - rx_q->rx_tail_addr = rx_q->dma_rx_phy + (rx_q->buf_alloc_num * - sizeof(struct dma_desc)); - stmmac_set_rx_tail_ptr(priv, priv->ioaddr, - rx_q->rx_tail_addr, rx_q->queue_index); + stmmac_set_queue_rx_tail_ptr(priv, rx_q, queue, rx_q->buf_alloc_num); - if (rx_q->xsk_pool && rx_q->buf_alloc_num) { - buf_size = xsk_pool_get_rx_frame_size(rx_q->xsk_pool); - stmmac_set_dma_bfsize(priv, priv->ioaddr, - buf_size, - rx_q->queue_index); - } else { - stmmac_set_dma_bfsize(priv, priv->ioaddr, - priv->dma_conf.dma_buf_sz, - rx_q->queue_index); - } + stmmac_set_queue_rx_buf_size(priv, rx_q, queue); stmmac_start_rx_dma(priv, queue); @@ -7027,14 +7073,12 @@ void stmmac_enable_tx_queue(struct stmmac_priv *priv, u32 queue) stmmac_clear_tx_descriptors(priv, &priv->dma_conf, queue); stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, - tx_q->dma_tx_phy, tx_q->queue_index); + tx_q->dma_tx_phy, queue); if (tx_q->tbs & STMMAC_TBS_AVAIL) - stmmac_enable_tbs(priv, priv->ioaddr, 1, tx_q->queue_index); + stmmac_enable_tbs(priv, priv->ioaddr, 1, queue); - tx_q->tx_tail_addr = tx_q->dma_tx_phy; - stmmac_set_tx_tail_ptr(priv, priv->ioaddr, - tx_q->tx_tail_addr, tx_q->queue_index); + stmmac_set_queue_tx_tail_ptr(priv, tx_q, queue, 0); stmmac_start_tx_dma(priv, queue); @@ -7046,7 +7090,7 @@ void stmmac_enable_tx_queue(struct stmmac_priv *priv, u32 queue) void stmmac_xdp_release(struct net_device *dev) { struct stmmac_priv *priv = netdev_priv(dev); - u32 chan; + u8 chan; /* Ensure tx function is not running */ netif_tx_disable(dev); @@ -7079,14 +7123,13 @@ void stmmac_xdp_release(struct net_device *dev) int stmmac_xdp_open(struct net_device *dev) { struct stmmac_priv *priv = netdev_priv(dev); - u32 rx_cnt = priv->plat->rx_queues_to_use; - u32 tx_cnt = priv->plat->tx_queues_to_use; - u32 dma_csr_ch = max(rx_cnt, tx_cnt); + u8 rx_cnt = priv->plat->rx_queues_to_use; + u8 tx_cnt = priv->plat->tx_queues_to_use; + u8 dma_csr_ch = max(rx_cnt, tx_cnt); struct stmmac_rx_queue *rx_q; struct stmmac_tx_queue *tx_q; - u32 buf_size; bool sph_en; - u32 chan; + u8 chan; int ret; ret = alloc_dma_desc_resources(priv, &priv->dma_conf); @@ -7121,22 +7164,10 @@ int stmmac_xdp_open(struct net_device *dev) stmmac_init_rx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, rx_q->dma_rx_phy, chan); - rx_q->rx_tail_addr = rx_q->dma_rx_phy + - (rx_q->buf_alloc_num * - sizeof(struct dma_desc)); - stmmac_set_rx_tail_ptr(priv, priv->ioaddr, - rx_q->rx_tail_addr, chan); - - if (rx_q->xsk_pool && rx_q->buf_alloc_num) { - buf_size = xsk_pool_get_rx_frame_size(rx_q->xsk_pool); - stmmac_set_dma_bfsize(priv, priv->ioaddr, - buf_size, - rx_q->queue_index); - } else { - stmmac_set_dma_bfsize(priv, priv->ioaddr, - priv->dma_conf.dma_buf_sz, - rx_q->queue_index); - } + stmmac_set_queue_rx_tail_ptr(priv, rx_q, chan, + rx_q->buf_alloc_num); + + stmmac_set_queue_rx_buf_size(priv, rx_q, chan); stmmac_enable_sph(priv, priv->ioaddr, sph_en, chan); } @@ -7148,9 +7179,7 @@ int stmmac_xdp_open(struct net_device *dev) stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, tx_q->dma_tx_phy, chan); - tx_q->tx_tail_addr = tx_q->dma_tx_phy; - stmmac_set_tx_tail_ptr(priv, priv->ioaddr, - tx_q->tx_tail_addr, chan); + stmmac_set_queue_tx_tail_ptr(priv, tx_q, chan, 0); hrtimer_setup(&tx_q->txtimer, stmmac_tx_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); } @@ -7222,10 +7251,10 @@ int stmmac_xsk_wakeup(struct net_device *dev, u32 queue, u32 flags) static void stmmac_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats) { struct stmmac_priv *priv = netdev_priv(dev); - u32 tx_cnt = priv->plat->tx_queues_to_use; - u32 rx_cnt = priv->plat->rx_queues_to_use; + u8 tx_cnt = priv->plat->tx_queues_to_use; + u8 rx_cnt = priv->plat->rx_queues_to_use; unsigned int start; - int q; + u8 q; for (q = 0; q < tx_cnt; q++) { struct stmmac_txq_stats *txq_stats = &priv->xstats.txq_stats[q]; @@ -7275,6 +7304,7 @@ static void stmmac_get_stats64(struct net_device *dev, struct rtnl_link_stats64 static const struct net_device_ops stmmac_netdev_ops = { .ndo_open = stmmac_open, .ndo_start_xmit = stmmac_xmit, + .ndo_features_check = stmmac_features_check, .ndo_stop = stmmac_release, .ndo_change_mtu = stmmac_change_mtu, .ndo_fix_features = stmmac_fix_features, @@ -7375,7 +7405,7 @@ static int stmmac_hw_init(struct stmmac_priv *priv) /* dwmac-sun8i only work in chain mode */ if (priv->plat->flags & STMMAC_FLAG_HAS_SUN8I) chain_mode = 1; - priv->chain_mode = chain_mode; + priv->chain_mode = !!chain_mode; /* Initialize HW Interface */ ret = stmmac_hwif_init(priv); @@ -7404,7 +7434,7 @@ static int stmmac_hw_init(struct stmmac_priv *priv) /* TXCOE doesn't work in thresh DMA mode */ if (priv->plat->force_thresh_dma_mode) - priv->plat->tx_coe = 0; + priv->plat->tx_coe = false; else priv->plat->tx_coe = priv->dma_cap.tx_coe; @@ -7436,9 +7466,6 @@ static int stmmac_hw_init(struct stmmac_priv *priv) devm_pm_set_wake_irq(priv->device, priv->wol_irq); } - if (priv->dma_cap.tsoen) - dev_info(priv->device, "TSO supported\n"); - if (priv->dma_cap.number_rx_queues && priv->plat->rx_queues_to_use > priv->dma_cap.number_rx_queues) { dev_warn(priv->device, @@ -7480,6 +7507,14 @@ static int stmmac_hw_init(struct stmmac_priv *priv) return ret; } + /* Set alternate descriptor size (which tells the hardware that + * descriptors are 8 32-bit words) when using extended descriptors + * with ring mode. Only applicable for pre-v4.0 cores. Platform glue + * is not expected to change this. + */ + priv->plat->dma_cfg->atds = priv->extend_desc && + priv->descriptor_mode == STMMAC_RING_MODE; + /* Rx Watchdog is available in the COREs newer than the 3.40. * In some case, for example on bugged HW this feature * has to be disable and this can be done by passing the @@ -7506,7 +7541,7 @@ static int stmmac_hw_init(struct stmmac_priv *priv) static void stmmac_napi_add(struct net_device *dev) { struct stmmac_priv *priv = netdev_priv(dev); - u32 queue, maxq; + u8 queue, maxq; maxq = max(priv->plat->rx_queues_to_use, priv->plat->tx_queues_to_use); @@ -7535,7 +7570,7 @@ static void stmmac_napi_add(struct net_device *dev) static void stmmac_napi_del(struct net_device *dev) { struct stmmac_priv *priv = netdev_priv(dev); - u32 queue, maxq; + u8 queue, maxq; maxq = max(priv->plat->rx_queues_to_use, priv->plat->tx_queues_to_use); @@ -7553,7 +7588,7 @@ static void stmmac_napi_del(struct net_device *dev) } } -int stmmac_reinit_queues(struct net_device *dev, u32 rx_cnt, u32 tx_cnt) +int stmmac_reinit_queues(struct net_device *dev, u8 rx_cnt, u8 tx_cnt) { struct stmmac_priv *priv = netdev_priv(dev); int ret = 0, i; @@ -7725,6 +7760,8 @@ struct plat_stmmacenet_data *stmmac_plat_dat_alloc(struct device *dev) if (!plat_dat) return NULL; + plat_dat->dma_cfg = &plat_dat->__dma_cfg; + /* Set the defaults: * - phy autodetection * - determine GMII_Address CR field from CSR clock @@ -7756,8 +7793,13 @@ static int __stmmac_dvr_probe(struct device *device, { struct net_device *ndev = NULL; struct stmmac_priv *priv; - u32 rxq; int i, ret = 0; + u8 rxq; + + if (!plat_dat->dma_cfg || !plat_dat->dma_cfg->pbl) { + dev_err(device, "invalid DMA configuration\n"); + return -EINVAL; + } ndev = devm_alloc_etherdev_mqs(device, sizeof(struct stmmac_priv), MTL_MAX_TX_QUEUES, MTL_MAX_RX_QUEUES); @@ -7793,12 +7835,12 @@ static int __stmmac_dvr_probe(struct device *device, priv->dev->irq = res->irq; priv->wol_irq = res->wol_irq; priv->sfty_irq = res->sfty_irq; - priv->sfty_ce_irq = res->sfty_ce_irq; - priv->sfty_ue_irq = res->sfty_ue_irq; - for (i = 0; i < MTL_MAX_RX_QUEUES; i++) - priv->rx_irq[i] = res->rx_irq[i]; - for (i = 0; i < MTL_MAX_TX_QUEUES; i++) - priv->tx_irq[i] = res->tx_irq[i]; + + if (priv->plat->flags & STMMAC_FLAG_MULTI_MSI_EN) { + ret = stmmac_msi_init(priv, res); + if (ret) + return ret; + } if (!is_zero_ether_addr(res->mac)) eth_hw_addr_set(priv->dev, res->mac); @@ -7875,13 +7917,7 @@ static int __stmmac_dvr_probe(struct device *device, ndev->hw_features |= NETIF_F_HW_TC; } - if ((priv->plat->flags & STMMAC_FLAG_TSO_EN) && (priv->dma_cap.tsoen)) { - ndev->hw_features |= NETIF_F_TSO | NETIF_F_TSO6; - if (priv->plat->core_type == DWMAC_CORE_GMAC4) - ndev->hw_features |= NETIF_F_GSO_UDP_L4; - priv->tso = true; - dev_info(priv->device, "TSO feature enabled\n"); - } + stmmac_set_gso_features(ndev); if (priv->dma_cap.sphen && !(priv->plat->flags & STMMAC_FLAG_SPH_DISABLE)) { @@ -8135,7 +8171,7 @@ int stmmac_suspend(struct device *dev) { struct net_device *ndev = dev_get_drvdata(dev); struct stmmac_priv *priv = netdev_priv(ndev); - u32 chan; + u8 chan; if (!ndev || !netif_running(ndev)) goto suspend_bsp; @@ -8210,9 +8246,9 @@ static void stmmac_reset_tx_queue(struct stmmac_priv *priv, u32 queue) */ static void stmmac_reset_queues_param(struct stmmac_priv *priv) { - u32 rx_cnt = priv->plat->rx_queues_to_use; - u32 tx_cnt = priv->plat->tx_queues_to_use; - u32 queue; + u8 rx_cnt = priv->plat->rx_queues_to_use; + u8 tx_cnt = priv->plat->tx_queues_to_use; + u8 queue; for (queue = 0; queue < rx_cnt; queue++) stmmac_reset_rx_queue(priv, queue); diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_mdio.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_mdio.c index a7c2496b39f2..afe98ff5bdcb 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_mdio.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_mdio.c @@ -234,8 +234,8 @@ static u32 stmmac_mdio_format_addr(struct stmmac_priv *priv, { const struct mii_regs *mii_regs = &priv->hw->mii; - return ((pa << mii_regs->addr_shift) & mii_regs->addr_mask) | - ((gr << mii_regs->reg_shift) & mii_regs->reg_mask) | + return field_prep(mii_regs->addr_mask, pa) | + field_prep(mii_regs->reg_mask, gr) | priv->gmii_address_bus_config | MII_ADDR_GBUSY; } @@ -430,7 +430,7 @@ int stmmac_pcs_setup(struct net_device *ndev) struct dw_xpcs *xpcs = NULL; int addr, ret; - devnode = priv->plat->port_node; + devnode = dev_fwnode(priv->device); if (priv->plat->pcs_init) { ret = priv->plat->pcs_init(priv); @@ -473,6 +473,53 @@ void stmmac_pcs_clean(struct net_device *ndev) priv->hw->xpcs = NULL; } +struct stmmac_clk_rate { + unsigned long rate; + u8 cr; +}; + +/* The standard clk_csr_i to GMII_Address CR field mapping. The rate provided + * in this table is the exclusive maximum frequency for the divisor. The + * comments for each entry give the divisor and the resulting range of MDC + * clock frequencies. + */ +static const struct stmmac_clk_rate stmmac_std_csr_to_mdc[] = { + { CSR_F_800M, ~0 }, + { CSR_F_500M, STMMAC_CSR_500_800M }, + { CSR_F_300M, STMMAC_CSR_300_500M }, + { CSR_F_250M, STMMAC_CSR_250_300M }, + { CSR_F_150M, STMMAC_CSR_150_250M }, + { CSR_F_100M, STMMAC_CSR_100_150M }, + { CSR_F_60M, STMMAC_CSR_60_100M }, + { CSR_F_35M, STMMAC_CSR_35_60M }, + { CSR_F_20M, STMMAC_CSR_20_35M }, + { 0, ~0 }, +}; + +/* The sun8i clk_csr_i to GMII_Address CR field mapping uses rate as the + * exclusive minimum frequency for the divisor. Note that the last entry + * is valid and also acts as the sentinel. + */ +static const struct stmmac_clk_rate stmmac_sun8i_csr_to_mdc[] = { + { 160000000, 3 }, + { 80000000, 2 }, + { 40000000, 1 }, + { 0, 0 }, +}; + +/* The xgmac clk_csr_i to GMII_Address CR field mapping similarly uses rate + * as the exclusive minimum frequency for the divisor, and again the last + * entry is valid and also the sentinel. + */ +static const struct stmmac_clk_rate stmmac_xgmac_csr_to_mdc[] = { + { 400000000, 5 }, + { 350000000, 4 }, + { 300000000, 3 }, + { 250000000, 2 }, + { 150000000, 1 }, + { 0, 0 }, +}; + /** * stmmac_clk_csr_set - dynamically set the MDC clock * @priv: driver private structure @@ -488,8 +535,10 @@ void stmmac_pcs_clean(struct net_device *ndev) */ static u32 stmmac_clk_csr_set(struct stmmac_priv *priv) { + const struct stmmac_clk_rate *rates; unsigned long clk_rate; u32 value = ~0; + int i; clk_rate = clk_get_rate(priv->plat->stmmac_clk); @@ -500,48 +549,17 @@ static u32 stmmac_clk_csr_set(struct stmmac_priv *priv) * the frequency of clk_csr_i. So we do not change the default * divider. */ - if (clk_rate < CSR_F_35M) - value = STMMAC_CSR_20_35M; - else if (clk_rate < CSR_F_60M) - value = STMMAC_CSR_35_60M; - else if (clk_rate < CSR_F_100M) - value = STMMAC_CSR_60_100M; - else if (clk_rate < CSR_F_150M) - value = STMMAC_CSR_100_150M; - else if (clk_rate < CSR_F_250M) - value = STMMAC_CSR_150_250M; - else if (clk_rate <= CSR_F_300M) - value = STMMAC_CSR_250_300M; - else if (clk_rate < CSR_F_500M) - value = STMMAC_CSR_300_500M; - else if (clk_rate < CSR_F_800M) - value = STMMAC_CSR_500_800M; - - if (priv->plat->flags & STMMAC_FLAG_HAS_SUN8I) { - if (clk_rate > 160000000) - value = 0x03; - else if (clk_rate > 80000000) - value = 0x02; - else if (clk_rate > 40000000) - value = 0x01; - else - value = 0; - } + rates = stmmac_std_csr_to_mdc; + if (priv->plat->flags & STMMAC_FLAG_HAS_SUN8I) + rates = stmmac_sun8i_csr_to_mdc; + if (priv->plat->core_type == DWMAC_CORE_XGMAC) + rates = stmmac_xgmac_csr_to_mdc; - if (priv->plat->core_type == DWMAC_CORE_XGMAC) { - if (clk_rate > 400000000) - value = 0x5; - else if (clk_rate > 350000000) - value = 0x4; - else if (clk_rate > 300000000) - value = 0x3; - else if (clk_rate > 250000000) - value = 0x2; - else if (clk_rate > 150000000) - value = 0x1; - else - value = 0x0; - } + for (i = 0; rates[i].rate; i++) + if (clk_rate > rates[i].rate) + break; + if (rates[i].cr != (u8)~0) + value = rates[i].cr; return value; } @@ -559,7 +577,7 @@ static void stmmac_mdio_bus_config(struct stmmac_priv *priv) else value = stmmac_clk_csr_set(priv); - value <<= priv->hw->mii.clk_csr_shift; + value <<= __ffs(priv->hw->mii.clk_csr_mask); if (value & ~priv->hw->mii.clk_csr_mask) dev_warn(priv->device, @@ -649,10 +667,7 @@ int stmmac_mdio_register(struct net_device *ndev) stmmac_xgmac2_mdio_read_c45(new_bus, 0, 0, 0); /* If fixed-link is set, skip PHY scanning */ - fwnode = priv->plat->port_node; - if (!fwnode) - fwnode = dev_fwnode(priv->device); - + fwnode = dev_fwnode(priv->device); if (fwnode) { fixed_node = fwnode_get_named_child_node(fwnode, "fixed-link"); if (fixed_node) { diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_pci.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_pci.c index 270ad066ced3..d584fd2daa6f 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_pci.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_pci.c @@ -25,7 +25,7 @@ static void common_default_data(struct plat_stmmacenet_data *plat) /* clk_csr_i = 20-35MHz & MDC = clk_csr_i/16 */ plat->clk_csr = STMMAC_CSR_20_35M; plat->core_type = DWMAC_CORE_GMAC; - plat->force_sf_dma_mode = 1; + plat->force_sf_dma_mode = true; plat->mdio_bus_data->needs_reset = true; } @@ -58,9 +58,9 @@ static int snps_gmac5_default_data(struct pci_dev *pdev, plat->clk_csr = STMMAC_CSR_250_300M; plat->core_type = DWMAC_CORE_GMAC4; - plat->force_sf_dma_mode = 1; + plat->force_sf_dma_mode = true; plat->flags |= STMMAC_FLAG_TSO_EN; - plat->pmt = 1; + plat->pmt = true; /* Set default number of RX and TX queues to use */ plat->tx_queues_to_use = 4; @@ -134,11 +134,6 @@ static int stmmac_pci_probe(struct pci_dev *pdev, if (!plat->mdio_bus_data) return -ENOMEM; - plat->dma_cfg = devm_kzalloc(&pdev->dev, sizeof(*plat->dma_cfg), - GFP_KERNEL); - if (!plat->dma_cfg) - return -ENOMEM; - plat->safety_feat_cfg = devm_kzalloc(&pdev->dev, sizeof(*plat->safety_feat_cfg), GFP_KERNEL); diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_pcs.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_pcs.c index 88fa359ea716..df37af5ab837 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_pcs.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_pcs.c @@ -16,6 +16,37 @@ #define GMAC_ANE_LPA 0x0c /* ANE link partener ability */ #define GMAC_TBI 0x14 /* TBI extend status */ +/* + * RGSMII status bitfield definitions. + */ +#define GMAC_RGSMII_LNKMOD BIT(0) +#define GMAC_RGSMII_SPEED_MASK GENMASK(2, 1) +#define GMAC_RGSMII_SPEED_125 2 +#define GMAC_RGSMII_SPEED_25 1 +#define GMAC_RGSMII_SPEED_2_5 0 +#define GMAC_RGSMII_LNKSTS BIT(3) + +static unsigned int dwmac_integrated_pcs_inband_caps(struct phylink_pcs *pcs, + phy_interface_t interface) +{ + struct stmmac_pcs *spcs = phylink_pcs_to_stmmac_pcs(pcs); + unsigned int ib_caps; + + if (phy_interface_mode_is_8023z(interface)) { + ib_caps = LINK_INBAND_DISABLE; + + /* If the PCS supports TBI/RTBI, then BASE-X negotiation is + * supported. + */ + if (spcs->support_tbi_rtbi) + ib_caps |= LINK_INBAND_ENABLE; + + return ib_caps; + } + + return 0; +} + static int dwmac_integrated_pcs_enable(struct phylink_pcs *pcs) { struct stmmac_pcs *spcs = phylink_pcs_to_stmmac_pcs(pcs); @@ -36,7 +67,65 @@ static void dwmac_integrated_pcs_get_state(struct phylink_pcs *pcs, unsigned int neg_mode, struct phylink_link_state *state) { - state->link = false; + struct stmmac_pcs *spcs = phylink_pcs_to_stmmac_pcs(pcs); + u32 status, lpa, rgsmii; + + status = readl(spcs->base + GMAC_AN_STATUS); + + if (phy_interface_mode_is_8023z(state->interface)) { + /* For BASE-X modes, the PCS block supports the advertisement + * and link partner advertisement registers using standard + * 802.3 format. The status register also has the link status + * and AN complete bits in the same bit location. This will + * only be used when AN is enabled. + */ + lpa = readl(spcs->base + GMAC_ANE_LPA); + + phylink_mii_c22_pcs_decode_state(state, neg_mode, status, lpa); + } else { + rgsmii = field_get(spcs->rgsmii_status_mask, + readl(spcs->rgsmii)); + + state->link = status & BMSR_LSTATUS && + rgsmii & GMAC_RGSMII_LNKSTS; + + if (state->link && neg_mode == PHYLINK_PCS_NEG_INBAND_ENABLED) { + state->duplex = rgsmii & GMAC_RGSMII_LNKMOD ? + DUPLEX_FULL : DUPLEX_HALF; + switch (FIELD_GET(GMAC_RGSMII_SPEED_MASK, rgsmii)) { + case GMAC_RGSMII_SPEED_2_5: + state->speed = SPEED_10; + break; + + case GMAC_RGSMII_SPEED_25: + state->speed = SPEED_100; + break; + + case GMAC_RGSMII_SPEED_125: + state->speed = SPEED_1000; + break; + + default: + state->link = false; + break; + } + } + } +} + +static int dwmac_integrated_pcs_config_aneg(struct stmmac_pcs *spcs, + phy_interface_t interface, + const unsigned long *advertising) +{ + bool changed = false; + u32 adv; + + adv = phylink_mii_c22_pcs_encode_advertisement(interface, advertising); + if (readl(spcs->base + GMAC_ANE_ADV) != adv) + changed = true; + writel(adv, spcs->base + GMAC_ANE_ADV); + + return changed; } static int dwmac_integrated_pcs_config(struct phylink_pcs *pcs, @@ -46,17 +135,46 @@ static int dwmac_integrated_pcs_config(struct phylink_pcs *pcs, bool permit_pause_to_mac) { struct stmmac_pcs *spcs = phylink_pcs_to_stmmac_pcs(pcs); + bool changed = false, ane = true; + + /* Only configure the advertisement and allow AN in BASE-X mode if + * the core supports TBI/RTBI. AN will be filtered out by via phylink + * and the .pcs_inband_caps() method above. + */ + if (phy_interface_mode_is_8023z(interface) && + spcs->support_tbi_rtbi) { + ane = neg_mode == PHYLINK_PCS_NEG_INBAND_ENABLED; + + changed = dwmac_integrated_pcs_config_aneg(spcs, interface, + advertising); + } - dwmac_ctrl_ane(spcs->base, 0, 1, spcs->priv->hw->reverse_sgmii_enable); + dwmac_ctrl_ane(spcs->base, 0, ane, + spcs->priv->hw->reverse_sgmii_enable); - return 0; + return changed; +} + +static void dwmac_integrated_pcs_an_restart(struct phylink_pcs *pcs) +{ + struct stmmac_pcs *spcs = phylink_pcs_to_stmmac_pcs(pcs); + void __iomem *an_control = spcs->base + GMAC_AN_CTRL(0); + u32 ctrl; + + /* We can only do AN restart if using TBI/RTBI mode */ + if (spcs->support_tbi_rtbi) { + ctrl = readl(an_control) | GMAC_AN_CTRL_RAN; + writel(ctrl, an_control); + } } static const struct phylink_pcs_ops dwmac_integrated_pcs_ops = { + .pcs_inband_caps = dwmac_integrated_pcs_inband_caps, .pcs_enable = dwmac_integrated_pcs_enable, .pcs_disable = dwmac_integrated_pcs_disable, .pcs_get_state = dwmac_integrated_pcs_get_state, .pcs_config = dwmac_integrated_pcs_config, + .pcs_an_restart = dwmac_integrated_pcs_an_restart, }; void stmmac_integrated_pcs_irq(struct stmmac_priv *priv, u32 status, @@ -84,14 +202,23 @@ void stmmac_integrated_pcs_irq(struct stmmac_priv *priv, u32 status, int stmmac_integrated_pcs_get_phy_intf_sel(struct phylink_pcs *pcs, phy_interface_t interface) { + struct stmmac_pcs *spcs = phylink_pcs_to_stmmac_pcs(pcs); + if (interface == PHY_INTERFACE_MODE_SGMII) return PHY_INTF_SEL_SGMII; + if (phy_interface_mode_is_8023z(interface)) { + if (spcs->support_tbi_rtbi) + return PHY_INTF_SEL_TBI; + else + return PHY_INTF_SEL_SGMII; + } + return -EINVAL; } -int stmmac_integrated_pcs_init(struct stmmac_priv *priv, unsigned int offset, - u32 int_mask) +int stmmac_integrated_pcs_init(struct stmmac_priv *priv, + const struct stmmac_pcs_info *pcs_info) { struct stmmac_pcs *spcs; @@ -100,11 +227,26 @@ int stmmac_integrated_pcs_init(struct stmmac_priv *priv, unsigned int offset, return -ENOMEM; spcs->priv = priv; - spcs->base = priv->ioaddr + offset; - spcs->int_mask = int_mask; + spcs->base = priv->ioaddr + pcs_info->pcs_offset; + spcs->rgsmii = priv->ioaddr + pcs_info->rgsmii_offset; + spcs->rgsmii_status_mask = pcs_info->rgsmii_status_mask; + spcs->int_mask = pcs_info->int_mask; spcs->pcs.ops = &dwmac_integrated_pcs_ops; + /* If the PCS supports extended status, then it supports BASE-X AN + * with a TBI interface to the SerDes. Otherwise, we can support + * BASE-X without AN using SGMII, which is required for qcom-ethqos. + */ + if (readl(spcs->base + GMAC_AN_STATUS) & BMSR_ESTATEN) + spcs->support_tbi_rtbi = true; + __set_bit(PHY_INTERFACE_MODE_SGMII, spcs->pcs.supported_interfaces); + __set_bit(PHY_INTERFACE_MODE_1000BASEX, spcs->pcs.supported_interfaces); + + /* Only allow 2500BASE-X if the SerDes has support. */ + if (priv->plat->flags & STMMAC_FLAG_SERDES_SUPPORTS_2500M) + __set_bit(PHY_INTERFACE_MODE_2500BASEX, + spcs->pcs.supported_interfaces); priv->integrated_pcs = spcs; diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_pcs.h b/drivers/net/ethernet/stmicro/stmmac/stmmac_pcs.h index 23bbd4f10bf8..b2b12d34b3dd 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_pcs.h +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_pcs.h @@ -27,11 +27,21 @@ struct stmmac_priv; +struct stmmac_pcs_info { + unsigned int pcs_offset; + unsigned int rgsmii_offset; + u32 rgsmii_status_mask; + u32 int_mask; +}; + struct stmmac_pcs { struct stmmac_priv *priv; void __iomem *base; + void __iomem *rgsmii; + u32 rgsmii_status_mask; u32 int_mask; struct phylink_pcs pcs; + bool support_tbi_rtbi; }; static inline struct stmmac_pcs * @@ -44,8 +54,8 @@ void stmmac_integrated_pcs_irq(struct stmmac_priv *priv, u32 status, struct stmmac_extra_stats *x); int stmmac_integrated_pcs_get_phy_intf_sel(struct phylink_pcs *pcs, phy_interface_t interface); -int stmmac_integrated_pcs_init(struct stmmac_priv *priv, unsigned int offset, - u32 int_mask); +int stmmac_integrated_pcs_init(struct stmmac_priv *priv, + const struct stmmac_pcs_info *pcs_info); /** * dwmac_ctrl_ane - To program the AN Control Register. diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.c index 5c9fd91a1db9..5cae2aa72906 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_platform.c @@ -109,10 +109,7 @@ static struct stmmac_axi *stmmac_axi_setup(struct platform_device *pdev) axi->axi_lpi_en = of_property_read_bool(np, "snps,lpi_en"); axi->axi_xit_frm = of_property_read_bool(np, "snps,xit_frm"); - axi->axi_kbbe = of_property_read_bool(np, "snps,kbbe"); axi->axi_fb = of_property_read_bool(np, "snps,fb"); - axi->axi_mb = of_property_read_bool(np, "snps,mb"); - axi->axi_rb = of_property_read_bool(np, "snps,rb"); if (of_property_read_u32(np, "snps,wr_osr_lmt", &axi->axi_wr_osr_lmt)) axi->axi_wr_osr_lmt = 1; @@ -138,6 +135,7 @@ static int stmmac_mtl_setup(struct platform_device *pdev, struct device_node *tx_node; u8 queue = 0; int ret = 0; + u32 value; /* First Queue must always be in DCB mode. As MTL_QUEUE_DCB = 1 we need * to always set this, otherwise Queue will be classified as AVB @@ -157,8 +155,11 @@ static int stmmac_mtl_setup(struct platform_device *pdev, } /* Processing RX queues common config */ - of_property_read_u32(rx_node, "snps,rx-queues-to-use", - &plat->rx_queues_to_use); + if (!of_property_read_u32(rx_node, "snps,rx-queues-to-use", &value)) { + if (value > U8_MAX) + value = U8_MAX; + plat->rx_queues_to_use = value; + } if (of_property_read_bool(rx_node, "snps,rx-sched-sp")) plat->rx_sched_algorithm = MTL_RX_ALGORITHM_SP; @@ -208,8 +209,11 @@ static int stmmac_mtl_setup(struct platform_device *pdev, } /* Processing TX queues common config */ - of_property_read_u32(tx_node, "snps,tx-queues-to-use", - &plat->tx_queues_to_use); + if (!of_property_read_u32(tx_node, "snps,tx-queues-to-use", &value)) { + if (value > U8_MAX) + value = U8_MAX; + plat->tx_queues_to_use = value; + } if (of_property_read_bool(tx_node, "snps,tx-sched-wrr")) plat->tx_sched_algorithm = MTL_TX_ALGORITHM_WRR; @@ -396,6 +400,7 @@ static const char * const stmmac_gmac4_compats[] = { "snps,dwmac-5.10a", "snps,dwmac-5.20", "snps,dwmac-5.30a", + "snps,dwmac-5.40a", NULL }; @@ -446,9 +451,6 @@ stmmac_probe_config_dt(struct platform_device *pdev, u8 *mac) * they are not converted to phylink. */ plat->phy_node = of_parse_phandle(np, "phy-handle", 0); - /* PHYLINK automatically parses the phy-handle property */ - plat->port_node = of_fwnode_handle(np); - /* Get max speed of operation from device tree */ of_property_read_u32(np, "max-speed", &plat->max_speed); @@ -517,47 +519,41 @@ stmmac_probe_config_dt(struct platform_device *pdev, u8 *mac) plat->multicast_filter_bins = dwmac1000_validate_mcast_bins( &pdev->dev, plat->multicast_filter_bins); plat->core_type = DWMAC_CORE_GMAC; - plat->pmt = 1; + plat->pmt = true; } if (of_device_is_compatible(np, "snps,dwmac-3.40a")) { plat->core_type = DWMAC_CORE_GMAC; - plat->enh_desc = 1; - plat->tx_coe = 1; - plat->bugged_jumbo = 1; - plat->pmt = 1; + plat->enh_desc = true; + plat->tx_coe = true; + plat->bugged_jumbo = true; + plat->pmt = true; } if (of_device_compatible_match(np, stmmac_gmac4_compats)) { plat->core_type = DWMAC_CORE_GMAC4; - plat->pmt = 1; + plat->pmt = true; if (of_property_read_bool(np, "snps,tso")) plat->flags |= STMMAC_FLAG_TSO_EN; } if (of_device_is_compatible(np, "snps,dwmac-3.610") || of_device_is_compatible(np, "snps,dwmac-3.710")) { - plat->enh_desc = 1; - plat->bugged_jumbo = 1; - plat->force_sf_dma_mode = 1; + plat->enh_desc = true; + plat->bugged_jumbo = true; + plat->force_sf_dma_mode = true; } if (of_device_is_compatible(np, "snps,dwxgmac")) { plat->core_type = DWMAC_CORE_XGMAC; - plat->pmt = 1; + plat->pmt = true; if (of_property_read_bool(np, "snps,tso")) plat->flags |= STMMAC_FLAG_TSO_EN; of_property_read_u32(np, "snps,multicast-filter-bins", &plat->multicast_filter_bins); } - dma_cfg = devm_kzalloc(&pdev->dev, sizeof(*dma_cfg), - GFP_KERNEL); - if (!dma_cfg) { - ret = ERR_PTR(-ENOMEM); - goto error_put_mdio; - } - plat->dma_cfg = dma_cfg; + dma_cfg = plat->dma_cfg; of_property_read_u32(np, "snps,pbl", &dma_cfg->pbl); if (!dma_cfg->pbl) @@ -697,9 +693,47 @@ struct clk *stmmac_pltfr_find_clk(struct plat_stmmacenet_data *plat_dat, } EXPORT_SYMBOL_GPL(stmmac_pltfr_find_clk); +/** + * stmmac_pltfr_get_irq_array - Read per-channel IRQs from platform device + * @pdev: platform device + * @fmt: IRQ name format string (e.g., "tx-queue-%d") + * @irqs: array to store IRQ numbers + * @num: maximum number of IRQs to read + * + * Return: 0 on success, -EPROBE_DEFER if IRQ is deferred, -EINVAL on error. + * Missing IRQs are set to 0 and iteration stops at first missing IRQ. + */ +static int stmmac_pltfr_get_irq_array(struct platform_device *pdev, + const char *fmt, int *irqs, size_t num) +{ + char name[16]; + int i; + + for (i = 0; i < num; i++) { + if (snprintf(name, sizeof(name), fmt, i) >= sizeof(name)) + return -EINVAL; + + irqs[i] = platform_get_irq_byname_optional(pdev, name); + if (irqs[i] == -EPROBE_DEFER) + return -EPROBE_DEFER; + + if (irqs[i] <= 0) { + dev_dbg(&pdev->dev, "IRQ %s not found\n", name); + + /* Stop silently on first unset irq */ + irqs[i] = 0; + break; + } + } + + return 0; +} + int stmmac_get_platform_resources(struct platform_device *pdev, struct stmmac_resources *stmmac_res) { + int ret; + memset(stmmac_res, 0, sizeof(*stmmac_res)); /* Get IRQ information early to have an ability to ask for deferred @@ -735,7 +769,24 @@ int stmmac_get_platform_resources(struct platform_device *pdev, stmmac_res->addr = devm_platform_ioremap_resource(pdev, 0); - return PTR_ERR_OR_ZERO(stmmac_res->addr); + if (IS_ERR(stmmac_res->addr)) + return PTR_ERR(stmmac_res->addr); + + /* TX channels irq */ + ret = stmmac_pltfr_get_irq_array(pdev, "tx-queue-%d", + stmmac_res->tx_irq, + MTL_MAX_TX_QUEUES); + if (ret) + return ret; + + /* RX channels irq */ + ret = stmmac_pltfr_get_irq_array(pdev, "rx-queue-%d", + stmmac_res->rx_irq, + MTL_MAX_RX_QUEUES); + if (ret) + return ret; + + return 0; } EXPORT_SYMBOL_GPL(stmmac_get_platform_resources); diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_ptp.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_ptp.c index 3e30172fa129..960249960004 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_ptp.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_ptp.c @@ -334,12 +334,14 @@ const struct ptp_clock_info dwmac1000_ptp_clock_ops = { */ void stmmac_ptp_register(struct stmmac_priv *priv) { - int i; - - for (i = 0; i < priv->dma_cap.pps_out_num; i++) { - if (i >= STMMAC_PPS_MAX) - break; - priv->pps[i].available = true; + unsigned int pps_out_num = priv->dma_cap.pps_out_num; + unsigned int n_ext_ts; + + if (pps_out_num > STMMAC_PPS_MAX) { + dev_warn(priv->device, + "pps outputs (%u) exceeds driver maximum, limiting to %u\n", + pps_out_num, STMMAC_PPS_MAX); + pps_out_num = STMMAC_PPS_MAX; } /* Calculate the clock domain crossing (CDC) error if necessary */ @@ -350,11 +352,12 @@ void stmmac_ptp_register(struct stmmac_priv *priv) /* Update the ptp clock parameters based on feature discovery, when * available */ - if (priv->dma_cap.pps_out_num) - priv->ptp_clock_ops.n_per_out = priv->dma_cap.pps_out_num; + if (pps_out_num) + priv->ptp_clock_ops.n_per_out = pps_out_num; - if (priv->dma_cap.aux_snapshot_n) - priv->ptp_clock_ops.n_ext_ts = priv->dma_cap.aux_snapshot_n; + n_ext_ts = priv->dma_cap.aux_snapshot_n; + if (n_ext_ts) + priv->ptp_clock_ops.n_ext_ts = n_ext_ts; if (priv->plat->ptp_max_adj) priv->ptp_clock_ops.max_adj = priv->plat->ptp_max_adj; diff --git a/drivers/net/ethernet/ti/Makefile b/drivers/net/ethernet/ti/Makefile index 6da50f4b7c2e..f4276c9a7762 100644 --- a/drivers/net/ethernet/ti/Makefile +++ b/drivers/net/ethernet/ti/Makefile @@ -6,30 +6,30 @@ obj-$(CONFIG_TI_PRUETH) += icssm-prueth.o icssm-prueth-y := icssm/icssm_prueth.o icssm/icssm_prueth_switch.o icssm/icssm_switchdev.o -obj-$(CONFIG_TI_CPSW) += cpsw-common.o -obj-$(CONFIG_TI_DAVINCI_EMAC) += cpsw-common.o -obj-$(CONFIG_TI_CPSW_SWITCHDEV) += cpsw-common.o +ti-cpsw-common-y += cpsw-common.o davinci_cpdma.o +ti-cpsw-priv-y += cpsw_priv.o cpsw_ethtool.o +ti-cpsw-ale-y += cpsw_ale.o +ti-cpsw-sl-y += cpsw_sl.o obj-$(CONFIG_TLAN) += tlan.o -obj-$(CONFIG_TI_DAVINCI_EMAC) += ti_davinci_emac.o -ti_davinci_emac-y := davinci_emac.o davinci_cpdma.o +obj-$(CONFIG_TI_DAVINCI_EMAC) += davinci_emac.o ti-cpsw-common.o obj-$(CONFIG_TI_DAVINCI_MDIO) += davinci_mdio.o obj-$(CONFIG_TI_CPSW_PHY_SEL) += cpsw-phy-sel.o obj-$(CONFIG_TI_CPTS) += cpts.o -obj-$(CONFIG_TI_CPSW) += ti_cpsw.o -ti_cpsw-y := cpsw.o davinci_cpdma.o cpsw_ale.o cpsw_priv.o cpsw_sl.o cpsw_ethtool.o -obj-$(CONFIG_TI_CPSW_SWITCHDEV) += ti_cpsw_new.o -ti_cpsw_new-y := cpsw_switchdev.o cpsw_new.o davinci_cpdma.o cpsw_ale.o cpsw_sl.o cpsw_priv.o cpsw_ethtool.o +obj-$(CONFIG_TI_CPSW) += ti_cpsw.o ti-cpsw-common.o ti-cpsw-priv.o ti-cpsw-ale.o ti-cpsw-sl.o +ti_cpsw-y := cpsw.o +obj-$(CONFIG_TI_CPSW_SWITCHDEV) += ti_cpsw_new.o ti-cpsw-common.o ti-cpsw-priv.o ti-cpsw-ale.o ti-cpsw-sl.o +ti_cpsw_new-y := cpsw_switchdev.o cpsw_new.o -obj-$(CONFIG_TI_KEYSTONE_NETCP) += keystone_netcp.o -keystone_netcp-y := netcp_core.o cpsw_ale.o -obj-$(CONFIG_TI_KEYSTONE_NETCP_ETHSS) += keystone_netcp_ethss.o -keystone_netcp_ethss-y := netcp_ethss.o netcp_sgmii.o netcp_xgbepcsr.o cpsw_ale.o +obj-$(CONFIG_TI_KEYSTONE_NETCP) += keystone_netcp.o ti-cpsw-ale.o +keystone_netcp-y := netcp_core.o +obj-$(CONFIG_TI_KEYSTONE_NETCP_ETHSS) += keystone_netcp_ethss.o ti-cpsw-ale.o +keystone_netcp_ethss-y := netcp_ethss.o netcp_sgmii.o netcp_xgbepcsr.o obj-$(CONFIG_TI_K3_CPPI_DESC_POOL) += k3-cppi-desc-pool.o -obj-$(CONFIG_TI_K3_AM65_CPSW_NUSS) += ti-am65-cpsw-nuss.o -ti-am65-cpsw-nuss-y := am65-cpsw-nuss.o cpsw_sl.o am65-cpsw-ethtool.o cpsw_ale.o +obj-$(CONFIG_TI_K3_AM65_CPSW_NUSS) += ti-am65-cpsw-nuss.o ti-cpsw-sl.o ti-cpsw-ale.o +ti-am65-cpsw-nuss-y := am65-cpsw-nuss.o am65-cpsw-ethtool.o ti-am65-cpsw-nuss-$(CONFIG_TI_AM65_CPSW_QOS) += am65-cpsw-qos.o ti-am65-cpsw-nuss-$(CONFIG_TI_K3_AM65_CPSW_SWITCHDEV) += am65-cpsw-switchdev.o obj-$(CONFIG_TI_K3_AM65_CPTS) += am65-cpts.o diff --git a/drivers/net/ethernet/ti/am65-cpsw-nuss.c b/drivers/net/ethernet/ti/am65-cpsw-nuss.c index 265ce5479915..7ac75fc8cdcf 100644 --- a/drivers/net/ethernet/ti/am65-cpsw-nuss.c +++ b/drivers/net/ethernet/ti/am65-cpsw-nuss.c @@ -1352,7 +1352,7 @@ static int am65_cpsw_nuss_rx_packets(struct am65_cpsw_rx_flow *flow, am65_cpsw_nuss_set_offload_fwd_mark(skb, ndev_priv->offload_fwd_mark); skb_put(skb, pkt_len); if (port->rx_ts_filter) - am65_cpts_rx_timestamp(common->cpts, skb); + am65_cpts_rx_timestamp(common->cpts, port_id, skb); skb_mark_for_recycle(skb); skb->protocol = eth_type_trans(skb, ndev); am65_cpsw_nuss_rx_csum(skb, csum_info); @@ -1607,7 +1607,7 @@ static netdev_tx_t am65_cpsw_nuss_ndo_slave_xmit(struct sk_buff *skb, /* SKB TX timestamp */ if (port->tx_ts_enabled) - am65_cpts_prep_tx_timestamp(common->cpts, skb); + am65_cpts_prep_tx_timestamp(common->cpts, port->port_id, skb); q_idx = skb_get_queue_mapping(skb); dev_dbg(dev, "%s skb_queue:%d\n", __func__, q_idx); @@ -3468,6 +3468,13 @@ static const struct am65_cpsw_pdata am64x_cpswxg_pdata = { .fdqring_mode = K3_RINGACC_RING_MODE_RING, }; +static const struct am65_cpsw_pdata j722s_cpswxg_pdata = { + .quirks = AM64_CPSW_QUIRK_DMA_RX_TDOWN_IRQ, + .ale_dev_id = "am64-cpswxg", + .fdqring_mode = K3_RINGACC_RING_MODE_RING, + .extra_modes = BIT(PHY_INTERFACE_MODE_SGMII), +}; + static const struct am65_cpsw_pdata j7200_cpswxg_pdata = { .quirks = 0, .ale_dev_id = "am64-cpswxg", @@ -3495,6 +3502,7 @@ static const struct of_device_id am65_cpsw_nuss_of_mtable[] = { { .compatible = "ti,am654-cpsw-nuss", .data = &am65x_sr1_0}, { .compatible = "ti,j721e-cpsw-nuss", .data = &j721e_pdata}, { .compatible = "ti,am642-cpsw-nuss", .data = &am64x_cpswxg_pdata}, + { .compatible = "ti,j722s-cpsw-nuss", .data = &j722s_cpswxg_pdata}, { .compatible = "ti,j7200-cpswxg-nuss", .data = &j7200_cpswxg_pdata}, { .compatible = "ti,j721e-cpswxg-nuss", .data = &j721e_cpswxg_pdata}, { .compatible = "ti,j784s4-cpswxg-nuss", .data = &j784s4_cpswxg_pdata}, diff --git a/drivers/net/ethernet/ti/am65-cpts.c b/drivers/net/ethernet/ti/am65-cpts.c index 8ffbfaa3ab18..efbf2ef976a3 100644 --- a/drivers/net/ethernet/ti/am65-cpts.c +++ b/drivers/net/ethernet/ti/am65-cpts.c @@ -796,11 +796,7 @@ static bool am65_cpts_match_tx_ts(struct am65_cpts *cpts, bool found = false; u32 mtype_seqid; - mtype_seqid = event->event1 & - (AM65_CPTS_EVENT_1_MESSAGE_TYPE_MASK | - AM65_CPTS_EVENT_1_EVENT_TYPE_MASK | - AM65_CPTS_EVENT_1_SEQUENCE_ID_MASK); - + mtype_seqid = event->event1; __skb_queue_head_init(&txq_list); spin_lock_irqsave(&cpts->txq.lock, flags); @@ -922,7 +918,6 @@ static u64 am65_cpts_find_rx_ts(struct am65_cpts *cpts, u32 skb_mtype_seqid) struct list_head *this, *next; struct am65_cpts_event *event; unsigned long flags; - u32 mtype_seqid; u64 ns = 0; spin_lock_irqsave(&cpts->lock, flags); @@ -934,12 +929,7 @@ static u64 am65_cpts_find_rx_ts(struct am65_cpts *cpts, u32 skb_mtype_seqid) continue; } - mtype_seqid = event->event1 & - (AM65_CPTS_EVENT_1_MESSAGE_TYPE_MASK | - AM65_CPTS_EVENT_1_SEQUENCE_ID_MASK | - AM65_CPTS_EVENT_1_EVENT_TYPE_MASK); - - if (mtype_seqid == skb_mtype_seqid) { + if (event->event1 == skb_mtype_seqid) { ns = event->timestamp; list_move(&event->list, &cpts->pool); break; @@ -950,7 +940,8 @@ static u64 am65_cpts_find_rx_ts(struct am65_cpts *cpts, u32 skb_mtype_seqid) return ns; } -void am65_cpts_rx_timestamp(struct am65_cpts *cpts, struct sk_buff *skb) +void am65_cpts_rx_timestamp(struct am65_cpts *cpts, unsigned int port_id, + struct sk_buff *skb) { struct am65_cpts_skb_cb_data *skb_cb = (struct am65_cpts_skb_cb_data *)skb->cb; struct skb_shared_hwtstamps *ssh; @@ -966,6 +957,7 @@ void am65_cpts_rx_timestamp(struct am65_cpts *cpts, struct sk_buff *skb) if (!ret) return; /* if not PTP class packet */ + skb_cb->skb_mtype_seqid |= port_id << AM65_CPTS_EVENT_1_PORT_NUMBER_SHIFT; skb_cb->skb_mtype_seqid |= (AM65_CPTS_EV_RX << AM65_CPTS_EVENT_1_EVENT_TYPE_SHIFT); dev_dbg(cpts->dev, "%s mtype seqid %08x\n", __func__, skb_cb->skb_mtype_seqid); @@ -1009,13 +1001,15 @@ EXPORT_SYMBOL_GPL(am65_cpts_tx_timestamp); /** * am65_cpts_prep_tx_timestamp - check and prepare tx packet for timestamping * @cpts: cpts handle + * @port_id: The port on which the skb will be sent * @skb: packet * * This functions should be called from .xmit(). * It checks if packet can be timestamped, fills internal cpts data * in skb-cb and marks packet as SKBTX_IN_PROGRESS. */ -void am65_cpts_prep_tx_timestamp(struct am65_cpts *cpts, struct sk_buff *skb) +void am65_cpts_prep_tx_timestamp(struct am65_cpts *cpts, unsigned int port_id, + struct sk_buff *skb) { struct am65_cpts_skb_cb_data *skb_cb = (void *)skb->cb; int ret; @@ -1026,6 +1020,7 @@ void am65_cpts_prep_tx_timestamp(struct am65_cpts *cpts, struct sk_buff *skb) ret = am65_skb_get_mtype_seqid(skb, &skb_cb->skb_mtype_seqid); if (!ret) return; + skb_cb->skb_mtype_seqid |= port_id << AM65_CPTS_EVENT_1_PORT_NUMBER_SHIFT; skb_cb->skb_mtype_seqid |= (AM65_CPTS_EV_TX << AM65_CPTS_EVENT_1_EVENT_TYPE_SHIFT); diff --git a/drivers/net/ethernet/ti/am65-cpts.h b/drivers/net/ethernet/ti/am65-cpts.h index 6099d772799d..5fa77b0fc837 100644 --- a/drivers/net/ethernet/ti/am65-cpts.h +++ b/drivers/net/ethernet/ti/am65-cpts.h @@ -22,9 +22,11 @@ void am65_cpts_release(struct am65_cpts *cpts); struct am65_cpts *am65_cpts_create(struct device *dev, void __iomem *regs, struct device_node *node); int am65_cpts_phc_index(struct am65_cpts *cpts); -void am65_cpts_rx_timestamp(struct am65_cpts *cpts, struct sk_buff *skb); +void am65_cpts_rx_timestamp(struct am65_cpts *cpts, unsigned int port_id, + struct sk_buff *skb); void am65_cpts_tx_timestamp(struct am65_cpts *cpts, struct sk_buff *skb); -void am65_cpts_prep_tx_timestamp(struct am65_cpts *cpts, struct sk_buff *skb); +void am65_cpts_prep_tx_timestamp(struct am65_cpts *cpts, unsigned int port_id, + struct sk_buff *skb); u64 am65_cpts_ns_gettime(struct am65_cpts *cpts); int am65_cpts_estf_enable(struct am65_cpts *cpts, int idx, struct am65_cpts_estf_cfg *cfg); @@ -49,6 +51,7 @@ static inline int am65_cpts_phc_index(struct am65_cpts *cpts) } static inline void am65_cpts_rx_timestamp(struct am65_cpts *cpts, + unsigned int port_id, struct sk_buff *skb) { } @@ -59,6 +62,7 @@ static inline void am65_cpts_tx_timestamp(struct am65_cpts *cpts, } static inline void am65_cpts_prep_tx_timestamp(struct am65_cpts *cpts, + unsigned int port_id, struct sk_buff *skb) { } diff --git a/drivers/net/ethernet/ti/cpsw.c b/drivers/net/ethernet/ti/cpsw.c index b0e18bdc2c85..aa3531e844e8 100644 --- a/drivers/net/ethernet/ti/cpsw.c +++ b/drivers/net/ethernet/ti/cpsw.c @@ -706,7 +706,7 @@ static void cpsw_init_host_port(struct cpsw_priv *priv) struct cpsw_common *cpsw = priv->cpsw; /* soft reset the controller and initialize ale */ - soft_reset("cpsw", &cpsw->regs->soft_reset); + cpsw_soft_reset("cpsw", &cpsw->regs->soft_reset); cpsw_ale_start(cpsw->ale); /* switch to vlan aware mode */ diff --git a/drivers/net/ethernet/ti/cpsw_ale.c b/drivers/net/ethernet/ti/cpsw_ale.c index be7b69319221..e202bba49480 100644 --- a/drivers/net/ethernet/ti/cpsw_ale.c +++ b/drivers/net/ethernet/ti/cpsw_ale.c @@ -493,6 +493,7 @@ int cpsw_ale_flush_multicast(struct cpsw_ale *ale, int port_mask, int vid) } return 0; } +EXPORT_SYMBOL_GPL(cpsw_ale_flush_multicast); static inline void cpsw_ale_set_vlan_entry_type(u32 *ale_entry, int flags, u16 vid) @@ -530,6 +531,7 @@ int cpsw_ale_add_ucast(struct cpsw_ale *ale, const u8 *addr, int port, cpsw_ale_write(ale, idx, ale_entry); return 0; } +EXPORT_SYMBOL_GPL(cpsw_ale_add_ucast); int cpsw_ale_del_ucast(struct cpsw_ale *ale, const u8 *addr, int port, int flags, u16 vid) @@ -545,6 +547,7 @@ int cpsw_ale_del_ucast(struct cpsw_ale *ale, const u8 *addr, int port, cpsw_ale_write(ale, idx, ale_entry); return 0; } +EXPORT_SYMBOL_GPL(cpsw_ale_del_ucast); int cpsw_ale_add_mcast(struct cpsw_ale *ale, const u8 *addr, int port_mask, int flags, u16 vid, int mcast_state) @@ -578,6 +581,7 @@ int cpsw_ale_add_mcast(struct cpsw_ale *ale, const u8 *addr, int port_mask, cpsw_ale_write(ale, idx, ale_entry); return 0; } +EXPORT_SYMBOL_GPL(cpsw_ale_add_mcast); int cpsw_ale_del_mcast(struct cpsw_ale *ale, const u8 *addr, int port_mask, int flags, u16 vid) @@ -607,6 +611,7 @@ int cpsw_ale_del_mcast(struct cpsw_ale *ale, const u8 *addr, int port_mask, cpsw_ale_write(ale, idx, ale_entry); return 0; } +EXPORT_SYMBOL_GPL(cpsw_ale_del_mcast); /* ALE NetCP NU switch specific vlan functions */ static void cpsw_ale_set_vlan_mcast(struct cpsw_ale *ale, u32 *ale_entry, @@ -676,6 +681,7 @@ int cpsw_ale_add_vlan(struct cpsw_ale *ale, u16 vid, int port_mask, int untag, cpsw_ale_write(ale, idx, ale_entry); return 0; } +EXPORT_SYMBOL_GPL(cpsw_ale_add_vlan); static void cpsw_ale_vlan_del_modify_int(struct cpsw_ale *ale, u32 *ale_entry, u16 vid, int port_mask) @@ -733,6 +739,7 @@ int cpsw_ale_vlan_del_modify(struct cpsw_ale *ale, u16 vid, int port_mask) return 0; } +EXPORT_SYMBOL_GPL(cpsw_ale_vlan_del_modify); int cpsw_ale_del_vlan(struct cpsw_ale *ale, u16 vid, int port_mask) { @@ -767,6 +774,7 @@ int cpsw_ale_del_vlan(struct cpsw_ale *ale, u16 vid, int port_mask) return 0; } +EXPORT_SYMBOL_GPL(cpsw_ale_del_vlan); int cpsw_ale_vlan_add_modify(struct cpsw_ale *ale, u16 vid, int port_mask, int untag_mask, int reg_mask, int unreg_mask) @@ -806,6 +814,7 @@ int cpsw_ale_vlan_add_modify(struct cpsw_ale *ale, u16 vid, int port_mask, return ret; } +EXPORT_SYMBOL_GPL(cpsw_ale_vlan_add_modify); void cpsw_ale_set_unreg_mcast(struct cpsw_ale *ale, int unreg_mcast_mask, bool add) @@ -833,6 +842,7 @@ void cpsw_ale_set_unreg_mcast(struct cpsw_ale *ale, int unreg_mcast_mask, cpsw_ale_write(ale, idx, ale_entry); } } +EXPORT_SYMBOL_GPL(cpsw_ale_set_unreg_mcast); static void cpsw_ale_vlan_set_unreg_mcast(struct cpsw_ale *ale, u32 *ale_entry, int allmulti) @@ -898,6 +908,7 @@ void cpsw_ale_set_allmulti(struct cpsw_ale *ale, int allmulti, int port) cpsw_ale_write(ale, idx, ale_entry); } } +EXPORT_SYMBOL_GPL(cpsw_ale_set_allmulti); struct ale_control_info { const char *name; @@ -1155,6 +1166,7 @@ int cpsw_ale_control_set(struct cpsw_ale *ale, int port, int control, return 0; } +EXPORT_SYMBOL_GPL(cpsw_ale_control_set); int cpsw_ale_control_get(struct cpsw_ale *ale, int port, int control) { @@ -1178,6 +1190,7 @@ int cpsw_ale_control_get(struct cpsw_ale *ale, int port, int control) tmp = readl_relaxed(ale->params.ale_regs + offset) >> shift; return tmp & BITMASK(info->bits); } +EXPORT_SYMBOL_GPL(cpsw_ale_control_get); int cpsw_ale_rx_ratelimit_mc(struct cpsw_ale *ale, int port, unsigned int ratelimit_pps) @@ -1200,6 +1213,7 @@ int cpsw_ale_rx_ratelimit_mc(struct cpsw_ale *ale, int port, unsigned int rateli port, val * ALE_RATE_LIMIT_MIN_PPS); return 0; } +EXPORT_SYMBOL_GPL(cpsw_ale_rx_ratelimit_mc); int cpsw_ale_rx_ratelimit_bc(struct cpsw_ale *ale, int port, unsigned int ratelimit_pps) @@ -1222,6 +1236,7 @@ int cpsw_ale_rx_ratelimit_bc(struct cpsw_ale *ale, int port, unsigned int rateli port, val * ALE_RATE_LIMIT_MIN_PPS); return 0; } +EXPORT_SYMBOL_GPL(cpsw_ale_rx_ratelimit_bc); static void cpsw_ale_timer(struct timer_list *t) { @@ -1311,6 +1326,7 @@ void cpsw_ale_start(struct cpsw_ale *ale) cpsw_ale_aging_start(ale); } +EXPORT_SYMBOL_GPL(cpsw_ale_start); void cpsw_ale_stop(struct cpsw_ale *ale) { @@ -1318,6 +1334,7 @@ void cpsw_ale_stop(struct cpsw_ale *ale) cpsw_ale_control_set(ale, 0, ALE_CLEAR, 1); cpsw_ale_control_set(ale, 0, ALE_ENABLE, 0); } +EXPORT_SYMBOL_GPL(cpsw_ale_stop); static const struct reg_field ale_fields_cpsw[] = { /* CPSW_ALE_IDVER_REG */ @@ -1618,6 +1635,7 @@ struct cpsw_ale *cpsw_ale_create(struct cpsw_ale_params *params) cpsw_ale_control_set(ale, 0, ALE_CLEAR, 1); return ale; } +EXPORT_SYMBOL_GPL(cpsw_ale_create); void cpsw_ale_dump(struct cpsw_ale *ale, u32 *data) { @@ -1628,6 +1646,7 @@ void cpsw_ale_dump(struct cpsw_ale *ale, u32 *data) data += ALE_ENTRY_WORDS; } } +EXPORT_SYMBOL_GPL(cpsw_ale_dump); void cpsw_ale_restore(struct cpsw_ale *ale, u32 *data) { @@ -1638,11 +1657,13 @@ void cpsw_ale_restore(struct cpsw_ale *ale, u32 *data) data += ALE_ENTRY_WORDS; } } +EXPORT_SYMBOL_GPL(cpsw_ale_restore); u32 cpsw_ale_get_num_entries(struct cpsw_ale *ale) { return ale ? ale->params.ale_entries : 0; } +EXPORT_SYMBOL_GPL(cpsw_ale_get_num_entries); /* Reads the specified policer index into ALE POLICER registers */ static void cpsw_ale_policer_read_idx(struct cpsw_ale *ale, u32 idx) @@ -1745,3 +1766,7 @@ void cpsw_ale_classifier_setup_default(struct cpsw_ale *ale, int num_rx_ch) 1); } } +EXPORT_SYMBOL_GPL(cpsw_ale_classifier_setup_default); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("TI N-Port Ethernet Switch Address Lookup Engine"); diff --git a/drivers/net/ethernet/ti/cpsw_ethtool.c b/drivers/net/ethernet/ti/cpsw_ethtool.c index a43f75ee269e..3f2682c461f9 100644 --- a/drivers/net/ethernet/ti/cpsw_ethtool.c +++ b/drivers/net/ethernet/ti/cpsw_ethtool.c @@ -144,6 +144,7 @@ u32 cpsw_get_msglevel(struct net_device *ndev) return priv->msg_enable; } +EXPORT_SYMBOL_GPL(cpsw_get_msglevel); void cpsw_set_msglevel(struct net_device *ndev, u32 value) { @@ -151,6 +152,7 @@ void cpsw_set_msglevel(struct net_device *ndev, u32 value) priv->msg_enable = value; } +EXPORT_SYMBOL_GPL(cpsw_set_msglevel); int cpsw_get_coalesce(struct net_device *ndev, struct ethtool_coalesce *coal, struct kernel_ethtool_coalesce *kernel_coal, @@ -161,6 +163,7 @@ int cpsw_get_coalesce(struct net_device *ndev, struct ethtool_coalesce *coal, coal->rx_coalesce_usecs = cpsw->coal_intvl; return 0; } +EXPORT_SYMBOL_GPL(cpsw_get_coalesce); int cpsw_set_coalesce(struct net_device *ndev, struct ethtool_coalesce *coal, struct kernel_ethtool_coalesce *kernel_coal, @@ -220,6 +223,7 @@ update_return: return 0; } +EXPORT_SYMBOL_GPL(cpsw_set_coalesce); int cpsw_get_sset_count(struct net_device *ndev, int sset) { @@ -234,6 +238,7 @@ int cpsw_get_sset_count(struct net_device *ndev, int sset) return -EOPNOTSUPP; } } +EXPORT_SYMBOL_GPL(cpsw_get_sset_count); static void cpsw_add_ch_strings(u8 **p, int ch_num, int rx_dir) { @@ -271,6 +276,7 @@ void cpsw_get_strings(struct net_device *ndev, u32 stringset, u8 *data) break; } } +EXPORT_SYMBOL_GPL(cpsw_get_strings); void cpsw_get_ethtool_stats(struct net_device *ndev, struct ethtool_stats *stats, u64 *data) @@ -303,6 +309,7 @@ void cpsw_get_ethtool_stats(struct net_device *ndev, } } } +EXPORT_SYMBOL_GPL(cpsw_get_ethtool_stats); void cpsw_get_pauseparam(struct net_device *ndev, struct ethtool_pauseparam *pause) @@ -313,6 +320,7 @@ void cpsw_get_pauseparam(struct net_device *ndev, pause->rx_pause = priv->rx_pause ? true : false; pause->tx_pause = priv->tx_pause ? true : false; } +EXPORT_SYMBOL_GPL(cpsw_get_pauseparam); void cpsw_get_wol(struct net_device *ndev, struct ethtool_wolinfo *wol) { @@ -326,6 +334,7 @@ void cpsw_get_wol(struct net_device *ndev, struct ethtool_wolinfo *wol) if (cpsw->slaves[slave_no].phy) phy_ethtool_get_wol(cpsw->slaves[slave_no].phy, wol); } +EXPORT_SYMBOL_GPL(cpsw_get_wol); int cpsw_set_wol(struct net_device *ndev, struct ethtool_wolinfo *wol) { @@ -338,6 +347,7 @@ int cpsw_set_wol(struct net_device *ndev, struct ethtool_wolinfo *wol) else return -EOPNOTSUPP; } +EXPORT_SYMBOL_GPL(cpsw_set_wol); int cpsw_get_regs_len(struct net_device *ndev) { @@ -346,6 +356,7 @@ int cpsw_get_regs_len(struct net_device *ndev) return cpsw_ale_get_num_entries(cpsw->ale) * ALE_ENTRY_WORDS * sizeof(u32); } +EXPORT_SYMBOL_GPL(cpsw_get_regs_len); void cpsw_get_regs(struct net_device *ndev, struct ethtool_regs *regs, void *p) { @@ -357,6 +368,7 @@ void cpsw_get_regs(struct net_device *ndev, struct ethtool_regs *regs, void *p) cpsw_ale_dump(cpsw->ale, reg); } +EXPORT_SYMBOL_GPL(cpsw_get_regs); int cpsw_ethtool_op_begin(struct net_device *ndev) { @@ -370,6 +382,7 @@ int cpsw_ethtool_op_begin(struct net_device *ndev) return ret; } +EXPORT_SYMBOL_GPL(cpsw_ethtool_op_begin); void cpsw_ethtool_op_complete(struct net_device *ndev) { @@ -377,6 +390,7 @@ void cpsw_ethtool_op_complete(struct net_device *ndev) pm_runtime_put(priv->cpsw->dev); } +EXPORT_SYMBOL_GPL(cpsw_ethtool_op_complete); void cpsw_get_channels(struct net_device *ndev, struct ethtool_channels *ch) { @@ -391,6 +405,7 @@ void cpsw_get_channels(struct net_device *ndev, struct ethtool_channels *ch) ch->tx_count = cpsw->tx_ch_num; ch->combined_count = 0; } +EXPORT_SYMBOL_GPL(cpsw_get_channels); int cpsw_get_link_ksettings(struct net_device *ndev, struct ethtool_link_ksettings *ecmd) @@ -405,6 +420,7 @@ int cpsw_get_link_ksettings(struct net_device *ndev, phy_ethtool_ksettings_get(cpsw->slaves[slave_no].phy, ecmd); return 0; } +EXPORT_SYMBOL_GPL(cpsw_get_link_ksettings); int cpsw_set_link_ksettings(struct net_device *ndev, const struct ethtool_link_ksettings *ecmd) @@ -418,6 +434,7 @@ int cpsw_set_link_ksettings(struct net_device *ndev, return phy_ethtool_ksettings_set(cpsw->slaves[slave_no].phy, ecmd); } +EXPORT_SYMBOL_GPL(cpsw_set_link_ksettings); int cpsw_get_eee(struct net_device *ndev, struct ethtool_keee *edata) { @@ -430,6 +447,7 @@ int cpsw_get_eee(struct net_device *ndev, struct ethtool_keee *edata) else return -EOPNOTSUPP; } +EXPORT_SYMBOL_GPL(cpsw_get_eee); int cpsw_nway_reset(struct net_device *ndev) { @@ -442,6 +460,7 @@ int cpsw_nway_reset(struct net_device *ndev) else return -EOPNOTSUPP; } +EXPORT_SYMBOL_GPL(cpsw_nway_reset); static void cpsw_suspend_data_pass(struct net_device *ndev) { @@ -639,6 +658,7 @@ err: cpsw_fail(cpsw); return ret; } +EXPORT_SYMBOL_GPL(cpsw_set_channels_common); void cpsw_get_ringparam(struct net_device *ndev, struct ethtool_ringparam *ering, @@ -654,6 +674,7 @@ void cpsw_get_ringparam(struct net_device *ndev, ering->rx_max_pending = cpsw->descs_pool_size - CPSW_MAX_QUEUES; ering->rx_pending = cpdma_get_num_rx_descs(cpsw->dma); } +EXPORT_SYMBOL_GPL(cpsw_get_ringparam); int cpsw_set_ringparam(struct net_device *ndev, struct ethtool_ringparam *ering, @@ -700,6 +721,7 @@ err: cpsw_fail(cpsw); return ret; } +EXPORT_SYMBOL_GPL(cpsw_set_ringparam); #if IS_ENABLED(CONFIG_TI_CPTS) int cpsw_get_ts_info(struct net_device *ndev, struct kernel_ethtool_ts_info *info) @@ -720,6 +742,7 @@ int cpsw_get_ts_info(struct net_device *ndev, struct kernel_ethtool_ts_info *inf (1 << HWTSTAMP_FILTER_PTP_V2_EVENT); return 0; } +EXPORT_SYMBOL_GPL(cpsw_get_ts_info); #else int cpsw_get_ts_info(struct net_device *ndev, struct kernel_ethtool_ts_info *info) { @@ -729,4 +752,5 @@ int cpsw_get_ts_info(struct net_device *ndev, struct kernel_ethtool_ts_info *inf info->rx_filters = 0; return 0; } +EXPORT_SYMBOL_GPL(cpsw_get_ts_info); #endif diff --git a/drivers/net/ethernet/ti/cpsw_new.c b/drivers/net/ethernet/ti/cpsw_new.c index 7f42f58a4b03..c5be359f3c66 100644 --- a/drivers/net/ethernet/ti/cpsw_new.c +++ b/drivers/net/ethernet/ti/cpsw_new.c @@ -573,7 +573,7 @@ static void cpsw_init_host_port(struct cpsw_priv *priv) u32 control_reg; /* soft reset the controller and initialize ale */ - soft_reset("cpsw", &cpsw->regs->soft_reset); + cpsw_soft_reset("cpsw", &cpsw->regs->soft_reset); cpsw_ale_start(cpsw->ale); /* switch to vlan aware mode */ diff --git a/drivers/net/ethernet/ti/cpsw_priv.c b/drivers/net/ethernet/ti/cpsw_priv.c index bc4fdf17a99e..1f6f374551cb 100644 --- a/drivers/net/ethernet/ti/cpsw_priv.c +++ b/drivers/net/ethernet/ti/cpsw_priv.c @@ -32,6 +32,7 @@ #define CPTS_N_ETX_TS 4 int (*cpsw_slave_index)(struct cpsw_common *cpsw, struct cpsw_priv *priv); +EXPORT_SYMBOL_GPL(cpsw_slave_index); void cpsw_intr_enable(struct cpsw_common *cpsw) { @@ -40,6 +41,7 @@ void cpsw_intr_enable(struct cpsw_common *cpsw) cpdma_ctlr_int_ctrl(cpsw->dma, true); } +EXPORT_SYMBOL_GPL(cpsw_intr_enable); void cpsw_intr_disable(struct cpsw_common *cpsw) { @@ -48,6 +50,7 @@ void cpsw_intr_disable(struct cpsw_common *cpsw) cpdma_ctlr_int_ctrl(cpsw->dma, false); } +EXPORT_SYMBOL_GPL(cpsw_intr_disable); void cpsw_tx_handler(void *token, int len, int status) { @@ -82,6 +85,7 @@ void cpsw_tx_handler(void *token, int len, int status) ndev->stats.tx_packets++; ndev->stats.tx_bytes += len; } +EXPORT_SYMBOL_GPL(cpsw_tx_handler); irqreturn_t cpsw_tx_interrupt(int irq, void *dev_id) { @@ -98,6 +102,7 @@ irqreturn_t cpsw_tx_interrupt(int irq, void *dev_id) napi_schedule(&cpsw->napi_tx); return IRQ_HANDLED; } +EXPORT_SYMBOL_GPL(cpsw_tx_interrupt); irqreturn_t cpsw_rx_interrupt(int irq, void *dev_id) { @@ -114,6 +119,7 @@ irqreturn_t cpsw_rx_interrupt(int irq, void *dev_id) napi_schedule(&cpsw->napi_rx); return IRQ_HANDLED; } +EXPORT_SYMBOL_GPL(cpsw_rx_interrupt); irqreturn_t cpsw_misc_interrupt(int irq, void *dev_id) { @@ -126,6 +132,7 @@ irqreturn_t cpsw_misc_interrupt(int irq, void *dev_id) return IRQ_HANDLED; } +EXPORT_SYMBOL_GPL(cpsw_misc_interrupt); int cpsw_tx_mq_poll(struct napi_struct *napi_tx, int budget) { @@ -158,6 +165,7 @@ int cpsw_tx_mq_poll(struct napi_struct *napi_tx, int budget) return num_tx; } +EXPORT_SYMBOL_GPL(cpsw_tx_mq_poll); int cpsw_tx_poll(struct napi_struct *napi_tx, int budget) { @@ -176,6 +184,7 @@ int cpsw_tx_poll(struct napi_struct *napi_tx, int budget) return num_tx; } +EXPORT_SYMBOL_GPL(cpsw_tx_poll); int cpsw_rx_mq_poll(struct napi_struct *napi_rx, int budget) { @@ -208,6 +217,7 @@ int cpsw_rx_mq_poll(struct napi_struct *napi_rx, int budget) return num_rx; } +EXPORT_SYMBOL_GPL(cpsw_rx_mq_poll); int cpsw_rx_poll(struct napi_struct *napi_rx, int budget) { @@ -226,6 +236,7 @@ int cpsw_rx_poll(struct napi_struct *napi_rx, int budget) return num_rx; } +EXPORT_SYMBOL_GPL(cpsw_rx_poll); void cpsw_rx_vlan_encap(struct sk_buff *skb) { @@ -268,14 +279,16 @@ void cpsw_rx_vlan_encap(struct sk_buff *skb) skb_pull(skb, VLAN_HLEN); } } +EXPORT_SYMBOL_GPL(cpsw_rx_vlan_encap); void cpsw_set_slave_mac(struct cpsw_slave *slave, struct cpsw_priv *priv) { slave_write(slave, mac_hi(priv->mac_addr), SA_HI); slave_write(slave, mac_lo(priv->mac_addr), SA_LO); } +EXPORT_SYMBOL_GPL(cpsw_set_slave_mac); -void soft_reset(const char *module, void __iomem *reg) +void cpsw_soft_reset(const char *module, void __iomem *reg) { unsigned long timeout = jiffies + HZ; @@ -286,6 +299,7 @@ void soft_reset(const char *module, void __iomem *reg) WARN(readl_relaxed(reg) & 1, "failed to soft-reset %s\n", module); } +EXPORT_SYMBOL_GPL(cpsw_soft_reset); void cpsw_ndo_tx_timeout(struct net_device *ndev, unsigned int txqueue) { @@ -305,6 +319,7 @@ void cpsw_ndo_tx_timeout(struct net_device *ndev, unsigned int txqueue) netif_trans_update(ndev); netif_tx_wake_all_queues(ndev); } +EXPORT_SYMBOL_GPL(cpsw_ndo_tx_timeout); static int cpsw_get_common_speed(struct cpsw_common *cpsw) { @@ -343,6 +358,7 @@ int cpsw_need_resplit(struct cpsw_common *cpsw) return 1; } +EXPORT_SYMBOL_GPL(cpsw_need_resplit); void cpsw_split_res(struct cpsw_common *cpsw) { @@ -428,6 +444,7 @@ void cpsw_split_res(struct cpsw_common *cpsw) if (budget) cpsw->rxv[0].budget += budget; } +EXPORT_SYMBOL_GPL(cpsw_split_res); int cpsw_init_common(struct cpsw_common *cpsw, void __iomem *ss_regs, int ale_ageout, phys_addr_t desc_mem_phys, @@ -548,6 +565,7 @@ int cpsw_init_common(struct cpsw_common *cpsw, void __iomem *ss_regs, return ret; } +EXPORT_SYMBOL_GPL(cpsw_init_common); #if IS_ENABLED(CONFIG_TI_CPTS) @@ -678,6 +696,7 @@ int cpsw_hwtstamp_set(struct net_device *dev, return 0; } +EXPORT_SYMBOL_GPL(cpsw_hwtstamp_set); int cpsw_hwtstamp_get(struct net_device *dev, struct kernel_hwtstamp_config *cfg) @@ -695,12 +714,14 @@ int cpsw_hwtstamp_get(struct net_device *dev, return 0; } +EXPORT_SYMBOL_GPL(cpsw_hwtstamp_get); #else int cpsw_hwtstamp_get(struct net_device *dev, struct kernel_hwtstamp_config *cfg) { return -EOPNOTSUPP; } +EXPORT_SYMBOL_GPL(cpsw_hwtstamp_set); int cpsw_hwtstamp_set(struct net_device *dev, struct kernel_hwtstamp_config *cfg, @@ -708,6 +729,7 @@ int cpsw_hwtstamp_set(struct net_device *dev, { return -EOPNOTSUPP; } +EXPORT_SYMBOL_GPL(cpsw_hwtstamp_get); #endif /*CONFIG_TI_CPTS*/ int cpsw_ndo_set_tx_maxrate(struct net_device *ndev, int queue, u32 rate) @@ -758,6 +780,7 @@ int cpsw_ndo_set_tx_maxrate(struct net_device *ndev, int queue, u32 rate) cpsw_split_res(cpsw); return ret; } +EXPORT_SYMBOL_GPL(cpsw_ndo_set_tx_maxrate); static int cpsw_tc_to_fifo(int tc, int num_tc) { @@ -782,6 +805,7 @@ bool cpsw_shp_is_off(struct cpsw_priv *priv) return !val; } +EXPORT_SYMBOL_GPL(cpsw_shp_is_off); static void cpsw_fifo_shp_on(struct cpsw_priv *priv, int fifo, int on) { @@ -1043,6 +1067,7 @@ int cpsw_ndo_setup_tc(struct net_device *ndev, enum tc_setup_type type, return -EOPNOTSUPP; } } +EXPORT_SYMBOL_GPL(cpsw_ndo_setup_tc); void cpsw_cbs_resume(struct cpsw_slave *slave, struct cpsw_priv *priv) { @@ -1056,6 +1081,7 @@ void cpsw_cbs_resume(struct cpsw_slave *slave, struct cpsw_priv *priv) cpsw_set_fifo_rlimit(priv, fifo, bw); } } +EXPORT_SYMBOL_GPL(cpsw_cbs_resume); void cpsw_mqprio_resume(struct cpsw_slave *slave, struct cpsw_priv *priv) { @@ -1078,6 +1104,7 @@ void cpsw_mqprio_resume(struct cpsw_slave *slave, struct cpsw_priv *priv) slave_write(slave, tx_prio_map, tx_prio_rg); } +EXPORT_SYMBOL_GPL(cpsw_mqprio_resume); int cpsw_fill_rx_channels(struct cpsw_priv *priv) { @@ -1123,6 +1150,7 @@ int cpsw_fill_rx_channels(struct cpsw_priv *priv) return 0; } +EXPORT_SYMBOL_GPL(cpsw_fill_rx_channels); static struct page_pool *cpsw_create_page_pool(struct cpsw_common *cpsw, int size) @@ -1208,6 +1236,7 @@ void cpsw_destroy_xdp_rxqs(struct cpsw_common *cpsw) cpsw->page_pool[ch] = NULL; } } +EXPORT_SYMBOL_GPL(cpsw_destroy_xdp_rxqs); int cpsw_create_xdp_rxqs(struct cpsw_common *cpsw) { @@ -1240,6 +1269,7 @@ err_cleanup: return ret; } +EXPORT_SYMBOL_GPL(cpsw_create_xdp_rxqs); static int cpsw_xdp_prog_setup(struct cpsw_priv *priv, struct netdev_bpf *bpf) { @@ -1267,6 +1297,7 @@ int cpsw_ndo_bpf(struct net_device *ndev, struct netdev_bpf *bpf) return -EINVAL; } } +EXPORT_SYMBOL_GPL(cpsw_ndo_bpf); int cpsw_xdp_tx_frame(struct cpsw_priv *priv, struct xdp_frame *xdpf, struct page *page, int port) @@ -1300,6 +1331,7 @@ int cpsw_xdp_tx_frame(struct cpsw_priv *priv, struct xdp_frame *xdpf, return ret; } +EXPORT_SYMBOL_GPL(cpsw_xdp_tx_frame); int cpsw_run_xdp(struct cpsw_priv *priv, int ch, struct xdp_buff *xdp, struct page *page, int port, int *len) @@ -1362,6 +1394,7 @@ drop: page_pool_recycle_direct(cpsw->page_pool[ch], page); return ret; } +EXPORT_SYMBOL_GPL(cpsw_run_xdp); static int cpsw_qos_clsflower_add_policer(struct cpsw_priv *priv, struct netlink_ext_ack *extack, @@ -1564,3 +1597,7 @@ void cpsw_qos_clsflower_resume(struct cpsw_priv *priv) cpsw_ale_rx_ratelimit_mc(priv->cpsw->ale, port_id, priv->ale_mc_ratelimit.rate_packet_ps); } +EXPORT_SYMBOL_GPL(cpsw_qos_clsflower_resume); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("TI CPSW Ethernet Switch Driver"); diff --git a/drivers/net/ethernet/ti/cpsw_priv.h b/drivers/net/ethernet/ti/cpsw_priv.h index acb6181c5c9e..fddd7a79f4b0 100644 --- a/drivers/net/ethernet/ti/cpsw_priv.h +++ b/drivers/net/ethernet/ti/cpsw_priv.h @@ -458,7 +458,7 @@ int cpsw_tx_poll(struct napi_struct *napi_tx, int budget); int cpsw_rx_mq_poll(struct napi_struct *napi_rx, int budget); int cpsw_rx_poll(struct napi_struct *napi_rx, int budget); void cpsw_rx_vlan_encap(struct sk_buff *skb); -void soft_reset(const char *module, void __iomem *reg); +void cpsw_soft_reset(const char *module, void __iomem *reg); void cpsw_set_slave_mac(struct cpsw_slave *slave, struct cpsw_priv *priv); void cpsw_ndo_tx_timeout(struct net_device *ndev, unsigned int txqueue); int cpsw_need_resplit(struct cpsw_common *cpsw); diff --git a/drivers/net/ethernet/ti/cpsw_sl.c b/drivers/net/ethernet/ti/cpsw_sl.c index 0c7531cb0f39..761719a348fa 100644 --- a/drivers/net/ethernet/ti/cpsw_sl.c +++ b/drivers/net/ethernet/ti/cpsw_sl.c @@ -200,6 +200,7 @@ u32 cpsw_sl_reg_read(struct cpsw_sl *sl, enum cpsw_sl_regs reg) dev_dbg(sl->dev, "cpsw_sl: reg: %04X r 0x%08X\n", sl->regs[reg], val); return val; } +EXPORT_SYMBOL_GPL(cpsw_sl_reg_read); void cpsw_sl_reg_write(struct cpsw_sl *sl, enum cpsw_sl_regs reg, u32 val) { @@ -212,6 +213,7 @@ void cpsw_sl_reg_write(struct cpsw_sl *sl, enum cpsw_sl_regs reg, u32 val) dev_dbg(sl->dev, "cpsw_sl: reg: %04X w 0x%08X\n", sl->regs[reg], val); writel(val, sl->sl_base + sl->regs[reg]); } +EXPORT_SYMBOL_GPL(cpsw_sl_reg_write); static const struct cpsw_sl_dev_id *cpsw_sl_match_id( const struct cpsw_sl_dev_id *id, @@ -252,6 +254,7 @@ struct cpsw_sl *cpsw_sl_get(const char *device_id, struct device *dev, return sl; } +EXPORT_SYMBOL_GPL(cpsw_sl_get); void cpsw_sl_reset(struct cpsw_sl *sl, unsigned long tmo) { @@ -270,6 +273,7 @@ void cpsw_sl_reset(struct cpsw_sl *sl, unsigned long tmo) if (cpsw_sl_reg_read(sl, CPSW_SL_SOFT_RESET) & CPSW_SL_SOFT_RESET_BIT) dev_err(sl->dev, "cpsw_sl failed to soft-reset.\n"); } +EXPORT_SYMBOL_GPL(cpsw_sl_reset); u32 cpsw_sl_ctl_set(struct cpsw_sl *sl, u32 ctl_funcs) { @@ -287,6 +291,7 @@ u32 cpsw_sl_ctl_set(struct cpsw_sl *sl, u32 ctl_funcs) return 0; } +EXPORT_SYMBOL_GPL(cpsw_sl_ctl_set); u32 cpsw_sl_ctl_clr(struct cpsw_sl *sl, u32 ctl_funcs) { @@ -304,11 +309,13 @@ u32 cpsw_sl_ctl_clr(struct cpsw_sl *sl, u32 ctl_funcs) return 0; } +EXPORT_SYMBOL_GPL(cpsw_sl_ctl_clr); void cpsw_sl_ctl_reset(struct cpsw_sl *sl) { cpsw_sl_reg_write(sl, CPSW_SL_MACCONTROL, 0); } +EXPORT_SYMBOL_GPL(cpsw_sl_ctl_reset); int cpsw_sl_wait_for_idle(struct cpsw_sl *sl, unsigned long tmo) { @@ -326,3 +333,7 @@ int cpsw_sl_wait_for_idle(struct cpsw_sl *sl, unsigned long tmo) return 0; } +EXPORT_SYMBOL_GPL(cpsw_sl_wait_for_idle); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("TI Ethernet Switch media-access-controller (MAC) submodule"); diff --git a/drivers/net/ethernet/ti/davinci_cpdma.c b/drivers/net/ethernet/ti/davinci_cpdma.c index d2eab5cd1e0c..41e89a19be53 100644 --- a/drivers/net/ethernet/ti/davinci_cpdma.c +++ b/drivers/net/ethernet/ti/davinci_cpdma.c @@ -531,6 +531,7 @@ struct cpdma_ctlr *cpdma_ctlr_create(struct cpdma_params *params) ctlr->num_chan = CPDMA_MAX_CHANNELS; return ctlr; } +EXPORT_SYMBOL_GPL(cpdma_ctlr_create); int cpdma_ctlr_start(struct cpdma_ctlr *ctlr) { @@ -591,6 +592,7 @@ int cpdma_ctlr_start(struct cpdma_ctlr *ctlr) spin_unlock_irqrestore(&ctlr->lock, flags); return 0; } +EXPORT_SYMBOL_GPL(cpdma_ctlr_start); int cpdma_ctlr_stop(struct cpdma_ctlr *ctlr) { @@ -623,6 +625,7 @@ int cpdma_ctlr_stop(struct cpdma_ctlr *ctlr) spin_unlock_irqrestore(&ctlr->lock, flags); return 0; } +EXPORT_SYMBOL_GPL(cpdma_ctlr_stop); int cpdma_ctlr_destroy(struct cpdma_ctlr *ctlr) { @@ -640,6 +643,7 @@ int cpdma_ctlr_destroy(struct cpdma_ctlr *ctlr) cpdma_desc_pool_destroy(ctlr); return ret; } +EXPORT_SYMBOL_GPL(cpdma_ctlr_destroy); int cpdma_ctlr_int_ctrl(struct cpdma_ctlr *ctlr, bool enable) { @@ -660,21 +664,25 @@ int cpdma_ctlr_int_ctrl(struct cpdma_ctlr *ctlr, bool enable) spin_unlock_irqrestore(&ctlr->lock, flags); return 0; } +EXPORT_SYMBOL_GPL(cpdma_ctlr_int_ctrl); void cpdma_ctlr_eoi(struct cpdma_ctlr *ctlr, u32 value) { dma_reg_write(ctlr, CPDMA_MACEOIVECTOR, value); } +EXPORT_SYMBOL_GPL(cpdma_ctlr_eoi); u32 cpdma_ctrl_rxchs_state(struct cpdma_ctlr *ctlr) { return dma_reg_read(ctlr, CPDMA_RXINTSTATMASKED); } +EXPORT_SYMBOL_GPL(cpdma_ctrl_rxchs_state); u32 cpdma_ctrl_txchs_state(struct cpdma_ctlr *ctlr) { return dma_reg_read(ctlr, CPDMA_TXINTSTATMASKED); } +EXPORT_SYMBOL_GPL(cpdma_ctrl_txchs_state); static void cpdma_chan_set_descs(struct cpdma_ctlr *ctlr, int rx, int desc_num, @@ -802,6 +810,7 @@ int cpdma_chan_set_weight(struct cpdma_chan *ch, int weight) spin_unlock_irqrestore(&ctlr->lock, flags); return ret; } +EXPORT_SYMBOL_GPL(cpdma_chan_set_weight); /* cpdma_chan_get_min_rate - get minimum allowed rate for channel * Should be called before cpdma_chan_set_rate. @@ -816,6 +825,7 @@ u32 cpdma_chan_get_min_rate(struct cpdma_ctlr *ctlr) return DIV_ROUND_UP(divident, divisor); } +EXPORT_SYMBOL_GPL(cpdma_chan_get_min_rate); /* cpdma_chan_set_rate - limits bandwidth for transmit channel. * The bandwidth * limited channels have to be in order beginning from lowest. @@ -860,6 +870,7 @@ err: spin_unlock_irqrestore(&ctlr->lock, flags); return ret; } +EXPORT_SYMBOL_GPL(cpdma_chan_set_rate); u32 cpdma_chan_get_rate(struct cpdma_chan *ch) { @@ -872,6 +883,7 @@ u32 cpdma_chan_get_rate(struct cpdma_chan *ch) return rate; } +EXPORT_SYMBOL_GPL(cpdma_chan_get_rate); struct cpdma_chan *cpdma_chan_create(struct cpdma_ctlr *ctlr, int chan_num, cpdma_handler_fn handler, int rx_type) @@ -931,6 +943,7 @@ struct cpdma_chan *cpdma_chan_create(struct cpdma_ctlr *ctlr, int chan_num, spin_unlock_irqrestore(&ctlr->lock, flags); return chan; } +EXPORT_SYMBOL_GPL(cpdma_chan_create); int cpdma_chan_get_rx_buf_num(struct cpdma_chan *chan) { @@ -943,6 +956,7 @@ int cpdma_chan_get_rx_buf_num(struct cpdma_chan *chan) return desc_num; } +EXPORT_SYMBOL_GPL(cpdma_chan_get_rx_buf_num); int cpdma_chan_destroy(struct cpdma_chan *chan) { @@ -964,6 +978,7 @@ int cpdma_chan_destroy(struct cpdma_chan *chan) spin_unlock_irqrestore(&ctlr->lock, flags); return 0; } +EXPORT_SYMBOL_GPL(cpdma_chan_destroy); int cpdma_chan_get_stats(struct cpdma_chan *chan, struct cpdma_chan_stats *stats) @@ -976,6 +991,7 @@ int cpdma_chan_get_stats(struct cpdma_chan *chan, spin_unlock_irqrestore(&chan->lock, flags); return 0; } +EXPORT_SYMBOL_GPL(cpdma_chan_get_stats); static void __cpdma_chan_submit(struct cpdma_chan *chan, struct cpdma_desc __iomem *desc) @@ -1100,6 +1116,7 @@ int cpdma_chan_idle_submit(struct cpdma_chan *chan, void *token, void *data, spin_unlock_irqrestore(&chan->lock, flags); return ret; } +EXPORT_SYMBOL_GPL(cpdma_chan_idle_submit); int cpdma_chan_idle_submit_mapped(struct cpdma_chan *chan, void *token, dma_addr_t data, int len, int directed) @@ -1125,6 +1142,7 @@ int cpdma_chan_idle_submit_mapped(struct cpdma_chan *chan, void *token, spin_unlock_irqrestore(&chan->lock, flags); return ret; } +EXPORT_SYMBOL_GPL(cpdma_chan_idle_submit_mapped); int cpdma_chan_submit(struct cpdma_chan *chan, void *token, void *data, int len, int directed) @@ -1150,6 +1168,7 @@ int cpdma_chan_submit(struct cpdma_chan *chan, void *token, void *data, spin_unlock_irqrestore(&chan->lock, flags); return ret; } +EXPORT_SYMBOL_GPL(cpdma_chan_submit); int cpdma_chan_submit_mapped(struct cpdma_chan *chan, void *token, dma_addr_t data, int len, int directed) @@ -1175,6 +1194,7 @@ int cpdma_chan_submit_mapped(struct cpdma_chan *chan, void *token, spin_unlock_irqrestore(&chan->lock, flags); return ret; } +EXPORT_SYMBOL_GPL(cpdma_chan_submit_mapped); bool cpdma_check_free_tx_desc(struct cpdma_chan *chan) { @@ -1189,6 +1209,7 @@ bool cpdma_check_free_tx_desc(struct cpdma_chan *chan) spin_unlock_irqrestore(&chan->lock, flags); return free_tx_desc; } +EXPORT_SYMBOL_GPL(cpdma_check_free_tx_desc); static void __cpdma_chan_free(struct cpdma_chan *chan, struct cpdma_desc __iomem *desc, @@ -1289,6 +1310,7 @@ int cpdma_chan_process(struct cpdma_chan *chan, int quota) } return used; } +EXPORT_SYMBOL_GPL(cpdma_chan_process); int cpdma_chan_start(struct cpdma_chan *chan) { @@ -1308,6 +1330,7 @@ int cpdma_chan_start(struct cpdma_chan *chan) return 0; } +EXPORT_SYMBOL_GPL(cpdma_chan_start); int cpdma_chan_stop(struct cpdma_chan *chan) { @@ -1370,6 +1393,7 @@ int cpdma_chan_stop(struct cpdma_chan *chan) spin_unlock_irqrestore(&chan->lock, flags); return 0; } +EXPORT_SYMBOL_GPL(cpdma_chan_stop); int cpdma_chan_int_ctrl(struct cpdma_chan *chan, bool enable) { @@ -1416,11 +1440,13 @@ int cpdma_get_num_rx_descs(struct cpdma_ctlr *ctlr) { return ctlr->num_rx_desc; } +EXPORT_SYMBOL_GPL(cpdma_get_num_rx_descs); int cpdma_get_num_tx_descs(struct cpdma_ctlr *ctlr) { return ctlr->num_tx_desc; } +EXPORT_SYMBOL_GPL(cpdma_get_num_tx_descs); int cpdma_set_num_rx_descs(struct cpdma_ctlr *ctlr, int num_rx_desc) { @@ -1442,3 +1468,4 @@ int cpdma_set_num_rx_descs(struct cpdma_ctlr *ctlr, int num_rx_desc) return ret; } +EXPORT_SYMBOL_GPL(cpdma_set_num_rx_descs); diff --git a/drivers/net/ethernet/ti/davinci_emac.c b/drivers/net/ethernet/ti/davinci_emac.c index ed8116fb05e9..3802122d0f1b 100644 --- a/drivers/net/ethernet/ti/davinci_emac.c +++ b/drivers/net/ethernet/ti/davinci_emac.c @@ -1389,15 +1389,6 @@ static int emac_devioctl(struct net_device *ndev, struct ifreq *ifrq, int cmd) return -EOPNOTSUPP; } -static int match_first_device(struct device *dev, const void *data) -{ - if (dev->parent && dev->parent->of_node) - return of_device_is_compatible(dev->parent->of_node, - "ti,davinci_mdio"); - - return !strncmp(dev_name(dev), "davinci_mdio", 12); -} - /** * emac_dev_open - EMAC device open * @ndev: The DaVinci EMAC network adapter @@ -1417,7 +1408,6 @@ static int emac_dev_open(struct net_device *ndev) int i = 0; struct emac_priv *priv = netdev_priv(ndev); struct phy_device *phydev = NULL; - struct device *phy = NULL; ret = pm_runtime_resume_and_get(&priv->pdev->dev); if (ret < 0) { @@ -1502,20 +1492,22 @@ static int emac_dev_open(struct net_device *ndev) } } - /* use the first phy on the bus if pdata did not give us a phy id */ + /* if no phy-handle and no fixed link, use the first phy on the bus */ if (!phydev && !priv->phy_id) { - /* NOTE: we can't use bus_find_device_by_name() here because - * the device name is not guaranteed to be 'davinci_mdio'. On - * some systems it can be 'davinci_mdio.0' so we need to use - * strncmp() against the first part of the string to correctly - * match it. - */ - phy = bus_find_device(&mdio_bus_type, NULL, NULL, - match_first_device); - if (phy) { - priv->phy_id = dev_name(phy); - if (!priv->phy_id || !*priv->phy_id) - put_device(phy); + struct device_node *np; + + np = of_find_compatible_node(NULL, NULL, "ti,davinci_mdio"); + if (np) { + struct mii_bus *bus = of_mdio_find_bus(np); + + if (bus) { + struct phy_device *phy = phy_find_first(bus); + + if (phy) + priv->phy_id = phydev_name(phy); + put_device(&bus->dev); /* of_mdio_find_bus */ + } + of_node_put(np); /* of_find_compatible_node */ } } @@ -1523,7 +1515,6 @@ static int emac_dev_open(struct net_device *ndev) phydev = phy_connect(ndev, priv->phy_id, &emac_adjust_link, PHY_INTERFACE_MODE_MII); - put_device(phy); /* reference taken by bus_find_device */ if (IS_ERR(phydev)) { dev_err(emac_dev, "could not connect to phy %s\n", priv->phy_id); diff --git a/drivers/net/ethernet/ti/icssg/icssg_classifier.c b/drivers/net/ethernet/ti/icssg/icssg_classifier.c index 833ca86d0b71..87a8577093db 100644 --- a/drivers/net/ethernet/ti/icssg/icssg_classifier.c +++ b/drivers/net/ethernet/ti/icssg/icssg_classifier.c @@ -27,7 +27,7 @@ #define FT1_DA0_MASK 0x8 #define FT1_DA1_MASK 0xc -#define FT1_N_REG(slize, n, reg) \ +#define FT1_N_REG(slice, n, reg) \ (offs[slice].ft1_slot_base + FT1_SLOT_SIZE * (n) + (reg)) #define FT1_LEN_MASK GENMASK(19, 16) @@ -62,7 +62,7 @@ enum ft1_cfg_type { #define FT3_T 0x18 #define FT3_T_MASK 0x1c -#define FT3_N_REG(slize, n, reg) \ +#define FT3_N_REG(slice, n, reg) \ (offs[slice].ft3_slot_base + FT3_SLOT_SIZE * (n) + (reg)) /* offsets from rx_class n's base */ @@ -74,6 +74,9 @@ enum ft1_cfg_type { #define RX_CLASS_N_REG(slice, n, reg) \ (offs[slice].rx_class_base + RX_CLASS_EN_SIZE * (n) + (reg)) +#define RX_CLASS_OR_REG(slice, n, reg) \ + (offs[slice].rx_class_or_base + RX_CLASS_EN_SIZE * (n) + (reg)) + /* RX Class Gates */ #define RX_CLASS_GATES_SIZE 0x4 /* bytes */ @@ -102,6 +105,22 @@ enum ft1_cfg_type { ((BIT(slot) << RX_CLASS_FT_FT1_MATCH_SHIFT) & \ RX_CLASS_FT_FT1_MATCH_MASK) +/* HSR/PRP protocol frame filtering */ +#define HSR_PTP_ETHERTYPE_OFFSET 0x12 +#define PRP_PTP_ETHERTYPE_OFFSET 0x0C +#define FT3_PTP_PATTERN 0xF788 +#define FT3_ETHERTYPE_MASK 0xFFFF0000U +#define FT3_VLAN_MODE_BOTH 0x1 +#define RX_CLASS_OR_DUP_PTP 0x4200 +#define RX_CLASS_OR_HSR_TAG 0x4000 +#define RX_CLASS_GATE_PTP 0x50 +#define RX_CLASS_DISABLE 0x0 + +/* HSR/PRP classifier indices */ +#define CLASSIFIER_PTP_DUP 10 +#define CLASSIFIER_HSR_TAG 11 +#define FT3_PTP_SLOT 14 + /* RX class type */ enum rx_class_sel_type { RX_CLASS_SEL_TYPE_OR = 0, @@ -133,6 +152,7 @@ struct miig_rt_offsets { u32 ft3_p_base; u32 ft_rx_ptr; u32 rx_class_base; + u32 rx_class_or_base; u32 rx_class_cfg1; u32 rx_class_cfg2; u32 rx_class_gates_base; @@ -161,6 +181,7 @@ static const struct miig_rt_offsets offs[] = { 0x308, 0x408, 0x40c, + 0x410, 0x48c, 0x490, 0x494, @@ -186,6 +207,7 @@ static const struct miig_rt_offsets offs[] = { 0x8d4, 0x9d4, 0x9d8, + 0x9dc, 0xa58, 0xa5c, 0xa60, @@ -467,3 +489,46 @@ void icssg_ft1_set_mac_addr(struct regmap *miig_rt, int slice, u8 *mac_addr) rx_class_ft1_cfg_set_type(miig_rt, slice, 0, FT1_CFG_TYPE_EQ); } EXPORT_SYMBOL_GPL(icssg_ft1_set_mac_addr); + +/** + * icssg_ft3_hsr_configurations - Configure filter table for HSR/PRP protocol frames + * @miig_rt: Pointer to the MII-G register map + * @slice: ICSSG slice number (0 or 1) + * @prueth: Pointer to prueth structure to determine HSR/PRP mode + * + * Configures FT3 to detect PTP frames (EtherType 0x88F7) in HSR/PRP tagged packets. + * HSR frames have a 6-byte tag, while PRP has no tag offset for EtherType detection. + */ +void icssg_ft3_hsr_configurations(struct regmap *miig_rt, int slice, + struct prueth *prueth) +{ + u8 offset = (prueth->hsr_prp_version == PRP_V1) ? + PRP_PTP_ETHERTYPE_OFFSET : HSR_PTP_ETHERTYPE_OFFSET; + + regmap_write(miig_rt, FT3_N_REG(slice, FT3_PTP_SLOT, FT3_START), offset); + regmap_write(miig_rt, FT3_N_REG(slice, FT3_PTP_SLOT, FT3_START_AUTO), 0); + regmap_write(miig_rt, FT3_N_REG(slice, FT3_PTP_SLOT, FT3_START_OFFSET), 0); + regmap_write(miig_rt, FT3_N_REG(slice, FT3_PTP_SLOT, FT3_JUMP_OFFSET), 0); + regmap_write(miig_rt, FT3_N_REG(slice, FT3_PTP_SLOT, FT3_LEN), 0); + regmap_write(miig_rt, FT3_N_REG(slice, FT3_PTP_SLOT, FT3_CFG), FT3_VLAN_MODE_BOTH); + regmap_write(miig_rt, FT3_N_REG(slice, FT3_PTP_SLOT, FT3_T), FT3_PTP_PATTERN); + regmap_write(miig_rt, FT3_N_REG(slice, FT3_PTP_SLOT, FT3_T_MASK), + FT3_ETHERTYPE_MASK); + + regmap_write(miig_rt, RX_CLASS_N_REG(slice, CLASSIFIER_PTP_DUP, RX_CLASS_AND_EN), + RX_CLASS_DISABLE); + regmap_write(miig_rt, RX_CLASS_OR_REG(slice, CLASSIFIER_PTP_DUP, RX_CLASS_AND_EN), + RX_CLASS_OR_DUP_PTP); + regmap_write(miig_rt, RX_CLASS_GATES_N_REG(slice, CLASSIFIER_PTP_DUP), + RX_CLASS_GATE_PTP); + + if (prueth->hsr_prp_version != PRP_V1) { + regmap_write(miig_rt, RX_CLASS_N_REG(slice, CLASSIFIER_HSR_TAG, RX_CLASS_AND_EN), + RX_CLASS_DISABLE); + regmap_write(miig_rt, RX_CLASS_OR_REG(slice, CLASSIFIER_HSR_TAG, RX_CLASS_AND_EN), + RX_CLASS_OR_HSR_TAG); + regmap_write(miig_rt, RX_CLASS_GATES_N_REG(slice, CLASSIFIER_HSR_TAG), + RX_CLASS_GATE_PTP); + } +} +EXPORT_SYMBOL_GPL(icssg_ft3_hsr_configurations); diff --git a/drivers/net/ethernet/ti/icssg/icssg_prueth.c b/drivers/net/ethernet/ti/icssg/icssg_prueth.c index 42a881bee109..591be5c8056b 100644 --- a/drivers/net/ethernet/ti/icssg/icssg_prueth.c +++ b/drivers/net/ethernet/ti/icssg/icssg_prueth.c @@ -260,6 +260,12 @@ static int prueth_emac_common_start(struct prueth *prueth) icssg_class_default(prueth->miig_rt, ICSS_SLICE0, 0, false); icssg_class_default(prueth->miig_rt, ICSS_SLICE1, 0, false); + /* Configure HSR/PRP protocol filtering if in HSR offload mode */ + if (prueth->is_hsr_offload_mode) { + icssg_ft3_hsr_configurations(prueth->miig_rt, ICSS_SLICE0, prueth); + icssg_ft3_hsr_configurations(prueth->miig_rt, ICSS_SLICE1, prueth); + } + if (prueth->is_switch_mode || prueth->is_hsr_offload_mode) icssg_init_fw_offload_mode(prueth); else @@ -663,28 +669,133 @@ static int icssg_prueth_del_mcast(struct net_device *ndev, const u8 *addr) return 0; } +/** + * icssg_is_addr_synced - Check if address is synced from HSR master + * @ndev: network device + * @addr: multicast MAC address + * + * Checks if the address is synced from HSR master (hsr0) via + * dev_mc_sync_multiple() or added directly to the slave port. + * + * Return: true if synced from HSR master, false if added directly + */ +static bool icssg_is_addr_synced(struct net_device *ndev, const u8 *addr) +{ + struct netdev_hw_addr *ha; + bool is_synced = false; + + netif_addr_lock_bh(ndev); + netdev_for_each_mc_addr(ha, ndev) { + if (ether_addr_equal(ha->addr, addr)) { + is_synced = ha->synced; + break; + } + } + netif_addr_unlock_bh(ndev); + + return is_synced; +} + +/** + * icssg_hsr_fdb_update - Update FDB and VLAN table for HSR multicast + * @emac: PRUETH EMAC private data + * @addr: multicast MAC address + * @vid: VLAN ID + * @port_membership: port membership bitmap (P0=0x1, P1=0x2, P2=0x4) + * @add: true to add entry, false to delete + * + * Updates both FDB and VLAN table entries for the given address. + */ +static void icssg_hsr_fdb_update(struct prueth_emac *emac, const u8 *addr, + u8 vid, u8 port_membership, bool add) +{ + icssg_fdb_add_del(emac, addr, vid, port_membership, add); + icssg_vtbl_modify(emac, vid, BIT(emac->port_id), + BIT(emac->port_id), add); +} + +/** + * icssg_prueth_hsr_fdb_add_del - Manage FDB port membership for HSR multicast + * @emac: PRUETH EMAC private data + * @addr: multicast MAC address + * @vid: VLAN ID + * @is_synced: true if synced from HSR master, false if added directly + * @is_vlan_path: true if called from VLAN interface path, false for direct path + * @add: true to add/update, false to remove + * + * Manages FDB port membership bits (P0/P1/P2) for multicast addresses. + * On add: accumulates membership with existing ports. + * On delete: removes only the specific port, cleans up orphaned P0 if needed. + */ static void icssg_prueth_hsr_fdb_add_del(struct prueth_emac *emac, - const u8 *addr, u8 vid, bool add) + const u8 *addr, u8 vid, + bool is_synced, bool is_vlan_path, + bool add) { - icssg_fdb_add_del(emac, addr, vid, - ICSSG_FDB_ENTRY_P0_MEMBERSHIP | - ICSSG_FDB_ENTRY_P1_MEMBERSHIP | - ICSSG_FDB_ENTRY_P2_MEMBERSHIP | - ICSSG_FDB_ENTRY_BLOCK, add); - - if (add) - icssg_vtbl_modify(emac, vid, BIT(emac->port_id), - BIT(emac->port_id), add); + int existing_membership, other_port_membership; + u8 port_membership; + + /* Set P0 (HSR master) bit when: + * - Address is synced from HSR master (is_synced=true), OR + * - In HSR offload mode AND it's a VLAN interface (is_vlan_path=true) + * This ensures VLAN interfaces on HSR master (hsr0.X) also get P0 set. + */ + if (is_synced || (emac->prueth->is_hsr_offload_mode && is_vlan_path)) + port_membership = ICSSG_FDB_ENTRY_P0_MEMBERSHIP | BIT(emac->port_id); + else + port_membership = BIT(emac->port_id); + existing_membership = icssg_fdb_lookup(emac, addr, vid); + if (existing_membership < 0) { + netdev_dbg(emac->ndev, + "FDB lookup failed for %pM: %d, assuming no existing entry\n", + addr, existing_membership); + existing_membership = 0; + } + + if (add) { + /* Accumulate with existing membership */ + port_membership |= existing_membership; + + netdev_dbg(emac->ndev, + "FDB add %pM: P%d%s -> membership 0x%x\n", + addr, emac->port_id, is_synced ? "+P0" : "", + port_membership); + + icssg_hsr_fdb_update(emac, addr, vid, port_membership, true); + } else { + other_port_membership = existing_membership & ~port_membership; + + /* Clean up orphaned P0 if neither HSR synced nor VLAN path */ + if (!is_synced && !is_vlan_path && + (existing_membership & ICSSG_FDB_ENTRY_P0_MEMBERSHIP)) + other_port_membership &= + ~ICSSG_FDB_ENTRY_P0_MEMBERSHIP; + + netdev_dbg(emac->ndev, + "FDB del %pM: 0x%x -> 0x%x\n", + addr, existing_membership, other_port_membership); + + icssg_hsr_fdb_update(emac, addr, vid, port_membership, false); + + if (other_port_membership) + icssg_hsr_fdb_update(emac, addr, vid, + other_port_membership, true); + } } static int icssg_prueth_hsr_add_mcast(struct net_device *ndev, const u8 *addr) { struct net_device *real_dev, *port_dev; + bool is_synced, is_vlan_path; struct prueth_emac *emac; u8 vlan_id, i; - vlan_id = is_vlan_dev(ndev) ? vlan_dev_vlan_id(ndev) : PRUETH_DFLT_VLAN_HSR; - real_dev = is_vlan_dev(ndev) ? vlan_dev_real_dev(ndev) : ndev; + is_vlan_path = is_vlan_dev(ndev); + vlan_id = is_vlan_path ? vlan_dev_vlan_id(ndev) : PRUETH_DFLT_VLAN_HSR; + real_dev = is_vlan_path ? vlan_dev_real_dev(ndev) : ndev; + + /* Check if this address is synced from HSR master */ + is_synced = icssg_is_addr_synced(ndev, addr); if (is_hsr_master(real_dev)) { for (i = HSR_PT_SLAVE_A; i < HSR_PT_INTERLINK; i++) { @@ -695,12 +806,14 @@ static int icssg_prueth_hsr_add_mcast(struct net_device *ndev, const u8 *addr) return -EINVAL; } icssg_prueth_hsr_fdb_add_del(emac, addr, vlan_id, + is_synced, is_vlan_path, true); dev_put(port_dev); } } else { emac = netdev_priv(real_dev); - icssg_prueth_hsr_fdb_add_del(emac, addr, vlan_id, true); + icssg_prueth_hsr_fdb_add_del(emac, addr, vlan_id, + is_synced, is_vlan_path, true); } return 0; @@ -709,11 +822,16 @@ static int icssg_prueth_hsr_add_mcast(struct net_device *ndev, const u8 *addr) static int icssg_prueth_hsr_del_mcast(struct net_device *ndev, const u8 *addr) { struct net_device *real_dev, *port_dev; + bool is_synced, is_vlan_path; struct prueth_emac *emac; u8 vlan_id, i; - vlan_id = is_vlan_dev(ndev) ? vlan_dev_vlan_id(ndev) : PRUETH_DFLT_VLAN_HSR; - real_dev = is_vlan_dev(ndev) ? vlan_dev_real_dev(ndev) : ndev; + is_vlan_path = is_vlan_dev(ndev); + vlan_id = is_vlan_path ? vlan_dev_vlan_id(ndev) : PRUETH_DFLT_VLAN_HSR; + real_dev = is_vlan_path ? vlan_dev_real_dev(ndev) : ndev; + + /* Check if this address was synced from HSR master */ + is_synced = icssg_is_addr_synced(ndev, addr); if (is_hsr_master(real_dev)) { for (i = HSR_PT_SLAVE_A; i < HSR_PT_INTERLINK; i++) { @@ -724,12 +842,14 @@ static int icssg_prueth_hsr_del_mcast(struct net_device *ndev, const u8 *addr) return -EINVAL; } icssg_prueth_hsr_fdb_add_del(emac, addr, vlan_id, + is_synced, is_vlan_path, false); dev_put(port_dev); } } else { emac = netdev_priv(real_dev); - icssg_prueth_hsr_fdb_add_del(emac, addr, vlan_id, false); + icssg_prueth_hsr_fdb_add_del(emac, addr, vlan_id, + is_synced, is_vlan_path, false); } return 0; @@ -1053,6 +1173,104 @@ static int emac_ndo_stop(struct net_device *ndev) return 0; } +/** + * icssg_hsr_handle_multicast_sync() - Handle HSR multicast overlapping memberships + * @emac: PRUETH EMAC private structure + * + * Post-process multicast addresses to handle overlapping memberships where + * the same address is added from multiple paths (hsr0 + eth1). The kernel + * increments refcount without triggering callbacks, so manually check refcount + * to detect and update FDB port membership accordingly. + */ +static void icssg_hsr_handle_multicast_sync(struct prueth_emac *emac) +{ + struct hsr_mcast_update { + struct list_head list; + u8 addr[ETH_ALEN]; + int refcount; + bool synced; + }; + struct hsr_mcast_update *update, *tmp; + struct net_device *ndev = emac->ndev; + u8 vlan_id = PRUETH_DFLT_VLAN_HSR; + struct netdev_hw_addr *ha; + int existing_membership; + LIST_HEAD(update_list); + u8 port_membership; + + /* Handle overlapping memberships: When the same address + * is added from multiple paths (hsr0 + eth1), kernel + * increments refcount without triggering callbacks. + * Manually check refcount to detect: + * - refcount=2 + synced: HSR only, membership = P0 + * - refcount>=3 + synced: HSR + direct, membership = P0|Px + * - !synced but P0 set: HSR removed, clean up orphaned P0 + * + * Collect addresses to update while holding the lock, then + * process them after releasing the lock to avoid sleeping + * while atomic (icssg_fdb_lookup/update can sleep). + */ + netif_addr_lock_bh(ndev); + netdev_for_each_mc_addr(ha, ndev) { + /* Queue this address for processing. + * We need to check existing_membership via FDB lookup + * (which can sleep), so defer that until after unlock. + * Save synced/refcount to reproduce original logic. + */ + update = kmalloc_obj(*update, GFP_ATOMIC); + if (!update) + continue; + + ether_addr_copy(update->addr, ha->addr); + update->synced = ha->synced; + update->refcount = ha->refcount; + list_add_tail(&update->list, &update_list); + } + netif_addr_unlock_bh(ndev); + + /* Process collected addresses outside spinlock */ + list_for_each_entry_safe(update, tmp, &update_list, list) { + existing_membership = icssg_fdb_lookup(emac, + update->addr, + vlan_id); + if (existing_membership < 0) { + netdev_dbg(ndev, + "FDB lookup failed for %pM: %d, skipping\n", + update->addr, existing_membership); + list_del(&update->list); + kfree(update); + continue; + } + + port_membership = existing_membership; + if (update->synced) { + port_membership |= + ICSSG_FDB_ENTRY_P0_MEMBERSHIP; + if (update->refcount >= 3) + port_membership |= BIT(emac->port_id); + else + port_membership &= ~BIT(emac->port_id); + } else if (existing_membership & + ICSSG_FDB_ENTRY_P0_MEMBERSHIP) { + port_membership &= + ~ICSSG_FDB_ENTRY_P0_MEMBERSHIP; + } + + if (existing_membership != port_membership) { + netdev_dbg(ndev, "Update %pM: 0x%x -> 0x%x\n", + update->addr, existing_membership, + port_membership); + + icssg_hsr_fdb_update(emac, update->addr, + vlan_id, port_membership, + true); + } + + list_del(&update->list); + kfree(update); + } +} + static void emac_ndo_set_rx_mode_work(struct work_struct *work) { struct prueth_emac *emac = container_of(work, struct prueth_emac, rx_mode_work); @@ -1079,8 +1297,13 @@ static void emac_ndo_set_rx_mode_work(struct work_struct *work) } if (emac->prueth->is_hsr_offload_mode) { + /* Track basic add/delete via callbacks */ __dev_mc_sync(ndev, icssg_prueth_hsr_add_mcast, icssg_prueth_hsr_del_mcast); + + /* Handle overlapping memberships */ + icssg_hsr_handle_multicast_sync(emac); + if (rtnl_trylock()) { vlan_for_each(emac->prueth->hsr_dev, icssg_update_vlan_mcast, emac); diff --git a/drivers/net/ethernet/ti/icssg/icssg_prueth.h b/drivers/net/ethernet/ti/icssg/icssg_prueth.h index 3d94fa5a7ac1..df93d15c5b78 100644 --- a/drivers/net/ethernet/ti/icssg/icssg_prueth.h +++ b/drivers/net/ethernet/ti/icssg/icssg_prueth.h @@ -423,6 +423,8 @@ void icssg_class_promiscuous_sr1(struct regmap *miig_rt, int slice); void icssg_class_add_mcast_sr1(struct regmap *miig_rt, int slice, struct net_device *ndev); void icssg_ft1_set_mac_addr(struct regmap *miig_rt, int slice, u8 *mac_addr); +void icssg_ft3_hsr_configurations(struct regmap *miig_rt, int slice, + struct prueth *prueth); /* config helpers */ void icssg_config_ipg(struct prueth_emac *emac); diff --git a/drivers/net/ethernet/wangxun/libwx/wx_ethtool.c b/drivers/net/ethernet/wangxun/libwx/wx_ethtool.c index f362e51c73ee..5df971aca9e3 100644 --- a/drivers/net/ethernet/wangxun/libwx/wx_ethtool.c +++ b/drivers/net/ethernet/wangxun/libwx/wx_ethtool.c @@ -211,7 +211,7 @@ void wx_get_pause_stats(struct net_device *netdev, hwstats = &wx->stats; stats->tx_pause_frames = hwstats->lxontxc + hwstats->lxofftxc; - stats->rx_pause_frames = hwstats->lxonoffrxc; + stats->rx_pause_frames = hwstats->lxonrxc + hwstats->lxoffrxc; } EXPORT_SYMBOL(wx_get_pause_stats); @@ -262,6 +262,39 @@ int wx_set_link_ksettings(struct net_device *netdev, } EXPORT_SYMBOL(wx_set_link_ksettings); +void wx_get_wol(struct net_device *netdev, + struct ethtool_wolinfo *wol) +{ + struct wx *wx = netdev_priv(netdev); + + if (!wx->wol_hw_supported) + return; + wol->supported = WAKE_MAGIC; + wol->wolopts = 0; + if (wx->wol & WX_PSR_WKUP_CTL_MAG) + wol->wolopts |= WAKE_MAGIC; +} +EXPORT_SYMBOL(wx_get_wol); + +int wx_set_wol(struct net_device *netdev, + struct ethtool_wolinfo *wol) +{ + struct wx *wx = netdev_priv(netdev); + struct pci_dev *pdev = wx->pdev; + + if (!wx->wol_hw_supported) + return -EOPNOTSUPP; + + wx->wol = 0; + if (wol->wolopts & WAKE_MAGIC) + wx->wol = WX_PSR_WKUP_CTL_MAG; + wr32(wx, WX_PSR_WKUP_CTL, wx->wol); + device_set_wakeup_enable(&pdev->dev, !!(wx->wol)); + + return 0; +} +EXPORT_SYMBOL(wx_set_wol); + void wx_get_pauseparam(struct net_device *netdev, struct ethtool_pauseparam *pause) { @@ -522,7 +555,7 @@ int wx_set_channels(struct net_device *dev, wx->ring_feature[RING_F_RSS].limit = count; - return 0; + return wx->setup_tc(dev, netdev_get_num_tc(dev)); } EXPORT_SYMBOL(wx_set_channels); diff --git a/drivers/net/ethernet/wangxun/libwx/wx_ethtool.h b/drivers/net/ethernet/wangxun/libwx/wx_ethtool.h index 727093970462..5b187d1587b1 100644 --- a/drivers/net/ethernet/wangxun/libwx/wx_ethtool.h +++ b/drivers/net/ethernet/wangxun/libwx/wx_ethtool.h @@ -18,6 +18,10 @@ int wx_get_link_ksettings(struct net_device *netdev, struct ethtool_link_ksettings *cmd); int wx_set_link_ksettings(struct net_device *netdev, const struct ethtool_link_ksettings *cmd); +void wx_get_wol(struct net_device *netdev, + struct ethtool_wolinfo *wol); +int wx_set_wol(struct net_device *netdev, + struct ethtool_wolinfo *wol); void wx_get_pauseparam(struct net_device *netdev, struct ethtool_pauseparam *pause); int wx_set_pauseparam(struct net_device *netdev, diff --git a/drivers/net/ethernet/wangxun/libwx/wx_hw.c b/drivers/net/ethernet/wangxun/libwx/wx_hw.c index bee9e245e792..d3772d01e00b 100644 --- a/drivers/net/ethernet/wangxun/libwx/wx_hw.c +++ b/drivers/net/ethernet/wangxun/libwx/wx_hw.c @@ -2513,6 +2513,8 @@ int wx_sw_init(struct wx *wx) return -ENOMEM; } + spin_lock_init(&wx->hw_stats_lock); + mutex_init(&wx->reset_lock); bitmap_zero(wx->state, WX_STATE_NBITS); bitmap_zero(wx->flags, WX_PF_FLAGS_NBITS); wx->misc_irq_domain = false; @@ -2772,6 +2774,15 @@ int wx_fc_enable(struct wx *wx, bool tx_pause, bool rx_pause) } } + if (rx_pause && tx_pause) + wx->fc.mode = wx_fc_full; + else if (rx_pause) + wx->fc.mode = wx_fc_rx_pause; + else if (tx_pause) + wx->fc.mode = wx_fc_tx_pause; + else + wx->fc.mode = wx_fc_none; + /* Disable any previous flow control settings */ mflcn_reg = rd32(wx, WX_MAC_RX_FLOW_CTRL); mflcn_reg &= ~WX_MAC_RX_FLOW_CTRL_RFE; @@ -2790,7 +2801,9 @@ int wx_fc_enable(struct wx *wx, bool tx_pause, bool rx_pause) /* Set up and enable Rx high/low water mark thresholds, enable XON. */ if (tx_pause && wx->fc.high_water) { - fcrtl = (wx->fc.low_water << 10) | WX_RDB_RFCL_XONE; + fcrtl = (wx->fc.low_water << 10); + if (wx->mac.type != wx_mac_sp) + fcrtl |= WX_RDB_RFCL_XONE; wr32(wx, WX_RDB_RFCL, fcrtl); fcrth = (wx->fc.high_water << 10) | WX_RDB_RFCH_XOFFE; } else { @@ -2831,6 +2844,21 @@ int wx_fc_enable(struct wx *wx, bool tx_pause, bool rx_pause) } EXPORT_SYMBOL(wx_fc_enable); +static void wx_update_xoff_rx_lfc(struct wx *wx) +{ + struct wx_hw_stats *hwstats = &wx->stats; + + if (wx->fc.mode != wx_fc_full && + wx->fc.mode != wx_fc_rx_pause) + return; + + if (wx->mac.type >= wx_mac_aml) + hwstats->lxoffrxc += rd32_wrap(wx, WX_MAC_LXOFFRXC_AML, + &wx->last_stats.lxoffrxc); + else + hwstats->lxoffrxc += rd64(wx, WX_MAC_LXOFFRXC); +} + /** * wx_update_stats - Update the board statistics counters. * @wx: board private structure @@ -2844,6 +2872,12 @@ void wx_update_stats(struct wx *wx) u64 restart_queue = 0, tx_busy = 0; u32 i; + if (!netif_running(wx->netdev) || + test_bit(WX_STATE_RESETTING, wx->state)) + return; + + spin_lock(&wx->hw_stats_lock); + /* gather some stats to the wx struct that are per queue */ for (i = 0; i < wx->num_rx_queues; i++) { struct wx_ring *rx_ring = wx->rx_ring[i]; @@ -2879,6 +2913,8 @@ void wx_update_stats(struct wx *wx) wx->restart_queue = restart_queue; wx->tx_busy = tx_busy; + wx_update_xoff_rx_lfc(wx); + hwstats->gprc += rd32(wx, WX_RDM_PKT_CNT); hwstats->gptc += rd32(wx, WX_TDM_PKT_CNT); hwstats->gorc += rd64(wx, WX_RDM_BYTE_CNT_LSB); @@ -2893,7 +2929,11 @@ void wx_update_stats(struct wx *wx) hwstats->mptc += rd64(wx, WX_TX_MC_FRAMES_GOOD_L); hwstats->roc += rd32(wx, WX_RX_OVERSIZE_FRAMES_GOOD); hwstats->ruc += rd32(wx, WX_RX_UNDERSIZE_FRAMES_GOOD); - hwstats->lxonoffrxc += rd32(wx, WX_MAC_LXONOFFRXC); + if (wx->mac.type >= wx_mac_aml) + hwstats->lxonrxc += rd32_wrap(wx, WX_MAC_LXONRXC_AML, + &wx->last_stats.lxonrxc); + else + hwstats->lxonrxc += rd32(wx, WX_MAC_LXONRXC); hwstats->lxontxc += rd32(wx, WX_RDB_LXONTXC); hwstats->lxofftxc += rd32(wx, WX_RDB_LXOFFTXC); hwstats->o2bgptc += rd32(wx, WX_TDM_OS2BMC_CNT); @@ -2907,11 +2947,12 @@ void wx_update_stats(struct wx *wx) hwstats->fdirmiss += rd32(wx, WX_RDB_FDIR_MISS); } - /* qmprc is not cleared on read, manual reset it */ - hwstats->qmprc = 0; for (i = wx->num_vfs * wx->num_rx_queues_per_pool; i < wx->mac.max_rx_queues; i++) - hwstats->qmprc += rd32(wx, WX_PX_MPRC(i)); + hwstats->qmprc += rd32_wrap(wx, WX_PX_MPRC(i), + &wx->last_stats.qmprc[i]); + + spin_unlock(&wx->hw_stats_lock); } EXPORT_SYMBOL(wx_update_stats); @@ -2926,8 +2967,11 @@ void wx_clear_hw_cntrs(struct wx *wx) { u16 i = 0; - for (i = 0; i < wx->mac.max_rx_queues; i++) + for (i = wx->num_vfs * wx->num_rx_queues_per_pool; + i < wx->mac.max_rx_queues; i++) { wr32(wx, WX_PX_MPRC(i), 0); + wx->last_stats.qmprc[i] = 0; + } rd32(wx, WX_RDM_PKT_CNT); rd32(wx, WX_TDM_PKT_CNT); @@ -2946,7 +2990,15 @@ void wx_clear_hw_cntrs(struct wx *wx) rd64(wx, WX_RX_LEN_ERROR_FRAMES_L); rd32(wx, WX_RDB_LXONTXC); rd32(wx, WX_RDB_LXOFFTXC); - rd32(wx, WX_MAC_LXONOFFRXC); + if (wx->mac.type >= wx_mac_aml) { + wr32(wx, WX_MAC_LXONRXC_AML, 0); + wr32(wx, WX_MAC_LXOFFRXC_AML, 0); + wx->last_stats.lxonrxc = 0; + wx->last_stats.lxoffrxc = 0; + } else { + rd32(wx, WX_MAC_LXONRXC); + rd64(wx, WX_MAC_LXOFFRXC); + } } EXPORT_SYMBOL(wx_clear_hw_cntrs); diff --git a/drivers/net/ethernet/wangxun/libwx/wx_type.h b/drivers/net/ethernet/wangxun/libwx/wx_type.h index 29e5c5470c94..0da5565ee4ff 100644 --- a/drivers/net/ethernet/wangxun/libwx/wx_type.h +++ b/drivers/net/ethernet/wangxun/libwx/wx_type.h @@ -79,7 +79,10 @@ #define WX_RX_LEN_ERROR_FRAMES_L 0x11978 #define WX_RX_UNDERSIZE_FRAMES_GOOD 0x11938 #define WX_RX_OVERSIZE_FRAMES_GOOD 0x1193C -#define WX_MAC_LXONOFFRXC 0x11E0C +#define WX_MAC_LXOFFRXC 0x11988 +#define WX_MAC_LXONRXC 0x11E0C +#define WX_MAC_LXOFFRXC_AML 0x11F80 +#define WX_MAC_LXONRXC_AML 0x11F84 /*********************** Receive DMA registers **************************/ #define WX_RDM_VF_RE(_i) (0x12004 + ((_i) * 4)) @@ -1148,9 +1151,18 @@ enum wx_isb_idx { WX_ISB_MAX }; +/* Flow Control Settings */ +enum wx_fc_mode { + wx_fc_none = 0, + wx_fc_rx_pause, + wx_fc_tx_pause, + wx_fc_full +}; + struct wx_fc_info { u32 high_water; /* Flow Ctrl High-water */ u32 low_water; /* Flow Ctrl Low-water */ + enum wx_fc_mode mode; /* Flow Control Mode */ }; /* Statistics counters collected by the MAC */ @@ -1167,7 +1179,8 @@ struct wx_hw_stats { u64 mptc; u64 roc; u64 ruc; - u64 lxonoffrxc; + u64 lxonrxc; + u64 lxoffrxc; u64 lxontxc; u64 lxofftxc; u64 o2bgptc; @@ -1182,6 +1195,12 @@ struct wx_hw_stats { u64 fdirmiss; }; +struct wx_last_stats { + u32 qmprc[128]; + u32 lxoffrxc; + u32 lxonrxc; +}; + enum wx_state { WX_STATE_RESETTING, WX_STATE_SWFW_BUSY, @@ -1354,6 +1373,8 @@ struct wx { bool default_up; struct wx_hw_stats stats; + struct wx_last_stats last_stats; + spinlock_t hw_stats_lock; /* spinlock for accessing to hw stats */ u64 tx_busy; u64 non_eop_descs; u64 restart_queue; @@ -1400,6 +1421,7 @@ struct wx { struct timer_list service_timer; struct work_struct service_task; + struct mutex reset_lock; /* mutex for reset */ }; #define WX_INTR_ALL (~0ULL) @@ -1462,6 +1484,18 @@ wr32ptp(struct wx *wx, u32 reg, u32 value) return wr32(wx, reg + 0xB500, value); } +static inline u32 +rd32_wrap(struct wx *wx, u32 reg, u32 *last) +{ + u32 val, delta; + + val = rd32(wx, reg); + delta = val - *last; + *last = val; + + return delta; +} + /* On some domestic CPU platforms, sometimes IO is not synchronized with * flushing memory, here use readl() to flush PCI read and write. */ @@ -1478,21 +1512,6 @@ static inline struct wx *phylink_to_wx(struct phylink_config *config) return container_of(config, struct wx, phylink_config); } -static inline int wx_set_state_reset(struct wx *wx) -{ - u8 timeout = 50; - - while (test_and_set_bit(WX_STATE_RESETTING, wx->state)) { - timeout--; - if (!timeout) - return -EBUSY; - - usleep_range(1000, 2000); - } - - return 0; -} - static inline unsigned int wx_rx_pg_order(struct wx_ring *ring) { #if (PAGE_SIZE < 8192) diff --git a/drivers/net/ethernet/wangxun/libwx/wx_vf_common.c b/drivers/net/ethernet/wangxun/libwx/wx_vf_common.c index ade2bfe563aa..29cdbed2e5ec 100644 --- a/drivers/net/ethernet/wangxun/libwx/wx_vf_common.c +++ b/drivers/net/ethernet/wangxun/libwx/wx_vf_common.c @@ -48,9 +48,10 @@ void wxvf_remove(struct pci_dev *pdev) struct wx *wx = pci_get_drvdata(pdev); struct net_device *netdev; - cancel_work_sync(&wx->service_task); netdev = wx->netdev; unregister_netdev(netdev); + timer_shutdown_sync(&wx->service_timer); + cancel_work_sync(&wx->service_task); kfree(wx->vfinfo); kfree(wx->rss_key); kfree(wx->mac_table); @@ -339,14 +340,16 @@ static void wxvf_down(struct wx *wx) static void wxvf_reinit_locked(struct wx *wx) { - while (test_and_set_bit(WX_STATE_RESETTING, wx->state)) - usleep_range(1000, 2000); + mutex_lock(&wx->reset_lock); + set_bit(WX_STATE_RESETTING, wx->state); + wxvf_down(wx); wx_free_irq(wx); wx_configure_vf(wx); wx_request_msix_irqs_vf(wx); wxvf_up_complete(wx); clear_bit(WX_STATE_RESETTING, wx->state); + mutex_unlock(&wx->reset_lock); } static void wxvf_reset_subtask(struct wx *wx) diff --git a/drivers/net/ethernet/wangxun/ngbe/ngbe_ethtool.c b/drivers/net/ethernet/wangxun/ngbe/ngbe_ethtool.c index fc04040957bf..b2e191982803 100644 --- a/drivers/net/ethernet/wangxun/ngbe/ngbe_ethtool.c +++ b/drivers/net/ethernet/wangxun/ngbe/ngbe_ethtool.c @@ -12,38 +12,6 @@ #include "ngbe_ethtool.h" #include "ngbe_type.h" -static void ngbe_get_wol(struct net_device *netdev, - struct ethtool_wolinfo *wol) -{ - struct wx *wx = netdev_priv(netdev); - - if (!wx->wol_hw_supported) - return; - wol->supported = WAKE_MAGIC; - wol->wolopts = 0; - if (wx->wol & WX_PSR_WKUP_CTL_MAG) - wol->wolopts |= WAKE_MAGIC; -} - -static int ngbe_set_wol(struct net_device *netdev, - struct ethtool_wolinfo *wol) -{ - struct wx *wx = netdev_priv(netdev); - struct pci_dev *pdev = wx->pdev; - - if (!wx->wol_hw_supported) - return -EOPNOTSUPP; - - wx->wol = 0; - if (wol->wolopts & WAKE_MAGIC) - wx->wol = WX_PSR_WKUP_CTL_MAG; - netdev->ethtool->wol_enabled = !!(wx->wol); - wr32(wx, WX_PSR_WKUP_CTL, wx->wol); - device_set_wakeup_enable(&pdev->dev, netdev->ethtool->wol_enabled); - - return 0; -} - static int ngbe_set_ringparam(struct net_device *netdev, struct ethtool_ringparam *ring, struct kernel_ethtool_ringparam *kernel_ring, @@ -64,9 +32,8 @@ static int ngbe_set_ringparam(struct net_device *netdev, new_rx_count == wx->rx_ring_count) return 0; - err = wx_set_state_reset(wx); - if (err) - return err; + mutex_lock(&wx->reset_lock); + set_bit(WX_STATE_RESETTING, wx->state); if (!netif_running(wx->netdev)) { for (i = 0; i < wx->num_tx_queues; i++) @@ -97,22 +64,10 @@ static int ngbe_set_ringparam(struct net_device *netdev, clear_reset: clear_bit(WX_STATE_RESETTING, wx->state); + mutex_unlock(&wx->reset_lock); return err; } -static int ngbe_set_channels(struct net_device *dev, - struct ethtool_channels *ch) -{ - int err; - - err = wx_set_channels(dev, ch); - if (err < 0) - return err; - - /* use setup TC to update any traffic class queue mapping */ - return ngbe_setup_tc(dev, netdev_get_num_tc(dev)); -} - static const struct ethtool_ops ngbe_ethtool_ops = { .supported_coalesce_params = ETHTOOL_COALESCE_USECS | ETHTOOL_COALESCE_TX_MAX_FRAMES_IRQ | @@ -122,8 +77,8 @@ static const struct ethtool_ops ngbe_ethtool_ops = { .get_link_ksettings = wx_get_link_ksettings, .set_link_ksettings = wx_set_link_ksettings, .nway_reset = wx_nway_reset, - .get_wol = ngbe_get_wol, - .set_wol = ngbe_set_wol, + .get_wol = wx_get_wol, + .set_wol = wx_set_wol, .get_sset_count = wx_get_sset_count, .get_strings = wx_get_strings, .get_ethtool_stats = wx_get_ethtool_stats, @@ -136,7 +91,7 @@ static const struct ethtool_ops ngbe_ethtool_ops = { .get_coalesce = wx_get_coalesce, .set_coalesce = wx_set_coalesce, .get_channels = wx_get_channels, - .set_channels = ngbe_set_channels, + .set_channels = wx_set_channels, .get_rxfh_fields = wx_get_rxfh_fields, .set_rxfh_fields = wx_set_rxfh_fields, .get_rxfh_indir_size = wx_rss_indir_size, diff --git a/drivers/net/ethernet/wangxun/ngbe/ngbe_main.c b/drivers/net/ethernet/wangxun/ngbe/ngbe_main.c index 58488e138beb..d8e3827a8b1f 100644 --- a/drivers/net/ethernet/wangxun/ngbe/ngbe_main.c +++ b/drivers/net/ethernet/wangxun/ngbe/ngbe_main.c @@ -57,16 +57,16 @@ static void ngbe_init_type_code(struct wx *wx) wx->mac.type = wx_mac_em; type_mask = (u16)(wx->subsystem_device_id & NGBE_OEM_MASK); - ncsi_mask = wx->subsystem_device_id & NGBE_NCSI_MASK; - wol_mask = wx->subsystem_device_id & NGBE_WOL_MASK; + ncsi_mask = wx->subsystem_device_id & WX_NCSI_MASK; + wol_mask = wx->subsystem_device_id & WX_WOL_MASK; val = rd32(wx, WX_CFG_PORT_ST); wx->mac_type = (val & BIT(7)) >> 7 ? em_mac_type_rgmii : em_mac_type_mdi; - wx->wol_hw_supported = (wol_mask == NGBE_WOL_SUP) ? 1 : 0; - wx->ncsi_enabled = (ncsi_mask == NGBE_NCSI_MASK || + wx->wol_hw_supported = (wol_mask == WX_WOL_SUP) ? 1 : 0; + wx->ncsi_enabled = (ncsi_mask == WX_NCSI_SUP || type_mask == NGBE_SUBID_OCP_CARD) ? 1 : 0; switch (type_mask) { @@ -139,6 +139,26 @@ static int ngbe_sw_init(struct wx *wx) } /** + * ngbe_service_task - manages and runs subtasks + * @work: pointer to work_struct containing our data + **/ +static void ngbe_service_task(struct work_struct *work) +{ + struct wx *wx = container_of(work, struct wx, service_task); + + wx_update_stats(wx); + + wx_service_event_complete(wx); +} + +static void ngbe_init_service(struct wx *wx) +{ + timer_setup(&wx->service_timer, wx_service_timer, 0); + INIT_WORK(&wx->service_task, ngbe_service_task); + clear_bit(WX_STATE_SERVICE_SCHED, wx->state); +} + +/** * ngbe_irq_enable - Enable default interrupt generation settings * @wx: board private structure * @queues: enable all queues interrupts @@ -368,6 +388,10 @@ static void ngbe_disable_device(struct wx *wx) wx_napi_disable_all(wx); netif_tx_stop_all_queues(netdev); netif_tx_disable(netdev); + + timer_delete_sync(&wx->service_timer); + cancel_work_sync(&wx->service_task); + if (wx->gpio_ctrl) ngbe_sfp_modules_txrx_powerctl(wx, false); wx_irq_disable(wx); @@ -407,6 +431,7 @@ void ngbe_up(struct wx *wx) wx_napi_enable_all(wx); /* enable transmits */ netif_tx_start_all_queues(wx->netdev); + mod_timer(&wx->service_timer, jiffies); /* clear any pending interrupts, may auto mask */ rd32(wx, WX_PX_IC(0)); @@ -520,9 +545,9 @@ static void ngbe_dev_shutdown(struct pci_dev *pdev, bool *enable_wake) if (wufc) { wx_set_rx_mode(netdev); wx_configure_rx(wx); - wr32(wx, NGBE_PSR_WKUP_CTL, wufc); + wr32(wx, WX_PSR_WKUP_CTL, wufc); } else { - wr32(wx, NGBE_PSR_WKUP_CTL, 0); + wr32(wx, WX_PSR_WKUP_CTL, 0); } pci_wake_from_d3(pdev, !!wufc); *enable_wake = !!wufc; @@ -742,10 +767,10 @@ static int ngbe_probe(struct pci_dev *pdev, wx->wol = 0; if (wx->wol_hw_supported) - wx->wol = NGBE_PSR_WKUP_CTL_MAG; + wx->wol = WX_PSR_WKUP_CTL_MAG; netdev->ethtool->wol_enabled = !!(wx->wol); - wr32(wx, NGBE_PSR_WKUP_CTL, wx->wol); + wr32(wx, WX_PSR_WKUP_CTL, wx->wol); device_set_wakeup_enable(&pdev->dev, wx->wol); /* Save off EEPROM version number and Option Rom version which @@ -770,9 +795,11 @@ static int ngbe_probe(struct pci_dev *pdev, eth_hw_addr_set(netdev, wx->mac.perm_addr); wx_mac_set_default_filter(wx, wx->mac.perm_addr); + ngbe_init_service(wx); + err = wx_init_interrupt_scheme(wx); if (err) - goto err_free_mac_table; + goto err_cancel_service; /* phy Interface Configuration */ err = ngbe_mdio_init(wx); @@ -792,6 +819,9 @@ err_register: wx_control_hw(wx, false); err_clear_interrupt_scheme: wx_clear_interrupt_scheme(wx); +err_cancel_service: + timer_delete_sync(&wx->service_timer); + cancel_work_sync(&wx->service_task); err_free_mac_table: kfree(wx->rss_key); kfree(wx->mac_table); @@ -820,6 +850,10 @@ static void ngbe_remove(struct pci_dev *pdev) netdev = wx->netdev; wx_disable_sriov(wx); unregister_netdev(netdev); + + timer_shutdown_sync(&wx->service_timer); + cancel_work_sync(&wx->service_task); + phylink_destroy(wx->phylink); pci_release_selected_regions(pdev, pci_select_bars(pdev, IORESOURCE_MEM)); diff --git a/drivers/net/ethernet/wangxun/ngbe/ngbe_type.h b/drivers/net/ethernet/wangxun/ngbe/ngbe_type.h index 3b2ca7f47e33..7077a0da4c98 100644 --- a/drivers/net/ethernet/wangxun/ngbe/ngbe_type.h +++ b/drivers/net/ethernet/wangxun/ngbe/ngbe_type.h @@ -37,11 +37,6 @@ #define NGBE_OEM_MASK 0x00FF -#define NGBE_NCSI_SUP 0x8000 -#define NGBE_NCSI_MASK 0x8000 -#define NGBE_WOL_SUP 0x4000 -#define NGBE_WOL_MASK 0x4000 - /**************** EM Registers ****************************/ /* chip control Registers */ #define NGBE_MIS_PRB_CTL 0x10010 @@ -93,18 +88,6 @@ #define NGBE_CFG_LAN_SPEED 0x14440 #define NGBE_CFG_PORT_ST 0x14404 -/* Wake up registers */ -#define NGBE_PSR_WKUP_CTL 0x15B80 -/* Wake Up Filter Control Bit */ -#define NGBE_PSR_WKUP_CTL_LNKC BIT(0) /* Link Status Change Wakeup Enable*/ -#define NGBE_PSR_WKUP_CTL_MAG BIT(1) /* Magic Packet Wakeup Enable */ -#define NGBE_PSR_WKUP_CTL_EX BIT(2) /* Directed Exact Wakeup Enable */ -#define NGBE_PSR_WKUP_CTL_MC BIT(3) /* Directed Multicast Wakeup Enable*/ -#define NGBE_PSR_WKUP_CTL_BC BIT(4) /* Broadcast Wakeup Enable */ -#define NGBE_PSR_WKUP_CTL_ARP BIT(5) /* ARP Request Packet Wakeup Enable*/ -#define NGBE_PSR_WKUP_CTL_IPV4 BIT(6) /* Directed IPv4 Pkt Wakeup Enable */ -#define NGBE_PSR_WKUP_CTL_IPV6 BIT(7) /* Directed IPv6 Pkt Wakeup Enable */ - #define NGBE_FW_EEPROM_CHECKSUM_CMD 0xE9 #define NGBE_FW_NVM_DATA_OFFSET 3 #define NGBE_FW_CMD_DEFAULT_CHECKSUM 0xFF /* checksum always 0xFF */ diff --git a/drivers/net/ethernet/wangxun/txgbe/txgbe_ethtool.c b/drivers/net/ethernet/wangxun/txgbe/txgbe_ethtool.c index 9157b8275be1..3e32aca72806 100644 --- a/drivers/net/ethernet/wangxun/txgbe/txgbe_ethtool.c +++ b/drivers/net/ethernet/wangxun/txgbe/txgbe_ethtool.c @@ -56,9 +56,8 @@ static int txgbe_set_ringparam(struct net_device *netdev, new_rx_count == wx->rx_ring_count) return 0; - err = wx_set_state_reset(wx); - if (err) - return err; + mutex_lock(&wx->reset_lock); + set_bit(WX_STATE_RESETTING, wx->state); if (!netif_running(wx->netdev)) { for (i = 0; i < wx->num_tx_queues; i++) @@ -88,22 +87,10 @@ static int txgbe_set_ringparam(struct net_device *netdev, clear_reset: clear_bit(WX_STATE_RESETTING, wx->state); + mutex_unlock(&wx->reset_lock); return err; } -static int txgbe_set_channels(struct net_device *dev, - struct ethtool_channels *ch) -{ - int err; - - err = wx_set_channels(dev, ch); - if (err < 0) - return err; - - /* use setup TC to update any traffic class queue mapping */ - return txgbe_setup_tc(dev, netdev_get_num_tc(dev)); -} - static int txgbe_get_ethtool_fdir_entry(struct txgbe *txgbe, struct ethtool_rxnfc *cmd) { @@ -587,7 +574,7 @@ static const struct ethtool_ops txgbe_ethtool_ops = { .get_coalesce = wx_get_coalesce, .set_coalesce = wx_set_coalesce, .get_channels = wx_get_channels, - .set_channels = txgbe_set_channels, + .set_channels = wx_set_channels, .get_rxnfc = txgbe_get_rxnfc, .set_rxnfc = txgbe_set_rxnfc, .get_rx_ring_count = txgbe_get_rx_ring_count, diff --git a/drivers/net/ethernet/wangxun/txgbe/txgbe_main.c b/drivers/net/ethernet/wangxun/txgbe/txgbe_main.c index 0de051450a82..ec32a5f422f2 100644 --- a/drivers/net/ethernet/wangxun/txgbe/txgbe_main.c +++ b/drivers/net/ethernet/wangxun/txgbe/txgbe_main.c @@ -130,6 +130,7 @@ static void txgbe_service_task(struct work_struct *work) txgbe_module_detection_subtask(wx); txgbe_link_config_subtask(wx); + wx_update_stats(wx); wx_service_event_complete(wx); } @@ -598,20 +599,16 @@ int txgbe_setup_tc(struct net_device *dev, u8 tc) static void txgbe_reinit_locked(struct wx *wx) { - int err = 0; - netif_trans_update(wx->netdev); - err = wx_set_state_reset(wx); - if (err) { - wx_err(wx, "wait device reset timeout\n"); - return; - } + mutex_lock(&wx->reset_lock); + set_bit(WX_STATE_RESETTING, wx->state); txgbe_down(wx); txgbe_up(wx); clear_bit(WX_STATE_RESETTING, wx->state); + mutex_unlock(&wx->reset_lock); } void txgbe_do_reset(struct net_device *netdev) @@ -950,12 +947,13 @@ static void txgbe_remove(struct pci_dev *pdev) struct txgbe *txgbe = wx->priv; struct net_device *netdev; - cancel_work_sync(&wx->service_task); - netdev = wx->netdev; wx_disable_sriov(wx); unregister_netdev(netdev); + timer_shutdown_sync(&wx->service_timer); + cancel_work_sync(&wx->service_task); + txgbe_remove_phy(txgbe); wx_free_isb_resources(wx); diff --git a/drivers/net/ethernet/wangxun/txgbe/txgbe_phy.c b/drivers/net/ethernet/wangxun/txgbe/txgbe_phy.c index 8ea7aa07ae4e..dc9f24314658 100644 --- a/drivers/net/ethernet/wangxun/txgbe/txgbe_phy.c +++ b/drivers/net/ethernet/wangxun/txgbe/txgbe_phy.c @@ -657,7 +657,9 @@ void txgbe_remove_phy(struct txgbe *txgbe) return; case wx_mac_sp: if (txgbe->wx->media_type == wx_media_copper) { + rtnl_lock(); phylink_disconnect_phy(txgbe->wx->phylink); + rtnl_unlock(); phylink_destroy(txgbe->wx->phylink); return; } |