1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
|
// SPDX-License-Identifier: GPL-2.0-only
/* Copyright (C) 2025 Intel Corporation */
#define DEFAULT_SYMBOL_NAMESPACE "LIBETH_XDP"
#include <linux/export.h>
#include <net/libeth/xdp.h>
/* ``XDP_TX`` bulking */
static void __cold
libeth_xdp_tx_return_one(const struct libeth_xdp_tx_frame *frm)
{
if (frm->len_fl & LIBETH_XDP_TX_MULTI)
libeth_xdp_return_frags(frm->data + frm->soff, true);
libeth_xdp_return_va(frm->data, true);
}
static void __cold
libeth_xdp_tx_return_bulk(const struct libeth_xdp_tx_frame *bq, u32 count)
{
for (u32 i = 0; i < count; i++) {
const struct libeth_xdp_tx_frame *frm = &bq[i];
if (!(frm->len_fl & LIBETH_XDP_TX_FIRST))
continue;
libeth_xdp_tx_return_one(frm);
}
}
static void __cold libeth_trace_xdp_exception(const struct net_device *dev,
const struct bpf_prog *prog,
u32 act)
{
trace_xdp_exception(dev, prog, act);
}
/**
* libeth_xdp_tx_exception - handle Tx exceptions of XDP frames
* @bq: XDP Tx frame bulk
* @sent: number of frames sent successfully (from this bulk)
* @flags: internal libeth_xdp flags
*
* Cold helper used by __libeth_xdp_tx_flush_bulk(), do not call directly.
* Reports XDP Tx exceptions, frees the frames that won't be sent or adjust
* the Tx bulk to try again later.
*/
void __cold libeth_xdp_tx_exception(struct libeth_xdp_tx_bulk *bq, u32 sent,
u32 flags)
{
const struct libeth_xdp_tx_frame *pos = &bq->bulk[sent];
u32 left = bq->count - sent;
libeth_trace_xdp_exception(bq->dev, bq->prog, XDP_TX);
if (!(flags & LIBETH_XDP_TX_DROP)) {
memmove(bq->bulk, pos, left * sizeof(*bq->bulk));
bq->count = left;
return;
}
libeth_xdp_tx_return_bulk(pos, left);
bq->count = 0;
}
EXPORT_SYMBOL_GPL(libeth_xdp_tx_exception);
/* Rx polling path */
/**
* libeth_xdp_return_buff_slow - free &libeth_xdp_buff
* @xdp: buffer to free/return
*
* Slowpath version of libeth_xdp_return_buff() to be called on exceptions,
* queue clean-ups etc., without unwanted inlining.
*/
void __cold libeth_xdp_return_buff_slow(struct libeth_xdp_buff *xdp)
{
__libeth_xdp_return_buff(xdp, false);
}
EXPORT_SYMBOL_GPL(libeth_xdp_return_buff_slow);
MODULE_DESCRIPTION("Common Ethernet library - XDP infra");
MODULE_IMPORT_NS("LIBETH");
MODULE_LICENSE("GPL");
|
