summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--drivers/vdpa/mlx5/core/mlx5_vdpa.h2
-rw-r--r--drivers/vdpa/mlx5/core/mr.c8
-rw-r--r--drivers/vdpa/mlx5/core/resources.c11
-rw-r--r--drivers/vdpa/vdpa_sim/vdpa_sim.c10
-rw-r--r--drivers/vdpa/vdpa_user/iova_domain.c11
-rw-r--r--drivers/vhost/iotlb.c47
-rw-r--r--drivers/vhost/scsi.c20
-rw-r--r--drivers/vhost/vdpa.c18
-rw-r--r--drivers/vhost/vhost.c16
-rw-r--r--kernel/futex/core.c85
10 files changed, 177 insertions, 51 deletions
diff --git a/drivers/vdpa/mlx5/core/mlx5_vdpa.h b/drivers/vdpa/mlx5/core/mlx5_vdpa.h
index 2cedf7e2dbc4..42f2f44b383c 100644
--- a/drivers/vdpa/mlx5/core/mlx5_vdpa.h
+++ b/drivers/vdpa/mlx5/core/mlx5_vdpa.h
@@ -11,6 +11,8 @@
#define MLX5V_ETH_HARD_MTU (ETH_HLEN + VLAN_HLEN + ETH_FCS_LEN)
+extern int mlx5_vdpa_max_iotlb_entries;
+
struct mlx5_vdpa_direct_mr {
u64 start;
u64 end;
diff --git a/drivers/vdpa/mlx5/core/mr.c b/drivers/vdpa/mlx5/core/mr.c
index 6d02ccf9eb91..77a479aeaa85 100644
--- a/drivers/vdpa/mlx5/core/mr.c
+++ b/drivers/vdpa/mlx5/core/mr.c
@@ -233,7 +233,8 @@ static int create_direct_keys(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_mr *
cmds[i].out = cmd_mem->out;
cmds[i].outlen = sizeof(cmd_mem->out);
cmds[i].in = cmd_mem->in;
- cmds[i].inlen = struct_size(cmd_mem, mtt, mttcount);
+ cmds[i].inlen = struct_size(cmd_mem, mtt, mttcount) -
+ offsetof(struct mlx5_create_mkey_mem, in);
fill_create_direct_mr(mvdev, dmr, cmd_mem);
@@ -776,6 +777,9 @@ static int _mlx5_vdpa_create_mr(struct mlx5_vdpa_dev *mvdev,
{
int err;
+ if (mlx5_vdpa_max_iotlb_entries < 2)
+ return -EINVAL;
+
if (iotlb)
err = create_user_mr(mvdev, mr, iotlb);
else
@@ -784,7 +788,7 @@ static int _mlx5_vdpa_create_mr(struct mlx5_vdpa_dev *mvdev,
if (err)
return err;
- mr->iotlb = vhost_iotlb_alloc(0, 0);
+ mr->iotlb = vhost_iotlb_alloc(mlx5_vdpa_max_iotlb_entries, 0);
if (!mr->iotlb) {
err = -ENOMEM;
goto err_mr;
diff --git a/drivers/vdpa/mlx5/core/resources.c b/drivers/vdpa/mlx5/core/resources.c
index aeae31d0cefa..28a4d7a35bf4 100644
--- a/drivers/vdpa/mlx5/core/resources.c
+++ b/drivers/vdpa/mlx5/core/resources.c
@@ -3,8 +3,14 @@
#include <linux/iova.h>
#include <linux/mlx5/driver.h>
+#include <linux/moduleparam.h>
#include "mlx5_vdpa.h"
+int mlx5_vdpa_max_iotlb_entries = 2048;
+module_param_named(max_iotlb_entries, mlx5_vdpa_max_iotlb_entries, int, 0444);
+MODULE_PARM_DESC(max_iotlb_entries,
+ "Maximum number of iotlb entries. (default: 2048)");
+
static int alloc_pd(struct mlx5_vdpa_dev *dev, u32 *pdn, u16 uid)
{
struct mlx5_core_dev *mdev = dev->mdev;
@@ -229,7 +235,10 @@ int mlx5_vdpa_destroy_mkey(struct mlx5_vdpa_dev *mvdev, u32 mkey)
static int init_ctrl_vq(struct mlx5_vdpa_dev *mvdev)
{
- mvdev->cvq.iotlb = vhost_iotlb_alloc(0, 0);
+ if (mlx5_vdpa_max_iotlb_entries < 2)
+ return -EINVAL;
+
+ mvdev->cvq.iotlb = vhost_iotlb_alloc(mlx5_vdpa_max_iotlb_entries, 0);
if (!mvdev->cvq.iotlb)
return -ENOMEM;
diff --git a/drivers/vdpa/vdpa_sim/vdpa_sim.c b/drivers/vdpa/vdpa_sim/vdpa_sim.c
index 8cb1cc2ea139..4d116644851d 100644
--- a/drivers/vdpa/vdpa_sim/vdpa_sim.c
+++ b/drivers/vdpa/vdpa_sim/vdpa_sim.c
@@ -34,7 +34,7 @@ MODULE_PARM_DESC(batch_mapping, "Batched mapping 1 -Enable; 0 - Disable");
static int max_iotlb_entries = 2048;
module_param(max_iotlb_entries, int, 0444);
MODULE_PARM_DESC(max_iotlb_entries,
- "Maximum number of iotlb entries for each address space. 0 means unlimited. (default: 2048)");
+ "Maximum number of iotlb entries for each address space. (default: 2048)");
static bool use_va = true;
module_param(use_va, bool, 0444);
@@ -201,6 +201,8 @@ struct vdpasim *vdpasim_create(struct vdpasim_dev_attr *dev_attr,
if (!dev_attr->alloc_size)
return ERR_PTR(-EINVAL);
+ if (max_iotlb_entries < 2)
+ return ERR_PTR(-EINVAL);
if (config->mask & BIT_ULL(VDPA_ATTR_DEV_FEATURES)) {
if (config->device_features &
@@ -261,8 +263,10 @@ struct vdpasim *vdpasim_create(struct vdpasim_dev_attr *dev_attr,
for (i = 0; i < vdpasim->dev_attr.nas; i++) {
vhost_iotlb_init(&vdpasim->iommu[i], max_iotlb_entries, 0);
- vhost_iotlb_add_range(&vdpasim->iommu[i], 0, ULONG_MAX, 0,
- VHOST_MAP_RW);
+ ret = vhost_iotlb_add_range(&vdpasim->iommu[i], 0, ULONG_MAX,
+ 0, VHOST_MAP_RW);
+ if (ret)
+ goto err_iommu;
vdpasim->iommu_pt[i] = true;
}
diff --git a/drivers/vdpa/vdpa_user/iova_domain.c b/drivers/vdpa/vdpa_user/iova_domain.c
index 4dc76c0d0d13..b6c958224b7c 100644
--- a/drivers/vdpa/vdpa_user/iova_domain.c
+++ b/drivers/vdpa/vdpa_user/iova_domain.c
@@ -12,11 +12,17 @@
#include <linux/file.h>
#include <linux/anon_inodes.h>
#include <linux/highmem.h>
+#include <linux/moduleparam.h>
#include <linux/vmalloc.h>
#include <linux/vdpa.h>
#include "iova_domain.h"
+static int max_iotlb_entries = 2048;
+module_param(max_iotlb_entries, int, 0444);
+MODULE_PARM_DESC(max_iotlb_entries,
+ "Maximum number of iotlb entries. (default: 2048)");
+
static int vduse_iotlb_add_range(struct vduse_iova_domain *domain,
u64 start, u64 last,
u64 addr, unsigned int perm,
@@ -622,11 +628,14 @@ vduse_domain_create(unsigned long iova_limit, size_t bounce_size)
if (iova_limit <= bounce_size)
return NULL;
+ if (max_iotlb_entries <= 0)
+ return NULL;
+
domain = kzalloc_obj(*domain);
if (!domain)
return NULL;
- domain->iotlb = vhost_iotlb_alloc(0, 0);
+ domain->iotlb = vhost_iotlb_alloc(max_iotlb_entries, 0);
if (!domain->iotlb)
goto err_iotlb;
diff --git a/drivers/vhost/iotlb.c b/drivers/vhost/iotlb.c
index e1414c774c34..a1d4376a5b87 100644
--- a/drivers/vhost/iotlb.c
+++ b/drivers/vhost/iotlb.c
@@ -20,6 +20,14 @@ INTERVAL_TREE_DEFINE(struct vhost_iotlb_map,
rb, __u64, __subtree_last,
START, LAST, static inline, vhost_iotlb_itree);
+static void vhost_iotlb_map_unlink(struct vhost_iotlb *iotlb,
+ struct vhost_iotlb_map *map)
+{
+ vhost_iotlb_itree_remove(map, &iotlb->root);
+ list_del(&map->link);
+ iotlb->nmaps--;
+}
+
/**
* vhost_iotlb_map_free - remove a map node and free it
* @iotlb: the IOTLB
@@ -28,10 +36,8 @@ INTERVAL_TREE_DEFINE(struct vhost_iotlb_map,
void vhost_iotlb_map_free(struct vhost_iotlb *iotlb,
struct vhost_iotlb_map *map)
{
- vhost_iotlb_itree_remove(map, &iotlb->root);
- list_del(&map->link);
+ vhost_iotlb_map_unlink(iotlb, map);
kfree(map);
- iotlb->nmaps--;
}
EXPORT_SYMBOL_GPL(vhost_iotlb_map_free);
@@ -57,14 +63,25 @@ int vhost_iotlb_add_range_ctx(struct vhost_iotlb *iotlb,
if (last < start)
return -EFAULT;
+ if (!iotlb->limit)
+ return -EINVAL;
+
/* If the range being mapped is [0, ULONG_MAX], split it into two entries
* otherwise its size would overflow u64.
*/
if (start == 0 && last == ULONG_MAX) {
u64 mid = last / 2;
- int err = vhost_iotlb_add_range_ctx(iotlb, start, mid, addr,
- perm, opaque);
+ int err;
+
+ if (iotlb->limit < 2)
+ return -ENOSPC;
+ if (!(iotlb->flags & VHOST_IOTLB_FLAG_RETIRE) &&
+ iotlb->nmaps > iotlb->limit - 2)
+ return -ENOSPC;
+
+ err = vhost_iotlb_add_range_ctx(iotlb, start, mid, addr,
+ perm, opaque);
if (err)
return err;
@@ -72,17 +89,19 @@ int vhost_iotlb_add_range_ctx(struct vhost_iotlb *iotlb,
start = mid + 1;
}
- if (iotlb->limit &&
- iotlb->nmaps == iotlb->limit &&
- iotlb->flags & VHOST_IOTLB_FLAG_RETIRE) {
- map = list_first_entry(&iotlb->list, typeof(*map), link);
- vhost_iotlb_map_free(iotlb, map);
+ if (iotlb->nmaps >= iotlb->limit) {
+ if (iotlb->flags & VHOST_IOTLB_FLAG_RETIRE) {
+ map = list_first_entry(&iotlb->list, typeof(*map), link);
+ vhost_iotlb_map_unlink(iotlb, map);
+ } else {
+ return -ENOSPC;
+ }
+ } else {
+ map = kmalloc_obj(*map, GFP_ATOMIC);
+ if (!map)
+ return -ENOMEM;
}
- map = kmalloc_obj(*map, GFP_ATOMIC);
- if (!map)
- return -ENOMEM;
-
map->start = start;
map->size = last - start + 1;
map->last = last;
diff --git a/drivers/vhost/scsi.c b/drivers/vhost/scsi.c
index 9a1253b9d8c5..7a1f39a327da 100644
--- a/drivers/vhost/scsi.c
+++ b/drivers/vhost/scsi.c
@@ -972,6 +972,9 @@ vhost_scsi_mapal(struct vhost_scsi *vs, struct vhost_scsi_cmd *cmd,
if (prot_bytes) {
sgl_count = vhost_scsi_calc_sgls(prot_iter, prot_bytes,
VHOST_SCSI_PREALLOC_PROT_SGLS);
+ if (sgl_count < 0)
+ return sgl_count;
+
cmd->prot_table.sgl = cmd->prot_sgl;
ret = sg_alloc_table_chained(&cmd->prot_table, sgl_count,
cmd->prot_table.sgl,
@@ -1416,6 +1419,11 @@ vhost_scsi_handle_vq(struct vhost_scsi *vs, struct vhost_virtqueue *vq)
* actual data payload length.
*/
if (prot_bytes) {
+ if (prot_bytes >= exp_data_len) {
+ vq_err(vq, "Protection data exceeds payload length\n");
+ goto err;
+ }
+
exp_data_len -= prot_bytes;
prot_iter = data_iter;
iov_iter_truncate(&prot_iter, prot_bytes);
@@ -2219,6 +2227,7 @@ static int vhost_scsi_set_features(struct vhost_scsi *vs, u64 features)
{
struct vhost_virtqueue *vq;
bool is_log, was_log;
+ u64 old_features;
int i;
if (features & ~VHOST_SCSI_FEATURES)
@@ -2234,6 +2243,14 @@ static int vhost_scsi_set_features(struct vhost_scsi *vs, u64 features)
if (!vs->dev.nvqs)
goto out;
+ old_features = vs->vqs[0].vq.acked_features;
+ if (vs->vs_tpg &&
+ ((features ^ old_features) &
+ ~(1ULL << VHOST_F_LOG_ALL))) {
+ mutex_unlock(&vs->dev.mutex);
+ return -EBUSY;
+ }
+
is_log = features & (1 << VHOST_F_LOG_ALL);
/*
* All VQs should have same feature.
@@ -2426,9 +2443,10 @@ vhost_scsi_ioctl(struct file *f,
default:
mutex_lock(&vs->dev.mutex);
r = vhost_dev_ioctl(&vs->dev, ioctl, argp);
- /* TODO: flush backend after dev ioctl. */
if (r == -ENOIOCTLCMD)
r = vhost_vring_ioctl(&vs->dev, ioctl, argp);
+ else
+ vhost_scsi_flush(vs);
mutex_unlock(&vs->dev.mutex);
return r;
}
diff --git a/drivers/vhost/vdpa.c b/drivers/vhost/vdpa.c
index ac55275fa0d0..c3d913bd7cac 100644
--- a/drivers/vhost/vdpa.c
+++ b/drivers/vhost/vdpa.c
@@ -34,6 +34,11 @@ enum {
#define VHOST_VDPA_DEV_MAX (1U << MINORBITS)
+static int max_iotlb_entries = 2048;
+module_param(max_iotlb_entries, int, 0444);
+MODULE_PARM_DESC(max_iotlb_entries,
+ "Maximum number of iotlb entries. (default: 2048)");
+
#define VHOST_VDPA_IOTLB_BUCKETS 16
struct vhost_vdpa_as {
@@ -109,12 +114,14 @@ static struct vhost_vdpa_as *vhost_vdpa_alloc_as(struct vhost_vdpa *v, u32 asid)
if (asid >= v->vdpa->nas)
return NULL;
+ if (max_iotlb_entries <= 0)
+ return NULL;
as = kmalloc_obj(*as);
if (!as)
return NULL;
- vhost_iotlb_init(&as->iotlb, 0, 0);
+ vhost_iotlb_init(&as->iotlb, max_iotlb_entries, 0);
as->id = asid;
hlist_add_head(&as->hash_link, head);
@@ -1102,6 +1109,7 @@ static int vhost_vdpa_pa_map(struct vhost_vdpa *v,
unsigned int gup_flags = FOLL_LONGTERM;
unsigned long npages, cur_base, map_pfn, last_pfn = 0;
unsigned long lock_limit, sz2pin, nchunks, i;
+ unsigned long page_offset;
u64 start = iova;
long pinned;
int ret = 0;
@@ -1114,7 +1122,13 @@ static int vhost_vdpa_pa_map(struct vhost_vdpa *v,
if (perm & VHOST_ACCESS_WO)
gup_flags |= FOLL_WRITE;
- npages = PFN_UP(size + (iova & ~PAGE_MASK));
+ page_offset = iova & ~PAGE_MASK;
+ if (size > ULONG_MAX - page_offset) {
+ ret = -EINVAL;
+ goto free;
+ }
+
+ npages = PFN_UP(size + page_offset);
if (!npages) {
ret = -EINVAL;
goto free;
diff --git a/drivers/vhost/vhost.c b/drivers/vhost/vhost.c
index 4c525b3e16ea..269efad90369 100644
--- a/drivers/vhost/vhost.c
+++ b/drivers/vhost/vhost.c
@@ -1137,6 +1137,9 @@ EXPORT_SYMBOL_GPL(vhost_dev_set_owner);
static struct vhost_iotlb *iotlb_alloc(void)
{
+ if (max_iotlb_entries <= 0)
+ return NULL;
+
return vhost_iotlb_alloc(max_iotlb_entries,
VHOST_IOTLB_FLAG_RETIRE);
}
@@ -1981,6 +1984,8 @@ static long vhost_set_memory(struct vhost_dev *d, struct vhost_memory __user *m)
return -EOPNOTSUPP;
if (mem.nregions > max_mem_regions)
return -E2BIG;
+ if (max_iotlb_entries <= 0)
+ return -EINVAL;
newmem = kvzalloc_flex(*newmem, regions, mem.nregions);
if (!newmem)
return -ENOMEM;
@@ -2126,6 +2131,14 @@ static long vhost_vring_set_num_addr(struct vhost_dev *d,
BUG();
}
+ /*
+ * The metadata cache holds the IOTLB mapping that backed the previous
+ * desc/avail/used addresses and vring size, both of which are being
+ * replaced here. iotlb_access_ok() takes a cache hit as proof that the
+ * region was validated, so the stale entries have to go.
+ */
+ __vhost_vq_meta_reset(vq);
+
mutex_unlock(&vq->mutex);
return r;
@@ -2275,6 +2288,9 @@ int vhost_init_device_iotlb(struct vhost_dev *d)
struct vhost_iotlb *niotlb, *oiotlb;
int i;
+ if (max_iotlb_entries <= 0)
+ return -EINVAL;
+
niotlb = iotlb_alloc();
if (!niotlb)
return -ENOMEM;
diff --git a/kernel/futex/core.c b/kernel/futex/core.c
index 179b26e9c934..2650d1e52803 100644
--- a/kernel/futex/core.c
+++ b/kernel/futex/core.c
@@ -982,8 +982,11 @@ retry:
return -1;
/*
- * Special case for regular (non PI) futexes. The unlock path in
- * user space has two race scenarios:
+ * Special case for regular (non PI) futexes. Ordinarily, we do
+ * not perform any processing here unless the current thread was
+ * the owner of the futex (by the TID check below).
+ *
+ * However, the unlock path has three race scenarios:
*
* 1. The unlock path releases the user space futex value and
* before it can execute the futex() syscall to wake up
@@ -992,42 +995,70 @@ retry:
* 2. A woken up waiter is killed before it can acquire the
* futex in user space.
*
- * In the second case, the wake up notification could be generated
- * by the unlock path in user space after setting the futex value
- * to zero or by the kernel after setting the OWNER_DIED bit below.
+ * 3. A woken up waiter is killed in user space after another
+ * thread has acquired the futex, but before it can set
+ * FUTEX_WAITERS.
+ *
+ * Note that, if userspace uses the FUTEX_ROBUST_UNLOCK flag, we
+ * will not see case 1 here.
+ *
+ * In the second and third case, the wake up notification could
+ * be generated from any of:
+ *
+ * i. An ordinary futex wakeup after unlock (with or
+ * without FUTEX_ROBUST_UNLOCK)
+ * ii. A robust wakeup from another thread's death
+ * iii. A previous round through this special case
+ *
+ * As a result, the futex world will be in one of four states:
+ *
+ * A. The futex word is 0 (unlocked)
+ * B. The futex word is owned by another thread
+ * (FUTEX_WAITERS is not set)
+ * C. The futex word is owned by another thread
+ * (FUTEX_WAITERS set)
+ * D. The futex's owner died and OWNER_DIED is set
+ * (the owner part of the word is 0)
*
- * In both cases the TID validation below prevents a wakeup of
- * potential waiters which can cause these waiters to block
- * forever.
+ * The key issue is that the kernel usually (at least from
+ * sources ii. and iii. or when so requested by userspace from
+ * source i.) only ever wakes *one* waiter at a time. If this
+ * waiter dies before acquiring the futex (or setting the
+ * FUTEX_WAITERS bit), the kernel *must* still wake the next
+ * waiter down the line to uphold the futex invariants and
+ * avoid lost wakeups. Note we do not need to handle state C,
+ * as it does not matter to us whether *we* successfully set
+ * the bit or a third thread did so in the meantime.
*
- * In both cases the following conditions are met:
+ * Therefore, in these cases we must issue an additional
+ * futex_wake(). Note however that we *must not* set OWNER_DIED
+ * here. Our thread is *not* the owner of the futex.
*
- * 1) task->futex.robust_list->list_op_pending != NULL
- * @pending_op == true
- * 2) The owner part of user space futex value == 0
+ * Thus to summarize, the conditions for needing the additional
+ * futex_wake() are:
+ *
+ * 1) @pending_op == true (the thread has not finished the
+ * mutex operation)
+ * 2) The futex word is in one of the states A, B or D
* 3) Regular futex: @pi == false
*
- * If these conditions are met, it is safe to attempt waking up a
- * potential waiter without touching the user space futex value and
- * trying to set the OWNER_DIED bit. If the futex value is zero,
- * the rest of the user space mutex state is consistent, so a woken
- * waiter will just take over the uncontended futex. Setting the
- * OWNER_DIED bit would create inconsistent state and malfunction
- * of the user space owner died handling. Otherwise, the OWNER_DIED
- * bit is already set, and the woken waiter is expected to deal with
- * this.
+ * Note in particular that in all of the states A-D the owner
+ * portion of the futex word differs from our thread's TID
+ * (unless the actual owner has the same TID in another PID
+ * namespace, but we cannot currently distinguish that
+ * scenario), so this can be a special-case wakeup in the bail
+ * path of the ordinary TID check.
*/
owner = uval & FUTEX_TID_MASK;
- if (pending_op && !pi && !owner) {
- futex_wake(uaddr, FLAGS_SIZE_32 | FLAGS_SHARED, NULL, 1,
- FUTEX_BITSET_MATCH_ANY);
+ if (owner != task_pid_vnr(curr)) {
+ if (pending_op && !pi && (!owner || !(uval & FUTEX_WAITERS))) {
+ futex_wake(uaddr, FLAGS_SIZE_32 | FLAGS_SHARED, NULL, 1,
+ FUTEX_BITSET_MATCH_ANY);
+ }
return 0;
}
- if (owner != task_pid_vnr(curr))
- return 0;
-
/*
* Ok, this dying thread is truly holding a futex
* of interest. Set the OWNER_DIED bit atomically
generated by cgit v1.2.3 (git 2.0.4) at 2026-08-05 02:13:37 +0000