diff options
Diffstat (limited to 'drivers/block')
41 files changed, 5647 insertions, 1513 deletions
diff --git a/drivers/block/drbd/Makefile b/drivers/block/drbd/Makefile index 67a8b352a1d5..5faaa8a8e7f0 100644 --- a/drivers/block/drbd/Makefile +++ b/drivers/block/drbd/Makefile @@ -3,7 +3,7 @@ drbd-y := drbd_buildtag.o drbd_bitmap.o drbd_proc.o drbd-y += drbd_worker.o drbd_receiver.o drbd_req.o drbd_actlog.o drbd-y += drbd_main.o drbd_strings.o drbd_nl.o drbd-y += drbd_interval.o drbd_state.o -drbd-y += drbd_nla.o +drbd-y += drbd_nl_gen.o drbd-$(CONFIG_DEBUG_FS) += drbd_debugfs.o obj-$(CONFIG_BLK_DEV_DRBD) += drbd.o diff --git a/drivers/block/drbd/drbd_buildtag.c b/drivers/block/drbd/drbd_buildtag.c index cb1aa66d7d5d..cd0389488f63 100644 --- a/drivers/block/drbd/drbd_buildtag.c +++ b/drivers/block/drbd/drbd_buildtag.c @@ -1,5 +1,5 @@ // SPDX-License-Identifier: GPL-2.0-only -#include <linux/drbd_config.h> +#include "drbd_config.h" #include <linux/module.h> const char *drbd_buildtag(void) diff --git a/drivers/block/drbd/drbd_config.h b/drivers/block/drbd/drbd_config.h new file mode 100644 index 000000000000..d215365c6bb1 --- /dev/null +++ b/drivers/block/drbd/drbd_config.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * drbd_config.h + * DRBD's compile time configuration. + */ + +#ifndef DRBD_CONFIG_H +#define DRBD_CONFIG_H + +extern const char *drbd_buildtag(void); + +#define REL_VERSION "8.4.11" +#define PRO_VERSION_MIN 86 +#define PRO_VERSION_MAX 101 + +#endif diff --git a/drivers/block/drbd/drbd_debugfs.c b/drivers/block/drbd/drbd_debugfs.c index 12460b584bcb..371abcd7e880 100644 --- a/drivers/block/drbd/drbd_debugfs.c +++ b/drivers/block/drbd/drbd_debugfs.c @@ -844,7 +844,7 @@ static int drbd_version_show(struct seq_file *m, void *ignored) { seq_printf(m, "# %s\n", drbd_buildtag()); seq_printf(m, "VERSION=%s\n", REL_VERSION); - seq_printf(m, "API_VERSION=%u\n", GENL_MAGIC_VERSION); + seq_printf(m, "API_VERSION=%u\n", DRBD_FAMILY_VERSION); seq_printf(m, "PRO_VERSION_MIN=%u\n", PRO_VERSION_MIN); seq_printf(m, "PRO_VERSION_MAX=%u\n", PRO_VERSION_MAX); return 0; diff --git a/drivers/block/drbd/drbd_int.h b/drivers/block/drbd/drbd_int.h index f6d6276974ee..48b45c3142f7 100644 --- a/drivers/block/drbd/drbd_int.h +++ b/drivers/block/drbd/drbd_int.h @@ -32,14 +32,16 @@ #include <net/tcp.h> #include <linux/lru_cache.h> #include <linux/prefetch.h> -#include <linux/drbd_genl_api.h> #include <linux/drbd.h> -#include <linux/drbd_config.h> +#include "drbd_config.h" +#include "drbd_nl_gen.h" #include "drbd_strings.h" #include "drbd_state.h" #include "drbd_protocol.h" #include "drbd_polymorph_printk.h" +extern struct genl_family drbd_nl_family; + /* shared module parameters, defined in drbd_main.c */ #ifdef CONFIG_DRBD_FAULT_INJECTION extern int drbd_enable_faults; diff --git a/drivers/block/drbd/drbd_main.c b/drivers/block/drbd/drbd_main.c index 200d464e984b..a2a841c89201 100644 --- a/drivers/block/drbd/drbd_main.c +++ b/drivers/block/drbd/drbd_main.c @@ -874,7 +874,7 @@ void drbd_gen_and_send_sync_uuid(struct drbd_peer_device *peer_device) if (uuid && uuid != UUID_JUST_CREATED) uuid = uuid + UUID_NEW_BM_OFFSET; else - get_random_bytes(&uuid, sizeof(u64)); + uuid = get_random_u64(); drbd_uuid_set(device, UI_BITMAP, uuid); drbd_print_uuids(device, "updated sync UUID"); drbd_md_sync(device); @@ -2324,7 +2324,7 @@ static void drbd_cleanup(void) if (retry.wq) destroy_workqueue(retry.wq); - drbd_genl_unregister(); + genl_unregister_family(&drbd_nl_family); idr_for_each_entry(&drbd_devices, device, i) drbd_delete_device(device); @@ -2846,7 +2846,7 @@ static int __init drbd_init(void) mutex_init(&resources_mutex); INIT_LIST_HEAD(&drbd_resources); - err = drbd_genl_register(); + err = genl_register_family(&drbd_nl_family); if (err) { pr_err("unable to register generic netlink family\n"); goto fail; @@ -2876,7 +2876,7 @@ static int __init drbd_init(void) pr_info("initialized. " "Version: " REL_VERSION " (api:%d/proto:%d-%d)\n", - GENL_MAGIC_VERSION, PRO_VERSION_MIN, PRO_VERSION_MAX); + DRBD_FAMILY_VERSION, PRO_VERSION_MIN, PRO_VERSION_MAX); pr_info("%s\n", drbd_buildtag()); pr_info("registered as block device major %d\n", DRBD_MAJOR); return 0; /* Success! */ @@ -3337,7 +3337,7 @@ void drbd_uuid_new_current(struct drbd_device *device) __must_hold(local) u64 val; unsigned long long bm_uuid; - get_random_bytes(&val, sizeof(u64)); + val = get_random_u64(); spin_lock_irq(&device->ldev->md.uuid_lock); bm_uuid = device->ldev->md.uuid[UI_BITMAP]; diff --git a/drivers/block/drbd/drbd_nl.c b/drivers/block/drbd/drbd_nl.c index e201f0087a0f..b77f901fc3ef 100644 --- a/drivers/block/drbd/drbd_nl.c +++ b/drivers/block/drbd/drbd_nl.c @@ -31,51 +31,13 @@ #include <net/genetlink.h> -/* .doit */ -// int drbd_adm_create_resource(struct sk_buff *skb, struct genl_info *info); -// int drbd_adm_delete_resource(struct sk_buff *skb, struct genl_info *info); - -int drbd_adm_new_minor(struct sk_buff *skb, struct genl_info *info); -int drbd_adm_del_minor(struct sk_buff *skb, struct genl_info *info); - -int drbd_adm_new_resource(struct sk_buff *skb, struct genl_info *info); -int drbd_adm_del_resource(struct sk_buff *skb, struct genl_info *info); -int drbd_adm_down(struct sk_buff *skb, struct genl_info *info); - -int drbd_adm_set_role(struct sk_buff *skb, struct genl_info *info); -int drbd_adm_attach(struct sk_buff *skb, struct genl_info *info); -int drbd_adm_disk_opts(struct sk_buff *skb, struct genl_info *info); -int drbd_adm_detach(struct sk_buff *skb, struct genl_info *info); -int drbd_adm_connect(struct sk_buff *skb, struct genl_info *info); -int drbd_adm_net_opts(struct sk_buff *skb, struct genl_info *info); -int drbd_adm_resize(struct sk_buff *skb, struct genl_info *info); -int drbd_adm_start_ov(struct sk_buff *skb, struct genl_info *info); -int drbd_adm_new_c_uuid(struct sk_buff *skb, struct genl_info *info); -int drbd_adm_disconnect(struct sk_buff *skb, struct genl_info *info); -int drbd_adm_invalidate(struct sk_buff *skb, struct genl_info *info); -int drbd_adm_invalidate_peer(struct sk_buff *skb, struct genl_info *info); -int drbd_adm_pause_sync(struct sk_buff *skb, struct genl_info *info); -int drbd_adm_resume_sync(struct sk_buff *skb, struct genl_info *info); -int drbd_adm_suspend_io(struct sk_buff *skb, struct genl_info *info); -int drbd_adm_resume_io(struct sk_buff *skb, struct genl_info *info); -int drbd_adm_outdate(struct sk_buff *skb, struct genl_info *info); -int drbd_adm_resource_opts(struct sk_buff *skb, struct genl_info *info); -int drbd_adm_get_status(struct sk_buff *skb, struct genl_info *info); -int drbd_adm_get_timeout_type(struct sk_buff *skb, struct genl_info *info); -/* .dumpit */ -int drbd_adm_get_status_all(struct sk_buff *skb, struct netlink_callback *cb); -int drbd_adm_dump_resources(struct sk_buff *skb, struct netlink_callback *cb); -int drbd_adm_dump_devices(struct sk_buff *skb, struct netlink_callback *cb); -int drbd_adm_dump_devices_done(struct netlink_callback *cb); -int drbd_adm_dump_connections(struct sk_buff *skb, struct netlink_callback *cb); -int drbd_adm_dump_connections_done(struct netlink_callback *cb); -int drbd_adm_dump_peer_devices(struct sk_buff *skb, struct netlink_callback *cb); -int drbd_adm_dump_peer_devices_done(struct netlink_callback *cb); -int drbd_adm_get_initial_state(struct sk_buff *skb, struct netlink_callback *cb); - -#include <linux/drbd_genl_api.h> -#include "drbd_nla.h" -#include <linux/genl_magic_func.h> +#include "drbd_nl_gen.h" + +static int drbd_genl_multicast_events(struct sk_buff *skb, gfp_t flags) +{ + return genlmsg_multicast(&drbd_nl_family, skb, 0, + DRBD_NLGRP_EVENTS, flags); +} static atomic_t drbd_genl_seq = ATOMIC_INIT(2); /* two. */ static atomic_t notify_genl_seq = ATOMIC_INIT(2); /* two. */ @@ -106,7 +68,7 @@ static int drbd_msg_put_info(struct sk_buff *skb, const char *info) if (!nla) return err; - err = nla_put_string(skb, T_info_text, info); + err = nla_put_string(skb, DRBD_A_DRBD_CFG_REPLY_INFO_TEXT, info); if (err) { nla_nest_cancel(skb, nla); return err; @@ -127,7 +89,7 @@ static int drbd_msg_sprintf_info(struct sk_buff *skb, const char *fmt, ...) if (!nla) return err; - txt = nla_reserve(skb, T_info_text, 256); + txt = nla_reserve(skb, DRBD_A_DRBD_CFG_REPLY_INFO_TEXT, 256); if (!txt) { nla_nest_cancel(skb, nla); return err; @@ -144,18 +106,55 @@ static int drbd_msg_sprintf_info(struct sk_buff *skb, const char *fmt, ...) return 0; } -/* This would be a good candidate for a "pre_doit" hook, - * and per-family private info->pointers. - * But we need to stay compatible with older kernels. - * If it returns successfully, adm_ctx members are valid. - * +/* Flags for drbd_adm_prepare() */ +#define DRBD_ADM_NEED_MINOR (1 << 0) +#define DRBD_ADM_NEED_RESOURCE (1 << 1) +#define DRBD_ADM_NEED_CONNECTION (1 << 2) + +/* Per-command flags for drbd_pre_doit() */ +static const unsigned int drbd_genl_cmd_flags[] = { + [DRBD_ADM_GET_STATUS] = DRBD_ADM_NEED_MINOR, + [DRBD_ADM_NEW_MINOR] = DRBD_ADM_NEED_RESOURCE, + [DRBD_ADM_DEL_MINOR] = DRBD_ADM_NEED_MINOR, + [DRBD_ADM_NEW_RESOURCE] = 0, + [DRBD_ADM_DEL_RESOURCE] = DRBD_ADM_NEED_RESOURCE, + [DRBD_ADM_RESOURCE_OPTS] = DRBD_ADM_NEED_RESOURCE, + [DRBD_ADM_CONNECT] = DRBD_ADM_NEED_RESOURCE, + [DRBD_ADM_CHG_NET_OPTS] = DRBD_ADM_NEED_CONNECTION, + [DRBD_ADM_DISCONNECT] = DRBD_ADM_NEED_CONNECTION, + [DRBD_ADM_ATTACH] = DRBD_ADM_NEED_MINOR, + [DRBD_ADM_CHG_DISK_OPTS] = DRBD_ADM_NEED_MINOR, + [DRBD_ADM_RESIZE] = DRBD_ADM_NEED_MINOR, + [DRBD_ADM_PRIMARY] = DRBD_ADM_NEED_MINOR, + [DRBD_ADM_SECONDARY] = DRBD_ADM_NEED_MINOR, + [DRBD_ADM_NEW_C_UUID] = DRBD_ADM_NEED_MINOR, + [DRBD_ADM_START_OV] = DRBD_ADM_NEED_MINOR, + [DRBD_ADM_DETACH] = DRBD_ADM_NEED_MINOR, + [DRBD_ADM_INVALIDATE] = DRBD_ADM_NEED_MINOR, + [DRBD_ADM_INVAL_PEER] = DRBD_ADM_NEED_MINOR, + [DRBD_ADM_PAUSE_SYNC] = DRBD_ADM_NEED_MINOR, + [DRBD_ADM_RESUME_SYNC] = DRBD_ADM_NEED_MINOR, + [DRBD_ADM_SUSPEND_IO] = DRBD_ADM_NEED_MINOR, + [DRBD_ADM_RESUME_IO] = DRBD_ADM_NEED_MINOR, + [DRBD_ADM_OUTDATE] = DRBD_ADM_NEED_MINOR, + [DRBD_ADM_GET_TIMEOUT_TYPE] = DRBD_ADM_NEED_MINOR, + [DRBD_ADM_DOWN] = DRBD_ADM_NEED_RESOURCE, +}; + +/* Detect attempts to change invariant attributes in a _change_ handler. */ +#define has_invariant(ntb, attr) \ +({ \ + bool __found = !!(ntb)[attr]; \ + if (__found) \ + pr_info("must not change invariant attr: %s\n", #attr); \ + __found; \ +}) + +/* * At this point, we still rely on the global genl_lock(). * If we want to avoid that, and allow "genl_family.parallel_ops", we may need * to add additional synchronization against object destruction/modification. */ -#define DRBD_ADM_NEED_MINOR 1 -#define DRBD_ADM_NEED_RESOURCE 2 -#define DRBD_ADM_NEED_CONNECTION 4 static int drbd_adm_prepare(struct drbd_config_context *adm_ctx, struct sk_buff *skb, struct genl_info *info, unsigned flags) { @@ -163,8 +162,6 @@ static int drbd_adm_prepare(struct drbd_config_context *adm_ctx, const u8 cmd = info->genlhdr->cmd; int err; - memset(adm_ctx, 0, sizeof(*adm_ctx)); - /* genl_rcv_msg only checks for CAP_NET_ADMIN on "GENL_ADMIN_PERM" :( */ if (cmd != DRBD_ADM_GET_STATUS && !capable(CAP_NET_ADMIN)) return -EPERM; @@ -176,7 +173,7 @@ static int drbd_adm_prepare(struct drbd_config_context *adm_ctx, } adm_ctx->reply_dh = genlmsg_put_reply(adm_ctx->reply_skb, - info, &drbd_genl_family, 0, cmd); + info, &drbd_nl_family, 0, cmd); /* put of a few bytes into a fresh skb of >= 4k will always succeed. * but anyways */ if (!adm_ctx->reply_dh) { @@ -189,9 +186,11 @@ static int drbd_adm_prepare(struct drbd_config_context *adm_ctx, adm_ctx->volume = VOLUME_UNSPECIFIED; if (info->attrs[DRBD_NLA_CFG_CONTEXT]) { + struct nlattr **ntb; struct nlattr *nla; - /* parse and validate only */ - err = drbd_cfg_context_from_attrs(NULL, info); + + /* parse and validate, get nested attribute table */ + err = drbd_cfg_context_ntb_from_attrs(&ntb, info); if (err) goto fail; @@ -200,18 +199,21 @@ static int drbd_adm_prepare(struct drbd_config_context *adm_ctx, err = nla_put_nohdr(adm_ctx->reply_skb, info->attrs[DRBD_NLA_CFG_CONTEXT]->nla_len, info->attrs[DRBD_NLA_CFG_CONTEXT]); - if (err) + if (err) { + kfree(ntb); goto fail; + } /* and assign stuff to the adm_ctx */ - nla = nested_attr_tb[__nla_type(T_ctx_volume)]; + nla = ntb[DRBD_A_DRBD_CFG_CONTEXT_CTX_VOLUME]; if (nla) adm_ctx->volume = nla_get_u32(nla); - nla = nested_attr_tb[__nla_type(T_ctx_resource_name)]; + nla = ntb[DRBD_A_DRBD_CFG_CONTEXT_CTX_RESOURCE_NAME]; if (nla) adm_ctx->resource_name = nla_data(nla); - adm_ctx->my_addr = nested_attr_tb[__nla_type(T_ctx_my_addr)]; - adm_ctx->peer_addr = nested_attr_tb[__nla_type(T_ctx_peer_addr)]; + adm_ctx->my_addr = ntb[DRBD_A_DRBD_CFG_CONTEXT_CTX_MY_ADDR]; + adm_ctx->peer_addr = ntb[DRBD_A_DRBD_CFG_CONTEXT_CTX_PEER_ADDR]; + kfree(ntb); if ((adm_ctx->my_addr && nla_len(adm_ctx->my_addr) > sizeof(adm_ctx->connection->my_addr)) || (adm_ctx->peer_addr && @@ -225,7 +227,7 @@ static int drbd_adm_prepare(struct drbd_config_context *adm_ctx, adm_ctx->device = minor_to_device(d_in->minor); /* We are protected by the global genl_lock(). - * But we may explicitly drop it/retake it in drbd_adm_set_role(), + * But we may explicitly drop it/retake it in drbd_nl_set_role(), * so make sure this object stays around. */ if (adm_ctx->device) kref_get(&adm_ctx->device->kref); @@ -300,9 +302,45 @@ fail: return err; } -static int drbd_adm_finish(struct drbd_config_context *adm_ctx, - struct genl_info *info, int retcode) +int drbd_pre_doit(const struct genl_split_ops *ops, + struct sk_buff *skb, struct genl_info *info) +{ + struct drbd_config_context *adm_ctx; + u8 cmd = info->genlhdr->cmd; + unsigned int flags; + int err; + + adm_ctx = kzalloc_obj(*adm_ctx); + if (!adm_ctx) + return -ENOMEM; + + flags = (cmd < ARRAY_SIZE(drbd_genl_cmd_flags)) + ? drbd_genl_cmd_flags[cmd] : 0; + + err = drbd_adm_prepare(adm_ctx, skb, info, flags); + if (err && !adm_ctx->reply_skb) { + /* Fatal error before reply_skb was allocated. */ + kfree(adm_ctx); + return err; + } + if (err) + adm_ctx->reply_dh->ret_code = err; + + info->user_ptr[0] = adm_ctx; + return 0; +} + +void drbd_post_doit(const struct genl_split_ops *ops, + struct sk_buff *skb, struct genl_info *info) { + struct drbd_config_context *adm_ctx = info->user_ptr[0]; + + if (!adm_ctx) + return; + + if (adm_ctx->reply_skb) + drbd_adm_send_reply(adm_ctx->reply_skb, info); + if (adm_ctx->device) { kref_put(&adm_ctx->device->kref, drbd_destroy_device); adm_ctx->device = NULL; @@ -316,12 +354,7 @@ static int drbd_adm_finish(struct drbd_config_context *adm_ctx, adm_ctx->resource = NULL; } - if (!adm_ctx->reply_skb) - return -ENOMEM; - - adm_ctx->reply_dh->ret_code = retcode; - drbd_adm_send_reply(adm_ctx->reply_skb, info); - return 0; + kfree(adm_ctx); } static void setup_khelper_env(struct drbd_connection *connection, char **envp) @@ -759,22 +792,21 @@ out: static const char *from_attrs_err_to_txt(int err) { return err == -ENOMSG ? "required attribute missing" : - err == -EOPNOTSUPP ? "unknown mandatory attribute" : err == -EEXIST ? "can not change invariant setting" : "invalid attribute value"; } -int drbd_adm_set_role(struct sk_buff *skb, struct genl_info *info) +static int drbd_nl_set_role(struct sk_buff *skb, struct genl_info *info) { - struct drbd_config_context adm_ctx; + struct drbd_config_context *adm_ctx = info->user_ptr[0]; struct set_role_parms parms; int err; enum drbd_ret_code retcode; enum drbd_state_rv rv; - retcode = drbd_adm_prepare(&adm_ctx, skb, info, DRBD_ADM_NEED_MINOR); - if (!adm_ctx.reply_skb) - return retcode; + if (!adm_ctx->reply_skb) + return 0; + retcode = adm_ctx->reply_dh->ret_code; if (retcode != NO_ERROR) goto out; @@ -783,27 +815,37 @@ int drbd_adm_set_role(struct sk_buff *skb, struct genl_info *info) err = set_role_parms_from_attrs(&parms, info); if (err) { retcode = ERR_MANDATORY_TAG; - drbd_msg_put_info(adm_ctx.reply_skb, from_attrs_err_to_txt(err)); + drbd_msg_put_info(adm_ctx->reply_skb, from_attrs_err_to_txt(err)); goto out; } } genl_unlock(); - mutex_lock(&adm_ctx.resource->adm_mutex); + mutex_lock(&adm_ctx->resource->adm_mutex); if (info->genlhdr->cmd == DRBD_ADM_PRIMARY) - rv = drbd_set_role(adm_ctx.device, R_PRIMARY, parms.assume_uptodate); + rv = drbd_set_role(adm_ctx->device, R_PRIMARY, parms.assume_uptodate); else - rv = drbd_set_role(adm_ctx.device, R_SECONDARY, 0); + rv = drbd_set_role(adm_ctx->device, R_SECONDARY, 0); - mutex_unlock(&adm_ctx.resource->adm_mutex); + mutex_unlock(&adm_ctx->resource->adm_mutex); genl_lock(); - drbd_adm_finish(&adm_ctx, info, rv); + adm_ctx->reply_dh->ret_code = rv; return 0; out: - drbd_adm_finish(&adm_ctx, info, retcode); + adm_ctx->reply_dh->ret_code = retcode; return 0; } +int drbd_nl_primary_doit(struct sk_buff *skb, struct genl_info *info) +{ + return drbd_nl_set_role(skb, info); +} + +int drbd_nl_secondary_doit(struct sk_buff *skb, struct genl_info *info) +{ + return drbd_nl_set_role(skb, info); +} + /* Initializes the md.*_offset members, so we are able to find * the on disk meta data. * @@ -898,7 +940,7 @@ char *ppsize(char *buf, unsigned long long size) * peer may not initiate a resize. */ /* Note these are not to be confused with - * drbd_adm_suspend_io/drbd_adm_resume_io, + * drbd_nl_suspend_io_doit/drbd_nl_resume_io_doit, * which are (sub) state changes triggered by admin (drbdsetup), * and can be long lived. * This changes an device->flag, is triggered by drbd internals, @@ -1510,24 +1552,25 @@ out: return err; } -int drbd_adm_disk_opts(struct sk_buff *skb, struct genl_info *info) +int drbd_nl_chg_disk_opts_doit(struct sk_buff *skb, struct genl_info *info) { - struct drbd_config_context adm_ctx; + struct drbd_config_context *adm_ctx = info->user_ptr[0]; enum drbd_ret_code retcode; struct drbd_device *device; struct disk_conf *new_disk_conf, *old_disk_conf; struct fifo_buffer *old_plan = NULL, *new_plan = NULL; + struct nlattr **ntb; int err; unsigned int fifo_size; - retcode = drbd_adm_prepare(&adm_ctx, skb, info, DRBD_ADM_NEED_MINOR); - if (!adm_ctx.reply_skb) - return retcode; + if (!adm_ctx->reply_skb) + return 0; + retcode = adm_ctx->reply_dh->ret_code; if (retcode != NO_ERROR) goto finish; - device = adm_ctx.device; - mutex_lock(&adm_ctx.resource->adm_mutex); + device = adm_ctx->device; + mutex_lock(&adm_ctx->resource->adm_mutex); /* we also need a disk * to change the options on */ @@ -1548,13 +1591,29 @@ int drbd_adm_disk_opts(struct sk_buff *skb, struct genl_info *info) if (should_set_defaults(info)) set_disk_conf_defaults(new_disk_conf); - err = disk_conf_from_attrs_for_change(new_disk_conf, info); + err = disk_conf_from_attrs(new_disk_conf, info); if (err && err != -ENOMSG) { retcode = ERR_MANDATORY_TAG; - drbd_msg_put_info(adm_ctx.reply_skb, from_attrs_err_to_txt(err)); + drbd_msg_put_info(adm_ctx->reply_skb, from_attrs_err_to_txt(err)); goto fail_unlock; } + err = disk_conf_ntb_from_attrs(&ntb, info); + if (!err) { + if (has_invariant(ntb, DRBD_A_DISK_CONF_BACKING_DEV) || + has_invariant(ntb, DRBD_A_DISK_CONF_META_DEV) || + has_invariant(ntb, DRBD_A_DISK_CONF_META_DEV_IDX) || + has_invariant(ntb, DRBD_A_DISK_CONF_DISK_SIZE) || + has_invariant(ntb, DRBD_A_DISK_CONF_MAX_BIO_BVECS)) { + retcode = ERR_MANDATORY_TAG; + drbd_msg_put_info(adm_ctx->reply_skb, + "cannot change invariant setting"); + kfree(ntb); + goto fail_unlock; + } + kfree(ntb); + } + if (!expect(device, new_disk_conf->resync_rate >= 1)) new_disk_conf->resync_rate = 1; @@ -1577,7 +1636,7 @@ int drbd_adm_disk_opts(struct sk_buff *skb, struct genl_info *info) if (err) { /* Could be just "busy". Ignore? * Introduce dedicated error code? */ - drbd_msg_put_info(adm_ctx.reply_skb, + drbd_msg_put_info(adm_ctx->reply_skb, "Try again without changing current al-extents setting"); retcode = ERR_NOMEM; goto fail_unlock; @@ -1640,9 +1699,9 @@ fail_unlock: success: put_ldev(device); out: - mutex_unlock(&adm_ctx.resource->adm_mutex); + mutex_unlock(&adm_ctx->resource->adm_mutex); finish: - drbd_adm_finish(&adm_ctx, info, retcode); + adm_ctx->reply_dh->ret_code = retcode; return 0; } @@ -1732,9 +1791,9 @@ void drbd_backing_dev_free(struct drbd_device *device, struct drbd_backing_dev * kfree(ldev); } -int drbd_adm_attach(struct sk_buff *skb, struct genl_info *info) +int drbd_nl_attach_doit(struct sk_buff *skb, struct genl_info *info) { - struct drbd_config_context adm_ctx; + struct drbd_config_context *adm_ctx = info->user_ptr[0]; struct drbd_device *device; struct drbd_peer_device *peer_device; struct drbd_connection *connection; @@ -1751,14 +1810,14 @@ int drbd_adm_attach(struct sk_buff *skb, struct genl_info *info) enum drbd_state_rv rv; struct net_conf *nc; - retcode = drbd_adm_prepare(&adm_ctx, skb, info, DRBD_ADM_NEED_MINOR); - if (!adm_ctx.reply_skb) - return retcode; + if (!adm_ctx->reply_skb) + return 0; + retcode = adm_ctx->reply_dh->ret_code; if (retcode != NO_ERROR) goto finish; - device = adm_ctx.device; - mutex_lock(&adm_ctx.resource->adm_mutex); + device = adm_ctx->device; + mutex_lock(&adm_ctx->resource->adm_mutex); peer_device = first_peer_device(device); connection = peer_device->connection; conn_reconfig_start(connection); @@ -1803,7 +1862,7 @@ int drbd_adm_attach(struct sk_buff *skb, struct genl_info *info) err = disk_conf_from_attrs(new_disk_conf, info); if (err) { retcode = ERR_MANDATORY_TAG; - drbd_msg_put_info(adm_ctx.reply_skb, from_attrs_err_to_txt(err)); + drbd_msg_put_info(adm_ctx->reply_skb, from_attrs_err_to_txt(err)); goto fail; } @@ -1954,7 +2013,7 @@ int drbd_adm_attach(struct sk_buff *skb, struct genl_info *info) drbd_warn(device, "truncating a consistent device during attach (%llu < %llu)\n", nsz, eff); } else { drbd_warn(device, "refusing to truncate a consistent device (%llu < %llu)\n", nsz, eff); - drbd_msg_sprintf_info(adm_ctx.reply_skb, + drbd_msg_sprintf_info(adm_ctx->reply_skb, "To-be-attached device has last effective > current size, and is consistent\n" "(%llu > %llu sectors). Refusing to attach.", eff, nsz); retcode = ERR_IMPLICIT_SHRINK; @@ -2130,8 +2189,8 @@ int drbd_adm_attach(struct sk_buff *skb, struct genl_info *info) kobject_uevent(&disk_to_dev(device->vdisk)->kobj, KOBJ_CHANGE); put_ldev(device); conn_reconfig_done(connection); - mutex_unlock(&adm_ctx.resource->adm_mutex); - drbd_adm_finish(&adm_ctx, info, retcode); + mutex_unlock(&adm_ctx->resource->adm_mutex); + adm_ctx->reply_dh->ret_code = retcode; return 0; force_diskless_dec: @@ -2150,9 +2209,9 @@ int drbd_adm_attach(struct sk_buff *skb, struct genl_info *info) kfree(new_disk_conf); lc_destroy(resync_lru); kfree(new_plan); - mutex_unlock(&adm_ctx.resource->adm_mutex); + mutex_unlock(&adm_ctx->resource->adm_mutex); finish: - drbd_adm_finish(&adm_ctx, info, retcode); + adm_ctx->reply_dh->ret_code = retcode; return 0; } @@ -2172,16 +2231,16 @@ static int adm_detach(struct drbd_device *device, int force) * Then we transition to D_DISKLESS, and wait for put_ldev() to return all * internal references as well. * Only then we have finally detached. */ -int drbd_adm_detach(struct sk_buff *skb, struct genl_info *info) +int drbd_nl_detach_doit(struct sk_buff *skb, struct genl_info *info) { - struct drbd_config_context adm_ctx; + struct drbd_config_context *adm_ctx = info->user_ptr[0]; enum drbd_ret_code retcode; struct detach_parms parms = { }; int err; - retcode = drbd_adm_prepare(&adm_ctx, skb, info, DRBD_ADM_NEED_MINOR); - if (!adm_ctx.reply_skb) - return retcode; + if (!adm_ctx->reply_skb) + return 0; + retcode = adm_ctx->reply_dh->ret_code; if (retcode != NO_ERROR) goto out; @@ -2189,16 +2248,16 @@ int drbd_adm_detach(struct sk_buff *skb, struct genl_info *info) err = detach_parms_from_attrs(&parms, info); if (err) { retcode = ERR_MANDATORY_TAG; - drbd_msg_put_info(adm_ctx.reply_skb, from_attrs_err_to_txt(err)); + drbd_msg_put_info(adm_ctx->reply_skb, from_attrs_err_to_txt(err)); goto out; } } - mutex_lock(&adm_ctx.resource->adm_mutex); - retcode = adm_detach(adm_ctx.device, parms.force_detach); - mutex_unlock(&adm_ctx.resource->adm_mutex); + mutex_lock(&adm_ctx->resource->adm_mutex); + retcode = adm_detach(adm_ctx->device, parms.force_detach); + mutex_unlock(&adm_ctx->resource->adm_mutex); out: - drbd_adm_finish(&adm_ctx, info, retcode); + adm_ctx->reply_dh->ret_code = retcode; return 0; } @@ -2370,25 +2429,26 @@ static void free_crypto(struct crypto *crypto) crypto_free_shash(crypto->verify_tfm); } -int drbd_adm_net_opts(struct sk_buff *skb, struct genl_info *info) +int drbd_nl_chg_net_opts_doit(struct sk_buff *skb, struct genl_info *info) { - struct drbd_config_context adm_ctx; + struct drbd_config_context *adm_ctx = info->user_ptr[0]; enum drbd_ret_code retcode; struct drbd_connection *connection; struct net_conf *old_net_conf, *new_net_conf = NULL; + struct nlattr **ntb; int err; int ovr; /* online verify running */ int rsr; /* re-sync running */ struct crypto crypto = { }; - retcode = drbd_adm_prepare(&adm_ctx, skb, info, DRBD_ADM_NEED_CONNECTION); - if (!adm_ctx.reply_skb) - return retcode; + if (!adm_ctx->reply_skb) + return 0; + retcode = adm_ctx->reply_dh->ret_code; if (retcode != NO_ERROR) goto finish; - connection = adm_ctx.connection; - mutex_lock(&adm_ctx.resource->adm_mutex); + connection = adm_ctx->connection; + mutex_lock(&adm_ctx->resource->adm_mutex); new_net_conf = kzalloc_obj(struct net_conf); if (!new_net_conf) { @@ -2403,7 +2463,7 @@ int drbd_adm_net_opts(struct sk_buff *skb, struct genl_info *info) old_net_conf = connection->net_conf; if (!old_net_conf) { - drbd_msg_put_info(adm_ctx.reply_skb, "net conf missing, try connect"); + drbd_msg_put_info(adm_ctx->reply_skb, "net conf missing, try connect"); retcode = ERR_INVALID_REQUEST; goto fail; } @@ -2412,13 +2472,26 @@ int drbd_adm_net_opts(struct sk_buff *skb, struct genl_info *info) if (should_set_defaults(info)) set_net_conf_defaults(new_net_conf); - err = net_conf_from_attrs_for_change(new_net_conf, info); + err = net_conf_from_attrs(new_net_conf, info); if (err && err != -ENOMSG) { retcode = ERR_MANDATORY_TAG; - drbd_msg_put_info(adm_ctx.reply_skb, from_attrs_err_to_txt(err)); + drbd_msg_put_info(adm_ctx->reply_skb, from_attrs_err_to_txt(err)); goto fail; } + err = net_conf_ntb_from_attrs(&ntb, info); + if (!err) { + if (has_invariant(ntb, DRBD_A_NET_CONF_DISCARD_MY_DATA) || + has_invariant(ntb, DRBD_A_NET_CONF_TENTATIVE)) { + retcode = ERR_MANDATORY_TAG; + drbd_msg_put_info(adm_ctx->reply_skb, + "cannot change invariant setting"); + kfree(ntb); + goto fail; + } + kfree(ntb); + } + retcode = check_net_options(connection, new_net_conf); if (retcode != NO_ERROR) goto fail; @@ -2485,9 +2558,9 @@ int drbd_adm_net_opts(struct sk_buff *skb, struct genl_info *info) done: conn_reconfig_done(connection); out: - mutex_unlock(&adm_ctx.resource->adm_mutex); + mutex_unlock(&adm_ctx->resource->adm_mutex); finish: - drbd_adm_finish(&adm_ctx, info, retcode); + adm_ctx->reply_dh->ret_code = retcode; return 0; } @@ -2511,12 +2584,12 @@ static void peer_device_to_info(struct peer_device_info *info, info->peer_resync_susp_dependency = device->state.aftr_isp; } -int drbd_adm_connect(struct sk_buff *skb, struct genl_info *info) +int drbd_nl_connect_doit(struct sk_buff *skb, struct genl_info *info) { struct connection_info connection_info; enum drbd_notification_type flags; unsigned int peer_devices = 0; - struct drbd_config_context adm_ctx; + struct drbd_config_context *adm_ctx = info->user_ptr[0]; struct drbd_peer_device *peer_device; struct net_conf *old_net_conf, *new_net_conf = NULL; struct crypto crypto = { }; @@ -2527,14 +2600,13 @@ int drbd_adm_connect(struct sk_buff *skb, struct genl_info *info) int i; int err; - retcode = drbd_adm_prepare(&adm_ctx, skb, info, DRBD_ADM_NEED_RESOURCE); - - if (!adm_ctx.reply_skb) - return retcode; + if (!adm_ctx->reply_skb) + return 0; + retcode = adm_ctx->reply_dh->ret_code; if (retcode != NO_ERROR) goto out; - if (!(adm_ctx.my_addr && adm_ctx.peer_addr)) { - drbd_msg_put_info(adm_ctx.reply_skb, "connection endpoint(s) missing"); + if (!(adm_ctx->my_addr && adm_ctx->peer_addr)) { + drbd_msg_put_info(adm_ctx->reply_skb, "connection endpoint(s) missing"); retcode = ERR_INVALID_REQUEST; goto out; } @@ -2544,15 +2616,15 @@ int drbd_adm_connect(struct sk_buff *skb, struct genl_info *info) * concurrent reconfiguration/addition/deletion */ for_each_resource(resource, &drbd_resources) { for_each_connection(connection, resource) { - if (nla_len(adm_ctx.my_addr) == connection->my_addr_len && - !memcmp(nla_data(adm_ctx.my_addr), &connection->my_addr, + if (nla_len(adm_ctx->my_addr) == connection->my_addr_len && + !memcmp(nla_data(adm_ctx->my_addr), &connection->my_addr, connection->my_addr_len)) { retcode = ERR_LOCAL_ADDR; goto out; } - if (nla_len(adm_ctx.peer_addr) == connection->peer_addr_len && - !memcmp(nla_data(adm_ctx.peer_addr), &connection->peer_addr, + if (nla_len(adm_ctx->peer_addr) == connection->peer_addr_len && + !memcmp(nla_data(adm_ctx->peer_addr), &connection->peer_addr, connection->peer_addr_len)) { retcode = ERR_PEER_ADDR; goto out; @@ -2560,8 +2632,8 @@ int drbd_adm_connect(struct sk_buff *skb, struct genl_info *info) } } - mutex_lock(&adm_ctx.resource->adm_mutex); - connection = first_connection(adm_ctx.resource); + mutex_lock(&adm_ctx->resource->adm_mutex); + connection = first_connection(adm_ctx->resource); conn_reconfig_start(connection); if (connection->cstate > C_STANDALONE) { @@ -2581,7 +2653,7 @@ int drbd_adm_connect(struct sk_buff *skb, struct genl_info *info) err = net_conf_from_attrs(new_net_conf, info); if (err && err != -ENOMSG) { retcode = ERR_MANDATORY_TAG; - drbd_msg_put_info(adm_ctx.reply_skb, from_attrs_err_to_txt(err)); + drbd_msg_put_info(adm_ctx->reply_skb, from_attrs_err_to_txt(err)); goto fail; } @@ -2597,11 +2669,11 @@ int drbd_adm_connect(struct sk_buff *skb, struct genl_info *info) drbd_flush_workqueue(&connection->sender_work); - mutex_lock(&adm_ctx.resource->conf_update); + mutex_lock(&adm_ctx->resource->conf_update); old_net_conf = connection->net_conf; if (old_net_conf) { retcode = ERR_NET_CONFIGURED; - mutex_unlock(&adm_ctx.resource->conf_update); + mutex_unlock(&adm_ctx->resource->conf_update); goto fail; } rcu_assign_pointer(connection->net_conf, new_net_conf); @@ -2612,10 +2684,10 @@ int drbd_adm_connect(struct sk_buff *skb, struct genl_info *info) connection->csums_tfm = crypto.csums_tfm; connection->verify_tfm = crypto.verify_tfm; - connection->my_addr_len = nla_len(adm_ctx.my_addr); - memcpy(&connection->my_addr, nla_data(adm_ctx.my_addr), connection->my_addr_len); - connection->peer_addr_len = nla_len(adm_ctx.peer_addr); - memcpy(&connection->peer_addr, nla_data(adm_ctx.peer_addr), connection->peer_addr_len); + connection->my_addr_len = nla_len(adm_ctx->my_addr); + memcpy(&connection->my_addr, nla_data(adm_ctx->my_addr), connection->my_addr_len); + connection->peer_addr_len = nla_len(adm_ctx->peer_addr); + memcpy(&connection->peer_addr, nla_data(adm_ctx->peer_addr), connection->peer_addr_len); idr_for_each_entry(&connection->peer_devices, peer_device, i) { peer_devices++; @@ -2633,7 +2705,7 @@ int drbd_adm_connect(struct sk_buff *skb, struct genl_info *info) notify_peer_device_state(NULL, 0, peer_device, &peer_device_info, NOTIFY_CREATE | flags); } mutex_unlock(¬ification_mutex); - mutex_unlock(&adm_ctx.resource->conf_update); + mutex_unlock(&adm_ctx->resource->conf_update); rcu_read_lock(); idr_for_each_entry(&connection->peer_devices, peer_device, i) { @@ -2646,8 +2718,8 @@ int drbd_adm_connect(struct sk_buff *skb, struct genl_info *info) rv = conn_request_state(connection, NS(conn, C_UNCONNECTED), CS_VERBOSE); conn_reconfig_done(connection); - mutex_unlock(&adm_ctx.resource->adm_mutex); - drbd_adm_finish(&adm_ctx, info, rv); + mutex_unlock(&adm_ctx->resource->adm_mutex); + adm_ctx->reply_dh->ret_code = rv; return 0; fail: @@ -2655,9 +2727,9 @@ fail: kfree(new_net_conf); conn_reconfig_done(connection); - mutex_unlock(&adm_ctx.resource->adm_mutex); + mutex_unlock(&adm_ctx->resource->adm_mutex); out: - drbd_adm_finish(&adm_ctx, info, retcode); + adm_ctx->reply_dh->ret_code = retcode; return 0; } @@ -2727,42 +2799,42 @@ repeat: return rv; } -int drbd_adm_disconnect(struct sk_buff *skb, struct genl_info *info) +int drbd_nl_disconnect_doit(struct sk_buff *skb, struct genl_info *info) { - struct drbd_config_context adm_ctx; + struct drbd_config_context *adm_ctx = info->user_ptr[0]; struct disconnect_parms parms; struct drbd_connection *connection; enum drbd_state_rv rv; enum drbd_ret_code retcode; int err; - retcode = drbd_adm_prepare(&adm_ctx, skb, info, DRBD_ADM_NEED_CONNECTION); - if (!adm_ctx.reply_skb) - return retcode; + if (!adm_ctx->reply_skb) + return 0; + retcode = adm_ctx->reply_dh->ret_code; if (retcode != NO_ERROR) goto fail; - connection = adm_ctx.connection; + connection = adm_ctx->connection; memset(&parms, 0, sizeof(parms)); if (info->attrs[DRBD_NLA_DISCONNECT_PARMS]) { err = disconnect_parms_from_attrs(&parms, info); if (err) { retcode = ERR_MANDATORY_TAG; - drbd_msg_put_info(adm_ctx.reply_skb, from_attrs_err_to_txt(err)); + drbd_msg_put_info(adm_ctx->reply_skb, from_attrs_err_to_txt(err)); goto fail; } } - mutex_lock(&adm_ctx.resource->adm_mutex); + mutex_lock(&adm_ctx->resource->adm_mutex); rv = conn_try_disconnect(connection, parms.force_disconnect); - mutex_unlock(&adm_ctx.resource->adm_mutex); + mutex_unlock(&adm_ctx->resource->adm_mutex); if (rv < SS_SUCCESS) { - drbd_adm_finish(&adm_ctx, info, rv); + adm_ctx->reply_dh->ret_code = rv; return 0; } retcode = NO_ERROR; fail: - drbd_adm_finish(&adm_ctx, info, retcode); + adm_ctx->reply_dh->ret_code = retcode; return 0; } @@ -2782,9 +2854,9 @@ void resync_after_online_grow(struct drbd_device *device) _drbd_request_state(device, NS(conn, C_WF_SYNC_UUID), CS_VERBOSE + CS_SERIALIZE); } -int drbd_adm_resize(struct sk_buff *skb, struct genl_info *info) +int drbd_nl_resize_doit(struct sk_buff *skb, struct genl_info *info) { - struct drbd_config_context adm_ctx; + struct drbd_config_context *adm_ctx = info->user_ptr[0]; struct disk_conf *old_disk_conf, *new_disk_conf = NULL; struct resize_parms rs; struct drbd_device *device; @@ -2795,14 +2867,14 @@ int drbd_adm_resize(struct sk_buff *skb, struct genl_info *info) sector_t u_size; int err; - retcode = drbd_adm_prepare(&adm_ctx, skb, info, DRBD_ADM_NEED_MINOR); - if (!adm_ctx.reply_skb) - return retcode; + if (!adm_ctx->reply_skb) + return 0; + retcode = adm_ctx->reply_dh->ret_code; if (retcode != NO_ERROR) goto finish; - mutex_lock(&adm_ctx.resource->adm_mutex); - device = adm_ctx.device; + mutex_lock(&adm_ctx->resource->adm_mutex); + device = adm_ctx->device; if (!get_ldev(device)) { retcode = ERR_NO_DISK; goto fail; @@ -2815,7 +2887,7 @@ int drbd_adm_resize(struct sk_buff *skb, struct genl_info *info) err = resize_parms_from_attrs(&rs, info); if (err) { retcode = ERR_MANDATORY_TAG; - drbd_msg_put_info(adm_ctx.reply_skb, from_attrs_err_to_txt(err)); + drbd_msg_put_info(adm_ctx->reply_skb, from_attrs_err_to_txt(err)); goto fail_ldev; } } @@ -2907,9 +2979,9 @@ int drbd_adm_resize(struct sk_buff *skb, struct genl_info *info) } fail: - mutex_unlock(&adm_ctx.resource->adm_mutex); + mutex_unlock(&adm_ctx->resource->adm_mutex); finish: - drbd_adm_finish(&adm_ctx, info, retcode); + adm_ctx->reply_dh->ret_code = retcode; return 0; fail_ldev: @@ -2918,63 +2990,63 @@ int drbd_adm_resize(struct sk_buff *skb, struct genl_info *info) goto fail; } -int drbd_adm_resource_opts(struct sk_buff *skb, struct genl_info *info) +int drbd_nl_resource_opts_doit(struct sk_buff *skb, struct genl_info *info) { - struct drbd_config_context adm_ctx; + struct drbd_config_context *adm_ctx = info->user_ptr[0]; enum drbd_ret_code retcode; struct res_opts res_opts; int err; - retcode = drbd_adm_prepare(&adm_ctx, skb, info, DRBD_ADM_NEED_RESOURCE); - if (!adm_ctx.reply_skb) - return retcode; + if (!adm_ctx->reply_skb) + return 0; + retcode = adm_ctx->reply_dh->ret_code; if (retcode != NO_ERROR) goto fail; - res_opts = adm_ctx.resource->res_opts; + res_opts = adm_ctx->resource->res_opts; if (should_set_defaults(info)) set_res_opts_defaults(&res_opts); err = res_opts_from_attrs(&res_opts, info); if (err && err != -ENOMSG) { retcode = ERR_MANDATORY_TAG; - drbd_msg_put_info(adm_ctx.reply_skb, from_attrs_err_to_txt(err)); + drbd_msg_put_info(adm_ctx->reply_skb, from_attrs_err_to_txt(err)); goto fail; } - mutex_lock(&adm_ctx.resource->adm_mutex); - err = set_resource_options(adm_ctx.resource, &res_opts); + mutex_lock(&adm_ctx->resource->adm_mutex); + err = set_resource_options(adm_ctx->resource, &res_opts); if (err) { retcode = ERR_INVALID_REQUEST; if (err == -ENOMEM) retcode = ERR_NOMEM; } - mutex_unlock(&adm_ctx.resource->adm_mutex); + mutex_unlock(&adm_ctx->resource->adm_mutex); fail: - drbd_adm_finish(&adm_ctx, info, retcode); + adm_ctx->reply_dh->ret_code = retcode; return 0; } -int drbd_adm_invalidate(struct sk_buff *skb, struct genl_info *info) +int drbd_nl_invalidate_doit(struct sk_buff *skb, struct genl_info *info) { - struct drbd_config_context adm_ctx; + struct drbd_config_context *adm_ctx = info->user_ptr[0]; struct drbd_device *device; int retcode; /* enum drbd_ret_code rsp. enum drbd_state_rv */ - retcode = drbd_adm_prepare(&adm_ctx, skb, info, DRBD_ADM_NEED_MINOR); - if (!adm_ctx.reply_skb) - return retcode; + if (!adm_ctx->reply_skb) + return 0; + retcode = adm_ctx->reply_dh->ret_code; if (retcode != NO_ERROR) goto out; - device = adm_ctx.device; + device = adm_ctx->device; if (!get_ldev(device)) { retcode = ERR_NO_DISK; goto out; } - mutex_lock(&adm_ctx.resource->adm_mutex); + mutex_lock(&adm_ctx->resource->adm_mutex); /* If there is still bitmap IO pending, probably because of a previous * resync just being finished, wait for it before requesting a new resync. @@ -2997,30 +3069,30 @@ int drbd_adm_invalidate(struct sk_buff *skb, struct genl_info *info) } else retcode = drbd_request_state(device, NS(conn, C_STARTING_SYNC_T)); drbd_resume_io(device); - mutex_unlock(&adm_ctx.resource->adm_mutex); + mutex_unlock(&adm_ctx->resource->adm_mutex); put_ldev(device); out: - drbd_adm_finish(&adm_ctx, info, retcode); + adm_ctx->reply_dh->ret_code = retcode; return 0; } static int drbd_adm_simple_request_state(struct sk_buff *skb, struct genl_info *info, union drbd_state mask, union drbd_state val) { - struct drbd_config_context adm_ctx; + struct drbd_config_context *adm_ctx = info->user_ptr[0]; enum drbd_ret_code retcode; - retcode = drbd_adm_prepare(&adm_ctx, skb, info, DRBD_ADM_NEED_MINOR); - if (!adm_ctx.reply_skb) - return retcode; + if (!adm_ctx->reply_skb) + return 0; + retcode = adm_ctx->reply_dh->ret_code; if (retcode != NO_ERROR) goto out; - mutex_lock(&adm_ctx.resource->adm_mutex); - retcode = drbd_request_state(adm_ctx.device, mask, val); - mutex_unlock(&adm_ctx.resource->adm_mutex); + mutex_lock(&adm_ctx->resource->adm_mutex); + retcode = drbd_request_state(adm_ctx->device, mask, val); + mutex_unlock(&adm_ctx->resource->adm_mutex); out: - drbd_adm_finish(&adm_ctx, info, retcode); + adm_ctx->reply_dh->ret_code = retcode; return 0; } @@ -3034,25 +3106,25 @@ static int drbd_bmio_set_susp_al(struct drbd_device *device, return rv; } -int drbd_adm_invalidate_peer(struct sk_buff *skb, struct genl_info *info) +int drbd_nl_inval_peer_doit(struct sk_buff *skb, struct genl_info *info) { - struct drbd_config_context adm_ctx; + struct drbd_config_context *adm_ctx = info->user_ptr[0]; int retcode; /* drbd_ret_code, drbd_state_rv */ struct drbd_device *device; - retcode = drbd_adm_prepare(&adm_ctx, skb, info, DRBD_ADM_NEED_MINOR); - if (!adm_ctx.reply_skb) - return retcode; + if (!adm_ctx->reply_skb) + return 0; + retcode = adm_ctx->reply_dh->ret_code; if (retcode != NO_ERROR) goto out; - device = adm_ctx.device; + device = adm_ctx->device; if (!get_ldev(device)) { retcode = ERR_NO_DISK; goto out; } - mutex_lock(&adm_ctx.resource->adm_mutex); + mutex_lock(&adm_ctx->resource->adm_mutex); /* If there is still bitmap IO pending, probably because of a previous * resync just being finished, wait for it before requesting a new resync. @@ -3078,48 +3150,48 @@ int drbd_adm_invalidate_peer(struct sk_buff *skb, struct genl_info *info) } else retcode = drbd_request_state(device, NS(conn, C_STARTING_SYNC_S)); drbd_resume_io(device); - mutex_unlock(&adm_ctx.resource->adm_mutex); + mutex_unlock(&adm_ctx->resource->adm_mutex); put_ldev(device); out: - drbd_adm_finish(&adm_ctx, info, retcode); + adm_ctx->reply_dh->ret_code = retcode; return 0; } -int drbd_adm_pause_sync(struct sk_buff *skb, struct genl_info *info) +int drbd_nl_pause_sync_doit(struct sk_buff *skb, struct genl_info *info) { - struct drbd_config_context adm_ctx; + struct drbd_config_context *adm_ctx = info->user_ptr[0]; enum drbd_ret_code retcode; - retcode = drbd_adm_prepare(&adm_ctx, skb, info, DRBD_ADM_NEED_MINOR); - if (!adm_ctx.reply_skb) - return retcode; + if (!adm_ctx->reply_skb) + return 0; + retcode = adm_ctx->reply_dh->ret_code; if (retcode != NO_ERROR) goto out; - mutex_lock(&adm_ctx.resource->adm_mutex); - if (drbd_request_state(adm_ctx.device, NS(user_isp, 1)) == SS_NOTHING_TO_DO) + mutex_lock(&adm_ctx->resource->adm_mutex); + if (drbd_request_state(adm_ctx->device, NS(user_isp, 1)) == SS_NOTHING_TO_DO) retcode = ERR_PAUSE_IS_SET; - mutex_unlock(&adm_ctx.resource->adm_mutex); + mutex_unlock(&adm_ctx->resource->adm_mutex); out: - drbd_adm_finish(&adm_ctx, info, retcode); + adm_ctx->reply_dh->ret_code = retcode; return 0; } -int drbd_adm_resume_sync(struct sk_buff *skb, struct genl_info *info) +int drbd_nl_resume_sync_doit(struct sk_buff *skb, struct genl_info *info) { - struct drbd_config_context adm_ctx; + struct drbd_config_context *adm_ctx = info->user_ptr[0]; union drbd_dev_state s; enum drbd_ret_code retcode; - retcode = drbd_adm_prepare(&adm_ctx, skb, info, DRBD_ADM_NEED_MINOR); - if (!adm_ctx.reply_skb) - return retcode; + if (!adm_ctx->reply_skb) + return 0; + retcode = adm_ctx->reply_dh->ret_code; if (retcode != NO_ERROR) goto out; - mutex_lock(&adm_ctx.resource->adm_mutex); - if (drbd_request_state(adm_ctx.device, NS(user_isp, 0)) == SS_NOTHING_TO_DO) { - s = adm_ctx.device->state; + mutex_lock(&adm_ctx->resource->adm_mutex); + if (drbd_request_state(adm_ctx->device, NS(user_isp, 0)) == SS_NOTHING_TO_DO) { + s = adm_ctx->device->state; if (s.conn == C_PAUSED_SYNC_S || s.conn == C_PAUSED_SYNC_T) { retcode = s.aftr_isp ? ERR_PIC_AFTER_DEP : s.peer_isp ? ERR_PIC_PEER_DEP : ERR_PAUSE_IS_CLEAR; @@ -3127,31 +3199,31 @@ int drbd_adm_resume_sync(struct sk_buff *skb, struct genl_info *info) retcode = ERR_PAUSE_IS_CLEAR; } } - mutex_unlock(&adm_ctx.resource->adm_mutex); + mutex_unlock(&adm_ctx->resource->adm_mutex); out: - drbd_adm_finish(&adm_ctx, info, retcode); + adm_ctx->reply_dh->ret_code = retcode; return 0; } -int drbd_adm_suspend_io(struct sk_buff *skb, struct genl_info *info) +int drbd_nl_suspend_io_doit(struct sk_buff *skb, struct genl_info *info) { return drbd_adm_simple_request_state(skb, info, NS(susp, 1)); } -int drbd_adm_resume_io(struct sk_buff *skb, struct genl_info *info) +int drbd_nl_resume_io_doit(struct sk_buff *skb, struct genl_info *info) { - struct drbd_config_context adm_ctx; + struct drbd_config_context *adm_ctx = info->user_ptr[0]; struct drbd_device *device; int retcode; /* enum drbd_ret_code rsp. enum drbd_state_rv */ - retcode = drbd_adm_prepare(&adm_ctx, skb, info, DRBD_ADM_NEED_MINOR); - if (!adm_ctx.reply_skb) - return retcode; + if (!adm_ctx->reply_skb) + return 0; + retcode = adm_ctx->reply_dh->ret_code; if (retcode != NO_ERROR) goto out; - mutex_lock(&adm_ctx.resource->adm_mutex); - device = adm_ctx.device; + mutex_lock(&adm_ctx->resource->adm_mutex); + device = adm_ctx->device; if (test_bit(NEW_CUR_UUID, &device->flags)) { if (get_ldev_if_state(device, D_ATTACHING)) { drbd_uuid_new_current(device); @@ -3173,7 +3245,7 @@ int drbd_adm_resume_io(struct sk_buff *skb, struct genl_info *info) * matching real data uuid exists). */ u64 val; - get_random_bytes(&val, sizeof(u64)); + val = get_random_u64(); drbd_set_ed_uuid(device, val); drbd_warn(device, "Resumed without access to data; please tear down before attempting to re-configure.\n"); } @@ -3188,13 +3260,13 @@ int drbd_adm_resume_io(struct sk_buff *skb, struct genl_info *info) tl_restart(first_peer_device(device)->connection, FAIL_FROZEN_DISK_IO); } drbd_resume_io(device); - mutex_unlock(&adm_ctx.resource->adm_mutex); + mutex_unlock(&adm_ctx->resource->adm_mutex); out: - drbd_adm_finish(&adm_ctx, info, retcode); + adm_ctx->reply_dh->ret_code = retcode; return 0; } -int drbd_adm_outdate(struct sk_buff *skb, struct genl_info *info) +int drbd_nl_outdate_doit(struct sk_buff *skb, struct genl_info *info) { return drbd_adm_simple_request_state(skb, info, NS(disk, D_OUTDATED)); } @@ -3209,16 +3281,20 @@ static int nla_put_drbd_cfg_context(struct sk_buff *skb, if (!nla) goto nla_put_failure; if (device && - nla_put_u32(skb, T_ctx_volume, device->vnr)) + nla_put_u32(skb, DRBD_A_DRBD_CFG_CONTEXT_CTX_VOLUME, device->vnr)) goto nla_put_failure; - if (nla_put_string(skb, T_ctx_resource_name, resource->name)) + if (nla_put_string(skb, DRBD_A_DRBD_CFG_CONTEXT_CTX_RESOURCE_NAME, resource->name)) goto nla_put_failure; if (connection) { if (connection->my_addr_len && - nla_put(skb, T_ctx_my_addr, connection->my_addr_len, &connection->my_addr)) + nla_put(skb, DRBD_A_DRBD_CFG_CONTEXT_CTX_MY_ADDR, + connection->my_addr_len, + &connection->my_addr)) goto nla_put_failure; if (connection->peer_addr_len && - nla_put(skb, T_ctx_peer_addr, connection->peer_addr_len, &connection->peer_addr)) + nla_put(skb, DRBD_A_DRBD_CFG_CONTEXT_CTX_PEER_ADDR, + connection->peer_addr_len, + &connection->peer_addr)) goto nla_put_failure; } nla_nest_end(skb, nla); @@ -3231,26 +3307,45 @@ nla_put_failure: } /* + * net_conf_to_skb() serializes the shared secret verbatim. Any path that can + * answer a request from an unprivileged process must pass exclude_sensitive, + * so the secret is blanked in a private copy before it reaches the skb. + */ +static int net_conf_to_skb_sanitized(struct sk_buff *skb, struct net_conf *nc, + bool exclude_sensitive) +{ + struct net_conf nc_clean; + + if (!exclude_sensitive) + return net_conf_to_skb(skb, nc); + + nc_clean = *nc; + memset(nc_clean.shared_secret, 0, sizeof(nc_clean.shared_secret)); + nc_clean.shared_secret_len = 0; + + return net_conf_to_skb(skb, &nc_clean); +} + +/* * The generic netlink dump callbacks are called outside the genl_lock(), so * they cannot use the simple attribute parsing code which uses global * attribute tables. */ static struct nlattr *find_cfg_context_attr(const struct nlmsghdr *nlh, int attr) { - const unsigned hdrlen = GENL_HDRLEN + GENL_MAGIC_FAMILY_HDRSZ; - const int maxtype = ARRAY_SIZE(drbd_cfg_context_nl_policy) - 1; + const unsigned int hdrlen = GENL_HDRLEN + sizeof(struct drbd_genlmsghdr); struct nlattr *nla; nla = nla_find(nlmsg_attrdata(nlh, hdrlen), nlmsg_attrlen(nlh, hdrlen), DRBD_NLA_CFG_CONTEXT); if (!nla) return NULL; - return drbd_nla_find_nested(maxtype, nla, __nla_type(attr)); + return nla_find_nested(nla, attr); } static void resource_to_info(struct resource_info *, struct drbd_resource *); -int drbd_adm_dump_resources(struct sk_buff *skb, struct netlink_callback *cb) +int drbd_nl_get_resources_dumpit(struct sk_buff *skb, struct netlink_callback *cb) { struct drbd_genlmsghdr *dh; struct drbd_resource *resource; @@ -3278,7 +3373,7 @@ found_resource: put_result: dh = genlmsg_put(skb, NETLINK_CB(cb->skb).portid, - cb->nlh->nlmsg_seq, &drbd_genl_family, + cb->nlh->nlmsg_seq, &drbd_nl_family, NLM_F_MULTI, DRBD_ADM_GET_RESOURCES); err = -ENOMEM; if (!dh) @@ -3288,15 +3383,15 @@ put_result: err = nla_put_drbd_cfg_context(skb, resource, NULL, NULL); if (err) goto out; - err = res_opts_to_skb(skb, &resource->res_opts, !capable(CAP_SYS_ADMIN)); + err = res_opts_to_skb(skb, &resource->res_opts); if (err) goto out; resource_to_info(&resource_info, resource); - err = resource_info_to_skb(skb, &resource_info, !capable(CAP_SYS_ADMIN)); + err = resource_info_to_skb(skb, &resource_info); if (err) goto out; resource_statistics.res_stat_write_ordering = resource->write_ordering; - err = resource_statistics_to_skb(skb, &resource_statistics, !capable(CAP_SYS_ADMIN)); + err = resource_statistics_to_skb(skb, &resource_statistics); if (err) goto out; cb->args[0] = (long)resource; @@ -3361,7 +3456,7 @@ int drbd_adm_dump_devices_done(struct netlink_callback *cb) { static void device_to_info(struct device_info *, struct drbd_device *); -int drbd_adm_dump_devices(struct sk_buff *skb, struct netlink_callback *cb) +int drbd_nl_get_devices_dumpit(struct sk_buff *skb, struct netlink_callback *cb) { struct nlattr *resource_filter; struct drbd_resource *resource; @@ -3374,12 +3469,15 @@ int drbd_adm_dump_devices(struct sk_buff *skb, struct netlink_callback *cb) resource = (struct drbd_resource *)cb->args[0]; if (!cb->args[0] && !cb->args[1]) { - resource_filter = find_cfg_context_attr(cb->nlh, T_ctx_resource_name); + resource_filter = find_cfg_context_attr(cb->nlh, + DRBD_A_DRBD_CFG_CONTEXT_CTX_RESOURCE_NAME); if (resource_filter) { retcode = ERR_RES_NOT_KNOWN; resource = drbd_find_resource(nla_data(resource_filter)); - if (!resource) + if (!resource) { + rcu_read_lock(); goto put_result; + } cb->args[0] = (long)resource; } } @@ -3401,7 +3499,7 @@ int drbd_adm_dump_devices(struct sk_buff *skb, struct netlink_callback *cb) put_result: dh = genlmsg_put(skb, NETLINK_CB(cb->skb).portid, - cb->nlh->nlmsg_seq, &drbd_genl_family, + cb->nlh->nlmsg_seq, &drbd_nl_family, NLM_F_MULTI, DRBD_ADM_GET_DEVICES); err = -ENOMEM; if (!dh) @@ -3417,18 +3515,18 @@ put_result: struct disk_conf *disk_conf = rcu_dereference(device->ldev->disk_conf); - err = disk_conf_to_skb(skb, disk_conf, !capable(CAP_SYS_ADMIN)); + err = disk_conf_to_skb(skb, disk_conf); put_ldev(device); if (err) goto out; } device_to_info(&device_info, device); - err = device_info_to_skb(skb, &device_info, !capable(CAP_SYS_ADMIN)); + err = device_info_to_skb(skb, &device_info); if (err) goto out; device_to_statistics(&device_statistics, device); - err = device_statistics_to_skb(skb, &device_statistics, !capable(CAP_SYS_ADMIN)); + err = device_statistics_to_skb(skb, &device_statistics); if (err) goto out; cb->args[1] = minor + 1; @@ -3450,7 +3548,7 @@ int drbd_adm_dump_connections_done(struct netlink_callback *cb) enum { SINGLE_RESOURCE, ITERATE_RESOURCES }; -int drbd_adm_dump_connections(struct sk_buff *skb, struct netlink_callback *cb) +int drbd_nl_get_connections_dumpit(struct sk_buff *skb, struct netlink_callback *cb) { struct nlattr *resource_filter; struct drbd_resource *resource = NULL, *next_resource; @@ -3463,7 +3561,8 @@ int drbd_adm_dump_connections(struct sk_buff *skb, struct netlink_callback *cb) rcu_read_lock(); resource = (struct drbd_resource *)cb->args[0]; if (!cb->args[0]) { - resource_filter = find_cfg_context_attr(cb->nlh, T_ctx_resource_name); + resource_filter = find_cfg_context_attr(cb->nlh, + DRBD_A_DRBD_CFG_CONTEXT_CTX_RESOURCE_NAME); if (resource_filter) { retcode = ERR_RES_NOT_KNOWN; resource = drbd_find_resource(nla_data(resource_filter)); @@ -3527,7 +3626,7 @@ found_resource: put_result: dh = genlmsg_put(skb, NETLINK_CB(cb->skb).portid, - cb->nlh->nlmsg_seq, &drbd_genl_family, + cb->nlh->nlmsg_seq, &drbd_nl_family, NLM_F_MULTI, DRBD_ADM_GET_CONNECTIONS); err = -ENOMEM; if (!dh) @@ -3542,16 +3641,17 @@ put_result: goto out; net_conf = rcu_dereference(connection->net_conf); if (net_conf) { - err = net_conf_to_skb(skb, net_conf, !capable(CAP_SYS_ADMIN)); + err = net_conf_to_skb_sanitized(skb, net_conf, + !capable(CAP_SYS_ADMIN)); if (err) goto out; } connection_to_info(&connection_info, connection); - err = connection_info_to_skb(skb, &connection_info, !capable(CAP_SYS_ADMIN)); + err = connection_info_to_skb(skb, &connection_info); if (err) goto out; connection_statistics.conn_congested = test_bit(NET_CONGESTED, &connection->flags); - err = connection_statistics_to_skb(skb, &connection_statistics, !capable(CAP_SYS_ADMIN)); + err = connection_statistics_to_skb(skb, &connection_statistics); if (err) goto out; cb->args[2] = (long)connection; @@ -3612,7 +3712,7 @@ int drbd_adm_dump_peer_devices_done(struct netlink_callback *cb) return put_resource_in_arg0(cb, 9); } -int drbd_adm_dump_peer_devices(struct sk_buff *skb, struct netlink_callback *cb) +int drbd_nl_get_peer_devices_dumpit(struct sk_buff *skb, struct netlink_callback *cb) { struct nlattr *resource_filter; struct drbd_resource *resource; @@ -3624,12 +3724,15 @@ int drbd_adm_dump_peer_devices(struct sk_buff *skb, struct netlink_callback *cb) resource = (struct drbd_resource *)cb->args[0]; if (!cb->args[0] && !cb->args[1]) { - resource_filter = find_cfg_context_attr(cb->nlh, T_ctx_resource_name); + resource_filter = find_cfg_context_attr(cb->nlh, + DRBD_A_DRBD_CFG_CONTEXT_CTX_RESOURCE_NAME); if (resource_filter) { retcode = ERR_RES_NOT_KNOWN; resource = drbd_find_resource(nla_data(resource_filter)); - if (!resource) + if (!resource) { + rcu_read_lock(); goto put_result; + } } cb->args[0] = (long)resource; } @@ -3669,7 +3772,7 @@ found_peer_device: put_result: dh = genlmsg_put(skb, NETLINK_CB(cb->skb).portid, - cb->nlh->nlmsg_seq, &drbd_genl_family, + cb->nlh->nlmsg_seq, &drbd_nl_family, NLM_F_MULTI, DRBD_ADM_GET_PEER_DEVICES); err = -ENOMEM; if (!dh) @@ -3685,11 +3788,11 @@ put_result: if (err) goto out; peer_device_to_info(&peer_device_info, peer_device); - err = peer_device_info_to_skb(skb, &peer_device_info, !capable(CAP_SYS_ADMIN)); + err = peer_device_info_to_skb(skb, &peer_device_info); if (err) goto out; peer_device_to_statistics(&peer_device_statistics, peer_device); - err = peer_device_statistics_to_skb(skb, &peer_device_statistics, !capable(CAP_SYS_ADMIN)); + err = peer_device_statistics_to_skb(skb, &peer_device_statistics); if (err) goto out; cb->args[1] = minor; @@ -3729,11 +3832,11 @@ static int nla_put_status_info(struct sk_buff *skb, struct drbd_device *device, /* If sib != NULL, this is drbd_bcast_event, which anyone can listen * to. So we better exclude_sensitive information. * - * If sib == NULL, this is drbd_adm_get_status, executed synchronously + * If sib == NULL, this is drbd_nl_get_status_doit, executed synchronously * in the context of the requesting user process. Exclude sensitive * information, unless current has superuser. * - * NOTE: for drbd_adm_get_status_all(), this is a netlink dump, and + * NOTE: for drbd_nl_get_status_dumpit(), this is a netlink dump, and * relies on the current implementation of netlink_dump(), which * executes the dump callback successively from netlink_recvmsg(), * always in the context of the receiving process */ @@ -3746,7 +3849,7 @@ static int nla_put_status_info(struct sk_buff *skb, struct drbd_device *device, if (nla_put_drbd_cfg_context(skb, resource, the_only_connection(resource), device)) goto nla_put_failure; - if (res_opts_to_skb(skb, &device->resource->res_opts, exclude_sensitive)) + if (res_opts_to_skb(skb, &device->resource->res_opts)) goto nla_put_failure; rcu_read_lock(); @@ -3754,14 +3857,14 @@ static int nla_put_status_info(struct sk_buff *skb, struct drbd_device *device, struct disk_conf *disk_conf; disk_conf = rcu_dereference(device->ldev->disk_conf); - err = disk_conf_to_skb(skb, disk_conf, exclude_sensitive); + err = disk_conf_to_skb(skb, disk_conf); } if (!err) { struct net_conf *nc; nc = rcu_dereference(first_peer_device(device)->connection->net_conf); if (nc) - err = net_conf_to_skb(skb, nc, exclude_sensitive); + err = net_conf_to_skb_sanitized(skb, nc, exclude_sensitive); } rcu_read_unlock(); if (err) @@ -3770,42 +3873,57 @@ static int nla_put_status_info(struct sk_buff *skb, struct drbd_device *device, nla = nla_nest_start_noflag(skb, DRBD_NLA_STATE_INFO); if (!nla) goto nla_put_failure; - if (nla_put_u32(skb, T_sib_reason, sib ? sib->sib_reason : SIB_GET_STATUS_REPLY) || - nla_put_u32(skb, T_current_state, device->state.i) || - nla_put_u64_0pad(skb, T_ed_uuid, device->ed_uuid) || - nla_put_u64_0pad(skb, T_capacity, get_capacity(device->vdisk)) || - nla_put_u64_0pad(skb, T_send_cnt, device->send_cnt) || - nla_put_u64_0pad(skb, T_recv_cnt, device->recv_cnt) || - nla_put_u64_0pad(skb, T_read_cnt, device->read_cnt) || - nla_put_u64_0pad(skb, T_writ_cnt, device->writ_cnt) || - nla_put_u64_0pad(skb, T_al_writ_cnt, device->al_writ_cnt) || - nla_put_u64_0pad(skb, T_bm_writ_cnt, device->bm_writ_cnt) || - nla_put_u32(skb, T_ap_bio_cnt, atomic_read(&device->ap_bio_cnt)) || - nla_put_u32(skb, T_ap_pending_cnt, atomic_read(&device->ap_pending_cnt)) || - nla_put_u32(skb, T_rs_pending_cnt, atomic_read(&device->rs_pending_cnt))) + if (nla_put_u32(skb, DRBD_A_STATE_INFO_SIB_REASON, + sib ? sib->sib_reason : SIB_GET_STATUS_REPLY) || + nla_put_u32(skb, DRBD_A_STATE_INFO_CURRENT_STATE, + device->state.i) || + nla_put_u64_64bit(skb, DRBD_A_STATE_INFO_ED_UUID, + device->ed_uuid, 0) || + nla_put_u64_64bit(skb, DRBD_A_STATE_INFO_CAPACITY, + get_capacity(device->vdisk), 0) || + nla_put_u64_64bit(skb, DRBD_A_STATE_INFO_SEND_CNT, + device->send_cnt, 0) || + nla_put_u64_64bit(skb, DRBD_A_STATE_INFO_RECV_CNT, + device->recv_cnt, 0) || + nla_put_u64_64bit(skb, DRBD_A_STATE_INFO_READ_CNT, + device->read_cnt, 0) || + nla_put_u64_64bit(skb, DRBD_A_STATE_INFO_WRIT_CNT, + device->writ_cnt, 0) || + nla_put_u64_64bit(skb, DRBD_A_STATE_INFO_AL_WRIT_CNT, + device->al_writ_cnt, 0) || + nla_put_u64_64bit(skb, DRBD_A_STATE_INFO_BM_WRIT_CNT, + device->bm_writ_cnt, 0) || + nla_put_u32(skb, DRBD_A_STATE_INFO_AP_BIO_CNT, + atomic_read(&device->ap_bio_cnt)) || + nla_put_u32(skb, DRBD_A_STATE_INFO_AP_PENDING_CNT, + atomic_read(&device->ap_pending_cnt)) || + nla_put_u32(skb, DRBD_A_STATE_INFO_RS_PENDING_CNT, + atomic_read(&device->rs_pending_cnt))) goto nla_put_failure; if (got_ldev) { int err; spin_lock_irq(&device->ldev->md.uuid_lock); - err = nla_put(skb, T_uuids, sizeof(si->uuids), device->ldev->md.uuid); + err = nla_put(skb, DRBD_A_STATE_INFO_UUIDS, + sizeof(si->uuids), + device->ldev->md.uuid); spin_unlock_irq(&device->ldev->md.uuid_lock); if (err) goto nla_put_failure; - if (nla_put_u32(skb, T_disk_flags, device->ldev->md.flags) || - nla_put_u64_0pad(skb, T_bits_total, drbd_bm_bits(device)) || - nla_put_u64_0pad(skb, T_bits_oos, - drbd_bm_total_weight(device))) + if (nla_put_u32(skb, DRBD_A_STATE_INFO_DISK_FLAGS, device->ldev->md.flags) || + nla_put_u64_64bit(skb, DRBD_A_STATE_INFO_BITS_TOTAL, drbd_bm_bits(device), 0) || + nla_put_u64_64bit(skb, DRBD_A_STATE_INFO_BITS_OOS, + drbd_bm_total_weight(device), 0)) goto nla_put_failure; if (C_SYNC_SOURCE <= device->state.conn && C_PAUSED_SYNC_T >= device->state.conn) { - if (nla_put_u64_0pad(skb, T_bits_rs_total, - device->rs_total) || - nla_put_u64_0pad(skb, T_bits_rs_failed, - device->rs_failed)) + if (nla_put_u64_64bit(skb, DRBD_A_STATE_INFO_BITS_RS_TOTAL, + device->rs_total, 0) || + nla_put_u64_64bit(skb, DRBD_A_STATE_INFO_BITS_RS_FAILED, + device->rs_failed, 0)) goto nla_put_failure; } } @@ -3816,17 +3934,17 @@ static int nla_put_status_info(struct sk_buff *skb, struct drbd_device *device, case SIB_GET_STATUS_REPLY: break; case SIB_STATE_CHANGE: - if (nla_put_u32(skb, T_prev_state, sib->os.i) || - nla_put_u32(skb, T_new_state, sib->ns.i)) + if (nla_put_u32(skb, DRBD_A_STATE_INFO_PREV_STATE, sib->os.i) || + nla_put_u32(skb, DRBD_A_STATE_INFO_NEW_STATE, sib->ns.i)) goto nla_put_failure; break; case SIB_HELPER_POST: - if (nla_put_u32(skb, T_helper_exit_code, + if (nla_put_u32(skb, DRBD_A_STATE_INFO_HELPER_EXIT_CODE, sib->helper_exit_code)) goto nla_put_failure; fallthrough; case SIB_HELPER_PRE: - if (nla_put_string(skb, T_helper, sib->helper_name)) + if (nla_put_string(skb, DRBD_A_STATE_INFO_HELPER, sib->helper_name)) goto nla_put_failure; break; } @@ -3841,25 +3959,26 @@ nla_put_failure: return err; } -int drbd_adm_get_status(struct sk_buff *skb, struct genl_info *info) +int drbd_nl_get_status_doit(struct sk_buff *skb, struct genl_info *info) { - struct drbd_config_context adm_ctx; + struct drbd_config_context *adm_ctx = info->user_ptr[0]; enum drbd_ret_code retcode; int err; - retcode = drbd_adm_prepare(&adm_ctx, skb, info, DRBD_ADM_NEED_MINOR); - if (!adm_ctx.reply_skb) - return retcode; + if (!adm_ctx->reply_skb) + return 0; + retcode = adm_ctx->reply_dh->ret_code; if (retcode != NO_ERROR) goto out; - err = nla_put_status_info(adm_ctx.reply_skb, adm_ctx.device, NULL); + err = nla_put_status_info(adm_ctx->reply_skb, adm_ctx->device, NULL); if (err) { - nlmsg_free(adm_ctx.reply_skb); + nlmsg_free(adm_ctx->reply_skb); + adm_ctx->reply_skb = NULL; return err; } out: - drbd_adm_finish(&adm_ctx, info, retcode); + adm_ctx->reply_dh->ret_code = retcode; return 0; } @@ -3930,7 +4049,7 @@ next_resource: } dh = genlmsg_put(skb, NETLINK_CB(cb->skb).portid, - cb->nlh->nlmsg_seq, &drbd_genl_family, + cb->nlh->nlmsg_seq, &drbd_nl_family, NLM_F_MULTI, DRBD_ADM_GET_STATUS); if (!dh) goto out; @@ -3950,7 +4069,7 @@ next_resource: struct net_conf *nc; nc = rcu_dereference(connection->net_conf); - if (nc && net_conf_to_skb(skb, nc, 1) != 0) + if (nc && net_conf_to_skb_sanitized(skb, nc, true) != 0) goto cancel; } goto done; @@ -3992,13 +4111,12 @@ out: * * Once things are setup properly, we call into get_one_status(). */ -int drbd_adm_get_status_all(struct sk_buff *skb, struct netlink_callback *cb) +int drbd_nl_get_status_dumpit(struct sk_buff *skb, struct netlink_callback *cb) { - const unsigned hdrlen = GENL_HDRLEN + GENL_MAGIC_FAMILY_HDRSZ; + const unsigned int hdrlen = GENL_HDRLEN + sizeof(struct drbd_genlmsghdr); struct nlattr *nla; const char *resource_name; struct drbd_resource *resource; - int maxtype; /* Is this a followup call? */ if (cb->args[0]) { @@ -4018,10 +4136,7 @@ int drbd_adm_get_status_all(struct sk_buff *skb, struct netlink_callback *cb) /* No explicit context given. Dump all. */ if (!nla) goto dump; - maxtype = ARRAY_SIZE(drbd_cfg_context_nl_policy) - 1; - nla = drbd_nla_find_nested(maxtype, nla, __nla_type(T_ctx_resource_name)); - if (IS_ERR(nla)) - return PTR_ERR(nla); + nla = nla_find_nested(nla, DRBD_A_DRBD_CFG_CONTEXT_CTX_RESOURCE_NAME); /* context given, but no name present? */ if (!nla) return -EINVAL; @@ -4044,48 +4159,49 @@ dump: return get_one_status(skb, cb); } -int drbd_adm_get_timeout_type(struct sk_buff *skb, struct genl_info *info) +int drbd_nl_get_timeout_type_doit(struct sk_buff *skb, struct genl_info *info) { - struct drbd_config_context adm_ctx; + struct drbd_config_context *adm_ctx = info->user_ptr[0]; enum drbd_ret_code retcode; struct timeout_parms tp; int err; - retcode = drbd_adm_prepare(&adm_ctx, skb, info, DRBD_ADM_NEED_MINOR); - if (!adm_ctx.reply_skb) - return retcode; + if (!adm_ctx->reply_skb) + return 0; + retcode = adm_ctx->reply_dh->ret_code; if (retcode != NO_ERROR) goto out; tp.timeout_type = - adm_ctx.device->state.pdsk == D_OUTDATED ? UT_PEER_OUTDATED : - test_bit(USE_DEGR_WFC_T, &adm_ctx.device->flags) ? UT_DEGRADED : + adm_ctx->device->state.pdsk == D_OUTDATED ? UT_PEER_OUTDATED : + test_bit(USE_DEGR_WFC_T, &adm_ctx->device->flags) ? UT_DEGRADED : UT_DEFAULT; - err = timeout_parms_to_priv_skb(adm_ctx.reply_skb, &tp); + err = timeout_parms_to_skb(adm_ctx->reply_skb, &tp); if (err) { - nlmsg_free(adm_ctx.reply_skb); + nlmsg_free(adm_ctx->reply_skb); + adm_ctx->reply_skb = NULL; return err; } out: - drbd_adm_finish(&adm_ctx, info, retcode); + adm_ctx->reply_dh->ret_code = retcode; return 0; } -int drbd_adm_start_ov(struct sk_buff *skb, struct genl_info *info) +int drbd_nl_start_ov_doit(struct sk_buff *skb, struct genl_info *info) { - struct drbd_config_context adm_ctx; + struct drbd_config_context *adm_ctx = info->user_ptr[0]; struct drbd_device *device; enum drbd_ret_code retcode; struct start_ov_parms parms; - retcode = drbd_adm_prepare(&adm_ctx, skb, info, DRBD_ADM_NEED_MINOR); - if (!adm_ctx.reply_skb) - return retcode; + if (!adm_ctx->reply_skb) + return 0; + retcode = adm_ctx->reply_dh->ret_code; if (retcode != NO_ERROR) goto out; - device = adm_ctx.device; + device = adm_ctx->device; /* resume from last known position, if possible */ parms.ov_start_sector = device->ov_start_sector; @@ -4094,11 +4210,11 @@ int drbd_adm_start_ov(struct sk_buff *skb, struct genl_info *info) int err = start_ov_parms_from_attrs(&parms, info); if (err) { retcode = ERR_MANDATORY_TAG; - drbd_msg_put_info(adm_ctx.reply_skb, from_attrs_err_to_txt(err)); + drbd_msg_put_info(adm_ctx->reply_skb, from_attrs_err_to_txt(err)); goto out; } } - mutex_lock(&adm_ctx.resource->adm_mutex); + mutex_lock(&adm_ctx->resource->adm_mutex); /* w_make_ov_request expects position to be aligned */ device->ov_start_sector = parms.ov_start_sector & ~(BM_SECT_PER_BIT-1); @@ -4111,40 +4227,40 @@ int drbd_adm_start_ov(struct sk_buff *skb, struct genl_info *info) retcode = drbd_request_state(device, NS(conn, C_VERIFY_S)); drbd_resume_io(device); - mutex_unlock(&adm_ctx.resource->adm_mutex); + mutex_unlock(&adm_ctx->resource->adm_mutex); out: - drbd_adm_finish(&adm_ctx, info, retcode); + adm_ctx->reply_dh->ret_code = retcode; return 0; } -int drbd_adm_new_c_uuid(struct sk_buff *skb, struct genl_info *info) +int drbd_nl_new_c_uuid_doit(struct sk_buff *skb, struct genl_info *info) { - struct drbd_config_context adm_ctx; + struct drbd_config_context *adm_ctx = info->user_ptr[0]; struct drbd_device *device; enum drbd_ret_code retcode; int skip_initial_sync = 0; int err; struct new_c_uuid_parms args; - retcode = drbd_adm_prepare(&adm_ctx, skb, info, DRBD_ADM_NEED_MINOR); - if (!adm_ctx.reply_skb) - return retcode; + if (!adm_ctx->reply_skb) + return 0; + retcode = adm_ctx->reply_dh->ret_code; if (retcode != NO_ERROR) goto out_nolock; - device = adm_ctx.device; + device = adm_ctx->device; memset(&args, 0, sizeof(args)); if (info->attrs[DRBD_NLA_NEW_C_UUID_PARMS]) { err = new_c_uuid_parms_from_attrs(&args, info); if (err) { retcode = ERR_MANDATORY_TAG; - drbd_msg_put_info(adm_ctx.reply_skb, from_attrs_err_to_txt(err)); + drbd_msg_put_info(adm_ctx->reply_skb, from_attrs_err_to_txt(err)); goto out_nolock; } } - mutex_lock(&adm_ctx.resource->adm_mutex); + mutex_lock(&adm_ctx->resource->adm_mutex); mutex_lock(device->state_mutex); /* Protects us against serialized state changes. */ if (!get_ldev(device)) { @@ -4189,9 +4305,9 @@ out_dec: put_ldev(device); out: mutex_unlock(device->state_mutex); - mutex_unlock(&adm_ctx.resource->adm_mutex); + mutex_unlock(&adm_ctx->resource->adm_mutex); out_nolock: - drbd_adm_finish(&adm_ctx, info, retcode); + adm_ctx->reply_dh->ret_code = retcode; return 0; } @@ -4221,17 +4337,17 @@ static void resource_to_info(struct resource_info *info, info->res_susp_fen = resource->susp_fen; } -int drbd_adm_new_resource(struct sk_buff *skb, struct genl_info *info) +int drbd_nl_new_resource_doit(struct sk_buff *skb, struct genl_info *info) { struct drbd_connection *connection; - struct drbd_config_context adm_ctx; + struct drbd_config_context *adm_ctx = info->user_ptr[0]; enum drbd_ret_code retcode; struct res_opts res_opts; int err; - retcode = drbd_adm_prepare(&adm_ctx, skb, info, 0); - if (!adm_ctx.reply_skb) - return retcode; + if (!adm_ctx->reply_skb) + return 0; + retcode = adm_ctx->reply_dh->ret_code; if (retcode != NO_ERROR) goto out; @@ -4239,18 +4355,18 @@ int drbd_adm_new_resource(struct sk_buff *skb, struct genl_info *info) err = res_opts_from_attrs(&res_opts, info); if (err && err != -ENOMSG) { retcode = ERR_MANDATORY_TAG; - drbd_msg_put_info(adm_ctx.reply_skb, from_attrs_err_to_txt(err)); + drbd_msg_put_info(adm_ctx->reply_skb, from_attrs_err_to_txt(err)); goto out; } - retcode = drbd_check_resource_name(&adm_ctx); + retcode = drbd_check_resource_name(adm_ctx); if (retcode != NO_ERROR) goto out; - if (adm_ctx.resource) { + if (adm_ctx->resource) { if (info->nlhdr->nlmsg_flags & NLM_F_EXCL) { retcode = ERR_INVALID_REQUEST; - drbd_msg_put_info(adm_ctx.reply_skb, "resource exists"); + drbd_msg_put_info(adm_ctx->reply_skb, "resource exists"); } /* else: still NO_ERROR */ goto out; @@ -4258,7 +4374,7 @@ int drbd_adm_new_resource(struct sk_buff *skb, struct genl_info *info) /* not yet safe for genl_family.parallel_ops */ mutex_lock(&resources_mutex); - connection = conn_create(adm_ctx.resource_name, &res_opts); + connection = conn_create(adm_ctx->resource_name, &res_opts); mutex_unlock(&resources_mutex); if (connection) { @@ -4273,7 +4389,7 @@ int drbd_adm_new_resource(struct sk_buff *skb, struct genl_info *info) retcode = ERR_NOMEM; out: - drbd_adm_finish(&adm_ctx, info, retcode); + adm_ctx->reply_dh->ret_code = retcode; return 0; } @@ -4284,40 +4400,40 @@ static void device_to_info(struct device_info *info, } -int drbd_adm_new_minor(struct sk_buff *skb, struct genl_info *info) +int drbd_nl_new_minor_doit(struct sk_buff *skb, struct genl_info *info) { - struct drbd_config_context adm_ctx; + struct drbd_config_context *adm_ctx = info->user_ptr[0]; struct drbd_genlmsghdr *dh = genl_info_userhdr(info); enum drbd_ret_code retcode; - retcode = drbd_adm_prepare(&adm_ctx, skb, info, DRBD_ADM_NEED_RESOURCE); - if (!adm_ctx.reply_skb) - return retcode; + if (!adm_ctx->reply_skb) + return 0; + retcode = adm_ctx->reply_dh->ret_code; if (retcode != NO_ERROR) goto out; if (dh->minor > MINORMASK) { - drbd_msg_put_info(adm_ctx.reply_skb, "requested minor out of range"); + drbd_msg_put_info(adm_ctx->reply_skb, "requested minor out of range"); retcode = ERR_INVALID_REQUEST; goto out; } - if (adm_ctx.volume > DRBD_VOLUME_MAX) { - drbd_msg_put_info(adm_ctx.reply_skb, "requested volume id out of range"); + if (adm_ctx->volume > DRBD_VOLUME_MAX) { + drbd_msg_put_info(adm_ctx->reply_skb, "requested volume id out of range"); retcode = ERR_INVALID_REQUEST; goto out; } /* drbd_adm_prepare made sure already * that first_peer_device(device)->connection and device->vnr match the request. */ - if (adm_ctx.device) { + if (adm_ctx->device) { if (info->nlhdr->nlmsg_flags & NLM_F_EXCL) retcode = ERR_MINOR_OR_VOLUME_EXISTS; /* else: still NO_ERROR */ goto out; } - mutex_lock(&adm_ctx.resource->adm_mutex); - retcode = drbd_create_device(&adm_ctx, dh->minor); + mutex_lock(&adm_ctx->resource->adm_mutex); + retcode = drbd_create_device(adm_ctx, dh->minor); if (retcode == NO_ERROR) { struct drbd_device *device; struct drbd_peer_device *peer_device; @@ -4348,9 +4464,9 @@ int drbd_adm_new_minor(struct sk_buff *skb, struct genl_info *info) } mutex_unlock(¬ification_mutex); } - mutex_unlock(&adm_ctx.resource->adm_mutex); + mutex_unlock(&adm_ctx->resource->adm_mutex); out: - drbd_adm_finish(&adm_ctx, info, retcode); + adm_ctx->reply_dh->ret_code = retcode; return 0; } @@ -4391,22 +4507,22 @@ static enum drbd_ret_code adm_del_minor(struct drbd_device *device) return ERR_MINOR_CONFIGURED; } -int drbd_adm_del_minor(struct sk_buff *skb, struct genl_info *info) +int drbd_nl_del_minor_doit(struct sk_buff *skb, struct genl_info *info) { - struct drbd_config_context adm_ctx; + struct drbd_config_context *adm_ctx = info->user_ptr[0]; enum drbd_ret_code retcode; - retcode = drbd_adm_prepare(&adm_ctx, skb, info, DRBD_ADM_NEED_MINOR); - if (!adm_ctx.reply_skb) - return retcode; + if (!adm_ctx->reply_skb) + return 0; + retcode = adm_ctx->reply_dh->ret_code; if (retcode != NO_ERROR) goto out; - mutex_lock(&adm_ctx.resource->adm_mutex); - retcode = adm_del_minor(adm_ctx.device); - mutex_unlock(&adm_ctx.resource->adm_mutex); + mutex_lock(&adm_ctx->resource->adm_mutex); + retcode = adm_del_minor(adm_ctx->device); + mutex_unlock(&adm_ctx->resource->adm_mutex); out: - drbd_adm_finish(&adm_ctx, info, retcode); + adm_ctx->reply_dh->ret_code = retcode; return 0; } @@ -4440,22 +4556,22 @@ static int adm_del_resource(struct drbd_resource *resource) return NO_ERROR; } -int drbd_adm_down(struct sk_buff *skb, struct genl_info *info) +int drbd_nl_down_doit(struct sk_buff *skb, struct genl_info *info) { - struct drbd_config_context adm_ctx; + struct drbd_config_context *adm_ctx = info->user_ptr[0]; struct drbd_resource *resource; struct drbd_connection *connection; struct drbd_device *device; int retcode; /* enum drbd_ret_code rsp. enum drbd_state_rv */ unsigned i; - retcode = drbd_adm_prepare(&adm_ctx, skb, info, DRBD_ADM_NEED_RESOURCE); - if (!adm_ctx.reply_skb) - return retcode; + if (!adm_ctx->reply_skb) + return 0; + retcode = adm_ctx->reply_dh->ret_code; if (retcode != NO_ERROR) goto finish; - resource = adm_ctx.resource; + resource = adm_ctx->resource; mutex_lock(&resource->adm_mutex); /* demote */ for_each_connection(connection, resource) { @@ -4464,14 +4580,14 @@ int drbd_adm_down(struct sk_buff *skb, struct genl_info *info) idr_for_each_entry(&connection->peer_devices, peer_device, i) { retcode = drbd_set_role(peer_device->device, R_SECONDARY, 0); if (retcode < SS_SUCCESS) { - drbd_msg_put_info(adm_ctx.reply_skb, "failed to demote"); + drbd_msg_put_info(adm_ctx->reply_skb, "failed to demote"); goto out; } } retcode = conn_try_disconnect(connection, 0); if (retcode < SS_SUCCESS) { - drbd_msg_put_info(adm_ctx.reply_skb, "failed to disconnect"); + drbd_msg_put_info(adm_ctx->reply_skb, "failed to disconnect"); goto out; } } @@ -4480,7 +4596,7 @@ int drbd_adm_down(struct sk_buff *skb, struct genl_info *info) idr_for_each_entry(&resource->devices, device, i) { retcode = adm_detach(device, 0); if (retcode < SS_SUCCESS || retcode > NO_ERROR) { - drbd_msg_put_info(adm_ctx.reply_skb, "failed to detach"); + drbd_msg_put_info(adm_ctx->reply_skb, "failed to detach"); goto out; } } @@ -4490,7 +4606,7 @@ int drbd_adm_down(struct sk_buff *skb, struct genl_info *info) retcode = adm_del_minor(device); if (retcode != NO_ERROR) { /* "can not happen" */ - drbd_msg_put_info(adm_ctx.reply_skb, "failed to delete volume"); + drbd_msg_put_info(adm_ctx->reply_skb, "failed to delete volume"); goto out; } } @@ -4499,28 +4615,28 @@ int drbd_adm_down(struct sk_buff *skb, struct genl_info *info) out: mutex_unlock(&resource->adm_mutex); finish: - drbd_adm_finish(&adm_ctx, info, retcode); + adm_ctx->reply_dh->ret_code = retcode; return 0; } -int drbd_adm_del_resource(struct sk_buff *skb, struct genl_info *info) +int drbd_nl_del_resource_doit(struct sk_buff *skb, struct genl_info *info) { - struct drbd_config_context adm_ctx; + struct drbd_config_context *adm_ctx = info->user_ptr[0]; struct drbd_resource *resource; enum drbd_ret_code retcode; - retcode = drbd_adm_prepare(&adm_ctx, skb, info, DRBD_ADM_NEED_RESOURCE); - if (!adm_ctx.reply_skb) - return retcode; + if (!adm_ctx->reply_skb) + return 0; + retcode = adm_ctx->reply_dh->ret_code; if (retcode != NO_ERROR) goto finish; - resource = adm_ctx.resource; + resource = adm_ctx->resource; mutex_lock(&resource->adm_mutex); retcode = adm_del_resource(resource); mutex_unlock(&resource->adm_mutex); finish: - drbd_adm_finish(&adm_ctx, info, retcode); + adm_ctx->reply_dh->ret_code = retcode; return 0; } @@ -4537,7 +4653,7 @@ void drbd_bcast_event(struct drbd_device *device, const struct sib_info *sib) goto failed; err = -EMSGSIZE; - d_out = genlmsg_put(msg, 0, seq, &drbd_genl_family, 0, DRBD_EVENT); + d_out = genlmsg_put(msg, 0, seq, &drbd_nl_family, 0, DRBD_ADM_EVENT); if (!d_out) /* cannot happen, but anyways. */ goto nla_put_failure; d_out->minor = device_to_minor(device); @@ -4568,7 +4684,7 @@ static int nla_put_notification_header(struct sk_buff *msg, .nh_type = type, }; - return drbd_notification_header_to_skb(msg, &nh, true); + return drbd_notification_header_to_skb(msg, &nh); } int notify_resource_state(struct sk_buff *skb, @@ -4592,7 +4708,7 @@ int notify_resource_state(struct sk_buff *skb, } err = -EMSGSIZE; - dh = genlmsg_put(skb, 0, seq, &drbd_genl_family, 0, DRBD_RESOURCE_STATE); + dh = genlmsg_put(skb, 0, seq, &drbd_nl_family, 0, DRBD_ADM_RESOURCE_STATE); if (!dh) goto nla_put_failure; dh->minor = -1U; @@ -4600,10 +4716,10 @@ int notify_resource_state(struct sk_buff *skb, if (nla_put_drbd_cfg_context(skb, resource, NULL, NULL) || nla_put_notification_header(skb, type) || ((type & ~NOTIFY_FLAGS) != NOTIFY_DESTROY && - resource_info_to_skb(skb, resource_info, true))) + resource_info_to_skb(skb, resource_info))) goto nla_put_failure; resource_statistics.res_stat_write_ordering = resource->write_ordering; - err = resource_statistics_to_skb(skb, &resource_statistics, !capable(CAP_SYS_ADMIN)); + err = resource_statistics_to_skb(skb, &resource_statistics); if (err) goto nla_put_failure; genlmsg_end(skb, dh); @@ -4644,7 +4760,7 @@ int notify_device_state(struct sk_buff *skb, } err = -EMSGSIZE; - dh = genlmsg_put(skb, 0, seq, &drbd_genl_family, 0, DRBD_DEVICE_STATE); + dh = genlmsg_put(skb, 0, seq, &drbd_nl_family, 0, DRBD_ADM_DEVICE_STATE); if (!dh) goto nla_put_failure; dh->minor = device->minor; @@ -4652,10 +4768,10 @@ int notify_device_state(struct sk_buff *skb, if (nla_put_drbd_cfg_context(skb, device->resource, NULL, device) || nla_put_notification_header(skb, type) || ((type & ~NOTIFY_FLAGS) != NOTIFY_DESTROY && - device_info_to_skb(skb, device_info, true))) + device_info_to_skb(skb, device_info))) goto nla_put_failure; device_to_statistics(&device_statistics, device); - device_statistics_to_skb(skb, &device_statistics, !capable(CAP_SYS_ADMIN)); + device_statistics_to_skb(skb, &device_statistics); genlmsg_end(skb, dh); if (multicast) { err = drbd_genl_multicast_events(skb, GFP_NOWAIT); @@ -4694,7 +4810,7 @@ int notify_connection_state(struct sk_buff *skb, } err = -EMSGSIZE; - dh = genlmsg_put(skb, 0, seq, &drbd_genl_family, 0, DRBD_CONNECTION_STATE); + dh = genlmsg_put(skb, 0, seq, &drbd_nl_family, 0, DRBD_ADM_CONNECTION_STATE); if (!dh) goto nla_put_failure; dh->minor = -1U; @@ -4702,10 +4818,10 @@ int notify_connection_state(struct sk_buff *skb, if (nla_put_drbd_cfg_context(skb, connection->resource, connection, NULL) || nla_put_notification_header(skb, type) || ((type & ~NOTIFY_FLAGS) != NOTIFY_DESTROY && - connection_info_to_skb(skb, connection_info, true))) + connection_info_to_skb(skb, connection_info))) goto nla_put_failure; connection_statistics.conn_congested = test_bit(NET_CONGESTED, &connection->flags); - connection_statistics_to_skb(skb, &connection_statistics, !capable(CAP_SYS_ADMIN)); + connection_statistics_to_skb(skb, &connection_statistics); genlmsg_end(skb, dh); if (multicast) { err = drbd_genl_multicast_events(skb, GFP_NOWAIT); @@ -4745,7 +4861,7 @@ int notify_peer_device_state(struct sk_buff *skb, } err = -EMSGSIZE; - dh = genlmsg_put(skb, 0, seq, &drbd_genl_family, 0, DRBD_PEER_DEVICE_STATE); + dh = genlmsg_put(skb, 0, seq, &drbd_nl_family, 0, DRBD_ADM_PEER_DEVICE_STATE); if (!dh) goto nla_put_failure; dh->minor = -1U; @@ -4753,10 +4869,10 @@ int notify_peer_device_state(struct sk_buff *skb, if (nla_put_drbd_cfg_context(skb, resource, peer_device->connection, peer_device->device) || nla_put_notification_header(skb, type) || ((type & ~NOTIFY_FLAGS) != NOTIFY_DESTROY && - peer_device_info_to_skb(skb, peer_device_info, true))) + peer_device_info_to_skb(skb, peer_device_info))) goto nla_put_failure; peer_device_to_statistics(&peer_device_statistics, peer_device); - peer_device_statistics_to_skb(skb, &peer_device_statistics, !capable(CAP_SYS_ADMIN)); + peer_device_statistics_to_skb(skb, &peer_device_statistics); genlmsg_end(skb, dh); if (multicast) { err = drbd_genl_multicast_events(skb, GFP_NOWAIT); @@ -4795,7 +4911,7 @@ void notify_helper(enum drbd_notification_type type, goto fail; err = -EMSGSIZE; - dh = genlmsg_put(skb, 0, seq, &drbd_genl_family, 0, DRBD_HELPER); + dh = genlmsg_put(skb, 0, seq, &drbd_nl_family, 0, DRBD_ADM_HELPER); if (!dh) goto fail; dh->minor = device ? device->minor : -1; @@ -4803,7 +4919,7 @@ void notify_helper(enum drbd_notification_type type, mutex_lock(¬ification_mutex); if (nla_put_drbd_cfg_context(skb, resource, connection, device) || nla_put_notification_header(skb, type) || - drbd_helper_info_to_skb(skb, &helper_info, true)) + drbd_helper_info_to_skb(skb, &helper_info)) goto unlock_fail; genlmsg_end(skb, dh); err = drbd_genl_multicast_events(skb, GFP_NOWAIT); @@ -4828,7 +4944,7 @@ static int notify_initial_state_done(struct sk_buff *skb, unsigned int seq) int err; err = -EMSGSIZE; - dh = genlmsg_put(skb, 0, seq, &drbd_genl_family, 0, DRBD_INITIAL_STATE_DONE); + dh = genlmsg_put(skb, 0, seq, &drbd_nl_family, 0, DRBD_ADM_INITIAL_STATE_DONE); if (!dh) goto nla_put_failure; dh->minor = -1U; @@ -4923,7 +5039,7 @@ out: return skb->len; } -int drbd_adm_get_initial_state(struct sk_buff *skb, struct netlink_callback *cb) +int drbd_nl_get_initial_state_dumpit(struct sk_buff *skb, struct netlink_callback *cb) { struct drbd_resource *resource; LIST_HEAD(head); @@ -4971,3 +5087,20 @@ int drbd_adm_get_initial_state(struct sk_buff *skb, struct netlink_callback *cb) cb->args[2] = cb->nlh->nlmsg_seq; return get_initial_state(skb, cb); } + +static const struct genl_multicast_group drbd_nl_mcgrps[] = { + [DRBD_NLGRP_EVENTS] = { .name = "events", }, +}; + +struct genl_family drbd_nl_family __ro_after_init = { + .name = "drbd", + .version = DRBD_FAMILY_VERSION, + .hdrsize = NLA_ALIGN(sizeof(struct drbd_genlmsghdr)), + .split_ops = drbd_nl_ops, + .n_split_ops = ARRAY_SIZE(drbd_nl_ops), + .mcgrps = drbd_nl_mcgrps, + .n_mcgrps = ARRAY_SIZE(drbd_nl_mcgrps), + .resv_start_op = 42, + .module = THIS_MODULE, + .netnsok = true, +}; diff --git a/drivers/block/drbd/drbd_nl_gen.c b/drivers/block/drbd/drbd_nl_gen.c new file mode 100644 index 000000000000..9753dc789bde --- /dev/null +++ b/drivers/block/drbd/drbd_nl_gen.c @@ -0,0 +1,2610 @@ +// SPDX-License-Identifier: ((GPL-2.0 WITH Linux-syscall-note) OR BSD-3-Clause) + +#include <net/netlink.h> +#include <net/genetlink.h> + +#include <linux/kernel.h> +#include <linux/slab.h> +#include "drbd_nl_gen.h" + +#include <uapi/linux/drbd_genl.h> +#include <linux/drbd.h> +#include <linux/drbd_limits.h> + +/* Common nested types */ +const struct nla_policy drbd_connection_info_nl_policy[DRBD_A_CONNECTION_INFO_CONN_ROLE + 1] = { + [DRBD_A_CONNECTION_INFO_CONN_CONNECTION_STATE] = { .type = NLA_U32, }, + [DRBD_A_CONNECTION_INFO_CONN_ROLE] = { .type = NLA_U32, }, +}; + +const struct nla_policy drbd_connection_statistics_nl_policy[DRBD_A_CONNECTION_STATISTICS_CONN_CONGESTED + 1] = { + [DRBD_A_CONNECTION_STATISTICS_CONN_CONGESTED] = { .type = NLA_U8, }, +}; + +const struct nla_policy drbd_detach_parms_nl_policy[DRBD_A_DETACH_PARMS_FORCE_DETACH + 1] = { + [DRBD_A_DETACH_PARMS_FORCE_DETACH] = { .type = NLA_U8, }, +}; + +const struct nla_policy drbd_device_info_nl_policy[DRBD_A_DEVICE_INFO_DEV_DISK_STATE + 1] = { + [DRBD_A_DEVICE_INFO_DEV_DISK_STATE] = { .type = NLA_U32, }, +}; + +const struct nla_policy drbd_device_statistics_nl_policy[DRBD_A_DEVICE_STATISTICS_HISTORY_UUIDS + 1] = { + [DRBD_A_DEVICE_STATISTICS_DEV_SIZE] = { .type = NLA_U64, }, + [DRBD_A_DEVICE_STATISTICS_DEV_READ] = { .type = NLA_U64, }, + [DRBD_A_DEVICE_STATISTICS_DEV_WRITE] = { .type = NLA_U64, }, + [DRBD_A_DEVICE_STATISTICS_DEV_AL_WRITES] = { .type = NLA_U64, }, + [DRBD_A_DEVICE_STATISTICS_DEV_BM_WRITES] = { .type = NLA_U64, }, + [DRBD_A_DEVICE_STATISTICS_DEV_UPPER_PENDING] = { .type = NLA_U32, }, + [DRBD_A_DEVICE_STATISTICS_DEV_LOWER_PENDING] = { .type = NLA_U32, }, + [DRBD_A_DEVICE_STATISTICS_DEV_UPPER_BLOCKED] = { .type = NLA_U8, }, + [DRBD_A_DEVICE_STATISTICS_DEV_LOWER_BLOCKED] = { .type = NLA_U8, }, + [DRBD_A_DEVICE_STATISTICS_DEV_AL_SUSPENDED] = { .type = NLA_U8, }, + [DRBD_A_DEVICE_STATISTICS_DEV_EXPOSED_DATA_UUID] = { .type = NLA_U64, }, + [DRBD_A_DEVICE_STATISTICS_DEV_CURRENT_UUID] = { .type = NLA_U64, }, + [DRBD_A_DEVICE_STATISTICS_DEV_DISK_FLAGS] = { .type = NLA_U32, }, + [DRBD_A_DEVICE_STATISTICS_HISTORY_UUIDS] = NLA_POLICY_MAX_LEN(DRBD_NL_HISTORY_UUIDS_SIZE), +}; + +const struct nla_policy drbd_disconnect_parms_nl_policy[DRBD_A_DISCONNECT_PARMS_FORCE_DISCONNECT + 1] = { + [DRBD_A_DISCONNECT_PARMS_FORCE_DISCONNECT] = { .type = NLA_U8, }, +}; + +const struct nla_policy drbd_disk_conf_nl_policy[DRBD_A_DISK_CONF_DISABLE_WRITE_SAME + 1] = { + [DRBD_A_DISK_CONF_BACKING_DEV] = { .type = NLA_NUL_STRING, .len = 128, }, + [DRBD_A_DISK_CONF_META_DEV] = { .type = NLA_NUL_STRING, .len = 128, }, + [DRBD_A_DISK_CONF_META_DEV_IDX] = { .type = NLA_U32, }, + [DRBD_A_DISK_CONF_DISK_SIZE] = { .type = NLA_U64, }, + [DRBD_A_DISK_CONF_MAX_BIO_BVECS] = { .type = NLA_U32, }, + [DRBD_A_DISK_CONF_ON_IO_ERROR] = { .type = NLA_U32, }, + [DRBD_A_DISK_CONF_FENCING] = { .type = NLA_U32, }, + [DRBD_A_DISK_CONF_RESYNC_RATE] = { .type = NLA_U32, }, + [DRBD_A_DISK_CONF_RESYNC_AFTER] = { .type = NLA_U32, }, + [DRBD_A_DISK_CONF_AL_EXTENTS] = { .type = NLA_U32, }, + [DRBD_A_DISK_CONF_C_PLAN_AHEAD] = { .type = NLA_U32, }, + [DRBD_A_DISK_CONF_C_DELAY_TARGET] = { .type = NLA_U32, }, + [DRBD_A_DISK_CONF_C_FILL_TARGET] = { .type = NLA_U32, }, + [DRBD_A_DISK_CONF_C_MAX_RATE] = { .type = NLA_U32, }, + [DRBD_A_DISK_CONF_C_MIN_RATE] = { .type = NLA_U32, }, + [DRBD_A_DISK_CONF_DISK_BARRIER] = { .type = NLA_U8, }, + [DRBD_A_DISK_CONF_DISK_FLUSHES] = { .type = NLA_U8, }, + [DRBD_A_DISK_CONF_DISK_DRAIN] = { .type = NLA_U8, }, + [DRBD_A_DISK_CONF_MD_FLUSHES] = { .type = NLA_U8, }, + [DRBD_A_DISK_CONF_DISK_TIMEOUT] = { .type = NLA_U32, }, + [DRBD_A_DISK_CONF_READ_BALANCING] = { .type = NLA_U32, }, + [DRBD_A_DISK_CONF_AL_UPDATES] = { .type = NLA_U8, }, + [DRBD_A_DISK_CONF_DISCARD_ZEROES_IF_ALIGNED] = { .type = NLA_U8, }, + [DRBD_A_DISK_CONF_RS_DISCARD_GRANULARITY] = { .type = NLA_U32, }, + [DRBD_A_DISK_CONF_DISABLE_WRITE_SAME] = { .type = NLA_U8, }, +}; + +const struct nla_policy drbd_drbd_cfg_context_nl_policy[DRBD_A_DRBD_CFG_CONTEXT_CTX_PEER_ADDR + 1] = { + [DRBD_A_DRBD_CFG_CONTEXT_CTX_VOLUME] = { .type = NLA_U32, }, + [DRBD_A_DRBD_CFG_CONTEXT_CTX_RESOURCE_NAME] = { .type = NLA_NUL_STRING, .len = 128, }, + [DRBD_A_DRBD_CFG_CONTEXT_CTX_MY_ADDR] = NLA_POLICY_MAX_LEN(128), + [DRBD_A_DRBD_CFG_CONTEXT_CTX_PEER_ADDR] = NLA_POLICY_MAX_LEN(128), +}; + +const struct nla_policy drbd_net_conf_nl_policy[DRBD_A_NET_CONF_SOCK_CHECK_TIMEO + 1] = { + [DRBD_A_NET_CONF_SHARED_SECRET] = { .type = NLA_NUL_STRING, .len = SHARED_SECRET_MAX, }, + [DRBD_A_NET_CONF_CRAM_HMAC_ALG] = { .type = NLA_NUL_STRING, .len = SHARED_SECRET_MAX, }, + [DRBD_A_NET_CONF_INTEGRITY_ALG] = { .type = NLA_NUL_STRING, .len = SHARED_SECRET_MAX, }, + [DRBD_A_NET_CONF_VERIFY_ALG] = { .type = NLA_NUL_STRING, .len = SHARED_SECRET_MAX, }, + [DRBD_A_NET_CONF_CSUMS_ALG] = { .type = NLA_NUL_STRING, .len = SHARED_SECRET_MAX, }, + [DRBD_A_NET_CONF_WIRE_PROTOCOL] = { .type = NLA_U32, }, + [DRBD_A_NET_CONF_CONNECT_INT] = { .type = NLA_U32, }, + [DRBD_A_NET_CONF_TIMEOUT] = { .type = NLA_U32, }, + [DRBD_A_NET_CONF_PING_INT] = { .type = NLA_U32, }, + [DRBD_A_NET_CONF_PING_TIMEO] = { .type = NLA_U32, }, + [DRBD_A_NET_CONF_SNDBUF_SIZE] = { .type = NLA_U32, }, + [DRBD_A_NET_CONF_RCVBUF_SIZE] = { .type = NLA_U32, }, + [DRBD_A_NET_CONF_KO_COUNT] = { .type = NLA_U32, }, + [DRBD_A_NET_CONF_MAX_BUFFERS] = { .type = NLA_U32, }, + [DRBD_A_NET_CONF_MAX_EPOCH_SIZE] = { .type = NLA_U32, }, + [DRBD_A_NET_CONF_UNPLUG_WATERMARK] = { .type = NLA_U32, }, + [DRBD_A_NET_CONF_AFTER_SB_0P] = { .type = NLA_U32, }, + [DRBD_A_NET_CONF_AFTER_SB_1P] = { .type = NLA_U32, }, + [DRBD_A_NET_CONF_AFTER_SB_2P] = { .type = NLA_U32, }, + [DRBD_A_NET_CONF_RR_CONFLICT] = { .type = NLA_U32, }, + [DRBD_A_NET_CONF_ON_CONGESTION] = { .type = NLA_U32, }, + [DRBD_A_NET_CONF_CONG_FILL] = { .type = NLA_U32, }, + [DRBD_A_NET_CONF_CONG_EXTENTS] = { .type = NLA_U32, }, + [DRBD_A_NET_CONF_TWO_PRIMARIES] = { .type = NLA_U8, }, + [DRBD_A_NET_CONF_DISCARD_MY_DATA] = { .type = NLA_U8, }, + [DRBD_A_NET_CONF_TCP_CORK] = { .type = NLA_U8, }, + [DRBD_A_NET_CONF_ALWAYS_ASBP] = { .type = NLA_U8, }, + [DRBD_A_NET_CONF_TENTATIVE] = { .type = NLA_U8, }, + [DRBD_A_NET_CONF_USE_RLE] = { .type = NLA_U8, }, + [DRBD_A_NET_CONF_CSUMS_AFTER_CRASH_ONLY] = { .type = NLA_U8, }, + [DRBD_A_NET_CONF_SOCK_CHECK_TIMEO] = { .type = NLA_U32, }, +}; + +const struct nla_policy drbd_new_c_uuid_parms_nl_policy[DRBD_A_NEW_C_UUID_PARMS_CLEAR_BM + 1] = { + [DRBD_A_NEW_C_UUID_PARMS_CLEAR_BM] = { .type = NLA_U8, }, +}; + +const struct nla_policy drbd_peer_device_info_nl_policy[DRBD_A_PEER_DEVICE_INFO_PEER_RESYNC_SUSP_DEPENDENCY + 1] = { + [DRBD_A_PEER_DEVICE_INFO_PEER_REPL_STATE] = { .type = NLA_U32, }, + [DRBD_A_PEER_DEVICE_INFO_PEER_DISK_STATE] = { .type = NLA_U32, }, + [DRBD_A_PEER_DEVICE_INFO_PEER_RESYNC_SUSP_USER] = { .type = NLA_U32, }, + [DRBD_A_PEER_DEVICE_INFO_PEER_RESYNC_SUSP_PEER] = { .type = NLA_U32, }, + [DRBD_A_PEER_DEVICE_INFO_PEER_RESYNC_SUSP_DEPENDENCY] = { .type = NLA_U32, }, +}; + +const struct nla_policy drbd_peer_device_statistics_nl_policy[DRBD_A_PEER_DEVICE_STATISTICS_PEER_DEV_FLAGS + 1] = { + [DRBD_A_PEER_DEVICE_STATISTICS_PEER_DEV_RECEIVED] = { .type = NLA_U64, }, + [DRBD_A_PEER_DEVICE_STATISTICS_PEER_DEV_SENT] = { .type = NLA_U64, }, + [DRBD_A_PEER_DEVICE_STATISTICS_PEER_DEV_PENDING] = { .type = NLA_U32, }, + [DRBD_A_PEER_DEVICE_STATISTICS_PEER_DEV_UNACKED] = { .type = NLA_U32, }, + [DRBD_A_PEER_DEVICE_STATISTICS_PEER_DEV_OUT_OF_SYNC] = { .type = NLA_U64, }, + [DRBD_A_PEER_DEVICE_STATISTICS_PEER_DEV_RESYNC_FAILED] = { .type = NLA_U64, }, + [DRBD_A_PEER_DEVICE_STATISTICS_PEER_DEV_BITMAP_UUID] = { .type = NLA_U64, }, + [DRBD_A_PEER_DEVICE_STATISTICS_PEER_DEV_FLAGS] = { .type = NLA_U32, }, +}; + +const struct nla_policy drbd_res_opts_nl_policy[DRBD_A_RES_OPTS_ON_NO_DATA + 1] = { + [DRBD_A_RES_OPTS_CPU_MASK] = { .type = NLA_NUL_STRING, .len = DRBD_CPU_MASK_SIZE, }, + [DRBD_A_RES_OPTS_ON_NO_DATA] = { .type = NLA_U32, }, +}; + +const struct nla_policy drbd_resize_parms_nl_policy[DRBD_A_RESIZE_PARMS_AL_STRIPE_SIZE + 1] = { + [DRBD_A_RESIZE_PARMS_RESIZE_SIZE] = { .type = NLA_U64, }, + [DRBD_A_RESIZE_PARMS_RESIZE_FORCE] = { .type = NLA_U8, }, + [DRBD_A_RESIZE_PARMS_NO_RESYNC] = { .type = NLA_U8, }, + [DRBD_A_RESIZE_PARMS_AL_STRIPES] = { .type = NLA_U32, }, + [DRBD_A_RESIZE_PARMS_AL_STRIPE_SIZE] = { .type = NLA_U32, }, +}; + +const struct nla_policy drbd_resource_info_nl_policy[DRBD_A_RESOURCE_INFO_RES_SUSP_FEN + 1] = { + [DRBD_A_RESOURCE_INFO_RES_ROLE] = { .type = NLA_U32, }, + [DRBD_A_RESOURCE_INFO_RES_SUSP] = { .type = NLA_U8, }, + [DRBD_A_RESOURCE_INFO_RES_SUSP_NOD] = { .type = NLA_U8, }, + [DRBD_A_RESOURCE_INFO_RES_SUSP_FEN] = { .type = NLA_U8, }, +}; + +const struct nla_policy drbd_resource_statistics_nl_policy[DRBD_A_RESOURCE_STATISTICS_RES_STAT_WRITE_ORDERING + 1] = { + [DRBD_A_RESOURCE_STATISTICS_RES_STAT_WRITE_ORDERING] = { .type = NLA_U32, }, +}; + +const struct nla_policy drbd_set_role_parms_nl_policy[DRBD_A_SET_ROLE_PARMS_ASSUME_UPTODATE + 1] = { + [DRBD_A_SET_ROLE_PARMS_ASSUME_UPTODATE] = { .type = NLA_U8, }, +}; + +const struct nla_policy drbd_start_ov_parms_nl_policy[DRBD_A_START_OV_PARMS_OV_STOP_SECTOR + 1] = { + [DRBD_A_START_OV_PARMS_OV_START_SECTOR] = { .type = NLA_U64, }, + [DRBD_A_START_OV_PARMS_OV_STOP_SECTOR] = { .type = NLA_U64, }, +}; + +/* DRBD_ADM_GET_STATUS - do */ +static const struct nla_policy drbd_get_status_do_nl_policy[DRBD_NLA_CFG_CONTEXT + 1] = { + [DRBD_NLA_CFG_CONTEXT] = NLA_POLICY_NESTED(drbd_drbd_cfg_context_nl_policy), +}; + +/* DRBD_ADM_GET_STATUS - dump */ +static const struct nla_policy drbd_get_status_dump_nl_policy[DRBD_NLA_CFG_CONTEXT + 1] = { + [DRBD_NLA_CFG_CONTEXT] = NLA_POLICY_NESTED(drbd_drbd_cfg_context_nl_policy), +}; + +/* DRBD_ADM_NEW_MINOR - do */ +static const struct nla_policy drbd_new_minor_nl_policy[DRBD_NLA_CFG_CONTEXT + 1] = { + [DRBD_NLA_CFG_CONTEXT] = NLA_POLICY_NESTED(drbd_drbd_cfg_context_nl_policy), +}; + +/* DRBD_ADM_DEL_MINOR - do */ +static const struct nla_policy drbd_del_minor_nl_policy[DRBD_NLA_CFG_CONTEXT + 1] = { + [DRBD_NLA_CFG_CONTEXT] = NLA_POLICY_NESTED(drbd_drbd_cfg_context_nl_policy), +}; + +/* DRBD_ADM_NEW_RESOURCE - do */ +static const struct nla_policy drbd_new_resource_nl_policy[DRBD_NLA_RESOURCE_OPTS + 1] = { + [DRBD_NLA_CFG_CONTEXT] = NLA_POLICY_NESTED(drbd_drbd_cfg_context_nl_policy), + [DRBD_NLA_RESOURCE_OPTS] = NLA_POLICY_NESTED(drbd_res_opts_nl_policy), +}; + +/* DRBD_ADM_DEL_RESOURCE - do */ +static const struct nla_policy drbd_del_resource_nl_policy[DRBD_NLA_CFG_CONTEXT + 1] = { + [DRBD_NLA_CFG_CONTEXT] = NLA_POLICY_NESTED(drbd_drbd_cfg_context_nl_policy), +}; + +/* DRBD_ADM_RESOURCE_OPTS - do */ +static const struct nla_policy drbd_resource_opts_nl_policy[DRBD_NLA_RESOURCE_OPTS + 1] = { + [DRBD_NLA_CFG_CONTEXT] = NLA_POLICY_NESTED(drbd_drbd_cfg_context_nl_policy), + [DRBD_NLA_RESOURCE_OPTS] = NLA_POLICY_NESTED(drbd_res_opts_nl_policy), +}; + +/* DRBD_ADM_CONNECT - do */ +static const struct nla_policy drbd_connect_nl_policy[DRBD_NLA_NET_CONF + 1] = { + [DRBD_NLA_CFG_CONTEXT] = NLA_POLICY_NESTED(drbd_drbd_cfg_context_nl_policy), + [DRBD_NLA_NET_CONF] = NLA_POLICY_NESTED(drbd_net_conf_nl_policy), +}; + +/* DRBD_ADM_DISCONNECT - do */ +static const struct nla_policy drbd_disconnect_nl_policy[DRBD_NLA_DISCONNECT_PARMS + 1] = { + [DRBD_NLA_CFG_CONTEXT] = NLA_POLICY_NESTED(drbd_drbd_cfg_context_nl_policy), + [DRBD_NLA_DISCONNECT_PARMS] = NLA_POLICY_NESTED(drbd_disconnect_parms_nl_policy), +}; + +/* DRBD_ADM_ATTACH - do */ +static const struct nla_policy drbd_attach_nl_policy[DRBD_NLA_DISK_CONF + 1] = { + [DRBD_NLA_CFG_CONTEXT] = NLA_POLICY_NESTED(drbd_drbd_cfg_context_nl_policy), + [DRBD_NLA_DISK_CONF] = NLA_POLICY_NESTED(drbd_disk_conf_nl_policy), +}; + +/* DRBD_ADM_RESIZE - do */ +static const struct nla_policy drbd_resize_nl_policy[DRBD_NLA_RESIZE_PARMS + 1] = { + [DRBD_NLA_CFG_CONTEXT] = NLA_POLICY_NESTED(drbd_drbd_cfg_context_nl_policy), + [DRBD_NLA_RESIZE_PARMS] = NLA_POLICY_NESTED(drbd_resize_parms_nl_policy), +}; + +/* DRBD_ADM_PRIMARY - do */ +static const struct nla_policy drbd_primary_nl_policy[DRBD_NLA_SET_ROLE_PARMS + 1] = { + [DRBD_NLA_CFG_CONTEXT] = NLA_POLICY_NESTED(drbd_drbd_cfg_context_nl_policy), + [DRBD_NLA_SET_ROLE_PARMS] = NLA_POLICY_NESTED(drbd_set_role_parms_nl_policy), +}; + +/* DRBD_ADM_SECONDARY - do */ +static const struct nla_policy drbd_secondary_nl_policy[DRBD_NLA_SET_ROLE_PARMS + 1] = { + [DRBD_NLA_CFG_CONTEXT] = NLA_POLICY_NESTED(drbd_drbd_cfg_context_nl_policy), + [DRBD_NLA_SET_ROLE_PARMS] = NLA_POLICY_NESTED(drbd_set_role_parms_nl_policy), +}; + +/* DRBD_ADM_NEW_C_UUID - do */ +static const struct nla_policy drbd_new_c_uuid_nl_policy[DRBD_NLA_NEW_C_UUID_PARMS + 1] = { + [DRBD_NLA_CFG_CONTEXT] = NLA_POLICY_NESTED(drbd_drbd_cfg_context_nl_policy), + [DRBD_NLA_NEW_C_UUID_PARMS] = NLA_POLICY_NESTED(drbd_new_c_uuid_parms_nl_policy), +}; + +/* DRBD_ADM_START_OV - do */ +static const struct nla_policy drbd_start_ov_nl_policy[DRBD_NLA_START_OV_PARMS + 1] = { + [DRBD_NLA_START_OV_PARMS] = NLA_POLICY_NESTED(drbd_start_ov_parms_nl_policy), +}; + +/* DRBD_ADM_DETACH - do */ +static const struct nla_policy drbd_detach_nl_policy[DRBD_NLA_DETACH_PARMS + 1] = { + [DRBD_NLA_CFG_CONTEXT] = NLA_POLICY_NESTED(drbd_drbd_cfg_context_nl_policy), + [DRBD_NLA_DETACH_PARMS] = NLA_POLICY_NESTED(drbd_detach_parms_nl_policy), +}; + +/* DRBD_ADM_INVALIDATE - do */ +static const struct nla_policy drbd_invalidate_nl_policy[DRBD_NLA_CFG_CONTEXT + 1] = { + [DRBD_NLA_CFG_CONTEXT] = NLA_POLICY_NESTED(drbd_drbd_cfg_context_nl_policy), +}; + +/* DRBD_ADM_INVAL_PEER - do */ +static const struct nla_policy drbd_inval_peer_nl_policy[DRBD_NLA_CFG_CONTEXT + 1] = { + [DRBD_NLA_CFG_CONTEXT] = NLA_POLICY_NESTED(drbd_drbd_cfg_context_nl_policy), +}; + +/* DRBD_ADM_PAUSE_SYNC - do */ +static const struct nla_policy drbd_pause_sync_nl_policy[DRBD_NLA_CFG_CONTEXT + 1] = { + [DRBD_NLA_CFG_CONTEXT] = NLA_POLICY_NESTED(drbd_drbd_cfg_context_nl_policy), +}; + +/* DRBD_ADM_RESUME_SYNC - do */ +static const struct nla_policy drbd_resume_sync_nl_policy[DRBD_NLA_CFG_CONTEXT + 1] = { + [DRBD_NLA_CFG_CONTEXT] = NLA_POLICY_NESTED(drbd_drbd_cfg_context_nl_policy), +}; + +/* DRBD_ADM_SUSPEND_IO - do */ +static const struct nla_policy drbd_suspend_io_nl_policy[DRBD_NLA_CFG_CONTEXT + 1] = { + [DRBD_NLA_CFG_CONTEXT] = NLA_POLICY_NESTED(drbd_drbd_cfg_context_nl_policy), +}; + +/* DRBD_ADM_RESUME_IO - do */ +static const struct nla_policy drbd_resume_io_nl_policy[DRBD_NLA_CFG_CONTEXT + 1] = { + [DRBD_NLA_CFG_CONTEXT] = NLA_POLICY_NESTED(drbd_drbd_cfg_context_nl_policy), +}; + +/* DRBD_ADM_OUTDATE - do */ +static const struct nla_policy drbd_outdate_nl_policy[DRBD_NLA_CFG_CONTEXT + 1] = { + [DRBD_NLA_CFG_CONTEXT] = NLA_POLICY_NESTED(drbd_drbd_cfg_context_nl_policy), +}; + +/* DRBD_ADM_GET_TIMEOUT_TYPE - do */ +static const struct nla_policy drbd_get_timeout_type_nl_policy[DRBD_NLA_CFG_CONTEXT + 1] = { + [DRBD_NLA_CFG_CONTEXT] = NLA_POLICY_NESTED(drbd_drbd_cfg_context_nl_policy), +}; + +/* DRBD_ADM_DOWN - do */ +static const struct nla_policy drbd_down_nl_policy[DRBD_NLA_CFG_CONTEXT + 1] = { + [DRBD_NLA_CFG_CONTEXT] = NLA_POLICY_NESTED(drbd_drbd_cfg_context_nl_policy), +}; + +/* DRBD_ADM_CHG_DISK_OPTS - do */ +static const struct nla_policy drbd_chg_disk_opts_nl_policy[DRBD_NLA_DISK_CONF + 1] = { + [DRBD_NLA_CFG_CONTEXT] = NLA_POLICY_NESTED(drbd_drbd_cfg_context_nl_policy), + [DRBD_NLA_DISK_CONF] = NLA_POLICY_NESTED(drbd_disk_conf_nl_policy), +}; + +/* DRBD_ADM_CHG_NET_OPTS - do */ +static const struct nla_policy drbd_chg_net_opts_nl_policy[DRBD_NLA_NET_CONF + 1] = { + [DRBD_NLA_CFG_CONTEXT] = NLA_POLICY_NESTED(drbd_drbd_cfg_context_nl_policy), + [DRBD_NLA_NET_CONF] = NLA_POLICY_NESTED(drbd_net_conf_nl_policy), +}; + +/* DRBD_ADM_GET_RESOURCES - dump */ +static const struct nla_policy drbd_get_resources_nl_policy[DRBD_NLA_RESOURCE_STATISTICS + 1] = { + [DRBD_NLA_CFG_CONTEXT] = NLA_POLICY_NESTED(drbd_drbd_cfg_context_nl_policy), + [DRBD_NLA_RESOURCE_INFO] = NLA_POLICY_NESTED(drbd_resource_info_nl_policy), + [DRBD_NLA_RESOURCE_STATISTICS] = NLA_POLICY_NESTED(drbd_resource_statistics_nl_policy), +}; + +/* DRBD_ADM_GET_DEVICES - dump */ +static const struct nla_policy drbd_get_devices_nl_policy[DRBD_NLA_DEVICE_STATISTICS + 1] = { + [DRBD_NLA_CFG_CONTEXT] = NLA_POLICY_NESTED(drbd_drbd_cfg_context_nl_policy), + [DRBD_NLA_DEVICE_INFO] = NLA_POLICY_NESTED(drbd_device_info_nl_policy), + [DRBD_NLA_DEVICE_STATISTICS] = NLA_POLICY_NESTED(drbd_device_statistics_nl_policy), +}; + +/* DRBD_ADM_GET_CONNECTIONS - dump */ +static const struct nla_policy drbd_get_connections_nl_policy[DRBD_NLA_CONNECTION_STATISTICS + 1] = { + [DRBD_NLA_CFG_CONTEXT] = NLA_POLICY_NESTED(drbd_drbd_cfg_context_nl_policy), + [DRBD_NLA_CONNECTION_INFO] = NLA_POLICY_NESTED(drbd_connection_info_nl_policy), + [DRBD_NLA_CONNECTION_STATISTICS] = NLA_POLICY_NESTED(drbd_connection_statistics_nl_policy), +}; + +/* DRBD_ADM_GET_PEER_DEVICES - dump */ +static const struct nla_policy drbd_get_peer_devices_nl_policy[DRBD_NLA_PEER_DEVICE_STATISTICS + 1] = { + [DRBD_NLA_CFG_CONTEXT] = NLA_POLICY_NESTED(drbd_drbd_cfg_context_nl_policy), + [DRBD_NLA_PEER_DEVICE_INFO] = NLA_POLICY_NESTED(drbd_peer_device_info_nl_policy), + [DRBD_NLA_PEER_DEVICE_STATISTICS] = NLA_POLICY_NESTED(drbd_peer_device_statistics_nl_policy), +}; + +/* DRBD_ADM_GET_INITIAL_STATE - dump */ +static const struct nla_policy drbd_get_initial_state_nl_policy[DRBD_NLA_CFG_CONTEXT + 1] = { + [DRBD_NLA_CFG_CONTEXT] = NLA_POLICY_NESTED(drbd_drbd_cfg_context_nl_policy), +}; + +/* Ops table for drbd */ +const struct genl_split_ops drbd_nl_ops[32] = { + { + .cmd = DRBD_ADM_GET_STATUS, + .pre_doit = drbd_pre_doit, + .doit = drbd_nl_get_status_doit, + .post_doit = drbd_post_doit, + .policy = drbd_get_status_do_nl_policy, + .maxattr = DRBD_NLA_CFG_CONTEXT, + .flags = GENL_CMD_CAP_DO, + }, + { + .cmd = DRBD_ADM_GET_STATUS, + .dumpit = drbd_nl_get_status_dumpit, + .policy = drbd_get_status_dump_nl_policy, + .maxattr = DRBD_NLA_CFG_CONTEXT, + .flags = GENL_CMD_CAP_DUMP, + }, + { + .cmd = DRBD_ADM_NEW_MINOR, + .pre_doit = drbd_pre_doit, + .doit = drbd_nl_new_minor_doit, + .post_doit = drbd_post_doit, + .policy = drbd_new_minor_nl_policy, + .maxattr = DRBD_NLA_CFG_CONTEXT, + .flags = GENL_ADMIN_PERM | GENL_CMD_CAP_DO, + }, + { + .cmd = DRBD_ADM_DEL_MINOR, + .pre_doit = drbd_pre_doit, + .doit = drbd_nl_del_minor_doit, + .post_doit = drbd_post_doit, + .policy = drbd_del_minor_nl_policy, + .maxattr = DRBD_NLA_CFG_CONTEXT, + .flags = GENL_ADMIN_PERM | GENL_CMD_CAP_DO, + }, + { + .cmd = DRBD_ADM_NEW_RESOURCE, + .pre_doit = drbd_pre_doit, + .doit = drbd_nl_new_resource_doit, + .post_doit = drbd_post_doit, + .policy = drbd_new_resource_nl_policy, + .maxattr = DRBD_NLA_RESOURCE_OPTS, + .flags = GENL_ADMIN_PERM | GENL_CMD_CAP_DO, + }, + { + .cmd = DRBD_ADM_DEL_RESOURCE, + .pre_doit = drbd_pre_doit, + .doit = drbd_nl_del_resource_doit, + .post_doit = drbd_post_doit, + .policy = drbd_del_resource_nl_policy, + .maxattr = DRBD_NLA_CFG_CONTEXT, + .flags = GENL_ADMIN_PERM | GENL_CMD_CAP_DO, + }, + { + .cmd = DRBD_ADM_RESOURCE_OPTS, + .pre_doit = drbd_pre_doit, + .doit = drbd_nl_resource_opts_doit, + .post_doit = drbd_post_doit, + .policy = drbd_resource_opts_nl_policy, + .maxattr = DRBD_NLA_RESOURCE_OPTS, + .flags = GENL_ADMIN_PERM | GENL_CMD_CAP_DO, + }, + { + .cmd = DRBD_ADM_CONNECT, + .pre_doit = drbd_pre_doit, + .doit = drbd_nl_connect_doit, + .post_doit = drbd_post_doit, + .policy = drbd_connect_nl_policy, + .maxattr = DRBD_NLA_NET_CONF, + .flags = GENL_ADMIN_PERM | GENL_CMD_CAP_DO, + }, + { + .cmd = DRBD_ADM_DISCONNECT, + .pre_doit = drbd_pre_doit, + .doit = drbd_nl_disconnect_doit, + .post_doit = drbd_post_doit, + .policy = drbd_disconnect_nl_policy, + .maxattr = DRBD_NLA_DISCONNECT_PARMS, + .flags = GENL_ADMIN_PERM | GENL_CMD_CAP_DO, + }, + { + .cmd = DRBD_ADM_ATTACH, + .pre_doit = drbd_pre_doit, + .doit = drbd_nl_attach_doit, + .post_doit = drbd_post_doit, + .policy = drbd_attach_nl_policy, + .maxattr = DRBD_NLA_DISK_CONF, + .flags = GENL_ADMIN_PERM | GENL_CMD_CAP_DO, + }, + { + .cmd = DRBD_ADM_RESIZE, + .pre_doit = drbd_pre_doit, + .doit = drbd_nl_resize_doit, + .post_doit = drbd_post_doit, + .policy = drbd_resize_nl_policy, + .maxattr = DRBD_NLA_RESIZE_PARMS, + .flags = GENL_ADMIN_PERM | GENL_CMD_CAP_DO, + }, + { + .cmd = DRBD_ADM_PRIMARY, + .pre_doit = drbd_pre_doit, + .doit = drbd_nl_primary_doit, + .post_doit = drbd_post_doit, + .policy = drbd_primary_nl_policy, + .maxattr = DRBD_NLA_SET_ROLE_PARMS, + .flags = GENL_ADMIN_PERM | GENL_CMD_CAP_DO, + }, + { + .cmd = DRBD_ADM_SECONDARY, + .pre_doit = drbd_pre_doit, + .doit = drbd_nl_secondary_doit, + .post_doit = drbd_post_doit, + .policy = drbd_secondary_nl_policy, + .maxattr = DRBD_NLA_SET_ROLE_PARMS, + .flags = GENL_ADMIN_PERM | GENL_CMD_CAP_DO, + }, + { + .cmd = DRBD_ADM_NEW_C_UUID, + .pre_doit = drbd_pre_doit, + .doit = drbd_nl_new_c_uuid_doit, + .post_doit = drbd_post_doit, + .policy = drbd_new_c_uuid_nl_policy, + .maxattr = DRBD_NLA_NEW_C_UUID_PARMS, + .flags = GENL_ADMIN_PERM | GENL_CMD_CAP_DO, + }, + { + .cmd = DRBD_ADM_START_OV, + .pre_doit = drbd_pre_doit, + .doit = drbd_nl_start_ov_doit, + .post_doit = drbd_post_doit, + .policy = drbd_start_ov_nl_policy, + .maxattr = DRBD_NLA_START_OV_PARMS, + .flags = GENL_ADMIN_PERM | GENL_CMD_CAP_DO, + }, + { + .cmd = DRBD_ADM_DETACH, + .pre_doit = drbd_pre_doit, + .doit = drbd_nl_detach_doit, + .post_doit = drbd_post_doit, + .policy = drbd_detach_nl_policy, + .maxattr = DRBD_NLA_DETACH_PARMS, + .flags = GENL_ADMIN_PERM | GENL_CMD_CAP_DO, + }, + { + .cmd = DRBD_ADM_INVALIDATE, + .pre_doit = drbd_pre_doit, + .doit = drbd_nl_invalidate_doit, + .post_doit = drbd_post_doit, + .policy = drbd_invalidate_nl_policy, + .maxattr = DRBD_NLA_CFG_CONTEXT, + .flags = GENL_ADMIN_PERM | GENL_CMD_CAP_DO, + }, + { + .cmd = DRBD_ADM_INVAL_PEER, + .pre_doit = drbd_pre_doit, + .doit = drbd_nl_inval_peer_doit, + .post_doit = drbd_post_doit, + .policy = drbd_inval_peer_nl_policy, + .maxattr = DRBD_NLA_CFG_CONTEXT, + .flags = GENL_ADMIN_PERM | GENL_CMD_CAP_DO, + }, + { + .cmd = DRBD_ADM_PAUSE_SYNC, + .pre_doit = drbd_pre_doit, + .doit = drbd_nl_pause_sync_doit, + .post_doit = drbd_post_doit, + .policy = drbd_pause_sync_nl_policy, + .maxattr = DRBD_NLA_CFG_CONTEXT, + .flags = GENL_ADMIN_PERM | GENL_CMD_CAP_DO, + }, + { + .cmd = DRBD_ADM_RESUME_SYNC, + .pre_doit = drbd_pre_doit, + .doit = drbd_nl_resume_sync_doit, + .post_doit = drbd_post_doit, + .policy = drbd_resume_sync_nl_policy, + .maxattr = DRBD_NLA_CFG_CONTEXT, + .flags = GENL_ADMIN_PERM | GENL_CMD_CAP_DO, + }, + { + .cmd = DRBD_ADM_SUSPEND_IO, + .pre_doit = drbd_pre_doit, + .doit = drbd_nl_suspend_io_doit, + .post_doit = drbd_post_doit, + .policy = drbd_suspend_io_nl_policy, + .maxattr = DRBD_NLA_CFG_CONTEXT, + .flags = GENL_ADMIN_PERM | GENL_CMD_CAP_DO, + }, + { + .cmd = DRBD_ADM_RESUME_IO, + .pre_doit = drbd_pre_doit, + .doit = drbd_nl_resume_io_doit, + .post_doit = drbd_post_doit, + .policy = drbd_resume_io_nl_policy, + .maxattr = DRBD_NLA_CFG_CONTEXT, + .flags = GENL_ADMIN_PERM | GENL_CMD_CAP_DO, + }, + { + .cmd = DRBD_ADM_OUTDATE, + .pre_doit = drbd_pre_doit, + .doit = drbd_nl_outdate_doit, + .post_doit = drbd_post_doit, + .policy = drbd_outdate_nl_policy, + .maxattr = DRBD_NLA_CFG_CONTEXT, + .flags = GENL_ADMIN_PERM | GENL_CMD_CAP_DO, + }, + { + .cmd = DRBD_ADM_GET_TIMEOUT_TYPE, + .pre_doit = drbd_pre_doit, + .doit = drbd_nl_get_timeout_type_doit, + .post_doit = drbd_post_doit, + .policy = drbd_get_timeout_type_nl_policy, + .maxattr = DRBD_NLA_CFG_CONTEXT, + .flags = GENL_ADMIN_PERM | GENL_CMD_CAP_DO, + }, + { + .cmd = DRBD_ADM_DOWN, + .pre_doit = drbd_pre_doit, + .doit = drbd_nl_down_doit, + .post_doit = drbd_post_doit, + .policy = drbd_down_nl_policy, + .maxattr = DRBD_NLA_CFG_CONTEXT, + .flags = GENL_ADMIN_PERM | GENL_CMD_CAP_DO, + }, + { + .cmd = DRBD_ADM_CHG_DISK_OPTS, + .pre_doit = drbd_pre_doit, + .doit = drbd_nl_chg_disk_opts_doit, + .post_doit = drbd_post_doit, + .policy = drbd_chg_disk_opts_nl_policy, + .maxattr = DRBD_NLA_DISK_CONF, + .flags = GENL_ADMIN_PERM | GENL_CMD_CAP_DO, + }, + { + .cmd = DRBD_ADM_CHG_NET_OPTS, + .pre_doit = drbd_pre_doit, + .doit = drbd_nl_chg_net_opts_doit, + .post_doit = drbd_post_doit, + .policy = drbd_chg_net_opts_nl_policy, + .maxattr = DRBD_NLA_NET_CONF, + .flags = GENL_ADMIN_PERM | GENL_CMD_CAP_DO, + }, + { + .cmd = DRBD_ADM_GET_RESOURCES, + .dumpit = drbd_nl_get_resources_dumpit, + .policy = drbd_get_resources_nl_policy, + .maxattr = DRBD_NLA_RESOURCE_STATISTICS, + .flags = GENL_CMD_CAP_DUMP, + }, + { + .cmd = DRBD_ADM_GET_DEVICES, + .dumpit = drbd_nl_get_devices_dumpit, + .done = drbd_adm_dump_devices_done, + .policy = drbd_get_devices_nl_policy, + .maxattr = DRBD_NLA_DEVICE_STATISTICS, + .flags = GENL_CMD_CAP_DUMP, + }, + { + .cmd = DRBD_ADM_GET_CONNECTIONS, + .dumpit = drbd_nl_get_connections_dumpit, + .done = drbd_adm_dump_connections_done, + .policy = drbd_get_connections_nl_policy, + .maxattr = DRBD_NLA_CONNECTION_STATISTICS, + .flags = GENL_CMD_CAP_DUMP, + }, + { + .cmd = DRBD_ADM_GET_PEER_DEVICES, + .dumpit = drbd_nl_get_peer_devices_dumpit, + .done = drbd_adm_dump_peer_devices_done, + .policy = drbd_get_peer_devices_nl_policy, + .maxattr = DRBD_NLA_PEER_DEVICE_STATISTICS, + .flags = GENL_CMD_CAP_DUMP, + }, + { + .cmd = DRBD_ADM_GET_INITIAL_STATE, + .dumpit = drbd_nl_get_initial_state_dumpit, + .policy = drbd_get_initial_state_nl_policy, + .maxattr = DRBD_NLA_CFG_CONTEXT, + .flags = GENL_CMD_CAP_DUMP, + }, +}; + +static const struct genl_multicast_group drbd_nl_mcgrps[] = { + [DRBD_NLGRP_EVENTS] = { "events", }, +}; + +static int __drbd_cfg_context_from_attrs(struct drbd_cfg_context *s, + struct nlattr ***ret_nested_attribute_table, + struct genl_info *info) +{ + const int maxtype = DRBD_A_DRBD_CFG_CONTEXT_CTX_PEER_ADDR; + struct nlattr *tla = info->attrs[DRBD_NLA_CFG_CONTEXT]; + struct nlattr **ntb; + struct nlattr *nla; + int err = 0; + + if (ret_nested_attribute_table) + *ret_nested_attribute_table = NULL; + if (!tla) + return -ENOMSG; + ntb = kzalloc_objs(*ntb, DRBD_A_DRBD_CFG_CONTEXT_CTX_PEER_ADDR + 1); + if (!ntb) + return -ENOMEM; + err = nla_parse_nested_deprecated(ntb, maxtype, tla, drbd_drbd_cfg_context_nl_policy, NULL); + if (err) + goto out; + + nla = ntb[DRBD_A_DRBD_CFG_CONTEXT_CTX_VOLUME]; + if (nla && s) + s->ctx_volume = nla_get_u32(nla); + + nla = ntb[DRBD_A_DRBD_CFG_CONTEXT_CTX_RESOURCE_NAME]; + if (nla && s) + s->ctx_resource_name_len = nla_strscpy(s->ctx_resource_name, nla, 128); + + nla = ntb[DRBD_A_DRBD_CFG_CONTEXT_CTX_MY_ADDR]; + if (nla && s) + s->ctx_my_addr_len = nla_memcpy(s->ctx_my_addr, nla, 128); + + nla = ntb[DRBD_A_DRBD_CFG_CONTEXT_CTX_PEER_ADDR]; + if (nla && s) + s->ctx_peer_addr_len = nla_memcpy(s->ctx_peer_addr, nla, 128); + +out: + if (ret_nested_attribute_table && (!err || err == -ENOMSG)) + *ret_nested_attribute_table = ntb; + else + kfree(ntb); + return err; +} + +int drbd_cfg_context_from_attrs(struct drbd_cfg_context *s, + struct genl_info *info) +{ + return __drbd_cfg_context_from_attrs(s, NULL, info); +} + +int drbd_cfg_context_ntb_from_attrs( + struct nlattr ***ret_nested_attribute_table, + struct genl_info *info) +{ + return __drbd_cfg_context_from_attrs(NULL, ret_nested_attribute_table, info); +} + +static int __disk_conf_from_attrs(struct disk_conf *s, + struct nlattr ***ret_nested_attribute_table, + struct genl_info *info) +{ + const int maxtype = DRBD_A_DISK_CONF_DISABLE_WRITE_SAME; + struct nlattr *tla = info->attrs[DRBD_NLA_DISK_CONF]; + struct nlattr **ntb; + struct nlattr *nla; + int err = 0; + + if (ret_nested_attribute_table) + *ret_nested_attribute_table = NULL; + if (!tla) + return -ENOMSG; + ntb = kzalloc_objs(*ntb, DRBD_A_DISK_CONF_DISABLE_WRITE_SAME + 1); + if (!ntb) + return -ENOMEM; + err = nla_parse_nested_deprecated(ntb, maxtype, tla, drbd_disk_conf_nl_policy, NULL); + if (err) + goto out; + + nla = ntb[DRBD_A_DISK_CONF_BACKING_DEV]; + if (nla) { + if (s) + s->backing_dev_len = nla_strscpy(s->backing_dev, nla, 128); + } else { + pr_info("<< missing required attr: backing_dev\n"); + err = -ENOMSG; + } + + nla = ntb[DRBD_A_DISK_CONF_META_DEV]; + if (nla) { + if (s) + s->meta_dev_len = nla_strscpy(s->meta_dev, nla, 128); + } else { + pr_info("<< missing required attr: meta_dev\n"); + err = -ENOMSG; + } + + nla = ntb[DRBD_A_DISK_CONF_META_DEV_IDX]; + if (nla) { + if (s) + s->meta_dev_idx = nla_get_s32(nla); + } else { + pr_info("<< missing required attr: meta_dev_idx\n"); + err = -ENOMSG; + } + + nla = ntb[DRBD_A_DISK_CONF_DISK_SIZE]; + if (nla && s) + s->disk_size = nla_get_u64(nla); + + nla = ntb[DRBD_A_DISK_CONF_MAX_BIO_BVECS]; + if (nla && s) + s->max_bio_bvecs = nla_get_u32(nla); + + nla = ntb[DRBD_A_DISK_CONF_ON_IO_ERROR]; + if (nla && s) + s->on_io_error = nla_get_u32(nla); + + nla = ntb[DRBD_A_DISK_CONF_FENCING]; + if (nla && s) + s->fencing = nla_get_u32(nla); + + nla = ntb[DRBD_A_DISK_CONF_RESYNC_RATE]; + if (nla && s) + s->resync_rate = nla_get_u32(nla); + + nla = ntb[DRBD_A_DISK_CONF_RESYNC_AFTER]; + if (nla && s) + s->resync_after = nla_get_s32(nla); + + nla = ntb[DRBD_A_DISK_CONF_AL_EXTENTS]; + if (nla && s) + s->al_extents = nla_get_u32(nla); + + nla = ntb[DRBD_A_DISK_CONF_C_PLAN_AHEAD]; + if (nla && s) + s->c_plan_ahead = nla_get_u32(nla); + + nla = ntb[DRBD_A_DISK_CONF_C_DELAY_TARGET]; + if (nla && s) + s->c_delay_target = nla_get_u32(nla); + + nla = ntb[DRBD_A_DISK_CONF_C_FILL_TARGET]; + if (nla && s) + s->c_fill_target = nla_get_u32(nla); + + nla = ntb[DRBD_A_DISK_CONF_C_MAX_RATE]; + if (nla && s) + s->c_max_rate = nla_get_u32(nla); + + nla = ntb[DRBD_A_DISK_CONF_C_MIN_RATE]; + if (nla && s) + s->c_min_rate = nla_get_u32(nla); + + nla = ntb[DRBD_A_DISK_CONF_DISK_BARRIER]; + if (nla && s) + s->disk_barrier = nla_get_u8(nla); + + nla = ntb[DRBD_A_DISK_CONF_DISK_FLUSHES]; + if (nla && s) + s->disk_flushes = nla_get_u8(nla); + + nla = ntb[DRBD_A_DISK_CONF_DISK_DRAIN]; + if (nla && s) + s->disk_drain = nla_get_u8(nla); + + nla = ntb[DRBD_A_DISK_CONF_MD_FLUSHES]; + if (nla && s) + s->md_flushes = nla_get_u8(nla); + + nla = ntb[DRBD_A_DISK_CONF_DISK_TIMEOUT]; + if (nla && s) + s->disk_timeout = nla_get_u32(nla); + + nla = ntb[DRBD_A_DISK_CONF_READ_BALANCING]; + if (nla && s) + s->read_balancing = nla_get_u32(nla); + + nla = ntb[DRBD_A_DISK_CONF_AL_UPDATES]; + if (nla && s) + s->al_updates = nla_get_u8(nla); + + nla = ntb[DRBD_A_DISK_CONF_DISCARD_ZEROES_IF_ALIGNED]; + if (nla && s) + s->discard_zeroes_if_aligned = nla_get_u8(nla); + + nla = ntb[DRBD_A_DISK_CONF_RS_DISCARD_GRANULARITY]; + if (nla && s) + s->rs_discard_granularity = nla_get_u32(nla); + + nla = ntb[DRBD_A_DISK_CONF_DISABLE_WRITE_SAME]; + if (nla && s) + s->disable_write_same = nla_get_u8(nla); + +out: + if (ret_nested_attribute_table && (!err || err == -ENOMSG)) + *ret_nested_attribute_table = ntb; + else + kfree(ntb); + return err; +} + +int disk_conf_from_attrs(struct disk_conf *s, + struct genl_info *info) +{ + return __disk_conf_from_attrs(s, NULL, info); +} + +int disk_conf_ntb_from_attrs( + struct nlattr ***ret_nested_attribute_table, + struct genl_info *info) +{ + return __disk_conf_from_attrs(NULL, ret_nested_attribute_table, info); +} + +static int __res_opts_from_attrs(struct res_opts *s, + struct nlattr ***ret_nested_attribute_table, + struct genl_info *info) +{ + const int maxtype = DRBD_A_RES_OPTS_ON_NO_DATA; + struct nlattr *tla = info->attrs[DRBD_NLA_RESOURCE_OPTS]; + struct nlattr **ntb; + struct nlattr *nla; + int err = 0; + + if (ret_nested_attribute_table) + *ret_nested_attribute_table = NULL; + if (!tla) + return -ENOMSG; + ntb = kzalloc_objs(*ntb, DRBD_A_RES_OPTS_ON_NO_DATA + 1); + if (!ntb) + return -ENOMEM; + err = nla_parse_nested_deprecated(ntb, maxtype, tla, drbd_res_opts_nl_policy, NULL); + if (err) + goto out; + + nla = ntb[DRBD_A_RES_OPTS_CPU_MASK]; + if (nla && s) + s->cpu_mask_len = nla_strscpy(s->cpu_mask, nla, DRBD_CPU_MASK_SIZE); + + nla = ntb[DRBD_A_RES_OPTS_ON_NO_DATA]; + if (nla && s) + s->on_no_data = nla_get_u32(nla); + +out: + if (ret_nested_attribute_table && (!err || err == -ENOMSG)) + *ret_nested_attribute_table = ntb; + else + kfree(ntb); + return err; +} + +int res_opts_from_attrs(struct res_opts *s, + struct genl_info *info) +{ + return __res_opts_from_attrs(s, NULL, info); +} + +int res_opts_ntb_from_attrs( + struct nlattr ***ret_nested_attribute_table, + struct genl_info *info) +{ + return __res_opts_from_attrs(NULL, ret_nested_attribute_table, info); +} + +static int __net_conf_from_attrs(struct net_conf *s, + struct nlattr ***ret_nested_attribute_table, + struct genl_info *info) +{ + const int maxtype = DRBD_A_NET_CONF_SOCK_CHECK_TIMEO; + struct nlattr *tla = info->attrs[DRBD_NLA_NET_CONF]; + struct nlattr **ntb; + struct nlattr *nla; + int err = 0; + + if (ret_nested_attribute_table) + *ret_nested_attribute_table = NULL; + if (!tla) + return -ENOMSG; + ntb = kzalloc_objs(*ntb, DRBD_A_NET_CONF_SOCK_CHECK_TIMEO + 1); + if (!ntb) + return -ENOMEM; + err = nla_parse_nested_deprecated(ntb, maxtype, tla, drbd_net_conf_nl_policy, NULL); + if (err) + goto out; + + nla = ntb[DRBD_A_NET_CONF_SHARED_SECRET]; + if (nla && s) + s->shared_secret_len = nla_strscpy(s->shared_secret, nla, SHARED_SECRET_MAX); + + nla = ntb[DRBD_A_NET_CONF_CRAM_HMAC_ALG]; + if (nla && s) + s->cram_hmac_alg_len = nla_strscpy(s->cram_hmac_alg, nla, SHARED_SECRET_MAX); + + nla = ntb[DRBD_A_NET_CONF_INTEGRITY_ALG]; + if (nla && s) + s->integrity_alg_len = nla_strscpy(s->integrity_alg, nla, SHARED_SECRET_MAX); + + nla = ntb[DRBD_A_NET_CONF_VERIFY_ALG]; + if (nla && s) + s->verify_alg_len = nla_strscpy(s->verify_alg, nla, SHARED_SECRET_MAX); + + nla = ntb[DRBD_A_NET_CONF_CSUMS_ALG]; + if (nla && s) + s->csums_alg_len = nla_strscpy(s->csums_alg, nla, SHARED_SECRET_MAX); + + nla = ntb[DRBD_A_NET_CONF_WIRE_PROTOCOL]; + if (nla && s) + s->wire_protocol = nla_get_u32(nla); + + nla = ntb[DRBD_A_NET_CONF_CONNECT_INT]; + if (nla && s) + s->connect_int = nla_get_u32(nla); + + nla = ntb[DRBD_A_NET_CONF_TIMEOUT]; + if (nla && s) + s->timeout = nla_get_u32(nla); + + nla = ntb[DRBD_A_NET_CONF_PING_INT]; + if (nla && s) + s->ping_int = nla_get_u32(nla); + + nla = ntb[DRBD_A_NET_CONF_PING_TIMEO]; + if (nla && s) + s->ping_timeo = nla_get_u32(nla); + + nla = ntb[DRBD_A_NET_CONF_SNDBUF_SIZE]; + if (nla && s) + s->sndbuf_size = nla_get_u32(nla); + + nla = ntb[DRBD_A_NET_CONF_RCVBUF_SIZE]; + if (nla && s) + s->rcvbuf_size = nla_get_u32(nla); + + nla = ntb[DRBD_A_NET_CONF_KO_COUNT]; + if (nla && s) + s->ko_count = nla_get_u32(nla); + + nla = ntb[DRBD_A_NET_CONF_MAX_BUFFERS]; + if (nla && s) + s->max_buffers = nla_get_u32(nla); + + nla = ntb[DRBD_A_NET_CONF_MAX_EPOCH_SIZE]; + if (nla && s) + s->max_epoch_size = nla_get_u32(nla); + + nla = ntb[DRBD_A_NET_CONF_UNPLUG_WATERMARK]; + if (nla && s) + s->unplug_watermark = nla_get_u32(nla); + + nla = ntb[DRBD_A_NET_CONF_AFTER_SB_0P]; + if (nla && s) + s->after_sb_0p = nla_get_u32(nla); + + nla = ntb[DRBD_A_NET_CONF_AFTER_SB_1P]; + if (nla && s) + s->after_sb_1p = nla_get_u32(nla); + + nla = ntb[DRBD_A_NET_CONF_AFTER_SB_2P]; + if (nla && s) + s->after_sb_2p = nla_get_u32(nla); + + nla = ntb[DRBD_A_NET_CONF_RR_CONFLICT]; + if (nla && s) + s->rr_conflict = nla_get_u32(nla); + + nla = ntb[DRBD_A_NET_CONF_ON_CONGESTION]; + if (nla && s) + s->on_congestion = nla_get_u32(nla); + + nla = ntb[DRBD_A_NET_CONF_CONG_FILL]; + if (nla && s) + s->cong_fill = nla_get_u32(nla); + + nla = ntb[DRBD_A_NET_CONF_CONG_EXTENTS]; + if (nla && s) + s->cong_extents = nla_get_u32(nla); + + nla = ntb[DRBD_A_NET_CONF_TWO_PRIMARIES]; + if (nla && s) + s->two_primaries = nla_get_u8(nla); + + nla = ntb[DRBD_A_NET_CONF_DISCARD_MY_DATA]; + if (nla && s) + s->discard_my_data = nla_get_u8(nla); + + nla = ntb[DRBD_A_NET_CONF_TCP_CORK]; + if (nla && s) + s->tcp_cork = nla_get_u8(nla); + + nla = ntb[DRBD_A_NET_CONF_ALWAYS_ASBP]; + if (nla && s) + s->always_asbp = nla_get_u8(nla); + + nla = ntb[DRBD_A_NET_CONF_TENTATIVE]; + if (nla && s) + s->tentative = nla_get_u8(nla); + + nla = ntb[DRBD_A_NET_CONF_USE_RLE]; + if (nla && s) + s->use_rle = nla_get_u8(nla); + + nla = ntb[DRBD_A_NET_CONF_CSUMS_AFTER_CRASH_ONLY]; + if (nla && s) + s->csums_after_crash_only = nla_get_u8(nla); + + nla = ntb[DRBD_A_NET_CONF_SOCK_CHECK_TIMEO]; + if (nla && s) + s->sock_check_timeo = nla_get_u32(nla); + +out: + if (ret_nested_attribute_table && (!err || err == -ENOMSG)) + *ret_nested_attribute_table = ntb; + else + kfree(ntb); + return err; +} + +int net_conf_from_attrs(struct net_conf *s, + struct genl_info *info) +{ + return __net_conf_from_attrs(s, NULL, info); +} + +int net_conf_ntb_from_attrs( + struct nlattr ***ret_nested_attribute_table, + struct genl_info *info) +{ + return __net_conf_from_attrs(NULL, ret_nested_attribute_table, info); +} + +static int __set_role_parms_from_attrs(struct set_role_parms *s, + struct nlattr ***ret_nested_attribute_table, + struct genl_info *info) +{ + const int maxtype = DRBD_A_SET_ROLE_PARMS_ASSUME_UPTODATE; + struct nlattr *tla = info->attrs[DRBD_NLA_SET_ROLE_PARMS]; + struct nlattr **ntb; + struct nlattr *nla; + int err = 0; + + if (ret_nested_attribute_table) + *ret_nested_attribute_table = NULL; + if (!tla) + return -ENOMSG; + ntb = kzalloc_objs(*ntb, DRBD_A_SET_ROLE_PARMS_ASSUME_UPTODATE + 1); + if (!ntb) + return -ENOMEM; + err = nla_parse_nested_deprecated(ntb, maxtype, tla, drbd_set_role_parms_nl_policy, NULL); + if (err) + goto out; + + nla = ntb[DRBD_A_SET_ROLE_PARMS_ASSUME_UPTODATE]; + if (nla && s) + s->assume_uptodate = nla_get_u8(nla); + +out: + if (ret_nested_attribute_table && (!err || err == -ENOMSG)) + *ret_nested_attribute_table = ntb; + else + kfree(ntb); + return err; +} + +int set_role_parms_from_attrs(struct set_role_parms *s, + struct genl_info *info) +{ + return __set_role_parms_from_attrs(s, NULL, info); +} + +int set_role_parms_ntb_from_attrs( + struct nlattr ***ret_nested_attribute_table, + struct genl_info *info) +{ + return __set_role_parms_from_attrs(NULL, ret_nested_attribute_table, info); +} + +static int __resize_parms_from_attrs(struct resize_parms *s, + struct nlattr ***ret_nested_attribute_table, + struct genl_info *info) +{ + const int maxtype = DRBD_A_RESIZE_PARMS_AL_STRIPE_SIZE; + struct nlattr *tla = info->attrs[DRBD_NLA_RESIZE_PARMS]; + struct nlattr **ntb; + struct nlattr *nla; + int err = 0; + + if (ret_nested_attribute_table) + *ret_nested_attribute_table = NULL; + if (!tla) + return -ENOMSG; + ntb = kzalloc_objs(*ntb, DRBD_A_RESIZE_PARMS_AL_STRIPE_SIZE + 1); + if (!ntb) + return -ENOMEM; + err = nla_parse_nested_deprecated(ntb, maxtype, tla, drbd_resize_parms_nl_policy, NULL); + if (err) + goto out; + + nla = ntb[DRBD_A_RESIZE_PARMS_RESIZE_SIZE]; + if (nla && s) + s->resize_size = nla_get_u64(nla); + + nla = ntb[DRBD_A_RESIZE_PARMS_RESIZE_FORCE]; + if (nla && s) + s->resize_force = nla_get_u8(nla); + + nla = ntb[DRBD_A_RESIZE_PARMS_NO_RESYNC]; + if (nla && s) + s->no_resync = nla_get_u8(nla); + + nla = ntb[DRBD_A_RESIZE_PARMS_AL_STRIPES]; + if (nla && s) + s->al_stripes = nla_get_u32(nla); + + nla = ntb[DRBD_A_RESIZE_PARMS_AL_STRIPE_SIZE]; + if (nla && s) + s->al_stripe_size = nla_get_u32(nla); + +out: + if (ret_nested_attribute_table && (!err || err == -ENOMSG)) + *ret_nested_attribute_table = ntb; + else + kfree(ntb); + return err; +} + +int resize_parms_from_attrs(struct resize_parms *s, + struct genl_info *info) +{ + return __resize_parms_from_attrs(s, NULL, info); +} + +int resize_parms_ntb_from_attrs( + struct nlattr ***ret_nested_attribute_table, + struct genl_info *info) +{ + return __resize_parms_from_attrs(NULL, ret_nested_attribute_table, info); +} + +static int __start_ov_parms_from_attrs(struct start_ov_parms *s, + struct nlattr ***ret_nested_attribute_table, + struct genl_info *info) +{ + const int maxtype = DRBD_A_START_OV_PARMS_OV_STOP_SECTOR; + struct nlattr *tla = info->attrs[DRBD_NLA_START_OV_PARMS]; + struct nlattr **ntb; + struct nlattr *nla; + int err = 0; + + if (ret_nested_attribute_table) + *ret_nested_attribute_table = NULL; + if (!tla) + return -ENOMSG; + ntb = kzalloc_objs(*ntb, DRBD_A_START_OV_PARMS_OV_STOP_SECTOR + 1); + if (!ntb) + return -ENOMEM; + err = nla_parse_nested_deprecated(ntb, maxtype, tla, drbd_start_ov_parms_nl_policy, NULL); + if (err) + goto out; + + nla = ntb[DRBD_A_START_OV_PARMS_OV_START_SECTOR]; + if (nla && s) + s->ov_start_sector = nla_get_u64(nla); + + nla = ntb[DRBD_A_START_OV_PARMS_OV_STOP_SECTOR]; + if (nla && s) + s->ov_stop_sector = nla_get_u64(nla); + +out: + if (ret_nested_attribute_table && (!err || err == -ENOMSG)) + *ret_nested_attribute_table = ntb; + else + kfree(ntb); + return err; +} + +int start_ov_parms_from_attrs(struct start_ov_parms *s, + struct genl_info *info) +{ + return __start_ov_parms_from_attrs(s, NULL, info); +} + +int start_ov_parms_ntb_from_attrs( + struct nlattr ***ret_nested_attribute_table, + struct genl_info *info) +{ + return __start_ov_parms_from_attrs(NULL, ret_nested_attribute_table, info); +} + +static int __new_c_uuid_parms_from_attrs(struct new_c_uuid_parms *s, + struct nlattr ***ret_nested_attribute_table, + struct genl_info *info) +{ + const int maxtype = DRBD_A_NEW_C_UUID_PARMS_CLEAR_BM; + struct nlattr *tla = info->attrs[DRBD_NLA_NEW_C_UUID_PARMS]; + struct nlattr **ntb; + struct nlattr *nla; + int err = 0; + + if (ret_nested_attribute_table) + *ret_nested_attribute_table = NULL; + if (!tla) + return -ENOMSG; + ntb = kzalloc_objs(*ntb, DRBD_A_NEW_C_UUID_PARMS_CLEAR_BM + 1); + if (!ntb) + return -ENOMEM; + err = nla_parse_nested_deprecated(ntb, maxtype, tla, drbd_new_c_uuid_parms_nl_policy, NULL); + if (err) + goto out; + + nla = ntb[DRBD_A_NEW_C_UUID_PARMS_CLEAR_BM]; + if (nla && s) + s->clear_bm = nla_get_u8(nla); + +out: + if (ret_nested_attribute_table && (!err || err == -ENOMSG)) + *ret_nested_attribute_table = ntb; + else + kfree(ntb); + return err; +} + +int new_c_uuid_parms_from_attrs(struct new_c_uuid_parms *s, + struct genl_info *info) +{ + return __new_c_uuid_parms_from_attrs(s, NULL, info); +} + +int new_c_uuid_parms_ntb_from_attrs( + struct nlattr ***ret_nested_attribute_table, + struct genl_info *info) +{ + return __new_c_uuid_parms_from_attrs(NULL, ret_nested_attribute_table, info); +} + +static int __disconnect_parms_from_attrs(struct disconnect_parms *s, + struct nlattr ***ret_nested_attribute_table, + struct genl_info *info) +{ + const int maxtype = DRBD_A_DISCONNECT_PARMS_FORCE_DISCONNECT; + struct nlattr *tla = info->attrs[DRBD_NLA_DISCONNECT_PARMS]; + struct nlattr **ntb; + struct nlattr *nla; + int err = 0; + + if (ret_nested_attribute_table) + *ret_nested_attribute_table = NULL; + if (!tla) + return -ENOMSG; + ntb = kzalloc_objs(*ntb, DRBD_A_DISCONNECT_PARMS_FORCE_DISCONNECT + 1); + if (!ntb) + return -ENOMEM; + err = nla_parse_nested_deprecated(ntb, maxtype, tla, drbd_disconnect_parms_nl_policy, NULL); + if (err) + goto out; + + nla = ntb[DRBD_A_DISCONNECT_PARMS_FORCE_DISCONNECT]; + if (nla && s) + s->force_disconnect = nla_get_u8(nla); + +out: + if (ret_nested_attribute_table && (!err || err == -ENOMSG)) + *ret_nested_attribute_table = ntb; + else + kfree(ntb); + return err; +} + +int disconnect_parms_from_attrs(struct disconnect_parms *s, + struct genl_info *info) +{ + return __disconnect_parms_from_attrs(s, NULL, info); +} + +int disconnect_parms_ntb_from_attrs( + struct nlattr ***ret_nested_attribute_table, + struct genl_info *info) +{ + return __disconnect_parms_from_attrs(NULL, ret_nested_attribute_table, info); +} + +static int __detach_parms_from_attrs(struct detach_parms *s, + struct nlattr ***ret_nested_attribute_table, + struct genl_info *info) +{ + const int maxtype = DRBD_A_DETACH_PARMS_FORCE_DETACH; + struct nlattr *tla = info->attrs[DRBD_NLA_DETACH_PARMS]; + struct nlattr **ntb; + struct nlattr *nla; + int err = 0; + + if (ret_nested_attribute_table) + *ret_nested_attribute_table = NULL; + if (!tla) + return -ENOMSG; + ntb = kzalloc_objs(*ntb, DRBD_A_DETACH_PARMS_FORCE_DETACH + 1); + if (!ntb) + return -ENOMEM; + err = nla_parse_nested_deprecated(ntb, maxtype, tla, drbd_detach_parms_nl_policy, NULL); + if (err) + goto out; + + nla = ntb[DRBD_A_DETACH_PARMS_FORCE_DETACH]; + if (nla && s) + s->force_detach = nla_get_u8(nla); + +out: + if (ret_nested_attribute_table && (!err || err == -ENOMSG)) + *ret_nested_attribute_table = ntb; + else + kfree(ntb); + return err; +} + +int detach_parms_from_attrs(struct detach_parms *s, + struct genl_info *info) +{ + return __detach_parms_from_attrs(s, NULL, info); +} + +int detach_parms_ntb_from_attrs( + struct nlattr ***ret_nested_attribute_table, + struct genl_info *info) +{ + return __detach_parms_from_attrs(NULL, ret_nested_attribute_table, info); +} + +static int __resource_info_from_attrs(struct resource_info *s, + struct nlattr ***ret_nested_attribute_table, + struct genl_info *info) +{ + const int maxtype = DRBD_A_RESOURCE_INFO_RES_SUSP_FEN; + struct nlattr *tla = info->attrs[DRBD_NLA_RESOURCE_INFO]; + struct nlattr **ntb; + struct nlattr *nla; + int err = 0; + + if (ret_nested_attribute_table) + *ret_nested_attribute_table = NULL; + if (!tla) + return -ENOMSG; + ntb = kzalloc_objs(*ntb, DRBD_A_RESOURCE_INFO_RES_SUSP_FEN + 1); + if (!ntb) + return -ENOMEM; + err = nla_parse_nested_deprecated(ntb, maxtype, tla, drbd_resource_info_nl_policy, NULL); + if (err) + goto out; + + nla = ntb[DRBD_A_RESOURCE_INFO_RES_ROLE]; + if (nla && s) + s->res_role = nla_get_u32(nla); + + nla = ntb[DRBD_A_RESOURCE_INFO_RES_SUSP]; + if (nla && s) + s->res_susp = nla_get_u8(nla); + + nla = ntb[DRBD_A_RESOURCE_INFO_RES_SUSP_NOD]; + if (nla && s) + s->res_susp_nod = nla_get_u8(nla); + + nla = ntb[DRBD_A_RESOURCE_INFO_RES_SUSP_FEN]; + if (nla && s) + s->res_susp_fen = nla_get_u8(nla); + +out: + if (ret_nested_attribute_table && (!err || err == -ENOMSG)) + *ret_nested_attribute_table = ntb; + else + kfree(ntb); + return err; +} + +int resource_info_from_attrs(struct resource_info *s, + struct genl_info *info) +{ + return __resource_info_from_attrs(s, NULL, info); +} + +int resource_info_ntb_from_attrs( + struct nlattr ***ret_nested_attribute_table, + struct genl_info *info) +{ + return __resource_info_from_attrs(NULL, ret_nested_attribute_table, info); +} + +static int __device_info_from_attrs(struct device_info *s, + struct nlattr ***ret_nested_attribute_table, + struct genl_info *info) +{ + const int maxtype = DRBD_A_DEVICE_INFO_DEV_DISK_STATE; + struct nlattr *tla = info->attrs[DRBD_NLA_DEVICE_INFO]; + struct nlattr **ntb; + struct nlattr *nla; + int err = 0; + + if (ret_nested_attribute_table) + *ret_nested_attribute_table = NULL; + if (!tla) + return -ENOMSG; + ntb = kzalloc_objs(*ntb, DRBD_A_DEVICE_INFO_DEV_DISK_STATE + 1); + if (!ntb) + return -ENOMEM; + err = nla_parse_nested_deprecated(ntb, maxtype, tla, drbd_device_info_nl_policy, NULL); + if (err) + goto out; + + nla = ntb[DRBD_A_DEVICE_INFO_DEV_DISK_STATE]; + if (nla && s) + s->dev_disk_state = nla_get_u32(nla); + +out: + if (ret_nested_attribute_table && (!err || err == -ENOMSG)) + *ret_nested_attribute_table = ntb; + else + kfree(ntb); + return err; +} + +int device_info_from_attrs(struct device_info *s, + struct genl_info *info) +{ + return __device_info_from_attrs(s, NULL, info); +} + +int device_info_ntb_from_attrs( + struct nlattr ***ret_nested_attribute_table, + struct genl_info *info) +{ + return __device_info_from_attrs(NULL, ret_nested_attribute_table, info); +} + +static int __connection_info_from_attrs(struct connection_info *s, + struct nlattr ***ret_nested_attribute_table, + struct genl_info *info) +{ + const int maxtype = DRBD_A_CONNECTION_INFO_CONN_ROLE; + struct nlattr *tla = info->attrs[DRBD_NLA_CONNECTION_INFO]; + struct nlattr **ntb; + struct nlattr *nla; + int err = 0; + + if (ret_nested_attribute_table) + *ret_nested_attribute_table = NULL; + if (!tla) + return -ENOMSG; + ntb = kzalloc_objs(*ntb, DRBD_A_CONNECTION_INFO_CONN_ROLE + 1); + if (!ntb) + return -ENOMEM; + err = nla_parse_nested_deprecated(ntb, maxtype, tla, drbd_connection_info_nl_policy, NULL); + if (err) + goto out; + + nla = ntb[DRBD_A_CONNECTION_INFO_CONN_CONNECTION_STATE]; + if (nla && s) + s->conn_connection_state = nla_get_u32(nla); + + nla = ntb[DRBD_A_CONNECTION_INFO_CONN_ROLE]; + if (nla && s) + s->conn_role = nla_get_u32(nla); + +out: + if (ret_nested_attribute_table && (!err || err == -ENOMSG)) + *ret_nested_attribute_table = ntb; + else + kfree(ntb); + return err; +} + +int connection_info_from_attrs(struct connection_info *s, + struct genl_info *info) +{ + return __connection_info_from_attrs(s, NULL, info); +} + +int connection_info_ntb_from_attrs( + struct nlattr ***ret_nested_attribute_table, + struct genl_info *info) +{ + return __connection_info_from_attrs(NULL, ret_nested_attribute_table, info); +} + +static int __peer_device_info_from_attrs(struct peer_device_info *s, + struct nlattr ***ret_nested_attribute_table, + struct genl_info *info) +{ + const int maxtype = DRBD_A_PEER_DEVICE_INFO_PEER_RESYNC_SUSP_DEPENDENCY; + struct nlattr *tla = info->attrs[DRBD_NLA_PEER_DEVICE_INFO]; + struct nlattr **ntb; + struct nlattr *nla; + int err = 0; + + if (ret_nested_attribute_table) + *ret_nested_attribute_table = NULL; + if (!tla) + return -ENOMSG; + ntb = kzalloc_objs(*ntb, + DRBD_A_PEER_DEVICE_INFO_PEER_RESYNC_SUSP_DEPENDENCY + 1); + if (!ntb) + return -ENOMEM; + err = nla_parse_nested_deprecated(ntb, maxtype, tla, drbd_peer_device_info_nl_policy, NULL); + if (err) + goto out; + + nla = ntb[DRBD_A_PEER_DEVICE_INFO_PEER_REPL_STATE]; + if (nla && s) + s->peer_repl_state = nla_get_u32(nla); + + nla = ntb[DRBD_A_PEER_DEVICE_INFO_PEER_DISK_STATE]; + if (nla && s) + s->peer_disk_state = nla_get_u32(nla); + + nla = ntb[DRBD_A_PEER_DEVICE_INFO_PEER_RESYNC_SUSP_USER]; + if (nla && s) + s->peer_resync_susp_user = nla_get_u32(nla); + + nla = ntb[DRBD_A_PEER_DEVICE_INFO_PEER_RESYNC_SUSP_PEER]; + if (nla && s) + s->peer_resync_susp_peer = nla_get_u32(nla); + + nla = ntb[DRBD_A_PEER_DEVICE_INFO_PEER_RESYNC_SUSP_DEPENDENCY]; + if (nla && s) + s->peer_resync_susp_dependency = nla_get_u32(nla); + +out: + if (ret_nested_attribute_table && (!err || err == -ENOMSG)) + *ret_nested_attribute_table = ntb; + else + kfree(ntb); + return err; +} + +int peer_device_info_from_attrs(struct peer_device_info *s, + struct genl_info *info) +{ + return __peer_device_info_from_attrs(s, NULL, info); +} + +int peer_device_info_ntb_from_attrs( + struct nlattr ***ret_nested_attribute_table, + struct genl_info *info) +{ + return __peer_device_info_from_attrs(NULL, ret_nested_attribute_table, info); +} + +static int __resource_statistics_from_attrs(struct resource_statistics *s, + struct nlattr ***ret_nested_attribute_table, + struct genl_info *info) +{ + const int maxtype = DRBD_A_RESOURCE_STATISTICS_RES_STAT_WRITE_ORDERING; + struct nlattr *tla = info->attrs[DRBD_NLA_RESOURCE_STATISTICS]; + struct nlattr **ntb; + struct nlattr *nla; + int err = 0; + + if (ret_nested_attribute_table) + *ret_nested_attribute_table = NULL; + if (!tla) + return -ENOMSG; + ntb = kzalloc_objs(*ntb, + DRBD_A_RESOURCE_STATISTICS_RES_STAT_WRITE_ORDERING + 1); + if (!ntb) + return -ENOMEM; + err = nla_parse_nested_deprecated(ntb, maxtype, tla, drbd_resource_statistics_nl_policy, NULL); + if (err) + goto out; + + nla = ntb[DRBD_A_RESOURCE_STATISTICS_RES_STAT_WRITE_ORDERING]; + if (nla && s) + s->res_stat_write_ordering = nla_get_u32(nla); + +out: + if (ret_nested_attribute_table && (!err || err == -ENOMSG)) + *ret_nested_attribute_table = ntb; + else + kfree(ntb); + return err; +} + +int resource_statistics_from_attrs(struct resource_statistics *s, + struct genl_info *info) +{ + return __resource_statistics_from_attrs(s, NULL, info); +} + +int resource_statistics_ntb_from_attrs( + struct nlattr ***ret_nested_attribute_table, + struct genl_info *info) +{ + return __resource_statistics_from_attrs(NULL, ret_nested_attribute_table, info); +} + +static int __device_statistics_from_attrs(struct device_statistics *s, + struct nlattr ***ret_nested_attribute_table, + struct genl_info *info) +{ + const int maxtype = DRBD_A_DEVICE_STATISTICS_HISTORY_UUIDS; + struct nlattr *tla = info->attrs[DRBD_NLA_DEVICE_STATISTICS]; + struct nlattr **ntb; + struct nlattr *nla; + int err = 0; + + if (ret_nested_attribute_table) + *ret_nested_attribute_table = NULL; + if (!tla) + return -ENOMSG; + ntb = kzalloc_objs(*ntb, DRBD_A_DEVICE_STATISTICS_HISTORY_UUIDS + 1); + if (!ntb) + return -ENOMEM; + err = nla_parse_nested_deprecated(ntb, maxtype, tla, drbd_device_statistics_nl_policy, NULL); + if (err) + goto out; + + nla = ntb[DRBD_A_DEVICE_STATISTICS_DEV_SIZE]; + if (nla && s) + s->dev_size = nla_get_u64(nla); + + nla = ntb[DRBD_A_DEVICE_STATISTICS_DEV_READ]; + if (nla && s) + s->dev_read = nla_get_u64(nla); + + nla = ntb[DRBD_A_DEVICE_STATISTICS_DEV_WRITE]; + if (nla && s) + s->dev_write = nla_get_u64(nla); + + nla = ntb[DRBD_A_DEVICE_STATISTICS_DEV_AL_WRITES]; + if (nla && s) + s->dev_al_writes = nla_get_u64(nla); + + nla = ntb[DRBD_A_DEVICE_STATISTICS_DEV_BM_WRITES]; + if (nla && s) + s->dev_bm_writes = nla_get_u64(nla); + + nla = ntb[DRBD_A_DEVICE_STATISTICS_DEV_UPPER_PENDING]; + if (nla && s) + s->dev_upper_pending = nla_get_u32(nla); + + nla = ntb[DRBD_A_DEVICE_STATISTICS_DEV_LOWER_PENDING]; + if (nla && s) + s->dev_lower_pending = nla_get_u32(nla); + + nla = ntb[DRBD_A_DEVICE_STATISTICS_DEV_UPPER_BLOCKED]; + if (nla && s) + s->dev_upper_blocked = nla_get_u8(nla); + + nla = ntb[DRBD_A_DEVICE_STATISTICS_DEV_LOWER_BLOCKED]; + if (nla && s) + s->dev_lower_blocked = nla_get_u8(nla); + + nla = ntb[DRBD_A_DEVICE_STATISTICS_DEV_AL_SUSPENDED]; + if (nla && s) + s->dev_al_suspended = nla_get_u8(nla); + + nla = ntb[DRBD_A_DEVICE_STATISTICS_DEV_EXPOSED_DATA_UUID]; + if (nla && s) + s->dev_exposed_data_uuid = nla_get_u64(nla); + + nla = ntb[DRBD_A_DEVICE_STATISTICS_DEV_CURRENT_UUID]; + if (nla && s) + s->dev_current_uuid = nla_get_u64(nla); + + nla = ntb[DRBD_A_DEVICE_STATISTICS_DEV_DISK_FLAGS]; + if (nla && s) + s->dev_disk_flags = nla_get_u32(nla); + + nla = ntb[DRBD_A_DEVICE_STATISTICS_HISTORY_UUIDS]; + if (nla && s) + s->history_uuids_len = nla_memcpy(s->history_uuids, nla, DRBD_NL_HISTORY_UUIDS_SIZE); + +out: + if (ret_nested_attribute_table && (!err || err == -ENOMSG)) + *ret_nested_attribute_table = ntb; + else + kfree(ntb); + return err; +} + +int device_statistics_from_attrs(struct device_statistics *s, + struct genl_info *info) +{ + return __device_statistics_from_attrs(s, NULL, info); +} + +int device_statistics_ntb_from_attrs( + struct nlattr ***ret_nested_attribute_table, + struct genl_info *info) +{ + return __device_statistics_from_attrs(NULL, ret_nested_attribute_table, info); +} + +static int __connection_statistics_from_attrs(struct connection_statistics *s, + struct nlattr ***ret_nested_attribute_table, + struct genl_info *info) +{ + const int maxtype = DRBD_A_CONNECTION_STATISTICS_CONN_CONGESTED; + struct nlattr *tla = info->attrs[DRBD_NLA_CONNECTION_STATISTICS]; + struct nlattr **ntb; + struct nlattr *nla; + int err = 0; + + if (ret_nested_attribute_table) + *ret_nested_attribute_table = NULL; + if (!tla) + return -ENOMSG; + ntb = kzalloc_objs(*ntb, + DRBD_A_CONNECTION_STATISTICS_CONN_CONGESTED + 1); + if (!ntb) + return -ENOMEM; + err = nla_parse_nested_deprecated(ntb, maxtype, tla, drbd_connection_statistics_nl_policy, NULL); + if (err) + goto out; + + nla = ntb[DRBD_A_CONNECTION_STATISTICS_CONN_CONGESTED]; + if (nla && s) + s->conn_congested = nla_get_u8(nla); + +out: + if (ret_nested_attribute_table && (!err || err == -ENOMSG)) + *ret_nested_attribute_table = ntb; + else + kfree(ntb); + return err; +} + +int connection_statistics_from_attrs(struct connection_statistics *s, + struct genl_info *info) +{ + return __connection_statistics_from_attrs(s, NULL, info); +} + +int connection_statistics_ntb_from_attrs( + struct nlattr ***ret_nested_attribute_table, + struct genl_info *info) +{ + return __connection_statistics_from_attrs(NULL, ret_nested_attribute_table, info); +} + +static int __peer_device_statistics_from_attrs(struct peer_device_statistics *s, + struct nlattr ***ret_nested_attribute_table, + struct genl_info *info) +{ + const int maxtype = DRBD_A_PEER_DEVICE_STATISTICS_PEER_DEV_FLAGS; + struct nlattr *tla = info->attrs[DRBD_NLA_PEER_DEVICE_STATISTICS]; + struct nlattr **ntb; + struct nlattr *nla; + int err = 0; + + if (ret_nested_attribute_table) + *ret_nested_attribute_table = NULL; + if (!tla) + return -ENOMSG; + ntb = kzalloc_objs(*ntb, + DRBD_A_PEER_DEVICE_STATISTICS_PEER_DEV_FLAGS + 1); + if (!ntb) + return -ENOMEM; + err = nla_parse_nested_deprecated(ntb, maxtype, tla, drbd_peer_device_statistics_nl_policy, NULL); + if (err) + goto out; + + nla = ntb[DRBD_A_PEER_DEVICE_STATISTICS_PEER_DEV_RECEIVED]; + if (nla && s) + s->peer_dev_received = nla_get_u64(nla); + + nla = ntb[DRBD_A_PEER_DEVICE_STATISTICS_PEER_DEV_SENT]; + if (nla && s) + s->peer_dev_sent = nla_get_u64(nla); + + nla = ntb[DRBD_A_PEER_DEVICE_STATISTICS_PEER_DEV_PENDING]; + if (nla && s) + s->peer_dev_pending = nla_get_u32(nla); + + nla = ntb[DRBD_A_PEER_DEVICE_STATISTICS_PEER_DEV_UNACKED]; + if (nla && s) + s->peer_dev_unacked = nla_get_u32(nla); + + nla = ntb[DRBD_A_PEER_DEVICE_STATISTICS_PEER_DEV_OUT_OF_SYNC]; + if (nla && s) + s->peer_dev_out_of_sync = nla_get_u64(nla); + + nla = ntb[DRBD_A_PEER_DEVICE_STATISTICS_PEER_DEV_RESYNC_FAILED]; + if (nla && s) + s->peer_dev_resync_failed = nla_get_u64(nla); + + nla = ntb[DRBD_A_PEER_DEVICE_STATISTICS_PEER_DEV_BITMAP_UUID]; + if (nla && s) + s->peer_dev_bitmap_uuid = nla_get_u64(nla); + + nla = ntb[DRBD_A_PEER_DEVICE_STATISTICS_PEER_DEV_FLAGS]; + if (nla && s) + s->peer_dev_flags = nla_get_u32(nla); + +out: + if (ret_nested_attribute_table && (!err || err == -ENOMSG)) + *ret_nested_attribute_table = ntb; + else + kfree(ntb); + return err; +} + +int peer_device_statistics_from_attrs(struct peer_device_statistics *s, + struct genl_info *info) +{ + return __peer_device_statistics_from_attrs(s, NULL, info); +} + +int peer_device_statistics_ntb_from_attrs( + struct nlattr ***ret_nested_attribute_table, + struct genl_info *info) +{ + return __peer_device_statistics_from_attrs(NULL, ret_nested_attribute_table, info); +} + +int drbd_cfg_reply_to_skb(struct sk_buff *skb, struct drbd_cfg_reply *s) +{ + struct nlattr *tla = nla_nest_start(skb, DRBD_NLA_CFG_REPLY); + + if (!tla) + goto nla_put_failure; + + if (nla_put(skb, DRBD_A_DRBD_CFG_REPLY_INFO_TEXT, min_t(int, 0, + s->info_text_len + (s->info_text_len < 0)), s->info_text)) + goto nla_put_failure; + + nla_nest_end(skb, tla); + return 0; + +nla_put_failure: + if (tla) + nla_nest_cancel(skb, tla); + return -EMSGSIZE; +} + +int drbd_cfg_context_to_skb(struct sk_buff *skb, struct drbd_cfg_context *s) +{ + struct nlattr *tla = nla_nest_start(skb, DRBD_NLA_CFG_CONTEXT); + + if (!tla) + goto nla_put_failure; + + if (nla_put_u32(skb, DRBD_A_DRBD_CFG_CONTEXT_CTX_VOLUME, s->ctx_volume)) + goto nla_put_failure; + if (nla_put(skb, DRBD_A_DRBD_CFG_CONTEXT_CTX_RESOURCE_NAME, min_t(int, 128, + s->ctx_resource_name_len + (s->ctx_resource_name_len < 128)), s->ctx_resource_name)) + goto nla_put_failure; + if (nla_put(skb, DRBD_A_DRBD_CFG_CONTEXT_CTX_MY_ADDR, min_t(int, 128, + s->ctx_my_addr_len), s->ctx_my_addr)) + goto nla_put_failure; + if (nla_put(skb, DRBD_A_DRBD_CFG_CONTEXT_CTX_PEER_ADDR, min_t(int, 128, + s->ctx_peer_addr_len), s->ctx_peer_addr)) + goto nla_put_failure; + + nla_nest_end(skb, tla); + return 0; + +nla_put_failure: + if (tla) + nla_nest_cancel(skb, tla); + return -EMSGSIZE; +} + +int disk_conf_to_skb(struct sk_buff *skb, struct disk_conf *s) +{ + struct nlattr *tla = nla_nest_start(skb, DRBD_NLA_DISK_CONF); + + if (!tla) + goto nla_put_failure; + + if (nla_put(skb, DRBD_A_DISK_CONF_BACKING_DEV, min_t(int, 128, + s->backing_dev_len + (s->backing_dev_len < 128)), s->backing_dev)) + goto nla_put_failure; + if (nla_put(skb, DRBD_A_DISK_CONF_META_DEV, min_t(int, 128, + s->meta_dev_len + (s->meta_dev_len < 128)), s->meta_dev)) + goto nla_put_failure; + if (nla_put_s32(skb, DRBD_A_DISK_CONF_META_DEV_IDX, s->meta_dev_idx)) + goto nla_put_failure; + if (nla_put_u64_64bit(skb, DRBD_A_DISK_CONF_DISK_SIZE, s->disk_size, 0)) + goto nla_put_failure; + if (nla_put_u32(skb, DRBD_A_DISK_CONF_MAX_BIO_BVECS, s->max_bio_bvecs)) + goto nla_put_failure; + if (nla_put_u32(skb, DRBD_A_DISK_CONF_ON_IO_ERROR, s->on_io_error)) + goto nla_put_failure; + if (nla_put_u32(skb, DRBD_A_DISK_CONF_FENCING, s->fencing)) + goto nla_put_failure; + if (nla_put_u32(skb, DRBD_A_DISK_CONF_RESYNC_RATE, s->resync_rate)) + goto nla_put_failure; + if (nla_put_s32(skb, DRBD_A_DISK_CONF_RESYNC_AFTER, s->resync_after)) + goto nla_put_failure; + if (nla_put_u32(skb, DRBD_A_DISK_CONF_AL_EXTENTS, s->al_extents)) + goto nla_put_failure; + if (nla_put_u32(skb, DRBD_A_DISK_CONF_C_PLAN_AHEAD, s->c_plan_ahead)) + goto nla_put_failure; + if (nla_put_u32(skb, DRBD_A_DISK_CONF_C_DELAY_TARGET, s->c_delay_target)) + goto nla_put_failure; + if (nla_put_u32(skb, DRBD_A_DISK_CONF_C_FILL_TARGET, s->c_fill_target)) + goto nla_put_failure; + if (nla_put_u32(skb, DRBD_A_DISK_CONF_C_MAX_RATE, s->c_max_rate)) + goto nla_put_failure; + if (nla_put_u32(skb, DRBD_A_DISK_CONF_C_MIN_RATE, s->c_min_rate)) + goto nla_put_failure; + if (nla_put_u8(skb, DRBD_A_DISK_CONF_DISK_BARRIER, s->disk_barrier)) + goto nla_put_failure; + if (nla_put_u8(skb, DRBD_A_DISK_CONF_DISK_FLUSHES, s->disk_flushes)) + goto nla_put_failure; + if (nla_put_u8(skb, DRBD_A_DISK_CONF_DISK_DRAIN, s->disk_drain)) + goto nla_put_failure; + if (nla_put_u8(skb, DRBD_A_DISK_CONF_MD_FLUSHES, s->md_flushes)) + goto nla_put_failure; + if (nla_put_u32(skb, DRBD_A_DISK_CONF_DISK_TIMEOUT, s->disk_timeout)) + goto nla_put_failure; + if (nla_put_u32(skb, DRBD_A_DISK_CONF_READ_BALANCING, s->read_balancing)) + goto nla_put_failure; + if (nla_put_u8(skb, DRBD_A_DISK_CONF_AL_UPDATES, s->al_updates)) + goto nla_put_failure; + if (nla_put_u8(skb, DRBD_A_DISK_CONF_DISCARD_ZEROES_IF_ALIGNED, s->discard_zeroes_if_aligned)) + goto nla_put_failure; + if (nla_put_u32(skb, DRBD_A_DISK_CONF_RS_DISCARD_GRANULARITY, s->rs_discard_granularity)) + goto nla_put_failure; + if (nla_put_u8(skb, DRBD_A_DISK_CONF_DISABLE_WRITE_SAME, s->disable_write_same)) + goto nla_put_failure; + + nla_nest_end(skb, tla); + return 0; + +nla_put_failure: + if (tla) + nla_nest_cancel(skb, tla); + return -EMSGSIZE; +} + +int res_opts_to_skb(struct sk_buff *skb, struct res_opts *s) +{ + struct nlattr *tla = nla_nest_start(skb, DRBD_NLA_RESOURCE_OPTS); + + if (!tla) + goto nla_put_failure; + + if (nla_put(skb, DRBD_A_RES_OPTS_CPU_MASK, min_t(int, DRBD_CPU_MASK_SIZE, + s->cpu_mask_len + (s->cpu_mask_len < DRBD_CPU_MASK_SIZE)), s->cpu_mask)) + goto nla_put_failure; + if (nla_put_u32(skb, DRBD_A_RES_OPTS_ON_NO_DATA, s->on_no_data)) + goto nla_put_failure; + + nla_nest_end(skb, tla); + return 0; + +nla_put_failure: + if (tla) + nla_nest_cancel(skb, tla); + return -EMSGSIZE; +} + +int net_conf_to_skb(struct sk_buff *skb, struct net_conf *s) +{ + struct nlattr *tla = nla_nest_start(skb, DRBD_NLA_NET_CONF); + + if (!tla) + goto nla_put_failure; + + if (nla_put(skb, DRBD_A_NET_CONF_SHARED_SECRET, min_t(int, SHARED_SECRET_MAX, + s->shared_secret_len + (s->shared_secret_len < SHARED_SECRET_MAX)), s->shared_secret)) + goto nla_put_failure; + if (nla_put(skb, DRBD_A_NET_CONF_CRAM_HMAC_ALG, min_t(int, SHARED_SECRET_MAX, + s->cram_hmac_alg_len + (s->cram_hmac_alg_len < SHARED_SECRET_MAX)), s->cram_hmac_alg)) + goto nla_put_failure; + if (nla_put(skb, DRBD_A_NET_CONF_INTEGRITY_ALG, min_t(int, SHARED_SECRET_MAX, + s->integrity_alg_len + (s->integrity_alg_len < SHARED_SECRET_MAX)), s->integrity_alg)) + goto nla_put_failure; + if (nla_put(skb, DRBD_A_NET_CONF_VERIFY_ALG, min_t(int, SHARED_SECRET_MAX, + s->verify_alg_len + (s->verify_alg_len < SHARED_SECRET_MAX)), s->verify_alg)) + goto nla_put_failure; + if (nla_put(skb, DRBD_A_NET_CONF_CSUMS_ALG, min_t(int, SHARED_SECRET_MAX, + s->csums_alg_len + (s->csums_alg_len < SHARED_SECRET_MAX)), s->csums_alg)) + goto nla_put_failure; + if (nla_put_u32(skb, DRBD_A_NET_CONF_WIRE_PROTOCOL, s->wire_protocol)) + goto nla_put_failure; + if (nla_put_u32(skb, DRBD_A_NET_CONF_CONNECT_INT, s->connect_int)) + goto nla_put_failure; + if (nla_put_u32(skb, DRBD_A_NET_CONF_TIMEOUT, s->timeout)) + goto nla_put_failure; + if (nla_put_u32(skb, DRBD_A_NET_CONF_PING_INT, s->ping_int)) + goto nla_put_failure; + if (nla_put_u32(skb, DRBD_A_NET_CONF_PING_TIMEO, s->ping_timeo)) + goto nla_put_failure; + if (nla_put_u32(skb, DRBD_A_NET_CONF_SNDBUF_SIZE, s->sndbuf_size)) + goto nla_put_failure; + if (nla_put_u32(skb, DRBD_A_NET_CONF_RCVBUF_SIZE, s->rcvbuf_size)) + goto nla_put_failure; + if (nla_put_u32(skb, DRBD_A_NET_CONF_KO_COUNT, s->ko_count)) + goto nla_put_failure; + if (nla_put_u32(skb, DRBD_A_NET_CONF_MAX_BUFFERS, s->max_buffers)) + goto nla_put_failure; + if (nla_put_u32(skb, DRBD_A_NET_CONF_MAX_EPOCH_SIZE, s->max_epoch_size)) + goto nla_put_failure; + if (nla_put_u32(skb, DRBD_A_NET_CONF_UNPLUG_WATERMARK, s->unplug_watermark)) + goto nla_put_failure; + if (nla_put_u32(skb, DRBD_A_NET_CONF_AFTER_SB_0P, s->after_sb_0p)) + goto nla_put_failure; + if (nla_put_u32(skb, DRBD_A_NET_CONF_AFTER_SB_1P, s->after_sb_1p)) + goto nla_put_failure; + if (nla_put_u32(skb, DRBD_A_NET_CONF_AFTER_SB_2P, s->after_sb_2p)) + goto nla_put_failure; + if (nla_put_u32(skb, DRBD_A_NET_CONF_RR_CONFLICT, s->rr_conflict)) + goto nla_put_failure; + if (nla_put_u32(skb, DRBD_A_NET_CONF_ON_CONGESTION, s->on_congestion)) + goto nla_put_failure; + if (nla_put_u32(skb, DRBD_A_NET_CONF_CONG_FILL, s->cong_fill)) + goto nla_put_failure; + if (nla_put_u32(skb, DRBD_A_NET_CONF_CONG_EXTENTS, s->cong_extents)) + goto nla_put_failure; + if (nla_put_u8(skb, DRBD_A_NET_CONF_TWO_PRIMARIES, s->two_primaries)) + goto nla_put_failure; + if (nla_put_u8(skb, DRBD_A_NET_CONF_DISCARD_MY_DATA, s->discard_my_data)) + goto nla_put_failure; + if (nla_put_u8(skb, DRBD_A_NET_CONF_TCP_CORK, s->tcp_cork)) + goto nla_put_failure; + if (nla_put_u8(skb, DRBD_A_NET_CONF_ALWAYS_ASBP, s->always_asbp)) + goto nla_put_failure; + if (nla_put_u8(skb, DRBD_A_NET_CONF_TENTATIVE, s->tentative)) + goto nla_put_failure; + if (nla_put_u8(skb, DRBD_A_NET_CONF_USE_RLE, s->use_rle)) + goto nla_put_failure; + if (nla_put_u8(skb, DRBD_A_NET_CONF_CSUMS_AFTER_CRASH_ONLY, s->csums_after_crash_only)) + goto nla_put_failure; + if (nla_put_u32(skb, DRBD_A_NET_CONF_SOCK_CHECK_TIMEO, s->sock_check_timeo)) + goto nla_put_failure; + + nla_nest_end(skb, tla); + return 0; + +nla_put_failure: + if (tla) + nla_nest_cancel(skb, tla); + return -EMSGSIZE; +} + +int set_role_parms_to_skb(struct sk_buff *skb, struct set_role_parms *s) +{ + struct nlattr *tla = nla_nest_start(skb, DRBD_NLA_SET_ROLE_PARMS); + + if (!tla) + goto nla_put_failure; + + if (nla_put_u8(skb, DRBD_A_SET_ROLE_PARMS_ASSUME_UPTODATE, s->assume_uptodate)) + goto nla_put_failure; + + nla_nest_end(skb, tla); + return 0; + +nla_put_failure: + if (tla) + nla_nest_cancel(skb, tla); + return -EMSGSIZE; +} + +int resize_parms_to_skb(struct sk_buff *skb, struct resize_parms *s) +{ + struct nlattr *tla = nla_nest_start(skb, DRBD_NLA_RESIZE_PARMS); + + if (!tla) + goto nla_put_failure; + + if (nla_put_u64_64bit(skb, DRBD_A_RESIZE_PARMS_RESIZE_SIZE, s->resize_size, 0)) + goto nla_put_failure; + if (nla_put_u8(skb, DRBD_A_RESIZE_PARMS_RESIZE_FORCE, s->resize_force)) + goto nla_put_failure; + if (nla_put_u8(skb, DRBD_A_RESIZE_PARMS_NO_RESYNC, s->no_resync)) + goto nla_put_failure; + if (nla_put_u32(skb, DRBD_A_RESIZE_PARMS_AL_STRIPES, s->al_stripes)) + goto nla_put_failure; + if (nla_put_u32(skb, DRBD_A_RESIZE_PARMS_AL_STRIPE_SIZE, s->al_stripe_size)) + goto nla_put_failure; + + nla_nest_end(skb, tla); + return 0; + +nla_put_failure: + if (tla) + nla_nest_cancel(skb, tla); + return -EMSGSIZE; +} + +int state_info_to_skb(struct sk_buff *skb, struct state_info *s) +{ + struct nlattr *tla = nla_nest_start(skb, DRBD_NLA_STATE_INFO); + + if (!tla) + goto nla_put_failure; + + if (nla_put_u32(skb, DRBD_A_STATE_INFO_SIB_REASON, s->sib_reason)) + goto nla_put_failure; + if (nla_put_u32(skb, DRBD_A_STATE_INFO_CURRENT_STATE, s->current_state)) + goto nla_put_failure; + if (nla_put_u64_64bit(skb, DRBD_A_STATE_INFO_CAPACITY, s->capacity, 0)) + goto nla_put_failure; + if (nla_put_u64_64bit(skb, DRBD_A_STATE_INFO_ED_UUID, s->ed_uuid, 0)) + goto nla_put_failure; + if (nla_put_u32(skb, DRBD_A_STATE_INFO_PREV_STATE, s->prev_state)) + goto nla_put_failure; + if (nla_put_u32(skb, DRBD_A_STATE_INFO_NEW_STATE, s->new_state)) + goto nla_put_failure; + if (nla_put(skb, DRBD_A_STATE_INFO_UUIDS, min_t(int, DRBD_NL_UUIDS_SIZE, + s->uuids_len), s->uuids)) + goto nla_put_failure; + if (nla_put_u32(skb, DRBD_A_STATE_INFO_DISK_FLAGS, s->disk_flags)) + goto nla_put_failure; + if (nla_put_u64_64bit(skb, DRBD_A_STATE_INFO_BITS_TOTAL, s->bits_total, 0)) + goto nla_put_failure; + if (nla_put_u64_64bit(skb, DRBD_A_STATE_INFO_BITS_OOS, s->bits_oos, 0)) + goto nla_put_failure; + if (nla_put_u64_64bit(skb, DRBD_A_STATE_INFO_BITS_RS_TOTAL, s->bits_rs_total, 0)) + goto nla_put_failure; + if (nla_put_u64_64bit(skb, DRBD_A_STATE_INFO_BITS_RS_FAILED, s->bits_rs_failed, 0)) + goto nla_put_failure; + if (nla_put(skb, DRBD_A_STATE_INFO_HELPER, min_t(int, 32, + s->helper_len + (s->helper_len < 32)), s->helper)) + goto nla_put_failure; + if (nla_put_u32(skb, DRBD_A_STATE_INFO_HELPER_EXIT_CODE, s->helper_exit_code)) + goto nla_put_failure; + if (nla_put_u64_64bit(skb, DRBD_A_STATE_INFO_SEND_CNT, s->send_cnt, 0)) + goto nla_put_failure; + if (nla_put_u64_64bit(skb, DRBD_A_STATE_INFO_RECV_CNT, s->recv_cnt, 0)) + goto nla_put_failure; + if (nla_put_u64_64bit(skb, DRBD_A_STATE_INFO_READ_CNT, s->read_cnt, 0)) + goto nla_put_failure; + if (nla_put_u64_64bit(skb, DRBD_A_STATE_INFO_WRIT_CNT, s->writ_cnt, 0)) + goto nla_put_failure; + if (nla_put_u64_64bit(skb, DRBD_A_STATE_INFO_AL_WRIT_CNT, s->al_writ_cnt, 0)) + goto nla_put_failure; + if (nla_put_u64_64bit(skb, DRBD_A_STATE_INFO_BM_WRIT_CNT, s->bm_writ_cnt, 0)) + goto nla_put_failure; + if (nla_put_u32(skb, DRBD_A_STATE_INFO_AP_BIO_CNT, s->ap_bio_cnt)) + goto nla_put_failure; + if (nla_put_u32(skb, DRBD_A_STATE_INFO_AP_PENDING_CNT, s->ap_pending_cnt)) + goto nla_put_failure; + if (nla_put_u32(skb, DRBD_A_STATE_INFO_RS_PENDING_CNT, s->rs_pending_cnt)) + goto nla_put_failure; + + nla_nest_end(skb, tla); + return 0; + +nla_put_failure: + if (tla) + nla_nest_cancel(skb, tla); + return -EMSGSIZE; +} + +int start_ov_parms_to_skb(struct sk_buff *skb, struct start_ov_parms *s) +{ + struct nlattr *tla = nla_nest_start(skb, DRBD_NLA_START_OV_PARMS); + + if (!tla) + goto nla_put_failure; + + if (nla_put_u64_64bit(skb, DRBD_A_START_OV_PARMS_OV_START_SECTOR, s->ov_start_sector, 0)) + goto nla_put_failure; + if (nla_put_u64_64bit(skb, DRBD_A_START_OV_PARMS_OV_STOP_SECTOR, s->ov_stop_sector, 0)) + goto nla_put_failure; + + nla_nest_end(skb, tla); + return 0; + +nla_put_failure: + if (tla) + nla_nest_cancel(skb, tla); + return -EMSGSIZE; +} + +int new_c_uuid_parms_to_skb(struct sk_buff *skb, struct new_c_uuid_parms *s) +{ + struct nlattr *tla = nla_nest_start(skb, DRBD_NLA_NEW_C_UUID_PARMS); + + if (!tla) + goto nla_put_failure; + + if (nla_put_u8(skb, DRBD_A_NEW_C_UUID_PARMS_CLEAR_BM, s->clear_bm)) + goto nla_put_failure; + + nla_nest_end(skb, tla); + return 0; + +nla_put_failure: + if (tla) + nla_nest_cancel(skb, tla); + return -EMSGSIZE; +} + +int timeout_parms_to_skb(struct sk_buff *skb, struct timeout_parms *s) +{ + struct nlattr *tla = nla_nest_start(skb, DRBD_NLA_TIMEOUT_PARMS); + + if (!tla) + goto nla_put_failure; + + if (nla_put_u32(skb, DRBD_A_TIMEOUT_PARMS_TIMEOUT_TYPE, s->timeout_type)) + goto nla_put_failure; + + nla_nest_end(skb, tla); + return 0; + +nla_put_failure: + if (tla) + nla_nest_cancel(skb, tla); + return -EMSGSIZE; +} + +int disconnect_parms_to_skb(struct sk_buff *skb, struct disconnect_parms *s) +{ + struct nlattr *tla = nla_nest_start(skb, DRBD_NLA_DISCONNECT_PARMS); + + if (!tla) + goto nla_put_failure; + + if (nla_put_u8(skb, DRBD_A_DISCONNECT_PARMS_FORCE_DISCONNECT, s->force_disconnect)) + goto nla_put_failure; + + nla_nest_end(skb, tla); + return 0; + +nla_put_failure: + if (tla) + nla_nest_cancel(skb, tla); + return -EMSGSIZE; +} + +int detach_parms_to_skb(struct sk_buff *skb, struct detach_parms *s) +{ + struct nlattr *tla = nla_nest_start(skb, DRBD_NLA_DETACH_PARMS); + + if (!tla) + goto nla_put_failure; + + if (nla_put_u8(skb, DRBD_A_DETACH_PARMS_FORCE_DETACH, s->force_detach)) + goto nla_put_failure; + + nla_nest_end(skb, tla); + return 0; + +nla_put_failure: + if (tla) + nla_nest_cancel(skb, tla); + return -EMSGSIZE; +} + +int resource_info_to_skb(struct sk_buff *skb, struct resource_info *s) +{ + struct nlattr *tla = nla_nest_start(skb, DRBD_NLA_RESOURCE_INFO); + + if (!tla) + goto nla_put_failure; + + if (nla_put_u32(skb, DRBD_A_RESOURCE_INFO_RES_ROLE, s->res_role)) + goto nla_put_failure; + if (nla_put_u8(skb, DRBD_A_RESOURCE_INFO_RES_SUSP, s->res_susp)) + goto nla_put_failure; + if (nla_put_u8(skb, DRBD_A_RESOURCE_INFO_RES_SUSP_NOD, s->res_susp_nod)) + goto nla_put_failure; + if (nla_put_u8(skb, DRBD_A_RESOURCE_INFO_RES_SUSP_FEN, s->res_susp_fen)) + goto nla_put_failure; + + nla_nest_end(skb, tla); + return 0; + +nla_put_failure: + if (tla) + nla_nest_cancel(skb, tla); + return -EMSGSIZE; +} + +int device_info_to_skb(struct sk_buff *skb, struct device_info *s) +{ + struct nlattr *tla = nla_nest_start(skb, DRBD_NLA_DEVICE_INFO); + + if (!tla) + goto nla_put_failure; + + if (nla_put_u32(skb, DRBD_A_DEVICE_INFO_DEV_DISK_STATE, s->dev_disk_state)) + goto nla_put_failure; + + nla_nest_end(skb, tla); + return 0; + +nla_put_failure: + if (tla) + nla_nest_cancel(skb, tla); + return -EMSGSIZE; +} + +int connection_info_to_skb(struct sk_buff *skb, struct connection_info *s) +{ + struct nlattr *tla = nla_nest_start(skb, DRBD_NLA_CONNECTION_INFO); + + if (!tla) + goto nla_put_failure; + + if (nla_put_u32(skb, DRBD_A_CONNECTION_INFO_CONN_CONNECTION_STATE, s->conn_connection_state)) + goto nla_put_failure; + if (nla_put_u32(skb, DRBD_A_CONNECTION_INFO_CONN_ROLE, s->conn_role)) + goto nla_put_failure; + + nla_nest_end(skb, tla); + return 0; + +nla_put_failure: + if (tla) + nla_nest_cancel(skb, tla); + return -EMSGSIZE; +} + +int peer_device_info_to_skb(struct sk_buff *skb, struct peer_device_info *s) +{ + struct nlattr *tla = nla_nest_start(skb, DRBD_NLA_PEER_DEVICE_INFO); + + if (!tla) + goto nla_put_failure; + + if (nla_put_u32(skb, DRBD_A_PEER_DEVICE_INFO_PEER_REPL_STATE, s->peer_repl_state)) + goto nla_put_failure; + if (nla_put_u32(skb, DRBD_A_PEER_DEVICE_INFO_PEER_DISK_STATE, s->peer_disk_state)) + goto nla_put_failure; + if (nla_put_u32(skb, DRBD_A_PEER_DEVICE_INFO_PEER_RESYNC_SUSP_USER, s->peer_resync_susp_user)) + goto nla_put_failure; + if (nla_put_u32(skb, DRBD_A_PEER_DEVICE_INFO_PEER_RESYNC_SUSP_PEER, s->peer_resync_susp_peer)) + goto nla_put_failure; + if (nla_put_u32(skb, DRBD_A_PEER_DEVICE_INFO_PEER_RESYNC_SUSP_DEPENDENCY, s->peer_resync_susp_dependency)) + goto nla_put_failure; + + nla_nest_end(skb, tla); + return 0; + +nla_put_failure: + if (tla) + nla_nest_cancel(skb, tla); + return -EMSGSIZE; +} + +int resource_statistics_to_skb(struct sk_buff *skb, struct resource_statistics *s) +{ + struct nlattr *tla = nla_nest_start(skb, DRBD_NLA_RESOURCE_STATISTICS); + + if (!tla) + goto nla_put_failure; + + if (nla_put_u32(skb, DRBD_A_RESOURCE_STATISTICS_RES_STAT_WRITE_ORDERING, s->res_stat_write_ordering)) + goto nla_put_failure; + + nla_nest_end(skb, tla); + return 0; + +nla_put_failure: + if (tla) + nla_nest_cancel(skb, tla); + return -EMSGSIZE; +} + +int device_statistics_to_skb(struct sk_buff *skb, struct device_statistics *s) +{ + struct nlattr *tla = nla_nest_start(skb, DRBD_NLA_DEVICE_STATISTICS); + + if (!tla) + goto nla_put_failure; + + if (nla_put_u64_64bit(skb, DRBD_A_DEVICE_STATISTICS_DEV_SIZE, s->dev_size, 0)) + goto nla_put_failure; + if (nla_put_u64_64bit(skb, DRBD_A_DEVICE_STATISTICS_DEV_READ, s->dev_read, 0)) + goto nla_put_failure; + if (nla_put_u64_64bit(skb, DRBD_A_DEVICE_STATISTICS_DEV_WRITE, s->dev_write, 0)) + goto nla_put_failure; + if (nla_put_u64_64bit(skb, DRBD_A_DEVICE_STATISTICS_DEV_AL_WRITES, s->dev_al_writes, 0)) + goto nla_put_failure; + if (nla_put_u64_64bit(skb, DRBD_A_DEVICE_STATISTICS_DEV_BM_WRITES, s->dev_bm_writes, 0)) + goto nla_put_failure; + if (nla_put_u32(skb, DRBD_A_DEVICE_STATISTICS_DEV_UPPER_PENDING, s->dev_upper_pending)) + goto nla_put_failure; + if (nla_put_u32(skb, DRBD_A_DEVICE_STATISTICS_DEV_LOWER_PENDING, s->dev_lower_pending)) + goto nla_put_failure; + if (nla_put_u8(skb, DRBD_A_DEVICE_STATISTICS_DEV_UPPER_BLOCKED, s->dev_upper_blocked)) + goto nla_put_failure; + if (nla_put_u8(skb, DRBD_A_DEVICE_STATISTICS_DEV_LOWER_BLOCKED, s->dev_lower_blocked)) + goto nla_put_failure; + if (nla_put_u8(skb, DRBD_A_DEVICE_STATISTICS_DEV_AL_SUSPENDED, s->dev_al_suspended)) + goto nla_put_failure; + if (nla_put_u64_64bit(skb, DRBD_A_DEVICE_STATISTICS_DEV_EXPOSED_DATA_UUID, s->dev_exposed_data_uuid, 0)) + goto nla_put_failure; + if (nla_put_u64_64bit(skb, DRBD_A_DEVICE_STATISTICS_DEV_CURRENT_UUID, s->dev_current_uuid, 0)) + goto nla_put_failure; + if (nla_put_u32(skb, DRBD_A_DEVICE_STATISTICS_DEV_DISK_FLAGS, s->dev_disk_flags)) + goto nla_put_failure; + if (nla_put(skb, DRBD_A_DEVICE_STATISTICS_HISTORY_UUIDS, min_t(int, DRBD_NL_HISTORY_UUIDS_SIZE, + s->history_uuids_len), s->history_uuids)) + goto nla_put_failure; + + nla_nest_end(skb, tla); + return 0; + +nla_put_failure: + if (tla) + nla_nest_cancel(skb, tla); + return -EMSGSIZE; +} + +int connection_statistics_to_skb(struct sk_buff *skb, struct connection_statistics *s) +{ + struct nlattr *tla = nla_nest_start(skb, DRBD_NLA_CONNECTION_STATISTICS); + + if (!tla) + goto nla_put_failure; + + if (nla_put_u8(skb, DRBD_A_CONNECTION_STATISTICS_CONN_CONGESTED, s->conn_congested)) + goto nla_put_failure; + + nla_nest_end(skb, tla); + return 0; + +nla_put_failure: + if (tla) + nla_nest_cancel(skb, tla); + return -EMSGSIZE; +} + +int peer_device_statistics_to_skb(struct sk_buff *skb, struct peer_device_statistics *s) +{ + struct nlattr *tla = nla_nest_start(skb, DRBD_NLA_PEER_DEVICE_STATISTICS); + + if (!tla) + goto nla_put_failure; + + if (nla_put_u64_64bit(skb, DRBD_A_PEER_DEVICE_STATISTICS_PEER_DEV_RECEIVED, s->peer_dev_received, 0)) + goto nla_put_failure; + if (nla_put_u64_64bit(skb, DRBD_A_PEER_DEVICE_STATISTICS_PEER_DEV_SENT, s->peer_dev_sent, 0)) + goto nla_put_failure; + if (nla_put_u32(skb, DRBD_A_PEER_DEVICE_STATISTICS_PEER_DEV_PENDING, s->peer_dev_pending)) + goto nla_put_failure; + if (nla_put_u32(skb, DRBD_A_PEER_DEVICE_STATISTICS_PEER_DEV_UNACKED, s->peer_dev_unacked)) + goto nla_put_failure; + if (nla_put_u64_64bit(skb, DRBD_A_PEER_DEVICE_STATISTICS_PEER_DEV_OUT_OF_SYNC, s->peer_dev_out_of_sync, 0)) + goto nla_put_failure; + if (nla_put_u64_64bit(skb, DRBD_A_PEER_DEVICE_STATISTICS_PEER_DEV_RESYNC_FAILED, s->peer_dev_resync_failed, 0)) + goto nla_put_failure; + if (nla_put_u64_64bit(skb, DRBD_A_PEER_DEVICE_STATISTICS_PEER_DEV_BITMAP_UUID, s->peer_dev_bitmap_uuid, 0)) + goto nla_put_failure; + if (nla_put_u32(skb, DRBD_A_PEER_DEVICE_STATISTICS_PEER_DEV_FLAGS, s->peer_dev_flags)) + goto nla_put_failure; + + nla_nest_end(skb, tla); + return 0; + +nla_put_failure: + if (tla) + nla_nest_cancel(skb, tla); + return -EMSGSIZE; +} + +int drbd_notification_header_to_skb(struct sk_buff *skb, struct drbd_notification_header *s) +{ + struct nlattr *tla = nla_nest_start(skb, DRBD_NLA_NOTIFICATION_HEADER); + + if (!tla) + goto nla_put_failure; + + if (nla_put_u32(skb, DRBD_A_DRBD_NOTIFICATION_HEADER_NH_TYPE, s->nh_type)) + goto nla_put_failure; + + nla_nest_end(skb, tla); + return 0; + +nla_put_failure: + if (tla) + nla_nest_cancel(skb, tla); + return -EMSGSIZE; +} + +int drbd_helper_info_to_skb(struct sk_buff *skb, struct drbd_helper_info *s) +{ + struct nlattr *tla = nla_nest_start(skb, DRBD_NLA_HELPER); + + if (!tla) + goto nla_put_failure; + + if (nla_put(skb, DRBD_A_DRBD_HELPER_INFO_HELPER_NAME, min_t(int, 32, + s->helper_name_len + (s->helper_name_len < 32)), s->helper_name)) + goto nla_put_failure; + if (nla_put_u32(skb, DRBD_A_DRBD_HELPER_INFO_HELPER_STATUS, s->helper_status)) + goto nla_put_failure; + + nla_nest_end(skb, tla); + return 0; + +nla_put_failure: + if (tla) + nla_nest_cancel(skb, tla); + return -EMSGSIZE; +} + +void set_disk_conf_defaults(struct disk_conf *x) +{ + x->on_io_error = DRBD_ON_IO_ERROR_DEF; + x->fencing = DRBD_FENCING_DEF; + x->resync_rate = DRBD_RESYNC_RATE_DEF; + x->resync_after = DRBD_MINOR_NUMBER_DEF; + x->al_extents = DRBD_AL_EXTENTS_DEF; + x->c_plan_ahead = DRBD_C_PLAN_AHEAD_DEF; + x->c_delay_target = DRBD_C_DELAY_TARGET_DEF; + x->c_fill_target = DRBD_C_FILL_TARGET_DEF; + x->c_max_rate = DRBD_C_MAX_RATE_DEF; + x->c_min_rate = DRBD_C_MIN_RATE_DEF; + x->disk_barrier = DRBD_DISK_BARRIER_DEF; + x->disk_flushes = DRBD_DISK_FLUSHES_DEF; + x->disk_drain = DRBD_DISK_DRAIN_DEF; + x->md_flushes = DRBD_MD_FLUSHES_DEF; + x->disk_timeout = DRBD_DISK_TIMEOUT_DEF; + x->read_balancing = DRBD_READ_BALANCING_DEF; + x->al_updates = DRBD_AL_UPDATES_DEF; + x->discard_zeroes_if_aligned = DRBD_DISCARD_ZEROES_IF_ALIGNED_DEF; + x->rs_discard_granularity = DRBD_RS_DISCARD_GRANULARITY_DEF; + x->disable_write_same = DRBD_DISABLE_WRITE_SAME_DEF; +} + +void set_res_opts_defaults(struct res_opts *x) +{ + memset(x->cpu_mask, 0, sizeof(x->cpu_mask)); + x->cpu_mask_len = 0; + x->on_no_data = DRBD_ON_NO_DATA_DEF; +} + +void set_net_conf_defaults(struct net_conf *x) +{ + memset(x->shared_secret, 0, sizeof(x->shared_secret)); + x->shared_secret_len = 0; + memset(x->cram_hmac_alg, 0, sizeof(x->cram_hmac_alg)); + x->cram_hmac_alg_len = 0; + memset(x->integrity_alg, 0, sizeof(x->integrity_alg)); + x->integrity_alg_len = 0; + memset(x->verify_alg, 0, sizeof(x->verify_alg)); + x->verify_alg_len = 0; + memset(x->csums_alg, 0, sizeof(x->csums_alg)); + x->csums_alg_len = 0; + x->wire_protocol = DRBD_PROTOCOL_DEF; + x->connect_int = DRBD_CONNECT_INT_DEF; + x->timeout = DRBD_TIMEOUT_DEF; + x->ping_int = DRBD_PING_INT_DEF; + x->ping_timeo = DRBD_PING_TIMEO_DEF; + x->sndbuf_size = DRBD_SNDBUF_SIZE_DEF; + x->rcvbuf_size = DRBD_RCVBUF_SIZE_DEF; + x->ko_count = DRBD_KO_COUNT_DEF; + x->max_buffers = DRBD_MAX_BUFFERS_DEF; + x->max_epoch_size = DRBD_MAX_EPOCH_SIZE_DEF; + x->unplug_watermark = DRBD_UNPLUG_WATERMARK_DEF; + x->after_sb_0p = DRBD_AFTER_SB_0P_DEF; + x->after_sb_1p = DRBD_AFTER_SB_1P_DEF; + x->after_sb_2p = DRBD_AFTER_SB_2P_DEF; + x->rr_conflict = DRBD_RR_CONFLICT_DEF; + x->on_congestion = DRBD_ON_CONGESTION_DEF; + x->cong_fill = DRBD_CONG_FILL_DEF; + x->cong_extents = DRBD_CONG_EXTENTS_DEF; + x->two_primaries = DRBD_ALLOW_TWO_PRIMARIES_DEF; + x->tcp_cork = DRBD_TCP_CORK_DEF; + x->always_asbp = DRBD_ALWAYS_ASBP_DEF; + x->use_rle = DRBD_USE_RLE_DEF; + x->csums_after_crash_only = DRBD_CSUMS_AFTER_CRASH_ONLY_DEF; + x->sock_check_timeo = DRBD_SOCKET_CHECK_TIMEO_DEF; +} + +void set_resize_parms_defaults(struct resize_parms *x) +{ + x->al_stripes = DRBD_AL_STRIPES_DEF; + x->al_stripe_size = DRBD_AL_STRIPE_SIZE_DEF; +} diff --git a/drivers/block/drbd/drbd_nl_gen.h b/drivers/block/drbd/drbd_nl_gen.h new file mode 100644 index 000000000000..f2140dd1ac4e --- /dev/null +++ b/drivers/block/drbd/drbd_nl_gen.h @@ -0,0 +1,395 @@ +/* SPDX-License-Identifier: ((GPL-2.0 WITH Linux-syscall-note) OR BSD-3-Clause) */ + +#ifndef _LINUX_DRBD_GEN_H +#define _LINUX_DRBD_GEN_H + +#include <net/netlink.h> +#include <net/genetlink.h> + +#include <uapi/linux/drbd_genl.h> +#include <linux/drbd.h> +#include <linux/drbd_limits.h> + +/* Common nested types */ +extern const struct nla_policy drbd_connection_info_nl_policy[DRBD_A_CONNECTION_INFO_CONN_ROLE + 1]; +extern const struct nla_policy drbd_connection_statistics_nl_policy[DRBD_A_CONNECTION_STATISTICS_CONN_CONGESTED + 1]; +extern const struct nla_policy drbd_detach_parms_nl_policy[DRBD_A_DETACH_PARMS_FORCE_DETACH + 1]; +extern const struct nla_policy drbd_device_info_nl_policy[DRBD_A_DEVICE_INFO_DEV_DISK_STATE + 1]; +extern const struct nla_policy drbd_device_statistics_nl_policy[DRBD_A_DEVICE_STATISTICS_HISTORY_UUIDS + 1]; +extern const struct nla_policy drbd_disconnect_parms_nl_policy[DRBD_A_DISCONNECT_PARMS_FORCE_DISCONNECT + 1]; +extern const struct nla_policy drbd_disk_conf_nl_policy[DRBD_A_DISK_CONF_DISABLE_WRITE_SAME + 1]; +extern const struct nla_policy drbd_drbd_cfg_context_nl_policy[DRBD_A_DRBD_CFG_CONTEXT_CTX_PEER_ADDR + 1]; +extern const struct nla_policy drbd_net_conf_nl_policy[DRBD_A_NET_CONF_SOCK_CHECK_TIMEO + 1]; +extern const struct nla_policy drbd_new_c_uuid_parms_nl_policy[DRBD_A_NEW_C_UUID_PARMS_CLEAR_BM + 1]; +extern const struct nla_policy drbd_peer_device_info_nl_policy[DRBD_A_PEER_DEVICE_INFO_PEER_RESYNC_SUSP_DEPENDENCY + 1]; +extern const struct nla_policy drbd_peer_device_statistics_nl_policy[DRBD_A_PEER_DEVICE_STATISTICS_PEER_DEV_FLAGS + 1]; +extern const struct nla_policy drbd_res_opts_nl_policy[DRBD_A_RES_OPTS_ON_NO_DATA + 1]; +extern const struct nla_policy drbd_resize_parms_nl_policy[DRBD_A_RESIZE_PARMS_AL_STRIPE_SIZE + 1]; +extern const struct nla_policy drbd_resource_info_nl_policy[DRBD_A_RESOURCE_INFO_RES_SUSP_FEN + 1]; +extern const struct nla_policy drbd_resource_statistics_nl_policy[DRBD_A_RESOURCE_STATISTICS_RES_STAT_WRITE_ORDERING + 1]; +extern const struct nla_policy drbd_set_role_parms_nl_policy[DRBD_A_SET_ROLE_PARMS_ASSUME_UPTODATE + 1]; +extern const struct nla_policy drbd_start_ov_parms_nl_policy[DRBD_A_START_OV_PARMS_OV_STOP_SECTOR + 1]; + +/* Ops table for drbd */ +extern const struct genl_split_ops drbd_nl_ops[32]; + +int drbd_pre_doit(const struct genl_split_ops *ops, struct sk_buff *skb, + struct genl_info *info); +void +drbd_post_doit(const struct genl_split_ops *ops, struct sk_buff *skb, + struct genl_info *info); +int drbd_adm_dump_devices_done(struct netlink_callback *cb); +int drbd_adm_dump_connections_done(struct netlink_callback *cb); +int drbd_adm_dump_peer_devices_done(struct netlink_callback *cb); + +int drbd_nl_get_status_doit(struct sk_buff *skb, struct genl_info *info); +int drbd_nl_get_status_dumpit(struct sk_buff *skb, struct netlink_callback *cb); +int drbd_nl_new_minor_doit(struct sk_buff *skb, struct genl_info *info); +int drbd_nl_del_minor_doit(struct sk_buff *skb, struct genl_info *info); +int drbd_nl_new_resource_doit(struct sk_buff *skb, struct genl_info *info); +int drbd_nl_del_resource_doit(struct sk_buff *skb, struct genl_info *info); +int drbd_nl_resource_opts_doit(struct sk_buff *skb, struct genl_info *info); +int drbd_nl_connect_doit(struct sk_buff *skb, struct genl_info *info); +int drbd_nl_disconnect_doit(struct sk_buff *skb, struct genl_info *info); +int drbd_nl_attach_doit(struct sk_buff *skb, struct genl_info *info); +int drbd_nl_resize_doit(struct sk_buff *skb, struct genl_info *info); +int drbd_nl_primary_doit(struct sk_buff *skb, struct genl_info *info); +int drbd_nl_secondary_doit(struct sk_buff *skb, struct genl_info *info); +int drbd_nl_new_c_uuid_doit(struct sk_buff *skb, struct genl_info *info); +int drbd_nl_start_ov_doit(struct sk_buff *skb, struct genl_info *info); +int drbd_nl_detach_doit(struct sk_buff *skb, struct genl_info *info); +int drbd_nl_invalidate_doit(struct sk_buff *skb, struct genl_info *info); +int drbd_nl_inval_peer_doit(struct sk_buff *skb, struct genl_info *info); +int drbd_nl_pause_sync_doit(struct sk_buff *skb, struct genl_info *info); +int drbd_nl_resume_sync_doit(struct sk_buff *skb, struct genl_info *info); +int drbd_nl_suspend_io_doit(struct sk_buff *skb, struct genl_info *info); +int drbd_nl_resume_io_doit(struct sk_buff *skb, struct genl_info *info); +int drbd_nl_outdate_doit(struct sk_buff *skb, struct genl_info *info); +int drbd_nl_get_timeout_type_doit(struct sk_buff *skb, struct genl_info *info); +int drbd_nl_down_doit(struct sk_buff *skb, struct genl_info *info); +int drbd_nl_chg_disk_opts_doit(struct sk_buff *skb, struct genl_info *info); +int drbd_nl_chg_net_opts_doit(struct sk_buff *skb, struct genl_info *info); +int drbd_nl_get_resources_dumpit(struct sk_buff *skb, + struct netlink_callback *cb); +int drbd_nl_get_devices_dumpit(struct sk_buff *skb, + struct netlink_callback *cb); +int drbd_nl_get_connections_dumpit(struct sk_buff *skb, + struct netlink_callback *cb); +int drbd_nl_get_peer_devices_dumpit(struct sk_buff *skb, + struct netlink_callback *cb); +int drbd_nl_get_initial_state_dumpit(struct sk_buff *skb, + struct netlink_callback *cb); + +enum { + DRBD_NLGRP_EVENTS, +}; + +struct drbd_cfg_reply { + char info_text[0]; + __u32 info_text_len; +}; + +struct drbd_cfg_context { + __u32 ctx_volume; + char ctx_resource_name[128]; + __u32 ctx_resource_name_len; + char ctx_my_addr[128]; + __u32 ctx_my_addr_len; + char ctx_peer_addr[128]; + __u32 ctx_peer_addr_len; +}; + +struct disk_conf { + char backing_dev[128]; + __u32 backing_dev_len; + char meta_dev[128]; + __u32 meta_dev_len; + __s32 meta_dev_idx; + __u64 disk_size; + __u32 max_bio_bvecs; + __u32 on_io_error; + __u32 fencing; + __u32 resync_rate; + __s32 resync_after; + __u32 al_extents; + __u32 c_plan_ahead; + __u32 c_delay_target; + __u32 c_fill_target; + __u32 c_max_rate; + __u32 c_min_rate; + unsigned char disk_barrier; + unsigned char disk_flushes; + unsigned char disk_drain; + unsigned char md_flushes; + __u32 disk_timeout; + __u32 read_balancing; + unsigned char al_updates; + unsigned char discard_zeroes_if_aligned; + __u32 rs_discard_granularity; + unsigned char disable_write_same; +}; + +struct res_opts { + char cpu_mask[DRBD_CPU_MASK_SIZE]; + __u32 cpu_mask_len; + __u32 on_no_data; +}; + +struct net_conf { + char shared_secret[SHARED_SECRET_MAX]; + __u32 shared_secret_len; + char cram_hmac_alg[SHARED_SECRET_MAX]; + __u32 cram_hmac_alg_len; + char integrity_alg[SHARED_SECRET_MAX]; + __u32 integrity_alg_len; + char verify_alg[SHARED_SECRET_MAX]; + __u32 verify_alg_len; + char csums_alg[SHARED_SECRET_MAX]; + __u32 csums_alg_len; + __u32 wire_protocol; + __u32 connect_int; + __u32 timeout; + __u32 ping_int; + __u32 ping_timeo; + __u32 sndbuf_size; + __u32 rcvbuf_size; + __u32 ko_count; + __u32 max_buffers; + __u32 max_epoch_size; + __u32 unplug_watermark; + __u32 after_sb_0p; + __u32 after_sb_1p; + __u32 after_sb_2p; + __u32 rr_conflict; + __u32 on_congestion; + __u32 cong_fill; + __u32 cong_extents; + unsigned char two_primaries; + unsigned char discard_my_data; + unsigned char tcp_cork; + unsigned char always_asbp; + unsigned char tentative; + unsigned char use_rle; + unsigned char csums_after_crash_only; + __u32 sock_check_timeo; +}; + +struct set_role_parms { + unsigned char assume_uptodate; +}; + +struct resize_parms { + __u64 resize_size; + unsigned char resize_force; + unsigned char no_resync; + __u32 al_stripes; + __u32 al_stripe_size; +}; + +struct state_info { + __u32 sib_reason; + __u32 current_state; + __u64 capacity; + __u64 ed_uuid; + __u32 prev_state; + __u32 new_state; + char uuids[DRBD_NL_UUIDS_SIZE]; + __u32 uuids_len; + __u32 disk_flags; + __u64 bits_total; + __u64 bits_oos; + __u64 bits_rs_total; + __u64 bits_rs_failed; + char helper[32]; + __u32 helper_len; + __u32 helper_exit_code; + __u64 send_cnt; + __u64 recv_cnt; + __u64 read_cnt; + __u64 writ_cnt; + __u64 al_writ_cnt; + __u64 bm_writ_cnt; + __u32 ap_bio_cnt; + __u32 ap_pending_cnt; + __u32 rs_pending_cnt; +}; + +struct start_ov_parms { + __u64 ov_start_sector; + __u64 ov_stop_sector; +}; + +struct new_c_uuid_parms { + unsigned char clear_bm; +}; + +struct timeout_parms { + __u32 timeout_type; +}; + +struct disconnect_parms { + unsigned char force_disconnect; +}; + +struct detach_parms { + unsigned char force_detach; +}; + +struct resource_info { + __u32 res_role; + unsigned char res_susp; + unsigned char res_susp_nod; + unsigned char res_susp_fen; +}; + +struct device_info { + __u32 dev_disk_state; +}; + +struct connection_info { + __u32 conn_connection_state; + __u32 conn_role; +}; + +struct peer_device_info { + __u32 peer_repl_state; + __u32 peer_disk_state; + __u32 peer_resync_susp_user; + __u32 peer_resync_susp_peer; + __u32 peer_resync_susp_dependency; +}; + +struct resource_statistics { + __u32 res_stat_write_ordering; +}; + +struct device_statistics { + __u64 dev_size; + __u64 dev_read; + __u64 dev_write; + __u64 dev_al_writes; + __u64 dev_bm_writes; + __u32 dev_upper_pending; + __u32 dev_lower_pending; + unsigned char dev_upper_blocked; + unsigned char dev_lower_blocked; + unsigned char dev_al_suspended; + __u64 dev_exposed_data_uuid; + __u64 dev_current_uuid; + __u32 dev_disk_flags; + char history_uuids[DRBD_NL_HISTORY_UUIDS_SIZE]; + __u32 history_uuids_len; +}; + +struct connection_statistics { + unsigned char conn_congested; +}; + +struct peer_device_statistics { + __u64 peer_dev_received; + __u64 peer_dev_sent; + __u32 peer_dev_pending; + __u32 peer_dev_unacked; + __u64 peer_dev_out_of_sync; + __u64 peer_dev_resync_failed; + __u64 peer_dev_bitmap_uuid; + __u32 peer_dev_flags; +}; + +struct drbd_notification_header { + __u32 nh_type; +}; + +struct drbd_helper_info { + char helper_name[32]; + __u32 helper_name_len; + __u32 helper_status; +}; + +int drbd_cfg_reply_to_skb(struct sk_buff *skb, struct drbd_cfg_reply *s); + +int drbd_cfg_context_from_attrs(struct drbd_cfg_context *s, struct genl_info *info); +int drbd_cfg_context_ntb_from_attrs(struct nlattr ***ret_nested_attribute_table, struct genl_info *info); +int drbd_cfg_context_to_skb(struct sk_buff *skb, struct drbd_cfg_context *s); + +int disk_conf_from_attrs(struct disk_conf *s, struct genl_info *info); +int disk_conf_ntb_from_attrs(struct nlattr ***ret_nested_attribute_table, struct genl_info *info); +int disk_conf_to_skb(struct sk_buff *skb, struct disk_conf *s); +void set_disk_conf_defaults(struct disk_conf *x); + +int res_opts_from_attrs(struct res_opts *s, struct genl_info *info); +int res_opts_ntb_from_attrs(struct nlattr ***ret_nested_attribute_table, struct genl_info *info); +int res_opts_to_skb(struct sk_buff *skb, struct res_opts *s); +void set_res_opts_defaults(struct res_opts *x); + +int net_conf_from_attrs(struct net_conf *s, struct genl_info *info); +int net_conf_ntb_from_attrs(struct nlattr ***ret_nested_attribute_table, struct genl_info *info); +int net_conf_to_skb(struct sk_buff *skb, struct net_conf *s); +void set_net_conf_defaults(struct net_conf *x); + +int set_role_parms_from_attrs(struct set_role_parms *s, struct genl_info *info); +int set_role_parms_ntb_from_attrs(struct nlattr ***ret_nested_attribute_table, struct genl_info *info); +int set_role_parms_to_skb(struct sk_buff *skb, struct set_role_parms *s); + +int resize_parms_from_attrs(struct resize_parms *s, struct genl_info *info); +int resize_parms_ntb_from_attrs(struct nlattr ***ret_nested_attribute_table, struct genl_info *info); +int resize_parms_to_skb(struct sk_buff *skb, struct resize_parms *s); +void set_resize_parms_defaults(struct resize_parms *x); + +int state_info_to_skb(struct sk_buff *skb, struct state_info *s); + +int start_ov_parms_from_attrs(struct start_ov_parms *s, struct genl_info *info); +int start_ov_parms_ntb_from_attrs(struct nlattr ***ret_nested_attribute_table, struct genl_info *info); +int start_ov_parms_to_skb(struct sk_buff *skb, struct start_ov_parms *s); + +int new_c_uuid_parms_from_attrs(struct new_c_uuid_parms *s, struct genl_info *info); +int new_c_uuid_parms_ntb_from_attrs(struct nlattr ***ret_nested_attribute_table, struct genl_info *info); +int new_c_uuid_parms_to_skb(struct sk_buff *skb, struct new_c_uuid_parms *s); + +int timeout_parms_to_skb(struct sk_buff *skb, struct timeout_parms *s); + +int disconnect_parms_from_attrs(struct disconnect_parms *s, struct genl_info *info); +int disconnect_parms_ntb_from_attrs(struct nlattr ***ret_nested_attribute_table, struct genl_info *info); +int disconnect_parms_to_skb(struct sk_buff *skb, struct disconnect_parms *s); + +int detach_parms_from_attrs(struct detach_parms *s, struct genl_info *info); +int detach_parms_ntb_from_attrs(struct nlattr ***ret_nested_attribute_table, struct genl_info *info); +int detach_parms_to_skb(struct sk_buff *skb, struct detach_parms *s); + +int resource_info_from_attrs(struct resource_info *s, struct genl_info *info); +int resource_info_ntb_from_attrs(struct nlattr ***ret_nested_attribute_table, struct genl_info *info); +int resource_info_to_skb(struct sk_buff *skb, struct resource_info *s); + +int device_info_from_attrs(struct device_info *s, struct genl_info *info); +int device_info_ntb_from_attrs(struct nlattr ***ret_nested_attribute_table, struct genl_info *info); +int device_info_to_skb(struct sk_buff *skb, struct device_info *s); + +int connection_info_from_attrs(struct connection_info *s, struct genl_info *info); +int connection_info_ntb_from_attrs(struct nlattr ***ret_nested_attribute_table, struct genl_info *info); +int connection_info_to_skb(struct sk_buff *skb, struct connection_info *s); + +int peer_device_info_from_attrs(struct peer_device_info *s, struct genl_info *info); +int peer_device_info_ntb_from_attrs(struct nlattr ***ret_nested_attribute_table, struct genl_info *info); +int peer_device_info_to_skb(struct sk_buff *skb, struct peer_device_info *s); + +int resource_statistics_from_attrs(struct resource_statistics *s, struct genl_info *info); +int resource_statistics_ntb_from_attrs(struct nlattr ***ret_nested_attribute_table, struct genl_info *info); +int resource_statistics_to_skb(struct sk_buff *skb, struct resource_statistics *s); + +int device_statistics_from_attrs(struct device_statistics *s, struct genl_info *info); +int device_statistics_ntb_from_attrs(struct nlattr ***ret_nested_attribute_table, struct genl_info *info); +int device_statistics_to_skb(struct sk_buff *skb, struct device_statistics *s); + +int connection_statistics_from_attrs(struct connection_statistics *s, struct genl_info *info); +int connection_statistics_ntb_from_attrs(struct nlattr ***ret_nested_attribute_table, struct genl_info *info); +int connection_statistics_to_skb(struct sk_buff *skb, struct connection_statistics *s); + +int peer_device_statistics_from_attrs(struct peer_device_statistics *s, struct genl_info *info); +int peer_device_statistics_ntb_from_attrs(struct nlattr ***ret_nested_attribute_table, struct genl_info *info); +int peer_device_statistics_to_skb(struct sk_buff *skb, struct peer_device_statistics *s); + +int drbd_notification_header_to_skb(struct sk_buff *skb, struct drbd_notification_header *s); + +int drbd_helper_info_to_skb(struct sk_buff *skb, struct drbd_helper_info *s); + +#endif /* _LINUX_DRBD_GEN_H */ diff --git a/drivers/block/drbd/drbd_nla.c b/drivers/block/drbd/drbd_nla.c deleted file mode 100644 index df0d241d3f6a..000000000000 --- a/drivers/block/drbd/drbd_nla.c +++ /dev/null @@ -1,56 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -#include <linux/kernel.h> -#include <net/netlink.h> -#include <linux/drbd_genl_api.h> -#include "drbd_nla.h" - -static int drbd_nla_check_mandatory(int maxtype, struct nlattr *nla) -{ - struct nlattr *head = nla_data(nla); - int len = nla_len(nla); - int rem; - - /* - * validate_nla (called from nla_parse_nested) ignores attributes - * beyond maxtype, and does not understand the DRBD_GENLA_F_MANDATORY flag. - * In order to have it validate attributes with the DRBD_GENLA_F_MANDATORY - * flag set also, check and remove that flag before calling - * nla_parse_nested. - */ - - nla_for_each_attr(nla, head, len, rem) { - if (nla->nla_type & DRBD_GENLA_F_MANDATORY) { - nla->nla_type &= ~DRBD_GENLA_F_MANDATORY; - if (nla_type(nla) > maxtype) - return -EOPNOTSUPP; - } - } - return 0; -} - -int drbd_nla_parse_nested(struct nlattr *tb[], int maxtype, struct nlattr *nla, - const struct nla_policy *policy) -{ - int err; - - err = drbd_nla_check_mandatory(maxtype, nla); - if (!err) - err = nla_parse_nested_deprecated(tb, maxtype, nla, policy, - NULL); - - return err; -} - -struct nlattr *drbd_nla_find_nested(int maxtype, struct nlattr *nla, int attrtype) -{ - int err; - /* - * If any nested attribute has the DRBD_GENLA_F_MANDATORY flag set and - * we don't know about that attribute, reject all the nested - * attributes. - */ - err = drbd_nla_check_mandatory(maxtype, nla); - if (err) - return ERR_PTR(err); - return nla_find_nested(nla, attrtype); -} diff --git a/drivers/block/drbd/drbd_nla.h b/drivers/block/drbd/drbd_nla.h deleted file mode 100644 index d3555df0d353..000000000000 --- a/drivers/block/drbd/drbd_nla.h +++ /dev/null @@ -1,9 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -#ifndef __DRBD_NLA_H -#define __DRBD_NLA_H - -extern int drbd_nla_parse_nested(struct nlattr *tb[], int maxtype, struct nlattr *nla, - const struct nla_policy *policy); -extern struct nlattr *drbd_nla_find_nested(int maxtype, struct nlattr *nla, int attrtype); - -#endif /* __DRBD_NLA_H */ diff --git a/drivers/block/drbd/drbd_proc.c b/drivers/block/drbd/drbd_proc.c index 1d0feafceadc..6d0c12c10260 100644 --- a/drivers/block/drbd/drbd_proc.c +++ b/drivers/block/drbd/drbd_proc.c @@ -228,7 +228,7 @@ int drbd_seq_show(struct seq_file *seq, void *v) }; seq_printf(seq, "version: " REL_VERSION " (api:%d/proto:%d-%d)\n%s\n", - GENL_MAGIC_VERSION, PRO_VERSION_MIN, PRO_VERSION_MAX, drbd_buildtag()); + DRBD_FAMILY_VERSION, PRO_VERSION_MIN, PRO_VERSION_MAX, drbd_buildtag()); /* cs .. connection state diff --git a/drivers/block/drbd/drbd_receiver.c b/drivers/block/drbd/drbd_receiver.c index 58b95bf4bdca..2135c14354a8 100644 --- a/drivers/block/drbd/drbd_receiver.c +++ b/drivers/block/drbd/drbd_receiver.c @@ -1810,6 +1810,11 @@ static int recv_dless_read(struct drbd_peer_device *peer_device, struct drbd_req data_size -= digest_size; } + if (data_size < 0) { + drbd_err(peer_device, "Invalid data reply size\n"); + return -EIO; + } + /* optimistically update recv_cnt. if receiving fails below, * we disconnect anyways, and counters will be reset. */ peer_device->device->recv_cnt += data_size>>9; diff --git a/drivers/block/floppy.c b/drivers/block/floppy.c index 92e446a64371..f04397b8e381 100644 --- a/drivers/block/floppy.c +++ b/drivers/block/floppy.c @@ -145,8 +145,6 @@ * Better audit of register_blkdev. */ -#define REALLY_SLOW_IO - #define DEBUGT 2 #define DPRINT(format, args...) \ @@ -182,7 +180,7 @@ static int print_unex = 1; #include <linux/major.h> #include <linux/mc146818rtc.h> /* CMOS defines */ #include <linux/mm.h> -#include <linux/mod_devicetable.h> +#include <linux/device-id/pnp.h> #include <linux/module.h> #include <linux/mutex.h> #include <linux/platform_device.h> @@ -5014,8 +5012,8 @@ MODULE_LICENSE("GPL"); /* This doesn't actually get used other than for module information */ static const struct pnp_device_id floppy_pnpids[] = { - {"PNP0700", 0}, - {} + { .id = "PNP0700" }, + { } }; MODULE_DEVICE_TABLE(pnp, floppy_pnpids); diff --git a/drivers/block/loop.c b/drivers/block/loop.c index 0000913f7efc..758c20678bf6 100644 --- a/drivers/block/loop.c +++ b/drivers/block/loop.c @@ -54,6 +54,7 @@ struct loop_device { struct file *lo_backing_file; unsigned int lo_min_dio_size; + unsigned int lo_dio_mem_align; struct block_device *lo_device; gfp_t old_gfp_mask; @@ -342,23 +343,19 @@ static int lo_rw_aio(struct loop_device *lo, struct loop_cmd *cmd, { struct iov_iter iter; struct req_iterator rq_iter; - struct bio_vec *bvec; struct request *rq = blk_mq_rq_from_pdu(cmd); - struct bio *bio = rq->bio; struct file *file = lo->lo_backing_file; - struct bio_vec tmp; - unsigned int offset; unsigned int nr_bvec; int ret; nr_bvec = blk_rq_nr_bvec(rq); if (rq->bio != rq->biotail) { + struct bio_vec tmp, *bvec; - bvec = kmalloc_objs(struct bio_vec, nr_bvec, GFP_NOIO); - if (!bvec) + cmd->bvec = kmalloc_objs(*cmd->bvec, nr_bvec, GFP_NOIO); + if (!cmd->bvec) return -EIO; - cmd->bvec = bvec; /* * The bios of the request may be started from the middle of @@ -366,26 +363,26 @@ static int lo_rw_aio(struct loop_device *lo, struct loop_cmd *cmd, * copy bio->bi_iov_vec to new bvec. The rq_for_each_bvec * API will take care of all details for us. */ + bvec = cmd->bvec; rq_for_each_bvec(tmp, rq, rq_iter) { *bvec = tmp; bvec++; } - bvec = cmd->bvec; - offset = 0; + iov_iter_bvec(&iter, rw, cmd->bvec, nr_bvec, blk_rq_bytes(rq)); + iter.iov_offset = 0; } else { /* * Same here, this bio may be started from the middle of the * 'bvec' because of bio splitting, so offset from the bvec * must be passed to iov iterator */ - offset = bio->bi_iter.bi_bvec_done; - bvec = __bvec_iter_bvec(bio->bi_io_vec, bio->bi_iter); + iov_iter_bvec(&iter, rw, + __bvec_iter_bvec(rq->bio->bi_io_vec, rq->bio->bi_iter), + nr_bvec, blk_rq_bytes(rq)); + iter.iov_offset = rq->bio->bi_iter.bi_offset; } atomic_set(&cmd->ref, 2); - iov_iter_bvec(&iter, rw, bvec, nr_bvec, blk_rq_bytes(rq)); - iter.iov_offset = offset; - cmd->iocb.ki_pos = pos; cmd->iocb.ki_filp = file; cmd->iocb.ki_ioprio = req_get_ioprio(rq); @@ -451,26 +448,39 @@ static void loop_reread_partitions(struct loop_device *lo) __func__, lo->lo_number, lo->lo_file_name, rc); } -static unsigned int loop_query_min_dio_size(struct loop_device *lo) +static void loop_update_dio_alignment(struct loop_device *lo) { struct file *file = lo->lo_backing_file; struct block_device *sb_bdev = file->f_mapping->host->i_sb->s_bdev; struct kstat st; /* - * Use the minimal dio alignment of the file system if provided. + * Use the dio alignment of the file system if provided. The incomoing + * request's bio_vec is forwarded to the backing file unchanged, so its + * required memory alignment becomes the device's dma_alignment when + * used for direct-io. The file system reports zeroed alignments if the + * file can't be used for direct-io at all, so fall back to the block + * device limits in that case. */ if (!vfs_getattr(&file->f_path, &st, STATX_DIOALIGN, 0) && - (st.result_mask & STATX_DIOALIGN)) - return st.dio_offset_align; + (st.result_mask & STATX_DIOALIGN) && st.dio_mem_align) { + lo->lo_min_dio_size = st.dio_offset_align; + lo->lo_dio_mem_align = min(st.dio_mem_align - 1, PAGE_SIZE - 1); + return; + } /* * In a perfect world this wouldn't be needed, but as of Linux 6.13 only * a handful of file systems support the STATX_DIOALIGN flag. */ - if (sb_bdev) - return bdev_logical_block_size(sb_bdev); - return SECTOR_SIZE; + if (sb_bdev) { + lo->lo_min_dio_size = bdev_logical_block_size(sb_bdev); + lo->lo_dio_mem_align = bdev_dma_alignment(sb_bdev); + return; + } + + lo->lo_min_dio_size = SECTOR_SIZE; + lo->lo_dio_mem_align = SECTOR_SIZE - 1; } static inline int is_loop_device(struct file *file) @@ -513,7 +523,7 @@ static void loop_assign_backing_file(struct loop_device *lo, struct file *file) lo->old_gfp_mask & ~(__GFP_IO | __GFP_FS)); if (lo->lo_backing_file->f_flags & O_DIRECT) lo->lo_flags |= LO_FLAGS_DIRECT_IO; - lo->lo_min_dio_size = loop_query_min_dio_size(lo); + loop_update_dio_alignment(lo); } static int loop_check_backing_file(struct file *file) @@ -944,6 +954,19 @@ static unsigned int loop_default_blocksize(struct loop_device *lo) return SECTOR_SIZE; } +static void loop_set_dma_limit(struct loop_device *lo, struct queue_limits *lim) +{ + /* + * Direct I/O forwards the user pages to the backing file unchanged, so + * track the backing's DMA alignment requirement as the mode is toggled. + */ + if (lo->lo_flags & LO_FLAGS_DIRECT_IO) + lim->dma_alignment = max_t(unsigned int, lo->lo_dio_mem_align, + SECTOR_SIZE - 1); + else + lim->dma_alignment = SECTOR_SIZE - 1; +} + static void loop_update_limits(struct loop_device *lo, struct queue_limits *lim, unsigned int bsize) { @@ -965,6 +988,7 @@ static void loop_update_limits(struct loop_device *lo, struct queue_limits *lim, lim->logical_block_size = bsize; lim->physical_block_size = bsize; lim->io_min = bsize; + loop_set_dma_limit(lo, lim); lim->features &= ~(BLK_FEAT_WRITE_CACHE | BLK_FEAT_ROTATIONAL); if (file->f_op->fsync && !(lo->lo_flags & LO_FLAGS_READ_ONLY)) lim->features |= BLK_FEAT_WRITE_CACHE; @@ -1117,6 +1141,7 @@ static void __loop_clr_fd(struct loop_device *lo) struct queue_limits lim; struct file *filp; gfp_t gfp = lo->old_gfp_mask; + int err; spin_lock_irq(&lo->lo_lock); filp = lo->lo_backing_file; @@ -1150,26 +1175,21 @@ static void __loop_clr_fd(struct loop_device *lo) disk_force_media_change(lo->lo_disk); - if (lo->lo_flags & LO_FLAGS_PARTSCAN) { - int err; - - /* - * open_mutex has been held already in release path, so don't - * acquire it if this function is called in such case. - * - * If the reread partition isn't from release path, lo_refcnt - * must be at least one and it can only become zero when the - * current holder is released. - */ - err = bdev_disk_changed(lo->lo_disk, false); - if (err) - pr_warn("%s: partition scan of loop%d failed (rc=%d)\n", - __func__, lo->lo_number, err); - /* Device is gone, no point in returning error */ - } + /* + * Remove all partitions, including partitions added manually with + * BLKPG, which may exist even if LO_FLAGS_PARTSCAN is not set. + * + * open_mutex has been held already in release path, so don't acquire + * it here. + */ + err = bdev_disk_changed(lo->lo_disk, false); + if (err) + pr_warn("%s: partition scan of loop%d failed (rc=%d)\n", + __func__, lo->lo_number, err); + /* Device is gone, no point in returning error */ /* - * lo->lo_state is set to Lo_unbound here after above partscan has + * lo->lo_state is set to Lo_unbound here after removing partitions has * finished. There cannot be anybody else entering __loop_clr_fd() as * Lo_rundown state protects us from all the other places trying to * change the 'lo' device. @@ -1420,6 +1440,7 @@ static int loop_set_dio(struct loop_device *lo, unsigned long arg) { bool use_dio = !!arg; unsigned int memflags; + struct queue_limits lim; if (lo->lo_state != Lo_bound) return -ENXIO; @@ -1433,11 +1454,14 @@ static int loop_set_dio(struct loop_device *lo, unsigned long arg) vfs_fsync(lo->lo_backing_file, 0); } + lim = queue_limits_start_update(lo->lo_queue); memflags = blk_mq_freeze_queue(lo->lo_queue); if (use_dio) lo->lo_flags |= LO_FLAGS_DIRECT_IO; else lo->lo_flags &= ~LO_FLAGS_DIRECT_IO; + loop_set_dma_limit(lo, &lim); + queue_limits_commit_update(lo->lo_queue, &lim); blk_mq_unfreeze_queue(lo->lo_queue, memflags); return 0; } diff --git a/drivers/block/mtip32xx/mtip32xx.c b/drivers/block/mtip32xx/mtip32xx.c index 567192e371a8..113bdb868c46 100644 --- a/drivers/block/mtip32xx/mtip32xx.c +++ b/drivers/block/mtip32xx/mtip32xx.c @@ -3048,6 +3048,8 @@ static int mtip_block_ioctl(struct block_device *dev, if (!dd) return -ENOTTY; + guard(mutex)(&dd->ioctl_mutex); + if (unlikely(test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag))) return -ENOTTY; @@ -3086,6 +3088,8 @@ static int mtip_block_compat_ioctl(struct block_device *dev, if (!dd) return -ENOTTY; + guard(mutex)(&dd->ioctl_mutex); + if (unlikely(test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag))) return -ENOTTY; @@ -3340,7 +3344,7 @@ static void mtip_free_cmd(struct blk_mq_tag_set *set, struct request *rq, } static int mtip_init_cmd(struct blk_mq_tag_set *set, struct request *rq, - unsigned int hctx_idx, unsigned int numa_node) + unsigned int hctx_idx, int numa_node) { struct driver_data *dd = set->driver_data; struct mtip_cmd *cmd = blk_mq_rq_to_pdu(rq); @@ -3405,6 +3409,7 @@ static int mtip_block_initialize(struct driver_data *dd) .max_segment_size = 0x400000, }; int rv = 0, wait_for_rebuild = 0; + bool disk_added = false; sector_t capacity; unsigned int index = 0; @@ -3438,6 +3443,7 @@ static int mtip_block_initialize(struct driver_data *dd) dev_err(&dd->pdev->dev, "Unable to allocate request queue\n"); rv = -ENOMEM; + dd->disk = NULL; goto block_queue_alloc_init_error; } dd->queue = dd->disk->queue; @@ -3496,6 +3502,7 @@ skip_create_disk: rv = device_add_disk(&dd->pdev->dev, dd->disk, mtip_disk_attr_groups); if (rv) goto read_capacity_error; + disk_added = true; if (dd->mtip_svc_handler) { set_bit(MTIP_DDF_INIT_DONE_BIT, &dd->dd_flag); @@ -3511,7 +3518,9 @@ start_service_thread: dev_err(&dd->pdev->dev, "service thread failed to start\n"); dd->mtip_svc_handler = NULL; rv = -EFAULT; - goto kthread_run_error; + if (disk_added) + goto kthread_run_error; + goto read_capacity_error; } wake_up_process(dd->mtip_svc_handler); if (wait_for_rebuild == MTIP_FTL_REBUILD_MAGIC) @@ -3522,6 +3531,10 @@ start_service_thread: kthread_run_error: /* Delete our gendisk. This also removes the device from /dev */ del_gendisk(dd->disk); + mtip_hw_debugfs_exit(dd); + blk_mq_free_tag_set(&dd->tags); + mtip_hw_exit(dd); + return rv; read_capacity_error: init_hw_cmds_error: mtip_hw_debugfs_exit(dd); @@ -3529,6 +3542,7 @@ disk_index_error: ida_free(&rssd_index_ida, index); ida_get_error: put_disk(dd->disk); + dd->disk = NULL; block_queue_alloc_init_error: blk_mq_free_tag_set(&dd->tags); block_queue_alloc_tag_error: @@ -3711,6 +3725,7 @@ static int mtip_pci_probe(struct pci_dev *pdev, dd = kzalloc_node(sizeof(struct driver_data), GFP_KERNEL, my_node); if (!dd) return -ENOMEM; + mutex_init(&dd->ioctl_mutex); /* Attach the private data to this PCI device. */ pci_set_drvdata(pdev, dd); @@ -3839,7 +3854,10 @@ msi_initialize_err: } iomap_err: - kfree(dd); + if (dd->disk) + put_disk(dd->disk); + else + kfree(dd); pci_set_drvdata(pdev, NULL); return rv; done: @@ -3874,6 +3892,7 @@ static void mtip_pci_remove(struct pci_dev *pdev) } set_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag); + mutex_lock(&dd->ioctl_mutex); if (test_bit(MTIP_DDF_INIT_DONE_BIT, &dd->dd_flag)) del_gendisk(dd->disk); @@ -3902,6 +3921,7 @@ static void mtip_pci_remove(struct pci_dev *pdev) /* De-initialize the protocol layer. */ mtip_hw_exit(dd); + mutex_unlock(&dd->ioctl_mutex); if (dd->isr_workq) { destroy_workqueue(dd->isr_workq); diff --git a/drivers/block/mtip32xx/mtip32xx.h b/drivers/block/mtip32xx/mtip32xx.h index f7328f19ac5c..0963c07b5845 100644 --- a/drivers/block/mtip32xx/mtip32xx.h +++ b/drivers/block/mtip32xx/mtip32xx.h @@ -12,6 +12,7 @@ #define __MTIP32XX_H__ #include <linux/spinlock.h> +#include <linux/mutex.h> #include <linux/rwsem.h> #include <linux/ata.h> #include <linux/interrupt.h> @@ -432,6 +433,7 @@ struct driver_data { struct request_queue *queue; /* Our request queue. */ struct blk_mq_tag_set tags; /* blk_mq tags */ + struct mutex ioctl_mutex; struct mtip_port *port; /* Pointer to the port data structure. */ diff --git a/drivers/block/n64cart.c b/drivers/block/n64cart.c index b9fdeff31caf..328da73b6f2c 100644 --- a/drivers/block/n64cart.c +++ b/drivers/block/n64cart.c @@ -12,6 +12,7 @@ #include <linux/init.h> #include <linux/module.h> #include <linux/platform_device.h> +#include <linux/string.h> enum { PI_DRAM_REG = 0, @@ -145,7 +146,7 @@ static int __init n64cart_probe(struct platform_device *pdev) disk->flags = GENHD_FL_NO_PART; disk->fops = &n64cart_fops; disk->private_data = &pdev->dev; - strcpy(disk->disk_name, "n64cart"); + strscpy(disk->disk_name, "n64cart"); set_capacity(disk, size >> SECTOR_SHIFT); set_disk_ro(disk, 1); diff --git a/drivers/block/nbd.c b/drivers/block/nbd.c index fe63f3c55d0d..ffce519bf008 100644 --- a/drivers/block/nbd.c +++ b/drivers/block/nbd.c @@ -166,6 +166,7 @@ static struct dentry *nbd_dbg_dir; static unsigned int nbds_max = 16; static int max_part = 16; +static int pre_defined_connections = 1; static int part_shift; static int nbd_dev_dbg_init(struct nbd_device *nbd); @@ -331,6 +332,26 @@ static void nbd_mark_nsock_dead(struct nbd_device *nbd, struct nbd_sock *nsock, nsock->sent = 0; } +static void nbd_apply_limits(struct queue_limits *lim, u32 flags) +{ + lim->features &= ~(BLK_FEAT_WRITE_CACHE | BLK_FEAT_FUA | BLK_FEAT_ROTATIONAL); + lim->max_hw_discard_sectors = 0; + lim->max_write_zeroes_sectors = 0; + + if (flags & NBD_FLAG_SEND_TRIM) + lim->max_hw_discard_sectors = UINT_MAX >> SECTOR_SHIFT; + if (flags & NBD_FLAG_SEND_FLUSH) { + lim->features |= BLK_FEAT_WRITE_CACHE; + if (flags & NBD_FLAG_SEND_FUA) + lim->features |= BLK_FEAT_FUA; + } + + if (flags & NBD_FLAG_ROTATIONAL) + lim->features |= BLK_FEAT_ROTATIONAL; + if (flags & NBD_FLAG_SEND_WRITE_ZEROES) + lim->max_write_zeroes_sectors = UINT_MAX >> SECTOR_SHIFT; +} + static int nbd_set_size(struct nbd_device *nbd, loff_t bytesize, loff_t blksize) { struct queue_limits lim; @@ -352,26 +373,14 @@ static int nbd_set_size(struct nbd_device *nbd, loff_t bytesize, loff_t blksize) return 0; lim = queue_limits_start_update(nbd->disk->queue); - if (nbd->config->flags & NBD_FLAG_SEND_TRIM) - lim.max_hw_discard_sectors = UINT_MAX >> SECTOR_SHIFT; - else - lim.max_hw_discard_sectors = 0; - if (!(nbd->config->flags & NBD_FLAG_SEND_FLUSH)) { - lim.features &= ~(BLK_FEAT_WRITE_CACHE | BLK_FEAT_FUA); - } else if (nbd->config->flags & NBD_FLAG_SEND_FUA) { - lim.features |= BLK_FEAT_WRITE_CACHE | BLK_FEAT_FUA; - } else { - lim.features |= BLK_FEAT_WRITE_CACHE; - lim.features &= ~BLK_FEAT_FUA; - } - if (nbd->config->flags & NBD_FLAG_ROTATIONAL) - lim.features |= BLK_FEAT_ROTATIONAL; - if (nbd->config->flags & NBD_FLAG_SEND_WRITE_ZEROES) - lim.max_write_zeroes_sectors = UINT_MAX >> SECTOR_SHIFT; - + nbd_apply_limits(&lim, nbd->config->flags); lim.logical_block_size = blksize; lim.physical_block_size = blksize; - error = queue_limits_commit_update_frozen(nbd->disk->queue, &lim); + /* No need freeze with 0 capacity and write cache disabled */ + if (!get_capacity(nbd->disk) && !blk_queue_write_cache(nbd->disk->queue)) + error = queue_limits_commit_update(nbd->disk->queue, &lim); + else + error = queue_limits_commit_update_frozen(nbd->disk->queue, &lim); if (error) return error; @@ -1061,40 +1070,31 @@ static int find_fallback(struct nbd_device *nbd, int index) int new_index = -1; struct nbd_sock *nsock = config->socks[index]; int fallback = nsock->fallback_index; + int i; if (test_bit(NBD_RT_DISCONNECTED, &config->runtime_flags)) return new_index; - if (config->num_connections <= 1) { - dev_err_ratelimited(disk_to_dev(nbd->disk), - "Dead connection, failed to find a fallback\n"); - return new_index; - } + if (config->num_connections <= 1) + goto no_fallback; if (fallback >= 0 && fallback < config->num_connections && !config->socks[fallback]->dead) return fallback; - if (nsock->fallback_index < 0 || - nsock->fallback_index >= config->num_connections || - config->socks[nsock->fallback_index]->dead) { - int i; - for (i = 0; i < config->num_connections; i++) { - if (i == index) - continue; - if (!config->socks[i]->dead) { - new_index = i; - break; - } - } - nsock->fallback_index = new_index; - if (new_index < 0) { - dev_err_ratelimited(disk_to_dev(nbd->disk), - "Dead connection, failed to find a fallback\n"); - return new_index; + for (i = 0; i < config->num_connections; i++) { + if (i != index && !config->socks[i]->dead) { + new_index = i; + break; } } - new_index = nsock->fallback_index; + nsock->fallback_index = new_index; + if (new_index >= 0) + return new_index; + +no_fallback: + dev_err_ratelimited(disk_to_dev(nbd->disk), + "Dead connection, failed to find a fallback\n"); return new_index; } @@ -1238,6 +1238,42 @@ static struct socket *nbd_get_socket(struct nbd_device *nbd, unsigned long fd, return sock; } +#ifdef CONFIG_DEBUG_LOCK_ALLOC +static struct lock_class_key nbd_key[3]; +static struct lock_class_key nbd_slock_key[3]; + +static void nbd_reclassify_socket(struct socket *sock) +{ + struct sock *sk = sock->sk; + + if (!sock_allow_reclassification(sk)) + return; + + switch (sk->sk_family) { + case AF_INET: + sock_lock_init_class_and_name(sk, "slock-AF_INET-NBD", + &nbd_slock_key[0], + "sk_lock-AF_INET-NBD", + &nbd_key[0]); + break; + case AF_INET6: + sock_lock_init_class_and_name(sk, "slock-AF_INET6-NBD", + &nbd_slock_key[1], + "sk_lock-AF_INET6-NBD", + &nbd_key[1]); + break; + case AF_UNIX: + sock_lock_init_class_and_name(sk, "slock-AF_UNIX-NBD", + &nbd_slock_key[2], + "sk_lock-AF_UNIX-NBD", + &nbd_key[2]); + break; + } +} +#else +static inline void nbd_reclassify_socket(struct socket *sock) {} +#endif + static int nbd_add_socket(struct nbd_device *nbd, unsigned long arg, bool netlink) { @@ -1245,21 +1281,22 @@ static int nbd_add_socket(struct nbd_device *nbd, unsigned long arg, struct socket *sock; struct nbd_sock **socks; struct nbd_sock *nsock; - unsigned int memflags; int err; /* Arg will be cast to int, check it to avoid overflow */ if (arg > INT_MAX) return -EINVAL; + + if (nbd->pid) { + dev_err(disk_to_dev(nbd->disk), + "Cannot add socket to a running device\n"); + return -EBUSY; + } + sock = nbd_get_socket(nbd, arg, &err); if (!sock) return err; - - /* - * We need to make sure we don't get any errant requests while we're - * reallocating the ->socks array. - */ - memflags = blk_mq_freeze_queue(nbd->disk->queue); + nbd_reclassify_socket(sock); if (!netlink && !nbd->task_setup && !test_bit(NBD_RT_BOUND, &config->runtime_flags)) @@ -1300,17 +1337,15 @@ static int nbd_add_socket(struct nbd_device *nbd, unsigned long arg, INIT_WORK(&nsock->work, nbd_pending_cmd_work); socks[config->num_connections++] = nsock; atomic_inc(&config->live_connections); - blk_mq_unfreeze_queue(nbd->disk->queue, memflags); return 0; put_socket: - blk_mq_unfreeze_queue(nbd->disk->queue, memflags); sockfd_put(sock); return err; } -static int nbd_reconnect_socket(struct nbd_device *nbd, unsigned long arg) +static int nbd_genl_reconnect_sock_cb(struct nbd_device *nbd, unsigned long arg) { struct nbd_config *config = nbd->config; struct socket *sock, *old; @@ -1365,11 +1400,12 @@ static int nbd_reconnect_socket(struct nbd_device *nbd, unsigned long arg) atomic_inc(&config->live_connections); wake_up(&config->conn_wait); + dev_info(nbd_to_dev(nbd), "reconnected socket\n"); return 0; } sockfd_put(sock); kfree(args); - return -ENOSPC; + return 1; } static void nbd_bdev_reset(struct nbd_device *nbd) @@ -1434,8 +1470,13 @@ static void nbd_config_put(struct nbd_device *nbd) if (refcount_dec_and_mutex_lock(&nbd->config_refs, &nbd->config_lock)) { struct nbd_config *config = nbd->config; + struct queue_limits lim; nbd_dev_dbg_close(nbd); invalidate_disk(nbd->disk); + /* reset queue limits to default */ + lim = queue_limits_start_update(nbd->disk->queue); + nbd_apply_limits(&lim, 0); + queue_limits_commit_update(nbd->disk->queue, &lim); if (nbd->config->bytesize) kobject_uevent(&nbd_to_dev(nbd)->kobj, KOBJ_CHANGE); if (test_and_clear_bit(NBD_RT_HAS_PID_FILE, @@ -1888,7 +1929,7 @@ static void nbd_dbg_close(void) #endif static int nbd_init_request(struct blk_mq_tag_set *set, struct request *rq, - unsigned int hctx_idx, unsigned int numa_node) + unsigned int hctx_idx, int numa_node) { struct nbd_cmd *cmd = blk_mq_rq_to_pdu(rq); cmd->nbd = set->driver_data; @@ -1904,7 +1945,8 @@ static const struct blk_mq_ops nbd_mq_ops = { .timeout = nbd_xmit_timeout, }; -static struct nbd_device *nbd_dev_add(int index, unsigned int refs) +static struct nbd_device *nbd_dev_add(int index, unsigned int refs, + int nr_hw_queues) { struct queue_limits lim = { .max_hw_sectors = 65536, @@ -1921,7 +1963,7 @@ static struct nbd_device *nbd_dev_add(int index, unsigned int refs) goto out; nbd->tag_set.ops = &nbd_mq_ops; - nbd->tag_set.nr_hw_queues = 1; + nbd->tag_set.nr_hw_queues = nr_hw_queues; nbd->tag_set.queue_depth = 128; nbd->tag_set.numa_node = NUMA_NO_NODE; nbd->tag_set.cmd_size = sizeof(struct nbd_cmd); @@ -2070,6 +2112,58 @@ static int nbd_genl_size_set(struct genl_info *info, struct nbd_device *nbd) return 0; } +/* + * Walk the NBD_ATTR_SOCKETS nested list can call @cb for each socket fd. + * + * Return the number of fds walked, or a negative errno. + */ +static int nbd_genl_foreach_sock(struct genl_info *info, + int (*cb)(struct nbd_device *nbd, unsigned long fd), + struct nbd_device *nbd) +{ + struct nlattr *attr; + int rem, count = 0; + + if (!info->attrs[NBD_ATTR_SOCKETS]) + return 0; + + nla_for_each_nested(attr, info->attrs[NBD_ATTR_SOCKETS], rem) { + struct nlattr *socks[NBD_SOCK_MAX + 1]; + int ret; + + if (nla_type(attr) != NBD_SOCK_ITEM) { + pr_err("socks must be embedded in a SOCK_ITEM attr\n"); + return -EINVAL; + } + + if (nla_parse_nested_deprecated(socks, NBD_SOCK_MAX, + attr, + nbd_sock_policy, + info->extack)) { + pr_err("error processing sock list\n"); + return -EINVAL; + } + + if (!socks[NBD_SOCK_FD]) + continue; + + count++; + if (cb) { + ret = cb(nbd, (int)nla_get_u32(socks[NBD_SOCK_FD])); + if (ret > 0) + return count; + if (ret < 0) + return ret; + } + } + return count; +} + +static int nbd_genl_connect_sock_cb(struct nbd_device *nbd, unsigned long fd) +{ + return nbd_add_socket(nbd, fd, true); +} + static int nbd_genl_connect(struct sk_buff *skb, struct genl_info *info) { struct nbd_device *nbd; @@ -2122,7 +2216,11 @@ again: mutex_unlock(&nbd_index_mutex); if (!nbd) { - nbd = nbd_dev_add(index, 2); + ret = nbd_genl_foreach_sock(info, NULL, NULL); + if (ret < 0) + return ret; + + nbd = nbd_dev_add(index, 2, ret > 0 ? ret : 1); if (IS_ERR(nbd)) { pr_err("failed to add new device\n"); return PTR_ERR(nbd); @@ -2189,36 +2287,9 @@ again: } } - if (info->attrs[NBD_ATTR_SOCKETS]) { - struct nlattr *attr; - int rem, fd; - - nla_for_each_nested(attr, info->attrs[NBD_ATTR_SOCKETS], - rem) { - struct nlattr *socks[NBD_SOCK_MAX+1]; - - if (nla_type(attr) != NBD_SOCK_ITEM) { - pr_err("socks must be embedded in a SOCK_ITEM attr\n"); - ret = -EINVAL; - goto out; - } - ret = nla_parse_nested_deprecated(socks, NBD_SOCK_MAX, - attr, - nbd_sock_policy, - info->extack); - if (ret != 0) { - pr_err("error processing sock list\n"); - ret = -EINVAL; - goto out; - } - if (!socks[NBD_SOCK_FD]) - continue; - fd = (int)nla_get_u32(socks[NBD_SOCK_FD]); - ret = nbd_add_socket(nbd, fd, true); - if (ret) - goto out; - } - } + ret = nbd_genl_foreach_sock(info, nbd_genl_connect_sock_cb, nbd); + if (ret < 0) + goto out; if (info->attrs[NBD_ATTR_BACKEND_IDENTIFIER]) { nbd->backend = nla_strdup(info->attrs[NBD_ATTR_BACKEND_IDENTIFIER], @@ -2403,40 +2474,10 @@ static int nbd_genl_reconfigure(struct sk_buff *skb, struct genl_info *info) } } - if (info->attrs[NBD_ATTR_SOCKETS]) { - struct nlattr *attr; - int rem, fd; - - nla_for_each_nested(attr, info->attrs[NBD_ATTR_SOCKETS], - rem) { - struct nlattr *socks[NBD_SOCK_MAX+1]; - - if (nla_type(attr) != NBD_SOCK_ITEM) { - pr_err("socks must be embedded in a SOCK_ITEM attr\n"); - ret = -EINVAL; - goto out; - } - ret = nla_parse_nested_deprecated(socks, NBD_SOCK_MAX, - attr, - nbd_sock_policy, - info->extack); - if (ret != 0) { - pr_err("error processing sock list\n"); - ret = -EINVAL; - goto out; - } - if (!socks[NBD_SOCK_FD]) - continue; - fd = (int)nla_get_u32(socks[NBD_SOCK_FD]); - ret = nbd_reconnect_socket(nbd, fd); - if (ret) { - if (ret == -ENOSPC) - ret = 0; - goto out; - } - dev_info(nbd_to_dev(nbd), "reconnected socket\n"); - } - } + ret = nbd_genl_foreach_sock(info, nbd_genl_reconnect_sock_cb, nbd); + /* foreach_sock returns a positive count on success; doit must return 0 */ + if (ret >= 0) + ret = 0; out: mutex_unlock(&nbd->config_lock); nbd_config_put(nbd); @@ -2672,6 +2713,12 @@ static int __init nbd_init(void) if (nbds_max > 1UL << (MINORBITS - part_shift)) return -EINVAL; + /* An excessively large value will be adjusted in blk_mq_alloc_tag_set */ + if (pre_defined_connections < 1) { + pr_err("pre_defined_connections must be >= 1\n"); + return -EINVAL; + } + if (register_blkdev(NBD_MAJOR, "nbd")) return -EIO; @@ -2688,8 +2735,12 @@ static int __init nbd_init(void) } nbd_dbg_init(); + /* + * Set to the intended connection count so nbd_start_device() can skip + * the queue-freezing blk_mq_update_nr_hw_queues() call. + */ for (i = 0; i < nbds_max; i++) - nbd_dev_add(i, 1); + nbd_dev_add(i, 1, pre_defined_connections); return 0; } @@ -2750,3 +2801,6 @@ module_param(nbds_max, int, 0444); MODULE_PARM_DESC(nbds_max, "number of network block devices to initialize (default: 16)"); module_param(max_part, int, 0444); MODULE_PARM_DESC(max_part, "number of partitions per device (default: 16)"); +module_param(pre_defined_connections, int, 0444); +MODULE_PARM_DESC(pre_defined_connections, +"number of connections for devices pre-created at module load (default: 1)"); diff --git a/drivers/block/null_blk/main.c b/drivers/block/null_blk/main.c index f8c0fd57e041..6beb1f5b71ac 100644 --- a/drivers/block/null_blk/main.c +++ b/drivers/block/null_blk/main.c @@ -66,7 +66,7 @@ struct nullb_page { #define NULLB_PAGE_FREE (MAP_SZ - 2) static LIST_HEAD(nullb_list); -static struct mutex lock; +static DEFINE_MUTEX(lock); static int null_major; static DEFINE_IDA(nullb_indexes); static struct blk_mq_tag_set tag_set; @@ -340,11 +340,20 @@ static ssize_t nullb_device_bool_attr_store(bool *val, const char *page, return count; } -/* The following macro should only be used with TYPE = {uint, ulong, bool}. */ +/* + * The following macro should only be used with TYPE = {uint, ulong, bool}. + * + * The device configuration is modified under the global lock to serialize + * attribute changes against null_add_dev() and null_del_dev(): without this, + * an attribute could be changed while null_add_dev() is running, that is, + * before NULLB_DEV_FL_CONFIGURED is set, which would let null_add_dev() + * observe inconsistent values for the device configuration. + */ #define NULLB_DEVICE_ATTR(NAME, TYPE, APPLY) \ static ssize_t \ nullb_device_##NAME##_show(struct config_item *item, char *page) \ { \ + guard(mutex)(&lock); \ return nullb_device_##TYPE##_attr_show( \ to_nullb_device(item)->NAME, page); \ } \ @@ -360,6 +369,7 @@ nullb_device_##NAME##_store(struct config_item *item, const char *page, \ ret = nullb_device_##TYPE##_attr_store(&new_value, page, count);\ if (ret < 0) \ return ret; \ + guard(mutex)(&lock); \ if (apply_fn) \ ret = apply_fn(dev, new_value); \ else if (test_bit(NULLB_DEV_FL_CONFIGURED, &dev->flags)) \ @@ -379,10 +389,21 @@ static int nullb_update_nr_hw_queues(struct nullb_device *dev, struct blk_mq_tag_set *set; int ret, nr_hw_queues; + lockdep_assert_held(&lock); + if (!dev->nullb) return 0; /* + * A shared tag_set is mapped via the module-wide queue counts, so a + * per-device resize is meaningless. On shrink it would also leave + * mq_map[] pointing at NULLed hctx slots, causing a NULL deref in + * blk_mq_map_swqueue(). Reject it. + */ + if (dev->shared_tags) + return -EINVAL; + + /* * Make sure at least one submit queue exists. */ if (!submit_queues) @@ -421,25 +442,13 @@ static int nullb_update_nr_hw_queues(struct nullb_device *dev, static int nullb_apply_submit_queues(struct nullb_device *dev, unsigned int submit_queues) { - int ret; - - mutex_lock(&lock); - ret = nullb_update_nr_hw_queues(dev, submit_queues, dev->poll_queues); - mutex_unlock(&lock); - - return ret; + return nullb_update_nr_hw_queues(dev, submit_queues, dev->poll_queues); } static int nullb_apply_poll_queues(struct nullb_device *dev, unsigned int poll_queues) { - int ret; - - mutex_lock(&lock); - ret = nullb_update_nr_hw_queues(dev, dev->submit_queues, poll_queues); - mutex_unlock(&lock); - - return ret; + return nullb_update_nr_hw_queues(dev, dev->submit_queues, poll_queues); } NULLB_DEVICE_ATTR(size, ulong, NULL); @@ -478,6 +487,7 @@ NULLB_DEVICE_ATTR(badblocks_partial_io, bool, NULL); static ssize_t nullb_device_power_show(struct config_item *item, char *page) { + guard(mutex)(&lock); return nullb_device_bool_attr_show(to_nullb_device(item)->power, page); } @@ -493,15 +503,15 @@ static ssize_t nullb_device_power_store(struct config_item *item, return ret; ret = count; - mutex_lock(&lock); + guard(mutex)(&lock); if (!dev->power && newp) { if (test_and_set_bit(NULLB_DEV_FL_UP, &dev->flags)) - goto out; + return ret; ret = null_add_dev(dev); if (ret) { clear_bit(NULLB_DEV_FL_UP, &dev->flags); - goto out; + return ret; } set_bit(NULLB_DEV_FL_CONFIGURED, &dev->flags); @@ -515,8 +525,6 @@ static ssize_t nullb_device_power_store(struct config_item *item, clear_bit(NULLB_DEV_FL_CONFIGURED, &dev->flags); } -out: - mutex_unlock(&lock); return ret; } @@ -580,6 +588,7 @@ static ssize_t nullb_device_zone_readonly_store(struct config_item *item, { struct nullb_device *dev = to_nullb_device(item); + guard(mutex)(&lock); return zone_cond_store(dev, page, count, BLK_ZONE_COND_READONLY); } CONFIGFS_ATTR_WO(nullb_device_, zone_readonly); @@ -589,6 +598,7 @@ static ssize_t nullb_device_zone_offline_store(struct config_item *item, { struct nullb_device *dev = to_nullb_device(item); + guard(mutex)(&lock); return zone_cond_store(dev, page, count, BLK_ZONE_COND_OFFLINE); } CONFIGFS_ATTR_WO(nullb_device_, zone_offline); @@ -707,10 +717,9 @@ nullb_group_drop_item(struct config_group *group, struct config_item *item) struct nullb_device *dev = to_nullb_device(item); if (test_and_clear_bit(NULLB_DEV_FL_UP, &dev->flags)) { - mutex_lock(&lock); + guard(mutex)(&lock); dev->power = false; null_del_dev(dev->nullb); - mutex_unlock(&lock); } nullb_del_fault_config(dev); config_item_put(item); @@ -836,7 +845,6 @@ static void null_free_dev(struct nullb_device *dev) if (!dev) return; - null_free_zoned_dev(dev); badblocks_exit(&dev->badblocks); kfree(dev); } @@ -1770,18 +1778,19 @@ static void null_del_dev(struct nullb *nullb) del_gendisk(nullb->disk); - if (test_bit(NULLB_DEV_FL_THROTTLED, &nullb->dev->flags)) { + if (test_bit(NULLB_DEV_FL_THROTTLED, &dev->flags)) { hrtimer_cancel(&nullb->bw_timer); atomic_long_set(&nullb->cur_bytes, LONG_MAX); blk_mq_start_stopped_hw_queues(nullb->q, true); } put_disk(nullb->disk); + null_free_zoned_dev(dev); if (nullb->tag_set == &nullb->__tag_set) blk_mq_free_tag_set(nullb->tag_set); kfree(nullb->queues); if (null_cache_active(nullb)) - null_free_device_storage(nullb->dev, true); + null_free_device_storage(dev, true); kfree(nullb); dev->nullb = NULL; } @@ -2081,14 +2090,13 @@ static struct nullb *null_find_dev_by_name(const char *name) { struct nullb *nullb = NULL, *nb; - mutex_lock(&lock); + guard(mutex)(&lock); list_for_each_entry(nb, &nullb_list, list) { if (strcmp(nb->disk_name, name) == 0) { nullb = nb; break; } } - mutex_unlock(&lock); return nullb; } @@ -2102,9 +2110,9 @@ static int null_create_dev(void) if (!dev) return -ENOMEM; - mutex_lock(&lock); - ret = null_add_dev(dev); - mutex_unlock(&lock); + scoped_guard(mutex, &lock) { + ret = null_add_dev(dev); + } if (ret) { null_free_dev(dev); return ret; @@ -2162,17 +2170,9 @@ static int __init null_init(void) config_group_init(&nullb_subsys.su_group); mutex_init(&nullb_subsys.su_mutex); - ret = configfs_register_subsystem(&nullb_subsys); - if (ret) - return ret; - - mutex_init(&lock); - null_major = register_blkdev(0, "nullb"); - if (null_major < 0) { - ret = null_major; - goto err_conf; - } + if (null_major < 0) + return null_major; for (i = 0; i < nr_devices; i++) { ret = null_create_dev(); @@ -2180,6 +2180,10 @@ static int __init null_init(void) goto err_dev; } + ret = configfs_register_subsystem(&nullb_subsys); + if (ret) + goto err_dev; + pr_info("module loaded\n"); return 0; @@ -2189,8 +2193,8 @@ err_dev: null_destroy_dev(nullb); } unregister_blkdev(null_major, "nullb"); -err_conf: - configfs_unregister_subsystem(&nullb_subsys); + if (tag_set.ops) + blk_mq_free_tag_set(&tag_set); return ret; } @@ -2200,19 +2204,17 @@ static void __exit null_exit(void) configfs_unregister_subsystem(&nullb_subsys); - unregister_blkdev(null_major, "nullb"); - - mutex_lock(&lock); - while (!list_empty(&nullb_list)) { - nullb = list_entry(nullb_list.next, struct nullb, list); - null_destroy_dev(nullb); + scoped_guard(mutex, &lock) { + while (!list_empty(&nullb_list)) { + nullb = list_entry(nullb_list.next, struct nullb, list); + null_destroy_dev(nullb); + } } - mutex_unlock(&lock); + + unregister_blkdev(null_major, "nullb"); if (tag_set.ops) blk_mq_free_tag_set(&tag_set); - - mutex_destroy(&lock); } module_init(null_init); diff --git a/drivers/block/null_blk/zoned.c b/drivers/block/null_blk/zoned.c index 384bdce6a9b7..7e9b5ffb9e84 100644 --- a/drivers/block/null_blk/zoned.c +++ b/drivers/block/null_blk/zoned.c @@ -18,6 +18,8 @@ static inline sector_t mb_to_sects(unsigned long mb) static inline unsigned int null_zone_no(struct nullb_device *dev, sector_t sect) { + if (WARN_ON_ONCE(!dev->zone_size_sects)) + return 0; return sect >> ilog2(dev->zone_size_sects); } @@ -56,8 +58,8 @@ int null_init_zoned_dev(struct nullb_device *dev, sector_t sector = 0; unsigned int i; - if (!is_power_of_2(dev->zone_size)) { - pr_err("zone_size must be power-of-two\n"); + if (!dev->zone_size || !is_power_of_2(dev->zone_size)) { + pr_err("zone_size must be non-zero power-of-two\n"); return -EINVAL; } if (dev->zone_size > dev->size) { @@ -88,6 +90,10 @@ int null_init_zoned_dev(struct nullb_device *dev, zone_capacity_sects = mb_to_sects(dev->zone_capacity); dev_capacity_sects = mb_to_sects(dev->size); dev->zone_size_sects = mb_to_sects(dev->zone_size); + if (!dev->zone_size_sects) { + pr_err("zone_size too large or too small, leads to zero sectors\n"); + return -EINVAL; + } dev->nr_zones = round_up(dev_capacity_sects, dev->zone_size_sects) >> ilog2(dev->zone_size_sects); diff --git a/drivers/block/rbd.c b/drivers/block/rbd.c index e7da06200c1e..1f1c2810f6ee 100644 --- a/drivers/block/rbd.c +++ b/drivers/block/rbd.c @@ -580,14 +580,7 @@ static const struct bus_type rbd_bus_type = { .bus_groups = rbd_bus_groups, }; -static void rbd_root_dev_release(struct device *dev) -{ -} - -static struct device rbd_root_dev = { - .init_name = "rbd", - .release = rbd_root_dev_release, -}; +static struct device *rbd_root_dev; static __printf(2, 3) void rbd_warn(struct rbd_device *rbd_dev, const char *fmt, ...) @@ -1957,10 +1950,15 @@ static int rbd_object_map_update_finish(struct rbd_obj_request *obj_req, bool has_current_state; void *p; - if (osd_req->r_result) + if (osd_req->r_result < 0) return osd_req->r_result; /* + * Writes aren't allowed to return a data payload. + */ + WARN_ON_ONCE(osd_req->r_result > 0); + + /* * Nothing to do for a snapshot object map. */ if (osd_req->r_num_ops == 1) @@ -3672,7 +3670,7 @@ static void __rbd_lock(struct rbd_device *rbd_dev, const char *cookie) struct rbd_client_id cid = rbd_get_cid(rbd_dev); rbd_dev->lock_state = RBD_LOCK_STATE_LOCKED; - strcpy(rbd_dev->lock_cookie, cookie); + strscpy(rbd_dev->lock_cookie, cookie); rbd_set_owner_cid(rbd_dev, &cid); queue_work(rbd_dev->task_wq, &rbd_dev->acquired_lock_work); } @@ -4565,24 +4563,12 @@ out: return ret; } -static void cancel_tasks_sync(struct rbd_device *rbd_dev) -{ - dout("%s rbd_dev %p\n", __func__, rbd_dev); - - cancel_work_sync(&rbd_dev->acquired_lock_work); - cancel_work_sync(&rbd_dev->released_lock_work); - cancel_delayed_work_sync(&rbd_dev->lock_dwork); - cancel_work_sync(&rbd_dev->unlock_work); -} - /* * header_rwsem must not be held to avoid a deadlock with * rbd_dev_refresh() when flushing notifies. */ static void rbd_unregister_watch(struct rbd_device *rbd_dev) { - cancel_tasks_sync(rbd_dev); - mutex_lock(&rbd_dev->watch_mutex); if (rbd_dev->watch_state == RBD_WATCH_STATE_REGISTERED) __rbd_unregister_watch(rbd_dev); @@ -5390,7 +5376,7 @@ static struct rbd_device *__rbd_dev_create(struct rbd_spec *spec) rbd_dev->dev.bus = &rbd_bus_type; rbd_dev->dev.type = &rbd_device_type; - rbd_dev->dev.parent = &rbd_root_dev; + rbd_dev->dev.parent = rbd_root_dev; device_initialize(&rbd_dev->dev); return rbd_dev; @@ -6094,12 +6080,9 @@ static int rbd_dev_v2_snap_context(struct rbd_device *rbd_dev, /* * Make sure the reported number of snapshot ids wouldn't go - * beyond the end of our buffer. But before checking that, - * make sure the computed size of the snapshot context we - * allocate is representable in a size_t. + * beyond the end of our buffer. */ - if (snap_count > (SIZE_MAX - sizeof (struct ceph_snap_context)) - / sizeof (u64)) { + if (snap_count > RBD_MAX_SNAP_COUNT) { ret = -EINVAL; goto out; } @@ -6548,10 +6531,18 @@ out_err: static void rbd_dev_image_unlock(struct rbd_device *rbd_dev) { + dout("%s rbd_dev %p\n", __func__, rbd_dev); + + disable_delayed_work_sync(&rbd_dev->lock_dwork); + disable_work_sync(&rbd_dev->unlock_work); + down_write(&rbd_dev->lock_rwsem); if (__rbd_is_lock_owner(rbd_dev)) __rbd_release_lock(rbd_dev); up_write(&rbd_dev->lock_rwsem); + + flush_work(&rbd_dev->acquired_lock_work); + flush_work(&rbd_dev->released_lock_work); } /* @@ -7165,7 +7156,7 @@ static ssize_t do_rbd_add(const char *buf, size_t count) rc = device_add_disk(&rbd_dev->dev, rbd_dev->disk, NULL); if (rc) - goto err_out_cleanup_disk; + goto err_out_device; spin_lock(&rbd_dev_list_lock); list_add_tail(&rbd_dev->node, &rbd_dev_list); @@ -7179,8 +7170,8 @@ out: module_put(THIS_MODULE); return rc; -err_out_cleanup_disk: - rbd_free_disk(rbd_dev); +err_out_device: + device_del(&rbd_dev->dev); err_out_image_lock: rbd_dev_image_unlock(rbd_dev); rbd_dev_device_release(rbd_dev); @@ -7331,15 +7322,13 @@ static int __init rbd_sysfs_init(void) { int ret; - ret = device_register(&rbd_root_dev); - if (ret < 0) { - put_device(&rbd_root_dev); - return ret; - } + rbd_root_dev = root_device_register("rbd"); + if (IS_ERR(rbd_root_dev)) + return PTR_ERR(rbd_root_dev); ret = bus_register(&rbd_bus_type); if (ret < 0) - device_unregister(&rbd_root_dev); + root_device_unregister(rbd_root_dev); return ret; } @@ -7347,7 +7336,7 @@ static int __init rbd_sysfs_init(void) static void __exit rbd_sysfs_cleanup(void) { bus_unregister(&rbd_bus_type); - device_unregister(&rbd_root_dev); + root_device_unregister(rbd_root_dev); } static int __init rbd_slab_init(void) diff --git a/drivers/block/rnbd/rnbd-srv.c b/drivers/block/rnbd/rnbd-srv.c index 10e8c438bb43..79c9a5fb418f 100644 --- a/drivers/block/rnbd/rnbd-srv.c +++ b/drivers/block/rnbd/rnbd-srv.c @@ -11,6 +11,7 @@ #include <linux/module.h> #include <linux/blkdev.h> +#include <linux/fs_struct.h> #include "rnbd-srv.h" #include "rnbd-srv-trace.h" @@ -734,7 +735,8 @@ static int process_msg_open(struct rnbd_srv_session *srv_sess, goto reject; } - bdev_file = bdev_file_open_by_path(full_path, open_flags, NULL, NULL); + scoped_with_init_fs() + bdev_file = bdev_file_open_by_path(full_path, open_flags, NULL, NULL); if (IS_ERR(bdev_file)) { ret = PTR_ERR(bdev_file); pr_err("Opening device '%s' on session %s failed, failed to open the block device, err: %pe\n", diff --git a/drivers/block/rnull/configfs.rs b/drivers/block/rnull/configfs.rs index 7c2eb5c0b722..32c10c3f4d0f 100644 --- a/drivers/block/rnull/configfs.rs +++ b/drivers/block/rnull/configfs.rs @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 -use super::{NullBlkDevice, THIS_MODULE}; +use super::NullBlkDevice; use kernel::{ block::mq::gen_disk::{GenDisk, GenDiskBuilder}, configfs::{self, AttributeOperations}, diff --git a/drivers/block/sunvdc.c b/drivers/block/sunvdc.c index 020bd9f1a7b6..2be8231dcd5b 100644 --- a/drivers/block/sunvdc.c +++ b/drivers/block/sunvdc.c @@ -525,6 +525,23 @@ static int __send_request(struct request *req) err = __vdc_tx_trigger(port); if (err < 0) { printk(KERN_ERR PFX "vdc_tx_trigger() failure, err=%d\n", err); + /* + * If the port was reset (-ENOTCONN), the dring and the + * LDC channel including all of its mappings are already + * torn down and reallocated - there is nothing to undo + * and @desc must not be touched. + * + * For any other failure the descriptor was never handed + * to the peer: unmap the cookies and free the descriptor + * again, so that a later retry of the request does not + * leak LDC map table entries. + */ + if (err != -ENOTCONN) { + ldc_unmap(port->vio.lp, desc->cookies, + desc->ncookies); + desc->hdr.state = VIO_DESC_FREE; + rqe->req = NULL; + } } else { port->req_id++; dr->prod = vio_dring_next(dr, dr->prod); @@ -539,6 +556,7 @@ static blk_status_t vdc_queue_rq(struct blk_mq_hw_ctx *hctx, struct vdc_port *port = hctx->queue->queuedata; struct vio_dring_state *dr; unsigned long flags; + int ret; dr = &port->vio.drings[VIO_DRIVER_TX_RING]; @@ -560,7 +578,13 @@ static blk_status_t vdc_queue_rq(struct blk_mq_hw_ctx *hctx, return BLK_STS_DEV_RESOURCE; } - if (__send_request(bd->rq) < 0) { + ret = __send_request(bd->rq); + if (ret == -EAGAIN) { + spin_unlock_irqrestore(&port->vio.lock, flags); + /* already spun for 10msec, defer 10msec and retry */ + blk_mq_delay_kick_requeue_list(hctx->queue, 10); + return BLK_STS_DEV_RESOURCE; + } else if (ret < 0) { spin_unlock_irqrestore(&port->vio.lock, flags); return BLK_STS_IOERR; } diff --git a/drivers/block/swim3.c b/drivers/block/swim3.c index 01f7aef3fcfb..d4f0d43e0a72 100644 --- a/drivers/block/swim3.c +++ b/drivers/block/swim3.c @@ -1290,4 +1290,5 @@ module_init(swim3_init) MODULE_LICENSE("GPL"); MODULE_AUTHOR("Paul Mackerras"); +MODULE_DESCRIPTION("SWIM3 floppy driver for PowerMacs"); MODULE_ALIAS_BLOCKDEV_MAJOR(FLOPPY_MAJOR); diff --git a/drivers/block/ublk_drv.c b/drivers/block/ublk_drv.c index 63aeb7a76a8c..66eb55e7162e 100644 --- a/drivers/block/ublk_drv.c +++ b/drivers/block/ublk_drv.c @@ -19,6 +19,7 @@ #include <linux/errno.h> #include <linux/major.h> #include <linux/wait.h> +#include <linux/wait_bit.h> #include <linux/blkdev.h> #include <linux/init.h> #include <linux/swap.h> @@ -26,7 +27,6 @@ #include <linux/compat.h> #include <linux/mutex.h> #include <linux/writeback.h> -#include <linux/completion.h> #include <linux/highmem.h> #include <linux/sysfs.h> #include <linux/miscdevice.h> @@ -46,6 +46,8 @@ #include <linux/kref.h> #include <linux/kfifo.h> #include <linux/blk-integrity.h> +#include <linux/maple_tree.h> +#include <linux/xarray.h> #include <uapi/linux/fs.h> #include <uapi/linux/ublk_cmd.h> @@ -58,6 +60,11 @@ #define UBLK_CMD_UPDATE_SIZE _IOC_NR(UBLK_U_CMD_UPDATE_SIZE) #define UBLK_CMD_QUIESCE_DEV _IOC_NR(UBLK_U_CMD_QUIESCE_DEV) #define UBLK_CMD_TRY_STOP_DEV _IOC_NR(UBLK_U_CMD_TRY_STOP_DEV) +#define UBLK_CMD_REG_BUF _IOC_NR(UBLK_U_CMD_REG_BUF) +#define UBLK_CMD_UNREG_BUF _IOC_NR(UBLK_U_CMD_UNREG_BUF) + +/* Default max shmem buffer size: 4GB (may be increased in future) */ +#define UBLK_SHMEM_BUF_SIZE_MAX (1ULL << 32) #define UBLK_IO_REGISTER_IO_BUF _IOC_NR(UBLK_U_IO_REGISTER_IO_BUF) #define UBLK_IO_UNREGISTER_IO_BUF _IOC_NR(UBLK_U_IO_UNREGISTER_IO_BUF) @@ -81,7 +88,9 @@ | (IS_ENABLED(CONFIG_BLK_DEV_INTEGRITY) ? UBLK_F_INTEGRITY : 0) \ | UBLK_F_SAFE_STOP_DEV \ | UBLK_F_BATCH_IO \ - | UBLK_F_NO_AUTO_PART_SCAN) + | UBLK_F_NO_AUTO_PART_SCAN \ + | UBLK_F_SHMEM_ZC \ + | UBLK_F_IO_DESC_SIZE) #define UBLK_F_ALL_RECOVERY_FLAGS (UBLK_F_USER_RECOVERY \ | UBLK_F_USER_RECOVERY_REISSUE \ @@ -99,6 +108,8 @@ UBLK_BATCH_F_HAS_BUF_ADDR | \ UBLK_BATCH_F_AUTO_BUF_REG_FALLBACK) +#define UBLK_MAX_IO_DESC_SIZE 256 + /* ublk batch fetch uring_cmd */ struct ublk_batch_fetch_cmd { struct list_head node; @@ -229,8 +240,9 @@ struct ublk_io { } ____cacheline_aligned_in_smp; struct ublk_queue { - int q_id; - int q_depth; + u16 q_id; + u16 q_depth; + u16 io_desc_size; unsigned long flags; struct ublksrv_io_desc *io_cmd_buf; @@ -240,7 +252,7 @@ struct ublk_queue { bool fail_io; /* copy of dev->state == UBLK_S_DEV_FAIL_IO */ spinlock_t cancel_lock; struct ublk_device *dev; - u32 nr_io_ready; + u16 nr_io_ready; /* * For supporting UBLK_F_BATCH_IO only. @@ -289,6 +301,13 @@ struct ublk_queue { struct ublk_io ios[] __counted_by(q_depth); }; +/* Maple tree value: maps a PFN range to buffer location */ +struct ublk_buf_range { + unsigned short buf_index; + unsigned short flags; + unsigned int base_offset; /* byte offset within buffer */ +}; + struct ublk_device { struct gendisk *ub_disk; @@ -312,8 +331,7 @@ struct ublk_device { struct ublk_params params; - struct completion completion; - u32 nr_queue_ready; + u16 nr_queue_ready; bool unprivileged_daemons; struct mutex cancel_mutex; bool canceling; @@ -323,6 +341,10 @@ struct ublk_device { bool block_open; /* protected by open_mutex */ + /* shared memory zero copy */ + struct maple_tree buf_tree; + struct ida buf_ida; + struct ublk_queue *queues[]; }; @@ -334,10 +356,12 @@ struct ublk_params_header { static void ublk_io_release(void *priv); static void ublk_stop_dev_unlocked(struct ublk_device *ub); +static bool ublk_try_buf_match(struct ublk_device *ub, struct request *rq, + u32 *buf_idx, u32 *buf_off); +static void ublk_buf_cleanup(struct ublk_device *ub); static void ublk_abort_queue(struct ublk_device *ub, struct ublk_queue *ubq); static inline struct request *__ublk_check_and_get_req(struct ublk_device *ub, u16 q_id, u16 tag, struct ublk_io *io); -static inline unsigned int ublk_req_build_flags(struct request *req); static void ublk_batch_dispatch(struct ublk_queue *ubq, const struct ublk_batch_io_data *data, struct ublk_batch_fetch_cmd *fcmd); @@ -383,9 +407,9 @@ static inline void ublk_io_evts_deinit(struct ublk_queue *q) } static inline struct ublksrv_io_desc * -ublk_get_iod(const struct ublk_queue *ubq, unsigned tag) +ublk_get_iod(const struct ublk_queue *ubq, u16 tag) { - return &ubq->io_cmd_buf[tag]; + return (void *)ubq->io_cmd_buf + tag * (size_t)ubq->io_desc_size; } static inline bool ublk_support_zero_copy(const struct ublk_queue *ubq) @@ -398,6 +422,21 @@ static inline bool ublk_dev_support_zero_copy(const struct ublk_device *ub) return ub->dev_info.flags & UBLK_F_SUPPORT_ZERO_COPY; } +static inline bool ublk_support_shmem_zc(const struct ublk_queue *ubq) +{ + return ubq->flags & UBLK_F_SHMEM_ZC; +} + +static inline bool ublk_iod_is_shmem_zc(const struct ublk_queue *ubq, u16 tag) +{ + return ublk_get_iod(ubq, tag)->op_flags & UBLK_IO_F_SHMEM_ZC; +} + +static inline bool ublk_dev_support_shmem_zc(const struct ublk_device *ub) +{ + return ub->dev_info.flags & UBLK_F_SHMEM_ZC; +} + static inline bool ublk_support_auto_buf_reg(const struct ublk_queue *ubq) { return ubq->flags & UBLK_F_AUTO_BUF_REG; @@ -433,11 +472,66 @@ static inline bool ublk_dev_support_integrity(const struct ublk_device *ub) return ub->dev_info.flags & UBLK_F_INTEGRITY; } +static inline unsigned int ublk_req_build_flags(struct request *req) +{ + unsigned flags = 0; + + if (req->cmd_flags & REQ_FAILFAST_DEV) + flags |= UBLK_IO_F_FAILFAST_DEV; + + if (req->cmd_flags & REQ_FAILFAST_TRANSPORT) + flags |= UBLK_IO_F_FAILFAST_TRANSPORT; + + if (req->cmd_flags & REQ_FAILFAST_DRIVER) + flags |= UBLK_IO_F_FAILFAST_DRIVER; + + if (req->cmd_flags & REQ_META) + flags |= UBLK_IO_F_META; + + if (req->cmd_flags & REQ_FUA) + flags |= UBLK_IO_F_FUA; + + if (req->cmd_flags & REQ_NOUNMAP) + flags |= UBLK_IO_F_NOUNMAP; + + if (req->cmd_flags & REQ_SWAP) + flags |= UBLK_IO_F_SWAP; + + if (blk_integrity_rq(req)) + flags |= UBLK_IO_F_INTEGRITY; + + return flags; +} + +static void ublk_init_iod(struct ublk_queue *ubq, struct request *req, + uint8_t ublk_op, uint32_t nr_sectors, + uint64_t start_sector) +{ + struct ublksrv_io_desc *iod = ublk_get_iod(ubq, req->tag); + struct ublk_io *io = &ubq->ios[req->tag]; + + iod->op_flags = ublk_op | ublk_req_build_flags(req); + iod->nr_sectors = nr_sectors; + iod->start_sector = start_sector; + + /* Try shmem zero-copy match before setting addr */ + if (ublk_support_shmem_zc(ubq) && blk_rq_has_data(req)) { + u32 buf_idx, buf_off; + + if (ublk_try_buf_match(ubq->dev, req, &buf_idx, &buf_off)) { + iod->op_flags |= UBLK_IO_F_SHMEM_ZC; + iod->addr = ublk_shmem_zc_addr(buf_idx, buf_off); + return; + } + } + + iod->addr = io->buf.addr; +} + #ifdef CONFIG_BLK_DEV_ZONED struct ublk_zoned_report_desc { __u64 sector; - __u32 operation; __u32 nr_zones; }; @@ -567,7 +661,6 @@ static int ublk_report_zones(struct gendisk *disk, sector_t sector, goto out; } - desc.operation = UBLK_IO_OP_REPORT_ZONES; desc.sector = sector; desc.nr_zones = zones_in_request; ret = ublk_zoned_insert_report_desc(req, &desc); @@ -611,11 +704,25 @@ out: return ret; } -static blk_status_t ublk_setup_iod_zoned(struct ublk_queue *ubq, - struct request *req) +static bool ublk_validate_req_zoned(const struct request *req) +{ + switch (req_op(req)) { + case REQ_OP_ZONE_OPEN: + case REQ_OP_ZONE_CLOSE: + case REQ_OP_ZONE_FINISH: + case REQ_OP_ZONE_RESET: + case REQ_OP_ZONE_APPEND: + case REQ_OP_ZONE_RESET_ALL: + return true; + case REQ_OP_DRV_IN: + return !!ublk_zoned_get_report_desc(req); + default: + return false; + } +} + +static void ublk_setup_iod_zoned(struct ublk_queue *ubq, struct request *req) { - struct ublksrv_io_desc *iod = ublk_get_iod(ubq, req->tag); - struct ublk_io *io = &ubq->ios[req->tag]; struct ublk_zoned_report_desc *desc; u32 ublk_op; @@ -640,31 +747,15 @@ static blk_status_t ublk_setup_iod_zoned(struct ublk_queue *ubq, break; case REQ_OP_DRV_IN: desc = ublk_zoned_get_report_desc(req); - if (!desc) - return BLK_STS_IOERR; - ublk_op = desc->operation; - switch (ublk_op) { - case UBLK_IO_OP_REPORT_ZONES: - iod->op_flags = ublk_op | ublk_req_build_flags(req); - iod->nr_zones = desc->nr_zones; - iod->start_sector = desc->sector; - return BLK_STS_OK; - default: - return BLK_STS_IOERR; - } - case REQ_OP_DRV_OUT: - /* We do not support drv_out */ - return BLK_STS_NOTSUPP; + ublk_init_iod(ubq, req, UBLK_IO_OP_REPORT_ZONES, desc->nr_zones, + desc->sector); + return; default: - return BLK_STS_IOERR; + WARN_ON_ONCE(1); + return; } - iod->op_flags = ublk_op | ublk_req_build_flags(req); - iod->nr_sectors = blk_rq_sectors(req); - iod->start_sector = blk_rq_pos(req); - iod->addr = io->buf.addr; - - return BLK_STS_OK; + ublk_init_iod(ubq, req, ublk_op, blk_rq_sectors(req), blk_rq_pos(req)); } #else @@ -685,10 +776,14 @@ static int ublk_revalidate_disk_zones(struct ublk_device *ub) return 0; } -static blk_status_t ublk_setup_iod_zoned(struct ublk_queue *ubq, - struct request *req) +static bool ublk_validate_req_zoned(const struct request *req) +{ + return false; +} + +static void ublk_setup_iod_zoned(struct ublk_queue *ubq, struct request *req) { - return BLK_STS_NOTSUPP; + WARN_ON_ONCE(1); } #endif @@ -779,7 +874,7 @@ static unsigned int unprivileged_ublks_added; /* protected by ublk_ctl_mutex */ static struct miscdevice ublk_misc; -static inline unsigned ublk_pos_to_hwq(loff_t pos) +static inline u16 ublk_pos_to_hwq(loff_t pos) { return ((pos - UBLKSRV_IO_BUF_OFFSET) >> UBLK_QID_OFF) & UBLK_QID_BITS_MASK; @@ -790,7 +885,7 @@ static inline unsigned ublk_pos_to_buf_off(loff_t pos) return (pos - UBLKSRV_IO_BUF_OFFSET) & UBLK_IO_BUF_BITS_MASK; } -static inline unsigned ublk_pos_to_tag(loff_t pos) +static inline u16 ublk_pos_to_tag(loff_t pos) { return ((pos - UBLKSRV_IO_BUF_OFFSET) >> UBLK_TAG_OFF) & UBLK_TAG_BITS_MASK; @@ -808,7 +903,7 @@ static void ublk_dev_param_basic_apply(struct ublk_device *ub) static int ublk_integrity_flags(u32 flags) { - int ret_flags = 0; + int ret_flags = BLK_SPLIT_INTERVAL_CAPABLE; if (flags & LBMD_PI_CAP_INTEGRITY) { flags &= ~LBMD_PI_CAP_INTEGRITY; @@ -862,13 +957,30 @@ static int ublk_validate_params(const struct ublk_device *ub) if (p->logical_bs_shift > PAGE_SHIFT || p->logical_bs_shift < 9) return -EINVAL; + /* + * 256M is a reasonable upper bound for physical block size, + * io_min and io_opt; it aligns with the maximum physical + * block size possible in NVMe. + */ + if (p->physical_bs_shift > ilog2(SZ_256M)) + return -EINVAL; + + if (p->io_min_shift > ilog2(SZ_256M)) + return -EINVAL; + + if (p->io_opt_shift > ilog2(SZ_256M)) + return -EINVAL; + if (p->logical_bs_shift > p->physical_bs_shift) return -EINVAL; if (p->max_sectors > (ub->dev_info.max_io_buf_bytes >> 9)) return -EINVAL; - if (ublk_dev_is_zoned(ub) && !p->chunk_sectors) + if (p->max_sectors < PAGE_SECTORS) + return -EINVAL; + + if (ublk_dev_is_zoned(ub) && !is_power_of_2(p->chunk_sectors)) return -EINVAL; } else return -EINVAL; @@ -1129,35 +1241,31 @@ static noinline void ublk_put_device(struct ublk_device *ub) } static inline struct ublk_queue *ublk_get_queue(struct ublk_device *dev, - int qid) + u16 qid) { return dev->queues[qid]; } -static inline bool ublk_rq_has_data(const struct request *rq) -{ - return bio_has_data(rq->bio); -} - static inline struct ublksrv_io_desc * -ublk_queue_cmd_buf(struct ublk_device *ub, int q_id) +ublk_queue_cmd_buf(struct ublk_device *ub, u16 q_id) { return ublk_get_queue(ub, q_id)->io_cmd_buf; } -static inline int __ublk_queue_cmd_buf_size(int depth) +static inline size_t __ublk_queue_cmd_buf_size(const struct ublk_device *ub, + u16 depth) { - return round_up(depth * sizeof(struct ublksrv_io_desc), PAGE_SIZE); + return round_up(depth * (size_t)ub->dev_info.io_desc_size, PAGE_SIZE); } -static inline int ublk_queue_cmd_buf_size(struct ublk_device *ub) +static inline size_t ublk_queue_cmd_buf_size(const struct ublk_device *ub) { - return __ublk_queue_cmd_buf_size(ub->dev_info.queue_depth); + return __ublk_queue_cmd_buf_size(ub, ub->dev_info.queue_depth); } -static int ublk_max_cmd_buf_size(void) +static size_t ublk_max_cmd_buf_size(const struct ublk_device *ub) { - return __ublk_queue_cmd_buf_size(UBLK_MAX_QUEUE_DEPTH); + return __ublk_queue_cmd_buf_size(ub, UBLK_MAX_QUEUE_DEPTH); } /* @@ -1281,10 +1389,18 @@ static bool ublk_copy_user_bvec(const struct bio_vec *bv, unsigned *offset, len = bv->bv_len - *offset; bv_buf = kmap_local_page(bv->bv_page) + bv->bv_offset + *offset; + /* + * Bio pages may originate from slab caches without a usercopy region + * (e.g. jbd2 frozen metadata buffers). This is the same data that + * the loop driver writes to its backing file — no exposure risk. + * The bvec length is always trusted, so the size check in + * check_copy_size() is not needed either. Use the unchecked + * helpers to avoid false positives on slab pages. + */ if (dir == ITER_DEST) - copied = copy_to_iter(bv_buf, len, uiter); + copied = _copy_to_iter(bv_buf, len, uiter); else - copied = copy_from_iter(bv_buf, len, uiter); + copied = _copy_from_iter(bv_buf, len, uiter); kunmap_local(bv_buf); @@ -1343,95 +1459,57 @@ static size_t ublk_copy_user_integrity(const struct request *req, static inline bool ublk_need_map_req(const struct request *req) { - return ublk_rq_has_data(req) && req_op(req) == REQ_OP_WRITE; + return blk_rq_has_data(req) && req_op(req) == REQ_OP_WRITE; } static inline bool ublk_need_unmap_req(const struct request *req) { - return ublk_rq_has_data(req) && + return blk_rq_has_data(req) && (req_op(req) == REQ_OP_READ || req_op(req) == REQ_OP_DRV_IN); } -static unsigned int ublk_map_io(const struct ublk_queue *ubq, - const struct request *req, +static unsigned int ublk_map_io(const struct request *req, const struct ublk_io *io) { - const unsigned int rq_bytes = blk_rq_bytes(req); + struct iov_iter iter; + const int dir = ITER_DEST; - if (!ublk_need_map_io(ubq)) - return rq_bytes; - - /* - * no zero copy, we delay copy WRITE request data into ublksrv - * context and the big benefit is that pinning pages in current - * context is pretty fast, see ublk_pin_user_pages - */ - if (ublk_need_map_req(req)) { - struct iov_iter iter; - const int dir = ITER_DEST; + if (import_ubuf(dir, u64_to_user_ptr(io->buf.addr), blk_rq_bytes(req), + &iter) < 0) + return 0; - import_ubuf(dir, u64_to_user_ptr(io->buf.addr), rq_bytes, &iter); - return ublk_copy_user_pages(req, 0, &iter, dir); - } - return rq_bytes; + return ublk_copy_user_pages(req, 0, &iter, dir); } -static unsigned int ublk_unmap_io(bool need_map, - const struct request *req, +static unsigned int ublk_unmap_io(const struct request *req, const struct ublk_io *io) { - const unsigned int rq_bytes = blk_rq_bytes(req); + struct iov_iter iter; + const int dir = ITER_SOURCE; - if (!need_map) - return rq_bytes; - - if (ublk_need_unmap_req(req)) { - struct iov_iter iter; - const int dir = ITER_SOURCE; - - WARN_ON_ONCE(io->res > rq_bytes); + if (import_ubuf(dir, u64_to_user_ptr(io->buf.addr), io->res, &iter) < 0) + return 0; - import_ubuf(dir, u64_to_user_ptr(io->buf.addr), io->res, &iter); - return ublk_copy_user_pages(req, 0, &iter, dir); - } - return rq_bytes; + return ublk_copy_user_pages(req, 0, &iter, dir); } -static inline unsigned int ublk_req_build_flags(struct request *req) +static bool ublk_validate_req(const struct ublk_queue *ubq, + const struct request *req) { - unsigned flags = 0; - - if (req->cmd_flags & REQ_FAILFAST_DEV) - flags |= UBLK_IO_F_FAILFAST_DEV; - - if (req->cmd_flags & REQ_FAILFAST_TRANSPORT) - flags |= UBLK_IO_F_FAILFAST_TRANSPORT; - - if (req->cmd_flags & REQ_FAILFAST_DRIVER) - flags |= UBLK_IO_F_FAILFAST_DRIVER; - - if (req->cmd_flags & REQ_META) - flags |= UBLK_IO_F_META; - - if (req->cmd_flags & REQ_FUA) - flags |= UBLK_IO_F_FUA; - - if (req->cmd_flags & REQ_NOUNMAP) - flags |= UBLK_IO_F_NOUNMAP; - - if (req->cmd_flags & REQ_SWAP) - flags |= UBLK_IO_F_SWAP; - - if (blk_integrity_rq(req)) - flags |= UBLK_IO_F_INTEGRITY; - - return flags; + switch (req_op(req)) { + case REQ_OP_READ: + case REQ_OP_WRITE: + case REQ_OP_FLUSH: + case REQ_OP_DISCARD: + case REQ_OP_WRITE_ZEROES: + return true; + default: + return ublk_queue_is_zoned(ubq) && ublk_validate_req_zoned(req); + } } -static blk_status_t ublk_setup_iod(struct ublk_queue *ubq, struct request *req) +static void ublk_setup_iod(struct ublk_queue *ubq, struct request *req) { - struct ublksrv_io_desc *iod = ublk_get_iod(ubq, req->tag); - struct ublk_io *io = &ubq->ios[req->tag]; u32 ublk_op; switch (req_op(req)) { @@ -1451,18 +1529,11 @@ static blk_status_t ublk_setup_iod(struct ublk_queue *ubq, struct request *req) ublk_op = UBLK_IO_OP_WRITE_ZEROES; break; default: - if (ublk_queue_is_zoned(ubq)) - return ublk_setup_iod_zoned(ubq, req); - return BLK_STS_IOERR; + ublk_setup_iod_zoned(ubq, req); + return; } - /* need to translate since kernel may change */ - iod->op_flags = ublk_op | ublk_req_build_flags(req); - iod->nr_sectors = blk_rq_sectors(req); - iod->start_sector = blk_rq_pos(req); - iod->addr = io->buf.addr; - - return BLK_STS_OK; + ublk_init_iod(ubq, req, ublk_op, blk_rq_sectors(req), blk_rq_pos(req)); } static inline struct ublk_uring_cmd_pdu *ublk_get_uring_cmd_pdu( @@ -1495,26 +1566,27 @@ static inline void __ublk_complete_rq(struct request *req, struct ublk_io *io, goto exit; } - /* - * FLUSH, DISCARD or WRITE_ZEROES usually won't return bytes returned, so end them - * directly. - * - * Both the two needn't unmap. - */ - if (req_op(req) != REQ_OP_READ && req_op(req) != REQ_OP_WRITE && - req_op(req) != REQ_OP_DRV_IN) + /* shmem zero copy: no data to unmap, pages already shared */ + if (!need_map || !ublk_need_unmap_req(req) || + ublk_iod_is_shmem_zc(req->mq_hctx->driver_data, req->tag)) goto exit; /* for READ request, writing data in iod->addr to rq buffers */ - unmapped_bytes = ublk_unmap_io(need_map, req, io); + unmapped_bytes = ublk_unmap_io(req, io); /* * Extremely impossible since we got data filled in just before * * Re-read simply for this unlikely case. */ - if (unlikely(unmapped_bytes < io->res)) + if (unlikely(unmapped_bytes < io->res)) { + if (unlikely(!unmapped_bytes)) { + res = BLK_STS_IOERR; + goto exit; + } + io->res = unmapped_bytes; + } /* * Run bio->bi_end_io() with softirqs disabled. If the final fput @@ -1586,7 +1658,7 @@ static inline void __ublk_abort_rq(struct ublk_queue *ubq, } static void -ublk_auto_buf_reg_fallback(const struct ublk_queue *ubq, unsigned tag) +ublk_auto_buf_reg_fallback(const struct ublk_queue *ubq, u16 tag) { struct ublksrv_io_desc *iod = ublk_get_iod(ubq, tag); @@ -1627,8 +1699,8 @@ ublk_auto_buf_register(const struct ublk_queue *ubq, struct request *req, { int ret; - ret = io_buffer_register_bvec(cmd, req, ublk_io_release, - io->buf.auto_reg.index, issue_flags); + ret = io_buffer_register_request(cmd, req, ublk_io_release, + io->buf.auto_reg.index, issue_flags); if (ret) { if (io->buf.auto_reg.flags & UBLK_AUTO_BUF_REG_FALLBACK) { ublk_auto_buf_reg_fallback(ubq, req->tag); @@ -1663,7 +1735,14 @@ static void ublk_auto_buf_dispatch(const struct ublk_queue *ubq, static bool ublk_start_io(const struct ublk_queue *ubq, struct request *req, struct ublk_io *io) { - unsigned mapped_bytes = ublk_map_io(ubq, req, io); + unsigned mapped_bytes; + + /* shmem zero copy: skip data copy, pages already shared */ + if (!ublk_need_map_io(ubq) || !ublk_need_map_req(req) || + ublk_iod_is_shmem_zc(ubq, req->tag)) + return true; + + mapped_bytes = ublk_map_io(req, io); /* partially mapped, update io descriptor */ if (unlikely(mapped_bytes != blk_rq_bytes(req))) { @@ -1691,9 +1770,10 @@ static bool ublk_start_io(const struct ublk_queue *ubq, struct request *req, static void ublk_dispatch_req(struct ublk_queue *ubq, struct request *req) { unsigned int issue_flags = IO_URING_CMD_TASK_WORK_ISSUE_FLAGS; - int tag = req->tag; + u16 tag = req->tag; struct ublk_io *io = &ubq->ios[tag]; + ublk_setup_iod(ubq, req); pr_devel("%s: complete: qid %d tag %d io_flags %x addr %llx\n", __func__, ubq->q_id, req->tag, io->flags, ublk_get_iod(ubq, req->tag)->addr); @@ -1729,7 +1809,7 @@ static void ublk_dispatch_req(struct ublk_queue *ubq, struct request *req) if (!ublk_start_io(ubq, req, io)) return; - if (ublk_support_auto_buf_reg(ubq) && ublk_rq_has_data(req)) { + if (ublk_support_auto_buf_reg(ubq) && blk_rq_has_data(req)) { ublk_auto_buf_dispatch(ubq, req, io, io->cmd, issue_flags); } else { ublk_init_req_ref(ubq, io); @@ -1747,10 +1827,11 @@ static bool __ublk_batch_prep_dispatch(struct ublk_queue *ubq, enum auto_buf_reg_res res = AUTO_BUF_REG_FALLBACK; struct io_uring_cmd *cmd = data->cmd; + ublk_setup_iod(ubq, req); if (!ublk_start_io(ubq, req, io)) return false; - if (ublk_support_auto_buf_reg(ubq) && ublk_rq_has_data(req)) { + if (ublk_support_auto_buf_reg(ubq) && blk_rq_has_data(req)) { res = ublk_auto_buf_register(ubq, req, io, cmd, data->issue_flags); @@ -1789,7 +1870,7 @@ static bool ublk_batch_prep_dispatch(struct ublk_queue *ubq, * Filter out UBLK_BATCH_IO_UNUSED_TAG entries from tag_buf. * Returns the new length after filtering. */ -static unsigned int ublk_filter_unused_tags(unsigned short *tag_buf, +static noinline unsigned int ublk_filter_unused_tags(unsigned short *tag_buf, unsigned int len) { unsigned int i, j; @@ -1805,6 +1886,41 @@ static unsigned int ublk_filter_unused_tags(unsigned short *tag_buf, return j; } +static noinline void ublk_batch_dispatch_fail(struct ublk_queue *ubq, + const struct ublk_batch_io_data *data, + unsigned short *tag_buf, size_t len, int ret) +{ + int i, res; + + /* + * Undo prep state for all IOs since userspace never received them. + * This restores IOs to pre-prepared state so they can be cleanly + * re-prepared when tags are pulled from FIFO again. + */ + for (i = 0; i < len; i++) { + struct ublk_io *io = &ubq->ios[tag_buf[i]]; + int index = -1; + + ublk_io_lock(io); + if (io->flags & UBLK_IO_FLAG_AUTO_BUF_REG) + index = io->buf.auto_reg.index; + io->flags &= ~(UBLK_IO_FLAG_OWNED_BY_SRV | UBLK_IO_FLAG_AUTO_BUF_REG); + io->flags |= UBLK_IO_FLAG_ACTIVE; + ublk_io_unlock(io); + + if (index != -1) + io_buffer_unregister(data->cmd, index, + data->issue_flags); + } + + res = kfifo_in_spinlocked_noirqsave(&ubq->evts_fifo, + tag_buf, len, &ubq->evts_lock); + + pr_warn_ratelimited("%s: copy tags or post CQE failure, move back " + "tags(%d %zu) ret %d\n", __func__, res, len, + ret); +} + #define MAX_NR_TAG 128 static int __ublk_batch_dispatch(struct ublk_queue *ubq, const struct ublk_batch_io_data *data, @@ -1848,37 +1964,8 @@ static int __ublk_batch_dispatch(struct ublk_queue *ubq, sel.val = ublk_batch_copy_io_tags(fcmd, sel.addr, tag_buf, len * tag_sz); ret = ublk_batch_fetch_post_cqe(fcmd, &sel, data->issue_flags); - if (unlikely(ret < 0)) { - int i, res; - - /* - * Undo prep state for all IOs since userspace never received them. - * This restores IOs to pre-prepared state so they can be cleanly - * re-prepared when tags are pulled from FIFO again. - */ - for (i = 0; i < len; i++) { - struct ublk_io *io = &ubq->ios[tag_buf[i]]; - int index = -1; - - ublk_io_lock(io); - if (io->flags & UBLK_IO_FLAG_AUTO_BUF_REG) - index = io->buf.auto_reg.index; - io->flags &= ~(UBLK_IO_FLAG_OWNED_BY_SRV | UBLK_IO_FLAG_AUTO_BUF_REG); - io->flags |= UBLK_IO_FLAG_ACTIVE; - ublk_io_unlock(io); - - if (index != -1) - io_buffer_unregister_bvec(data->cmd, index, - data->issue_flags); - } - - res = kfifo_in_spinlocked_noirqsave(&ubq->evts_fifo, - tag_buf, len, &ubq->evts_lock); - - pr_warn_ratelimited("%s: copy tags or post CQE failure, move back " - "tags(%d %zu) ret %d\n", __func__, res, len, - ret); - } + if (unlikely(ret < 0)) + ublk_batch_dispatch_fail(ubq, data, tag_buf, len, ret); return ret; } @@ -2054,8 +2141,6 @@ static enum blk_eh_timer_return ublk_timeout(struct request *rq) static blk_status_t ublk_prep_req(struct ublk_queue *ubq, struct request *rq, bool check_cancel) { - blk_status_t res; - if (unlikely(READ_ONCE(ubq->fail_io))) return BLK_STS_TARGET; @@ -2076,8 +2161,7 @@ static blk_status_t ublk_prep_req(struct ublk_queue *ubq, struct request *rq, return BLK_STS_IOERR; /* fill iod to slot in io cmd buffer */ - res = ublk_setup_iod(ubq, rq); - if (unlikely(res != BLK_STS_OK)) + if (unlikely(!ublk_validate_req(ubq, rq))) return BLK_STS_IOERR; blk_mq_start_request(rq); @@ -2275,7 +2359,7 @@ static const struct blk_mq_ops ublk_batch_mq_ops = { static void ublk_queue_reinit(struct ublk_device *ub, struct ublk_queue *ubq) { - int i; + u16 i; ubq->nr_io_ready = 0; @@ -2320,10 +2404,16 @@ static int ublk_ch_open(struct inode *inode, struct file *filp) static void ublk_reset_ch_dev(struct ublk_device *ub) { - int i; + u16 i; - for (i = 0; i < ub->dev_info.nr_hw_queues; i++) - ublk_queue_reinit(ub, ublk_get_queue(ub, i)); + for (i = 0; i < ub->dev_info.nr_hw_queues; i++) { + struct ublk_queue *ubq = ublk_get_queue(ub, i); + + /* Sync with ublk_cancel_cmd() */ + spin_lock(&ubq->cancel_lock); + ublk_queue_reinit(ub, ubq); + spin_unlock(&ubq->cancel_lock); + } /* set to NULL, otherwise new tasks cannot mmap io_cmd_buf */ ub->mm = NULL; @@ -2386,7 +2476,7 @@ out: static void ublk_set_canceling(struct ublk_device *ub, bool canceling) __must_hold(&ub->cancel_mutex) { - int i; + u16 i; ub->canceling = canceling; for (i = 0; i < ub->dev_info.nr_hw_queues; i++) @@ -2395,7 +2485,7 @@ static void ublk_set_canceling(struct ublk_device *ub, bool canceling) static bool ublk_check_and_reset_active_ref(struct ublk_device *ub) { - int i, j; + u16 i, j; if (!ublk_dev_need_req_ref(ub)) return false; @@ -2428,7 +2518,7 @@ static void ublk_ch_release_work_fn(struct work_struct *work) struct ublk_device *ub = container_of(work, struct ublk_device, exit_work.work); struct gendisk *disk; - int i; + u16 i; /* * For zero-copy and auto buffer register modes, I/O references @@ -2545,9 +2635,10 @@ static int ublk_ch_mmap(struct file *filp, struct vm_area_struct *vma) { struct ublk_device *ub = filp->private_data; size_t sz = vma->vm_end - vma->vm_start; - unsigned max_sz = ublk_max_cmd_buf_size(); + size_t max_sz = ublk_max_cmd_buf_size(ub); unsigned long pfn, end, phys_off = vma->vm_pgoff << PAGE_SHIFT; - int q_id, ret = 0; + int ret = 0; + u16 q_id; spin_lock(&ub->lock); if (!ub->mm) @@ -2562,6 +2653,12 @@ static int ublk_ch_mmap(struct file *filp, struct vm_area_struct *vma) if (vma->vm_flags & VM_WRITE) return -EPERM; + /* + * The per-queue command buffer is kernel-written ABI; prevent + * the daemon from upgrading to writable via mprotect(). + */ + vm_flags_clear(vma, VM_MAYWRITE); + end = UBLKSRV_CMD_BUF_OFFSET + ub->dev_info.nr_hw_queues * max_sz; if (phys_off < UBLKSRV_CMD_BUF_OFFSET || phys_off >= end) return -EINVAL; @@ -2620,7 +2717,7 @@ static void ublk_abort_batch_queue(struct ublk_device *ub, */ static void ublk_abort_queue(struct ublk_device *ub, struct ublk_queue *ubq) { - int i; + u16 i; for (i = 0; i < ubq->q_depth; i++) { struct ublk_io *io = &ubq->ios[i]; @@ -2637,33 +2734,38 @@ static void ublk_start_cancel(struct ublk_device *ub) { struct gendisk *disk = ublk_get_disk(ub); - /* Our disk has been dead */ - if (!disk) - return; - mutex_lock(&ub->cancel_mutex); if (ub->canceling) goto out; - /* - * Now we are serialized with ublk_queue_rq() - * - * Make sure that ubq->canceling is set when queue is frozen, - * because ublk_queue_rq() has to rely on this flag for avoiding to - * touch completed uring_cmd - */ - blk_mq_quiesce_queue(disk->queue); - ublk_set_canceling(ub, true); - blk_mq_unquiesce_queue(disk->queue); + + if (disk) { + /* + * Quiesce to serialize with ublk_queue_rq(), ensuring + * ubq->canceling is visible when the queue resumes. + */ + blk_mq_quiesce_queue(disk->queue); + ublk_set_canceling(ub, true); + blk_mq_unquiesce_queue(disk->queue); + } else { + /* + * Disk not yet allocated by ublk_ctrl_start_dev(), so + * there is no request queue and ublk_queue_rq() cannot + * be running. Just set the flag; if start_dev proceeds + * later, new I/O will see canceling and be aborted. + */ + ublk_set_canceling(ub, true); + } out: mutex_unlock(&ub->cancel_mutex); ublk_put_disk(disk); } -static void ublk_cancel_cmd(struct ublk_queue *ubq, unsigned tag, +static void ublk_cancel_cmd(struct ublk_queue *ubq, u16 tag, unsigned int issue_flags) { struct ublk_io *io = &ubq->ios[tag]; struct ublk_device *ub = ubq->dev; + struct io_uring_cmd *cmd = NULL; struct request *req; bool done; @@ -2686,12 +2788,15 @@ static void ublk_cancel_cmd(struct ublk_queue *ubq, unsigned tag, spin_lock(&ubq->cancel_lock); done = !!(io->flags & UBLK_IO_FLAG_CANCELED); - if (!done) + if (!done) { io->flags |= UBLK_IO_FLAG_CANCELED; + cmd = io->cmd; + io->cmd = NULL; + } spin_unlock(&ubq->cancel_lock); - if (!done) - io_uring_cmd_done(io->cmd, UBLK_IO_RES_ABORT, issue_flags); + if (!done && cmd) + io_uring_cmd_done(cmd, UBLK_IO_RES_ABORT, issue_flags); } /* @@ -2806,7 +2911,7 @@ static inline bool ublk_dev_ready(const struct ublk_device *ub) static void ublk_cancel_queue(struct ublk_queue *ubq) { - int i; + u16 i; if (ublk_support_batch_io(ubq)) { ublk_batch_cancel_queue(ubq); @@ -2820,7 +2925,7 @@ static void ublk_cancel_queue(struct ublk_queue *ubq) /* Cancel all pending commands, must be called after del_gendisk() returns */ static void ublk_cancel_dev(struct ublk_device *ub) { - int i; + u16 i; for (i = 0; i < ub->dev_info.nr_hw_queues; i++) ublk_cancel_queue(ublk_get_queue(ub, i)); @@ -2854,7 +2959,7 @@ static void ublk_wait_tagset_rqs_idle(struct ublk_device *ub) static void ublk_force_abort_dev(struct ublk_device *ub) { - int i; + u16 i; pr_devel("%s: force abort ub: dev_id %d state %s\n", __func__, ub->dev_info.dev_id, @@ -2910,22 +3015,27 @@ static void ublk_stop_dev(struct ublk_device *ub) ublk_cancel_dev(ub); } +static void ublk_reset_io_flags(struct ublk_queue *ubq, struct ublk_io *io) +{ + /* UBLK_IO_FLAG_CANCELED can be cleared now */ + spin_lock(&ubq->cancel_lock); + io->flags &= ~UBLK_IO_FLAG_CANCELED; + spin_unlock(&ubq->cancel_lock); +} + /* reset per-queue io flags */ static void ublk_queue_reset_io_flags(struct ublk_queue *ubq) { - int j; - - /* UBLK_IO_FLAG_CANCELED can be cleared now */ spin_lock(&ubq->cancel_lock); - for (j = 0; j < ubq->q_depth; j++) - ubq->ios[j].flags &= ~UBLK_IO_FLAG_CANCELED; ubq->canceling = false; spin_unlock(&ubq->cancel_lock); ubq->fail_io = false; + ubq->force_abort = false; } /* device can only be started after all IOs are ready */ -static void ublk_mark_io_ready(struct ublk_device *ub, u16 q_id) +static void ublk_mark_io_ready(struct ublk_device *ub, u16 q_id, + struct ublk_io *io) __must_hold(&ub->mutex) { struct ublk_queue *ubq = ublk_get_queue(ub, q_id); @@ -2934,6 +3044,7 @@ static void ublk_mark_io_ready(struct ublk_device *ub, u16 q_id) ub->unprivileged_daemons = true; ubq->nr_io_ready++; + ublk_reset_io_flags(ubq, io); /* Check if this specific queue is now fully ready */ if (ublk_queue_ready(ubq)) { @@ -2951,12 +3062,12 @@ static void ublk_mark_io_ready(struct ublk_device *ub, u16 q_id) if (ublk_dev_ready(ub)) { /* * All queues ready - clear device-level canceling flag - * and complete the recovery/initialization. + * and wake ublk_dev_ready() waiters. */ mutex_lock(&ub->cancel_mutex); ub->canceling = false; mutex_unlock(&ub->cancel_mutex); - complete_all(&ub->completion); + wake_up_var(&ub->nr_queue_ready); } } @@ -2973,18 +3084,19 @@ static inline int ublk_check_cmd_op(u32 cmd_op) return 0; } -static inline int ublk_set_auto_buf_reg(struct ublk_io *io, struct io_uring_cmd *cmd) +/* Must run before ublk_fill_io_cmd() / __ublk_fetch(). */ +static inline int ublk_validate_io_buf(const struct ublk_device *ub, + struct io_uring_cmd *cmd, + struct ublk_auto_buf_reg *buf) { - struct ublk_auto_buf_reg buf; - - buf = ublk_sqe_addr_to_auto_buf_reg(READ_ONCE(cmd->sqe->addr)); + if (!ublk_dev_support_auto_buf_reg(ub)) + return 0; - if (buf.reserved0 || buf.reserved1) + *buf = ublk_sqe_addr_to_auto_buf_reg(READ_ONCE(cmd->sqe->addr)); + if (buf->reserved0 || buf->reserved1) return -EINVAL; - - if (buf.flags & ~UBLK_AUTO_BUF_REG_F_MASK) + if (buf->flags & ~UBLK_AUTO_BUF_REG_F_MASK) return -EINVAL; - io->buf.auto_reg = buf; return 0; } @@ -3005,17 +3117,25 @@ static void ublk_clear_auto_buf_reg(struct ublk_io *io, * responsibility for unregistering the buffer, otherwise * this ublk request gets stuck. */ - if (io->buf_ctx_handle == io_uring_cmd_ctx_handle(cmd)) + if (buf_idx && + io->buf_ctx_handle == io_uring_cmd_ctx_handle(cmd)) *buf_idx = io->buf.auto_reg.index; } } -static int ublk_handle_auto_buf_reg(struct ublk_io *io, - struct io_uring_cmd *cmd, - u16 *buf_idx) +static inline void ublk_apply_io_buf(const struct ublk_device *ub, + struct ublk_io *io, + struct io_uring_cmd *cmd, + unsigned long buf_addr, + const struct ublk_auto_buf_reg *auto_buf, + u16 *buf_idx) { - ublk_clear_auto_buf_reg(io, cmd, buf_idx); - return ublk_set_auto_buf_reg(io, cmd); + if (ublk_dev_support_auto_buf_reg(ub)) { + ublk_clear_auto_buf_reg(io, cmd, buf_idx); + io->buf.auto_reg = *auto_buf; + } else { + io->buf.addr = buf_addr; + } } /* Once we return, `io->req` can't be used any more */ @@ -3032,21 +3152,9 @@ ublk_fill_io_cmd(struct ublk_io *io, struct io_uring_cmd *cmd) return req; } -static inline int -ublk_config_io_buf(const struct ublk_device *ub, struct ublk_io *io, - struct io_uring_cmd *cmd, unsigned long buf_addr, - u16 *buf_idx) -{ - if (ublk_dev_support_auto_buf_reg(ub)) - return ublk_handle_auto_buf_reg(io, cmd, buf_idx); - - io->buf.addr = buf_addr; - return 0; -} - static inline void ublk_prep_cancel(struct io_uring_cmd *cmd, unsigned int issue_flags, - struct ublk_queue *ubq, unsigned int tag) + struct ublk_queue *ubq, u16 tag) { struct ublk_uring_cmd_pdu *pdu = ublk_get_uring_cmd_pdu(cmd); @@ -3091,8 +3199,8 @@ static int ublk_register_io_buf(struct io_uring_cmd *cmd, if (!req) return -EINVAL; - ret = io_buffer_register_bvec(cmd, req, ublk_io_release, index, - issue_flags); + ret = io_buffer_register_request(cmd, req, ublk_io_release, index, + issue_flags); if (ret) { ublk_put_req_ref(io, req); return ret; @@ -3120,11 +3228,11 @@ ublk_daemon_register_io_buf(struct io_uring_cmd *cmd, return ublk_register_io_buf(cmd, ub, q_id, tag, io, index, issue_flags); - if (!ublk_dev_support_zero_copy(ub) || !ublk_rq_has_data(req)) + if (!ublk_dev_support_zero_copy(ub) || !blk_rq_has_data(req)) return -EINVAL; - ret = io_buffer_register_bvec(cmd, req, ublk_io_release, index, - issue_flags); + ret = io_buffer_register_request(cmd, req, ublk_io_release, index, + issue_flags); if (ret) return ret; @@ -3139,7 +3247,7 @@ static int ublk_unregister_io_buf(struct io_uring_cmd *cmd, if (!(ub->dev_info.flags & UBLK_F_SUPPORT_ZERO_COPY)) return -EINVAL; - return io_buffer_unregister_bvec(cmd, index, issue_flags); + return io_buffer_unregister(cmd, index, issue_flags); } static int ublk_check_fetch_buf(const struct ublk_device *ub, __u64 buf_addr) @@ -3184,6 +3292,7 @@ static int __ublk_fetch(struct io_uring_cmd *cmd, struct ublk_device *ub, static int ublk_fetch(struct io_uring_cmd *cmd, struct ublk_device *ub, struct ublk_io *io, __u64 buf_addr, u16 q_id) { + struct ublk_auto_buf_reg auto_buf; int ret; /* @@ -3192,11 +3301,13 @@ static int ublk_fetch(struct io_uring_cmd *cmd, struct ublk_device *ub, * FETCH, so it is fine even for IO_URING_F_NONBLOCK. */ mutex_lock(&ub->mutex); - ret = __ublk_fetch(cmd, ub, io, q_id); - if (!ret) - ret = ublk_config_io_buf(ub, io, cmd, buf_addr, NULL); + ret = ublk_validate_io_buf(ub, cmd, &auto_buf); if (!ret) - ublk_mark_io_ready(ub, q_id); + ret = __ublk_fetch(cmd, ub, io, q_id); + if (!ret) { + ublk_apply_io_buf(ub, io, cmd, buf_addr, &auto_buf, NULL); + ublk_mark_io_ready(ub, q_id, io); + } mutex_unlock(&ub->mutex); return ret; } @@ -3280,7 +3391,7 @@ static int ublk_ch_uring_cmd_local(struct io_uring_cmd *cmd, goto out; /* - * io_buffer_unregister_bvec() doesn't access the ubq or io, + * io_buffer_unregister() doesn't access the ubq or io, * so no need to validate the q_id, tag, or task */ if (_IOC_NR(cmd_op) == UBLK_IO_UNREGISTER_IO_BUF) @@ -3339,25 +3450,28 @@ static int ublk_ch_uring_cmd_local(struct io_uring_cmd *cmd, case UBLK_IO_REGISTER_IO_BUF: return ublk_daemon_register_io_buf(cmd, ub, q_id, tag, io, addr, issue_flags); - case UBLK_IO_COMMIT_AND_FETCH_REQ: + case UBLK_IO_COMMIT_AND_FETCH_REQ: { + struct ublk_auto_buf_reg auto_buf; + ret = ublk_check_commit_and_fetch(ub, io, addr); if (ret) goto out; + ret = ublk_validate_io_buf(ub, cmd, &auto_buf); + if (ret) + goto out; io->res = result; req = ublk_fill_io_cmd(io, cmd); - ret = ublk_config_io_buf(ub, io, cmd, addr, &buf_idx); + ublk_apply_io_buf(ub, io, cmd, addr, &auto_buf, &buf_idx); if (buf_idx != UBLK_INVALID_BUF_IDX) - io_buffer_unregister_bvec(cmd, buf_idx, issue_flags); + io_buffer_unregister(cmd, buf_idx, issue_flags); compl = ublk_need_complete_req(ub, io); if (req_op(req) == REQ_OP_ZONE_APPEND) req->__sector = addr; if (compl) __ublk_complete_rq(req, io, ublk_dev_need_map_io(ub), NULL); - - if (ret) - goto out; break; + } case UBLK_IO_NEED_GET_DATA: /* * ublk_get_data() may fail and fallback to requeue, so keep @@ -3365,8 +3479,7 @@ static int ublk_ch_uring_cmd_local(struct io_uring_cmd *cmd, * request */ req = ublk_fill_io_cmd(io, cmd); - ret = ublk_config_io_buf(ub, io, cmd, addr, NULL); - WARN_ON_ONCE(ret); + io->buf.addr = addr; if (likely(ublk_get_data(ubq, io, req))) { __ublk_prep_compl_io_cmd(io, req); return UBLK_IO_RES_OK; @@ -3403,7 +3516,7 @@ static inline struct request *__ublk_check_and_get_req(struct ublk_device *ub, if (unlikely(!blk_mq_request_started(req) || req->tag != tag)) goto fail_put; - if (!ublk_rq_has_data(req)) + if (!blk_rq_has_data(req)) goto fail_put; return req; @@ -3416,8 +3529,10 @@ static void ublk_ch_uring_cmd_cb(struct io_tw_req tw_req, io_tw_token_t tw) { unsigned int issue_flags = IO_URING_CMD_TASK_WORK_ISSUE_FLAGS; struct io_uring_cmd *cmd = io_uring_cmd_from_tw(tw_req); - int ret = ublk_ch_uring_cmd_local(cmd, issue_flags); + int ret = -ECANCELED; + if (!tw.cancel) + ret = ublk_ch_uring_cmd_local(cmd, issue_flags); if (ret != -EIOCBQUEUED) io_uring_cmd_done(cmd, ret, issue_flags); } @@ -3479,6 +3594,7 @@ ublk_batch_auto_buf_reg(const struct ublk_batch_io *uc, #define UBLK_CMD_BATCH_TMP_BUF_SZ (48 * 10) struct ublk_batch_io_iter { void __user *uaddr; + const u8 *kaddr; unsigned done, total; unsigned char elem_bytes; /* copy to this buffer from user space */ @@ -3527,7 +3643,10 @@ static int ublk_walk_cmd_buf(struct ublk_batch_io_iter *iter, while (iter->done < iter->total) { unsigned int len = min(sizeof(iter->buf), iter->total - iter->done); - if (copy_from_user(iter->buf, iter->uaddr + iter->done, len)) { + if (iter->kaddr) { + memcpy(iter->buf, iter->kaddr + iter->done, len); + } else if (copy_from_user(iter->buf, iter->uaddr + iter->done, + len)) { pr_warn("ublk%d: read batch cmd buffer failed\n", data->ub->dev_info.dev_id); return -EFAULT; @@ -3604,7 +3723,7 @@ static int ublk_batch_prep_io(struct ublk_queue *ubq, ublk_io_unlock(io); if (!ret) - ublk_mark_io_ready(data->ub, ubq->q_id); + ublk_mark_io_ready(data->ub, ubq->q_id, io); return ret; } @@ -3618,14 +3737,21 @@ static int ublk_handle_batch_prep_cmd(const struct ublk_batch_io_data *data) .total = uc->nr_elem * uc->elem_bytes, .elem_bytes = uc->elem_bytes, }; + void *cmd_buf; int ret; + cmd_buf = vmemdup_user(iter.uaddr, iter.total); + if (IS_ERR(cmd_buf)) + return PTR_ERR(cmd_buf); + iter.kaddr = cmd_buf; + mutex_lock(&data->ub->mutex); ret = ublk_walk_cmd_buf(&iter, data, ublk_batch_prep_io); if (ret && iter.done) ublk_batch_revert_prep_cmd(&iter, data); mutex_unlock(&data->ub->mutex); + kvfree(cmd_buf); return ret; } @@ -3665,11 +3791,11 @@ static int ublk_batch_commit_io(struct ublk_queue *ubq, ret = ublk_batch_commit_io_check(ubq, io, &buf); if (!ret) { io->res = elem->result; - io->buf = buf; req = ublk_fill_io_cmd(io, data->cmd); if (auto_reg) ublk_clear_auto_buf_reg(io, data->cmd, &buf_idx); + io->buf = buf; compl = ublk_need_complete_req(data->ub, io); } ublk_io_unlock(io); @@ -3682,7 +3808,7 @@ static int ublk_batch_commit_io(struct ublk_queue *ubq, } if (buf_idx != UBLK_INVALID_BUF_IDX) - io_buffer_unregister_bvec(data->cmd, buf_idx, data->issue_flags); + io_buffer_unregister(data->cmd, buf_idx, data->issue_flags); if (req_op(req) == REQ_OP_ZONE_APPEND) req->__sector = ublk_batch_zone_lba(uc, elem); if (compl) @@ -3837,8 +3963,8 @@ static int ublk_handle_non_batch_cmd(struct io_uring_cmd *cmd, const struct ublksrv_io_cmd *ub_cmd = io_uring_sqe_cmd(cmd->sqe, struct ublksrv_io_cmd); struct ublk_device *ub = cmd->file->private_data; - unsigned tag = READ_ONCE(ub_cmd->tag); - unsigned q_id = READ_ONCE(ub_cmd->q_id); + u16 tag = READ_ONCE(ub_cmd->tag); + u16 q_id = READ_ONCE(ub_cmd->q_id); unsigned index = READ_ONCE(ub_cmd->addr); struct ublk_queue *ubq; struct ublk_io *io; @@ -3976,7 +4102,7 @@ ublk_user_copy(struct kiocb *iocb, struct iov_iter *iter, int dir) return -EINVAL; req = io->req; - if (!ublk_rq_has_data(req)) + if (!blk_rq_has_data(req)) return -EINVAL; } else { req = __ublk_check_and_get_req(ub, q_id, tag, io); @@ -4044,7 +4170,8 @@ static const struct file_operations ublk_ch_batch_io_fops = { static void __ublk_deinit_queue(struct ublk_device *ub, struct ublk_queue *ubq) { - int size, i; + size_t size; + u16 i; size = ublk_queue_cmd_buf_size(ub); @@ -4065,7 +4192,7 @@ static void __ublk_deinit_queue(struct ublk_device *ub, struct ublk_queue *ubq) kvfree(ubq); } -static void ublk_deinit_queue(struct ublk_device *ub, int q_id) +static void ublk_deinit_queue(struct ublk_device *ub, u16 q_id) { struct ublk_queue *ubq = ub->queues[q_id]; @@ -4076,7 +4203,7 @@ static void ublk_deinit_queue(struct ublk_device *ub, int q_id) ub->queues[q_id] = NULL; } -static int ublk_get_queue_numa_node(struct ublk_device *ub, int q_id) +static int ublk_get_queue_numa_node(struct ublk_device *ub, u16 q_id) { unsigned int cpu; @@ -4089,14 +4216,16 @@ static int ublk_get_queue_numa_node(struct ublk_device *ub, int q_id) return NUMA_NO_NODE; } -static int ublk_init_queue(struct ublk_device *ub, int q_id) +static int ublk_init_queue(struct ublk_device *ub, u16 q_id) { - int depth = ub->dev_info.queue_depth; + u16 depth = ub->dev_info.queue_depth; gfp_t gfp_flags = GFP_KERNEL | __GFP_ZERO; struct ublk_queue *ubq; struct page *page; int numa_node; - int size, i, ret; + size_t size; + int ret; + u16 i; /* Determine NUMA node based on queue's CPU affinity */ numa_node = ublk_get_queue_numa_node(ub, q_id); @@ -4120,6 +4249,7 @@ static int ublk_init_queue(struct ublk_device *ub, int q_id) return -ENOMEM; } ubq->io_cmd_buf = page_address(page); + ubq->io_desc_size = ub->dev_info.io_desc_size; for (i = 0; i < ubq->q_depth; i++) spin_lock_init(&ubq->ios[i].lock); @@ -4141,7 +4271,7 @@ fail: static void ublk_deinit_queues(struct ublk_device *ub) { - int i; + u16 i; for (i = 0; i < ub->dev_info.nr_hw_queues; i++) ublk_deinit_queue(ub, i); @@ -4149,7 +4279,8 @@ static void ublk_deinit_queues(struct ublk_device *ub) static int ublk_init_queues(struct ublk_device *ub) { - int i, ret; + int ret; + u16 i; for (i = 0; i < ub->dev_info.nr_hw_queues; i++) { ret = ublk_init_queue(ub, i); @@ -4157,7 +4288,6 @@ static int ublk_init_queues(struct ublk_device *ub) goto fail; } - init_completion(&ub->completion); return 0; fail: @@ -4200,6 +4330,7 @@ static void ublk_cdev_rel(struct device *dev) { struct ublk_device *ub = container_of(dev, struct ublk_device, cdev_dev); + ublk_buf_cleanup(ub); blk_mq_free_tag_set(&ub->tag_set); ublk_deinit_queues(ub); ublk_free_dev_number(ub); @@ -4300,6 +4431,26 @@ static bool ublk_validate_user_pid(struct ublk_device *ub, pid_t ublksrv_pid) return ub->ublksrv_tgid == ublksrv_pid; } +/* + * Wait until all queues have fetched their I/O commands, and return with + * ub->mutex held and readiness guaranteed: then every queue's ->canceling + * is cleared. Ready may regress between wakeup and mutex_lock() (F_BATCH + * UNPREP, daemon death), so re-check it under the mutex and wait again. + */ +static int ublk_wait_dev_ready_and_lock(struct ublk_device *ub) +{ + while (true) { + if (wait_var_event_interruptible(&ub->nr_queue_ready, + ublk_dev_ready(ub))) + return -EINTR; + + mutex_lock(&ub->mutex); + if (ublk_dev_ready(ub)) + return 0; + mutex_unlock(&ub->mutex); + } +} + static int ublk_ctrl_start_dev(struct ublk_device *ub, const struct ublksrv_ctrl_cmd *header) { @@ -4382,15 +4533,10 @@ static int ublk_ctrl_start_dev(struct ublk_device *ub, }; } - if (wait_for_completion_interruptible(&ub->completion) != 0) + if (ublk_wait_dev_ready_and_lock(ub)) return -EINTR; - if (!ublk_validate_user_pid(ub, ublksrv_pid)) - return -EINVAL; - - mutex_lock(&ub->mutex); - /* device may become not ready in case of F_BATCH */ - if (!ublk_dev_ready(ub)) { + if (!ublk_validate_user_pid(ub, ublksrv_pid)) { ret = -EINVAL; goto out_unlock; } @@ -4588,6 +4734,15 @@ static int ublk_ctrl_add_dev(const struct ublksrv_ctrl_cmd *header) if (info.flags & UBLK_F_INTEGRITY && !(info.flags & UBLK_F_USER_COPY)) return -EINVAL; + if (info.flags & UBLK_F_IO_DESC_SIZE) { + if (info.io_desc_size < sizeof(struct ublksrv_io_desc) || + info.io_desc_size % _Alignof(struct ublksrv_io_desc) || + info.io_desc_size > UBLK_MAX_IO_DESC_SIZE) + return -EINVAL; + } else { + info.io_desc_size = sizeof(struct ublksrv_io_desc); + } + /* the created device is always owned by current user */ ublk_store_owner_uid_gid(&info.owner_uid, &info.owner_gid); @@ -4621,6 +4776,8 @@ static int ublk_ctrl_add_dev(const struct ublksrv_ctrl_cmd *header) mutex_init(&ub->mutex); spin_lock_init(&ub->lock); mutex_init(&ub->cancel_mutex); + mt_init(&ub->buf_tree); + ida_init(&ub->buf_ida); INIT_WORK(&ub->partition_scan_work, ublk_partition_scan_work); ret = ublk_alloc_dev_number(ub, header->dev_id); @@ -4633,6 +4790,15 @@ static int ublk_ctrl_add_dev(const struct ublksrv_ctrl_cmd *header) ub->dev_info.dev_id = ub->ub_number; /* + * ->state and ->ublksrv_pid are owned by the driver and only read back + * by userspace, but they come from the copied-in dev_info, so reset + * them. Otherwise a device added with ->state != DEAD looks live while + * ->ub_disk is still NULL. + */ + ub->dev_info.state = UBLK_S_DEV_DEAD; + ub->dev_info.ublksrv_pid = -1; + + /* * 64bit flags will be copied back to userspace as feature * negotiation result, so have to clear flags which driver * doesn't support yet, then userspace can get correct flags @@ -4907,13 +5073,15 @@ static int ublk_ctrl_set_params(struct ublk_device *ub, */ ret = -EACCES; } else if (copy_from_user(&ub->params, argp, ph.len)) { + /* zero out partial copy so no stale params survive */ + memset(&ub->params, 0, sizeof(ub->params)); ret = -EFAULT; } else { /* clear all we don't support yet */ ub->params.types &= UBLK_PARAM_TYPE_ALL; ret = ublk_validate_params(ub); if (ret) - ub->params.types = 0; + memset(&ub->params, 0, sizeof(ub->params)); } mutex_unlock(&ub->mutex); @@ -4950,7 +5118,6 @@ static int ublk_ctrl_start_recovery(struct ublk_device *ub) goto out_unlock; } pr_devel("%s: start recovery for dev id %d\n", __func__, ub->ub_number); - init_completion(&ub->completion); ret = 0; out_unlock: mutex_unlock(&ub->mutex); @@ -4966,16 +5133,17 @@ static int ublk_ctrl_end_recovery(struct ublk_device *ub, pr_devel("%s: Waiting for all FETCH_REQs, dev id %d...\n", __func__, header->dev_id); - if (wait_for_completion_interruptible(&ub->completion)) + if (ublk_wait_dev_ready_and_lock(ub)) return -EINTR; pr_devel("%s: All FETCH_REQs received, dev id %d\n", __func__, header->dev_id); - if (!ublk_validate_user_pid(ub, ublksrv_pid)) - return -EINVAL; + if (!ublk_validate_user_pid(ub, ublksrv_pid)) { + ret = -EINVAL; + goto out_unlock; + } - mutex_lock(&ub->mutex); if (ublk_nosrv_should_stop_dev(ub)) goto out_unlock; @@ -5028,7 +5196,7 @@ out: struct count_busy { const struct ublk_queue *ubq; - unsigned int nr_busy; + u16 nr_busy; }; static bool ublk_count_busy_req(struct request *rq, void *data) @@ -5066,8 +5234,7 @@ static int ublk_wait_for_idle_io(struct ublk_device *ub, return 0; while (elapsed < timeout_ms && !signal_pending(current)) { - unsigned int queues_cancelable = 0; - int i; + u16 i, queues_cancelable = 0; for (i = 0; i < ub->dev_info.nr_hw_queues; i++) { struct ublk_queue *ubq = ublk_get_queue(ub, i); @@ -5171,6 +5338,340 @@ exit: return err; } +/* + * Lock for maple tree modification: acquire ub->mutex, then freeze queue + * if device is started. If device is not yet started, only mutex is + * needed since no I/O path can access the tree. + * + * This ordering (mutex -> freeze) is safe because ublk_stop_dev_unlocked() + * already holds ub->mutex when calling del_gendisk() which freezes the queue. +*/ +static unsigned int ublk_lock_buf_tree(struct ublk_device *ub) +{ + unsigned int memflags = 0; + + mutex_lock(&ub->mutex); + if (ub->ub_disk) + memflags = blk_mq_freeze_queue(ub->ub_disk->queue); + + return memflags; +} + +static void ublk_unlock_buf_tree(struct ublk_device *ub, unsigned int memflags) +{ + if (ub->ub_disk) + blk_mq_unfreeze_queue(ub->ub_disk->queue, memflags); + mutex_unlock(&ub->mutex); +} + +/* Erase coalesced PFN ranges from the maple tree matching buf_index */ +static void ublk_buf_erase_ranges(struct ublk_device *ub, int buf_index) +{ + MA_STATE(mas, &ub->buf_tree, 0, ULONG_MAX); + struct ublk_buf_range *range; + + mas_lock(&mas); + mas_for_each(&mas, range, ULONG_MAX) { + if (range->buf_index == buf_index) { + mas_erase(&mas); + kfree(range); + } + } + mas_unlock(&mas); +} + +static int __ublk_ctrl_reg_buf(struct ublk_device *ub, + struct page **pages, unsigned long nr_pages, + int index, unsigned short flags) +{ + unsigned long i; + int ret; + + for (i = 0; i < nr_pages; i++) { + unsigned long pfn = page_to_pfn(pages[i]); + unsigned long start = i; + struct ublk_buf_range *range; + + /* Find run of consecutive PFNs */ + while (i + 1 < nr_pages && + page_to_pfn(pages[i + 1]) == pfn + (i - start) + 1) + i++; + + range = kzalloc_obj(*range); + if (!range) { + ret = -ENOMEM; + goto unwind; + } + range->buf_index = index; + range->flags = flags; + range->base_offset = start << PAGE_SHIFT; + + ret = mtree_insert_range(&ub->buf_tree, pfn, + pfn + (i - start), + range, GFP_KERNEL); + if (ret) { + kfree(range); + goto unwind; + } + } + return 0; + +unwind: + ublk_buf_erase_ranges(ub, index); + return ret; +} + +/* + * Register a shared memory buffer for zero-copy I/O. + * Pins pages, builds PFN maple tree, freezes/unfreezes the queue + * internally. Returns buffer index (>= 0) on success. + */ +static int ublk_ctrl_reg_buf(struct ublk_device *ub, + struct ublksrv_ctrl_cmd *header) +{ + void __user *argp = (void __user *)(unsigned long)header->addr; + struct ublk_shmem_buf_reg buf_reg; + unsigned long nr_pages; + struct page **pages = NULL; + unsigned int gup_flags; + unsigned int memflags; + long pinned; + int index; + int ret; + + if (!ublk_dev_support_shmem_zc(ub)) + return -EOPNOTSUPP; + + memset(&buf_reg, 0, sizeof(buf_reg)); + if (copy_from_user(&buf_reg, argp, + min_t(size_t, header->len, sizeof(buf_reg)))) + return -EFAULT; + + if (buf_reg.flags & ~UBLK_SHMEM_BUF_READ_ONLY) + return -EINVAL; + + if (buf_reg.reserved) + return -EINVAL; + + if (!buf_reg.len || buf_reg.len > UBLK_SHMEM_BUF_SIZE_MAX || + !PAGE_ALIGNED(buf_reg.len) || !PAGE_ALIGNED(buf_reg.addr)) + return -EINVAL; + + nr_pages = buf_reg.len >> PAGE_SHIFT; + + /* Pin pages before any locks (may sleep) */ + pages = kvmalloc_objs(*pages, nr_pages); + if (!pages) + return -ENOMEM; + + gup_flags = FOLL_LONGTERM; + if (!(buf_reg.flags & UBLK_SHMEM_BUF_READ_ONLY)) + gup_flags |= FOLL_WRITE; + + pinned = pin_user_pages_fast(buf_reg.addr, nr_pages, gup_flags, pages); + if (pinned < 0) { + ret = pinned; + goto err_free_pages; + } + if (pinned != nr_pages) { + ret = -EFAULT; + goto err_unpin; + } + + memflags = ublk_lock_buf_tree(ub); + + index = ida_alloc_max(&ub->buf_ida, USHRT_MAX, GFP_KERNEL); + if (index < 0) { + ret = index; + goto err_unlock; + } + + ret = __ublk_ctrl_reg_buf(ub, pages, nr_pages, index, buf_reg.flags); + if (ret) { + ida_free(&ub->buf_ida, index); + goto err_unlock; + } + + ublk_unlock_buf_tree(ub, memflags); + kvfree(pages); + return index; + +err_unlock: + ublk_unlock_buf_tree(ub, memflags); +err_unpin: + unpin_user_pages(pages, pinned); +err_free_pages: + kvfree(pages); + return ret; +} + +static void ublk_unpin_range_pages(unsigned long base_pfn, + unsigned long nr_pages) +{ +#define UBLK_UNPIN_BATCH 32 + struct page *pages[UBLK_UNPIN_BATCH]; + unsigned long off; + + for (off = 0; off < nr_pages; ) { + unsigned int batch = min_t(unsigned long, + nr_pages - off, UBLK_UNPIN_BATCH); + unsigned int j; + + for (j = 0; j < batch; j++) + pages[j] = pfn_to_page(base_pfn + off + j); + unpin_user_pages(pages, batch); + off += batch; + } +} + +/* + * Inner loop: erase up to UBLK_REMOVE_BATCH matching ranges under + * mas_lock, collecting the page ranges in a fixed-size array. Then + * drop the lock and unpin pages + free ranges outside spinlock context. + * + * Returns true if the tree walk completed, false if more ranges remain. + */ +#define UBLK_REMOVE_BATCH 64 + +struct ublk_unpin_range { + unsigned long base_pfn; + unsigned long nr_pages; +}; + +static bool __ublk_shmem_remove_ranges(struct ublk_device *ub, + int buf_index, int *ret) +{ + MA_STATE(mas, &ub->buf_tree, 0, ULONG_MAX); + struct ublk_buf_range *range; + struct ublk_unpin_range to_unpin[UBLK_REMOVE_BATCH]; + unsigned int count = 0; + unsigned int i; + bool done = false; + + mas_lock(&mas); + mas_for_each(&mas, range, ULONG_MAX) { + if (buf_index >= 0 && range->buf_index != buf_index) + continue; + + *ret = 0; + to_unpin[count].base_pfn = mas.index; + to_unpin[count].nr_pages = mas.last - mas.index + 1; + mas_erase(&mas); + kfree(range); + if (++count >= UBLK_REMOVE_BATCH) + goto unlock; + } + done = true; +unlock: + mas_unlock(&mas); + + for (i = 0; i < count; i++) + ublk_unpin_range_pages(to_unpin[i].base_pfn, + to_unpin[i].nr_pages); + + return done; +} + +/* + * Remove ranges from the maple tree matching buf_index, unpin pages + * and free range structs. If buf_index < 0, remove all ranges. + * Processes ranges in batches to avoid holding the maple tree spinlock + * across potentially expensive page unpinning. + */ +static int ublk_shmem_remove_ranges(struct ublk_device *ub, int buf_index) +{ + int ret = -ENOENT; + + while (!__ublk_shmem_remove_ranges(ub, buf_index, &ret)) + cond_resched(); + return ret; +} + +static int ublk_ctrl_unreg_buf(struct ublk_device *ub, + struct ublksrv_ctrl_cmd *header) +{ + int index = (int)header->data[0]; + unsigned int memflags; + int ret; + + if (!ublk_dev_support_shmem_zc(ub)) + return -EOPNOTSUPP; + + if (index < 0 || index > USHRT_MAX) + return -EINVAL; + + memflags = ublk_lock_buf_tree(ub); + + ret = ublk_shmem_remove_ranges(ub, index); + if (!ret) + ida_free(&ub->buf_ida, index); + + ublk_unlock_buf_tree(ub, memflags); + return ret; +} + +static void ublk_buf_cleanup(struct ublk_device *ub) +{ + ublk_shmem_remove_ranges(ub, -1); + mtree_destroy(&ub->buf_tree); + ida_destroy(&ub->buf_ida); +} + +/* Check if request pages match a registered shared memory buffer */ +static bool ublk_try_buf_match(struct ublk_device *ub, + struct request *rq, + u32 *buf_idx, u32 *buf_off) +{ + struct req_iterator iter; + struct bio_vec bv; + int index = -1; + unsigned long expected_offset = 0; + bool first = true; + + rq_for_each_bvec(bv, rq, iter) { + unsigned long pfn = page_to_pfn(bv.bv_page); + unsigned long end_pfn = pfn + + ((bv.bv_offset + bv.bv_len - 1) >> PAGE_SHIFT); + struct ublk_buf_range *range; + unsigned long off; + MA_STATE(mas, &ub->buf_tree, pfn, pfn); + + range = mas_walk(&mas); + if (!range) + return false; + + /* verify all pages in this bvec fall within the range */ + if (end_pfn > mas.last) + return false; + + off = range->base_offset + + (pfn - mas.index) * PAGE_SIZE + bv.bv_offset; + + if (first) { + /* Read-only buffer can't serve READ (kernel writes) */ + if ((range->flags & UBLK_SHMEM_BUF_READ_ONLY) && + req_op(rq) != REQ_OP_WRITE) + return false; + index = range->buf_index; + expected_offset = off; + *buf_off = off; + first = false; + } else { + if (range->buf_index != index) + return false; + if (off != expected_offset) + return false; + } + expected_offset += bv.bv_len; + } + + if (first) + return false; + + *buf_idx = index; + return true; +} + static int ublk_ctrl_uring_cmd_permission(struct ublk_device *ub, u32 cmd_op, struct ublksrv_ctrl_cmd *header) { @@ -5228,6 +5729,8 @@ static int ublk_ctrl_uring_cmd_permission(struct ublk_device *ub, case UBLK_CMD_UPDATE_SIZE: case UBLK_CMD_QUIESCE_DEV: case UBLK_CMD_TRY_STOP_DEV: + case UBLK_CMD_REG_BUF: + case UBLK_CMD_UNREG_BUF: mask = MAY_READ | MAY_WRITE; break; default: @@ -5352,6 +5855,12 @@ static int ublk_ctrl_uring_cmd(struct io_uring_cmd *cmd, case UBLK_CMD_TRY_STOP_DEV: ret = ublk_ctrl_try_stop_dev(ub); break; + case UBLK_CMD_REG_BUF: + ret = ublk_ctrl_reg_buf(ub, &header); + break; + case UBLK_CMD_UNREG_BUF: + ret = ublk_ctrl_unreg_buf(ub, &header); + break; default: ret = -EOPNOTSUPP; break; diff --git a/drivers/block/virtio_blk.c b/drivers/block/virtio_blk.c index b1c9a27fe00f..32bf3ba07a9d 100644 --- a/drivers/block/virtio_blk.c +++ b/drivers/block/virtio_blk.c @@ -689,6 +689,8 @@ static int virtblk_report_zones(struct gendisk *disk, sector_t sector, nz = min_t(u64, virtio64_to_cpu(vblk->vdev, report->nr_zones), nr_zones); + nz = min_t(u64, nz, + (buflen - sizeof(*report)) / sizeof(report->zones[0])); if (!nz) break; diff --git a/drivers/block/xen-blkfront.c b/drivers/block/xen-blkfront.c index f765970578f9..8dad7bf5f664 100644 --- a/drivers/block/xen-blkfront.c +++ b/drivers/block/xen-blkfront.c @@ -2080,6 +2080,15 @@ static int blkfront_resume(struct xenbus_device *dev) continue; /* + * For requests split across multiple slots, process the + * underlying request only once: skip the linked, sg-less + * secondary slot. + */ + if (shadow[j].associated_id != NO_ASSOCIATED_ID && + shadow[j].num_sg == 0) + continue; + + /* * Get the bios in the request so we can re-queue them. */ if (req_op(shadow[j].request) == REQ_OP_FLUSH || diff --git a/drivers/block/zloop.c b/drivers/block/zloop.c index 51c043342127..f0ca221524db 100644 --- a/drivers/block/zloop.c +++ b/drivers/block/zloop.c @@ -17,6 +17,7 @@ #include <linux/mutex.h> #include <linux/parser.h> #include <linux/seq_file.h> +#include <linux/xattr.h> /* * Options for adding (and removing) a device. @@ -34,6 +35,8 @@ enum { ZLOOP_OPT_BUFFERED_IO = (1 << 8), ZLOOP_OPT_ZONE_APPEND = (1 << 9), ZLOOP_OPT_ORDERED_ZONE_APPEND = (1 << 10), + ZLOOP_OPT_DISCARD_WRITE_CACHE = (1 << 11), + ZLOOP_OPT_MAX_OPEN_ZONES = (1 << 12), }; static const match_table_t zloop_opt_tokens = { @@ -48,6 +51,8 @@ static const match_table_t zloop_opt_tokens = { { ZLOOP_OPT_BUFFERED_IO, "buffered_io" }, { ZLOOP_OPT_ZONE_APPEND, "zone_append=%u" }, { ZLOOP_OPT_ORDERED_ZONE_APPEND, "ordered_zone_append" }, + { ZLOOP_OPT_DISCARD_WRITE_CACHE, "discard_write_cache" }, + { ZLOOP_OPT_MAX_OPEN_ZONES, "max_open_zones=%u" }, { ZLOOP_OPT_ERR, NULL } }; @@ -56,6 +61,7 @@ static const match_table_t zloop_opt_tokens = { #define ZLOOP_DEF_ZONE_SIZE ((256ULL * SZ_1M) >> SECTOR_SHIFT) #define ZLOOP_DEF_NR_ZONES 64 #define ZLOOP_DEF_NR_CONV_ZONES 8 +#define ZLOOP_DEF_MAX_OPEN_ZONES 0 #define ZLOOP_DEF_BASE_DIR "/var/local/zloop" #define ZLOOP_DEF_NR_QUEUES 1 #define ZLOOP_DEF_QUEUE_DEPTH 128 @@ -73,12 +79,14 @@ struct zloop_options { sector_t zone_size; sector_t zone_capacity; unsigned int nr_conv_zones; + unsigned int max_open_zones; char *base_dir; unsigned int nr_queues; unsigned int queue_depth; bool buffered_io; bool zone_append; bool ordered_zone_append; + bool discard_write_cache; }; /* @@ -95,7 +103,12 @@ enum zloop_zone_flags { ZLOOP_ZONE_SEQ_ERROR, }; +/* + * Zone descriptor. + * Locking order: z.lock -> z.wp_lock -> zlo.open_zones_lock + */ struct zloop_zone { + struct list_head open_zone_entry; struct file *file; unsigned long flags; @@ -119,6 +132,7 @@ struct zloop_device { bool buffered_io; bool zone_append; bool ordered_zone_append; + bool discard_write_cache; const char *base_dir; struct file *data_dir; @@ -128,7 +142,13 @@ struct zloop_device { sector_t zone_capacity; unsigned int nr_zones; unsigned int nr_conv_zones; + unsigned int max_open_zones; unsigned int block_size; + unsigned int dio_mem_align; + + spinlock_t open_zones_lock; + struct list_head open_zones_lru_list; + unsigned int nr_open_zones; struct zloop_zone zones[] __counted_by(nr_zones); }; @@ -153,12 +173,145 @@ static unsigned int rq_zone_no(struct request *rq) return blk_rq_pos(rq) >> zlo->zone_shift; } +/* + * Open an already open zone. This is mostly a no-op, except for the imp open -> + * exp open condition change that may happen. We also move a zone at the tail of + * the list of open zones so that if we need to + * implicitly close one open zone, we can do so in LRU order. + */ +static inline void zloop_lru_rotate_open_zone(struct zloop_device *zlo, + struct zloop_zone *zone) +{ + if (zlo->max_open_zones) { + spin_lock(&zlo->open_zones_lock); + list_move_tail(&zone->open_zone_entry, + &zlo->open_zones_lru_list); + spin_unlock(&zlo->open_zones_lock); + } +} + +static inline void zloop_lru_remove_open_zone(struct zloop_device *zlo, + struct zloop_zone *zone) +{ + if (zone->cond == BLK_ZONE_COND_IMP_OPEN || + zone->cond == BLK_ZONE_COND_EXP_OPEN) { + spin_lock(&zlo->open_zones_lock); + list_del_init(&zone->open_zone_entry); + zlo->nr_open_zones--; + spin_unlock(&zlo->open_zones_lock); + } +} + +static inline bool zloop_can_open_zone(struct zloop_device *zlo) +{ + return !zlo->max_open_zones || zlo->nr_open_zones < zlo->max_open_zones; +} + +/* + * If we have reached the maximum open zones limit, attempt to close an + * implicitly open zone (if we have any) so that we can implicitly open another + * zone without exceeding the maximum number of open zones. + */ +static bool zloop_close_imp_open_zone(struct zloop_device *zlo) +{ + struct zloop_zone *zone; + + lockdep_assert_held(&zlo->open_zones_lock); + + if (zloop_can_open_zone(zlo)) + return true; + + list_for_each_entry(zone, &zlo->open_zones_lru_list, open_zone_entry) { + if (zone->cond == BLK_ZONE_COND_IMP_OPEN) { + zone->cond = BLK_ZONE_COND_CLOSED; + list_del_init(&zone->open_zone_entry); + zlo->nr_open_zones--; + return true; + } + } + + return false; +} + +static bool zloop_open_closed_or_empty_zone(struct zloop_device *zlo, + struct zloop_zone *zone, + bool explicit) +{ + spin_lock(&zlo->open_zones_lock); + + if (explicit) { + /* + * Explicit open: we cannot allow this if we have reached the + * maximum open zones limit. + */ + if (!zloop_can_open_zone(zlo)) + goto fail; + zone->cond = BLK_ZONE_COND_EXP_OPEN; + } else { + /* + * Implicit open case: if we have reached the maximum open zones + * limit, try to close an implicitly open zone first. + */ + if (!zloop_close_imp_open_zone(zlo)) + goto fail; + zone->cond = BLK_ZONE_COND_IMP_OPEN; + } + + zlo->nr_open_zones++; + list_add_tail(&zone->open_zone_entry, + &zlo->open_zones_lru_list); + + spin_unlock(&zlo->open_zones_lock); + + return true; + +fail: + spin_unlock(&zlo->open_zones_lock); + + return false; +} + +static bool zloop_do_open_zone(struct zloop_device *zlo, + struct zloop_zone *zone, bool explicit) +{ + switch (zone->cond) { + case BLK_ZONE_COND_IMP_OPEN: + case BLK_ZONE_COND_EXP_OPEN: + if (explicit) + zone->cond = BLK_ZONE_COND_EXP_OPEN; + zloop_lru_rotate_open_zone(zlo, zone); + return true; + case BLK_ZONE_COND_EMPTY: + case BLK_ZONE_COND_CLOSED: + return zloop_open_closed_or_empty_zone(zlo, zone, explicit); + default: + return false; + } +} + +static void zloop_mark_full(struct zloop_device *zlo, struct zloop_zone *zone) +{ + lockdep_assert_held(&zone->wp_lock); + + zloop_lru_remove_open_zone(zlo, zone); + zone->cond = BLK_ZONE_COND_FULL; + zone->wp = ULLONG_MAX; +} + +static void zloop_mark_empty(struct zloop_device *zlo, struct zloop_zone *zone) +{ + lockdep_assert_held(&zone->wp_lock); + + zloop_lru_remove_open_zone(zlo, zone); + zone->cond = BLK_ZONE_COND_EMPTY; + zone->wp = zone->start; +} + static int zloop_update_seq_zone(struct zloop_device *zlo, unsigned int zone_no) { struct zloop_zone *zone = &zlo->zones[zone_no]; struct kstat stat; sector_t file_sectors; - unsigned long flags; int ret; lockdep_assert_held(&zone->lock); @@ -178,24 +331,24 @@ static int zloop_update_seq_zone(struct zloop_device *zlo, unsigned int zone_no) return -EINVAL; } - if (file_sectors & ((zlo->block_size >> SECTOR_SHIFT) - 1)) { - pr_err("Zone %u file size not aligned to block size %u\n", - zone_no, zlo->block_size); + if (!IS_ALIGNED(stat.size, zlo->block_size)) { + pr_err("Zone %u file size (%llu) not aligned to block size %u\n", + zone_no, stat.size, zlo->block_size); return -EINVAL; } - spin_lock_irqsave(&zone->wp_lock, flags); + spin_lock(&zone->wp_lock); if (!file_sectors) { - zone->cond = BLK_ZONE_COND_EMPTY; - zone->wp = zone->start; + zloop_mark_empty(zlo, zone); } else if (file_sectors == zlo->zone_capacity) { - zone->cond = BLK_ZONE_COND_FULL; - zone->wp = ULLONG_MAX; + zloop_mark_full(zlo, zone); } else { - zone->cond = BLK_ZONE_COND_CLOSED; + if (zone->cond != BLK_ZONE_COND_IMP_OPEN && + zone->cond != BLK_ZONE_COND_EXP_OPEN) + zone->cond = BLK_ZONE_COND_CLOSED; zone->wp = zone->start + file_sectors; } - spin_unlock_irqrestore(&zone->wp_lock, flags); + spin_unlock(&zone->wp_lock); return 0; } @@ -216,19 +369,8 @@ static int zloop_open_zone(struct zloop_device *zlo, unsigned int zone_no) goto unlock; } - switch (zone->cond) { - case BLK_ZONE_COND_EXP_OPEN: - break; - case BLK_ZONE_COND_EMPTY: - case BLK_ZONE_COND_CLOSED: - case BLK_ZONE_COND_IMP_OPEN: - zone->cond = BLK_ZONE_COND_EXP_OPEN; - break; - case BLK_ZONE_COND_FULL: - default: + if (!zloop_do_open_zone(zlo, zone, true)) ret = -EIO; - break; - } unlock: mutex_unlock(&zone->lock); @@ -239,7 +381,6 @@ unlock: static int zloop_close_zone(struct zloop_device *zlo, unsigned int zone_no) { struct zloop_zone *zone = &zlo->zones[zone_no]; - unsigned long flags; int ret = 0; if (test_bit(ZLOOP_ZONE_CONV, &zone->flags)) @@ -258,12 +399,13 @@ static int zloop_close_zone(struct zloop_device *zlo, unsigned int zone_no) break; case BLK_ZONE_COND_IMP_OPEN: case BLK_ZONE_COND_EXP_OPEN: - spin_lock_irqsave(&zone->wp_lock, flags); + spin_lock(&zone->wp_lock); + zloop_lru_remove_open_zone(zlo, zone); if (zone->wp == zone->start) zone->cond = BLK_ZONE_COND_EMPTY; else zone->cond = BLK_ZONE_COND_CLOSED; - spin_unlock_irqrestore(&zone->wp_lock, flags); + spin_unlock(&zone->wp_lock); break; case BLK_ZONE_COND_EMPTY: case BLK_ZONE_COND_FULL: @@ -281,7 +423,6 @@ unlock: static int zloop_reset_zone(struct zloop_device *zlo, unsigned int zone_no) { struct zloop_zone *zone = &zlo->zones[zone_no]; - unsigned long flags; int ret = 0; if (test_bit(ZLOOP_ZONE_CONV, &zone->flags)) @@ -299,11 +440,10 @@ static int zloop_reset_zone(struct zloop_device *zlo, unsigned int zone_no) goto unlock; } - spin_lock_irqsave(&zone->wp_lock, flags); - zone->cond = BLK_ZONE_COND_EMPTY; - zone->wp = zone->start; + spin_lock(&zone->wp_lock); + zloop_mark_empty(zlo, zone); clear_bit(ZLOOP_ZONE_SEQ_ERROR, &zone->flags); - spin_unlock_irqrestore(&zone->wp_lock, flags); + spin_unlock(&zone->wp_lock); unlock: mutex_unlock(&zone->lock); @@ -328,7 +468,6 @@ static int zloop_reset_all_zones(struct zloop_device *zlo) static int zloop_finish_zone(struct zloop_device *zlo, unsigned int zone_no) { struct zloop_zone *zone = &zlo->zones[zone_no]; - unsigned long flags; int ret = 0; if (test_bit(ZLOOP_ZONE_CONV, &zone->flags)) @@ -340,17 +479,17 @@ static int zloop_finish_zone(struct zloop_device *zlo, unsigned int zone_no) zone->cond == BLK_ZONE_COND_FULL) goto unlock; - if (vfs_truncate(&zone->file->f_path, zlo->zone_size << SECTOR_SHIFT)) { + if (vfs_truncate(&zone->file->f_path, + zlo->zone_capacity << SECTOR_SHIFT)) { set_bit(ZLOOP_ZONE_SEQ_ERROR, &zone->flags); ret = -EIO; goto unlock; } - spin_lock_irqsave(&zone->wp_lock, flags); - zone->cond = BLK_ZONE_COND_FULL; - zone->wp = ULLONG_MAX; + spin_lock(&zone->wp_lock); + zloop_mark_full(zlo, zone); clear_bit(ZLOOP_ZONE_SEQ_ERROR, &zone->flags); - spin_unlock_irqrestore(&zone->wp_lock, flags); + spin_unlock(&zone->wp_lock); unlock: mutex_unlock(&zone->lock); @@ -378,125 +517,22 @@ static void zloop_rw_complete(struct kiocb *iocb, long ret) zloop_put_cmd(cmd); } -static void zloop_rw(struct zloop_cmd *cmd) +static int zloop_do_rw(struct zloop_cmd *cmd) { struct request *rq = blk_mq_rq_from_pdu(cmd); + int rw = req_op(rq) == REQ_OP_READ ? ITER_DEST : ITER_SOURCE; + unsigned int nr_bvec = blk_rq_nr_bvec(rq); struct zloop_device *zlo = rq->q->queuedata; - unsigned int zone_no = rq_zone_no(rq); - sector_t sector = blk_rq_pos(rq); - sector_t nr_sectors = blk_rq_sectors(rq); - bool is_append = req_op(rq) == REQ_OP_ZONE_APPEND; - bool is_write = req_op(rq) == REQ_OP_WRITE || is_append; - int rw = is_write ? ITER_SOURCE : ITER_DEST; + struct zloop_zone *zone = &zlo->zones[rq_zone_no(rq)]; struct req_iterator rq_iter; - struct zloop_zone *zone; struct iov_iter iter; - struct bio_vec tmp; - unsigned long flags; - sector_t zone_end; - unsigned int nr_bvec; - int ret; - - atomic_set(&cmd->ref, 2); - cmd->sector = sector; - cmd->nr_sectors = nr_sectors; - cmd->ret = 0; - - if (WARN_ON_ONCE(is_append && !zlo->zone_append)) { - ret = -EIO; - goto out; - } - - /* We should never get an I/O beyond the device capacity. */ - if (WARN_ON_ONCE(zone_no >= zlo->nr_zones)) { - ret = -EIO; - goto out; - } - zone = &zlo->zones[zone_no]; - zone_end = zone->start + zlo->zone_capacity; - - /* - * The block layer should never send requests that are not fully - * contained within the zone. - */ - if (WARN_ON_ONCE(sector + nr_sectors > zone->start + zlo->zone_size)) { - ret = -EIO; - goto out; - } - - if (test_and_clear_bit(ZLOOP_ZONE_SEQ_ERROR, &zone->flags)) { - mutex_lock(&zone->lock); - ret = zloop_update_seq_zone(zlo, zone_no); - mutex_unlock(&zone->lock); - if (ret) - goto out; - } - - if (!test_bit(ZLOOP_ZONE_CONV, &zone->flags) && is_write) { - mutex_lock(&zone->lock); - - spin_lock_irqsave(&zone->wp_lock, flags); - - /* - * Zone append operations always go at the current write - * pointer, but regular write operations must already be - * aligned to the write pointer when submitted. - */ - if (is_append) { - /* - * If ordered zone append is in use, we already checked - * and set the target sector in zloop_queue_rq(). - */ - if (!zlo->ordered_zone_append) { - if (zone->cond == BLK_ZONE_COND_FULL || - zone->wp + nr_sectors > zone_end) { - spin_unlock_irqrestore(&zone->wp_lock, - flags); - ret = -EIO; - goto unlock; - } - sector = zone->wp; - } - cmd->sector = sector; - } else if (sector != zone->wp) { - spin_unlock_irqrestore(&zone->wp_lock, flags); - pr_err("Zone %u: unaligned write: sect %llu, wp %llu\n", - zone_no, sector, zone->wp); - ret = -EIO; - goto unlock; - } - - /* Implicitly open the target zone. */ - if (zone->cond == BLK_ZONE_COND_CLOSED || - zone->cond == BLK_ZONE_COND_EMPTY) - zone->cond = BLK_ZONE_COND_IMP_OPEN; - - /* - * Advance the write pointer, unless ordered zone append is in - * use. If the write fails, the write pointer position will be - * corrected when the next I/O starts execution. - */ - if (!is_append || !zlo->ordered_zone_append) { - zone->wp += nr_sectors; - if (zone->wp == zone_end) { - zone->cond = BLK_ZONE_COND_FULL; - zone->wp = ULLONG_MAX; - } - } - - spin_unlock_irqrestore(&zone->wp_lock, flags); - } - - nr_bvec = blk_rq_nr_bvec(rq); if (rq->bio != rq->biotail) { - struct bio_vec *bvec; + struct bio_vec tmp, *bvec; cmd->bvec = kmalloc_objs(*cmd->bvec, nr_bvec, GFP_NOIO); - if (!cmd->bvec) { - ret = -EIO; - goto unlock; - } + if (!cmd->bvec) + return -EIO; /* * The bios of the request may be started from the middle of @@ -519,10 +555,10 @@ static void zloop_rw(struct zloop_cmd *cmd) iov_iter_bvec(&iter, rw, __bvec_iter_bvec(rq->bio->bi_io_vec, rq->bio->bi_iter), nr_bvec, blk_rq_bytes(rq)); - iter.iov_offset = rq->bio->bi_iter.bi_bvec_done; + iter.iov_offset = rq->bio->bi_iter.bi_offset; } - cmd->iocb.ki_pos = (sector - zone->start) << SECTOR_SHIFT; + cmd->iocb.ki_pos = (cmd->sector - zone->start) << SECTOR_SHIFT; cmd->iocb.ki_filp = zone->file; cmd->iocb.ki_complete = zloop_rw_complete; if (!zlo->buffered_io) @@ -530,18 +566,162 @@ static void zloop_rw(struct zloop_cmd *cmd) cmd->iocb.ki_ioprio = IOPRIO_PRIO_VALUE(IOPRIO_CLASS_NONE, 0); if (rw == ITER_SOURCE) - ret = zone->file->f_op->write_iter(&cmd->iocb, &iter); - else - ret = zone->file->f_op->read_iter(&cmd->iocb, &iter); -unlock: - if (!test_bit(ZLOOP_ZONE_CONV, &zone->flags) && is_write) + return zone->file->f_op->write_iter(&cmd->iocb, &iter); + return zone->file->f_op->read_iter(&cmd->iocb, &iter); +} + +static int zloop_seq_write_prep(struct zloop_cmd *cmd) +{ + struct request *rq = blk_mq_rq_from_pdu(cmd); + struct zloop_device *zlo = rq->q->queuedata; + unsigned int zone_no = rq_zone_no(rq); + sector_t nr_sectors = blk_rq_sectors(rq); + bool is_append = req_op(rq) == REQ_OP_ZONE_APPEND; + struct zloop_zone *zone = &zlo->zones[zone_no]; + sector_t zone_end = zone->start + zlo->zone_capacity; + int ret = 0; + + spin_lock(&zone->wp_lock); + + /* + * Zone append operations always go at the current write pointer, but + * regular write operations must already be aligned to the write pointer + * when submitted. + */ + if (is_append) { + /* + * If ordered zone append is in use, we already checked and set + * the target sector in zloop_queue_rq(). + */ + if (!zlo->ordered_zone_append) { + if (zone->cond == BLK_ZONE_COND_FULL || + zone->wp + nr_sectors > zone_end) { + ret = -EIO; + goto out_unlock; + } + cmd->sector = zone->wp; + } + } else { + if (cmd->sector != zone->wp) { + pr_err("Zone %u: unaligned write: sect %llu, wp %llu\n", + zone_no, cmd->sector, zone->wp); + ret = -EIO; + goto out_unlock; + } + } + + /* Implicitly open the target zone. */ + if (!zloop_do_open_zone(zlo, zone, false)) { + ret = -EIO; + goto out_unlock; + } + + /* + * Advance the write pointer, unless ordered zone append is in use. If + * the write fails, the write pointer position will be corrected when + * the next I/O starts execution. + */ + if (!is_append || !zlo->ordered_zone_append) { + zone->wp += nr_sectors; + if (zone->wp == zone_end) + zloop_mark_full(zlo, zone); + } +out_unlock: + spin_unlock(&zone->wp_lock); + return ret; +} + +static void zloop_rw(struct zloop_cmd *cmd) +{ + struct request *rq = blk_mq_rq_from_pdu(cmd); + struct zloop_device *zlo = rq->q->queuedata; + unsigned int zone_no = rq_zone_no(rq); + sector_t nr_sectors = blk_rq_sectors(rq); + bool is_append = req_op(rq) == REQ_OP_ZONE_APPEND; + bool is_write = req_op(rq) == REQ_OP_WRITE || is_append; + struct zloop_zone *zone; + int ret = -EIO; + + atomic_set(&cmd->ref, 2); + cmd->sector = blk_rq_pos(rq); + cmd->nr_sectors = nr_sectors; + cmd->ret = 0; + + if (WARN_ON_ONCE(is_append && !zlo->zone_append)) + goto out; + + /* We should never get an I/O beyond the device capacity. */ + if (WARN_ON_ONCE(zone_no >= zlo->nr_zones)) + goto out; + + zone = &zlo->zones[zone_no]; + + /* + * The block layer should never send requests that are not fully + * contained within the zone. + */ + if (WARN_ON_ONCE(cmd->sector + nr_sectors > + zone->start + zlo->zone_size)) + goto out; + + if (test_and_clear_bit(ZLOOP_ZONE_SEQ_ERROR, &zone->flags)) { + mutex_lock(&zone->lock); + ret = zloop_update_seq_zone(zlo, zone_no); mutex_unlock(&zone->lock); + if (ret) + goto out; + } + + if (!test_bit(ZLOOP_ZONE_CONV, &zone->flags) && is_write) { + mutex_lock(&zone->lock); + ret = zloop_seq_write_prep(cmd); + if (!ret) + ret = zloop_do_rw(cmd); + mutex_unlock(&zone->lock); + } else { + ret = zloop_do_rw(cmd); + } out: if (ret != -EIOCBQUEUED) zloop_rw_complete(&cmd->iocb, ret); zloop_put_cmd(cmd); } +static inline bool zloop_zone_is_active(struct zloop_zone *zone) +{ + switch (zone->cond) { + case BLK_ZONE_COND_EXP_OPEN: + case BLK_ZONE_COND_IMP_OPEN: + case BLK_ZONE_COND_CLOSED: + return true; + default: + return false; + } +} + +static int zloop_record_safe_wps(struct zloop_device *zlo) +{ + unsigned int i; + int ret; + + for (i = 0; i < zlo->nr_zones; i++) { + struct zloop_zone *zone = &zlo->zones[i]; + struct file *file = zone->file; + + if (!zloop_zone_is_active(zone)) + continue; + ret = vfs_setxattr(file_mnt_idmap(file), file_dentry(file), + "user.zloop.wp", &zone->wp, sizeof(zone->wp), 0); + if (ret) { + pr_err("%pg: failed to record write pointer (%d)\n", + zlo->disk->part0, ret); + return ret; + } + } + + return 0; +} + /* * Sync the entire FS containing the zone files instead of walking all files. */ @@ -550,6 +730,12 @@ static int zloop_flush(struct zloop_device *zlo) struct super_block *sb = file_inode(zlo->data_dir)->i_sb; int ret; + if (zlo->discard_write_cache) { + ret = zloop_record_safe_wps(zlo); + if (ret) + return ret; + } + down_read(&sb->s_umount); ret = sync_filesystem(sb); up_read(&sb->s_umount); @@ -679,24 +865,21 @@ static bool zloop_set_zone_append_sector(struct request *rq) struct zloop_zone *zone = &zlo->zones[zone_no]; sector_t zone_end = zone->start + zlo->zone_capacity; sector_t nr_sectors = blk_rq_sectors(rq); - unsigned long flags; - spin_lock_irqsave(&zone->wp_lock, flags); + spin_lock(&zone->wp_lock); if (zone->cond == BLK_ZONE_COND_FULL || zone->wp + nr_sectors > zone_end) { - spin_unlock_irqrestore(&zone->wp_lock, flags); + spin_unlock(&zone->wp_lock); return false; } rq->__sector = zone->wp; zone->wp += blk_rq_sectors(rq); - if (zone->wp >= zone_end) { - zone->cond = BLK_ZONE_COND_FULL; - zone->wp = ULLONG_MAX; - } + if (zone->wp >= zone_end) + zloop_mark_full(zlo, zone); - spin_unlock_irqrestore(&zone->wp_lock, flags); + spin_unlock(&zone->wp_lock); return true; } @@ -708,8 +891,10 @@ static blk_status_t zloop_queue_rq(struct blk_mq_hw_ctx *hctx, struct zloop_cmd *cmd = blk_mq_rq_to_pdu(rq); struct zloop_device *zlo = rq->q->queuedata; - if (data_race(READ_ONCE(zlo->state)) == Zlo_deleting) + if (data_race(READ_ONCE(zlo->state)) == Zlo_deleting) { + rq->rq_flags |= RQF_QUIET; return BLK_STS_IOERR; + } /* * If we need to strongly order zone append operations, set the request @@ -755,7 +940,6 @@ static int zloop_report_zones(struct gendisk *disk, sector_t sector, struct zloop_device *zlo = disk->private_data; struct blk_zone blkz = {}; unsigned int first, i; - unsigned long flags; int ret; first = disk_zone_no(disk, sector); @@ -779,9 +963,9 @@ static int zloop_report_zones(struct gendisk *disk, sector_t sector, blkz.start = zone->start; blkz.len = zlo->zone_size; - spin_lock_irqsave(&zone->wp_lock, flags); + spin_lock(&zone->wp_lock); blkz.wp = zone->wp; - spin_unlock_irqrestore(&zone->wp_lock, flags); + spin_unlock(&zone->wp_lock); blkz.cond = zone->cond; if (test_bit(ZLOOP_ZONE_CONV, &zone->flags)) { blkz.type = BLK_ZONE_TYPE_CONVENTIONAL; @@ -855,20 +1039,32 @@ static int zloop_get_block_size(struct zloop_device *zlo, struct kstat st; /* - * If the FS block size is lower than or equal to 4K, use that as the - * device block size. Otherwise, fallback to the FS direct IO alignment - * constraint if that is provided, and to the FS underlying device - * physical block size if the direct IO alignment is unknown. + * Use the dio alignment of the file system if provided. The incoming + * request's bio_vec is forwarded to the backing file unchanged, so its + * required memory alignment becomes the device's dma_alignment when + * used for direct-io. The file system reports zeroed alignments if the + * file can't be used for direct-io at all, so fall back to the block + * device limits in that case. */ - if (file_inode(zone->file)->i_sb->s_blocksize <= SZ_4K) - zlo->block_size = file_inode(zone->file)->i_sb->s_blocksize; - else if (!vfs_getattr(&zone->file->f_path, &st, STATX_DIOALIGN, 0) && - (st.result_mask & STATX_DIOALIGN)) + if (!vfs_getattr(&zone->file->f_path, &st, STATX_DIOALIGN, 0) && + (st.result_mask & STATX_DIOALIGN) && st.dio_mem_align) { zlo->block_size = st.dio_offset_align; - else if (sb_bdev) + zlo->dio_mem_align = min(st.dio_mem_align - 1, PAGE_SIZE - 1); + } else if (sb_bdev) { zlo->block_size = bdev_physical_block_size(sb_bdev); - else + zlo->dio_mem_align = bdev_dma_alignment(sb_bdev); + } else { zlo->block_size = SECTOR_SIZE; + zlo->dio_mem_align = SECTOR_SIZE - 1; + } + + /* + * Prefer the FS block size for the device block size when it is no + * larger than 4K; otherwise keep the direct I/O / physical block size + * selected above. + */ + if (file_inode(zone->file)->i_sb->s_blocksize <= SZ_4K) + zlo->block_size = file_inode(zone->file)->i_sb->s_blocksize; if (zlo->zone_capacity & ((zlo->block_size >> SECTOR_SHIFT) - 1)) { pr_err("Zone capacity is not aligned to block size %u\n", @@ -889,6 +1085,7 @@ static int zloop_init_zone(struct zloop_device *zlo, struct zloop_options *opts, int ret; mutex_init(&zone->lock); + INIT_LIST_HEAD(&zone->open_zone_entry); spin_lock_init(&zone->wp_lock); zone->start = (sector_t)zone_no << zlo->zone_shift; @@ -1009,12 +1206,20 @@ static int zloop_ctl_add(struct zloop_options *opts) goto out; } + if (opts->max_open_zones > nr_zones - opts->nr_conv_zones) { + pr_err("Invalid maximum number of open zones %u\n", + opts->max_open_zones); + goto out; + } + zlo = kvzalloc_flex(*zlo, zones, nr_zones); if (!zlo) { ret = -ENOMEM; goto out; } WRITE_ONCE(zlo->state, Zlo_creating); + spin_lock_init(&zlo->open_zones_lock); + INIT_LIST_HEAD(&zlo->open_zones_lru_list); ret = mutex_lock_killable(&zloop_ctl_mutex); if (ret) @@ -1042,10 +1247,12 @@ static int zloop_ctl_add(struct zloop_options *opts) zlo->zone_capacity = zlo->zone_size; zlo->nr_zones = nr_zones; zlo->nr_conv_zones = opts->nr_conv_zones; + zlo->max_open_zones = opts->max_open_zones; zlo->buffered_io = opts->buffered_io; zlo->zone_append = opts->zone_append; if (zlo->zone_append) zlo->ordered_zone_append = opts->ordered_zone_append; + zlo->discard_write_cache = opts->discard_write_cache; zlo->workqueue = alloc_workqueue("zloop%d", WQ_UNBOUND | WQ_FREEZABLE, opts->nr_queues * opts->queue_depth, zlo->id); @@ -1086,8 +1293,13 @@ static int zloop_ctl_add(struct zloop_options *opts) lim.physical_block_size = zlo->block_size; lim.logical_block_size = zlo->block_size; + /* Direct I/O forwards the request pages to the backing files as-is. */ + if (!opts->buffered_io) + lim.dma_alignment = max_t(unsigned int, zlo->dio_mem_align, + SECTOR_SIZE - 1); if (zlo->zone_append) lim.max_hw_zone_append_sectors = lim.max_hw_sectors; + lim.max_open_zones = zlo->max_open_zones; zlo->tag_set.ops = &zloop_mq_ops; zlo->tag_set.nr_hw_queues = opts->nr_queues; @@ -1168,6 +1380,45 @@ out: return ret; } +static void zloop_forget_cache(struct zloop_device *zlo) +{ + unsigned int i; + int ret; + + pr_info("%pg: discarding volatile write cache\n", zlo->disk->part0); + + for (i = 0; i < zlo->nr_zones; i++) { + struct zloop_zone *zone = &zlo->zones[i]; + struct file *file = zone->file; + sector_t old_wp; + + if (!zloop_zone_is_active(zone)) + continue; + + ret = vfs_getxattr(file_mnt_idmap(file), file_dentry(file), + "user.zloop.wp", &old_wp, sizeof(old_wp)); + if (ret == -ENODATA) { + old_wp = 0; + } else if (ret != sizeof(old_wp)) { + pr_err("%pg: failed to retrieve write pointer (%d)\n", + zlo->disk->part0, ret); + continue; + } + + if (old_wp > zone->wp) + continue; + /* + * This should not happen, if we recored a full zone, it can't + * be active. + */ + if (WARN_ON_ONCE(old_wp == ULLONG_MAX)) + continue; + + vfs_truncate(&file->f_path, + (old_wp - zone->start) << SECTOR_SHIFT); + } +} + static int zloop_ctl_remove(struct zloop_options *opts) { struct zloop_device *zlo; @@ -1202,6 +1453,10 @@ static int zloop_ctl_remove(struct zloop_options *opts) return ret; del_gendisk(zlo->disk); + + if (zlo->discard_write_cache) + zloop_forget_cache(zlo); + put_disk(zlo->disk); pr_info("Removed device %d\n", opts->id); @@ -1224,6 +1479,7 @@ static int zloop_parse_options(struct zloop_options *opts, const char *buf) opts->capacity = ZLOOP_DEF_ZONE_SIZE * ZLOOP_DEF_NR_ZONES; opts->zone_size = ZLOOP_DEF_ZONE_SIZE; opts->nr_conv_zones = ZLOOP_DEF_NR_CONV_ZONES; + opts->max_open_zones = ZLOOP_DEF_MAX_OPEN_ZONES; opts->nr_queues = ZLOOP_DEF_NR_QUEUES; opts->queue_depth = ZLOOP_DEF_QUEUE_DEPTH; opts->buffered_io = ZLOOP_DEF_BUFFERED_IO; @@ -1302,6 +1558,13 @@ static int zloop_parse_options(struct zloop_options *opts, const char *buf) } opts->nr_conv_zones = token; break; + case ZLOOP_OPT_MAX_OPEN_ZONES: + if (match_uint(args, &token)) { + ret = -EINVAL; + goto out; + } + opts->max_open_zones = token; + break; case ZLOOP_OPT_BASE_DIR: p = match_strdup(args); if (!p) { @@ -1353,6 +1616,9 @@ static int zloop_parse_options(struct zloop_options *opts, const char *buf) case ZLOOP_OPT_ORDERED_ZONE_APPEND: opts->ordered_zone_append = true; break; + case ZLOOP_OPT_DISCARD_WRITE_CACHE: + opts->discard_write_cache = true; + break; case ZLOOP_OPT_ERR: default: pr_warn("unknown parameter or missing value '%s'\n", p); diff --git a/drivers/block/zram/backend_842.c b/drivers/block/zram/backend_842.c index 10d9d5c60f53..3846a04c69d7 100644 --- a/drivers/block/zram/backend_842.c +++ b/drivers/block/zram/backend_842.c @@ -1,5 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-or-later +#define pr_fmt(fmt) "842: " fmt + #include <linux/kernel.h> #include <linux/slab.h> #include <linux/sw842.h> @@ -13,6 +15,14 @@ static void release_params_842(struct zcomp_params *params) static int setup_params_842(struct zcomp_params *params) { + if (params->dict_sz) { + pr_err("dictionary is not supported\n"); + return -EOPNOTSUPP; + } + if (params->level != ZCOMP_PARAM_NOT_SET) { + pr_err("compression level is not supported\n"); + return -EOPNOTSUPP; + } return 0; } diff --git a/drivers/block/zram/backend_deflate.c b/drivers/block/zram/backend_deflate.c index f92a52a720d1..f71b11bcac78 100644 --- a/drivers/block/zram/backend_deflate.c +++ b/drivers/block/zram/backend_deflate.c @@ -1,5 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-or-later +#define pr_fmt(fmt) "deflate: " fmt + #include <linux/kernel.h> #include <linux/slab.h> #include <linux/vmalloc.h> @@ -22,10 +24,29 @@ static void deflate_release_params(struct zcomp_params *params) static int deflate_setup_params(struct zcomp_params *params) { - if (params->level == ZCOMP_PARAM_NOT_SET) + if (params->dict_sz) { + pr_err("dictionary is not supported\n"); + return -EOPNOTSUPP; + } + + if (params->level == ZCOMP_PARAM_NOT_SET) { params->level = Z_DEFAULT_COMPRESSION; - if (params->deflate.winbits == ZCOMP_PARAM_NOT_SET) + } else if (params->level < Z_DEFAULT_COMPRESSION || + params->level > Z_BEST_COMPRESSION) { + pr_err("invalid compression level %d\n", params->level); + return -EINVAL; + } + + if (params->deflate.winbits == ZCOMP_PARAM_NOT_SET) { params->deflate.winbits = DEFLATE_DEF_WINBITS; + } else { + s32 wb = params->deflate.winbits; + + if ((wb < -15 || wb > -9) && (wb < 9 || wb > 15)) { + pr_err("invalid winbits %d\n", wb); + return -EINVAL; + } + } return 0; } diff --git a/drivers/block/zram/backend_lz4.c b/drivers/block/zram/backend_lz4.c index 04e186614760..1e4ad31d39a6 100644 --- a/drivers/block/zram/backend_lz4.c +++ b/drivers/block/zram/backend_lz4.c @@ -1,3 +1,7 @@ +// SPDX-License-Identifier: GPL-2.0-or-later + +#define pr_fmt(fmt) "lz4: " fmt + #include <linux/kernel.h> #include <linux/lz4.h> #include <linux/slab.h> @@ -14,12 +18,41 @@ struct lz4_ctx { static void lz4_release_params(struct zcomp_params *params) { + LZ4_stream_t *dict_stream = params->drv_data; + + params->drv_data = NULL; + if (!dict_stream) + return; + + kfree(dict_stream); } static int lz4_setup_params(struct zcomp_params *params) { - if (params->level == ZCOMP_PARAM_NOT_SET) + LZ4_stream_t *dict_stream; + int ret; + + if (params->level == ZCOMP_PARAM_NOT_SET) { params->level = LZ4_ACCELERATION_DEFAULT; + } else if (params->level < LZ4_ACCELERATION_DEFAULT) { + pr_err("invalid compression level %d\n", params->level); + return -EINVAL; + } + + if (!params->dict || !params->dict_sz) + return 0; + + dict_stream = kzalloc_obj(*dict_stream); + if (!dict_stream) + return -ENOMEM; + + ret = LZ4_loadDict(dict_stream, + params->dict, params->dict_sz); + if (ret != params->dict_sz) { + kfree(dict_stream); + return -EINVAL; + } + params->drv_data = dict_stream; return 0; } @@ -79,9 +112,7 @@ static int lz4_compress(struct zcomp_params *params, struct zcomp_ctx *ctx, zctx->mem); } else { /* Cstrm needs to be reset */ - ret = LZ4_loadDict(zctx->cstrm, params->dict, params->dict_sz); - if (ret != params->dict_sz) - return -EINVAL; + memcpy(zctx->cstrm, params->drv_data, sizeof(*zctx->cstrm)); ret = LZ4_compress_fast_continue(zctx->cstrm, req->src, req->dst, req->src_len, req->dst_len, params->level); diff --git a/drivers/block/zram/backend_lz4hc.c b/drivers/block/zram/backend_lz4hc.c index f6a336acfe20..d8aa01bb258f 100644 --- a/drivers/block/zram/backend_lz4hc.c +++ b/drivers/block/zram/backend_lz4hc.c @@ -1,3 +1,7 @@ +// SPDX-License-Identifier: GPL-2.0-or-later + +#define pr_fmt(fmt) "lz4hc: " fmt + #include <linux/kernel.h> #include <linux/lz4.h> #include <linux/slab.h> @@ -18,8 +22,18 @@ static void lz4hc_release_params(struct zcomp_params *params) static int lz4hc_setup_params(struct zcomp_params *params) { - if (params->level == ZCOMP_PARAM_NOT_SET) + if (params->level == ZCOMP_PARAM_NOT_SET) { params->level = LZ4HC_DEFAULT_CLEVEL; + } else if (params->level < 1 || params->level > LZ4HC_MAX_CLEVEL) { + /* + * Use < 1 rather than < LZ4HC_MIN_CLEVEL here because + * LZ4HC_compress_generic() only clamps levels below 1 + * (levels 1 and 2 are valid). LZ4HC_MIN_CLEVEL (3) is + * advisory and not enforced by the library. + */ + pr_err("invalid compression level %d\n", params->level); + return -EINVAL; + } return 0; } diff --git a/drivers/block/zram/backend_lzo.c b/drivers/block/zram/backend_lzo.c index 4c906beaae6b..d83f92cf757c 100644 --- a/drivers/block/zram/backend_lzo.c +++ b/drivers/block/zram/backend_lzo.c @@ -1,5 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-or-later +#define pr_fmt(fmt) "lzo: " fmt + #include <linux/kernel.h> #include <linux/slab.h> #include <linux/lzo.h> @@ -12,6 +14,14 @@ static void lzo_release_params(struct zcomp_params *params) static int lzo_setup_params(struct zcomp_params *params) { + if (params->dict_sz) { + pr_err("dictionary is not supported\n"); + return -EOPNOTSUPP; + } + if (params->level != ZCOMP_PARAM_NOT_SET) { + pr_err("compression level is not supported\n"); + return -EOPNOTSUPP; + } return 0; } diff --git a/drivers/block/zram/backend_lzorle.c b/drivers/block/zram/backend_lzorle.c index 10640c96cbfc..1b120d062c92 100644 --- a/drivers/block/zram/backend_lzorle.c +++ b/drivers/block/zram/backend_lzorle.c @@ -1,5 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-or-later +#define pr_fmt(fmt) "lzo-rle: " fmt + #include <linux/kernel.h> #include <linux/slab.h> #include <linux/lzo.h> @@ -12,6 +14,14 @@ static void lzorle_release_params(struct zcomp_params *params) static int lzorle_setup_params(struct zcomp_params *params) { + if (params->dict_sz) { + pr_err("dictionary is not supported\n"); + return -EOPNOTSUPP; + } + if (params->level != ZCOMP_PARAM_NOT_SET) { + pr_err("compression level is not supported\n"); + return -EOPNOTSUPP; + } return 0; } diff --git a/drivers/block/zram/backend_zstd.c b/drivers/block/zram/backend_zstd.c index d00b548056dc..08da3810cffd 100644 --- a/drivers/block/zram/backend_zstd.c +++ b/drivers/block/zram/backend_zstd.c @@ -1,5 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-or-later +#define pr_fmt(fmt) "zstd: " fmt + #include <linux/kernel.h> #include <linux/slab.h> #include <linux/vmalloc.h> @@ -58,8 +60,13 @@ static int zstd_setup_params(struct zcomp_params *params) return -ENOMEM; params->drv_data = zp; - if (params->level == ZCOMP_PARAM_NOT_SET) + if (params->level == ZCOMP_PARAM_NOT_SET) { params->level = zstd_default_clevel(); + } else if (params->level < zstd_min_clevel() || + params->level > zstd_max_clevel()) { + pr_err("invalid compression level %d\n", params->level); + goto error; + } zp->cprm = zstd_get_params(params->level, PAGE_SIZE); @@ -85,7 +92,6 @@ static int zstd_setup_params(struct zcomp_params *params) return 0; error: - zstd_release_params(params); return -EINVAL; } @@ -161,7 +167,6 @@ static int zstd_create(struct zcomp_params *params, struct zcomp_ctx *ctx) return 0; error: - zstd_release_params(params); zstd_destroy(ctx); return -EINVAL; } diff --git a/drivers/block/zram/zcomp.c b/drivers/block/zram/zcomp.c index a771a8ecc540..974c4691887e 100644 --- a/drivers/block/zram/zcomp.c +++ b/drivers/block/zram/zcomp.c @@ -84,9 +84,14 @@ static const struct zcomp_ops *lookup_backend_ops(const char *comp) return backends[i]; } -bool zcomp_available_algorithm(const char *comp) +const char *zcomp_lookup_backend_name(const char *comp) { - return lookup_backend_ops(comp) != NULL; + const struct zcomp_ops *backend = lookup_backend_ops(comp); + + if (backend) + return backend->name; + + return NULL; } /* show available compressors */ diff --git a/drivers/block/zram/zcomp.h b/drivers/block/zram/zcomp.h index eacfd3f7d61d..81a0f3f6ff48 100644 --- a/drivers/block/zram/zcomp.h +++ b/drivers/block/zram/zcomp.h @@ -80,7 +80,7 @@ struct zcomp { int zcomp_cpu_up_prepare(unsigned int cpu, struct hlist_node *node); int zcomp_cpu_dead(unsigned int cpu, struct hlist_node *node); ssize_t zcomp_available_show(const char *comp, char *buf, ssize_t at); -bool zcomp_available_algorithm(const char *comp); +const char *zcomp_lookup_backend_name(const char *comp); struct zcomp *zcomp_create(const char *alg, struct zcomp_params *params); void zcomp_destroy(struct zcomp *comp); diff --git a/drivers/block/zram/zram_drv.c b/drivers/block/zram/zram_drv.c index af679375b193..a9b3bb1d3bef 100644 --- a/drivers/block/zram/zram_drv.c +++ b/drivers/block/zram/zram_drv.c @@ -33,6 +33,7 @@ #include <linux/cpuhotplug.h> #include <linux/part_stat.h> #include <linux/kernel_read_file.h> +#include <linux/rcupdate.h> #include "zram_drv.h" @@ -55,16 +56,7 @@ static size_t huge_class_size; static const struct block_device_operations zram_devops; -static void slot_free(struct zram *zram, u32 index); -#define slot_dep_map(zram, index) (&(zram)->table[(index)].dep_map) - -static void slot_lock_init(struct zram *zram, u32 index) -{ - static struct lock_class_key __key; - - lockdep_init_map(slot_dep_map(zram, index), "zram->table[index].lock", - &__key, 0); -} +static void slot_free(struct zram *zram, unsigned long index); /* * entry locking rules: @@ -78,34 +70,34 @@ static void slot_lock_init(struct zram *zram, u32 index) * 4) Use TRY lock variant when in atomic context * - must check return value and handle locking failers */ -static __must_check bool slot_trylock(struct zram *zram, u32 index) +static __must_check bool slot_trylock(struct zram *zram, unsigned long index) { unsigned long *lock = &zram->table[index].__lock; - if (!test_and_set_bit_lock(ZRAM_ENTRY_LOCK, lock)) { - mutex_acquire(slot_dep_map(zram, index), 0, 1, _RET_IP_); - lock_acquired(slot_dep_map(zram, index), _RET_IP_); + if (!test_and_set_bit_lock(ZRAM_ENTRY_LOCK_BIT, lock)) { + mutex_acquire(&zram->table_lock_map, 0, 1, _RET_IP_); + lock_acquired(&zram->table_lock_map, _RET_IP_); return true; } return false; } -static void slot_lock(struct zram *zram, u32 index) +static void slot_lock(struct zram *zram, unsigned long index) { unsigned long *lock = &zram->table[index].__lock; - mutex_acquire(slot_dep_map(zram, index), 0, 0, _RET_IP_); - wait_on_bit_lock(lock, ZRAM_ENTRY_LOCK, TASK_UNINTERRUPTIBLE); - lock_acquired(slot_dep_map(zram, index), _RET_IP_); + mutex_acquire(&zram->table_lock_map, 0, 0, _RET_IP_); + wait_on_bit_lock(lock, ZRAM_ENTRY_LOCK_BIT, TASK_UNINTERRUPTIBLE); + lock_acquired(&zram->table_lock_map, _RET_IP_); } -static void slot_unlock(struct zram *zram, u32 index) +static void slot_unlock(struct zram *zram, unsigned long index) { unsigned long *lock = &zram->table[index].__lock; - mutex_release(slot_dep_map(zram, index), _RET_IP_); - clear_and_wake_up_bit(ZRAM_ENTRY_LOCK, lock); + mutex_release(&zram->table_lock_map, _RET_IP_); + clear_and_wake_up_bit(ZRAM_ENTRY_LOCK_BIT, lock); } static inline bool init_done(struct zram *zram) @@ -118,55 +110,56 @@ static inline struct zram *dev_to_zram(struct device *dev) return (struct zram *)dev_to_disk(dev)->private_data; } -static unsigned long get_slot_handle(struct zram *zram, u32 index) +static unsigned long get_slot_handle(struct zram *zram, unsigned long index) { return zram->table[index].handle; } -static void set_slot_handle(struct zram *zram, u32 index, unsigned long handle) +static void set_slot_handle(struct zram *zram, unsigned long index, + unsigned long handle) { zram->table[index].handle = handle; } -static bool test_slot_flag(struct zram *zram, u32 index, +static bool test_slot_flag(struct zram *zram, unsigned long index, enum zram_pageflags flag) { return zram->table[index].attr.flags & BIT(flag); } -static void set_slot_flag(struct zram *zram, u32 index, +static void set_slot_flag(struct zram *zram, unsigned long index, enum zram_pageflags flag) { zram->table[index].attr.flags |= BIT(flag); } -static void clear_slot_flag(struct zram *zram, u32 index, +static void clear_slot_flag(struct zram *zram, unsigned long index, enum zram_pageflags flag) { zram->table[index].attr.flags &= ~BIT(flag); } -static size_t get_slot_size(struct zram *zram, u32 index) +static size_t get_slot_size(struct zram *zram, unsigned long index) { return zram->table[index].attr.flags & (BIT(ZRAM_FLAG_SHIFT) - 1); } -static void set_slot_size(struct zram *zram, u32 index, size_t size) +static void set_slot_size(struct zram *zram, unsigned long index, size_t size) { unsigned long flags = zram->table[index].attr.flags >> ZRAM_FLAG_SHIFT; zram->table[index].attr.flags = (flags << ZRAM_FLAG_SHIFT) | size; } -static inline bool slot_allocated(struct zram *zram, u32 index) +static inline bool slot_allocated(struct zram *zram, unsigned long index) { return get_slot_size(zram, index) || test_slot_flag(zram, index, ZRAM_SAME) || test_slot_flag(zram, index, ZRAM_WB); } -static inline void set_slot_comp_priority(struct zram *zram, u32 index, - u32 prio) +static inline void set_slot_comp_priority(struct zram *zram, + unsigned long index, u32 prio) { prio &= ZRAM_COMP_PRIORITY_MASK; /* @@ -178,14 +171,14 @@ static inline void set_slot_comp_priority(struct zram *zram, u32 index, zram->table[index].attr.flags |= (prio << ZRAM_COMP_PRIORITY_BIT1); } -static inline u32 get_slot_comp_priority(struct zram *zram, u32 index) +static inline u32 get_slot_comp_priority(struct zram *zram, unsigned long index) { u32 prio = zram->table[index].attr.flags >> ZRAM_COMP_PRIORITY_BIT1; return prio & ZRAM_COMP_PRIORITY_MASK; } -static void mark_slot_accessed(struct zram *zram, u32 index) +static void mark_slot_accessed(struct zram *zram, unsigned long index) { clear_slot_flag(zram, index, ZRAM_IDLE); clear_slot_flag(zram, index, ZRAM_PP_SLOT); @@ -292,7 +285,7 @@ static void release_pp_ctl(struct zram *zram, struct zram_pp_ctl *ctl) } static bool place_pp_slot(struct zram *zram, struct zram_pp_ctl *ctl, - u32 index) + unsigned long index) { struct zram_pp_slot *pps; u32 bid; @@ -426,7 +419,7 @@ static void mark_idle(struct zram *zram, ktime_t cutoff) { int is_idle = 1; unsigned long nr_pages = zram->disksize >> PAGE_SHIFT; - int index; + unsigned long index; for (index = 0; index < nr_pages; index++) { /* @@ -493,8 +486,9 @@ static ssize_t idle_store(struct device *dev, struct device_attribute *attr, #define INVALID_BDEV_BLOCK (~0UL) static int read_from_zspool_raw(struct zram *zram, struct page *page, - u32 index); -static int read_from_zspool(struct zram *zram, struct page *page, u32 index); + unsigned long index); +static int read_from_zspool(struct zram *zram, struct page *page, + unsigned long index); struct zram_wb_ctl { /* idle list is accessed only by the writeback task, no concurency */ @@ -504,6 +498,7 @@ struct zram_wb_ctl { wait_queue_head_t done_wait; spinlock_t done_lock; atomic_t num_inflight; + struct rcu_head rcu; }; struct zram_wb_req { @@ -529,7 +524,7 @@ struct zram_rb_req { /* error status (sync read) */ int error; }; - u32 index; + unsigned long index; }; #define FOUR_K(x) ((x) * (1 << (PAGE_SHIFT - 12))) @@ -847,7 +842,7 @@ static void release_wb_ctl(struct zram_wb_ctl *wb_ctl) release_wb_req(req); } - kfree(wb_ctl); + kfree_rcu(wb_ctl, rcu); } static struct zram_wb_ctl *init_wb_ctl(struct zram *zram) @@ -917,7 +912,7 @@ static void zram_account_writeback_submit(struct zram *zram) static int zram_writeback_complete(struct zram *zram, struct zram_wb_req *req) { - u32 index = req->pps->index; + unsigned long index = req->pps->index; int err; err = blk_status_to_errno(req->bio.bi_status); @@ -964,11 +959,13 @@ static void zram_writeback_endio(struct bio *bio) struct zram_wb_ctl *wb_ctl = bio->bi_private; unsigned long flags; + rcu_read_lock(); spin_lock_irqsave(&wb_ctl->done_lock, flags); list_add(&req->entry, &wb_ctl->done_reqs); spin_unlock_irqrestore(&wb_ctl->done_lock, flags); wake_up(&wb_ctl->done_wait); + rcu_read_unlock(); } static void zram_submit_wb_request(struct zram *zram, @@ -1037,7 +1034,7 @@ static int zram_writeback_slots(struct zram *zram, struct zram_wb_req *req = NULL; struct zram_pp_slot *pps; int ret = 0, err = 0; - u32 index = 0; + unsigned long index = 0; while ((pps = select_pp_slot(ctl))) { if (zram->wb_limit_enable && !zram->bd_wb_limit) { @@ -1123,6 +1120,9 @@ next: if (req) release_wb_req(req); + if (blk_idx != INVALID_BDEV_BLOCK) + zram_release_bdev_block(zram, blk_idx); + while (atomic_read(&wb_ctl->num_inflight) > 0) { wait_event(wb_ctl->done_wait, !list_empty(&wb_ctl->done_reqs)); err = zram_complete_done_reqs(zram, wb_ctl); @@ -1196,11 +1196,11 @@ static int parse_mode(char *val, u32 *mode) return 0; } -static int scan_slots_for_writeback(struct zram *zram, u32 mode, - unsigned long lo, unsigned long hi, - struct zram_pp_ctl *ctl) +static void scan_slots_for_writeback(struct zram *zram, u32 mode, + unsigned long lo, unsigned long hi, + struct zram_pp_ctl *ctl) { - u32 index = lo; + unsigned long index = lo; while (index < hi) { bool ok = true; @@ -1230,8 +1230,6 @@ next: break; index++; } - - return 0; } static ssize_t writeback_store(struct device *dev, @@ -1239,8 +1237,8 @@ static ssize_t writeback_store(struct device *dev, const char *buf, size_t len) { struct zram *zram = dev_to_zram(dev); - u64 nr_pages = zram->disksize >> PAGE_SHIFT; - unsigned long lo = 0, hi = nr_pages; + unsigned long nr_pages; + unsigned long lo = 0, hi; struct zram_pp_ctl *pp_ctl = NULL; struct zram_wb_ctl *wb_ctl = NULL; char *args, *param, *val; @@ -1254,6 +1252,9 @@ static ssize_t writeback_store(struct device *dev, if (!zram->backing_dev) return -ENODEV; + nr_pages = zram->disksize >> PAGE_SHIFT; + hi = nr_pages; + pp_ctl = init_pp_ctl(); if (!pp_ctl) return -ENOMEM; @@ -1337,7 +1338,8 @@ out: return ret; } -static int decompress_bdev_page(struct zram *zram, struct page *page, u32 index) +static int decompress_bdev_page(struct zram *zram, struct page *page, + unsigned long index) { struct zcomp_strm *zstrm; unsigned int size; @@ -1379,7 +1381,7 @@ static void zram_deferred_decompress(struct work_struct *w) struct zram_rb_req *req = container_of(w, struct zram_rb_req, work); struct page *page = bio_first_page_all(req->bio); struct zram *zram = req->zram; - u32 index = req->index; + unsigned long index = req->index; int ret; ret = decompress_bdev_page(zram, page, index); @@ -1429,21 +1431,21 @@ static void zram_async_read_endio(struct bio *bio) queue_work(system_highpri_wq, &req->work); } -static void read_from_bdev_async(struct zram *zram, struct page *page, - u32 index, unsigned long blk_idx, - struct bio *parent) +static int read_from_bdev_async(struct zram *zram, struct page *page, + unsigned long index, unsigned long blk_idx, + struct bio *parent) { struct zram_rb_req *req; struct bio *bio; req = kmalloc_obj(*req, GFP_NOIO); if (!req) - return; + return -ENOMEM; bio = bio_alloc(zram->bdev, 1, parent->bi_opf, GFP_NOIO); if (!bio) { kfree(req); - return; + return -ENOMEM; } req->zram = zram; @@ -1459,6 +1461,8 @@ static void read_from_bdev_async(struct zram *zram, struct page *page, __bio_add_page(bio, page, PAGE_SIZE, 0); bio_inc_remaining(parent); submit_bio(bio); + + return 0; } static void zram_sync_read(struct work_struct *w) @@ -1478,8 +1482,8 @@ static void zram_sync_read(struct work_struct *w) * chained IO with parent IO in same context, it's a deadlock. To avoid that, * use a worker thread context. */ -static int read_from_bdev_sync(struct zram *zram, struct page *page, u32 index, - unsigned long blk_idx) +static int read_from_bdev_sync(struct zram *zram, struct page *page, + unsigned long index, unsigned long blk_idx) { struct zram_rb_req req; @@ -1498,8 +1502,9 @@ static int read_from_bdev_sync(struct zram *zram, struct page *page, u32 index, return decompress_bdev_page(zram, page, index); } -static int read_from_bdev(struct zram *zram, struct page *page, u32 index, - unsigned long blk_idx, struct bio *parent) +static int read_from_bdev(struct zram *zram, struct page *page, + unsigned long index, unsigned long blk_idx, + struct bio *parent) { atomic64_inc(&zram->stats.bd_reads); if (!parent) { @@ -1507,13 +1512,13 @@ static int read_from_bdev(struct zram *zram, struct page *page, u32 index, return -EIO; return read_from_bdev_sync(zram, page, index, blk_idx); } - read_from_bdev_async(zram, page, index, blk_idx, parent); - return 0; + return read_from_bdev_async(zram, page, index, blk_idx, parent); } #else static inline void reset_bdev(struct zram *zram) {}; -static int read_from_bdev(struct zram *zram, struct page *page, u32 index, - unsigned long blk_idx, struct bio *parent) +static int read_from_bdev(struct zram *zram, struct page *page, + unsigned long index, unsigned long blk_idx, + struct bio *parent) { return -EIO; } @@ -1541,9 +1546,10 @@ static ssize_t read_block_state(struct file *file, char __user *buf, size_t count, loff_t *ppos) { char *kbuf; - ssize_t index, written = 0; + unsigned long index; + ssize_t written = 0; struct zram *zram = file->private_data; - unsigned long nr_pages = zram->disksize >> PAGE_SHIFT; + unsigned long nr_pages; kbuf = kvmalloc(count, GFP_KERNEL); if (!kbuf) @@ -1555,6 +1561,8 @@ static ssize_t read_block_state(struct file *file, char __user *buf, return -EINVAL; } + nr_pages = zram->disksize >> PAGE_SHIFT; + for (index = *ppos; index < nr_pages; index++) { int copied; @@ -1563,7 +1571,7 @@ static ssize_t read_block_state(struct file *file, char __user *buf, goto next; copied = snprintf(kbuf + written, count, - "%12zd %12u.%06d %c%c%c%c%c%c\n", + "%12lu %12u.%06d %c%c%c%c%c%c\n", index, zram->table[index].attr.ac_time, 0, test_slot_flag(zram, index, ZRAM_SAME) ? 's' : '.', test_slot_flag(zram, index, ZRAM_WB) ? 'w' : '.', @@ -1619,59 +1627,77 @@ static void zram_debugfs_register(struct zram *zram) {}; static void zram_debugfs_unregister(struct zram *zram) {}; #endif -static void comp_algorithm_set(struct zram *zram, u32 prio, const char *alg) +/* Only algo parameter given, lookup by algo name */ +static int lookup_algo_priority(struct zram *zram, const char *algo, + u32 min_prio) +{ + s32 prio; + + for (prio = min_prio; prio < ZRAM_MAX_COMPS; prio++) { + if (!zram->comp_algs[prio]) + continue; + + if (!strcmp(zram->comp_algs[prio], algo)) + return prio; + } + + return -EINVAL; +} + +/* Both algo and priority parameters given, validate them */ +static int validate_algo_priority(struct zram *zram, const char *algo, u32 prio) { - /* Do not free statically defined compression algorithms */ - if (zram->comp_algs[prio] != default_compressor) - kfree(zram->comp_algs[prio]); + if (prio >= ZRAM_MAX_COMPS) + return -EINVAL; + /* No algo at given priority */ + if (!zram->comp_algs[prio]) + return -EINVAL; + /* A different algo at given priority */ + if (strcmp(zram->comp_algs[prio], algo)) + return -EINVAL; + return 0; +} +static void comp_algorithm_set(struct zram *zram, u32 prio, const char *alg) +{ zram->comp_algs[prio] = alg; } +static void comp_params_reset(struct zram *zram, u32 prio) +{ + struct zcomp_params *params = &zram->params[prio]; + + vfree(params->dict); + params->level = ZCOMP_PARAM_NOT_SET; + params->deflate.winbits = ZCOMP_PARAM_NOT_SET; + params->dict_sz = 0; + params->dict = NULL; +} + static int __comp_algorithm_store(struct zram *zram, u32 prio, const char *buf) { - char *compressor; + const char *alg; size_t sz; sz = strlen(buf); if (sz >= ZRAM_MAX_ALGO_NAME_SZ) return -E2BIG; - compressor = kstrdup(buf, GFP_KERNEL); - if (!compressor) - return -ENOMEM; - - /* ignore trailing newline */ - if (sz > 0 && compressor[sz - 1] == '\n') - compressor[sz - 1] = 0x00; - - if (!zcomp_available_algorithm(compressor)) { - kfree(compressor); + alg = zcomp_lookup_backend_name(buf); + if (!alg) return -EINVAL; - } guard(rwsem_write)(&zram->dev_lock); if (init_done(zram)) { - kfree(compressor); pr_info("Can't change algorithm for initialized device\n"); return -EBUSY; } - comp_algorithm_set(zram, prio, compressor); + comp_algorithm_set(zram, prio, alg); + comp_params_reset(zram, prio); return 0; } -static void comp_params_reset(struct zram *zram, u32 prio) -{ - struct zcomp_params *params = &zram->params[prio]; - - vfree(params->dict); - params->level = ZCOMP_PARAM_NOT_SET; - params->deflate.winbits = ZCOMP_PARAM_NOT_SET; - params->dict_sz = 0; - params->dict = NULL; -} - static int comp_params_store(struct zram *zram, u32 prio, s32 level, const char *dict_path, struct deflate_params *deflate_params) @@ -1686,8 +1712,16 @@ static int comp_params_store(struct zram *zram, u32 prio, s32 level, INT_MAX, NULL, READING_POLICY); - if (sz < 0) + if (sz < 0) { + pr_err("failed to load dictionary %s (err=%zd)\n", + dict_path, sz); + return sz; + } + if (sz == 0) { + pr_err("failed to load dictionary %s (empty file)\n", + dict_path); return -EINVAL; + } } zram->params[prio].dict_sz = sz; @@ -1705,6 +1739,7 @@ static ssize_t algorithm_params_store(struct device *dev, char *args, *param, *val, *algo = NULL, *dict_path = NULL; struct deflate_params deflate_params; struct zram *zram = dev_to_zram(dev); + bool prio_param = false; int ret; deflate_params.winbits = ZCOMP_PARAM_NOT_SET; @@ -1717,6 +1752,7 @@ static ssize_t algorithm_params_store(struct device *dev, return -EINVAL; if (!strcmp(param, "priority")) { + prio_param = true; ret = kstrtoint(val, 10, &prio); if (ret) return ret; @@ -1748,24 +1784,26 @@ static ssize_t algorithm_params_store(struct device *dev, } } - /* Lookup priority by algorithm name */ - if (algo) { - s32 p; + guard(rwsem_write)(&zram->dev_lock); + if (init_done(zram)) + return -EBUSY; - prio = -EINVAL; - for (p = ZRAM_PRIMARY_COMP; p < ZRAM_MAX_COMPS; p++) { - if (!zram->comp_algs[p]) - continue; + if (prio_param) { + if (prio < ZRAM_PRIMARY_COMP || prio >= ZRAM_MAX_COMPS) + return -EINVAL; + } - if (!strcmp(zram->comp_algs[p], algo)) { - prio = p; - break; - } - } + if (algo && prio_param) { + ret = validate_algo_priority(zram, algo, prio); + if (ret) + return ret; } - if (prio < ZRAM_PRIMARY_COMP || prio >= ZRAM_MAX_COMPS) - return -EINVAL; + if (algo && !prio_param) { + prio = lookup_algo_priority(zram, algo, ZRAM_PRIMARY_COMP); + if (prio < 0) + return -EINVAL; + } ret = comp_params_store(zram, prio, level, dict_path, &deflate_params); return ret ? ret : len; @@ -1940,8 +1978,8 @@ static ssize_t debug_stat_show(struct device *dev, static void zram_meta_free(struct zram *zram, u64 disksize) { - size_t num_pages = disksize >> PAGE_SHIFT; - size_t index; + unsigned long num_pages = disksize >> PAGE_SHIFT; + unsigned long index; if (!zram->table) return; @@ -1953,11 +1991,12 @@ static void zram_meta_free(struct zram *zram, u64 disksize) zs_destroy_pool(zram->mem_pool); vfree(zram->table); zram->table = NULL; + lockdep_unregister_key(&zram->table_lock_key); } static bool zram_meta_alloc(struct zram *zram, u64 disksize) { - size_t num_pages, index; + unsigned long num_pages; num_pages = disksize >> PAGE_SHIFT; zram->table = vzalloc(array_size(num_pages, sizeof(*zram->table))); @@ -1974,13 +2013,13 @@ static bool zram_meta_alloc(struct zram *zram, u64 disksize) if (!huge_class_size) huge_class_size = zs_huge_class_size(zram->mem_pool); - for (index = 0; index < num_pages; index++) - slot_lock_init(zram, index); + lockdep_register_key(&zram->table_lock_key); + lockdep_init_map(&zram->table_lock_map, "zram->table[index].lock", &zram->table_lock_key, 0); return true; } -static void slot_free(struct zram *zram, u32 index) +static void slot_free(struct zram *zram, unsigned long index) { unsigned long handle; @@ -2034,7 +2073,7 @@ out: } static int read_same_filled_page(struct zram *zram, struct page *page, - u32 index) + unsigned long index) { void *mem; @@ -2045,7 +2084,7 @@ static int read_same_filled_page(struct zram *zram, struct page *page, } static int read_incompressible_page(struct zram *zram, struct page *page, - u32 index) + unsigned long index) { unsigned long handle; void *src, *dst; @@ -2060,7 +2099,8 @@ static int read_incompressible_page(struct zram *zram, struct page *page, return 0; } -static int read_compressed_page(struct zram *zram, struct page *page, u32 index) +static int read_compressed_page(struct zram *zram, struct page *page, + unsigned long index) { struct zcomp_strm *zstrm; unsigned long handle; @@ -2085,7 +2125,8 @@ static int read_compressed_page(struct zram *zram, struct page *page, u32 index) } #if defined CONFIG_ZRAM_WRITEBACK -static int read_from_zspool_raw(struct zram *zram, struct page *page, u32 index) +static int read_from_zspool_raw(struct zram *zram, struct page *page, + unsigned long index) { struct zcomp_strm *zstrm; unsigned long handle; @@ -2107,6 +2148,8 @@ static int read_from_zspool_raw(struct zram *zram, struct page *page, u32 index) zs_obj_read_end(zram->mem_pool, handle, size, src); zcomp_stream_put(zstrm); + memzero_page(page, size, PAGE_SIZE - size); + return 0; } #endif @@ -2115,7 +2158,8 @@ static int read_from_zspool_raw(struct zram *zram, struct page *page, u32 index) * Reads (decompresses if needed) a page from zspool (zsmalloc). * Corresponding ZRAM slot should be locked. */ -static int read_from_zspool(struct zram *zram, struct page *page, u32 index) +static int read_from_zspool(struct zram *zram, struct page *page, + unsigned long index) { if (test_slot_flag(zram, index, ZRAM_SAME) || !get_slot_handle(zram, index)) @@ -2127,8 +2171,8 @@ static int read_from_zspool(struct zram *zram, struct page *page, u32 index) return read_incompressible_page(zram, page, index); } -static int zram_read_page(struct zram *zram, struct page *page, u32 index, - struct bio *parent) +static int zram_read_page(struct zram *zram, struct page *page, + unsigned long index, struct bio *parent) { int ret; @@ -2150,7 +2194,7 @@ static int zram_read_page(struct zram *zram, struct page *page, u32 index, /* Should NEVER happen. Return bio error if it does. */ if (WARN_ON(ret < 0)) - pr_err("Decompression failed! err=%d, page=%u\n", ret, index); + pr_err("Decompression failed! err=%d, page=%lu\n", ret, index); return ret; } @@ -2160,7 +2204,7 @@ static int zram_read_page(struct zram *zram, struct page *page, u32 index, * always expects a full page for the output. */ static int zram_bvec_read_partial(struct zram *zram, struct bio_vec *bvec, - u32 index, int offset) + unsigned long index, int offset) { struct page *page = alloc_page(GFP_NOIO); int ret; @@ -2175,7 +2219,7 @@ static int zram_bvec_read_partial(struct zram *zram, struct bio_vec *bvec, } static int zram_bvec_read(struct zram *zram, struct bio_vec *bvec, - u32 index, int offset, struct bio *bio) + unsigned long index, int offset, struct bio *bio) { if (is_partial_io(bvec)) return zram_bvec_read_partial(zram, bvec, index, offset); @@ -2183,7 +2227,7 @@ static int zram_bvec_read(struct zram *zram, struct bio_vec *bvec, } static int write_same_filled_page(struct zram *zram, unsigned long fill, - u32 index) + unsigned long index) { slot_lock(zram, index); slot_free(zram, index); @@ -2198,7 +2242,7 @@ static int write_same_filled_page(struct zram *zram, unsigned long fill, } static int write_incompressible_page(struct zram *zram, struct page *page, - u32 index) + unsigned long index) { unsigned long handle; void *src; @@ -2238,7 +2282,8 @@ static int write_incompressible_page(struct zram *zram, struct page *page, return 0; } -static int zram_write_page(struct zram *zram, struct page *page, u32 index) +static int zram_write_page(struct zram *zram, struct page *page, + unsigned long index) { int ret = 0; unsigned long handle; @@ -2305,7 +2350,7 @@ static int zram_write_page(struct zram *zram, struct page *page, u32 index) * This is a partial IO. Read the full page before writing the changes. */ static int zram_bvec_write_partial(struct zram *zram, struct bio_vec *bvec, - u32 index, int offset, struct bio *bio) + unsigned long index, int offset) { struct page *page = alloc_page(GFP_NOIO); int ret; @@ -2313,7 +2358,7 @@ static int zram_bvec_write_partial(struct zram *zram, struct bio_vec *bvec, if (!page) return -ENOMEM; - ret = zram_read_page(zram, page, index, bio); + ret = zram_read_page(zram, page, index, NULL); if (!ret) { memcpy_from_bvec(page_address(page) + offset, bvec); ret = zram_write_page(zram, page, index); @@ -2323,10 +2368,10 @@ static int zram_bvec_write_partial(struct zram *zram, struct bio_vec *bvec, } static int zram_bvec_write(struct zram *zram, struct bio_vec *bvec, - u32 index, int offset, struct bio *bio) + unsigned long index, int offset) { if (is_partial_io(bvec)) - return zram_bvec_write_partial(zram, bvec, index, offset, bio); + return zram_bvec_write_partial(zram, bvec, index, offset); return zram_write_page(zram, bvec->bv_page, index); } @@ -2334,8 +2379,20 @@ static int zram_bvec_write(struct zram *zram, struct bio_vec *bvec, #define RECOMPRESS_IDLE (1 << 0) #define RECOMPRESS_HUGE (1 << 1) -static int scan_slots_for_recompress(struct zram *zram, u32 mode, u32 prio_max, - struct zram_pp_ctl *ctl) +static bool highest_priority_algorithm(struct zram *zram, u32 prio) +{ + u32 p; + + for (p = prio + 1; p < ZRAM_MAX_COMPS; p++) { + if (zram->comp_algs[p]) + return false; + } + + return true; +} + +static void scan_slots_for_recompress(struct zram *zram, u32 mode, u32 prio, + struct zram_pp_ctl *ctl) { unsigned long nr_pages = zram->disksize >> PAGE_SHIFT; unsigned long index; @@ -2360,8 +2417,8 @@ static int scan_slots_for_recompress(struct zram *zram, u32 mode, u32 prio_max, test_slot_flag(zram, index, ZRAM_INCOMPRESSIBLE)) goto next; - /* Already compressed with same of higher priority */ - if (get_slot_comp_priority(zram, index) + 1 >= prio_max) + /* Already compressed with same or higher priority */ + if (get_slot_comp_priority(zram, index) >= prio) goto next; ok = place_pp_slot(zram, ctl, index); @@ -2370,8 +2427,6 @@ next: if (!ok) break; } - - return 0; } /* @@ -2381,9 +2436,9 @@ next: * * Corresponding ZRAM slot should be locked. */ -static int recompress_slot(struct zram *zram, u32 index, struct page *page, - u64 *num_recomp_pages, u32 threshold, u32 prio, - u32 prio_max) +static int recompress_slot(struct zram *zram, unsigned long index, + struct page *page, u64 *num_recomp_pages, + u32 threshold, u32 prio) { struct zcomp_strm *zstrm = NULL; unsigned long handle_old; @@ -2417,51 +2472,10 @@ static int recompress_slot(struct zram *zram, u32 index, struct page *page, */ clear_slot_flag(zram, index, ZRAM_IDLE); - class_index_old = zs_lookup_class_index(zram->mem_pool, comp_len_old); - - prio = max(prio, get_slot_comp_priority(zram, index) + 1); - /* - * Recompression slots scan should not select slots that are - * already compressed with a higher priority algorithm, but - * just in case - */ - if (prio >= prio_max) - return 0; - - /* - * Iterate the secondary comp algorithms list (in order of priority) - * and try to recompress the page. - */ - for (; prio < prio_max; prio++) { - if (!zram->comps[prio]) - continue; - - zstrm = zcomp_stream_get(zram->comps[prio]); - src = kmap_local_page(page); - ret = zcomp_compress(zram->comps[prio], zstrm, - src, &comp_len_new); - kunmap_local(src); - - if (ret) { - zcomp_stream_put(zstrm); - zstrm = NULL; - break; - } - - class_index_new = zs_lookup_class_index(zram->mem_pool, - comp_len_new); - - /* Continue until we make progress */ - if (class_index_new >= class_index_old || - (threshold && comp_len_new >= threshold)) { - zcomp_stream_put(zstrm); - zstrm = NULL; - continue; - } - - /* Recompression was successful so break out */ - break; - } + zstrm = zcomp_stream_get(zram->comps[prio]); + src = kmap_local_page(page); + ret = zcomp_compress(zram->comps[prio], zstrm, src, &comp_len_new); + kunmap_local(src); /* * Decrement the limit (if set) on pages we can recompress, even @@ -2472,21 +2486,27 @@ static int recompress_slot(struct zram *zram, u32 index, struct page *page, if (*num_recomp_pages) *num_recomp_pages -= 1; - /* Compression error */ - if (ret) + if (ret) { + zcomp_stream_put(zstrm); return ret; + } + + class_index_old = zs_lookup_class_index(zram->mem_pool, comp_len_old); + class_index_new = zs_lookup_class_index(zram->mem_pool, comp_len_new); + + if (class_index_new >= class_index_old || + (threshold && comp_len_new >= threshold)) { + zcomp_stream_put(zstrm); - if (!zstrm) { /* * Secondary algorithms failed to re-compress the page * in a way that would save memory. * - * Mark the object incompressible if the max-priority - * algorithm couldn't re-compress it. + * Mark the object incompressible if the max-priority (the + * last configured one) algorithm couldn't re-compress it. */ - if (prio < zram->num_active_comps) - return 0; - set_slot_flag(zram, index, ZRAM_INCOMPRESSIBLE); + if (highest_priority_algorithm(zram, prio)) + set_slot_flag(zram, index, ZRAM_INCOMPRESSIBLE); return 0; } @@ -2531,15 +2551,13 @@ static ssize_t recompress_store(struct device *dev, char *args, *param, *val, *algo = NULL; u64 num_recomp_pages = ULLONG_MAX; struct zram_pp_ctl *ctl = NULL; - struct zram_pp_slot *pps; + s32 prio = ZRAM_SECONDARY_COMP; u32 mode = 0, threshold = 0; - u32 prio, prio_max; + struct zram_pp_slot *pps; struct page *page = NULL; + bool prio_param = false; ssize_t ret; - prio = ZRAM_SECONDARY_COMP; - prio_max = zram->num_active_comps; - args = skip_spaces(buf); while (*args) { args = next_arg(args, ¶m, &val); @@ -2554,6 +2572,8 @@ static ssize_t recompress_store(struct device *dev, mode = RECOMPRESS_HUGE; if (!strcmp(val, "huge_idle")) mode = RECOMPRESS_IDLE | RECOMPRESS_HUGE; + if (!mode) + return -EINVAL; continue; } @@ -2585,14 +2605,10 @@ static ssize_t recompress_store(struct device *dev, } if (!strcmp(param, "priority")) { - ret = kstrtouint(val, 10, &prio); + prio_param = true; + ret = kstrtoint(val, 10, &prio); if (ret) return ret; - - if (prio == ZRAM_PRIMARY_COMP) - prio = ZRAM_SECONDARY_COMP; - - prio_max = prio + 1; continue; } } @@ -2604,32 +2620,26 @@ static ssize_t recompress_store(struct device *dev, if (!init_done(zram)) return -EINVAL; - if (algo) { - bool found = false; - - for (; prio < ZRAM_MAX_COMPS; prio++) { - if (!zram->comp_algs[prio]) - continue; - - if (!strcmp(zram->comp_algs[prio], algo)) { - prio_max = prio + 1; - found = true; - break; - } - } + if (prio_param) { + if (prio < ZRAM_SECONDARY_COMP || prio >= ZRAM_MAX_COMPS) + return -EINVAL; + } - if (!found) { - ret = -EINVAL; - goto out; - } + if (algo && prio_param) { + ret = validate_algo_priority(zram, algo, prio); + if (ret) + return ret; } - prio_max = min(prio_max, (u32)zram->num_active_comps); - if (prio >= prio_max) { - ret = -EINVAL; - goto out; + if (algo && !prio_param) { + prio = lookup_algo_priority(zram, algo, ZRAM_SECONDARY_COMP); + if (prio < 0) + return -EINVAL; } + if (!zram->comps[prio]) + return -EINVAL; + page = alloc_page(GFP_KERNEL); if (!page) { ret = -ENOMEM; @@ -2642,7 +2652,7 @@ static ssize_t recompress_store(struct device *dev, goto out; } - scan_slots_for_recompress(zram, mode, prio_max, ctl); + scan_slots_for_recompress(zram, mode, prio, ctl); ret = len; while ((pps = select_pp_slot(ctl))) { @@ -2656,8 +2666,7 @@ static ssize_t recompress_store(struct device *dev, goto next; err = recompress_slot(zram, pps->index, page, - &num_recomp_pages, threshold, - prio, prio_max); + &num_recomp_pages, threshold, prio); next: slot_unlock(zram, pps->index); release_pp_slot(zram, pps); @@ -2681,7 +2690,7 @@ out: static void zram_bio_discard(struct zram *zram, struct bio *bio) { size_t n = bio->bi_iter.bi_size; - u32 index = bio->bi_iter.bi_sector >> SECTORS_PER_PAGE_SHIFT; + unsigned long index = bio->bi_iter.bi_sector >> SECTORS_PER_PAGE_SHIFT; u32 offset = (bio->bi_iter.bi_sector & (SECTORS_PER_PAGE - 1)) << SECTOR_SHIFT; @@ -2697,7 +2706,7 @@ static void zram_bio_discard(struct zram *zram, struct bio *bio) */ if (offset) { if (n <= (PAGE_SIZE - offset)) - return; + goto end_bio; n -= (PAGE_SIZE - offset); index++; @@ -2712,6 +2721,7 @@ static void zram_bio_discard(struct zram *zram, struct bio *bio) n -= PAGE_SIZE; } +end_bio: bio_endio(bio); } @@ -2721,7 +2731,7 @@ static void zram_bio_read(struct zram *zram, struct bio *bio) struct bvec_iter iter = bio->bi_iter; do { - u32 index = iter.bi_sector >> SECTORS_PER_PAGE_SHIFT; + unsigned long index = iter.bi_sector >> SECTORS_PER_PAGE_SHIFT; u32 offset = (iter.bi_sector & (SECTORS_PER_PAGE - 1)) << SECTOR_SHIFT; struct bio_vec bv = bio_iter_iovec(bio, iter); @@ -2752,14 +2762,14 @@ static void zram_bio_write(struct zram *zram, struct bio *bio) struct bvec_iter iter = bio->bi_iter; do { - u32 index = iter.bi_sector >> SECTORS_PER_PAGE_SHIFT; + unsigned long index = iter.bi_sector >> SECTORS_PER_PAGE_SHIFT; u32 offset = (iter.bi_sector & (SECTORS_PER_PAGE - 1)) << SECTOR_SHIFT; struct bio_vec bv = bio_iter_iovec(bio, iter); bv.bv_len = min_t(u32, bv.bv_len, PAGE_SIZE - offset); - if (zram_bvec_write(zram, &bv, index, offset, bio) < 0) { + if (zram_bvec_write(zram, &bv, index, offset) < 0) { atomic64_inc(&zram->stats.failed_writes); bio->bi_status = BLK_STS_IOERR; break; @@ -2837,17 +2847,13 @@ static void zram_destroy_comps(struct zram *zram) if (!comp) continue; zcomp_destroy(comp); - zram->num_active_comps--; } - for (prio = ZRAM_PRIMARY_COMP; prio < ZRAM_MAX_COMPS; prio++) { - /* Do not free statically defined compression algorithms */ - if (zram->comp_algs[prio] != default_compressor) - kfree(zram->comp_algs[prio]); + for (prio = ZRAM_PRIMARY_COMP; prio < ZRAM_MAX_COMPS; prio++) zram->comp_algs[prio] = NULL; - } zram_comp_params_reset(zram); + comp_algorithm_set(zram, ZRAM_PRIMARY_COMP, default_compressor); } static void zram_reset_device(struct zram *zram) @@ -2865,13 +2871,12 @@ static void zram_reset_device(struct zram *zram) zram_destroy_comps(zram); memset(&zram->stats, 0, sizeof(zram->stats)); reset_bdev(zram); - - comp_algorithm_set(zram, ZRAM_PRIMARY_COMP, default_compressor); } static ssize_t disksize_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t len) { + unsigned long num_pages; u64 disksize; struct zcomp *comp; struct zram *zram = dev_to_zram(dev); @@ -2889,6 +2894,11 @@ static ssize_t disksize_store(struct device *dev, struct device_attribute *attr, } disksize = PAGE_ALIGN(disksize); + num_pages = disksize >> PAGE_SHIFT; + /* Slots are addressed by an unsigned long index */ + if (!num_pages || ((u64)num_pages << PAGE_SHIFT) != disksize) + return -EINVAL; + if (!zram_meta_alloc(zram, disksize)) return -ENOMEM; @@ -2906,7 +2916,6 @@ static ssize_t disksize_store(struct device *dev, struct device_attribute *attr, } zram->comps[prio] = comp; - zram->num_active_comps++; } zram->disksize = disksize; set_capacity_and_notify(zram->disk, zram->disksize >> SECTOR_SHIFT); diff --git a/drivers/block/zram/zram_drv.h b/drivers/block/zram/zram_drv.h index f0de8f8218f5..7a55d751417e 100644 --- a/drivers/block/zram/zram_drv.h +++ b/drivers/block/zram/zram_drv.h @@ -15,6 +15,7 @@ #ifndef _ZRAM_DRV_H_ #define _ZRAM_DRV_H_ +#include <asm/byteorder.h> #include <linux/rwsem.h> #include <linux/zsmalloc.h> @@ -58,6 +59,19 @@ enum zram_pageflags { }; /* + * The slot lock is a bit-wait lock on the whole __lock word, while + * flags and ac_time alias that word as two u32s. The lock bit must + * land in the slot that ZRAM_ENTRY_LOCK reserves in attr.flags; on + * 64-bit big-endian the flags word maps to the upper half of __lock, + * so the bit position has to be shifted up. + */ +#if defined(CONFIG_64BIT) && defined(__BIG_ENDIAN) +#define ZRAM_ENTRY_LOCK_BIT (ZRAM_ENTRY_LOCK + 32) +#else +#define ZRAM_ENTRY_LOCK_BIT ZRAM_ENTRY_LOCK +#endif + +/* * Allocated for each disk page. We use bit-lock (ZRAM_ENTRY_LOCK bit * of flags) to save memory. There can be plenty of entries and standard * locking primitives (e.g. mutex) will significantly increase sizeof() @@ -74,7 +88,6 @@ struct zram_table_entry { #endif } attr; }; - struct lockdep_map dep_map; }; struct zram_stats { @@ -107,6 +120,8 @@ struct zram_stats { struct zram { struct zram_table_entry *table; + struct lockdep_map table_lock_map; + struct lock_class_key table_lock_key; struct zs_pool *mem_pool; struct zcomp *comps[ZRAM_MAX_COMPS]; struct zcomp_params params[ZRAM_MAX_COMPS]; @@ -125,7 +140,6 @@ struct zram { */ u64 disksize; /* bytes */ const char *comp_algs[ZRAM_MAX_COMPS]; - s8 num_active_comps; /* * zram is claimed so open request will be failed */ |