diff options
| author | Takashi Iwai <tiwai@suse.de> | 2026-11-05 16:56:23 +0100 |
|---|---|---|
| committer | Takashi Iwai <tiwai@suse.de> | 2026-11-05 16:56:23 +0100 |
| commit | 22733b6e2e1ebf75228a1e15b9fb9760a86137cc (patch) | |
| tree | c7388fd1cb44ed49347418ae058ead753d2c473b /include/linux | |
| parent | d54c3cc224dab59cb6b4db43cceaf84a4b39c87a (diff) | |
| parent | cf09d02dc0c49593fdd9e0ec2f17251cba39a94e (diff) | |
Merge tag 'asoc-fix-v7.4-rc1' of https://git.kernel.org/pub/scm/linux/kernel/git/broonie/sound into for-linus
ASoC: Fixes for v7.4
More fixes that have come in since -rc1, a lot of this is quirks but
there's some bug fixes as well. The bug fixes should mostly be those
that seem to be affecting actual users rather than just being pure
static analysis work. I also took some fixes for missing Kconfig
dependencies which won't have any runtime impact.
Diffstat (limited to 'include/linux')
411 files changed, 11236 insertions, 3349 deletions
diff --git a/include/linux/acpi.h b/include/linux/acpi.h index ddacac812094..9d3f039ebd94 100644 --- a/include/linux/acpi.h +++ b/include/linux/acpi.h @@ -25,17 +25,19 @@ struct irq_domain_ops; #define _LINUX #endif #include <acpi/acpi.h> +#include <acpi/acpi_bus.h> #include <acpi/acpi_numa.h> #ifdef CONFIG_ACPI +DEFINE_FREE(acpi_object_free, union acpi_object *, if (_T) ACPI_FREE(_T)); + #include <linux/list.h> #include <linux/dynamic_debug.h> #include <linux/module.h> #include <linux/mutex.h> #include <linux/fw_table.h> -#include <acpi/acpi_bus.h> #include <acpi/acpi_drivers.h> #include <acpi/acpi_io.h> #include <asm/acpi.h> @@ -105,8 +107,6 @@ static inline const char *acpi_dev_name(struct acpi_device *adev) return dev_name(&adev->dev); } -struct device *acpi_get_first_physical_node(struct acpi_device *adev); - enum acpi_irq_model_id { ACPI_IRQ_MODEL_PIC = 0, ACPI_IRQ_MODEL_IOAPIC, @@ -434,7 +434,7 @@ extern acpi_handle ec_get_handle(void); extern bool acpi_is_pnp_device(struct acpi_device *); -#if defined(CONFIG_ACPI_WMI) || defined(CONFIG_ACPI_WMI_MODULE) +#if IS_ENABLED(CONFIG_ACPI_WMI) typedef void (*wmi_notify_handler) (union acpi_object *data, void *context); @@ -938,11 +938,6 @@ static inline const char *acpi_dev_name(struct acpi_device *adev) return NULL; } -static inline struct device *acpi_get_first_physical_node(struct acpi_device *adev) -{ - return NULL; -} - static inline void acpi_early_init(void) { } static inline void acpi_subsystem_init(void) { } @@ -1305,34 +1300,34 @@ void __acpi_handle_debug(struct _ddebug *descriptor, acpi_handle handle, const c #endif /* - * acpi_handle_<level>: Print message with ACPI prefix and object path + * acpi_handle_<level> - Print a message with ACPI prefix and object path * - * These interfaces acquire the global namespace mutex to obtain an object - * path. In interrupt context, it shows the object path as <n/a>. + * In thread context, the global namespace mutex is acquired to obtain the + * object path. In interrupt context, the object path is shown as <n/a>. */ #define acpi_handle_emerg(handle, fmt, ...) \ - acpi_handle_printk(KERN_EMERG, handle, fmt, ##__VA_ARGS__) + acpi_handle_printk(KERN_EMERG, handle, dev_fmt(fmt), ##__VA_ARGS__) #define acpi_handle_alert(handle, fmt, ...) \ - acpi_handle_printk(KERN_ALERT, handle, fmt, ##__VA_ARGS__) + acpi_handle_printk(KERN_ALERT, handle, dev_fmt(fmt), ##__VA_ARGS__) #define acpi_handle_crit(handle, fmt, ...) \ - acpi_handle_printk(KERN_CRIT, handle, fmt, ##__VA_ARGS__) + acpi_handle_printk(KERN_CRIT, handle, dev_fmt(fmt), ##__VA_ARGS__) #define acpi_handle_err(handle, fmt, ...) \ - acpi_handle_printk(KERN_ERR, handle, fmt, ##__VA_ARGS__) + acpi_handle_printk(KERN_ERR, handle, dev_fmt(fmt), ##__VA_ARGS__) #define acpi_handle_warn(handle, fmt, ...) \ - acpi_handle_printk(KERN_WARNING, handle, fmt, ##__VA_ARGS__) + acpi_handle_printk(KERN_WARNING, handle, dev_fmt(fmt), ##__VA_ARGS__) #define acpi_handle_notice(handle, fmt, ...) \ - acpi_handle_printk(KERN_NOTICE, handle, fmt, ##__VA_ARGS__) + acpi_handle_printk(KERN_NOTICE, handle, dev_fmt(fmt), ##__VA_ARGS__) #define acpi_handle_info(handle, fmt, ...) \ - acpi_handle_printk(KERN_INFO, handle, fmt, ##__VA_ARGS__) + acpi_handle_printk(KERN_INFO, handle, dev_fmt(fmt), ##__VA_ARGS__) #if defined(DEBUG) #define acpi_handle_debug(handle, fmt, ...) \ - acpi_handle_printk(KERN_DEBUG, handle, fmt, ##__VA_ARGS__) + acpi_handle_printk(KERN_DEBUG, handle, dev_fmt(fmt), ##__VA_ARGS__) #else #if defined(CONFIG_DYNAMIC_DEBUG) #define acpi_handle_debug(handle, fmt, ...) \ _dynamic_func_call(fmt, __acpi_handle_debug, \ - handle, pr_fmt(fmt), ##__VA_ARGS__) + handle, dev_fmt(fmt), ##__VA_ARGS__) #else #define acpi_handle_debug(handle, fmt, ...) \ ({ \ @@ -1642,6 +1637,7 @@ static inline int acpi_pptt_get_cpumask_from_cache_id(u32 cache_id, #endif void acpi_arch_init(void); +void acpi_arch_late_init(void); #ifdef CONFIG_ACPI_PCC void acpi_init_pcc(void); diff --git a/include/linux/acpi_iovt.h b/include/linux/acpi_iovt.h new file mode 100644 index 000000000000..663b4c679bc5 --- /dev/null +++ b/include/linux/acpi_iovt.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#ifndef __ACPI_IOVT_H__ +#define __ACPI_IOVT_H__ + +#ifdef CONFIG_ACPI_IOVT +int iovt_iommu_configure_id(struct device *dev, const u32 *id_in); +#else +static inline int iovt_iommu_configure_id(struct device *dev, const u32 *id_in) +{ return -ENODEV; } +#endif + +#endif /* __ACPI_IOVT_H__ */ diff --git a/include/linux/aer.h b/include/linux/aer.h index df0f5c382286..043ee76b6f53 100644 --- a/include/linux/aer.h +++ b/include/linux/aer.h @@ -24,6 +24,13 @@ #define PCIE_STD_MAX_TLP_PREFIXLOG 4 #define PCIE_STD_MAX_TLP_HEADERLOG (PCIE_STD_NUM_TLP_HEADERLOG + 10) +/* + * Size of the AER Capability register block in hardware, long enough for the + * longest Flit mode TLP Log. struct aer_capability_regs below is larger and + * laid out differently; use aer_cap_regs_unpack() to convert. + */ +#define PCIE_AER_CAP_HW_SIZE 96 + struct pci_dev; struct pcie_tlp_log { @@ -68,6 +75,8 @@ static inline void pci_aer_unmask_internal_errors(struct pci_dev *dev) { } void pci_print_aer(struct pci_dev *dev, int aer_severity, struct aer_capability_regs *aer); +void aer_cap_regs_unpack(struct aer_capability_regs *regs, const void *raw, + size_t raw_len); int cper_severity_to_aer(int cper_severity); void aer_recover_queue(int domain, unsigned int bus, unsigned int devfn, int severity, struct aer_capability_regs *aer_regs); diff --git a/include/linux/amd-iommu.h b/include/linux/amd-iommu.h index edcee9f5335a..104817e8d897 100644 --- a/include/linux/amd-iommu.h +++ b/include/linux/amd-iommu.h @@ -11,11 +11,11 @@ #include <linux/types.h> struct amd_iommu; +struct pci_dev; #ifdef CONFIG_AMD_IOMMU struct task_struct; -struct pci_dev; extern void amd_iommu_detect(void); @@ -76,4 +76,15 @@ static inline int amd_iommu_snp_disable(void) { return 0; } static inline bool amd_iommu_sev_tio_supported(void) { return false; } #endif +#ifdef CONFIG_AMD_IOMMU +int amd_iommu_enable_perfopt(struct pci_dev *pdev); +void amd_iommu_disable_perfopt(struct pci_dev *pdev); +#else +static inline int amd_iommu_enable_perfopt(struct pci_dev *pdev) +{ + return 0; +} +static inline void amd_iommu_disable_perfopt(struct pci_dev *pdev) { } +#endif + #endif /* _ASM_X86_AMD_IOMMU_H */ diff --git a/include/linux/apple-gmux.h b/include/linux/apple-gmux.h index 206d97ffda79..d7939c3a08fd 100644 --- a/include/linux/apple-gmux.h +++ b/include/linux/apple-gmux.h @@ -109,7 +109,7 @@ static inline bool apple_gmux_detect(struct pnp_dev *pnp_dev, enum apple_gmux_ty if (!adev) return false; - dev = get_device(acpi_get_first_physical_node(adev)); + dev = acpi_bus_get_primary_device(adev); acpi_dev_put(adev); if (!dev) return false; diff --git a/include/linux/arm-rmi-cmds.h b/include/linux/arm-rmi-cmds.h new file mode 100644 index 000000000000..a84e50ac851c --- /dev/null +++ b/include/linux/arm-rmi-cmds.h @@ -0,0 +1,581 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2026 ARM Ltd. + */ + +#ifndef __LINUX_ARM_RMI_CMDS_H_ +#define __LINUX_ARM_RMI_CMDS_H_ + +#include <linux/arm-smccc-rmi.h> +#include <linux/bug.h> +#include <linux/gfp.h> +#include <linux/processor.h> +#include <linux/string.h> +#include <linux/types.h> + +#define RMI_MAX_ADDR_LIST 256 + +struct rmi_sro_state { + struct arm_smccc_1_2_regs regs; + int addr_count; + unsigned long addr_list[RMI_MAX_ADDR_LIST]; +}; + +#define RMM_BLOCKED_RETRY_COUNT 2 +/* + * rmi_smccc_invoke: Invoke the RMI call and return the results, retrying the + * command when status is RMI_BUSY. If we encounter RMI_BLOCKED, we retry + * it one more time before we give up. The caller is supposed to handle the + * result and reissue if required. + * + * We don't expect to see RMI_BLOCKED on a practical system, except when + * there are parallel requests that results in long standing operation, + * with one blocking the other. + * + * @regs: Input parameters filled in. Updated with the output results + * after the call. + */ +static inline void rmi_smccc_invoke(struct arm_smccc_1_2_regs *regs) +{ + struct arm_smccc_1_2_regs args = *regs; + long status; + int i = 0; + + while (i < RMM_BLOCKED_RETRY_COUNT) { + arm_smccc_1_2_smc(&args, regs); + + status = RMI_RESULT_STATUS(regs->a0); + if (status != RMI_BUSY && status != RMI_BLOCKED) + break; + if (status == RMI_BLOCKED) + i++; + cpu_relax(); + } +} + +/* + * rmi_smccc_invoke_once: Invoke the RMI call and return the results. Do not + * retry the command. Let the caller deal with RMI_BUSY or RMI_BLOCKED. + * + * @regs: Input parameters filled in. Updated with the output results + * after the call. + */ +static __always_inline void rmi_smccc_invoke_once(struct arm_smccc_1_2_regs *regs) +{ + arm_smccc_1_2_smc(regs, regs); +} + +unsigned long rmi_feat_reg(unsigned int index); + +int rmi_delegate_range(phys_addr_t phys, unsigned long size, + phys_addr_t *out_phys); +int rmi_undelegate_range(phys_addr_t phys, unsigned long size); +int rmi_free_delegated_page(phys_addr_t phys); + +static inline int rmi_delegate_page(phys_addr_t phys) +{ + return rmi_delegate_range(phys, PAGE_SIZE, NULL); +} + +static inline int rmi_undelegate_page(phys_addr_t phys) +{ + return rmi_undelegate_range(phys, PAGE_SIZE); +} + +long rmi_sro_memxfer_execute(struct rmi_sro_state *sro, gfp_t gfp); +void rmi_sro_free(struct rmi_sro_state *sro); +long rmi_sro_execute(struct arm_smccc_1_2_regs *regs); + +/* + * Resetting the addr_count is sufficient to ignore the addr_list contents. + */ +#define rmi_sro_memxfer_cmd(sro, gfp, ...) ({ \ + struct rmi_sro_state *__sro = (sro); \ + __sro->addr_count = 0; \ + __sro->regs = (struct arm_smccc_1_2_regs){ __VA_ARGS__ }; \ + long __ret = rmi_sro_memxfer_execute(__sro, gfp); \ + rmi_sro_free(__sro); \ + __ret; \ +}) + +#ifdef CONFIG_ARM_RMM_RMI + +bool is_rmm_active(void); +bool is_rmi_available(void); + +#else + +static inline bool is_rmm_active(void) +{ + return false; +} + +static inline bool is_rmi_available(void) +{ + return false; +} +#endif /* CONFIG_ARM_RMM_RMI */ + +/** + * rmi_rtt_data_map_init() - Create a mapping at protected IPA, copying contents + * from a given non-secure source granule. + * @rd: PA of the RD + * @data: PA of the target granule mapped in the guest + * @ipa: IPA at which the granule @data will be mapped in the guest + * @src: PA of the source granule with contents + * @flags: RMI_MEASURE_CONTENT if the contents should be measured + * + * Create a mapping from Protected IPA space to conventional memory, copying + * contents from a Non-secure Granule provided by the caller. + * + * Return: 0 on success, positive RMI result code or negative Linux error code + */ +static inline long rmi_rtt_data_map_init(unsigned long rd, unsigned long data, + unsigned long ipa, unsigned long src, + unsigned long flags) +{ + struct arm_smccc_1_2_regs regs = { + SMC_RMI_RTT_DATA_MAP_INIT, rd, data, ipa, src, flags + }; + + return rmi_sro_execute(®s); +} + +/** + * rmi_rtt_data_map() - Create mappings in protected IPA range with unknown contents + * @rd: PA of the RD + * @base: Base of the target IPA range + * @top: Top of the target IPA range + * @flags: Flags + * @oaddr: Output address set descriptor + * @out_top: Top address of range which was processed. + * + * Return: 0 on success, positive RMI result code or negative Linux error code. + * NOTE: out_top is only valid with ret == RMI_SUCCESS + */ +static inline long rmi_rtt_data_map(unsigned long rd, + unsigned long base, + unsigned long top, + unsigned long flags, + unsigned long oaddr, + unsigned long *out_top) +{ + struct arm_smccc_1_2_regs regs = { + SMC_RMI_RTT_DATA_MAP, rd, base, top, flags, oaddr + }; + long ret; + + ret = rmi_sro_execute(®s); + + if (ret == RMI_SUCCESS && out_top) + *out_top = regs.a1; + + return ret; +} + +/** + * rmi_rtt_data_unmap() - Remove mappings to conventional memory at a protected + * IPA range + * @rd: PA of the RD + * @base: Base of the target IPA range + * @top: Top of the target IPA range + * @flags: Flags + * @oaddr: Output address set descriptor + * @out_top: Returns top IPA of range which has been unmapped + * @out_range: Output address range + * @out_count: Number of entries in output address list + * + * Removes mappings to convention memory with a target Protected IPA range. + * + * Return: 0 on success, positive RMI result code or negative Linux error code + * NOTE: out_top is only valid with ret == RMI_SUCCESS + */ +static inline long rmi_rtt_data_unmap(unsigned long rd, + unsigned long base, + unsigned long top, + unsigned long flags, + unsigned long oaddr, + unsigned long *out_top, + unsigned long *out_range, + unsigned long *out_count) +{ + struct arm_smccc_1_2_regs regs = { + SMC_RMI_RTT_DATA_UNMAP, rd, base, top, flags, oaddr + }; + long ret; + + ret = rmi_sro_execute(®s); + + if (ret != RMI_SUCCESS) + return ret; + + if (out_top) + *out_top = regs.a1; + if (out_range) + *out_range = regs.a2; + if (out_count) + *out_count = regs.a3; + + return RMI_SUCCESS; +} + +/** + * rmi_psci_complete() - Complete pending PSCI command + * @calling_rec: PA of the calling REC + * @status: Status of the PSCI request + * + * Completes a pending PSCI command. + * + * Return: RMI return code + */ +static inline long rmi_psci_complete(unsigned long calling_rec, + unsigned long status) +{ + struct arm_smccc_1_2_regs regs = { + SMC_RMI_PSCI_COMPLETE, calling_rec, status, + }; + + rmi_smccc_invoke(®s); + return regs.a0; +} + +/** + * rmi_realm_activate() - Activate a realm + * @rd: PA of the RD + * + * Mark a realm as Active, signalling that creation is completed, allowing + * execution of the realm. + * + * Return: RMI return code + */ +static inline long rmi_realm_activate(unsigned long rd) +{ + struct arm_smccc_1_2_regs regs = { + SMC_RMI_REALM_ACTIVATE, rd, + }; + + rmi_smccc_invoke(®s); + return regs.a0; +} + +/** + * rmi_realm_create() - Create a realm + * @rd: PA of the RD + * @params: PA of realm parameters + * @sro: Preallocated SRO context + * + * Create a new realm using the given parameters. + * + * Return: 0 on success, positive RMI result code or negative Linux error code + */ +static inline long rmi_realm_create(unsigned long rd, unsigned long params, + struct rmi_sro_state *sro) +{ + return rmi_sro_memxfer_cmd(sro, GFP_KERNEL, + SMC_RMI_REALM_CREATE, rd, params); +} + +/** + * rmi_realm_terminate() - Terminate a realm + * @rd: PA of the RD + * @sro: Preallocated SRO context + * + * Terminates a realm, moving it into a ZOMBIE state + * + * Return: 0 on success, positive RMI result code or negative Linux error code + */ +static inline long rmi_realm_terminate(unsigned long rd, + struct rmi_sro_state *sro) +{ + return rmi_sro_memxfer_cmd(sro, GFP_KERNEL, + SMC_RMI_REALM_TERMINATE, rd); +} + +/** + * rmi_realm_destroy() - Destroy a realm + * @rd: PA of the RD + * @sro: Preallocated SRO context + * + * Destroys a realm, all objects belonging to the realm must be destroyed first. + * + * Return: 0 on success, positive RMI result code or negative Linux error code + */ +static inline long rmi_realm_destroy(unsigned long rd, + struct rmi_sro_state *sro) +{ + return rmi_sro_memxfer_cmd(sro, GFP_KERNEL, + SMC_RMI_REALM_DESTROY, rd); +} + +/** + * rmi_rec_create() - Create a REC + * @rd: PA of the RD + * @rec: PA of the target REC + * @params: PA of REC parameters + * @sro: Allocated SRO context to be used + * + * Create a REC using the parameters specified in the struct rec_params pointed + * to by @params. + * + * Return: 0 on success, positive RMI result code or negative Linux error code + */ +static inline long rmi_rec_create(unsigned long rd, + unsigned long rec, + unsigned long params, + struct rmi_sro_state *sro) +{ + return rmi_sro_memxfer_cmd(sro, GFP_KERNEL, + SMC_RMI_REC_CREATE, rd, rec, params); +} + +/** + * rmi_rec_destroy() - Destroy a REC + * @rec: PA of the target REC + * @sro: Allocated SRO context to be used + * + * Destroys a REC. The REC must not be running. + * + * Return: 0 on success, positive RMI result code or negative Linux error code + */ +static __always_inline long rmi_rec_destroy(unsigned long rec, + struct rmi_sro_state *sro) +{ + return rmi_sro_memxfer_cmd(sro, GFP_KERNEL, SMC_RMI_REC_DESTROY, rec); +} + +/** + * rmi_rec_enter() - Enter a REC + * @rec: PA of the target REC + * @run_ptr: PA of RecRun structure + * + * Starts (or continues) execution within a REC. + * + * Return: RMI result + */ +static __always_inline long rmi_rec_enter(unsigned long rec, unsigned long run_ptr) +{ + struct arm_smccc_1_2_regs regs = { + SMC_RMI_REC_ENTER, rec, run_ptr, + }; + + rmi_smccc_invoke_once(®s); + return regs.a0; +} + +/** + * rmi_rtt_create() - Creates an RTT + * @rd: PA of the RD + * @rtt: PA of the target RTT + * @ipa: Base of the IPA range described by the RTT + * @level: Depth of the RTT within the tree + * + * Creates an RTT (Realm Translation Table) at the specified level for the + * translation of the specified address within the realm. + * + * Return: 0 on success, positive RMI result code or negative Linux error code + */ +static inline long rmi_rtt_create(unsigned long rd, unsigned long rtt, + unsigned long ipa, long level) +{ + struct arm_smccc_1_2_regs regs = { + SMC_RMI_RTT_CREATE, rd, rtt, ipa, level + }; + + return rmi_sro_execute(®s); +} + +/** + * rmi_rtt_destroy() - Destroy an RTT + * @rd: PA of the RD + * @ipa: Base of the IPA range described by the RTT + * @level: RTT level + * @out_rtt: Pointer to write the PA of the RTT which was destroyed + * @out_top: Pointer to write the top IPA of non-live RTT entries, from entry + * at which the RTT walk terminated. + * + * Destroys an RTT. The RTT must be non-live, i.e. none of the entries in the + * table are in ASSIGNED or TABLE state. + * + * Return: 0 on success, positive RMI result code or negative Linux error code. + */ +static inline long rmi_rtt_destroy(unsigned long rd, + unsigned long ipa, + long level, + unsigned long *out_rtt, + unsigned long *out_top) +{ + struct arm_smccc_1_2_regs regs = { + SMC_RMI_RTT_DESTROY, rd, ipa, level + }; + long ret = rmi_sro_execute(®s); + + switch (RMI_RESULT_STATUS(ret)) { + case RMI_SUCCESS: + if (out_rtt) + *out_rtt = regs.a1; + fallthrough; + case RMI_ERROR_RTT: + if (out_top) + *out_top = regs.a2; + break; + default: + break; + } + + return ret; +} + +/** + * rmi_rtt_fold() - Fold an RTT + * @rd: PA of the RD + * @ipa: Base of the IPA range described by the RTT + * @level: Depth of the RTT within the tree + * @out_rtt: Pointer to write the PA of the RTT which was destroyed + * + * Folds an RTT. If all entries with the RTT are 'homogeneous' the RTT can be + * folded into the parent and the RTT destroyed. + * + * Return: 0 on success, positive RMI result code or negative Linux error code + */ +static inline long rmi_rtt_fold(unsigned long rd, unsigned long ipa, + long level, unsigned long *out_rtt) +{ + struct arm_smccc_1_2_regs regs = { + SMC_RMI_RTT_FOLD, rd, ipa, level + }; + long ret = rmi_sro_execute(®s); + + if (ret == RMI_SUCCESS && out_rtt) + *out_rtt = regs.a1; + + return ret; +} + +/** + * rmi_rtt_init_ripas() - Set RIPAS for new realm + * @rd: PA of the RD + * @base: Base of target IPA region + * @top: Top of target IPA region + * @out_top: Top IPA of range whose RIPAS was modified + * + * Sets the RIPAS of a target IPA range to RAM, for a realm in the NEW state. + * + * Return: 0 on success, positive RMI result code or negative Linux error code + */ +static inline long rmi_rtt_init_ripas(unsigned long rd, unsigned long base, + unsigned long top, unsigned long *out_top) +{ + struct arm_smccc_1_2_regs regs = { + SMC_RMI_RTT_INIT_RIPAS, rd, base, top + }; + long ret = rmi_sro_execute(®s); + + if (ret == RMI_SUCCESS && out_top) + *out_top = regs.a1; + + return ret; +} + +/** + * rmi_rtt_unprot_map() - Map unprotected granules into a realm + * @rd: PA of the RD + * @base: Base IPA of the mapping + * @top: Top of the target IPA range + * @flags: Flags + * @oaddr: Output address set descriptor + * @out_top: Top IPA of range which has been mapped + * + * Create mappings to memory within a target unprotected IPA range. + * + * Return: 0 on success, positive RMI result code or negative Linux error code + */ +static inline long rmi_rtt_unprot_map(unsigned long rd, + unsigned long base, + unsigned long top, + unsigned long flags, + unsigned long oaddr, + unsigned long *out_top) +{ + struct arm_smccc_1_2_regs regs = { + SMC_RMI_RTT_UNPROT_MAP, rd, base, top, flags, oaddr + }; + long ret = rmi_sro_execute(®s); + + if (ret == RMI_SUCCESS && out_top) + *out_top = regs.a1; + + return ret; +} + +/** + * rmi_rtt_set_ripas() - Set RIPAS for an running realm + * @rd: PA of the RD + * @rec: PA of the REC making the request + * @base: Base of target IPA region + * @top: Top of target IPA region + * @out_top: Pointer to write top IPA of range whose RIPAS was modified + * + * Completes a request made by the realm to change the RIPAS of a target IPA + * range. + * + * Return: 0 on success, positive RMI result code or negative Linux error code + */ +static inline long rmi_rtt_set_ripas(unsigned long rd, unsigned long rec, + unsigned long base, unsigned long top, + unsigned long *out_top) +{ + struct arm_smccc_1_2_regs regs = { + SMC_RMI_RTT_SET_RIPAS, rd, rec, base, top + }; + long ret = rmi_sro_execute(®s); + + if (ret == RMI_SUCCESS && out_top) + *out_top = regs.a1; + + return ret; +} + +/** + * rmi_rtt_unprot_unmap() - Remove mappings within an unprotected IPA range + * @rd: PA of the RD + * @base: Base IPA of the mapping + * @top: Top of the target IPA range + * @flags: Flags + * @oaddr: Output address set descriptor + * @out_top: Top IPA which has been unmapped + * @out_range: Output address range + * @out_count: Number of entries in output address list + * + * Removes mappings to memory within a target unprotected IPA range. + * + * Return: 0 on success, positive RMI result code or negative Linux error code + * NOTE: out_top is only valid with ret == RMI_SUCCESS. + */ +static inline long rmi_rtt_unprot_unmap(unsigned long rd, + unsigned long base, + unsigned long top, + unsigned long flags, + unsigned long oaddr, + unsigned long *out_top, + unsigned long *out_range, + unsigned long *out_count) +{ + struct arm_smccc_1_2_regs regs = { + SMC_RMI_RTT_UNPROT_UNMAP, rd, base, top, flags, oaddr + }; + long ret = rmi_sro_execute(®s); + + if (ret != RMI_SUCCESS) + return ret; + + if (out_top) + *out_top = regs.a1; + if (out_range) + *out_range = regs.a2; + if (out_count) + *out_count = regs.a3; + + return RMI_SUCCESS; +} + +#endif /* __LINUX_ARM_RMI_CMDS_H_ */ diff --git a/include/linux/arm-rsi-cmds.h b/include/linux/arm-rsi-cmds.h new file mode 100644 index 000000000000..3f7a6a833993 --- /dev/null +++ b/include/linux/arm-rsi-cmds.h @@ -0,0 +1,239 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2023 ARM Ltd. + */ + +#ifndef __LINUX_ARM_RSI_CMDS_H_ +#define __LINUX_ARM_RSI_CMDS_H_ + +#include <linux/arm-smccc-rsi.h> +#include <linux/jump_label.h> +#include <linux/string.h> +#include <asm/memory.h> + +#ifdef CONFIG_ARM_RMM_RSI +DECLARE_STATIC_KEY_FALSE(rsi_present); + +void __init arm64_rsi_init(void); + +bool arm64_rsi_is_protected(phys_addr_t base, size_t size); + +static inline bool is_realm_world(void) +{ + return static_branch_unlikely(&rsi_present); +} +#else +static inline void arm64_rsi_init(void) { } + +static inline bool arm64_rsi_is_protected(phys_addr_t base, size_t size) +{ + return false; +} + +static inline bool is_realm_world(void) { return false; } +#endif + +#define RSI_GRANULE_SHIFT 12 +#define RSI_GRANULE_SIZE (_AC(1, UL) << RSI_GRANULE_SHIFT) + +enum ripas { + RSI_RIPAS_EMPTY = 0, + RSI_RIPAS_RAM = 1, + RSI_RIPAS_DESTROYED = 2, + RSI_RIPAS_DEV = 3, +}; + +static inline unsigned long rsi_request_version(unsigned long req, + unsigned long *out_lower, + unsigned long *out_higher) +{ + struct arm_smccc_res res; + + arm_smccc_smc(SMC_RSI_ABI_VERSION, req, 0, 0, 0, 0, 0, 0, &res); + + if (out_lower) + *out_lower = res.a1; + if (out_higher) + *out_higher = res.a2; + + return res.a0; +} + +static inline unsigned long rsi_get_realm_config(struct realm_config *cfg) +{ + struct arm_smccc_res res; + + arm_smccc_smc(SMC_RSI_REALM_CONFIG, virt_to_phys(cfg), + 0, 0, 0, 0, 0, 0, &res); + return res.a0; +} + +static inline unsigned long rsi_ipa_state_get(phys_addr_t start, + phys_addr_t end, + enum ripas *state, + phys_addr_t *top) +{ + struct arm_smccc_res res; + + arm_smccc_smc(SMC_RSI_IPA_STATE_GET, + start, end, 0, 0, 0, 0, 0, + &res); + + if (res.a0 == RSI_SUCCESS) { + if (top) + *top = res.a1; + if (state) + *state = res.a2; + } + + return res.a0; +} + +static inline long rsi_set_addr_range_state(phys_addr_t start, + phys_addr_t end, + enum ripas state, + unsigned long flags, + phys_addr_t *top) +{ + struct arm_smccc_res res; + + arm_smccc_smc(SMC_RSI_IPA_STATE_SET, start, end, state, + flags, 0, 0, 0, &res); + + if (top) + *top = res.a1; + + if (res.a2 != RSI_ACCEPT) + return -EPERM; + + return res.a0; +} + +static inline int rsi_set_memory_range(phys_addr_t start, phys_addr_t end, + enum ripas state, unsigned long flags) +{ + unsigned long ret; + phys_addr_t top; + + while (start != end) { + ret = rsi_set_addr_range_state(start, end, state, flags, &top); + if (ret || top < start || top > end) + return -EINVAL; + start = top; + } + + return 0; +} + +/* + * Convert the specified range to RAM. Do not use this if you rely on the + * contents of a page that may already be in RAM state. + */ +static inline int rsi_set_memory_range_protected(phys_addr_t start, + phys_addr_t end) +{ + return rsi_set_memory_range(start, end, RSI_RIPAS_RAM, + RSI_CHANGE_DESTROYED); +} + +/* + * Convert the specified range to RAM. Do not convert any pages that may have + * been DESTROYED, without our permission. + */ +static inline int rsi_set_memory_range_protected_safe(phys_addr_t start, + phys_addr_t end) +{ + return rsi_set_memory_range(start, end, RSI_RIPAS_RAM, + RSI_NO_CHANGE_DESTROYED); +} + +static inline int rsi_set_memory_range_shared(phys_addr_t start, + phys_addr_t end) +{ + return rsi_set_memory_range(start, end, RSI_RIPAS_EMPTY, + RSI_CHANGE_DESTROYED); +} + +#define RSI_ATTEST_CHALLENGE_MIN_SIZE 32 +#define RSI_ATTEST_CHALLENGE_MAX_SIZE 64 + +struct rsi_attestation_token_init_args { + unsigned long fid; + u8 challenge[RSI_ATTEST_CHALLENGE_MAX_SIZE]; +}; + +/** + * rsi_attestation_token_init - Initialise the operation to retrieve an + * attestation token. + * + * @challenge: The challenge data to be used in the attestation token + * generation. + * @size: Size of the challenge data in bytes. + * + * Initialises the attestation token generation and returns an upper bound + * on the attestation token size that can be used to allocate an adequate + * buffer. The caller is expected to subsequently call + * rsi_attestation_token_continue() to retrieve the attestation token data on + * the same CPU. + * + * Returns: + * On success, returns the upper limit of the attestation report size. + * Otherwise, -EINVAL + */ +static inline long +rsi_attestation_token_init(const u8 *challenge, unsigned long size) +{ + union { + struct arm_smccc_1_2_regs regs; + struct rsi_attestation_token_init_args init; + } args = { 0 }; + + if (!challenge || size < RSI_ATTEST_CHALLENGE_MIN_SIZE || + size > RSI_ATTEST_CHALLENGE_MAX_SIZE) + return -EINVAL; + + args.init.fid = SMC_RSI_ATTESTATION_TOKEN_INIT; + memcpy(args.init.challenge, challenge, size); + arm_smccc_1_2_smc(&args.regs, &args.regs); + + if (args.regs.a0 == RSI_SUCCESS) + return args.regs.a1; + + return -EINVAL; +} + +/** + * rsi_attestation_token_continue - Continue the operation to retrieve an + * attestation token. + * + * @granule: {I}PA of the Granule to which the token will be written. + * @offset: Offset within Granule to start of buffer in bytes. + * @size: The size of the buffer. + * @len: The number of bytes written to the buffer. + * + * Retrieves up to a RSI_GRANULE_SIZE worth of token data per call. The caller + * is expected to call rsi_attestation_token_init() before calling this + * function to retrieve the attestation token. + * + * Return: + * * %RSI_SUCCESS - Attestation token retrieved successfully. + * * %RSI_INCOMPLETE - Token generation is not complete. + * * %RSI_ERROR_INPUT - A parameter was not valid. + * * %RSI_ERROR_STATE - Attestation not in progress. + */ +static inline unsigned long rsi_attestation_token_continue(phys_addr_t granule, + unsigned long offset, + unsigned long size, + unsigned long *len) +{ + struct arm_smccc_res res; + + arm_smccc_1_1_invoke(SMC_RSI_ATTESTATION_TOKEN_CONTINUE, + granule, offset, size, 0, &res); + + if (len) + *len = res.a1; + return res.a0; +} + +#endif /* __LINUX_ARM_RSI_CMDS_H_ */ diff --git a/include/linux/arm-smccc-bus.h b/include/linux/arm-smccc-bus.h new file mode 100644 index 000000000000..05c0c850510e --- /dev/null +++ b/include/linux/arm-smccc-bus.h @@ -0,0 +1,48 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2026 Arm Limited + */ +#ifndef __LINUX_ARM_SMCCC_BUS_H +#define __LINUX_ARM_SMCCC_BUS_H + +#include <linux/device.h> +#include <linux/device-id/arm_smccc.h> +#include <linux/module.h> + +struct arm_smccc_device { + struct device dev; + u32 func_id; +}; + +#define to_arm_smccc_device(d) container_of_const(d, struct arm_smccc_device, dev) + +struct arm_smccc_driver { + struct device_driver driver; + + const char *name; + int (*probe)(struct arm_smccc_device *sdev); + void (*remove)(struct arm_smccc_device *sdev); + const struct arm_smccc_device_id *id_table; +}; + +#define to_arm_smccc_driver(d) \ + container_of_const(d, struct arm_smccc_driver, driver) + +int arm_smccc_driver_register(struct arm_smccc_driver *driver, + struct module *owner, const char *mod_name); +void arm_smccc_driver_unregister(struct arm_smccc_driver *driver); +struct arm_smccc_device *arm_smccc_device_register(const char *name, u32 func_id); +void arm_smccc_device_unregister(struct arm_smccc_device *smcc_dev); + +#define arm_smccc_register(driver) \ + arm_smccc_driver_register(driver, THIS_MODULE, KBUILD_MODNAME) +#define arm_smccc_unregister(driver) \ + arm_smccc_driver_unregister(driver) + +#define module_arm_smccc_driver(__arm_smccc_driver) \ + module_driver(__arm_smccc_driver, arm_smccc_register, \ + arm_smccc_unregister) + +extern const struct bus_type arm_smccc_bus_type; + +#endif /* __LINUX_ARM_SMCCC_BUS_H */ diff --git a/include/linux/arm-smccc-rmi.h b/include/linux/arm-smccc-rmi.h new file mode 100644 index 000000000000..655726b37462 --- /dev/null +++ b/include/linux/arm-smccc-rmi.h @@ -0,0 +1,516 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2023-2026 ARM Ltd. + * + * The values and structures in this file are from the Realm Management Monitor + * specification (DEN0137) version 2.0-bet3: + * https://developer.arm.com/documentation/den0137/2-0bet3/ + */ + +#ifndef __LINUX_ARM_SMCCC_RMI_H_ +#define __LINUX_ARM_SMCCC_RMI_H_ + +#include <linux/arm-smccc.h> +#include <linux/bitfield.h> +#include <linux/bits.h> +#include <linux/build_bug.h> +#include <linux/sizes.h> +#include <linux/stddef.h> + +#include <asm/page.h> + +#define SMC_RMI_CALL(func) \ + ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL, \ + ARM_SMCCC_SMC_64, \ + ARM_SMCCC_OWNER_STANDARD, \ + (func)) + +#define SMC_RMI_VERSION SMC_RMI_CALL(0x0150) + +#define SMC_RMI_RTT_DATA_MAP_INIT SMC_RMI_CALL(0x0153) + +#define SMC_RMI_REALM_ACTIVATE SMC_RMI_CALL(0x0157) +#define SMC_RMI_REALM_CREATE SMC_RMI_CALL(0x0158) +#define SMC_RMI_REALM_DESTROY SMC_RMI_CALL(0x0159) +#define SMC_RMI_REC_CREATE SMC_RMI_CALL(0x015a) +#define SMC_RMI_REC_DESTROY SMC_RMI_CALL(0x015b) +#define SMC_RMI_REC_ENTER SMC_RMI_CALL(0x015c) +#define SMC_RMI_RTT_CREATE SMC_RMI_CALL(0x015d) +#define SMC_RMI_RTT_DESTROY SMC_RMI_CALL(0x015e) + +#define SMC_RMI_RTT_READ_ENTRY SMC_RMI_CALL(0x0161) + +#define SMC_RMI_RTT_DEV_VALIDATE SMC_RMI_CALL(0x0163) +#define SMC_RMI_PSCI_COMPLETE SMC_RMI_CALL(0x0164) +#define SMC_RMI_FEATURES SMC_RMI_CALL(0x0165) +#define SMC_RMI_RTT_FOLD SMC_RMI_CALL(0x0166) + +#define SMC_RMI_RTT_INIT_RIPAS SMC_RMI_CALL(0x0168) +#define SMC_RMI_RTT_SET_RIPAS SMC_RMI_CALL(0x0169) +#define SMC_RMI_VSMMU_CREATE SMC_RMI_CALL(0x016a) +#define SMC_RMI_VSMMU_DESTROY SMC_RMI_CALL(0x016b) + +#define SMC_RMI_RMM_CONFIG_SET SMC_RMI_CALL(0x016e) +#define SMC_RMI_PSMMU_IRQ_NOTIFY SMC_RMI_CALL(0x016f) +#define SMC_RMI_ATTEST_PLAT_TOKEN_REFRESH SMC_RMI_CALL(0x0170) + +#define SMC_RMI_PDEV_ABORT SMC_RMI_CALL(0x0174) +#define SMC_RMI_PDEV_COMMUNICATE SMC_RMI_CALL(0x0175) +#define SMC_RMI_PDEV_CREATE SMC_RMI_CALL(0x0176) +#define SMC_RMI_PDEV_DESTROY SMC_RMI_CALL(0x0177) +#define SMC_RMI_PDEV_GET_STATE SMC_RMI_CALL(0x0178) + +#define SMC_RMI_PDEV_STREAM_KEY_REFRESH SMC_RMI_CALL(0x017a) +#define SMC_RMI_PDEV_SET_PUBKEY SMC_RMI_CALL(0x017b) +#define SMC_RMI_PDEV_STOP SMC_RMI_CALL(0x017c) +#define SMC_RMI_RTT_AUX_CREATE SMC_RMI_CALL(0x017d) +#define SMC_RMI_RTT_AUX_DESTROY SMC_RMI_CALL(0x017e) +#define SMC_RMI_RTT_AUX_FOLD SMC_RMI_CALL(0x017f) + +#define SMC_RMI_VDEV_ABORT SMC_RMI_CALL(0x0185) +#define SMC_RMI_VDEV_COMMUNICATE SMC_RMI_CALL(0x0186) +#define SMC_RMI_VDEV_CREATE SMC_RMI_CALL(0x0187) +#define SMC_RMI_VDEV_DESTROY SMC_RMI_CALL(0x0188) +#define SMC_RMI_VDEV_GET_STATE SMC_RMI_CALL(0x0189) +#define SMC_RMI_VDEV_UNLOCK SMC_RMI_CALL(0x018a) +#define SMC_RMI_RTT_SET_S2AP SMC_RMI_CALL(0x018b) + +#define SMC_RMI_VDEV_GET_INTERFACE_REPORT SMC_RMI_CALL(0x01d0) +#define SMC_RMI_VDEV_GET_MEASUREMENTS SMC_RMI_CALL(0x01d1) +#define SMC_RMI_VDEV_LOCK SMC_RMI_CALL(0x01d2) +#define SMC_RMI_VDEV_START SMC_RMI_CALL(0x01d3) + +#define SMC_RMI_VSMMU_EVENT_HANDLE SMC_RMI_CALL(0x01d6) +#define SMC_RMI_PSMMU_ACTIVATE SMC_RMI_CALL(0x01d7) +#define SMC_RMI_PSMMU_DEACTIVATE SMC_RMI_CALL(0x01d8) + +#define SMC_RMI_PSMMU_ST_L2_CREATE SMC_RMI_CALL(0x01db) +#define SMC_RMI_PSMMU_ST_L2_DESTROY SMC_RMI_CALL(0x01dc) +#define SMC_RMI_DPT_L0_CREATE SMC_RMI_CALL(0x01dd) +#define SMC_RMI_DPT_L0_DESTROY SMC_RMI_CALL(0x01de) +#define SMC_RMI_DPT_L1_CREATE SMC_RMI_CALL(0x01df) +#define SMC_RMI_DPT_L1_DESTROY SMC_RMI_CALL(0x01e0) +#define SMC_RMI_GRANULE_TRACKING_GET SMC_RMI_CALL(0x01e1) + +#define SMC_RMI_GRANULE_TRACKING_SET SMC_RMI_CALL(0x01e3) + +#define SMC_RMI_RMM_CONFIG_GET SMC_RMI_CALL(0x01ec) + +#define SMC_RMI_RMM_STATE_GET SMC_RMI_CALL(0x01ee) + +#define SMC_RMI_PSMMU_EVENT_CONSUME SMC_RMI_CALL(0x01f0) +#define SMC_RMI_GRANULE_RANGE_DELEGATE SMC_RMI_CALL(0x01f1) +#define SMC_RMI_GRANULE_RANGE_UNDELEGATE SMC_RMI_CALL(0x01f2) +#define SMC_RMI_GPT_L1_CREATE SMC_RMI_CALL(0x01f3) +#define SMC_RMI_GPT_L1_DESTROY SMC_RMI_CALL(0x01f4) +#define SMC_RMI_RTT_DATA_MAP SMC_RMI_CALL(0x01f5) +#define SMC_RMI_RTT_DATA_UNMAP SMC_RMI_CALL(0x01f6) +#define SMC_RMI_RTT_DEV_MAP SMC_RMI_CALL(0x01f7) +#define SMC_RMI_RTT_DEV_UNMAP SMC_RMI_CALL(0x01f8) +#define SMC_RMI_RTT_ARCH_DEV_MAP SMC_RMI_CALL(0x01f9) +#define SMC_RMI_RTT_ARCH_DEV_UNMAP SMC_RMI_CALL(0x01fa) +#define SMC_RMI_RTT_UNPROT_MAP SMC_RMI_CALL(0x01fb) +#define SMC_RMI_RTT_UNPROT_UNMAP SMC_RMI_CALL(0x01fc) +#define SMC_RMI_RTT_AUX_PROT_MAP SMC_RMI_CALL(0x01fd) +#define SMC_RMI_RTT_AUX_PROT_UNMAP SMC_RMI_CALL(0x01fe) +#define SMC_RMI_RTT_AUX_UNPROT_MAP SMC_RMI_CALL(0x01ff) +#define SMC_RMI_RTT_AUX_UNPROT_UNMAP SMC_RMI_CALL(0x0200) +#define SMC_RMI_REALM_TERMINATE SMC_RMI_CALL(0x0201) +#define SMC_RMI_RMM_ACTIVATE SMC_RMI_CALL(0x0202) +#define SMC_RMI_OP_CONTINUE SMC_RMI_CALL(0x0203) +#define SMC_RMI_PDEV_STREAM_CONNECT SMC_RMI_CALL(0x0204) +#define SMC_RMI_PDEV_STREAM_DISCONNECT SMC_RMI_CALL(0x0205) +#define SMC_RMI_PDEV_STREAM_COMPLETE SMC_RMI_CALL(0x0206) +#define SMC_RMI_PDEV_STREAM_KEY_PURGE SMC_RMI_CALL(0x0207) +#define SMC_RMI_OP_MEM_DONATE SMC_RMI_CALL(0x0208) +#define SMC_RMI_OP_MEM_RECLAIM SMC_RMI_CALL(0x0209) +#define SMC_RMI_OP_CANCEL SMC_RMI_CALL(0x020a) +#define SMC_RMI_VSMMU_FEATURES SMC_RMI_CALL(0x020b) +#define SMC_RMI_VSMMU_CMD_GET SMC_RMI_CALL(0x020c) +#define SMC_RMI_VSMMU_CMD_COMPLETE SMC_RMI_CALL(0x020d) +#define SMC_RMI_PSMMU_INFO SMC_RMI_CALL(0x020e) +#define SMC_RMI_RMM_DEACTIVATE SMC_RMI_CALL(0x020f) +#define SMC_RMI_PDEV_STREAM_INFO SMC_RMI_CALL(0x0210) +#define SMC_RMI_GPT_INFO SMC_RMI_CALL(0x0211) + +#define RMI_ABI_MAJOR_VERSION 2 +#define RMI_ABI_MINOR_VERSION 0 + +#define RMI_ABI_VERSION_MAJOR_MASK GENMASK(30, 16) +#define RMI_ABI_VERSION_MINOR_MASK GENMASK(15, 0) + +#define RMI_ABI_VERSION_GET_MAJOR(v) FIELD_GET(RMI_ABI_VERSION_MAJOR_MASK, (v)) +#define RMI_ABI_VERSION_GET_MINOR(v) FIELD_GET(RMI_ABI_VERSION_MINOR_MASK, (v)) +#define RMI_ABI_VERSION(major, minor) \ + (FIELD_PREP(RMI_ABI_VERSION_MAJOR_MASK, (major)) | \ + FIELD_PREP(RMI_ABI_VERSION_MINOR_MASK, (minor))) + +/* RmiResult Type definitions */ +#define RMI_RESULT_STATUS_MASK GENMASK(7, 0) +/* RmiResultDataLevel field */ +#define RMI_RESULT_DATA_LEVEL_MASK GENMASK(15, 8) +/* RmiResultDataIncomplete fields */ +#define RMI_RESULT_MEMREQ_MASK GENMASK(9, 8) +#define RMI_RESULT_CAN_CANCEL_MASK BIT(10) + +#define RMI_RESULT_STATUS(ret) FIELD_GET(RMI_RESULT_STATUS_MASK, ret) +#define RMI_RESULT_DATA_LEVEL(ret) FIELD_GET(RMI_RESULT_DATA_LEVEL_MASK, ret) +#define RMI_RESULT_MEMREQ(ret) FIELD_GET(RMI_RESULT_MEMREQ_MASK, ret) +#define RMI_RESULT_CAN_CANCEL(ret) FIELD_GET(RMI_RESULT_CAN_CANCEL_MASK, ret) + +#define RMI_SUCCESS 0 +#define RMI_ERROR_INPUT 1 +#define RMI_ERROR_REALM 2 +#define RMI_ERROR_REC 3 +#define RMI_ERROR_RTT 4 +#define RMI_ERROR_NOT_SUPPORTED 5 +#define RMI_ERROR_DEVICE 6 +#define RMI_ERROR_RTT_AUX 7 +#define RMI_ERROR_PSMMU_ST 8 +#define RMI_ERROR_DPT 9 +#define RMI_BUSY 10 +#define RMI_ERROR_GLOBAL 11 +#define RMI_ERROR_TRACKING 12 +#define RMI_INCOMPLETE 13 +#define RMI_BLOCKED 14 +#define RMI_ERROR_GPT 15 +#define RMI_ERROR_GRANULE 16 + +#define RMI_CONTINUE_KEEP_GOING 0 +#define RMI_CONTINUE_STOP 1 + +#define RMI_OP_MEM_REQ_NONE 0 +#define RMI_OP_MEM_REQ_DONATE 1 +#define RMI_OP_MEM_REQ_RECLAIM 2 + +#define RMI_OP_CANNOT_CANCEL 0 +#define RMI_OP_CAN_CANCEL 1 + +#define RMI_DONATE_STATE_MASK GENMASK(18, 17) +#define RMI_DONATE_CONTIG_MASK BIT(16) +#define RMI_DONATE_COUNT_MASK GENMASK(15, 2) +#define RMI_DONATE_BLOCK_SIZE_MASK GENMASK(1, 0) + +#define RMI_DONATE_STATE(req) FIELD_GET(RMI_DONATE_STATE_MASK, req) +#define RMI_DONATE_CONTIG(req) FIELD_GET(RMI_DONATE_CONTIG_MASK, req) +#define RMI_DONATE_COUNT(req) FIELD_GET(RMI_DONATE_COUNT_MASK, req) +#define RMI_DONATE_BLOCK_SIZE(req) FIELD_GET(RMI_DONATE_BLOCK_SIZE_MASK, req) + +#define RMI_OP_MEM_DELEGATED 0 +#define RMI_OP_MEM_UNDELEGATED 1 +#define RMI_OP_MEM_CONDITIONAL 2 + +#define RMI_OP_MEM_NON_CONTIG 0 +#define RMI_OP_MEM_CONTIG 1 + +#define RMI_ADDR_TYPE_NONE 0 +#define RMI_ADDR_TYPE_SINGLE 1 +#define RMI_ADDR_TYPE_LIST 2 + +#define RMI_ADDR_RANGE_STATE_MASK GENMASK_ULL(63, 62) +#define RMI_ADDR_RANGE_ADDR_MASK GENMASK_ULL(51, PAGE_SHIFT) +#define RMI_ADDR_RANGE_COUNT_MASK GENMASK(PAGE_SHIFT - 1, 2) +#define RMI_ADDR_RANGE_BLOCK_SIZE_MASK GENMASK(1, 0) + +#define RMI_ADDR_RANGE_STATE(r) FIELD_GET(RMI_ADDR_RANGE_STATE_MASK, (r)) +#define RMI_ADDR_RANGE_ADDR(r) ((r) & RMI_ADDR_RANGE_ADDR_MASK) +#define RMI_ADDR_RANGE_COUNT(r) FIELD_GET(RMI_ADDR_RANGE_COUNT_MASK, (r)) +#define RMI_ADDR_RANGE_BLOCK_SIZE(r) FIELD_GET(RMI_ADDR_RANGE_BLOCK_SIZE_MASK, (r)) + +enum rmi_ripas { + RMI_EMPTY = 0, + RMI_RAM = 1, + RMI_DESTROYED = 2, + RMI_DEV = 3, +}; + +#define RMI_NO_MEASURE_CONTENT 0 +#define RMI_MEASURE_CONTENT 1 + +#define RMI_FEATURE_REGISTER_0_S2OASZ GENMASK_ULL(40, 33) +#define RMI_FEATURE_REGISTER_0_L0GPT_BLOCK_DELEGATE BIT_ULL(32) +#define RMI_FEATURE_REGISTER_0_PMU_NUM_CTRS GENMASK(31, 27) +#define RMI_FEATURE_REGISTER_0_PMU BIT(26) +#define RMI_FEATURE_REGISTER_0_NUM_WPS GENMASK(25, 20) +#define RMI_FEATURE_REGISTER_0_NUM_BPS GENMASK(19, 14) +#define RMI_FEATURE_REGISTER_0_SVE_VL GENMASK(13, 10) +#define RMI_FEATURE_REGISTER_0_SVE BIT(9) +#define RMI_FEATURE_REGISTER_0_LPA2 BIT(8) +#define RMI_FEATURE_REGISTER_0_S2SZ GENMASK(7, 0) + +#define RMI_FEATURE_REGISTER_1_PPS GENMASK(16, 14) +#define RMI_FEATURE_REGISTER_1_L0GPTSZ GENMASK(13, 10) +#define RMI_FEATURE_REGISTER_1_MAX_RECS_ORDER GENMASK(9, 6) +#define RMI_FEATURE_REGISTER_1_HASH_SHA_512 BIT(5) +#define RMI_FEATURE_REGISTER_1_HASH_SHA_384 BIT(4) +#define RMI_FEATURE_REGISTER_1_HASH_SHA_256 BIT(3) +#define RMI_FEATURE_REGISTER_1_RMI_GRAN_SZ_64KB BIT(2) +#define RMI_FEATURE_REGISTER_1_RMI_GRAN_SZ_16KB BIT(1) +#define RMI_FEATURE_REGISTER_1_RMI_GRAN_SZ_4KB BIT(0) + +#define RMI_FEATURE_REGISTER_2_REALM_MAX_VDEVS_ORDER GENMASK(14, 10) +#define RMI_FEATURE_REGISTER_2_NON_TEE_STREAM BIT(9) +#define RMI_FEATURE_REGISTER_2_VDEV_KROU BIT(8) +#define RMI_FEATURE_REGISTER_2_PDEV_MAX_VDEVS_ORDER GENMASK(7, 4) +#define RMI_FEATURE_REGISTER_2_ATS BIT(3) +#define RMI_FEATURE_REGISTER_2_VSMMU BIT(2) +#define RMI_FEATURE_REGISTER_2_DA_COH BIT(1) +#define RMI_FEATURE_REGISTER_2_DA BIT(0) + +#define RMI_FEATURE_REGISTER_3_RTT_S2AP_INDIRECT BIT(6) +#define RMI_FEATURE_REGISTER_3_RTT_PLANE GENMASK(5, 4) +#define RMI_FEATURE_REGISTER_3_MAX_NUM_AUX_PLANES GENMASK(3, 0) + +#define RMI_FEATURE_REGISTER_4_MEC_COUNT GENMASK_ULL(63, 0) + +#define RMI_MEM_CATEGORY_CONVENTIONAL 0 +#define RMI_MEM_CATEGORY_DEV_NCOH 1 +#define RMI_MEM_CATEGORY_DEV_COH 2 +#define RMI_MEM_CATEGORY_NONE 3 + +#define RMI_TRACKING_RESERVED 0 +#define RMI_TRACKING_NONE 1 +#define RMI_TRACKING_FINE 2 +#define RMI_TRACKING_COARSE 3 +#define RMI_TRACKING_INTERMEDIATE 4 + +#define RMI_GRANULE_SIZE_4KB 0 +#define RMI_GRANULE_SIZE_16KB 1 +#define RMI_GRANULE_SIZE_64KB 2 + +#define RMI_GPT_PAR_RESERVED 0U +#define RMI_GPT_PAR_PLAT 1U +#define RMI_GPT_PAR_HOST_NOT_CREATED 2U +#define RMI_GPT_PAR_HOST_CREATED 3U + +/* + * Note many of these fields are smaller than u64 but all fields have u64 + * alignment, so use u64 to ensure correct alignment. + */ +struct rmm_config { + union { /* 0x0 */ + struct { + u64 tracking_region_size; + u64 rmi_granule_size; + }; + u8 sizer[SZ_4K]; + }; +}; + +static_assert(sizeof(struct rmm_config) == SZ_4K); + +#define RMI_REALM_PARAM_FLAG_MEC_POLICY_MASK GENMASK(8, 7) +#define RMI_REALM_PARAM_FLAG_LFA_POLICY_MASK GENMASK(6, 5) +#define RMI_REALM_PARAM_FLAG_DA BIT(3) +#define RMI_REALM_PARAM_FLAG_PMU BIT(2) +#define RMI_REALM_PARAM_FLAG_SVE BIT(1) + +#define RMI_HASH_SHA_256 0 +#define RMI_HASH_SHA_512 1 +#define RMI_HASH_SHA_384 2 + +struct realm_params { + union { /* 0x0 */ + struct { + u64 flags0; + u64 s2sz; + u64 sve_vl; + u64 num_bps; + u64 num_wps; + u64 pmu_num_ctrs; + u64 hash_algo; + u64 num_aux_planes; + }; + u8 sizer0[0x400]; + }; + union { /* 0x400 */ + struct { + u8 rpv[64]; + u64 ats_plane; + }; + u8 sizer1[0x400]; + }; + union { /* 0x800 */ + struct { + u64 sizer2; + u64 rtt_base; + s64 rtt_level_start; + u64 rtt_num_start; + u64 flags1; + u64 max_num_vdevs; + }; + u8 sizer3[0x700]; + }; + union { /* 0xf00 */ + struct { + u8 sizer4[0x80]; + u64 aux_rtt_base[3]; + }; + u8 sizer5[0x100]; + }; +}; + +static_assert(offsetof(struct realm_params, rpv) == 0x400); +static_assert(offsetof(struct realm_params, rtt_base) == 0x808); +static_assert(offsetof(struct realm_params, aux_rtt_base) == 0xf80); +static_assert(sizeof(struct realm_params) == SZ_4K); + +/* + * The number of GPRs (starting from X0) that are configured by the host when + * a REC is created. + */ +#define REC_CREATE_NR_GPRS 8 + +#define REC_PARAMS_FLAG_RUNNABLE BIT(0) + +struct rec_params { + union { /* 0x0 */ + u64 flags; + u8 sizer0[0x100]; + }; + union { /* 0x100 */ + u64 mpidr; + u8 sizer1[0x100]; + }; + union { /* 0x200 */ + u64 pc; + u8 sizer2[0x100]; + }; + union { /* 0x300 */ + u64 gprs[REC_CREATE_NR_GPRS]; + u8 sizer3[0xd00]; + }; +}; + +static_assert(offsetof(struct rec_params, mpidr) == 0x100); +static_assert(offsetof(struct rec_params, pc) == 0x200); +static_assert(offsetof(struct rec_params, gprs) == 0x300); +static_assert(sizeof(struct rec_params) == SZ_4K); + +#define REC_ENTER_FLAG_FORCE_P0 BIT(7) +#define REC_ENTER_FLAG_DEV_MEM_RESPONSE BIT(6) +#define REC_ENTER_FLAG_S2AP_RESPONSE BIT(5) +#define REC_ENTER_FLAG_RIPAS_RESPONSE BIT(4) +#define REC_ENTER_FLAG_TRAP_WFE BIT(3) +#define REC_ENTER_FLAG_TRAP_WFI BIT(2) +#define REC_ENTER_FLAG_INJECT_SEA BIT(1) +#define REC_ENTER_FLAG_EMULATED_MMIO BIT(0) + +#define REC_RUN_GPRS 31 + +struct rec_enter { + union { /* 0x000 */ + u64 flags; + u8 sizer0[0x200]; + }; + union { /* 0x200 */ + u64 gprs[REC_RUN_GPRS]; + u8 sizer1[0x600]; + }; +}; + +static_assert(offsetof(struct rec_enter, gprs) == 0x200); +static_assert(sizeof(struct rec_enter) == SZ_2K); + +#define RMI_EXIT_SYNC 0x00 +#define RMI_EXIT_IRQ 0x01 +#define RMI_EXIT_FIQ 0x02 +#define RMI_EXIT_PSCI 0x03 +#define RMI_EXIT_RIPAS_CHANGE 0x04 +#define RMI_EXIT_HOST_CALL 0x05 +#define RMI_EXIT_SERROR 0x06 +#define RMI_EXIT_S2AP_CHANGE 0x07 +#define RMI_EXIT_VDEV_VALIDATE_MAPPING 0x08 +#define RMI_EXIT_VSMMU_COMMAND 0x0a + +struct rec_exit { + union { /* 0x000 */ + u8 exit_reason; + u8 sizer0[0x100]; + }; + union { /* 0x100 */ + struct { + u64 esr; + u64 far; + u64 hpfar; + u64 rtt_tree; + }; + u8 sizer1[0x100]; + }; + union { /* 0x200 */ + u64 gprs[REC_RUN_GPRS]; + u8 sizer2[0x200]; + }; + union { /* 0x400 */ + struct { + u64 cntp_ctl; + u64 cntp_cval; + u64 cntv_ctl; + u64 cntv_cval; + }; + u8 sizer4[0x100]; + }; + union { /* 0x500 */ + struct { + u64 ripas_base; + u64 ripas_top; + u8 ripas_value; + u8 sizer5[0xf]; + u64 s2ap_base; + u64 s2ap_top; + u64 vdev_id_1; + u64 vdev_id_2; + u64 dev_mem_base; + u64 dev_mem_top; + u64 dev_mem_pa; + }; + u8 sizer6[0x100]; + }; + union { /* 0x600 */ + struct { + u16 imm; + u8 sizer7[0x6]; + u64 plane; + }; + u8 sizer8[0x100]; + }; + union { /* 0x700 */ + struct { + u8 pmu_ovf_status; + u8 sizer9[0xf]; + u64 vsmmu; + }; + u8 sizer10[0x100]; + }; +}; + +static_assert(sizeof(struct rec_exit) == SZ_2K); +static_assert(offsetof(struct rec_exit, esr) == 0x100); +static_assert(offsetof(struct rec_exit, gprs) == 0x200); +static_assert(offsetof(struct rec_exit, cntp_ctl) == 0x400); +static_assert(offsetof(struct rec_exit, ripas_base) == 0x500); +static_assert(offsetof(struct rec_exit, imm) == 0x600); +static_assert(offsetof(struct rec_exit, pmu_ovf_status) == 0x700); + +struct rec_run { + struct rec_enter enter; + struct rec_exit exit; +}; + +static_assert(sizeof(struct rec_run) == SZ_4K); + +/* RMI_RTT_UNPROT_MAP_FLAGS definitions */ +#define RMI_RTT_UNPROT_MAP_FLAGS_S2AP_MASK GENMASK(22, 19) +#define RMI_RTT_UNPROT_MAP_FLAGS_MEMATTR_MASK GENMASK(18, 16) +#define RMI_RTT_UNPROT_MAP_FLAGS_LIST_COUNT_MASK GENMASK(15, 2) +#define RMI_RTT_UNPROT_MAP_FLAGS_OADDR_TYPE_MASK GENMASK(1, 0) + +/* RMI_RTT_PROT_MAP_FLAGS definitions */ +#define RMI_RTT_PROT_MAP_FLAGS_LIST_COUNT_MASK GENMASK(15, 2) +#define RMI_RTT_PROT_MAP_FLAGS_OADDR_TYPE_MASK GENMASK(1, 0) + +/* S2AP Direct Encodings, used in RMI_RTT_UNPROT_MAP_FLAGS_S2AP */ +#define RMI_S2AP_DIRECT_READ BIT(1) +#define RMI_S2AP_DIRECT_WRITE BIT(0) + +#endif /* __LINUX_ARM_SMCCC_RMI_H_ */ diff --git a/include/linux/arm-smccc-rsi.h b/include/linux/arm-smccc-rsi.h new file mode 100644 index 000000000000..cca881ddfdba --- /dev/null +++ b/include/linux/arm-smccc-rsi.h @@ -0,0 +1,193 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2023 ARM Ltd. + */ + +#ifndef __LINUX_ARM_SMCCC_RSI_H_ +#define __LINUX_ARM_SMCCC_RSI_H_ + +#include <linux/arm-smccc.h> + +/* + * This file describes the Realm Services Interface (RSI) Application Binary + * Interface (ABI) for SMC calls made from within the Realm to the RMM and + * serviced by the RMM. + */ + +/* + * The major version number of the RSI implementation. This is increased when + * the binary format or semantics of the SMC calls change. + */ +#define RSI_ABI_VERSION_MAJOR UL(1) + +/* + * The minor version number of the RSI implementation. This is increased when + * a bug is fixed, or a feature is added without breaking binary compatibility. + */ +#define RSI_ABI_VERSION_MINOR UL(0) + +#define RSI_ABI_VERSION ((RSI_ABI_VERSION_MAJOR << 16) | \ + RSI_ABI_VERSION_MINOR) + +#define RSI_ABI_VERSION_GET_MAJOR(_version) ((_version) >> 16) +#define RSI_ABI_VERSION_GET_MINOR(_version) ((_version) & 0xFFFF) + +#define RSI_SUCCESS UL(0) +#define RSI_ERROR_INPUT UL(1) +#define RSI_ERROR_STATE UL(2) +#define RSI_INCOMPLETE UL(3) +#define RSI_ERROR_UNKNOWN UL(4) + +#define SMC_RSI_FID(n) ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL, \ + ARM_SMCCC_SMC_64, \ + ARM_SMCCC_OWNER_STANDARD, \ + n) + +/* + * Returns RSI version. + * + * arg1 == Requested interface revision + * ret0 == Status / error + * ret1 == Lower implemented interface revision + * ret2 == Higher implemented interface revision + */ +#define SMC_RSI_ABI_VERSION SMC_RSI_FID(0x190) + +/* + * Read feature register. + * + * arg1 == Feature register index + * ret0 == Status / error + * ret1 == Feature register value + */ +#define SMC_RSI_FEATURES SMC_RSI_FID(0x191) + +/* + * Read measurement for the current Realm. + * + * arg1 == Index, which measurements slot to read + * ret0 == Status / error + * ret1 == Measurement value, bytes: 0 - 7 + * ret2 == Measurement value, bytes: 8 - 15 + * ret3 == Measurement value, bytes: 16 - 23 + * ret4 == Measurement value, bytes: 24 - 31 + * ret5 == Measurement value, bytes: 32 - 39 + * ret6 == Measurement value, bytes: 40 - 47 + * ret7 == Measurement value, bytes: 48 - 55 + * ret8 == Measurement value, bytes: 56 - 63 + */ +#define SMC_RSI_MEASUREMENT_READ SMC_RSI_FID(0x192) + +/* + * Extend Realm Extensible Measurement (REM) value. + * + * arg1 == Index, which measurements slot to extend + * arg2 == Size of realm measurement in bytes, max 64 bytes + * arg3 == Measurement value, bytes: 0 - 7 + * arg4 == Measurement value, bytes: 8 - 15 + * arg5 == Measurement value, bytes: 16 - 23 + * arg6 == Measurement value, bytes: 24 - 31 + * arg7 == Measurement value, bytes: 32 - 39 + * arg8 == Measurement value, bytes: 40 - 47 + * arg9 == Measurement value, bytes: 48 - 55 + * arg10 == Measurement value, bytes: 56 - 63 + * ret0 == Status / error + */ +#define SMC_RSI_MEASUREMENT_EXTEND SMC_RSI_FID(0x193) + +/* + * Initialize the operation to retrieve an attestation token. + * + * arg1 == Challenge value, bytes: 0 - 7 + * arg2 == Challenge value, bytes: 8 - 15 + * arg3 == Challenge value, bytes: 16 - 23 + * arg4 == Challenge value, bytes: 24 - 31 + * arg5 == Challenge value, bytes: 32 - 39 + * arg6 == Challenge value, bytes: 40 - 47 + * arg7 == Challenge value, bytes: 48 - 55 + * arg8 == Challenge value, bytes: 56 - 63 + * ret0 == Status / error + * ret1 == Upper bound of token size in bytes + */ +#define SMC_RSI_ATTESTATION_TOKEN_INIT SMC_RSI_FID(0x194) + +/* + * Continue the operation to retrieve an attestation token. + * + * arg1 == The IPA of token buffer + * arg2 == Offset within the granule of the token buffer + * arg3 == Size of the granule buffer + * ret0 == Status / error + * ret1 == Length of token bytes copied to the granule buffer + */ +#define SMC_RSI_ATTESTATION_TOKEN_CONTINUE SMC_RSI_FID(0x195) + +#ifndef __ASSEMBLER__ + +struct realm_config { + union { + struct { + unsigned long ipa_bits; /* Width of IPA in bits */ + u8 hash_algo; /* Hash algorithm */ + }; + u8 pad[0x200]; + }; + union { + u8 rpv[64]; /* Realm Personalization Value */ + u8 pad2[0xe00]; + }; + /* + * The RMM requires the configuration structure to be aligned to a 4k + * boundary, ensure this happens by aligning this structure. + */ +} __aligned(0x1000); + +#endif /* __ASSEMBLER__ */ + +/* + * Read configuration for the current Realm. + * + * arg1 == struct realm_config addr + * ret0 == Status / error + */ +#define SMC_RSI_REALM_CONFIG SMC_RSI_FID(0x196) + +/* + * Request RIPAS of a target IPA range to be changed to a specified value. + * + * arg1 == Base IPA address of target region + * arg2 == Top of the region + * arg3 == RIPAS value + * arg4 == flags + * ret0 == Status / error + * ret1 == Top of modified IPA range + * ret2 == Whether the Host accepted or rejected the request + */ +#define SMC_RSI_IPA_STATE_SET SMC_RSI_FID(0x197) + +#define RSI_NO_CHANGE_DESTROYED UL(0) +#define RSI_CHANGE_DESTROYED UL(1) + +#define RSI_ACCEPT UL(0) +#define RSI_REJECT UL(1) + +/* + * Get RIPAS of a target IPA range. + * + * arg1 == Base IPA of target region + * arg2 == End of target IPA region + * ret0 == Status / error + * ret1 == Top of IPA region which has the reported RIPAS value + * ret2 == RIPAS value + */ +#define SMC_RSI_IPA_STATE_GET SMC_RSI_FID(0x198) + +/* + * Make a Host call. + * + * arg1 == IPA of host call structure + * ret0 == Status / error + */ +#define SMC_RSI_HOST_CALL SMC_RSI_FID(0x199) + +#endif /* __LINUX_ARM_SMCCC_RSI_H_ */ diff --git a/include/linux/arm_mpam.h b/include/linux/arm_mpam.h index f92a36187a52..002f56e15362 100644 --- a/include/linux/arm_mpam.h +++ b/include/linux/arm_mpam.h @@ -12,7 +12,7 @@ struct mpam_msc; enum mpam_msc_iface { MPAM_IFACE_MMIO, /* a real MPAM MSC */ - MPAM_IFACE_PCC, /* a fake MPAM MSC */ + MPAM_IFACE_PCC, /* using the MPAM-Fb firmware redirection */ }; enum mpam_class_types { diff --git a/include/linux/audit.h b/include/linux/audit.h index 45abb3722d30..767e349d4834 100644 --- a/include/linux/audit.h +++ b/include/linux/audit.h @@ -449,6 +449,7 @@ extern void __audit_tk_injoffset(struct timespec64 offset); extern void __audit_ntp_log(const struct audit_ntp_data *ad); extern void __audit_log_nfcfg(const char *name, u8 af, unsigned int nentries, enum audit_nfcfgop op, gfp_t gfp); +extern void __audit_log_fsopen(const char *fs_name); static inline void audit_ipc_obj(struct kern_ipc_perm *ipcp) { @@ -547,7 +548,7 @@ static inline void audit_openat2_how(struct open_how *how) static inline void audit_log_kern_module(const char *name) { - if (!audit_dummy_context()) + if (unlikely(!audit_dummy_context())) __audit_log_kern_module(name); } @@ -563,7 +564,7 @@ static inline void audit_tk_injoffset(struct timespec64 offset) if (offset.tv_sec == 0 && offset.tv_nsec == 0) return; - if (!audit_dummy_context()) + if (unlikely(!audit_dummy_context())) __audit_tk_injoffset(offset); } @@ -586,7 +587,7 @@ static inline void audit_ntp_set_new(struct audit_ntp_data *ad, static inline void audit_ntp_log(const struct audit_ntp_data *ad) { - if (!audit_dummy_context()) + if (unlikely(!audit_dummy_context())) __audit_ntp_log(ad); } @@ -598,6 +599,12 @@ static inline void audit_log_nfcfg(const char *name, u8 af, __audit_log_nfcfg(name, af, nentries, op, gfp); } +static inline void audit_log_fsopen(const char *fs_name) +{ + if (!audit_dummy_context()) + __audit_log_fsopen(fs_name); +} + extern int audit_n_rules; extern int audit_signals; #else /* CONFIG_AUDITSYSCALL */ @@ -730,6 +737,9 @@ static inline void audit_log_nfcfg(const char *name, u8 af, enum audit_nfcfgop op, gfp_t gfp) { } +static inline void audit_log_fsopen(const char *fs_name) +{ } + #define audit_n_rules 0 #define audit_signals 0 #endif /* CONFIG_AUDITSYSCALL */ diff --git a/include/linux/binfmts.h b/include/linux/binfmts.h index f686a37f7a0a..2e87faf9a8c2 100644 --- a/include/linux/binfmts.h +++ b/include/linux/binfmts.h @@ -128,7 +128,8 @@ struct linux_binfmt { struct module *module; int (*load_binary)(struct linux_binprm *); #ifdef CONFIG_COREDUMP - int (*core_dump)(struct coredump_params *cprm); + /* Returns true if the whole coredump was written. */ + bool (*core_dump)(struct coredump_params *cprm); unsigned long min_coredump; /* minimal dump size */ #endif } __randomize_layout; diff --git a/include/linux/bio-integrity.h b/include/linux/bio-integrity.h index 0ea2a8bf7efb..a954c97be0b3 100644 --- a/include/linux/bio-integrity.h +++ b/include/linux/bio-integrity.h @@ -151,7 +151,6 @@ void bio_integrity_setup_default(struct bio *bio); unsigned int fs_bio_integrity_alloc(struct bio *bio); void fs_bio_integrity_free(struct bio *bio); void fs_bio_integrity_generate(struct bio *bio); -int fs_bio_integrity_verify(struct bio *bio, sector_t sector, - unsigned int size); +int fs_bio_integrity_verify(struct bio *bio, struct bvec_iter *data_iter); #endif /* _LINUX_BIO_INTEGRITY_H */ diff --git a/include/linux/bio.h b/include/linux/bio.h index bb3235497e67..7a794ce723b8 100644 --- a/include/linux/bio.h +++ b/include/linux/bio.h @@ -80,7 +80,8 @@ static inline bool bio_no_advance_iter(const struct bio *bio) { return bio_op(bio) == REQ_OP_DISCARD || bio_op(bio) == REQ_OP_SECURE_ERASE || - bio_op(bio) == REQ_OP_WRITE_ZEROES; + bio_op(bio) == REQ_OP_WRITE_ZEROES || + op_is_dmabuf(bio->bi_opf); } static inline void *bio_data(struct bio *bio) @@ -113,11 +114,13 @@ static inline void bio_advance_iter(const struct bio *bio, { iter->bi_sector += bytes >> 9; - if (bio_no_advance_iter(bio)) + if (bio_no_advance_iter(bio)) { iter->bi_size -= bytes; - else + iter->bi_offset += bytes; + } else { bvec_iter_advance(bio->bi_io_vec, iter, bytes); /* TODO: It is reasonable to complete bio with error here. */ + } } /* @bytes should be less or equal to bvec[i->bi_idx].bv_len */ @@ -127,10 +130,12 @@ static inline void bio_advance_iter_single(const struct bio *bio, { iter->bi_sector += bytes >> 9; - if (bio_no_advance_iter(bio)) + if (bio_no_advance_iter(bio)) { iter->bi_size -= bytes; - else + iter->bi_offset += bytes; + } else { bvec_iter_advance_single(bio->bi_io_vec, iter, bytes); + } } void __bio_advance(struct bio *, unsigned bytes); @@ -434,12 +439,12 @@ static inline void bio_wouldblock_error(struct bio *bio) /* * Calculate number of bvec segments that should be allocated to fit data - * pointed by @iter. If @iter is backed by bvec it's going to be reused - * instead of allocating a new one. + * pointed by @iter. If @iter is backed by a bvec or a dmabuf, the bvec array / + * the dma map are going to be reused, and so no extra allocation is required. */ static inline int bio_iov_vecs_to_alloc(struct iov_iter *iter, int max_segs) { - if (iov_iter_is_bvec(iter)) + if (iov_iter_is_bvec(iter) || iov_iter_is_dmabuf_map(iter)) return 0; return iov_iter_npages(iter, max_segs); } @@ -479,6 +484,7 @@ static inline void bio_init_inline(struct bio *bio, struct block_device *bdev, extern void bio_uninit(struct bio *); void bio_reset(struct bio *bio, struct block_device *bdev, blk_opf_t opf); void bio_reuse(struct bio *bio, blk_opf_t opf); +void bio_prepare_reissue(struct bio *bio, struct block_device *bdev); void bio_chain(struct bio *, struct bio *); void bio_await(struct bio *bio, void *priv, void (*submit)(struct bio *bio, void *priv)); @@ -516,16 +522,18 @@ int bdev_rw_virt(struct block_device *bdev, sector_t sector, void *data, size_t len, enum req_op op); int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter, - unsigned mem_align_mask, unsigned len_align_mask); + unsigned maxlen, unsigned mem_align_mask, + unsigned len_align_mask); bool bio_iov_iter_set(struct bio *bio, const struct iov_iter *iter); void __bio_release_pages(struct bio *bio, bool mark_dirty); extern void bio_set_pages_dirty(struct bio *bio); extern void bio_check_pages_dirty(struct bio *bio); -int bio_iov_iter_bounce(struct bio *bio, struct iov_iter *iter, size_t maxlen, - size_t minsize); -void bio_iov_iter_unbounce(struct bio *bio, bool is_error, bool mark_dirty); +int bio_alloc_bounce_folios(struct bio *bio, size_t total_len, size_t minsize); +void bio_free_folios(struct bio *bio); +int bio_iov_iter_bounce_write(struct bio *bio, struct iov_iter *iter, + size_t maxlen, size_t minsize); extern void bio_copy_data(struct bio *dst, struct bio *src); extern void bio_free_pages(struct bio *bio); diff --git a/include/linux/bitfield-fix-width.h b/include/linux/bitfield-fix-width.h new file mode 100644 index 000000000000..f73b8cd65601 --- /dev/null +++ b/include/linux/bitfield-fix-width.h @@ -0,0 +1,241 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#ifndef _LINUX_BITFIELD_FIX_WIDTH_H +#define _LINUX_BITFIELD_FIX_WIDTH_H + +#include <linux/compiler.h> +#include <linux/types.h> +#include <asm/byteorder.h> + +extern void __compiletime_error("value doesn't fit into mask") +__field_overflow(void); + +extern void __compiletime_error("bad bitfield mask") +__bad_mask(void); + +static __always_inline +u64 field_multiplier(u64 field) +{ + if ((field | (field - 1)) & ((field | (field - 1)) + 1)) + __bad_mask(); + return field & -field; +} + +static __always_inline +u64 field_mask(u64 field) +{ + return field / field_multiplier(field); +} + +#define field_max(field) ((typeof(field))field_mask(field)) + +#define __assert_field(v, field) \ + do { \ + if (__builtin_constant_p(v) && \ + ((v) & ~field_mask(field))) \ + __field_overflow(); \ + } while (0) + +static __always_inline __must_check +__u8 u8_encode_bits(u8 v, u8 field) +{ + __assert_field(v, field); + return (v & field_mask(field)) * field_multiplier(field); +} + +static __always_inline __must_check +__u8 u8_replace_bits(__u8 old, u8 val, u8 field) +{ + return (old & ~field) | u8_encode_bits(val, field); +} + +static __always_inline +void u8p_replace_bits(__u8 *p, u8 val, u8 field) +{ + *p = (*p & ~field) | u8_encode_bits(val, field); +} + +static __always_inline __must_check +u8 u8_get_bits(__u8 v, u8 field) +{ + return (v & field) / field_multiplier(field); +} + +static __always_inline __must_check +__le16 le16_encode_bits(u16 v, u16 field) +{ + __assert_field(v, field); + return cpu_to_le16((v & field_mask(field)) * field_multiplier(field)); +} + +static __always_inline +void le16p_replace_bits(__le16 *p, u16 val, u16 field) +{ + *p = (*p & ~cpu_to_le16(field)) | le16_encode_bits(val, field); +} + +static __always_inline __must_check +u16 le16_get_bits(__le16 v, u16 field) +{ + return (le16_to_cpu(v) & field) / field_multiplier(field); +} + +static __always_inline __must_check +__be16 be16_encode_bits(u16 v, u16 field) +{ + __assert_field(v, field); + return cpu_to_be16((v & field_mask(field)) * field_multiplier(field)); +} + +static __always_inline __must_check +u16 be16_get_bits(__be16 v, u16 field) +{ + return (be16_to_cpu(v) & field) / field_multiplier(field); +} + +static __always_inline __must_check +__be16 be16_replace_bits(__be16 old, u16 val, u16 field) +{ + return (old & ~cpu_to_be16(field)) | be16_encode_bits(val, field); +} + +static __always_inline __must_check +__u16 u16_encode_bits(u16 v, u16 field) +{ + __assert_field(v, field); + return (v & field_mask(field)) * field_multiplier(field); +} + +static __always_inline __must_check +__u16 u16_replace_bits(__u16 old, u16 val, u16 field) +{ + return (old & ~field) | u16_encode_bits(val, field); +} + +static __always_inline +void u16p_replace_bits(__u16 *p, u16 val, u16 field) +{ + *p = (*p & ~field) | u16_encode_bits(val, field); +} + +static __always_inline __must_check +u16 u16_get_bits(__u16 v, u16 field) +{ + return (v & field) / field_multiplier(field); +} + +static __always_inline __must_check +__le32 le32_encode_bits(u32 v, u32 field) +{ + __assert_field(v, field); + return cpu_to_le32((v & field_mask(field)) * field_multiplier(field)); +} + +static __always_inline +void le32p_replace_bits(__le32 *p, u32 val, u32 field) +{ + *p = (*p & ~cpu_to_le32(field)) | le32_encode_bits(val, field); +} + +static __always_inline __must_check +u32 le32_get_bits(__le32 v, u32 field) +{ + return (le32_to_cpu(v) & field) / field_multiplier(field); +} + +static __always_inline __must_check +__be32 be32_encode_bits(u32 v, u32 field) +{ + __assert_field(v, field); + return cpu_to_be32((v & field_mask(field)) * field_multiplier(field)); +} + +static __always_inline +void be32p_replace_bits(__be32 *p, u32 val, u32 field) +{ + *p = (*p & ~cpu_to_be32(field)) | be32_encode_bits(val, field); +} + +static __always_inline __must_check +u32 be32_get_bits(__be32 v, u32 field) +{ + return (be32_to_cpu(v) & field) / field_multiplier(field); +} + +static __always_inline __must_check +__u32 u32_encode_bits(u32 v, u32 field) +{ + __assert_field(v, field); + return (v & field_mask(field)) * field_multiplier(field); +} + +static __always_inline __must_check +__u32 u32_replace_bits(__u32 old, u32 val, u32 field) +{ + return (old & ~field) | u32_encode_bits(val, field); +} + +static __always_inline +void u32p_replace_bits(__u32 *p, u32 val, u32 field) +{ + *p = (*p & ~field) | u32_encode_bits(val, field); +} + +static __always_inline __must_check +u32 u32_get_bits(__u32 v, u32 field) +{ + return (v & field) / field_multiplier(field); +} + +static __always_inline __must_check +__le64 le64_encode_bits(u64 v, u64 field) +{ + __assert_field(v, field); + return cpu_to_le64((v & field_mask(field)) * field_multiplier(field)); +} + +static __always_inline __must_check +u64 le64_get_bits(__le64 v, u64 field) +{ + return (le64_to_cpu(v) & field) / field_multiplier(field); +} + +static __always_inline __must_check +__be64 be64_encode_bits(u64 v, u64 field) +{ + __assert_field(v, field); + return cpu_to_be64((v & field_mask(field)) * field_multiplier(field)); +} + +static __always_inline __must_check +u64 be64_get_bits(__be64 v, u64 field) +{ + return (be64_to_cpu(v) & field) / field_multiplier(field); +} + +static __always_inline __must_check +__u64 u64_encode_bits(u64 v, u64 field) +{ + __assert_field(v, field); + return (v & field_mask(field)) * field_multiplier(field); +} + +static __always_inline __must_check +__u64 u64_replace_bits(__u64 old, u64 val, u64 field) +{ + return (old & ~field) | u64_encode_bits(val, field); +} + +static __always_inline +void u64p_replace_bits(__u64 *p, u64 val, u64 field) +{ + *p = (*p & ~field) | u64_encode_bits(val, field); +} + +static __always_inline __must_check +u64 u64_get_bits(__u64 v, u64 field) +{ + return (v & field) / field_multiplier(field); +} + +#endif /* _LINUX_BITFIELD_FIX_WIDTH_H */ diff --git a/include/linux/bitfield.h b/include/linux/bitfield.h index 14f86e455a67..e3f7315c1d53 100644 --- a/include/linux/bitfield.h +++ b/include/linux/bitfield.h @@ -10,7 +10,7 @@ #include <linux/build_bug.h> #include <linux/compiler.h> #include <linux/typecheck.h> -#include <asm/byteorder.h> +#include <linux/bitfield-fix-width.h> /* * Bitfield access macros @@ -211,53 +211,6 @@ *(_reg_p) |= (((typeof(_mask))(_val) << __bf_shf(_mask)) & (_mask)); \ }) -extern void __compiletime_error("value doesn't fit into mask") -__field_overflow(void); -extern void __compiletime_error("bad bitfield mask") -__bad_mask(void); -static __always_inline u64 field_multiplier(u64 field) -{ - if ((field | (field - 1)) & ((field | (field - 1)) + 1)) - __bad_mask(); - return field & -field; -} -static __always_inline u64 field_mask(u64 field) -{ - return field / field_multiplier(field); -} -#define field_max(field) ((typeof(field))field_mask(field)) -#define ____MAKE_OP(type,base,to,from) \ -static __always_inline __##type __must_check type##_encode_bits(base v, base field) \ -{ \ - if (__builtin_constant_p(v) && (v & ~field_mask(field))) \ - __field_overflow(); \ - return to((v & field_mask(field)) * field_multiplier(field)); \ -} \ -static __always_inline __##type __must_check type##_replace_bits(__##type old, \ - base val, base field) \ -{ \ - return (old & ~to(field)) | type##_encode_bits(val, field); \ -} \ -static __always_inline void type##p_replace_bits(__##type *p, \ - base val, base field) \ -{ \ - *p = (*p & ~to(field)) | type##_encode_bits(val, field); \ -} \ -static __always_inline base __must_check type##_get_bits(__##type v, base field) \ -{ \ - return (from(v) & field)/field_multiplier(field); \ -} -#define __MAKE_OP(size) \ - ____MAKE_OP(le##size,u##size,cpu_to_le##size,le##size##_to_cpu) \ - ____MAKE_OP(be##size,u##size,cpu_to_be##size,be##size##_to_cpu) \ - ____MAKE_OP(u##size,u##size,,) -____MAKE_OP(u8,u8,,) -__MAKE_OP(16) -__MAKE_OP(32) -__MAKE_OP(64) -#undef __MAKE_OP -#undef ____MAKE_OP - #define __field_prep(mask, val) \ ({ \ auto __mask = (mask); \ diff --git a/include/linux/bitmap.h b/include/linux/bitmap.h index 7df1573a409c..adafbcf2016b 100644 --- a/include/linux/bitmap.h +++ b/include/linux/bitmap.h @@ -52,6 +52,7 @@ struct device; * bitmap_complement(dst, src, nbits) *dst = ~(*src) * bitmap_equal(src1, src2, nbits) Are *src1 and *src2 equal? * bitmap_intersects(src1, src2, nbits) Do *src1 and *src2 overlap? + * bitmap_intersects_and(src1, src2, src3, nbits) Do *src1, *src2 and *src3 overlap? * bitmap_subset(src1, src2, nbits) Is *src1 a subset of *src2? * bitmap_empty(src, nbits) Are all bits zero in *src? * bitmap_full(src, nbits) Are all bits set in *src? @@ -181,6 +182,9 @@ void __bitmap_replace(unsigned long *dst, const unsigned long *mask, unsigned int nbits); bool __bitmap_intersects(const unsigned long *bitmap1, const unsigned long *bitmap2, unsigned int nbits); +bool __bitmap_intersects_and(const unsigned long *bitmap1, + const unsigned long *bitmap2, + const unsigned long *bitmap3, unsigned int nbits); bool __bitmap_subset(const unsigned long *bitmap1, const unsigned long *bitmap2, unsigned int nbits); unsigned int __bitmap_weight(const unsigned long *bitmap, unsigned int nbits); @@ -446,6 +450,16 @@ bool bitmap_intersects(const unsigned long *src1, const unsigned long *src2, uns } static __always_inline +bool bitmap_intersects_and(const unsigned long *src1, const unsigned long *src2, + const unsigned long *src3, unsigned int nbits) +{ + if (small_const_nbits(nbits)) + return ((*src1 & *src2 & *src3) & BITMAP_LAST_WORD_MASK(nbits)) != 0; + else + return __bitmap_intersects_and(src1, src2, src3, nbits); +} + +static __always_inline bool bitmap_subset(const unsigned long *src1, const unsigned long *src2, unsigned int nbits) { if (small_const_nbits(nbits)) diff --git a/include/linux/blk-mq.h b/include/linux/blk-mq.h index af878597afb8..1a2f8d48adc4 100644 --- a/include/linux/blk-mq.h +++ b/include/linux/blk-mq.h @@ -523,10 +523,12 @@ enum hctx_type { * @tag_list_lock: Serializes tag_list accesses. * @tag_list: List of the request queues that use this tag set. See also * request_queue.tag_set_list. + * @quiesce_lock: Serializes blk_mq_[un]quiesce_tagset() against each other + * and against a request queue being added to @tag_list. + * @quiesce_depth: How often blk_mq_quiesce_tagset() has been called without + * blk_mq_unquiesce_tagset(). * @srcu: Use as lock when type of the request queue is blocking * (BLK_MQ_F_BLOCKING). - * @tags_srcu: SRCU used to defer freeing of tags page_list to prevent - * use-after-free when iterating tags. * @update_nr_hwq_lock: * Synchronize updating nr_hw_queues with add/del disk & * switching elevator. @@ -550,8 +552,9 @@ struct blk_mq_tag_set { struct mutex tag_list_lock; struct list_head tag_list; + spinlock_t quiesce_lock; + unsigned int quiesce_depth; struct srcu_struct *srcu; - struct srcu_struct tags_srcu; struct rw_semaphore update_nr_hwq_lock; }; @@ -1017,6 +1020,13 @@ static inline void *blk_mq_rq_to_pdu(struct request *rq) return rq + 1; } +static inline bool blk_mq_rq_is_dmabuf(struct request *rq) +{ + if (!IS_ENABLED(CONFIG_DMA_SHARED_BUFFER)) + return false; + return rq->bio && op_is_dmabuf(rq->bio->bi_opf); +} + static inline struct blk_mq_hw_ctx *queue_hctx(struct request_queue *q, int id) { struct blk_mq_hw_ctx *hctx; diff --git a/include/linux/blk_types.h b/include/linux/blk_types.h index 98e21b4cbf32..0cc09b975d8f 100644 --- a/include/linux/blk_types.h +++ b/include/linux/blk_types.h @@ -233,7 +233,12 @@ struct bio { atomic_t __bi_remaining; /* The actual vec list, preserved by bio_reset() */ - struct bio_vec *bi_io_vec; + union { + struct bio_vec *bi_io_vec; + /* Driver specific dma map, valid IFF REQ_DMABUF is set */ + struct dma_buf_io_map *bi_dmabuf_map; + }; + struct bvec_iter bi_iter; union { @@ -402,6 +407,7 @@ enum req_flag_bits { __REQ_DRV, /* for driver use */ __REQ_FS_PRIVATE, /* for file system (submitter) use */ __REQ_ATOMIC, /* for atomic write operations */ + __REQ_DMABUF, /* Using premmaped dma buffers */ /* * Command specific flags, keep last: */ @@ -434,6 +440,7 @@ enum req_flag_bits { #define REQ_DRV (__force blk_opf_t)(1ULL << __REQ_DRV) #define REQ_FS_PRIVATE (__force blk_opf_t)(1ULL << __REQ_FS_PRIVATE) #define REQ_ATOMIC (__force blk_opf_t)(1ULL << __REQ_ATOMIC) +#define REQ_DMABUF (__force blk_opf_t)(1ULL << __REQ_DMABUF) #define REQ_NOUNMAP (__force blk_opf_t)(1ULL << __REQ_NOUNMAP) @@ -487,6 +494,11 @@ static inline bool op_is_discard(blk_opf_t op) return (op & REQ_OP_MASK) == REQ_OP_DISCARD; } +static inline bool op_is_dmabuf(blk_opf_t op) +{ + return op & REQ_DMABUF; +} + /* * Check if a bio or request operation is a zone management operation. */ diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index 4f7905c3412b..0c2010230632 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -191,14 +191,17 @@ struct gendisk { #ifdef CONFIG_BLK_DEV_ZONED /* * Zoned block device information. Reads of this information must be - * protected with blk_queue_enter() / blk_queue_exit(). Modifying this - * information is only allowed while no requests are being processed. - * See also blk_mq_freeze_queue() and blk_mq_unfreeze_queue(). + * protected with blk_queue_enter() / blk_queue_exit() or by holding a + * lock on zone_revalidate_mutex. blk_revalidate_disk_zones() may modify + * this information while no requests are being processed (disk queue + * frozen with blk_mq_freeze_queue()) and while holding a lock on + * zone_revalidate_mutex. */ + struct mutex zone_revalidate_mutex; unsigned int nr_zones; unsigned int zone_capacity; unsigned int last_zone_capacity; - u8 __rcu *zones_cond; + u8 __rcu *zones_state; unsigned int zone_wplugs_hash_bits; atomic_t nr_zone_wplugs; spinlock_t zone_wplugs_hash_lock; @@ -1596,6 +1599,8 @@ struct block_device_operations { /* returns the length of the identifier or a negative errno: */ int (*get_unique_id)(struct gendisk *disk, u8 id[16], enum blk_unique_id id_type); + int (*init_dma_buf_io_ctx)(struct block_device *, + struct dma_buf_io_ctx *); struct module *owner; const struct pr_ops *pr_ops; @@ -1816,8 +1821,11 @@ static inline int bio_split_rw_at(struct bio *bio, */ static inline unsigned int max_integrity_io_size(struct queue_limits *lim) { - return min_t(unsigned int, lim->max_segment_size, - (BLK_INTEGRITY_MAX_SIZE / lim->integrity.metadata_size) << + unsigned int buf_size = min_t(unsigned int, lim->max_segment_size, + BLK_INTEGRITY_MAX_SIZE); + + return min_t(u64, UINT_MAX, + (u64)(buf_size / lim->integrity.metadata_size) << lim->integrity.interval_exp); } diff --git a/include/linux/bnge/hsi.h b/include/linux/bnge/hsi.h index 1f7bd96415a5..d4fe662d60b1 100644 --- a/include/linux/bnge/hsi.h +++ b/include/linux/bnge/hsi.h @@ -8607,6 +8607,7 @@ struct hwrm_cfa_l2_filter_alloc_input { #define CFA_L2_FILTER_ALLOC_REQ_ENABLES_MIRROR_VNIC_ID 0x10000UL #define CFA_L2_FILTER_ALLOC_REQ_ENABLES_NUM_VLANS 0x20000UL #define CFA_L2_FILTER_ALLOC_REQ_ENABLES_T_NUM_VLANS 0x40000UL + #define CFA_L2_FILTER_ALLOC_REQ_ENABLES_RFS_RING_TBL_IDX 0x80000UL u8 l2_addr[6]; u8 num_vlans; u8 t_num_vlans; @@ -8663,7 +8664,8 @@ struct hwrm_cfa_l2_filter_alloc_input { #define CFA_L2_FILTER_ALLOC_REQ_PRI_HINT_MIN 0x4UL #define CFA_L2_FILTER_ALLOC_REQ_PRI_HINT_LAST CFA_L2_FILTER_ALLOC_REQ_PRI_HINT_MIN u8 unused_5; - __le32 unused_6; + __le16 rfs_ring_tbl_idx; + __le16 unused_6; __le64 l2_filter_id_hint; }; @@ -8900,6 +8902,95 @@ struct hwrm_cfa_tunnel_filter_free_output { u8 valid; }; +/* hwrm_cfa_ntuple_filter_alloc_input (size:1024b/128B) */ +struct hwrm_cfa_ntuple_filter_alloc_input { + __le16 req_type; + __le16 cmpl_ring; + __le16 seq_id; + __le16 target_id; + __le64 resp_addr; + __le32 flags; + #define CFA_NTUPLE_FILTER_ALLOC_REQ_FLAGS_LOOPBACK 0x1UL + #define CFA_NTUPLE_FILTER_ALLOC_REQ_FLAGS_DROP 0x2UL + #define CFA_NTUPLE_FILTER_ALLOC_REQ_FLAGS_METER 0x4UL + #define CFA_NTUPLE_FILTER_ALLOC_REQ_FLAGS_DEST_FID 0x8UL + #define CFA_NTUPLE_FILTER_ALLOC_REQ_FLAGS_ARP_REPLY 0x10UL + #define CFA_NTUPLE_FILTER_ALLOC_REQ_FLAGS_DEST_RFS_RING_IDX 0x20UL + #define CFA_NTUPLE_FILTER_ALLOC_REQ_FLAGS_NO_L2_CONTEXT 0x40UL + __le32 enables; + #define CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_L2_FILTER_ID 0x1UL + #define CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_ETHERTYPE 0x2UL + #define CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_TUNNEL_TYPE 0x4UL + #define CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_MACADDR 0x8UL + #define CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IPADDR_TYPE 0x10UL + #define CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR 0x20UL + #define CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR_MASK 0x40UL + #define CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR 0x80UL + #define CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR_MASK 0x100UL + #define CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_IP_PROTOCOL 0x200UL + #define CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT 0x400UL + #define CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_SRC_PORT_MASK 0x800UL + #define CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT 0x1000UL + #define CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_PORT_MASK 0x2000UL + #define CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_PRI_HINT 0x4000UL + #define CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_NTUPLE_FILTER_ID 0x8000UL + #define CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_ID 0x10000UL + #define CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_MIRROR_VNIC_ID 0x20000UL + #define CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_DST_MACADDR 0x40000UL + #define CFA_NTUPLE_FILTER_ALLOC_REQ_ENABLES_RFS_RING_TBL_IDX 0x80000UL + __le64 l2_filter_id; + u8 src_macaddr[6]; + __be16 ethertype; + u8 ip_addr_type; + #define CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_UNKNOWN 0x0UL + #define CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV4 0x4UL + #define CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV6 0x6UL + #define CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_LAST CFA_NTUPLE_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV6 + u8 ip_protocol; + #define CFA_NTUPLE_FILTER_ALLOC_REQ_IP_PROTOCOL_UNKNOWN 0x0UL + #define CFA_NTUPLE_FILTER_ALLOC_REQ_IP_PROTOCOL_TCP 0x6UL + #define CFA_NTUPLE_FILTER_ALLOC_REQ_IP_PROTOCOL_UDP 0x11UL + #define CFA_NTUPLE_FILTER_ALLOC_REQ_IP_PROTOCOL_ICMP 0x1UL + #define CFA_NTUPLE_FILTER_ALLOC_REQ_IP_PROTOCOL_ICMPV6 0x3aUL + #define CFA_NTUPLE_FILTER_ALLOC_REQ_IP_PROTOCOL_RSVD 0xffUL + #define CFA_NTUPLE_FILTER_ALLOC_REQ_IP_PROTOCOL_LAST CFA_NTUPLE_FILTER_ALLOC_REQ_IP_PROTOCOL_RSVD + __le16 dst_id; + __le16 rfs_ring_tbl_idx; + u8 tunnel_type; + #define CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_NONTUNNEL 0x0UL + #define CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_VXLAN 0x1UL + #define CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_NVGRE 0x2UL + #define CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_L2GRE 0x3UL + #define CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_IPIP 0x4UL + #define CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_GENEVE 0x5UL + #define CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_MPLS 0x6UL + #define CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_STT 0x7UL + #define CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_IPGRE 0x8UL + #define CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_VXLAN_V4 0x9UL + #define CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_IPGRE_V1 0xaUL + #define CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_L2_ETYPE 0xbUL + #define CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_VXLAN_GPE_V6 0xcUL + #define CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_VXLAN_GPE 0x10UL + #define CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_ANYTUNNEL 0xffUL + #define CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_LAST CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_ANYTUNNEL + u8 pri_hint; + #define CFA_NTUPLE_FILTER_ALLOC_REQ_PRI_HINT_NO_PREFER 0x0UL + #define CFA_NTUPLE_FILTER_ALLOC_REQ_PRI_HINT_ABOVE 0x1UL + #define CFA_NTUPLE_FILTER_ALLOC_REQ_PRI_HINT_BELOW 0x2UL + #define CFA_NTUPLE_FILTER_ALLOC_REQ_PRI_HINT_HIGHEST 0x3UL + #define CFA_NTUPLE_FILTER_ALLOC_REQ_PRI_HINT_LOWEST 0x4UL + #define CFA_NTUPLE_FILTER_ALLOC_REQ_PRI_HINT_LAST CFA_NTUPLE_FILTER_ALLOC_REQ_PRI_HINT_LOWEST + __be32 src_ipaddr[4]; + __be32 src_ipaddr_mask[4]; + __be32 dst_ipaddr[4]; + __be32 dst_ipaddr_mask[4]; + __be16 src_port; + __be16 src_port_mask; + __be16 dst_port; + __be16 dst_port_mask; + __le64 ntuple_filter_id_hint; +}; + /* hwrm_cfa_ntuple_filter_alloc_output (size:192b/24B) */ struct hwrm_cfa_ntuple_filter_alloc_output { __le16 error_code; @@ -9014,6 +9105,49 @@ struct hwrm_cfa_ntuple_filter_cfg_output { u8 valid; }; +/* hwrm_cfa_adv_flow_mgnt_qcaps_input (size:256b/32B) */ +struct hwrm_cfa_adv_flow_mgnt_qcaps_input { + __le16 req_type; + __le16 cmpl_ring; + __le16 seq_id; + __le16 target_id; + __le64 resp_addr; + __le32 unused_0[4]; +}; + +/* hwrm_cfa_adv_flow_mgnt_qcaps_output (size:128b/16B) */ +struct hwrm_cfa_adv_flow_mgnt_qcaps_output { + __le16 error_code; + __le16 req_type; + __le16 seq_id; + __le16 resp_len; + __le32 flags; + #define CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_FLOW_HND_16BIT_SUPPORTED 0x1UL + #define CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_FLOW_HND_64BIT_SUPPORTED 0x2UL + #define CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_FLOW_BATCH_DELETE_SUPPORTED 0x4UL + #define CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_FLOW_RESET_ALL_SUPPORTED 0x8UL + #define CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_NTUPLE_FLOW_DEST_FUNC_SUPPORTED 0x10UL + #define CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_TX_EEM_FLOW_SUPPORTED 0x20UL + #define CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_RX_EEM_FLOW_SUPPORTED 0x40UL + #define CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_FLOW_COUNTER_ALLOC_SUPPORTED 0x80UL + #define CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_RFS_RING_TBL_IDX_SUPPORTED 0x100UL + #define CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_UNTAGGED_VLAN_SUPPORTED 0x200UL + #define CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_XDP_SUPPORTED 0x400UL + #define CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_L2_HEADER_SOURCE_FIELDS_SUPPORTED 0x800UL + #define CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_NTUPLE_FLOW_RX_ARP_SUPPORTED 0x1000UL + #define CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_RFS_RING_TBL_IDX_V2_SUPPORTED 0x2000UL + #define CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_NTUPLE_FLOW_RX_ETHERTYPE_IP_SUPPORTED 0x4000UL + #define CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_TRUFLOW_CAPABLE 0x8000UL + #define CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_L2_FILTER_TRAFFIC_TYPE_L2_ROCE_SUPPORTED 0x10000UL + #define CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_LAG_SUPPORTED 0x20000UL + #define CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_NTUPLE_FLOW_NO_L2CTX_SUPPORTED 0x40000UL + #define CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_NIC_FLOW_STATS_SUPPORTED 0x80000UL + #define CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_NTUPLE_FLOW_RX_EXT_IP_PROTO_SUPPORTED 0x100000UL + #define CFA_ADV_FLOW_MGNT_QCAPS_RESP_FLAGS_RFS_RING_TBL_IDX_V3_SUPPORTED 0x200000UL + u8 unused_0[3]; + u8 valid; +}; + /* hwrm_tunnel_dst_port_alloc_input (size:192b/24B) */ struct hwrm_tunnel_dst_port_alloc_input { __le16 req_type; diff --git a/include/linux/bootconfig.h b/include/linux/bootconfig.h index deda507500da..260a6f46042e 100644 --- a/include/linux/bootconfig.h +++ b/include/linux/bootconfig.h @@ -27,6 +27,10 @@ bool __init cmdline_has_extra_options(void); #define BOOTCONFIG_ALIGN (1 << BOOTCONFIG_ALIGN_SHIFT) #define BOOTCONFIG_ALIGN_MASK (BOOTCONFIG_ALIGN - 1) +/* Bootconfig footer is [size][csum][BOOTCONFIG_MAGIC]. */ +#define BOOTCONFIG_FOOTER_SIZE \ + (sizeof(uint32_t) * 2 + BOOTCONFIG_MAGIC_LEN) + /** * xbc_calc_checksum() - Calculate checksum of bootconfig * @data: Bootconfig data. @@ -266,7 +270,15 @@ int __init xbc_node_compose_key_after(struct xbc_node *root, struct xbc_node *node, char *buf, size_t size); /* Render key/value pairs under @root as a flat cmdline string */ -int __init xbc_snprint_cmdline(char *buf, size_t size, struct xbc_node *root); +typedef bool (*xbc_cmdline_filter_fn)(struct xbc_node *node, const char *key, void *data); + +int __init xbc_snprint_cmdline_filter(char *buf, size_t size, struct xbc_node *root, + xbc_cmdline_filter_fn filter, void *data); + +static inline int __init xbc_snprint_cmdline(char *buf, size_t size, struct xbc_node *root) +{ + return xbc_snprint_cmdline_filter(buf, size, root, NULL, NULL); +} /** * xbc_node_compose_key() - Compose full key string of the XBC node @@ -291,12 +303,7 @@ int __init xbc_init(const char *buf, size_t size, const char **emsg, int *epos); int __init xbc_get_info(int *node_size, size_t *data_size); /* XBC cleanup data structures */ -void __init _xbc_exit(bool early); - -static __always_inline void xbc_exit(void) -{ - _xbc_exit(false); -} +void __init xbc_exit(void); /* XBC embedded bootconfig data in kernel */ #ifdef CONFIG_BOOT_CONFIG_EMBED diff --git a/include/linux/bpf-cgroup-defs.h b/include/linux/bpf-cgroup-defs.h index c9e6b26abab6..0147b8bec973 100644 --- a/include/linux/bpf-cgroup-defs.h +++ b/include/linux/bpf-cgroup-defs.h @@ -47,6 +47,7 @@ enum cgroup_bpf_attach_type { CGROUP_INET6_GETSOCKNAME, CGROUP_UNIX_GETSOCKNAME, CGROUP_INET_SOCK_RELEASE, + CGROUP_TCP_SOCK_OPS, CGROUP_LSM_START, CGROUP_LSM_END = CGROUP_LSM_START + CGROUP_LSM_NUM - 1, MAX_CGROUP_BPF_ATTACH_TYPE diff --git a/include/linux/bpf-cgroup.h b/include/linux/bpf-cgroup.h index 4d0cc65976a1..02ef7f899b3d 100644 --- a/include/linux/bpf-cgroup.h +++ b/include/linux/bpf-cgroup.h @@ -100,6 +100,8 @@ struct bpf_cgroup_storage { struct bpf_cgroup_link { struct bpf_link link; struct cgroup *cgroup; + struct bpf_map *map; + wait_queue_head_t wait_hup; }; struct bpf_prog_list { @@ -110,6 +112,18 @@ struct bpf_prog_list { u32 flags; }; +#define bpf_cgroup_struct_ops_foreach(var, item, cgrp, atype) \ + for (item = rcu_dereference((cgrp)->bpf.effective[atype])->items;\ + ((var) = READ_ONCE(item->kdata)); \ + item++) + +static inline bool cgroup_bpf_is_struct_ops_atype(enum cgroup_bpf_attach_type atype) +{ + return atype == CGROUP_TCP_SOCK_OPS; +} +void cgroup_bpf_struct_ops_register(int atype, u32 type_id, void *cfi_stubs, bool mult_trace); +int cgroup_bpf_struct_ops_attach(struct bpf_map *map, const union bpf_attr *attr); + void __init cgroup_bpf_lifetime_notifier_init(void); int __cgroup_bpf_run_filter_skb(struct sock *sk, @@ -331,27 +345,19 @@ static inline bool cgroup_bpf_sock_enabled(struct sock *sk, * calling bpf_setsockopt on listener-sk will not make sense anyway, * so passing 'sock_ops->sk == req_sk' to the bpf prog is appropriate here. */ -#define BPF_CGROUP_RUN_PROG_SOCK_OPS_SK(sock_ops, sk) \ -({ \ - int __ret = 0; \ - if (cgroup_bpf_enabled(CGROUP_SOCK_OPS)) \ - __ret = __cgroup_bpf_run_filter_sock_ops(sk, \ - sock_ops, \ - CGROUP_SOCK_OPS); \ - __ret; \ -}) - -#define BPF_CGROUP_RUN_PROG_SOCK_OPS(sock_ops) \ -({ \ - int __ret = 0; \ - if (cgroup_bpf_enabled(CGROUP_SOCK_OPS) && (sock_ops)->sk) { \ - typeof(sk) __sk = sk_to_full_sk((sock_ops)->sk); \ - if (__sk && sk_fullsock(__sk)) \ - __ret = __cgroup_bpf_run_filter_sock_ops(__sk, \ - sock_ops, \ - CGROUP_SOCK_OPS); \ - } \ - __ret; \ +#define __BPF_CGROUP_RUN_PROG_SOCK_OPS_SK(sock_ops, sk) \ + __cgroup_bpf_run_filter_sock_ops(sk, sock_ops, \ + CGROUP_SOCK_OPS) + +#define __BPF_CGROUP_RUN_PROG_SOCK_OPS(sock_ops) \ +({ \ + int __ret = 0; \ + typeof(sk) __sk = sk_to_full_sk((sock_ops)->sk); \ + if (__sk && sk_fullsock(__sk)) \ + __ret = __cgroup_bpf_run_filter_sock_ops(__sk, \ + sock_ops, \ + CGROUP_SOCK_OPS); \ + __ret; \ }) #define BPF_CGROUP_RUN_PROG_DEVICE_CGROUP(atype, major, minor, access) \ @@ -479,6 +485,20 @@ static inline int bpf_percpu_cgroup_storage_update(struct bpf_map *map, return 0; } +static inline bool cgroup_bpf_is_struct_ops_atype(int atype) +{ + return false; +} +static inline void cgroup_bpf_struct_ops_register(int atype, u32 type_id, void *cfi_stubs, + bool mult_trace) +{ +} +static inline int cgroup_bpf_struct_ops_attach(struct bpf_map *map, + const union bpf_attr *attr) +{ + return -EOPNOTSUPP; +} + #define cgroup_bpf_enabled(atype) (0) #define BPF_CGROUP_RUN_SA_PROG_LOCK(sk, uaddr, uaddrlen, atype, t_ctx) ({ 0; }) #define BPF_CGROUP_RUN_SA_PROG(sk, uaddr, uaddrlen, atype) ({ 0; }) @@ -501,7 +521,7 @@ static inline int bpf_percpu_cgroup_storage_update(struct bpf_map *map, #define BPF_CGROUP_RUN_PROG_UDP4_RECVMSG_LOCK(sk, uaddr, uaddrlen) ({ 0; }) #define BPF_CGROUP_RUN_PROG_UDP6_RECVMSG_LOCK(sk, uaddr, uaddrlen) ({ 0; }) #define BPF_CGROUP_RUN_PROG_UNIX_RECVMSG_LOCK(sk, uaddr, uaddrlen) ({ 0; }) -#define BPF_CGROUP_RUN_PROG_SOCK_OPS(sock_ops) ({ 0; }) +#define __BPF_CGROUP_RUN_PROG_SOCK_OPS(sock_ops) ({ 0; }) #define BPF_CGROUP_RUN_PROG_DEVICE_CGROUP(atype, major, minor, access) ({ 0; }) #define BPF_CGROUP_RUN_PROG_SYSCTL(head,table,write,buf,count,pos) ({ 0; }) #define BPF_CGROUP_RUN_PROG_GETSOCKOPT(sock, level, optname, optval, \ diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 0ecb9418dfbd..ff0436781293 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -17,6 +17,7 @@ #include <linux/numa.h> #include <linux/mm_types.h> #include <linux/wait.h> +#include <linux/irq_work_types.h> #include <linux/refcount.h> #include <linux/mutex.h> #include <linux/module.h> @@ -90,6 +91,7 @@ struct bpf_map_ops { struct bpf_map *(*map_alloc)(union bpf_attr *attr); void (*map_release)(struct bpf_map *map, struct file *map_file); void (*map_free)(struct bpf_map *map); + void (*map_free_pre_rcu)(struct bpf_map *map); int (*map_get_next_key)(struct bpf_map *map, void *key, void *next_key); void (*map_release_uref)(struct bpf_map *map); void *(*map_lookup_elem_sys_only)(struct bpf_map *map, void *key); @@ -214,6 +216,7 @@ enum btf_field_type { BPF_UPTR = (1 << 11), BPF_RES_SPIN_LOCK = (1 << 12), BPF_TASK_WORK = (1 << 13), + BPF_RCU_HEAD = (1 << 14), }; enum bpf_cgroup_storage_type { @@ -268,6 +271,7 @@ struct btf_record { int wq_off; int refcount_off; int task_work_off; + int rcu_head_off; struct btf_field fields[]; }; @@ -337,12 +341,23 @@ struct bpf_map { bool frozen; /* write-once; write-protected by freeze_mutex */ bool free_after_mult_rcu_gp; bool free_after_rcu_gp; + bool was_inner_map; atomic64_t sleepable_refcnt; s64 __percpu *elem_count; u64 cookie; /* write-once */ char *excl_prog_sha; + /* + * Which programs use the map, see bpf_map_claim(): 0 - none so far, + * aux of the program - only that one, the same with BPF_MAP_USER_PATCHED + * set - only that one and it stored the addresses of its functions into + * the map, BPF_MAP_USER_MANY - more than one. + */ + unsigned long user; }; +#define BPF_MAP_USER_MANY 1UL +#define BPF_MAP_USER_PATCHED 1UL + static inline const char *btf_field_type_name(enum btf_field_type type) { switch (type) { @@ -373,6 +388,8 @@ static inline const char *btf_field_type_name(enum btf_field_type type) return "bpf_refcount"; case BPF_TASK_WORK: return "bpf_task_work"; + case BPF_RCU_HEAD: + return "bpf_rcu_head"; default: WARN_ON_ONCE(1); return "unknown"; @@ -413,6 +430,8 @@ static inline u32 btf_field_type_size(enum btf_field_type type) return sizeof(struct bpf_refcount); case BPF_TASK_WORK: return sizeof(struct bpf_task_work); + case BPF_RCU_HEAD: + return sizeof(struct bpf_rcu_head); default: WARN_ON_ONCE(1); return 0; @@ -447,6 +466,8 @@ static inline u32 btf_field_type_align(enum btf_field_type type) return __alignof__(struct bpf_refcount); case BPF_TASK_WORK: return __alignof__(struct bpf_task_work); + case BPF_RCU_HEAD: + return __alignof__(struct bpf_rcu_head); default: WARN_ON_ONCE(1); return 0; @@ -479,6 +500,7 @@ static inline void bpf_obj_init_field(const struct btf_field *field, void *addr) case BPF_KPTR_PERCPU: case BPF_UPTR: case BPF_TASK_WORK: + case BPF_RCU_HEAD: break; default: WARN_ON_ONCE(1); @@ -572,7 +594,7 @@ static inline void bpf_obj_memcpy(struct btf_record *rec, if (long_memcpy) bpf_long_memcpy(dst, src, size); else - memcpy(dst, src, size); + data_race(memcpy(dst, src, size)); return; } @@ -580,10 +602,10 @@ static inline void bpf_obj_memcpy(struct btf_record *rec, u32 next_off = rec->fields[i].offset; u32 sz = next_off - curr_off; - memcpy(dst + curr_off, src + curr_off, sz); + data_race(memcpy(dst + curr_off, src + curr_off, sz)); curr_off += rec->fields[i].size + sz; } - memcpy(dst + curr_off, src + curr_off, size - curr_off); + data_race(memcpy(dst + curr_off, src + curr_off, size - curr_off)); } static inline void copy_map_value(struct bpf_map *map, void *dst, void *src) @@ -780,7 +802,10 @@ enum bpf_type_flag { */ PTR_UNTRUSTED = BIT(6 + BPF_BASE_TYPE_BITS), - /* MEM can be uninitialized. */ + /* + * MEM can be uninitialized. Generic memory outputs need not be fully + * initialized by the callee. + */ MEM_UNINIT = BIT(7 + BPF_BASE_TYPE_BITS), /* DYNPTR points to memory local to the bpf program. */ @@ -874,7 +899,7 @@ enum bpf_type_flag { /* function argument constraints */ enum bpf_arg_type { - ARG_DONTCARE = 0, /* unused argument in helper function */ + ARG_UNUSED = 0, /* unused argument; terminates argument iteration */ /* the following constraints used to prototype * bpf_map_lookup/update/delete_elem() functions @@ -909,6 +934,24 @@ enum bpf_arg_type { ARG_PTR_TO_TIMER, /* pointer to bpf_timer */ ARG_KPTR_XCHG_DEST, /* pointer to destination that kptrs are bpf_kptr_xchg'd into */ ARG_PTR_TO_DYNPTR, /* pointer to bpf_dynptr. See bpf_type_flag for dynptr type */ + + ARG_CONST_SCALAR, /* scalar known at verification time */ + ARG_CONST_MEM_SIZE, /* ARG_MEM_SIZE that must be constant */ + ARG_PTR_TO_ALLOC_BTF_ID, /* pointer to an allocated object */ + ARG_PTR_TO_REFCOUNTED_KPTR, /* pointer to a refcounted local kptr */ + ARG_PTR_TO_ITER, /* pointer to an iterator */ + ARG_PTR_TO_LIST_HEAD, /* pointer to bpf_list_head */ + ARG_PTR_TO_LIST_NODE, /* pointer to bpf_list_node */ + ARG_PTR_TO_RB_ROOT, /* pointer to bpf_rb_root */ + ARG_PTR_TO_RB_NODE, /* pointer to bpf_rb_node */ + ARG_PTR_TO_WORKQUEUE, /* pointer to bpf_wq */ + ARG_PTR_TO_TASK_WORK, /* pointer to bpf_task_work */ + ARG_PTR_TO_RCU_HEAD, /* pointer to bpf_rcu_head */ + ARG_PTR_TO_IRQ_FLAG, /* pointer to saved IRQ flags on the stack */ + ARG_PTR_TO_RES_SPIN_LOCK, /* pointer to bpf_res_spin_lock */ + ARG_PTR_TO_CTX_OUT, /* hook output argument passed through from ctx */ + ARG_PTR_TO_PROG_AUX, /* pointer to the caller's bpf_prog_aux */ + ARG_IGNORE, /* argument the verifier does not check at all */ __BPF_ARG_TYPE_MAX, /* Extended arg_types. */ @@ -977,6 +1020,13 @@ static_assert(__BPF_RET_TYPE_MAX <= BPF_BASE_TYPE_LIMIT); */ #define MAX_BPF_FUNC_REG_ARGS 5 +/* A by-value argument takes two eightbytes at most, so the maximum number of + * argument slots of any function is 2 * MAX_BPF_FUNC_ARGS. A local array may + * need that size for processing, although eventually the maximum slots will + * be capped at MAX_BPF_FUNC_ARGS. + */ +#define MAX_BPF_FUNC_ARG_SLOTS (2 * MAX_BPF_FUNC_ARGS) + /* eBPF function prototype used by verifier to allow BPF_CALLs from eBPF programs * to in-kernel helper functions and for adjusting imm32 field in BPF_CALL * instructions after verifying @@ -1005,13 +1055,13 @@ struct bpf_func_proto { }; union { struct { - u32 *arg1_btf_id; - u32 *arg2_btf_id; - u32 *arg3_btf_id; - u32 *arg4_btf_id; - u32 *arg5_btf_id; + const u32 *arg1_btf_id; + const u32 *arg2_btf_id; + const u32 *arg3_btf_id; + const u32 *arg4_btf_id; + const u32 *arg5_btf_id; }; - u32 *arg_btf_id[MAX_BPF_FUNC_ARGS]; + const u32 *arg_btf_id[MAX_BPF_FUNC_ARGS]; struct { size_t arg1_size; size_t arg2_size; @@ -1114,6 +1164,7 @@ struct bpf_insn_access_aux { u32 ref_id; }; }; + u32 mem_size; struct bpf_verifier_log *log; /* for verbose logs */ bool is_retval; /* is accessing function return value ? */ }; @@ -1194,6 +1245,9 @@ struct bpf_prog_offload { u32 jited_len; }; +/* The argument is aligned to 16 bytes. */ +#define BTF_FMODEL_ALIGN16_ARG BIT(0) + /* The argument is signed. */ #define BTF_FMODEL_SIGNED_ARG BIT(1) @@ -1211,6 +1265,11 @@ struct btf_func_model { u8 arg_flags[MAX_BPF_FUNC_ARGS]; }; +static inline u32 btf_func_model_arg_slots(const struct btf_func_model *m, u32 arg) +{ + return (m->arg_size[arg] + sizeof(u64) - 1) / sizeof(u64); +} + /* Restore arguments before returning from trampoline to let original function * continue executing. This flag is used for fentry progs when there are no * fexit progs. @@ -1684,12 +1743,21 @@ static inline void bpf_trampoline_multi_detach(struct bpf_prog *prog, static inline void bpf_trampoline_set_flags(struct bpf_trampoline *tr, u32 flags) {} #endif +/* The contexts that a global function is called in, and is verified for. */ +enum bpf_func_context { + BPF_FUNC_CTX_NON_SLEEPABLE, + BPF_FUNC_CTX_SLEEPABLE, + /* Sleepable code has disabled preemption or interrupts: no RCU read lock. */ + BPF_FUNC_CTX_NO_RCU_LOCK, + BPF_FUNC_CTX_CNT, +}; + struct bpf_func_info_aux { u16 linkage; bool unreliable; - /* Indexed by in_sleepable. */ - bool called[2]; - bool verified[2]; + bool called[BPF_FUNC_CTX_CNT]; + bool verified[BPF_FUNC_CTX_CNT]; + bool called_with_pkt_ptrs; }; enum bpf_jit_poke_reason { @@ -1721,6 +1789,7 @@ struct bpf_ctx_arg_aux { struct btf *btf; u32 btf_id; u32 ref_id; + u32 mem_size; bool refcounted; }; @@ -1749,12 +1818,18 @@ enum { }; struct bpf_stream { - atomic_t capacity; + refcount_t refcnt; + atomic_t capacity; /* bytes reserved against the stream limit */ + atomic_t readable; /* published bytes available to readers */ struct llist_head log; /* list of in-flight stream elements in LIFO order */ struct mutex lock; /* lock protecting backlog_{head,tail} */ struct llist_node *backlog_head; /* list of in-flight stream elements in FIFO order */ struct llist_node *backlog_tail; /* tail of the list above */ + wait_queue_head_t waitq; + struct irq_work notify_work; + bool notify_used; /* notify_work was queued at least once */ + bool dead; }; struct bpf_stream_stage { @@ -1773,6 +1848,7 @@ enum bpf_sig_keyring { BPF_SIG_KEYRING_SECONDARY, BPF_SIG_KEYRING_PLATFORM, BPF_SIG_KEYRING_USER, + BPF_SIG_KEYRING_BPF, }; struct bpf_prog_aux { @@ -1792,6 +1868,9 @@ struct bpf_prog_aux { u32 ctx_arg_info_size; u32 max_rdonly_access; u32 max_rdwr_access; + /* [min, max): the bytes of a rdwr buffer that may be written */ + u32 min_rdwr_write; + u32 max_rdwr_write; u32 subprog_start; struct btf *attach_btf; struct bpf_ctx_arg_aux *ctx_arg_info; @@ -1807,24 +1886,28 @@ struct bpf_prog_aux { bool attach_btf_trace; /* true if attaching to BTF-enabled raw tp */ bool attach_tracing_prog; /* true if tracing another tracing program */ bool tramp_linked; /* true if it was ever linked to a trampoline */ + bool sleepable_async_cb; /* has a bpf_wq or bpf_task_work callback */ bool func_proto_unreliable; bool tail_call_reachable; bool xdp_has_frags; bool exception_cb; bool exception_boundary; - bool is_extended; /* true if extended by freplace program */ bool jits_use_priv_stack; bool priv_stack_requested; bool changes_pkt_data; + bool has_prologue; bool might_sleep; + bool can_unwind; bool kprobe_write_ctx; + /* has insns that work with the arena base, see bpf_prog_has_arena_insns() */ + bool uses_arena_insns; struct { s32 keyring_serial; u8 keyring_type; u8 verdict; } sig; u64 prog_array_member_cnt; /* counts how many times as member of prog_array */ - struct mutex ext_mutex; /* mutex for is_extended and prog_array_member_cnt */ + struct mutex ext_mutex; /* mutex for freplace_link_cnt and prog_array_member_cnt */ struct bpf_arena *arena; void (*recursion_detected)(struct bpf_prog *prog); /* callback if recursion is detected */ /* BTF_KIND_FUNC_PROTO for valid attach_btf_id */ @@ -1856,6 +1939,7 @@ struct bpf_prog_aux { char name[BPF_OBJ_NAME_LEN]; u64 (*bpf_exception_cb)(u64 cookie, u64 sp, u64 bp, u64, u64); u16 stack_arg_sp_adjust; + u16 freplace_link_cnt; /* counts freplace links extending this prog */ #ifdef CONFIG_SECURITY void *security; #endif @@ -1892,9 +1976,10 @@ struct bpf_prog_aux { union { struct work_struct work; struct rcu_head rcu; + struct rcu_work rwork; }; - struct bpf_stream stream[2]; - struct mutex st_ops_assoc_mutex; + struct bpf_stream *stream[2]; + spinlock_t st_ops_assoc_lock; /* protects st_ops_assoc */ struct bpf_map __rcu *st_ops_assoc; }; @@ -1940,6 +2025,15 @@ struct bpf_prog { }; }; +/* + * Does the program, or a callback of it, run under RCU Tasks Trace protection + * only, outside of any RCU read side critical section? + */ +static inline bool bpf_prog_runs_sleepable(const struct bpf_prog *prog) +{ + return prog->sleepable || prog->aux->sleepable_async_cb; +} + struct bpf_array_aux { /* Programs with direct jumps into programs part of this array. */ struct list_head poke_progs; @@ -2137,6 +2231,20 @@ struct btf_member; * unloaded while in use. * @name: The name of the struct bpf_struct_ops object. * @func_models: Func models + * @cgroup_atype: A value in enum cgroup_bpf_attach_type for cgroup attachment. + * 0 means the struct_ops type does not support cgroup attachment. + * If cgroup_atype is non-zero, the @reg and @unreg must be NULL + * because the attachment/detachment will be handled by the bpf core. + * @free_after_tasks_rcu_gp: Set to true if it needs the bpf core to wait for + * a tasks_rcu gp before freeing the struct_ops map + * and its progs. It is unnecessary if the @unreg + * has waited for the correct rcu gp or the @unreg + * has ensured all struct_ops prog has finished running. + * @free_after_mult_rcu_gp: Set to true if the cgroup prog array of this + * struct_ops needs a tasks_trace_rcu gp, besides a + * regular rcu gp, before it is freed. It is usually + * needed if the struct_ops has sleepable prog. A + * struct_ops map always waits for both. */ struct bpf_struct_ops { const struct bpf_verifier_ops *verifier_ops; @@ -2155,6 +2263,9 @@ struct bpf_struct_ops { struct module *owner; const char *name; struct btf_func_model func_models[BPF_STRUCT_OPS_MAX_NR_MEMBERS]; + int cgroup_atype; + bool free_after_tasks_rcu_gp; + bool free_after_mult_rcu_gp; }; /* Every member of a struct_ops type has an instance even a member is not @@ -2289,6 +2400,12 @@ u32 bpf_struct_ops_id(const void *kdata); int bpf_struct_ops_for_each_prog(const void *kdata, int (*cb)(struct bpf_prog *prog, void *data), void *data); +void *bpf_struct_ops_map_kdata(struct bpf_map *map); +void *bpf_struct_ops_map_cfi_stubs(struct bpf_map *map); +bool bpf_struct_ops_valid_to_reg(struct bpf_map *map); +int bpf_struct_ops_link_update_check(struct bpf_map *new_map, struct bpf_map *old_map, + struct bpf_map *expected_old_map); +int bpf_struct_ops_map_cgroup_atype(struct bpf_map *map); #ifdef CONFIG_NET /* Define it here to avoid the use of forward declaration */ @@ -2353,6 +2470,32 @@ static inline void bpf_map_struct_ops_info_fill(struct bpf_map_info *info, struc static inline void bpf_struct_ops_desc_release(struct bpf_struct_ops_desc *st_ops_desc) { } +static inline u32 bpf_struct_ops_id(void *kdata) +{ + return 0; +} +static inline void *bpf_struct_ops_map_kdata(struct bpf_map *map) +{ + return NULL; +} +static inline int bpf_struct_ops_map_cgroup_atype(struct bpf_map *map) +{ + return 0; +} +static inline void *bpf_struct_ops_map_cfi_stubs(struct bpf_map *map) +{ + return NULL; +} +static inline bool bpf_struct_ops_valid_to_reg(struct bpf_map *map) +{ + return false; +} +static inline int bpf_struct_ops_link_update_check(struct bpf_map *new_map, + struct bpf_map *old_map, + struct bpf_map *expected_old_map) +{ + return -EOPNOTSUPP; +} #endif @@ -2528,7 +2671,10 @@ u64 bpf_event_output(struct bpf_map *map, u64 flags, void *meta, u64 meta_size, * since other cpus are walking the array of pointers in parallel. */ struct bpf_prog_array_item { - struct bpf_prog *prog; + union { + struct bpf_prog *prog; + void *kdata; + }; union { struct bpf_cgroup_storage *cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE]; u64 bpf_cookie; @@ -2570,6 +2716,7 @@ int bpf_prog_array_copy(struct bpf_prog_array *old_array, struct bpf_prog *include_prog, u64 bpf_cookie, struct bpf_prog_array **new_array); +struct bpf_prog *bpf_prog_dummy(void); struct bpf_run_ctx {}; @@ -2588,6 +2735,7 @@ struct bpf_trace_run_ctx { struct bpf_tramp_run_ctx { struct bpf_run_ctx run_ctx; u64 bpf_cookie; + int retval; struct bpf_run_ctx *saved_run_ctx; }; @@ -2821,8 +2969,10 @@ struct bpf_map *bpf_map_get_curr_or_next(u32 *id); struct bpf_prog *bpf_prog_get_curr_or_next(u32 *id); -int bpf_map_alloc_pages(const struct bpf_map *map, int nid, - unsigned long nr_pages, struct page **page_array); +struct page *bpf_alloc_page(int nid, bool sleepable); +int bpf_alloc_pages(int nid, unsigned long nr_pages, + struct llist_head *pages, bool sleepable); +void bpf_free_pages(struct llist_head *pages); #ifdef CONFIG_MEMCG void bpf_map_memcg_enter(const struct bpf_map *map, struct mem_cgroup **old_memcg, struct mem_cgroup **new_memcg); @@ -3008,6 +3158,8 @@ typedef int (*bpf_iter_attach_target_t)(struct bpf_prog *prog, union bpf_iter_link_info *linfo, struct bpf_iter_aux_info *aux); typedef void (*bpf_iter_detach_target_t)(struct bpf_iter_aux_info *aux); +typedef int (*bpf_iter_check_prog_t)(const struct bpf_prog *prog, + const struct bpf_iter_aux_info *aux); typedef void (*bpf_iter_show_fdinfo_t) (const struct bpf_iter_aux_info *aux, struct seq_file *seq); typedef int (*bpf_iter_fill_link_info_t)(const struct bpf_iter_aux_info *aux, @@ -3017,7 +3169,8 @@ typedef const struct bpf_func_proto * const struct bpf_prog *prog); enum bpf_iter_feature { - BPF_ITER_RESCHED = BIT(0), + BPF_ITER_RESCHED = BIT(0), /* may cond_resched() between objects */ + BPF_ITER_NOSLEEP = BIT(1), /* ->show() can't run sleepable progs */ }; #define BPF_ITER_CTX_ARG_MAX 2 @@ -3025,6 +3178,8 @@ struct bpf_iter_reg { const char *target; bpf_iter_attach_target_t attach_target; bpf_iter_detach_target_t detach_target; + /* may the program run on what the link is attached to? */ + bpf_iter_check_prog_t check_prog; bpf_iter_show_fdinfo_t show_fdinfo; bpf_iter_fill_link_info_t fill_link_info; bpf_iter_get_func_proto_t get_func_proto; @@ -3061,6 +3216,8 @@ void bpf_iter_map_show_fdinfo(const struct bpf_iter_aux_info *aux, struct seq_file *seq); int bpf_iter_map_fill_link_info(const struct bpf_iter_aux_info *aux, struct bpf_link_info *info); +int bpf_iter_map_check_value_write(const struct bpf_prog *prog, + const struct bpf_map *map); int map_set_for_each_callback_args(struct bpf_verifier_env *env, struct bpf_func_state *caller, @@ -3218,7 +3375,7 @@ const struct btf_func_model * bpf_jit_find_kfunc_model(const struct bpf_prog *prog, const struct bpf_insn *insn); int bpf_get_kfunc_addr(const struct bpf_prog *prog, u32 func_id, - u16 btf_fd_idx, u8 **func_addr); + u16 desc_idx, u8 **func_addr); struct bpf_core_ctx { struct bpf_verifier_log *log; @@ -3561,7 +3718,7 @@ bpf_jit_find_kfunc_model(const struct bpf_prog *prog, static inline int bpf_get_kfunc_addr(const struct bpf_prog *prog, u32 func_id, - u16 btf_fd_idx, u8 **func_addr) + u16 desc_idx, u8 **func_addr) { return -ENOTSUPP; } @@ -3667,6 +3824,7 @@ int bpf_dev_bound_kfunc_check(struct bpf_verifier_log *log, void *bpf_dev_bound_resolve_kfunc(struct bpf_prog *prog, u32 func_id); int bpf_prog_dev_bound_init(struct bpf_prog *prog, union bpf_attr *attr); int bpf_prog_dev_bound_inherit(struct bpf_prog *new_prog, struct bpf_prog *old_prog); +void bpf_prog_dev_bound_moved(struct bpf_prog *prog); void bpf_dev_bound_netdev_unregister(struct net_device *dev); static inline bool bpf_prog_is_dev_bound(const struct bpf_prog_aux *aux) @@ -3728,6 +3886,10 @@ static inline int bpf_prog_dev_bound_inherit(struct bpf_prog *new_prog, return -EOPNOTSUPP; } +static inline void bpf_prog_dev_bound_moved(struct bpf_prog *prog) +{ +} + static inline void bpf_dev_bound_netdev_unregister(struct net_device *dev) { } @@ -3858,6 +4020,8 @@ struct bpf_key { #if defined(CONFIG_KEYS) && defined(CONFIG_BPF_SYSCALL) struct bpf_key *bpf_lookup_user_key(s32 serial, u64 flags); struct bpf_key *bpf_lookup_system_key(u64 id); +struct bpf_key *bpf_lookup_keyring(void); +bool bpf_keyring_enforced(void); void bpf_key_put(struct bpf_key *bkey); int bpf_verify_pkcs7_signature(const struct bpf_dynptr *data_p, const struct bpf_dynptr *sig_p, @@ -3878,6 +4042,16 @@ static inline struct bpf_key *bpf_lookup_system_key(u64 id) return NULL; } +static inline struct bpf_key *bpf_lookup_keyring(void) +{ + return NULL; +} + +static inline bool bpf_keyring_enforced(void) +{ + return false; +} + static inline void bpf_key_put(struct bpf_key *bkey) { } @@ -4138,9 +4312,10 @@ void bpf_bprintf_cleanup(struct bpf_bprintf_data *data); int bpf_try_get_buffers(struct bpf_bprintf_buffers **bufs); void bpf_put_buffers(void); -void bpf_prog_stream_init(struct bpf_prog *prog); +int bpf_prog_stream_init(struct bpf_prog *prog, gfp_t gfp_extra_flags); void bpf_prog_stream_free(struct bpf_prog *prog); -int bpf_prog_stream_read(struct bpf_prog *prog, enum bpf_stream_id stream_id, void __user *buf, int len); +int bpf_prog_stream_read(struct bpf_prog *prog, enum bpf_stream_id stream_id, void __user *buf, u32 len); +int bpf_prog_stream_new_fd(struct bpf_prog *prog, enum bpf_stream_id stream_id, u32 flags); void bpf_stream_stage_init(struct bpf_stream_stage *ss); void bpf_stream_stage_free(struct bpf_stream_stage *ss); __printf(2, 3) @@ -4203,7 +4378,7 @@ struct bpf_prog *bpf_prog_find_from_stack(void); int bpf_insn_array_init(struct bpf_map *map, const struct bpf_prog *prog); int bpf_insn_array_ready(struct bpf_map *map); void bpf_insn_array_release(struct bpf_map *map); -void bpf_insn_array_adjust(struct bpf_map *map, u32 off, u32 len); +void bpf_insn_array_adjust(struct bpf_map *map, u32 first, u32 len); void bpf_insn_array_adjust_after_remove(struct bpf_map *map, u32 off, u32 len); #ifdef CONFIG_BPF_SYSCALL @@ -4228,6 +4403,9 @@ static inline bool bpf_map_is_percpu_map(enum bpf_map_type map_type) } } +#define BPF_EXIST_FLAGS (BPF_EXIST | BPF_NOEXIST) +#define BPF_CPU_FLAGS (BPF_F_CPU | BPF_F_ALL_CPUS) + static inline int bpf_map_check_op_flags(struct bpf_map *map, u64 flags, u64 allowed_flags) { u32 cpu; diff --git a/include/linux/bpf_compiler_types.h b/include/linux/bpf_compiler_types.h new file mode 100644 index 000000000000..5c9e71acfdac --- /dev/null +++ b/include/linux/bpf_compiler_types.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __LINUX_BPF_COMPILER_TYPES_H +#define __LINUX_BPF_COMPILER_TYPES_H + +#include <linux/compiler_types.h> + +/* + * Qualifiers for the BPF verifier rather than address spaces: __arena marks a + * pointer into a BPF arena, __nullable a pointer that may be NULL. They reach + * BPF programs as BTF type tags on the members of kernel structs shared with + * them; to kernel code and to sparse the pointers are ordinary. + */ +#ifdef __CHECKER__ +# define __arena +# define __nullable +#else +# define __arena BTF_TYPE_TAG(arena) +# define __nullable BTF_TYPE_TAG(nullable) +#endif + +#endif /* __LINUX_BPF_COMPILER_TYPES_H */ diff --git a/include/linux/bpf_crypto.h b/include/linux/bpf_crypto.h deleted file mode 100644 index a41e71d4e2d9..000000000000 --- a/include/linux/bpf_crypto.h +++ /dev/null @@ -1,24 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* Copyright (c) 2024 Meta Platforms, Inc. and affiliates. */ -#ifndef _BPF_CRYPTO_H -#define _BPF_CRYPTO_H - -struct bpf_crypto_type { - void *(*alloc_tfm)(const char *algo); - void (*free_tfm)(void *tfm); - int (*has_algo)(const char *algo); - int (*setkey)(void *tfm, const u8 *key, unsigned int keylen); - int (*setauthsize)(void *tfm, unsigned int authsize); - int (*encrypt)(void *tfm, const u8 *src, u8 *dst, unsigned int len, u8 *iv); - int (*decrypt)(void *tfm, const u8 *src, u8 *dst, unsigned int len, u8 *iv); - unsigned int (*ivsize)(void *tfm); - unsigned int (*statesize)(void *tfm); - u32 (*get_flags)(void *tfm); - struct module *owner; - char name[14]; -}; - -int bpf_crypto_register_type(const struct bpf_crypto_type *type); -int bpf_crypto_unregister_type(const struct bpf_crypto_type *type); - -#endif /* _BPF_CRYPTO_H */ diff --git a/include/linux/bpf_local_storage.h b/include/linux/bpf_local_storage.h index 9e4f5c45c974..6c643ac51079 100644 --- a/include/linux/bpf_local_storage.h +++ b/include/linux/bpf_local_storage.h @@ -104,6 +104,19 @@ struct bpf_local_storage { refcount_t owner_refcnt;/* Used to pin owner when map_free is uncharging */ }; +/* + * Drop a reference got by refcount_inc_not_zero(&local_storage->owner_refcnt). + * bpf_local_storage_destroy() may have dropped the owner's reference meanwhile + * and wait for the count to reach zero, so this one can be the last. + */ +static inline void +bpf_local_storage_unpin_owner(struct bpf_local_storage *local_storage) +{ + bool last __maybe_unused; + + last = refcount_dec_and_test(&local_storage->owner_refcnt); +} + /* U16_MAX is much more than enough for sk local storage * considering a tcp_sock is ~2k. */ diff --git a/include/linux/bpf_lsm.h b/include/linux/bpf_lsm.h index dda272d78f01..eb4a38eef87e 100644 --- a/include/linux/bpf_lsm.h +++ b/include/linux/bpf_lsm.h @@ -21,6 +21,13 @@ extern bool bpf_lsm_initialized __ro_after_init; #include <linux/lsm_hook_defs.h> #undef LSM_HOOK +/* + * Number of xattr slots the BPF LSM reserves in the array handed to + * security_inode_init_security(), i.e. the maximum number of labels + * a policy may attach to an inode while it is being created. + */ +#define BPF_LSM_INODE_INIT_XATTRS 2 + struct bpf_storage_blob { struct bpf_local_storage __rcu *storage; }; @@ -28,7 +35,7 @@ struct bpf_storage_blob { extern struct lsm_blob_sizes bpf_lsm_blob_sizes; int bpf_lsm_verify_prog(struct bpf_verifier_log *vlog, - const struct bpf_prog *prog); + struct bpf_prog *prog); bool bpf_lsm_is_sleepable_hook(u32 btf_id); bool bpf_lsm_is_trusted(const struct bpf_prog *prog); @@ -71,7 +78,7 @@ static inline bool bpf_lsm_is_trusted(const struct bpf_prog *prog) } static inline int bpf_lsm_verify_prog(struct bpf_verifier_log *vlog, - const struct bpf_prog *prog) + struct bpf_prog *prog) { return -EOPNOTSUPP; } diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index b83ec99f1a13..7baeb831281f 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -9,6 +9,20 @@ #include <linux/filter.h> /* for MAX_BPF_STACK */ #include <linux/tnum.h> #include <linux/cnum.h> +#include <linux/math64.h> /* for div_s64_rem() */ + +/* + * Mathematical modulo, the residue of 'v' modulo 'step', normalized to [0, step). + * This differs from C's '%' operator, which truncates the quotient toward zero + * and so returns a remainder with the sign of the dividend (e.g. -1 % 3 == -1, not 2). + */ +static inline u16 imod(s64 v, u16 step) +{ + s32 rem; + + div_s64_rem(v, step, &rem); + return rem < 0 ? rem + step : rem; +} /* Maximum variable offset umax_value permitted when resolving memory accesses. * In practice this is far bigger than any realistic pointer offset; this limit @@ -19,11 +33,12 @@ * that converting umax_value to int cannot overflow. */ #define BPF_MAX_VAR_SIZ (1 << 29) -/* size of tmp_str_buf in bpf_verifier. - * we need at least 306 bytes to fit full stack mask representation - * (in the "-8,-16,...,-512" form) +/* + * size of tmp_str_buf in bpf_verifier. + * we need at least 1399 bytes to fit full stack mask representation + * (in the "-8,-16,...,-2048" form) */ -#define TMP_STR_BUF_LEN 320 +#define TMP_STR_BUF_LEN 1408 /* Patch buffer size */ #define INSN_BUF_SIZE 32 @@ -124,6 +139,13 @@ struct bpf_reg_state { */ struct cnum64 r64; /* 64-bit range as circular number */ struct cnum32 r32; /* 32-bit range as circular number */ + /* + * The value described by this register, interpreted as s64, lies on + * a line described by a linear equation base + step * k. + * Invariant: base < step. + */ + u16 base; + u16 step; /* For PTR_TO_PACKET, used to find other pointers with the same variable * offset, so they can share range knowledge. * For PTR_TO_MAP_VALUE_OR_NULL this is used to share which map value we @@ -162,9 +184,9 @@ struct bpf_reg_state { * other registers. Kept outside the metadata union for ID remapping * during state comparisons. */ - u32 map_uid; + u32 map_uid:31; /* if (!precise && SCALAR_VALUE) min/max/tnum don't affect safety */ - bool precise; + u32 precise:1; }; static inline s64 reg_smin(const struct bpf_reg_state *reg) @@ -243,55 +265,18 @@ enum bpf_stack_slot_type { #define BPF_REG_SIZE 8 /* size of eBPF register in bytes */ +/* + * Largest number of BPF_REG_SIZE stack slots a single frame can have, sized + * for the largest stack budget any JIT supports. A frame may use any part of + * its program's budget; check_max_stack_depth() enforces the budget on the + * combined depth of frames sharing the kernel stack and on each frame using + * a private stack. + */ +#define MAX_BPF_STACK_SLOTS (MAX_BPF_STACK_JIT / BPF_REG_SIZE) + /* 4-byte stack slot granularity for liveness analysis */ #define BPF_HALF_REG_SIZE 4 #define STACK_SLOT_SZ 4 -#define STACK_SLOTS (MAX_BPF_STACK / BPF_HALF_REG_SIZE) /* 128 */ - -typedef struct { - u64 v[2]; -} spis_t; - -#define SPIS_ZERO ((spis_t){}) -#define SPIS_ALL ((spis_t){{ U64_MAX, U64_MAX }}) - -static inline bool spis_is_zero(spis_t s) -{ - return s.v[0] == 0 && s.v[1] == 0; -} - -static inline bool spis_equal(spis_t a, spis_t b) -{ - return a.v[0] == b.v[0] && a.v[1] == b.v[1]; -} - -static inline spis_t spis_or(spis_t a, spis_t b) -{ - return (spis_t){{ a.v[0] | b.v[0], a.v[1] | b.v[1] }}; -} - -static inline spis_t spis_and(spis_t a, spis_t b) -{ - return (spis_t){{ a.v[0] & b.v[0], a.v[1] & b.v[1] }}; -} - -static inline spis_t spis_not(spis_t s) -{ - return (spis_t){{ ~s.v[0], ~s.v[1] }}; -} - -static inline bool spis_test_bit(spis_t s, u32 slot) -{ - return s.v[slot / 64] & BIT_ULL(slot % 64); -} - -static inline void spis_or_range(spis_t *mask, u32 lo, u32 hi) -{ - u32 w; - - for (w = lo; w <= hi && w < STACK_SLOTS; w++) - mask->v[w / 64] |= BIT_ULL(w % 64); -} #define BPF_REGMASK_ARGS ((1 << BPF_REG_1) | (1 << BPF_REG_2) | \ (1 << BPF_REG_3) | (1 << BPF_REG_4) | \ @@ -343,6 +328,25 @@ struct bpf_retval_range { bool return_32bit; }; +/* SCEV/widening register space: r0..r10 plus every stack slot of a frame. */ +#define BPF_SCEV_REGS_NUM (MAX_BPF_REG + MAX_BPF_STACK_SLOTS) + +struct bpf_loop_iters { + u32 min_header_count; /* min number of times header is executed */ + u32 max_header_count; /* max number of times header is executed */ + unsigned long precise_regs[BITS_TO_LONGS(BPF_SCEV_REGS_NUM)]; +}; + +struct loop_stack_entry { + struct bpf_verifier_state *entry_state; + u32 min_header_count; + u32 max_header_count; + u32 loop_id:31; + u32 terminates:1; +}; + +#define LOOP_STACK_SIZE 16 + /* state of the program: * type of all registers and stack info */ @@ -375,6 +379,11 @@ struct bpf_func_state { bool in_async_callback_fn; bool in_exception_callback_fn; bool no_stack_arg_load; + /* + * The helper invokes this callback under rcu_read_lock(), so the + * callback and its callees run in an RCU read-side critical section. + */ + bool in_rcu_callback_fn; /* For callback calling functions that limit number of possible * callback executions (e.g. bpf_loop) keeps track of current * simulated iteration number. @@ -388,6 +397,11 @@ struct bpf_func_state { u32 callback_depth; /* Instructions processed in this frame and callees on the current path. */ u32 insns_subtotal; + /* + * Control-flow loop nesting at the current insn within this frame's + * subprogram (loops never cross subprogram boundaries). + */ + u32 loop_stack_cnt; /* The following fields should be last. See copy_func_state() */ /* The state of the stack. Each element of the array describes BPF_REG_SIZE @@ -405,6 +419,7 @@ struct bpf_func_state { u16 out_stack_arg_cnt; /* Number of outgoing on-stack argument slots */ struct bpf_reg_state *stack_arg_regs; /* Outgoing on-stack arguments */ + struct loop_stack_entry *loop_stack; }; #define MAX_CALL_FRAMES 16 @@ -421,30 +436,37 @@ enum { INSN_F_STACK_ARG_ACCESS = BIT(3), }; +/* Registers linked to one jump condition that a history entry can record */ +#define BPF_LINKED_REGS_MAX 5 + struct bpf_jmp_history_entry { /* insn idx can't be bigger than 1 million */ u32 idx : 20; u32 frame : 4; /* stack access frame number */ - u32 spi : 6; /* stack slot index (0..63) */ - u32 : 2; - u32 prev_idx : 20; /* special INSN_F_xxx flags */ u32 flags : 4; - u32 : 8; + u32 : 4; + u32 prev_idx : 20; + u32 spi : 12; /* stack slot index */ /* - * additional registers that need precision tracking when this - * jump is backtracked, vector of five 11-bit records + * Scalar registers and spilled scalars linked to the condition of + * this jump, which need precision tracking together when the jump is + * backtracked. Each is packed as 4 bits of frame number, one bit + * telling a register from a stack slot and 11 bits of register or + * slot index, see linked_regs_pack(). */ - u64 linked_regs; + u16 linked_regs[BPF_LINKED_REGS_MAX]; + u8 linked_regs_cnt; }; static_assert(MAX_CALL_FRAMES <= (1 << 4)); -static_assert(MAX_BPF_STACK / 8 <= (1 << 6)); +static_assert(MAX_BPF_STACK_SLOTS <= (1 << 12)); /* Maximum number of bpf_reg_state objects that can exist at once */ #define MAX_STACK_ARG_SLOTS (MAX_BPF_FUNC_ARGS - MAX_BPF_FUNC_REG_ARGS) -#define BPF_ID_MAP_SIZE ((MAX_BPF_REG + MAX_BPF_STACK / BPF_REG_SIZE + \ - MAX_STACK_ARG_SLOTS) * MAX_CALL_FRAMES) +#define BPF_ID_MAP_SIZE ((MAX_BPF_REG + MAX_BPF_STACK_SLOTS + MAX_STACK_ARG_SLOTS) * \ + MAX_CALL_FRAMES) + struct bpf_verifier_state { /* call stack tracking */ struct bpf_func_state *frame[MAX_CALL_FRAMES]; @@ -510,8 +532,13 @@ struct bpf_verifier_state { bool speculative; bool in_sleepable; + /* The callee left with unwind_exit. Next insn is unwind_goto. */ + bool unwound; - /* first and last insn idx of this verifier state */ + /* + * First and last insn idx of this verifier state. + * last_insn_idx is -1 if no instructions have been executed yet. + */ u32 first_insn_idx; u32 last_insn_idx; /* if this state is a backedge state then equal_state @@ -530,12 +557,31 @@ struct bpf_verifier_state { u32 may_goto_depth; }; +/* Number of BPF_REG_SIZE stack slots tracked for the frame so far. */ +static inline u32 bpf_stack_nr_slots(const struct bpf_func_state *frame) +{ + return frame->allocated_stack / BPF_REG_SIZE; +} + +/* + * Stack slot @spi of @frame, covering bytes [fp - (spi + 1) * 8, fp - spi * 8). + * The caller must ensure spi < bpf_stack_nr_slots(frame), see bpf_grow_stack_state(). + */ +static inline struct bpf_stack_state *bpf_stack_slot(const struct bpf_func_state *frame, u32 spi) +{ + return &frame->stack[spi]; +} + static inline struct bpf_reg_state * bpf_get_spilled_reg(int slot, struct bpf_func_state *frame, u32 mask) { - if (slot < frame->allocated_stack / BPF_REG_SIZE && - (1 << frame->stack[slot].slot_type[BPF_REG_SIZE - 1]) & mask) - return &frame->stack[slot].spilled_ptr; + struct bpf_stack_state *ss; + + if (slot >= bpf_stack_nr_slots(frame)) + return NULL; + ss = bpf_stack_slot(frame, slot); + if ((1 << ss->slot_type[BPF_REG_SIZE - 1]) & mask) + return &ss->spilled_ptr; return NULL; } @@ -551,7 +597,7 @@ bpf_get_spilled_stack_arg(int slot, struct bpf_func_state *frame) /* Iterate over 'frame', setting 'reg' to either NULL or a spilled register. */ #define bpf_for_each_spilled_reg(iter, frame, reg, mask) \ for (iter = 0, reg = bpf_get_spilled_reg(iter, frame, mask); \ - iter < frame->allocated_stack / BPF_REG_SIZE; \ + iter < bpf_stack_nr_slots(frame); \ iter++, reg = bpf_get_spilled_reg(iter, frame, mask)) /* Iterate over 'frame', setting 'reg' to either NULL or a spilled stack arg. */ @@ -576,7 +622,7 @@ bpf_get_spilled_stack_arg(int slot, struct bpf_func_state *frame) bpf_for_each_spilled_reg(___j, __state, __reg, __mask) { \ if (!__reg) \ continue; \ - __stack = &__state->stack[___j]; \ + __stack = bpf_stack_slot(__state, ___j); \ (void)(__expr); \ } \ __stack = NULL; \ @@ -640,6 +686,52 @@ struct bpf_iarray { u32 items[]; }; +#define iarray_for_each(item, arr) \ + for (int ___idx = 0; \ + ___idx < (arr)->cnt && ({ item = (arr)->items[___idx]; 1; }); \ + ___idx++) + +#define MAX_BACKEDGES 16 +#define MAX_LOOP_EXITS 256 + +/* Number of exit conditions dominating a backedge. */ +#define MAX_LATCHES 4 + +struct bpf_backedge { + int from; + /* + * Conditional jumps within the loop that dominate 'from' and have + * a successor outside the loop, nearest first. latches[0] is the + * latch; the remaining entries are additional exit conditions. + * Each usable condition gives an upper bound on the iteration count, + * so keeping only a subset is sound. + */ + int latches[MAX_LATCHES]; + u8 latches_cnt; +}; + +struct bpf_loop_exit { + int from; /* instruction inside the loop */ + int to; /* instruction outside the loop */ +}; + +struct bpf_loop { + struct bpf_backedge backedges[MAX_BACKEDGES]; + /* edges exiting from this loop, includes edges from nested loops */ + struct bpf_loop_exit *exits; + int backedges_cnt; + int exits_cnt; + bool irreducible; + bool backedges_overflow; + bool exits_overflow; + /* + * Loop-entry registers used to compute stack addresses for loads, + * stores and calls requiring fixed stack offsets, in this and nested + * loops. + */ + unsigned long store_base_regs[BITS_TO_LONGS(BPF_SCEV_REGS_NUM)]; +}; + struct bpf_insn_aux_data { union { enum bpf_reg_type ptr_type; /* pointer type for load/store insns */ @@ -671,45 +763,63 @@ struct bpf_insn_aux_data { /* remember the offset of node field within type to rewrite */ u64 insert_off; }; - struct bpf_iarray *jt; /* jump table for gotox or bpf_tailcall call instruction */ struct btf_struct_meta *kptr_struct_meta; u64 map_key_state; /* constant (32 bit) key tracking for maps */ + /* + * Per-instruction summary of stack slots in the current frame + * that this instruction may write to. + */ + DECLARE_BITMAP(may_write_mask, MAX_BPF_STACK_SLOTS); + DECLARE_BITMAP(live_stack_before, MAX_BPF_STACK_SLOTS); + u16 stack_ptrs; /* bitmask of regs that may hold a frame pointer here (arg_track) */ int ctx_field_size; /* the ctx field size for load insn, maybe 0 */ u32 seen; /* this insn was processed by the verifier at env->pass_cnt */ - bool nospec; /* do not execute this instruction speculatively */ - bool nospec_result; /* result is unsafe under speculation, nospec must follow */ - bool zext_dst; /* this insn zero extends dst reg */ - bool needs_zext; /* alu op needs to clear upper bits */ - bool prevent_zext; /* alu op cannot be zext (already used with 64-bit scalars) */ - bool non_sleepable; /* helper/kfunc may be called from non-sleepable context */ - bool is_iter_next; /* bpf_iter_<type>_next() kfunc call */ - bool call_with_percpu_alloc_ptr; /* {this,per}_cpu_ptr() with prog percpu alloc */ - u8 alu_state; /* used in combination with alu_limit */ + u64 nospec:1; /* do not execute this instruction speculatively */ + u64 nospec_result:1; /* result is unsafe under speculation, nospec must follow */ + u64 zext_dst:1; /* this insn zero extends dst reg */ + u64 needs_zext:1; /* alu op needs to clear upper bits */ + u64 prevent_zext:1; /* alu op cannot be zext (already used with 64-bit scalars) */ + u64 non_sleepable:1; /* helper/kfunc may be called from non-sleepable context */ + u64 is_iter_next:1; /* bpf_iter_<type>_next() kfunc call */ + u64 call_with_percpu_alloc_ptr:1; /* {this,per}_cpu_ptr() with prog percpu alloc */ + u64 bb_end:1; /* last instruction of a basic block */ + u64 need_scev:1; + u64 alu_state:5; /* used in combination with alu_limit */ /* true if STX or LDX instruction is a part of a spill/fill * pattern for a bpf_fastcall call. */ - u8 fastcall_pattern:1; + u64 fastcall_pattern:1; /* for CALL instructions, a number of spill/fill pairs in the * bpf_fastcall pattern. */ - u8 fastcall_spills_num:3; - u8 arg_prog:4; + u64 fastcall_spills_num:3; + u64 arg_prog:4; + /* + * What a load found, on any path: an __arena pointer member of a + * kernel struct, which arena_member_fixup() rewrites, one that is + * __nullable as well, or anything else through any pointer. + */ + u64 arena_member:1; + u64 arena_member_nullable:1; + u64 non_arena_load:1; - /* below fields are initialized once */ - unsigned int orig_idx; /* original instruction index */ - u32 jmp_point:1; - u32 prune_point:1; + /* below flags are initialized once */ + u64 jmp_point:1; + u64 prune_point:1; /* ensure we check state equivalence and save state checkpoint and * this instruction, regardless of any heuristics */ - u32 force_checkpoint:1; + u64 force_checkpoint:1; /* true if instruction is a call to a helper function that * accepts callback function as a parameter. */ - u32 calls_callback:1; - u32 indirect_target:1; /* if it is an indirect jump target */ + u64 calls_callback:1; + u64 indirect_target:1; /* if it is an indirect jump target */ + u64 non_stack_access:1; /* instruction can access non-stack memory */ /* true if some jump or call instruction targets this instruction */ - u32 jump_target:1; + u64 jump_target:1; + + unsigned int orig_idx; /* original instruction index, initialized once */ /* * CFG strongly connected component this instruction belongs to, * zero if it is a singleton SCC. @@ -729,12 +839,23 @@ struct bpf_insn_aux_data { u16 const_reg_map_mask; u16 const_reg_subprog_mask; u32 const_reg_vals[10]; + /* + * Header index of the innermost loop containing this instruction, -1 if none. + * For a loop header, identifies the parent loop instead. + */ + s32 loop_header; + /* additional information about the loop if this instruction is a loop header */ + struct bpf_loop *loop; }; #define MAX_USED_MAPS 64 /* max number of maps accessed by one eBPF program */ #define MAX_USED_BTFS 64 /* max number of BTFs accessed by one BPF program */ -#define BPF_VERIFIER_TMP_LOG_SIZE 1024 +/* + * Longest line the verifier log can carry: a full stack mask of + * MAX_BPF_STACK_SLOTS slots, see TMP_STR_BUF_LEN, plus its prefix. + */ +#define BPF_VERIFIER_TMP_LOG_SIZE 2048 struct bpf_verifier_log { /* Logical start and end positions of a "log window" of the verifier log. @@ -787,6 +908,21 @@ int bpf_log_attr_finalize(struct bpf_log_attr *attr, struct bpf_verifier_log *lo #define BPF_MAX_SUBPROGS 256 +/* + * A pointer to a static subprog in the value of a frozen read-only array map: + * a 64-bit value that is the offset in bytes of the first instruction of + * the subprog in the program. + */ +struct bpf_func_ptr { + struct bpf_map *map; + u32 map_off; /* offset of the pointer in the value of the map */ + u32 orig_off; /* what the map has: the first instruction of the subprog */ + u32 xlated_off; /* the same after instructions were patched and removed */ +}; + +/* the subprog that a bpf_func_ptr pointed to was removed as dead code */ +#define BPF_FUNC_PTR_DELETED ((u32)-1) + struct bpf_subprog_arg_info { enum bpf_arg_type arg_type; union { @@ -806,7 +942,12 @@ struct bpf_subprog_info { u32 start; /* insn idx of function entry point */ u32 linfo_idx; /* The idx to the main_prog->aux->linfo */ u32 postorder_start; /* The idx to the env->cfg.insn_postorder */ - u32 exit_idx; /* Index of one of the BPF_EXIT instructions in this subprogram */ + /* + * Index of one of the BPF_EXIT instructions in this subprogram, or + * U32_MAX when it has none. + */ + u32 exit_idx; + struct bpf_iarray *jt; /* jump table shared by all gotox of this subprogram */ u16 stack_depth; /* max. stack depth used by this function */ u16 stack_extra; u32 insns_total; @@ -817,17 +958,20 @@ struct bpf_subprog_info { s16 fastcall_stack_off; bool has_tail_call: 1; bool might_throw: 1; + bool can_unwind: 1; /* has unwind_exit */ bool tail_call_reachable: 1; bool has_ld_abs: 1; bool is_cb: 1; bool is_async_cb: 1; bool is_exception_cb: 1; bool args_cached: 1; + /* true if the return value is passed in the R0:R2 register pair */ + bool ret_reg_pair: 1; /* true if bpf_fastcall stack region is used by functions that can't be inlined */ bool keep_fastcall_stack: 1; bool changes_pkt_data: 1; bool might_sleep: 1; - u8 arg_cnt:4; + u8 arg_slot_cnt:4; enum priv_stack_mode priv_stack_mode; struct bpf_subprog_arg_info args[MAX_BPF_FUNC_ARGS]; @@ -837,8 +981,8 @@ struct bpf_subprog_info { static inline u16 bpf_in_stack_arg_cnt(const struct bpf_subprog_info *sub) { - if (sub->arg_cnt > MAX_BPF_FUNC_REG_ARGS) - return sub->arg_cnt - MAX_BPF_FUNC_REG_ARGS; + if (sub->arg_slot_cnt > MAX_BPF_FUNC_REG_ARGS) + return sub->arg_slot_cnt - MAX_BPF_FUNC_REG_ARGS; return 0; } @@ -849,7 +993,7 @@ struct backtrack_state { struct bpf_verifier_env *env; u32 frame; u32 reg_masks[MAX_CALL_FRAMES]; - u64 stack_masks[MAX_CALL_FRAMES]; + unsigned long stack_masks[MAX_CALL_FRAMES][BITS_TO_LONGS(MAX_BPF_STACK_SLOTS)]; u8 stack_arg_masks[MAX_CALL_FRAMES]; }; @@ -905,6 +1049,7 @@ struct bpf_scc_info { }; struct bpf_liveness; +struct scev; struct bpf_fd_array { union { @@ -954,6 +1099,8 @@ struct bpf_verifier_env { bool bypass_spec_v1; bool bypass_spec_v4; bool seen_direct_write; + /* a global function is verified for BPF_FUNC_CTX_NO_RCU_LOCK */ + bool no_rcu_read_lock; bool seen_exception; bool signature; u32 insn_aux_data_len; @@ -961,9 +1108,24 @@ struct bpf_verifier_env { const struct bpf_line_info *prev_linfo; struct bpf_verifier_log log; struct bpf_diag *diag; + struct bpf_func_proto bpf_subprog_scratch; struct bpf_subprog_info subprog_info[BPF_MAX_SUBPROGS + 2]; /* max + 2 for the fake and exception subprogs */ /* subprog indices sorted in topological order: leaves first, callers last */ int subprog_topo_order[BPF_MAX_SUBPROGS + 2]; + /* + * Pointers to static subprogs found in frozen read-only maps of the + * program, see resolve_func_ptrs(). Sorted by map and map_off. + */ + struct bpf_func_ptr *func_ptrs; + u32 func_ptr_cnt; + bool has_callx; + /* + * Call graph edges created by callx instructions. A bitmap of + * subprog_cnt * subprog_cnt bits, where bit (caller * subprog_cnt + callee) + * is set when the main verification pass sees 'caller' calling 'callee' + * via callx. Allocated when the first such edge is recorded. + */ + unsigned long *callx_edges; union { struct bpf_idmap idmap_scratch; struct bpf_idset idset_scratch; @@ -976,14 +1138,17 @@ struct bpf_verifier_env { * see bpf_subprog_info->postorder_start. */ int *insn_postorder; + int *postorder_nums; int cur_stack; /* current position in the insn_postorder vector */ int cur_postorder; + u32 gotox_edges; + bool subprog_jts_ready; } cfg; struct backtrack_state bt; struct bpf_jmp_history_entry *cur_hist_ent; - /* Per-callsite copy of parent's converged at_stack_in for cross-frame fills. */ - struct arg_track **callsite_at_stack; + /* Per-callsite snapshot of the parent's tracked spill slots for cross-frame fills. */ + struct spill_snapshot **callsite_at_stack; u32 pass_cnt; /* number of times do_check() was called */ u32 subprog_cnt; /* number of instructions analyzed by the verifier */ @@ -992,6 +1157,8 @@ struct bpf_verifier_env { u32 prev_jmps_processed, jmps_processed; /* maximum combined stack depth */ u32 max_stack_depth; + /* stack budget of the program, see bpf_prog_stack_limit() */ + u32 stack_limit; /* total verification time */ u64 verification_time; /* maximum number of verifier states kept in 'branching' instructions */ @@ -1027,7 +1194,7 @@ struct bpf_verifier_env { */ u32 scratched_regs; /* Same as scratched_regs but for stack slots */ - u64 scratched_stack_slots; + DECLARE_BITMAP(scratched_stack_slots, MAX_BPF_STACK_SLOTS); u64 prev_log_pos, prev_insn_print_pos; /* buffer used to temporary hold constants as scalar registers */ struct bpf_reg_state fake_reg[1]; @@ -1047,6 +1214,8 @@ struct bpf_verifier_env { u32 scc_cnt; struct bpf_iarray *succ; struct bpf_iarray *gotox_tmp_buf; + int *idoms; + struct scev *scev; }; static inline struct bpf_func_info_aux *subprog_aux(struct bpf_verifier_env *env, int subprog) @@ -1059,8 +1228,13 @@ static inline struct bpf_subprog_info *subprog_info(struct bpf_verifier_env *env return &env->subprog_info[subprog]; } +static inline bool bpf_ret_reg_pair(struct bpf_verifier_env *env, int subprog) +{ + return subprog_info(env, subprog)->ret_reg_pair; +} + struct bpf_call_summary { - u8 num_params; + u8 arg_slot_cnt; bool is_void; bool fastcall; }; @@ -1083,6 +1257,12 @@ static inline bool bpf_pseudo_kfunc_call(const struct bpf_insn *insn) insn->src_reg == BPF_PSEUDO_KFUNC_CALL; } +/* callx: indirect call of a bpf subprog whose address is in insn->dst_reg */ +static inline bool bpf_is_callx(const struct bpf_insn *insn) +{ + return insn->code == (BPF_JMP | BPF_CALL | BPF_X); +} + __printf(2, 0) void bpf_verifier_vlog(struct bpf_verifier_log *log, const char *fmt, va_list args); __printf(2, 3) void bpf_verifier_log_write(struct bpf_verifier_env *env, @@ -1097,6 +1277,8 @@ int bpf_vlog_finalize(struct bpf_verifier_log *log, u32 *log_size_actual); __printf(3, 4) void verbose_linfo(struct bpf_verifier_env *env, u32 insn_off, const char *prefix_fmt, ...); +void bpf_verbose_snum(struct bpf_verifier_env *env, s64 num); +void bpf_verbose_snum32(struct bpf_verifier_env *env, s32 num); #define verifier_bug_if(cond, env, fmt, args...) \ ({ \ @@ -1131,6 +1313,12 @@ static inline bool bpf_is_force_checkpoint(struct bpf_verifier_env *env, int ins return env->insn_aux_data[insn_idx].force_checkpoint; } +static inline void clear_scalar_id(struct bpf_reg_state *reg) +{ + reg->id = 0; + reg->delta = 0; +} + static inline void mark_calls_callback(struct bpf_verifier_env *env, int idx) { env->insn_aux_data[idx].calls_callback = true; @@ -1209,11 +1397,14 @@ int bpf_check_attach_target(struct bpf_verifier_log *log, const struct bpf_prog *tgt_prog, u32 btf_id, struct bpf_attach_target_info *tgt_info); +int bpf_check_ext_prologue(struct bpf_verifier_log *log, + const struct bpf_prog *tgt_prog, u32 btf_id); void bpf_free_kfunc_btf_tab(struct bpf_kfunc_btf_tab *tab); int mark_chain_precision(struct bpf_verifier_env *env, int regno); int bpf_is_state_visited(struct bpf_verifier_env *env, int insn_idx); +int bpf_split_cur_state(struct bpf_verifier_env *env); int bpf_update_branch_counts(struct bpf_verifier_env *env, struct bpf_verifier_state *st); void bpf_clear_jmp_history(struct bpf_verifier_state *state); @@ -1223,11 +1414,14 @@ struct list_head *bpf_explored_state(struct bpf_verifier_env *env, int idx); void bpf_free_verifier_state(struct bpf_verifier_state *state, bool free_self); void bpf_free_backedges(struct bpf_scc_visit *visit); int bpf_push_jmp_history(struct bpf_verifier_env *env, struct bpf_verifier_state *cur, - int insn_flags, int spi, int frame, u64 linked_regs); + int insn_flags, int spi, int frame, const u16 *linked_regs, + u8 linked_regs_cnt); void bpf_bt_sync_linked_regs(struct backtrack_state *bt, struct bpf_jmp_history_entry *hist); void bpf_mark_reg_not_init(const struct bpf_verifier_env *env, struct bpf_reg_state *reg); +void bpf_mark_reg_known_scalar(struct bpf_reg_state *reg, u64 imm); void bpf_mark_reg_unknown_imprecise(struct bpf_reg_state *reg); +int bpf_grow_stack_state(struct bpf_verifier_env *env, struct bpf_func_state *state, int size); void bpf_mark_all_scalars_precise(struct bpf_verifier_env *env, struct bpf_verifier_state *st); void bpf_clear_singular_ids(struct bpf_verifier_env *env, struct bpf_verifier_state *st); @@ -1241,6 +1435,24 @@ static inline int bpf_get_spi(s32 off) } /* + * Stack a program may use in total: combined over the frames of a call + * chain on the kernel stack, or per frame on a private stack. Any single + * frame may reach that deep. Only a JIT that lays out such frames may go + * beyond MAX_BPF_STACK, the interpreter's frame size, and only one whose + * tail calls let the target set up its own frame: without subprogram + * tail calls, do_misc_fixups() gives every program with tail calls a + * MAX_BPF_STACK frame, which a deeper frame would overrun. + */ +static inline u32 bpf_prog_stack_limit(const struct bpf_prog *prog) +{ + /* an offloaded program never runs on the host JIT, whatever it supports */ + if (prog->jit_requested && !bpf_prog_is_offloaded(prog->aux) && + bpf_jit_supports_large_stack() && bpf_jit_supports_subprog_tailcalls()) + return MAX_BPF_STACK_JIT; + return MAX_BPF_STACK; +} + +/* * Return the function state a stack pointer register refers to. frameno * shares storage with other pointer metadata, so return NULL for any * other register type instead of indexing frame[] with aliased bytes. @@ -1273,6 +1485,15 @@ static inline bool bpf_is_spilled_reg(const struct bpf_stack_state *stack) return stack->slot_type[BPF_REG_SIZE - 1] == STACK_SPILL; } +static inline int bpf_spill_size(const struct bpf_stack_state *stack) +{ + int i, spill_size = 0; + + for (i = BPF_REG_SIZE - 1; i >= 0 && stack->slot_type[i] == STACK_SPILL; i--) + spill_size++; + return spill_size; +} + static inline bool bpf_is_spilled_scalar_reg(const struct bpf_stack_state *stack) { return bpf_is_spilled_reg(stack) && stack->spilled_ptr.type == SCALAR_VALUE; @@ -1290,12 +1511,7 @@ static inline void bpf_bt_set_frame_reg(struct backtrack_state *bt, u32 frame, u static inline void bpf_bt_set_frame_slot(struct backtrack_state *bt, u32 frame, u32 slot) { - bt->stack_masks[frame] |= 1ull << slot; -} - -static inline void bpf_bt_set_frame_slot_mask(struct backtrack_state *bt, u32 frame, u64 mask) -{ - bt->stack_masks[frame] |= mask; + __set_bit(slot, bt->stack_masks[frame]); } static inline void bt_set_frame_stack_arg_slot(struct backtrack_state *bt, u32 frame, u32 slot) @@ -1310,10 +1526,17 @@ static inline bool bt_is_frame_reg_set(struct backtrack_state *bt, u32 frame, u3 static inline bool bt_is_frame_slot_set(struct backtrack_state *bt, u32 frame, u32 slot) { - return bt->stack_masks[frame] & (1ull << slot); + return test_bit(slot, bt->stack_masks[frame]); } bool bpf_map_is_rdonly(const struct bpf_map *map); +struct bpf_func_ptr *bpf_map_func_ptrs(struct bpf_verifier_env *env, + const struct bpf_map *map, u32 *cnt); +struct bpf_func_ptr *bpf_map_range_func_ptrs(struct bpf_verifier_env *env, + const struct bpf_map *map, + u64 off, u64 size, u32 *cnt); +void bpf_adjust_func_ptrs(struct bpf_verifier_env *env, u32 off, u32 len); +void bpf_adjust_func_ptrs_after_remove(struct bpf_verifier_env *env, u32 off, u32 len); int bpf_map_direct_read(struct bpf_map *map, int off, int size, u64 *val, bool is_ldsx); @@ -1441,7 +1664,7 @@ static inline void mark_reg_scratched(struct bpf_verifier_env *env, u32 regno) static inline void mark_stack_slot_scratched(struct bpf_verifier_env *env, u32 spi) { - env->scratched_stack_slots |= 1ULL << spi; + __set_bit(spi, env->scratched_stack_slots); } static inline bool reg_scratched(const struct bpf_verifier_env *env, u32 regno) @@ -1449,27 +1672,28 @@ static inline bool reg_scratched(const struct bpf_verifier_env *env, u32 regno) return (env->scratched_regs >> regno) & 1; } -static inline bool stack_slot_scratched(const struct bpf_verifier_env *env, u64 regno) +static inline bool stack_slot_scratched(const struct bpf_verifier_env *env, u32 spi) { - return (env->scratched_stack_slots >> regno) & 1; + return test_bit(spi, env->scratched_stack_slots); } static inline bool verifier_state_scratched(const struct bpf_verifier_env *env) { - return env->scratched_regs || env->scratched_stack_slots; + return env->scratched_regs || + !bitmap_empty(env->scratched_stack_slots, MAX_BPF_STACK_SLOTS); } static inline void mark_verifier_state_clean(struct bpf_verifier_env *env) { env->scratched_regs = 0U; - env->scratched_stack_slots = 0ULL; + bitmap_zero(env->scratched_stack_slots, MAX_BPF_STACK_SLOTS); } /* Used for printing the entire verifier state. */ static inline void mark_verifier_state_scratched(struct bpf_verifier_env *env) { env->scratched_regs = ~0U; - env->scratched_stack_slots = ~0ULL; + bitmap_fill(env->scratched_stack_slots, MAX_BPF_STACK_SLOTS); } static inline bool bpf_stack_narrow_access_ok(int off, int fill_size, int spill_size) @@ -1504,14 +1728,25 @@ struct bpf_subprog_info *bpf_find_containing_subprog(struct bpf_verifier_env *en const char *bpf_subprog_name(const struct bpf_verifier_env *env, int subprog); int bpf_jmp_offset(struct bpf_insn *insn); struct bpf_iarray *bpf_insn_successors(struct bpf_verifier_env *env, u32 idx); -void bpf_fmt_stack_mask(char *buf, ssize_t buf_sz, u64 stack_mask); +void bpf_fmt_stack_mask(char *buf, ssize_t buf_sz, const unsigned long *stack_mask); bool bpf_subprog_is_global(const struct bpf_verifier_env *env, int subprog); +/* Kinds of member a by-value struct or union may be composed of. */ +enum btf_member_kind { + BTF_MEMBER_SCALAR = BIT(0), /* an int or an enum */ + BTF_MEMBER_ARENA_PTR = BIT(1), /* a pointer carrying the "arena" type tag */ +}; + +bool btf_struct_is_composed_of(struct bpf_verifier_env *env, const struct btf *btf, + const struct btf_type *t, u32 member_kinds); +u32 btf_func_arg_align(const struct btf *btf, const struct btf_type *t); + int bpf_find_subprog(struct bpf_verifier_env *env, int off); bool bpf_is_throw_kfunc(struct bpf_insn *insn); int bpf_compute_const_regs(struct bpf_verifier_env *env); int bpf_prune_dead_branches(struct bpf_verifier_env *env); int bpf_check_cfg(struct bpf_verifier_env *env); +void bpf_free_subprog_jts(struct bpf_verifier_env *env); int bpf_compute_postorder(struct bpf_verifier_env *env); int bpf_compute_scc(struct bpf_verifier_env *env); @@ -1540,12 +1775,15 @@ struct ref_obj_desc { }; /* - * A memory argument a call fills in. The verifier allows the stack to be uninitialized if - * the range is a known constant. Stack slots are marked as STACK_MISC by check_mem_access(). + * Generic MEM_UNINIT arguments, indexed by ABI slot. var_size_mask excludes + * variable-sized buffers from raw mode without losing the output annotation. + * size records constant ranges to mark initialized after checking all arguments, + * only when the caller is allowed to read uninitialized stack memory. */ struct arg_raw_mem_desc { - u8 regno; - int size; + u16 mask; + u16 var_size_mask; + int size[MAX_BPF_FUNC_ARGS]; }; /* Size of PTR_TO_MEM returned, taken from a constant allocation-size argument */ @@ -1565,6 +1803,7 @@ struct bpf_call_arg_meta { struct btf *btf; u32 func_id; const struct bpf_func_proto *fn; + const struct btf_type *func_proto; u8 release_regno; u32 ret_btf_id; u32 subprogno; @@ -1572,20 +1811,25 @@ struct bpf_call_arg_meta { struct bpf_dynptr_desc dynptr; struct ref_obj_desc ref_obj; struct ret_mem_desc ret_mem; + struct arg_raw_mem_desc arg_raw_mem; + + /* Only set by subprog */ + bool subprog_may_change_pkt; /* Only set by kfunc */ bool r0_rdonly; u32 kfunc_flags; - const struct btf_type *func_proto; const char *func_name; struct arg_constant_desc arg_constant; + /* The type of the dynptr that the kfunc writes through */ + enum bpf_dynptr_type dynptr_write_type; /* arg_{btf,btf_id,owning_ref} are used by kfunc-specific handling, * generally to pass info about user-defined local kptr types to later * verification logic * bpf_obj_drop/bpf_percpu_obj_drop * Record the local kptr type to be drop'd - * bpf_refcount_acquire (via KF_ARG_PTR_TO_REFCOUNTED_KPTR arg type) + * bpf_refcount_acquire (via ARG_PTR_TO_REFCOUNTED_KPTR arg type) * Record the local kptr type to be refcount_incr'd and use * arg_owning_ref to determine whether refcount_acquire should be * fallible @@ -1593,7 +1837,6 @@ struct bpf_call_arg_meta { struct btf *arg_btf; u32 arg_btf_id; bool arg_owning_ref; - bool arg_prog; struct { struct btf_field *field; @@ -1611,7 +1854,6 @@ struct bpf_call_arg_meta { s64 const_map_key; struct btf *ret_btf; struct btf_field *kptr_field; - struct arg_raw_mem_desc arg_raw_mem; }; int bpf_get_helper_proto(struct bpf_verifier_env *env, int func_id, @@ -1638,17 +1880,37 @@ bool bpf_is_may_goto_insn(struct bpf_insn *insn); void bpf_verbose_insn(struct bpf_verifier_env *env, struct bpf_insn *insn); bool bpf_get_call_summary(struct bpf_verifier_env *env, struct bpf_insn *call, struct bpf_call_summary *cs); -s64 bpf_helper_stack_access_bytes(struct bpf_verifier_env *env, - struct bpf_insn *insn, int arg, - int insn_idx); -s64 bpf_kfunc_stack_access_bytes(struct bpf_verifier_env *env, - struct bpf_insn *insn, int arg, - int insn_idx); +/* Stack effects used for state pruning; must_write implies may_write. */ +struct arg_access_info { + u32 size; /* Maximum extent; U32_MAX if unknown. */ + u8 may_read:1; /* Incoming contents or initialization may be needed. */ + u8 may_write:1; + u8 must_write:1; /* Prior verifier state is destroyed throughout size. */ +}; + +struct arg_access_info +bpf_helper_stack_access_bytes(struct bpf_verifier_env *env, + struct bpf_insn *insn, int arg, int insn_idx); +struct arg_access_info +bpf_kfunc_stack_access_bytes(struct bpf_verifier_env *env, + struct bpf_insn *insn, int arg, int insn_idx); +struct arg_access_info +bpf_global_subprog_stack_access_bytes(struct bpf_verifier_env *env, + struct bpf_insn *insn, int arg, int insn_idx); int bpf_compute_subprog_arg_access(struct bpf_verifier_env *env); +bool bpf_same_memory_origin(const struct bpf_reg_state *reg_a, + const struct bpf_reg_state *reg_b); +int bpf_set_reg_range(struct bpf_verifier_env *env, struct bpf_reg_state *reg, + struct cnum64 r64, struct cnum32 r32, u16 base, u16 step); +bool bpf_reg_union_compatible(const struct bpf_reg_state *a, const struct bpf_reg_state *b); +int bpf_reg_union(struct bpf_verifier_env *env, struct bpf_reg_state *acc, + const struct bpf_reg_state *src); int bpf_stack_liveness_init(struct bpf_verifier_env *env); void bpf_stack_liveness_free(struct bpf_verifier_env *env); int bpf_live_stack_query_init(struct bpf_verifier_env *env, struct bpf_verifier_state *st); +const unsigned long *bpf_may_write_mask(struct bpf_verifier_env *env, u32 insn_idx); +bool bpf_needs_fixed_stack_off(struct bpf_verifier_env *env, int insn_idx); bool bpf_stack_slot_alive(struct bpf_verifier_env *env, u32 frameno, u32 spi); int bpf_compute_live_registers(struct bpf_verifier_env *env); @@ -1680,6 +1942,21 @@ static inline u64 bpf_map_key_immediate(const struct bpf_insn_aux_data *aux) return aux->map_key_state & ~(BPF_MAP_KEY_SEEN | BPF_MAP_KEY_POISON); } +static inline bool bpf_is_mem_insn(struct bpf_insn *insn) +{ + if (BPF_CLASS(insn->code) != BPF_ST && + BPF_CLASS(insn->code) != BPF_STX && + BPF_CLASS(insn->code) != BPF_LDX) + return false; + + if (insn->code == (BPF_ST | BPF_NOSPEC)) + return false; + + return (BPF_MODE(insn->code) == BPF_MEM || + BPF_MODE(insn->code) == BPF_MEMSX || + BPF_MODE(insn->code) == BPF_ATOMIC); +} + #define MAX_PACKET_OFF 0xffff #define CALLER_SAVED_REGS 6 @@ -1690,30 +1967,34 @@ enum bpf_reg_arg_type { }; #define MAX_KFUNC_DESCS 256 +/* Each kfunc can have its canonical and one specialized call target. */ +#define MAX_KFUNC_CALL_DESCS (MAX_KFUNC_DESCS * 2) +static_assert(MAX_KFUNC_CALL_DESCS <= S16_MAX + 1); struct bpf_kfunc_desc { struct btf_func_model func_model; struct bpf_func_proto proto; u32 func_id; - s32 imm; u16 offset; unsigned long addr; }; struct bpf_kfunc_desc_tab { u32 nr_descs; + u32 nr_base_descs; /* Sorted by func_id (BTF ID) and offset (fd_array offset) during - * verification. JITs do lookups by bpf_insn, where func_id may not be - * available, therefore at the end of verification do_misc_fixups() - * sorts this by imm and offset. + * verification. JITs use the descriptor index stored in the finalized + * call's off field. The first nr_base_descs entries are the canonical + * descriptors used for verifier lookups. Call specialization may append + * immutable descriptors for additional targets. * - * Grown one entry at a time by bpf_add_kfunc_call(). + * Grown one entry at a time by bpf_add_kfunc_call() and during + * call specialization. */ struct bpf_kfunc_desc descs[]; }; /* Functions exported from verifier.c, used by fixups.c */ -void bpf_clear_insn_aux_data(struct bpf_verifier_env *env, int start, int len); void bpf_mark_subprog_exc_cb(struct bpf_verifier_env *env, int subprog); bool bpf_allow_tail_call_in_subprogs(struct bpf_verifier_env *env); bool bpf_verifier_inlines_helper_call(struct bpf_verifier_env *env, s32 imm); @@ -1726,6 +2007,7 @@ int bpf_check_attach_btf_id_multi(struct btf *btf, struct bpf_prog *prog, u32 bt struct bpf_attach_target_info *tgt_info); /* Functions in fixups.c, called from bpf_check() */ +void bpf_clear_insn_aux_data(struct bpf_verifier_env *env, int start, int len); int bpf_remove_fastcall_spills_fills(struct bpf_verifier_env *env); int bpf_optimize_bpf_loop(struct bpf_verifier_env *env); void bpf_opt_hard_wire_dead_code_branches(struct bpf_verifier_env *env); @@ -1738,4 +2020,23 @@ int bpf_fixup_call_args(struct bpf_verifier_env *env); int bpf_do_misc_fixups(struct bpf_verifier_env *env); int bpf_insn_def32(struct bpf_prog *prog, struct bpf_insn *insn); +int bpf_flip_opcode(u32 opcode); +u8 bpf_rev_opcode(u8 opcode); + +int bpf_compute_idoms(struct bpf_verifier_env *env); +int bpf_compute_loops(struct bpf_verifier_env *env); +int bpf_loop_at_index(struct bpf_verifier_env *env, u32 idx); +bool bpf_is_nested_loop(struct bpf_verifier_env *env, int inner_header, int outer_header); + +int bpf_init_scev(struct bpf_verifier_env *env); +void bpf_free_scev(struct bpf_verifier_env *env); +int bpf_compute_scev(struct bpf_verifier_env *env); + +int bpf_compute_loop_iters(struct bpf_verifier_env *env, struct bpf_verifier_state *st, + struct bpf_loop_iters *iters); +int bpf_widen_scev_regs(struct bpf_verifier_env *env, struct bpf_verifier_state *st, + struct bpf_verifier_state *loop_entry, struct bpf_loop_iters *iters); +int bpf_clamp_scev_regs(struct bpf_verifier_env *env, struct bpf_func_state *st, u32 insn_idx, + struct bpf_verifier_state *entry_state, u32 max_header_count); + #endif /* _LINUX_BPF_VERIFIER_H */ diff --git a/include/linux/bsg-lib.h b/include/linux/bsg-lib.h index 14fa93268630..a697246feeac 100644 --- a/include/linux/bsg-lib.h +++ b/include/linux/bsg-lib.h @@ -66,7 +66,8 @@ void bsg_job_done(struct bsg_job *job, int result, unsigned int reply_payload_rcv_len); struct request_queue *bsg_setup_queue(struct device *dev, const char *name, struct queue_limits *lim, bsg_job_fn *job_fn, - bsg_timeout_fn *timeout, int dd_job_size); + bsg_timeout_fn *timeout, int dd_job_size, + unsigned int reply_len); void bsg_remove_queue(struct request_queue *q); void bsg_job_put(struct bsg_job *job); int __must_check bsg_job_get(struct bsg_job *job); diff --git a/include/linux/btf.h b/include/linux/btf.h index 7c62ea17b116..a18254b7521b 100644 --- a/include/linux/btf.h +++ b/include/linux/btf.h @@ -5,6 +5,7 @@ #define _LINUX_BTF_H 1 #include <linux/types.h> +#include <linux/module.h> #include <linux/bpfptr.h> #include <linux/bsearch.h> #include <linux/btf_ids.h> @@ -81,6 +82,7 @@ #define KF_IMPLICIT_ARGS (1 << 16) /* kfunc has implicit arguments supplied by the verifier */ #define KF_SPINLOCK_SAFE (1 << 17) /* kfunc is allowed inside bpf_spin_lock-ed region */ #define KF_PERFMON (1 << 18) /* kfunc requires CAP_PERFMON */ +#define KF_ITER_RET_INDEPENDENT (1 << 19) /* next result outlives the current element */ /* * Tag marking a kernel function as a kfunc. This is meant to minimize the @@ -118,6 +120,19 @@ struct btf_show; struct btf_id_set; struct bpf_prog; +#define BTF_LINK_SHA256_LEN 32 + +/* + * A .BTF.link or .BTF.inline.link section identifies the module which + * carries a vmlinux BTF section. The module name is NUL terminated and the + * field is padded to __MODULE_NAME_LEN. + */ +struct btf_link { + char module_name[__MODULE_NAME_LEN]; + u8 sha256[BTF_LINK_SHA256_LEN]; + u32 btf_size; +} __packed; + typedef int (*btf_kfunc_filter_t)(const struct bpf_prog *prog, u32 kfunc_id); struct btf_kfunc_id_set { @@ -236,6 +251,7 @@ struct btf_record *btf_parse_fields(const struct btf *btf, const struct btf_type u32 field_mask, u32 value_size); int btf_check_and_fixup_fields(const struct btf *btf, struct btf_record *rec); bool btf_type_is_void(const struct btf_type *t); +bool btf_type_is_arena_ptr(const struct btf *btf, const struct btf_type *t); s32 btf_find_by_name_kind(const struct btf *btf, const char *name, u8 kind); s32 bpf_find_btf_id(const char *name, u32 kind, struct btf **btf_p); struct btf *btf_get_module_btf(const struct module *module); @@ -261,6 +277,11 @@ const char *btf_type_str(const struct btf_type *t); i < btf_type_vlen(datasec_type); \ i++, member++) +#define for_each_loc(i, locsec_type, member) \ + for (i = 0, member = btf_type_loc_secinfo(locsec_type); \ + i < btf_type_vlen(locsec_type); \ + i++, member++) + static inline bool btf_type_is_ptr(const struct btf_type *t) { return BTF_INFO_KIND(t->info) == BTF_KIND_PTR; @@ -329,6 +350,21 @@ static inline u64 btf_enum64_value(const struct btf_enum64 *e) return ((u64)e->val_hi32 << 32) | e->val_lo32; } +static inline struct btf_loc_param *btf_loc_param(const struct btf_type *t) +{ + return (struct btf_loc_param *)(t + 1); +} + +static inline __u32 *btf_loc_proto_params(const struct btf_type *t) +{ + return (__u32 *)(t + 1); +} + +static inline struct btf_loc *btf_type_loc_secinfo(const struct btf_type *t) +{ + return (struct btf_loc *)(t + 1); +} + static inline bool btf_is_composite(const struct btf_type *t) { u16 kind = btf_kind(t); @@ -559,7 +595,7 @@ struct btf_field_desc { /* member struct size, or zero, if no members */ int m_sz; /* repeated per-member offsets */ - int m_off_cnt, m_offs[1]; + int m_off_cnt, m_offs[2]; }; struct btf_field_iter { @@ -573,7 +609,8 @@ struct btf_field_iter { #ifdef CONFIG_BPF_SYSCALL const struct btf_type *btf_type_by_id(const struct btf *btf, u32 type_id); void btf_set_base_btf(struct btf *btf, const struct btf *base_btf); -int btf_relocate(struct btf *btf, const struct btf *base_btf, __u32 **map_ids); +int btf_relocate(struct btf *btf, const struct btf *base_btf, __u32 **map_ids, + __u32 **map_strs); int btf_field_iter_init(struct btf_field_iter *it, struct btf_type *t, enum btf_field_iter_kind iter_kind); __u32 *btf_field_iter_next(struct btf_field_iter *it); @@ -603,6 +640,29 @@ bool btf_types_are_same(const struct btf *btf1, u32 id1, const struct btf *btf2, u32 id2); int btf_check_iter_arg(struct btf *btf, const struct btf_type *func, int arg_idx); +#if IS_ENABLED(CONFIG_SYSFS) +struct bin_attribute *sysfs_btf_add(const char *name, void *data, size_t data_size, + bool mmap, const char *lazy_module_name); +bool sysfs_btf_update(struct bin_attribute *attr, void *data); +void sysfs_btf_remove(struct bin_attribute *attr); +#else +static inline struct bin_attribute * +sysfs_btf_add(const char *name, void *data, size_t data_size, bool mmap, + const char *lazy_module_name) +{ + return NULL; +} + +static inline bool sysfs_btf_update(struct bin_attribute *attr, void *data) +{ + return false; +} + +static inline void sysfs_btf_remove(struct bin_attribute *attr) +{ +} +#endif + static inline bool btf_type_is_struct_ptr(struct btf *btf, const struct btf_type *t) { if (!btf_type_is_ptr(t)) @@ -624,7 +684,7 @@ static inline void btf_set_base_btf(struct btf *btf, const struct btf *base_btf) } static inline int btf_relocate(void *log, struct btf *btf, const struct btf *base_btf, - __u32 **map_ids) + __u32 **map_ids, __u32 **map_strs) { return -EOPNOTSUPP; } diff --git a/include/linux/buffer_head.h b/include/linux/buffer_head.h index fd2c7115c054..f8782b719026 100644 --- a/include/linux/buffer_head.h +++ b/include/linux/buffer_head.h @@ -59,10 +59,7 @@ struct address_space; struct buffer_head { unsigned long b_state; /* buffer state bitmap (see above) */ struct buffer_head *b_this_page;/* circular list of page's buffers */ - union { - struct page *b_page; /* the page this bh is mapped to */ - struct folio *b_folio; /* the folio this bh is mapped to */ - }; + struct folio *b_folio; /* the folio this bh is mapped to */ sector_t b_blocknr; /* start block number */ size_t b_size; /* size of mapping */ @@ -172,15 +169,39 @@ static __always_inline int buffer_uptodate(const struct buffer_head *bh) static inline unsigned long bh_offset(const struct buffer_head *bh) { - return (unsigned long)(bh)->b_data & (page_size(bh->b_page) - 1); + return (unsigned long)(bh)->b_data & (folio_size(bh->b_folio) - 1); } -/* If we *know* page->private refers to buffer_heads */ -#define page_buffers(page) \ - ({ \ - BUG_ON(!PagePrivate(page)); \ - ((struct buffer_head *)page_private(page)); \ - }) +/** + * kmap_local_bh - Map the data of a buffer. + * @bh: The buffer. + * + * Buffers usually live in the page cache, but a few are built over memory + * which is not. Those carry no folio and b_data is already a kernel address + * which is always mapped, so there is nothing to do for them. Pair with + * kunmap_local_bh(). + * + * Return: A pointer to the buffer's data. + */ +static inline void *kmap_local_bh(const struct buffer_head *bh) +{ + if (!bh->b_folio) + return bh->b_data; + return kmap_local_folio(bh->b_folio, bh_offset(bh)); +} + +/** + * kunmap_local_bh - Unmap the data of a buffer. + * @bh: The buffer. + * @addr: The address returned by kmap_local_bh(). + */ +static inline void kunmap_local_bh(const struct buffer_head *bh, void *addr) +{ + if (bh->b_folio) + kunmap_local(addr); +} + +/* If we *know* folio->private refers to buffer_heads */ #define folio_buffers(folio) folio_get_private(folio) void buffer_check_dirty_writeback(struct folio *folio, @@ -197,7 +218,6 @@ void folio_set_bh(struct buffer_head *bh, struct folio *folio, unsigned long offset); struct buffer_head *folio_alloc_buffers(struct folio *folio, unsigned long size, gfp_t gfp); -struct buffer_head *alloc_page_buffers(struct page *page, unsigned long size); struct buffer_head *create_empty_buffers(struct folio *folio, unsigned long blocksize, unsigned long b_state); void end_buffer_read_sync(struct buffer_head *bh, int uptodate); @@ -338,20 +358,58 @@ static inline void bforget(struct buffer_head *bh) __bforget(bh); } -static inline struct buffer_head * -sb_bread(struct super_block *sb, sector_t block) +/** + * sb_bread - Read a block. + * @sb: The superblock to read from. + * @block: Block number in units of block size. + * + * Read a specified block, and return the buffer head that refers + * to it. The memory is allocated from the movable area so that it can + * be migrated. The returned buffer head has its refcount increased. + * The caller should call brelse() when it has finished with the buffer. + * + * Context: May sleep waiting for I/O. + * Return: NULL if the block was unreadable. + */ +static inline +struct buffer_head *sb_bread(struct super_block *sb, sector_t block) { return __bread_gfp(sb->s_bdev, block, sb->s_blocksize, __GFP_MOVABLE); } -static inline struct buffer_head * -sb_bread_unmovable(struct super_block *sb, sector_t block) +/** + * sb_bread_unmovable - Read a block. + * @sb: The superblock to read from. + * @block: Block number in units of block size. + * + * Read a specified block, and return the buffer head that refers to it. + * The memory is allocated from the unmovable area so that pointers into + * it remain valid after compaction runs. The returned buffer head has + * its refcount increased. The caller should call brelse() when it has + * finished with the buffer. + * + * Context: May sleep waiting for I/O. + * Return: NULL if the block was unreadable. + */ +static inline +struct buffer_head *sb_bread_unmovable(struct super_block *sb, sector_t block) { return __bread_gfp(sb->s_bdev, block, sb->s_blocksize, 0); } -static inline void -sb_breadahead(struct super_block *sb, sector_t block) +/** + * sb_breadahead - Start readahead. + * @sb: Superblock identifying the block device. + * @block: The block to read. + * + * Read this block. The I/O will be flagged as being readahead rather + * than immediate read, but (unlike the page cache), surrounding blocks + * will not be read. + * + * Context: May sleep in order to allocate memory. + */ +static inline +void sb_breadahead(struct super_block *sb, sector_t block) { __breadahead(sb->s_bdev, block, sb->s_blocksize); } diff --git a/include/linux/bvec.h b/include/linux/bvec.h index fc566ee1c1ff..b63914ff56e3 100644 --- a/include/linux/bvec.h +++ b/include/linux/bvec.h @@ -108,7 +108,8 @@ struct bvec_iter { unsigned int bi_idx; /* - * Current offset in the bvec entry pointed to by `bi_idx`. + * Current offset in the bvec entry pointed to by `bi_idx` or into + * a dma-buf map. */ unsigned int bi_offset; } __packed __aligned(4); diff --git a/include/linux/cache.h b/include/linux/cache.h index e69768f50d53..b6e857b985bc 100644 --- a/include/linux/cache.h +++ b/include/linux/cache.h @@ -89,6 +89,7 @@ */ #ifndef INTERNODE_CACHE_SHIFT #define INTERNODE_CACHE_SHIFT L1_CACHE_SHIFT +#define INTERNODE_CACHE_BYTES (1 << INTERNODE_CACHE_SHIFT) #endif #if !defined(____cacheline_internodealigned_in_smp) diff --git a/include/linux/cacheinfo.h b/include/linux/cacheinfo.h index fc879ac4cc4f..56fa646df0d1 100644 --- a/include/linux/cacheinfo.h +++ b/include/linux/cacheinfo.h @@ -82,6 +82,7 @@ struct cpu_cacheinfo { }; struct cpu_cacheinfo *get_cpu_cacheinfo(unsigned int cpu); +int get_cpu_cacheinfo_id(int cpu, int level); int early_cache_level(unsigned int cpu); int init_cache_level(unsigned int cpu); int init_of_cache_level(unsigned int cpu); @@ -137,17 +138,6 @@ static inline struct cacheinfo *get_cpu_cacheinfo_level(int cpu, int level) return NULL; } -/* - * Get the id of the cache associated with @cpu at level @level. - * cpuhp lock must be held. - */ -static inline int get_cpu_cacheinfo_id(int cpu, int level) -{ - struct cacheinfo *ci = get_cpu_cacheinfo_level(cpu, level); - - return ci ? ci->id : -1; -} - #if defined(CONFIG_ARM64) || defined(CONFIG_ARM) #define use_arch_cache_info() (true) #else diff --git a/include/linux/capability.h b/include/linux/capability.h index f8532d92fcad..622137f66f09 100644 --- a/include/linux/capability.h +++ b/include/linux/capability.h @@ -186,9 +186,9 @@ static inline bool ns_capable_setid(struct user_namespace *ns, int cap) } #endif /* CONFIG_MULTIUSER */ bool privileged_wrt_inode_uidgid(struct user_namespace *ns, - struct mnt_idmap *idmap, + const struct mnt_idmap *idmap, const struct inode *inode); -bool capable_wrt_inode_uidgid(struct mnt_idmap *idmap, +bool capable_wrt_inode_uidgid(const struct mnt_idmap *idmap, const struct inode *inode, int cap); extern bool file_ns_capable(const struct file *file, struct user_namespace *ns, int cap); extern bool ptracer_capable(struct task_struct *tsk, struct user_namespace *ns); @@ -215,11 +215,11 @@ static inline bool checkpoint_restore_ns_capable_noaudit(struct user_namespace * } /* audit system wants to get cap info from files as well */ -int get_vfs_caps_from_disk(struct mnt_idmap *idmap, +int get_vfs_caps_from_disk(const struct mnt_idmap *idmap, const struct dentry *dentry, struct cpu_vfs_cap_data *cpu_caps); -int cap_convert_nscap(struct mnt_idmap *idmap, struct dentry *dentry, +int cap_convert_nscap(const struct mnt_idmap *idmap, struct dentry *dentry, const void **ivalue, size_t size); #endif /* !_LINUX_CAPABILITY_H */ diff --git a/include/linux/ccp.h b/include/linux/ccp.h index 868924dec5a1..e6c599243f11 100644 --- a/include/linux/ccp.h +++ b/include/linux/ccp.h @@ -26,7 +26,7 @@ struct ccp_cmd; /** * ccp_present - check if a CCP device is present * - * Returns zero if a CCP device is present, -ENODEV otherwise. + * Returns: zero if a CCP device is present, -ENODEV otherwise. */ int ccp_present(void); @@ -38,7 +38,7 @@ int ccp_present(void); /** * ccp_version - get the version of the CCP * - * Returns a positive version number, or zero if no CCP + * Returns: a positive version number, or zero if no CCP */ unsigned int ccp_version(void); @@ -61,9 +61,9 @@ unsigned int ccp_version(void); * will be -EINPROGRESS. Any other "err" value during callback is * the result of the operation. * - * The cmd has been successfully queued if: - * the return code is -EINPROGRESS or - * the return code is -EBUSY and CCP_CMD_MAY_BACKLOG flag is set + * Returns: The cmd has been successfully queued if: + * * the return code is -EINPROGRESS or + * * the return code is -EBUSY and CCP_CMD_MAY_BACKLOG flag is set */ int ccp_enqueue_cmd(struct ccp_cmd *cmd); @@ -89,7 +89,7 @@ static inline int ccp_enqueue_cmd(struct ccp_cmd *cmd) /***** AES engine *****/ /** - * ccp_aes_type - AES key size + * enum ccp_aes_type - AES key size * * @CCP_AES_TYPE_128: 128-bit key * @CCP_AES_TYPE_192: 192-bit key @@ -99,11 +99,12 @@ enum ccp_aes_type { CCP_AES_TYPE_128 = 0, CCP_AES_TYPE_192, CCP_AES_TYPE_256, + /* private: */ CCP_AES_TYPE__LAST, }; /** - * ccp_aes_mode - AES operation mode + * enum ccp_aes_mode - AES operation mode * * @CCP_AES_MODE_ECB: ECB mode * @CCP_AES_MODE_CBC: CBC mode @@ -111,6 +112,10 @@ enum ccp_aes_type { * @CCP_AES_MODE_CFB: CFB mode * @CCP_AES_MODE_CTR: CTR mode * @CCP_AES_MODE_CMAC: CMAC mode + * @CCP_AES_MODE_GHASH: GHASH mode + * @CCP_AES_MODE_GCTR: GCTR mode + * @CCP_AES_MODE_GCM: GCM mode + * @CCP_AES_MODE_GMAC: GMAC mode */ enum ccp_aes_mode { CCP_AES_MODE_ECB = 0, @@ -123,11 +128,12 @@ enum ccp_aes_mode { CCP_AES_MODE_GCTR, CCP_AES_MODE_GCM, CCP_AES_MODE_GMAC, + /* private: */ CCP_AES_MODE__LAST, }; /** - * ccp_aes_mode - AES operation mode + * enum ccp_aes_action - AES operation mode * * @CCP_AES_ACTION_DECRYPT: AES decrypt operation * @CCP_AES_ACTION_ENCRYPT: AES encrypt operation @@ -135,6 +141,7 @@ enum ccp_aes_mode { enum ccp_aes_action { CCP_AES_ACTION_DECRYPT = 0, CCP_AES_ACTION_ENCRYPT, + /* private: */ CCP_AES_ACTION__LAST, }; /* Overloaded field */ @@ -146,6 +153,7 @@ enum ccp_aes_action { * @type: AES operation key size * @mode: AES operation mode * @action: AES operation (decrypt/encrypt) + * @authsize: AES block request size * @key: key to be used for this AES operation * @key_len: length in bytes of key * @iv: IV to be used for this AES operation @@ -156,6 +164,7 @@ enum ccp_aes_action { * @cmac_final: indicates final operation when running in CMAC mode * @cmac_key: K1/K2 key used in final CMAC operation * @cmac_key_len: length in bytes of cmac_key + * @aad_len: length in bytes of Additional Authenticated Data * * Variables required to be set when calling ccp_enqueue_cmd(): * - type, mode, action, key, key_len, src, dst, src_len @@ -192,7 +201,7 @@ struct ccp_aes_engine { /***** XTS-AES engine *****/ /** - * ccp_xts_aes_unit_size - XTS unit size + * enum ccp_xts_aes_unit_size - XTS unit size * * @CCP_XTS_AES_UNIT_SIZE_16: Unit size of 16 bytes * @CCP_XTS_AES_UNIT_SIZE_512: Unit size of 512 bytes @@ -206,11 +215,13 @@ enum ccp_xts_aes_unit_size { CCP_XTS_AES_UNIT_SIZE_1024, CCP_XTS_AES_UNIT_SIZE_2048, CCP_XTS_AES_UNIT_SIZE_4096, + /* private: */ CCP_XTS_AES_UNIT_SIZE__LAST, }; /** * struct ccp_xts_aes_engine - CCP XTS AES operation + * @type: ccp_aes_type - AES key size * @action: AES operation (decrypt/encrypt) * @unit_size: unit size of the XTS operation * @key: key to be used for this XTS AES operation @@ -247,11 +258,13 @@ struct ccp_xts_aes_engine { /***** SHA engine *****/ /** - * ccp_sha_type - type of SHA operation + * enum ccp_sha_type - type of SHA operation * * @CCP_SHA_TYPE_1: SHA-1 operation * @CCP_SHA_TYPE_224: SHA-224 operation * @CCP_SHA_TYPE_256: SHA-256 operation + * @CCP_SHA_TYPE_384: SHA-384 operation + * @CCP_SHA_TYPE_512: SHA-512 operation */ enum ccp_sha_type { CCP_SHA_TYPE_1 = 1, @@ -259,6 +272,7 @@ enum ccp_sha_type { CCP_SHA_TYPE_256, CCP_SHA_TYPE_384, CCP_SHA_TYPE_512, + /* private: */ CCP_SHA_TYPE__LAST, }; @@ -384,7 +398,7 @@ struct ccp_rsa_engine { /***** Passthru engine *****/ /** - * ccp_passthru_bitwise - type of bitwise passthru operation + * enum ccp_passthru_bitwise - type of bitwise passthru operation * * @CCP_PASSTHRU_BITWISE_NOOP: no bitwise operation performed * @CCP_PASSTHRU_BITWISE_AND: perform bitwise AND of src with mask @@ -398,11 +412,12 @@ enum ccp_passthru_bitwise { CCP_PASSTHRU_BITWISE_OR, CCP_PASSTHRU_BITWISE_XOR, CCP_PASSTHRU_BITWISE_MASK, + /* private: */ CCP_PASSTHRU_BITWISE__LAST, }; /** - * ccp_passthru_byteswap - type of byteswap passthru operation + * enum ccp_passthru_byteswap - type of byteswap passthru operation * * @CCP_PASSTHRU_BYTESWAP_NOOP: no byte swapping performed * @CCP_PASSTHRU_BYTESWAP_32BIT: swap bytes within 32-bit words @@ -412,6 +427,7 @@ enum ccp_passthru_byteswap { CCP_PASSTHRU_BYTESWAP_NOOP = 0, CCP_PASSTHRU_BYTESWAP_32BIT, CCP_PASSTHRU_BYTESWAP_256BIT, + /* private: */ CCP_PASSTHRU_BYTESWAP__LAST, }; @@ -450,8 +466,8 @@ struct ccp_passthru_engine { * @byte_swap: byteswap operation to perform * @mask: mask to be applied to data * @mask_len: length in bytes of mask - * @src: data to be used for this operation - * @dst: data produced by this operation + * @src_dma: data to be used for this operation + * @dst_dma: data produced by this operation * @src_len: length in bytes of data used for this operation * @final: indicate final pass-through operation * @@ -478,7 +494,7 @@ struct ccp_passthru_nomap_engine { #define CCP_ECC_MAX_OUTPUTS 3 /** - * ccp_ecc_function - type of ECC function + * enum ccp_ecc_function - type of ECC function * * @CCP_ECC_FUNCTION_MMUL_384BIT: 384-bit modular multiplication * @CCP_ECC_FUNCTION_MADD_384BIT: 384-bit modular addition @@ -564,8 +580,9 @@ struct ccp_ecc_point_math { * @function: ECC function to perform * @mod: ECC modulus * @mod_len: length in bytes of modulus - * @mm: module math parameters - * @pm: point math parameters + * @u: union for math parameters + * @u.mm: module math parameters + * @u.pm: point math parameters * @ecc_result: result of the ECC operation * * Variables required to be set when calling ccp_enqueue_cmd(): @@ -589,11 +606,11 @@ struct ccp_ecc_engine { /** - * ccp_engine - CCP operation identifiers + * enum ccp_engine - CCP operation identifiers * * @CCP_ENGINE_AES: AES operation - * @CCP_ENGINE_XTS_AES: 128-bit XTS AES operation - * @CCP_ENGINE_RSVD1: unused + * @CCP_ENGINE_XTS_AES_128: 128-bit XTS AES operation + * @CCP_ENGINE_DES3: 3DES operation * @CCP_ENGINE_SHA: SHA operation * @CCP_ENGINE_RSA: RSA operation * @CCP_ENGINE_PASSTHRU: pass-through operation @@ -609,6 +626,7 @@ enum ccp_engine { CCP_ENGINE_PASSTHRU, CCP_ENGINE_ZLIB_DECOMPRESS, CCP_ENGINE_ECC, + /* private: */ CCP_ENGINE__LAST, }; diff --git a/include/linux/cdrom.h b/include/linux/cdrom.h index 260d7968cf72..19e21a6cb762 100644 --- a/include/linux/cdrom.h +++ b/include/linux/cdrom.h @@ -11,7 +11,6 @@ #ifndef _LINUX_CDROM_H #define _LINUX_CDROM_H -#include <linux/fs.h> /* not really needed, later.. */ #include <linux/list.h> #include <linux/blkdev.h> #include <scsi/scsi_common.h> diff --git a/include/linux/ceph/ceph_features.h b/include/linux/ceph/ceph_features.h index 3a47acd9cc14..1e4dce93a8a1 100644 --- a/include/linux/ceph/ceph_features.h +++ b/include/linux/ceph/ceph_features.h @@ -174,6 +174,18 @@ DEFINE_CEPH_FEATURE_DEPRECATED(63, 1, RESERVED_BROKEN, LUMINOUS) // client-facin /* + * Only advertise CEPH_FEATURE_MDS_INLINE_DATA when inline data support is + * built in. Without it the MDS refuses to hand out files that still have + * inline data instead of letting the client read the data pool objects + * that do not hold it yet. + */ +#ifdef CONFIG_CEPH_FS_INLINE_DATA +#define CEPH_FEATURE_MDS_INLINE_DATA_SUPPORTED CEPH_FEATURE_MDS_INLINE_DATA +#else +#define CEPH_FEATURE_MDS_INLINE_DATA_SUPPORTED 0 +#endif + +/* * Features supported. */ #define CEPH_FEATURES_SUPPORTED_DEFAULT \ @@ -204,7 +216,7 @@ DEFINE_CEPH_FEATURE_DEPRECATED(63, 1, RESERVED_BROKEN, LUMINOUS) // client-facin CEPH_FEATURE_CRUSH_V2 | \ CEPH_FEATURE_EXPORT_PEER | \ CEPH_FEATURE_OSDMAP_ENC | \ - CEPH_FEATURE_MDS_INLINE_DATA | \ + CEPH_FEATURE_MDS_INLINE_DATA_SUPPORTED |\ CEPH_FEATURE_CRUSH_TUNABLES3 | \ CEPH_FEATURE_OSD_PRIMARY_AFFINITY | \ CEPH_FEATURE_MSGR_KEEPALIVE2 | \ diff --git a/include/linux/ceph/ceph_fs.h b/include/linux/ceph/ceph_fs.h index 69ac3e55a3fe..d70bbce1a1b2 100644 --- a/include/linux/ceph/ceph_fs.h +++ b/include/linux/ceph/ceph_fs.h @@ -414,6 +414,7 @@ extern const char *ceph_mds_op_name(int op); #define CEPH_O_CREAT 00000100 #define CEPH_O_EXCL 00000200 #define CEPH_O_TRUNC 00001000 +#define CEPH_O_LAZY 00020000 #define CEPH_O_DIRECTORY 00200000 #define CEPH_O_NOFOLLOW 00400000 @@ -707,8 +708,13 @@ int ceph_flags_to_mode(int flags); CEPH_CAP_LINK_SHARED | \ CEPH_CAP_FILE_SHARED | \ CEPH_CAP_XATTR_SHARED) +#ifdef CONFIG_CEPH_FS_INLINE_DATA #define CEPH_STAT_CAP_INLINE_DATA (CEPH_CAP_FILE_SHARED | \ CEPH_CAP_FILE_RD) +#else +/* no inline data to fetch, so no extra caps are needed for it */ +#define CEPH_STAT_CAP_INLINE_DATA 0 +#endif #define CEPH_STAT_RSTAT CEPH_CAP_FILE_WREXTEND #define CEPH_CAP_ANY_SHARED (CEPH_CAP_AUTH_SHARED | \ diff --git a/include/linux/cgroup-defs.h b/include/linux/cgroup-defs.h index 3754d697854b..cbfe8937647c 100644 --- a/include/linux/cgroup-defs.h +++ b/include/linux/cgroup-defs.h @@ -556,6 +556,14 @@ struct cgroup { struct cgroup_subsys_state __rcu *subsys[CGROUP_SUBSYS_COUNT]; /* + * Effective css for each subsystem: the css of the nearest ancestor, + * including this cgroup, that has the subsystem enabled, or the root + * css if the subsystem isn't bound to this hierarchy. Updated under + * cgroup_mutex and read under RCU. + */ + struct cgroup_subsys_state __rcu *e_css[CGROUP_SUBSYS_COUNT]; + + /* * Keep track of total number of dying CSSes at and below this cgroup. * Protected by cgroup_mutex. */ diff --git a/include/linux/cgroup.h b/include/linux/cgroup.h index 5dfa915a630e..2afb4cb2bb4f 100644 --- a/include/linux/cgroup.h +++ b/include/linux/cgroup.h @@ -154,6 +154,9 @@ struct cgroup *cgroup_get_from_path(const char *path); struct cgroup *cgroup_get_from_fd(int fd); struct cgroup *cgroup_v1v2_get_from_fd(int fd); +/* Was this controller blocked from v1 hierarchies by cgroup_no_v1= ? */ +bool cgroup1_ssid_disabled(int ssid); + int cgroup_attach_task_all(struct task_struct *from, struct task_struct *); int cgroup_transfer_tasks(struct cgroup *to, struct cgroup *from); diff --git a/include/linux/cleanup.h b/include/linux/cleanup.h index b1b5698cbf1b..1fb8058b897d 100644 --- a/include/linux/cleanup.h +++ b/include/linux/cleanup.h @@ -261,10 +261,6 @@ const volatile void * __must_check_fn(const volatile void *val) * CLASS(name, var)(args...): * declare the variable @var as an instance of the named class * - * CLASS_INIT(name, var, init_expr): - * declare the variable @var as an instance of the named class with - * custom initialization expression. - * * Ex. * * DEFINE_CLASS(fdget, struct fd, fdput(_T), fdget(fd), int fd) @@ -302,9 +298,6 @@ static __always_inline class_##_name##_t class_##_name##ext##_constructor(_init_ class_##_name##_t var __cleanup(class_##_name##_destructor) = \ class_##_name##_constructor -#define CLASS_INIT(_name, _var, _init_expr) \ - class_##_name##_t _var __cleanup(class_##_name##_destructor) = (_init_expr) - #define __scoped_class(_name, var, _label, args...) \ for (CLASS(_name, var)(args); ; ({ goto _label; })) \ if (0) { \ diff --git a/include/linux/clk-provider.h b/include/linux/clk-provider.h index dc8e161f9c2c..a355d1ee5189 100644 --- a/include/linux/clk-provider.h +++ b/include/linux/clk-provider.h @@ -7,6 +7,7 @@ #define __LINUX_CLK_PROVIDER_H #include <dt-bindings/clock/clock.h> +#include <linux/bits.h> #include <linux/of.h> #include <linux/of_clk.h> @@ -50,6 +51,7 @@ struct clk; struct clk_hw; struct clk_core; struct dentry; +struct module; /** * struct clk_rate_request - Structure encoding the clk constraints that @@ -351,8 +353,9 @@ struct clk_init_data { * @core: pointer to the struct clk_core instance that points back to this * struct clk_hw instance * - * @clk: pointer to the per-user struct clk instance that can be used to call - * into the clk API + * @clk: pointer to the per-user struct clk instance. This will be removed at + * some point in the future, Please consider the field obsolete and do not use + * it. Use clk_hw_get_clk() to get a per-user struct clk from a struct clk_hw. * * @init: pointer to struct clk_init_data that contains the init data shared * with the common clock framework. This pointer will be set to NULL once @@ -755,7 +758,7 @@ struct clk_divider { spinlock_t *lock; }; -#define clk_div_mask(width) ((1 << (width)) - 1) +#define clk_div_mask(width) GENMASK((width) - 1, 0) #define to_clk_divider(_hw) container_of(_hw, struct clk_divider, hw) #define CLK_DIVIDER_ONE_BASED BIT(0) @@ -1474,10 +1477,24 @@ static inline struct clk_hw *__clk_get_hw(struct clk *clk) } #endif -struct clk *clk_hw_get_clk(struct clk_hw *hw, const char *con_id); +struct clk *__clk_hw_get_clk(struct clk_hw *hw, const char *con_id, + struct module *owner); struct clk *devm_clk_hw_get_clk(struct device *dev, struct clk_hw *hw, const char *con_id); +/** + * clk_hw_get_clk - get clk consumer given a clk_hw + * @hw: clk_hw associated with the clk being consumed + * @con_id: connection ID string on device + * + * Return: new clk consumer + * This is the function to be used by providers which need + * to get a consumer clk and act on the clock element + * Calls to this function must be balanced with calls to clk_put() + */ +#define clk_hw_get_clk(hw, con_id) \ + __clk_hw_get_clk((hw), (con_id), THIS_MODULE) + unsigned int clk_hw_get_num_parents(const struct clk_hw *hw); struct clk_hw *clk_hw_get_parent(const struct clk_hw *hw); struct clk_hw *clk_hw_get_parent_by_index(const struct clk_hw *hw, diff --git a/include/linux/clk/tegra.h b/include/linux/clk/tegra.h index 3650e926e93f..f1034e14c1a4 100644 --- a/include/linux/clk/tegra.h +++ b/include/linux/clk/tegra.h @@ -170,7 +170,7 @@ struct tegra210_clk_emc_provider { struct module *owner; struct device *dev; - struct tegra210_clk_emc_config *configs; + struct tegra210_clk_emc_config *configs __counted_by_ptr(num_configs); unsigned int num_configs; int (*set_rate)(struct device *dev, diff --git a/include/linux/clk/ti.h b/include/linux/clk/ti.h index 54a3fa370004..61c4abf19995 100644 --- a/include/linux/clk/ti.h +++ b/include/linux/clk/ti.h @@ -51,7 +51,7 @@ struct clk_omap_reg { * @autoidle_mask: mask of the DPLL autoidle mode bitfield in @autoidle_reg * @freqsel_mask: mask of the DPLL jitter correction bitfield in @control_reg * @dcc_mask: mask of the DPLL DCC correction bitfield @mult_div1_reg - * @dcc_rate: rate atleast which DCC @dcc_mask must be set + * @dcc_rate: rate at least which DCC @dcc_mask must be set * @idlest_mask: mask of the DPLL idle status bitfield in @idlest_reg * @lpmode_mask: mask of the DPLL low-power mode bitfield in @control_reg * @m4xen_mask: mask of the DPLL M4X multiplier bitfield in @control_reg diff --git a/include/linux/cmpxchg-emu.h b/include/linux/cmpxchg-emu.h index 998deec67740..2db70f1e3925 100644 --- a/include/linux/cmpxchg-emu.h +++ b/include/linux/cmpxchg-emu.h @@ -4,12 +4,14 @@ * lacking direct support for these sizes. These are implemented in terms * of 4-byte cmpxchg operations. * - * Copyright (C) 2024 Paul E. McKenney. + * Copyright (C) 2024 Paul E. McKenney <paulmck@kernel.org> + * Copyright (C) 2026 Bradley Morgan <brads@mainlining.org> */ #ifndef __LINUX_CMPXCHG_EMU_H #define __LINUX_CMPXCHG_EMU_H uintptr_t cmpxchg_emu_u8(volatile u8 *p, uintptr_t old, uintptr_t new); +unsigned long cmpxchg_emu_u16(volatile u16 *p, unsigned long old, unsigned long new); #endif /* __LINUX_CMPXCHG_EMU_H */ diff --git a/include/linux/cnum.h b/include/linux/cnum.h index 49b7d0c7645d..49160318adf9 100644 --- a/include/linux/cnum.h +++ b/include/linux/cnum.h @@ -39,6 +39,7 @@ u32 cnum32_umin(struct cnum32 cnum); u32 cnum32_umax(struct cnum32 cnum); s32 cnum32_smin(struct cnum32 cnum); s32 cnum32_smax(struct cnum32 cnum); +struct cnum32 cnum32_union(struct cnum32 a, struct cnum32 b); struct cnum32 cnum32_intersect(struct cnum32 a, struct cnum32 b); void cnum32_intersect_with(struct cnum32 *dst, struct cnum32 src); void cnum32_intersect_with_urange(struct cnum32 *dst, u32 min, u32 max); @@ -65,6 +66,7 @@ u64 cnum64_umin(struct cnum64 cnum); u64 cnum64_umax(struct cnum64 cnum); s64 cnum64_smin(struct cnum64 cnum); s64 cnum64_smax(struct cnum64 cnum); +struct cnum64 cnum64_union(struct cnum64 a, struct cnum64 b); struct cnum64 cnum64_intersect(struct cnum64 a, struct cnum64 b); void cnum64_intersect_with(struct cnum64 *dst, struct cnum64 src); void cnum64_intersect_with_urange(struct cnum64 *dst, u64 min, u64 max); @@ -78,5 +80,6 @@ bool cnum64_is_subset(struct cnum64 outer, struct cnum64 inner); struct cnum32 cnum32_from_cnum64(struct cnum64 cnum); struct cnum64 cnum64_cnum32_intersect(struct cnum64 a, struct cnum32 b); +struct cnum64 cnum64_intersect_linear(struct cnum64 in, u16 base, u16 step); #endif /* _LINUX_CNUM_H */ diff --git a/include/linux/compaction.h b/include/linux/compaction.h index 66a2f70e9e01..691c09f0a697 100644 --- a/include/linux/compaction.h +++ b/include/linux/compaction.h @@ -81,10 +81,6 @@ static inline unsigned long compact_gap(unsigned int order) return min(2UL << order, COMPACT_CLUSTER_MAX); } -static inline int current_is_kcompactd(void) -{ - return current->flags & PF_KCOMPACTD; -} #ifdef CONFIG_COMPACTION @@ -103,7 +99,7 @@ extern void compaction_defer_reset(struct zone *zone, int order, bool compaction_zonelist_suitable(struct alloc_context *ac, int order, int alloc_flags, gfp_t gfp_mask); - +bool current_is_kcompactd(void); extern void __meminit kcompactd_run(int nid); extern void __meminit kcompactd_stop(int nid); extern void wakeup_kcompactd(pg_data_t *pgdat, int order, int highest_zoneidx); @@ -120,6 +116,11 @@ static inline bool compaction_suitable(struct zone *zone, int order, return false; } +static inline bool current_is_kcompactd(void) +{ + return false; +} + static inline void kcompactd_run(int nid) { } diff --git a/include/linux/compat.h b/include/linux/compat.h index 8da0a15c95f4..ad37ca617e65 100644 --- a/include/linux/compat.h +++ b/include/linux/compat.h @@ -18,6 +18,7 @@ #include <linux/aio_abi.h> /* for aio_context_t */ #include <linux/uaccess.h> #include <linux/unistd.h> +#include <linux/error-injection.h> #include <asm/compat.h> #include <asm/siginfo.h> diff --git a/include/linux/compiler.h b/include/linux/compiler.h index cb2f6050bdf7..0ea7150dba7b 100644 --- a/include/linux/compiler.h +++ b/include/linux/compiler.h @@ -239,6 +239,23 @@ void ftrace_likely_update(struct ftrace_likely_data *f, int val, # define TYPEOF_UNQUAL(exp) __typeof__(exp) #endif +/* + * TYPEOF_NO_ADDRESS_SPACE() - typeof() without the address space qualifiers + * + * No operator strips only the address space qualifiers: typeof() keeps every + * qualifier and TYPEOF_UNQUAL() drops every qualifier, const and volatile + * included. + * + * Approximate one by dropping the qualifiers for sparse only, as it is the + * only one that knows about address spaces. The compiler keeps seeing the fully + * qualified type, so a missing const or volatile will still throw a warning. + */ +#ifdef __CHECKER__ +# define TYPEOF_NO_ADDRESS_SPACE(exp) TYPEOF_UNQUAL(exp) +#else +# define TYPEOF_NO_ADDRESS_SPACE(exp) __typeof__(exp) +#endif + #endif /* __KERNEL__ */ #if defined(CONFIG_CFI) && !defined(__DISABLE_EXPORTS) && !defined(BUILD_VDSO) @@ -275,6 +292,11 @@ static inline void *offset_to_ptr(const int *off) #define __ADDRESSABLE(sym) \ ___ADDRESSABLE(sym, __section(".discard.addressable")) +/* Enforce static storage duration. */ +#define ASSERT_STATIC_STORAGE(name) \ + static typeof(name) * const __always_unused \ + name##_storage_check = &(name) + /* * This returns a constant expression while determining if an argument is * a constant expression, most importantly without evaluating the argument. diff --git a/include/linux/configfs.h b/include/linux/configfs.h index ef65c75beeaa..5bead9173ec1 100644 --- a/include/linux/configfs.h +++ b/include/linux/configfs.h @@ -66,8 +66,11 @@ struct config_item_type { struct module *ct_owner; const struct configfs_item_operations *ct_item_ops; const struct configfs_group_operations *ct_group_ops; - struct configfs_attribute **ct_attrs; - struct configfs_bin_attribute **ct_bin_attrs; + union { + struct configfs_attribute **ct_attrs; + const struct configfs_attribute *const *ct_attrs_const; + }; + const struct configfs_bin_attribute *const *ct_bin_attrs; }; /** @@ -160,41 +163,41 @@ struct configfs_bin_attribute { ssize_t (*write)(struct config_item *, const void *, size_t); }; -#define CONFIGFS_BIN_ATTR(_pfx, _name, _priv, _maxsz) \ -static struct configfs_bin_attribute _pfx##attr_##_name = { \ - .cb_attr = { \ - .ca_name = __stringify(_name), \ - .ca_mode = S_IRUGO | S_IWUSR, \ - .ca_owner = THIS_MODULE, \ - }, \ - .cb_private = _priv, \ - .cb_max_size = _maxsz, \ - .read = _pfx##_name##_read, \ - .write = _pfx##_name##_write, \ +#define CONFIGFS_BIN_ATTR(_pfx, _name, _priv, _maxsz) \ +static const struct configfs_bin_attribute _pfx##attr_##_name = { \ + .cb_attr = { \ + .ca_name = __stringify(_name), \ + .ca_mode = S_IRUGO | S_IWUSR, \ + .ca_owner = THIS_MODULE, \ + }, \ + .cb_private = _priv, \ + .cb_max_size = _maxsz, \ + .read = _pfx##_name##_read, \ + .write = _pfx##_name##_write, \ } -#define CONFIGFS_BIN_ATTR_RO(_pfx, _name, _priv, _maxsz) \ -static struct configfs_bin_attribute _pfx##attr_##_name = { \ - .cb_attr = { \ - .ca_name = __stringify(_name), \ - .ca_mode = S_IRUGO, \ - .ca_owner = THIS_MODULE, \ - }, \ - .cb_private = _priv, \ - .cb_max_size = _maxsz, \ - .read = _pfx##_name##_read, \ +#define CONFIGFS_BIN_ATTR_RO(_pfx, _name, _priv, _maxsz) \ +static const struct configfs_bin_attribute _pfx##attr_##_name = { \ + .cb_attr = { \ + .ca_name = __stringify(_name), \ + .ca_mode = S_IRUGO, \ + .ca_owner = THIS_MODULE, \ + }, \ + .cb_private = _priv, \ + .cb_max_size = _maxsz, \ + .read = _pfx##_name##_read, \ } -#define CONFIGFS_BIN_ATTR_WO(_pfx, _name, _priv, _maxsz) \ -static struct configfs_bin_attribute _pfx##attr_##_name = { \ - .cb_attr = { \ - .ca_name = __stringify(_name), \ - .ca_mode = S_IWUSR, \ - .ca_owner = THIS_MODULE, \ - }, \ - .cb_private = _priv, \ - .cb_max_size = _maxsz, \ - .write = _pfx##_name##_write, \ +#define CONFIGFS_BIN_ATTR_WO(_pfx, _name, _priv, _maxsz) \ +static const struct configfs_bin_attribute _pfx##attr_##_name = { \ + .cb_attr = { \ + .ca_name = __stringify(_name), \ + .ca_mode = S_IWUSR, \ + .ca_owner = THIS_MODULE, \ + }, \ + .cb_private = _priv, \ + .cb_max_size = _maxsz, \ + .write = _pfx##_name##_write, \ } /* @@ -220,8 +223,8 @@ struct configfs_group_operations { struct config_group *(*make_group)(struct config_group *group, const char *name); void (*disconnect_notify)(struct config_group *group, struct config_item *item); void (*drop_item)(struct config_group *group, struct config_item *item); - bool (*is_visible)(struct config_item *item, struct configfs_attribute *attr, int n); - bool (*is_bin_visible)(struct config_item *item, struct configfs_bin_attribute *attr, + bool (*is_visible)(struct config_item *item, const struct configfs_attribute *attr, int n); + bool (*is_bin_visible)(struct config_item *item, const struct configfs_bin_attribute *attr, int n); }; diff --git a/include/linux/console.h b/include/linux/console.h index df23148987ee..cb8be9ea4a76 100644 --- a/include/linux/console.h +++ b/include/linux/console.h @@ -55,9 +55,12 @@ enum vc_intensity; * Invoked by csi_M and printing to the console. * @con_switch: notifier about the console switch; it is supposed to return * true if a redraw is needed. - * @con_blank: blank/unblank the console. The target mode is passed in @blank. - * @mode_switch is set if changing from/to text/graphics. The hook - * is supposed to return true if a redraw is needed. + * @con_blank: blank the console. The preferred VESA blank mode is passed in + * @blank. @mode_switch is set if changing from/to text/graphics. + * The hook is supposed to return true if a redraw is needed. + * @con_unblank: unblank the console. The hook is supposed to return true if + * a redraw is needed. @mode_switch is set if changing from/to + * text/graphics. * @con_font_set: set console @vc font to @font with height @vpitch. @flags can * be %KD_FONT_FLAG_DONT_RECALC. (optional) * @con_font_get: fetch the current font on @vc of height @vpitch into @font. @@ -99,6 +102,7 @@ struct consw { bool (*con_switch)(struct vc_data *vc); bool (*con_blank)(struct vc_data *vc, enum vesa_blank_mode blank, bool mode_switch); + bool (*con_unblank)(struct vc_data *vc, bool mode_switch); int (*con_font_set)(struct vc_data *vc, const struct console_font *font, unsigned int vpitch, unsigned int flags); diff --git a/include/linux/console_struct.h b/include/linux/console_struct.h index fe915afdece5..de9e91429a88 100644 --- a/include/linux/console_struct.h +++ b/include/linux/console_struct.h @@ -204,6 +204,10 @@ struct vc { extern struct vc vc_cons [MAX_NR_CONSOLES]; extern void vc_SAK(struct work_struct *work); +/* + * Cursors + */ + #define CUR_MAKE(size, change, set) ((size) | ((change) << 8) | \ ((set) << 16)) #define CUR_SIZE(c) ((c) & 0x00000f) @@ -222,6 +226,10 @@ extern void vc_SAK(struct work_struct *work); #define CUR_CHANGE(c) ((c) & 0x00ff00) #define CUR_SET(c) (((c) & 0xff0000) >> 8) +unsigned int vc_cursor_valid_size(unsigned int cursor_size); +unsigned int vc_cursor_start(unsigned int cell_height, unsigned int cursor_size); +unsigned int vc_cursor_end(unsigned int cell_height, unsigned int cursor_size); + bool con_is_visible(const struct vc_data *vc); #endif /* _LINUX_CONSOLE_STRUCT_H */ diff --git a/include/linux/coredump.h b/include/linux/coredump.h index 7b38ee2e7913..8e857770dd21 100644 --- a/include/linux/coredump.h +++ b/include/linux/coredump.h @@ -5,10 +5,22 @@ #include <linux/types.h> #include <linux/mm.h> #include <linux/fs.h> +#include <linux/pid_types.h> #include <linux/sched/coredump.h> +#include <uapi/linux/coredump.h> #include <asm/siginfo.h> #ifdef CONFIG_COREDUMP +/** + * enum coredump_state - what happened while the coredump was written + * @COREDUMP_STATE_STARTED: the dumper committed to writing a coredump + * @COREDUMP_STATE_TRUNCATED: the dumper stopped before it had written all of it + */ +enum coredump_state { + COREDUMP_STATE_STARTED = (1U << 0), + COREDUMP_STATE_TRUNCATED = (1U << 1), +}; + struct core_vma_metadata { unsigned long start, end; vm_flags_t flags; @@ -21,18 +33,27 @@ struct coredump_params { const kernel_siginfo_t *siginfo; struct file *file; unsigned long limit; - /* MMF_DUMP_FILTER_* bits, snapshot of mm->flags at dump start. */ - unsigned long mm_flags; + /* COREDUMP_MEMORY_* types to dump, the task's or the server's. */ + u64 memory_types; /* Snapshot of dumpable at dump start. */ enum task_dumpable dumpable; int cpu; + /* COREDUMP_* options negotiated with the coredump server. */ + u64 mask; + /* COREDUMP_STATE_* raised while the coredump is written. */ + enum coredump_state state; + /* Record header scratch, NULL unless the coredump is a record stream. */ + struct coredump_record_header *record_hdr; + /* Bytes handed to the file, record headers included. */ loff_t written; + /* Offset in the coredump, record headers excluded. */ loff_t pos; loff_t to_skip; int vma_count; size_t vma_data_size; struct core_vma_metadata *vma_meta; - struct pid *pid; + /* Dumping thread and its thread-group leader by pid type. */ + DECLARE_PIDS(pid, PIDTYPE_TGID); }; extern unsigned int core_file_note_size_limit; @@ -41,13 +62,13 @@ extern unsigned int core_file_note_size_limit; * These are the only things you should do on a core-file: use only these * functions to write out all the necessary info. */ -extern void dump_skip_to(struct coredump_params *cprm, unsigned long to); -extern void dump_skip(struct coredump_params *cprm, size_t nr); -extern int dump_emit(struct coredump_params *cprm, const void *addr, int nr); -extern int dump_align(struct coredump_params *cprm, int align); -int dump_user_range(struct coredump_params *cprm, unsigned long start, - unsigned long len); -extern void vfs_coredump(const kernel_siginfo_t *siginfo); +void dump_skip_to(struct coredump_params *cprm, unsigned long to); +void dump_skip(struct coredump_params *cprm, size_t nr); +bool dump_emit(struct coredump_params *cprm, const void *addr, int nr); +bool dump_align(struct coredump_params *cprm, int align); +bool dump_user_range(struct coredump_params *cprm, unsigned long start, + unsigned long len); +void vfs_coredump(const kernel_siginfo_t *siginfo); /* * Logging for the coredump code, ratelimited. diff --git a/include/linux/coresight.h b/include/linux/coresight.h index ddf18c970e34..edf2a2fc697c 100644 --- a/include/linux/coresight.h +++ b/include/linux/coresight.h @@ -142,6 +142,8 @@ struct csdev_access { }) #define CORESIGHT_DESC_CPU_BOUND BIT(0) +/* Device can't be activated from sysfs, only via Perf. */ +#define CORESIGHT_DESC_NO_SYSFS_MODE BIT(1) /** * struct coresight_desc - description of a component required from drivers @@ -310,6 +312,8 @@ struct coresight_device { struct list_head config_csdev_list; raw_spinlock_t cscfg_csdev_lock; void *active_cscfg_ctxt; + /* CORESIGHT_DESC_[x] flags */ + u32 flags; }; /* diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h index feb32949aeea..1b6dba159bb2 100644 --- a/include/linux/cpuhotplug.h +++ b/include/linux/cpuhotplug.h @@ -209,12 +209,6 @@ enum cpuhp_state { CPUHP_AP_PERF_S390_SF_ONLINE, CPUHP_AP_PERF_ARM_CCI_ONLINE, CPUHP_AP_PERF_ARM_CCN_ONLINE, - CPUHP_AP_PERF_ARM_HISI_CPA_ONLINE, - CPUHP_AP_PERF_ARM_HISI_DDRC_ONLINE, - CPUHP_AP_PERF_ARM_HISI_HHA_ONLINE, - CPUHP_AP_PERF_ARM_HISI_L3_ONLINE, - CPUHP_AP_PERF_ARM_HISI_PA_ONLINE, - CPUHP_AP_PERF_ARM_HISI_SLLC_ONLINE, CPUHP_AP_PERF_ARM_HISI_PCIE_PMU_ONLINE, CPUHP_AP_PERF_ARM_HNS3_PMU_ONLINE, CPUHP_AP_PERF_ARM_L2X0_ONLINE, diff --git a/include/linux/cpumask.h b/include/linux/cpumask.h index 4c8bb6953107..bf89bb3f30f6 100644 --- a/include/linux/cpumask.h +++ b/include/linux/cpumask.h @@ -121,12 +121,20 @@ extern struct cpumask __cpu_enabled_mask; extern struct cpumask __cpu_present_mask; extern struct cpumask __cpu_active_mask; extern struct cpumask __cpu_dying_mask; + +#ifdef CONFIG_PREFERRED_CPU +extern struct cpumask __cpu_preferred_mask; +#else +#define __cpu_preferred_mask __cpu_active_mask +#endif + #define cpu_possible_mask ((const struct cpumask *)&__cpu_possible_mask) #define cpu_online_mask ((const struct cpumask *)&__cpu_online_mask) #define cpu_enabled_mask ((const struct cpumask *)&__cpu_enabled_mask) #define cpu_present_mask ((const struct cpumask *)&__cpu_present_mask) #define cpu_active_mask ((const struct cpumask *)&__cpu_active_mask) #define cpu_dying_mask ((const struct cpumask *)&__cpu_dying_mask) +#define cpu_preferred_mask ((const struct cpumask *)&__cpu_preferred_mask) extern atomic_t __num_online_cpus; extern unsigned int __num_possible_cpus; @@ -825,6 +833,24 @@ bool cpumask_intersects(const struct cpumask *src1p, const struct cpumask *src2p } /** + * cpumask_intersects_and - (*src1p & *src2p & *src3p) != 0 + * @src1p: the first input + * @src2p: the second input + * @src3p: the third input + * + * Return: true if AND of the three cpumasks is non-empty, + * otherwise false + */ +static __always_inline +bool cpumask_intersects_and(const struct cpumask *src1p, + const struct cpumask *src2p, + const struct cpumask *src3p) +{ + return bitmap_intersects_and(cpumask_bits(src1p), cpumask_bits(src2p), + cpumask_bits(src3p), small_cpumask_bits); +} + +/** * cpumask_subset - (*src1p & ~*src2p) == 0 * @src1p: the first input * @src2p: the second input @@ -1163,6 +1189,12 @@ void init_cpu_possible(const struct cpumask *src); #define set_cpu_active(cpu, active) assign_cpu((cpu), &__cpu_active_mask, (active)) #define set_cpu_dying(cpu, dying) assign_cpu((cpu), &__cpu_dying_mask, (dying)) +#ifdef CONFIG_PREFERRED_CPU +#define set_cpu_preferred(cpu, preferred) assign_cpu((cpu), &__cpu_preferred_mask, (preferred)) +#else +#define set_cpu_preferred(cpu, preferred) do { } while (0) +#endif + void set_cpu_online(unsigned int cpu, bool online); void set_cpu_possible(unsigned int cpu, bool possible); @@ -1257,6 +1289,11 @@ static __always_inline bool cpu_dying(unsigned int cpu) return cpumask_test_cpu(cpu, cpu_dying_mask); } +static __always_inline bool cpu_preferred(unsigned int cpu) +{ + return cpumask_test_cpu(cpu, cpu_preferred_mask); +} + #else #define num_online_cpus() 1U @@ -1295,6 +1332,11 @@ static __always_inline bool cpu_dying(unsigned int cpu) return false; } +static __always_inline bool cpu_preferred(unsigned int cpu) +{ + return cpu == 0; +} + #endif /* NR_CPUS > 1 */ #define cpu_is_offline(cpu) unlikely(!cpu_online(cpu)) diff --git a/include/linux/cred.h b/include/linux/cred.h index 6ef1750c93e2..49c26af37349 100644 --- a/include/linux/cred.h +++ b/include/linux/cred.h @@ -180,12 +180,18 @@ static inline bool cap_ambient_invariant_ok(const struct cred *cred) static inline const struct cred *override_creds(const struct cred *override_cred) { - return rcu_replace_pointer(current->cred, override_cred, 1); + const struct cred *old = current->cred; + + current->cred = override_cred; + return old; } static inline const struct cred *revert_creds(const struct cred *revert_cred) { - return rcu_replace_pointer(current->cred, revert_cred, 1); + const struct cred *override_cred = current->cred; + + current->cred = revert_cred; + return override_cred; } DEFINE_CLASS(override_creds, @@ -293,11 +299,10 @@ DEFINE_FREE(put_cred, struct cred *, if (!IS_ERR_OR_NULL(_T)) put_cred(_T)) /** * current_cred - Access the current task's subjective credentials * - * Access the subjective credentials of the current task. RCU-safe, - * since nobody else can modify it. + * Access the subjective credentials of the current task. + * Nobody else can modify it. */ -#define current_cred() \ - rcu_dereference_protected(current->cred, 1) +#define current_cred() (current->cred) /** * current_real_cred - Access the current task's objective credentials diff --git a/include/linux/damon.h b/include/linux/damon.h index 0c8b7ddef9ab..63050eb2206a 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -117,7 +117,6 @@ struct damon_target { * @DAMOS_MIGRATE_HOT: Migrate the regions prioritizing warmer regions. * @DAMOS_MIGRATE_COLD: Migrate the regions prioritizing colder regions. * @DAMOS_STAT: Do nothing but count the stat. - * @NR_DAMOS_ACTIONS: Total number of DAMOS actions * * The support of each action is up to running &struct damon_operations. * Refer to 'Operation Action' section of Documentation/mm/damon/design.rst for @@ -137,7 +136,6 @@ enum damos_action { DAMOS_MIGRATE_HOT, DAMOS_MIGRATE_COLD, DAMOS_STAT, /* Do nothing but only record the stat */ - NR_DAMOS_ACTIONS, }; /** @@ -153,9 +151,7 @@ enum damos_action { * @DAMOS_QUOTA_INACTIVE_MEM_BP: Inactive to total LRU memory ratio. * @DAMOS_QUOTA_NODE_ELIGIBLE_MEM_BP: Scheme-eligible memory ratio of a * node in basis points (0-10000). - * @NR_DAMOS_QUOTA_GOAL_METRICS: Number of DAMOS quota goal metrics. - * - * Metrics equal to larger than @NR_DAMOS_QUOTA_GOAL_METRICS are unsupported. + * @DAMOS_QUOTA_HUGEPAGE_MEM_BP: Huge page to total used memory ratio. */ enum damos_quota_goal_metric { DAMOS_QUOTA_USER_INPUT, @@ -167,12 +163,13 @@ enum damos_quota_goal_metric { DAMOS_QUOTA_ACTIVE_MEM_BP, DAMOS_QUOTA_INACTIVE_MEM_BP, DAMOS_QUOTA_NODE_ELIGIBLE_MEM_BP, - NR_DAMOS_QUOTA_GOAL_METRICS, + DAMOS_QUOTA_HUGEPAGE_MEM_BP, }; /** * struct damos_quota_goal - DAMOS scheme quota auto-tuning goal. * @metric: Metric to be used for representing the goal. + * @complement: Use the complement of the metric. * @target_value: Target value of @metric to achieve with the tuning. * @current_value: Current value of @metric. * @nid: Node id. @@ -195,6 +192,7 @@ enum damos_quota_goal_metric { */ struct damos_quota_goal { enum damos_quota_goal_metric metric; + bool complement; unsigned long target_value; unsigned long current_value; /* metric-dependent fields */ @@ -311,12 +309,10 @@ struct damos_quota { * * @DAMOS_WMARK_NONE: Ignore the watermarks of the given scheme. * @DAMOS_WMARK_FREE_MEM_RATE: Free memory rate of the system in [0,1000]. - * @NR_DAMOS_WMARK_METRICS: Total number of DAMOS watermark metrics */ enum damos_wmark_metric { DAMOS_WMARK_NONE, DAMOS_WMARK_FREE_MEM_RATE, - NR_DAMOS_WMARK_METRICS, }; /** @@ -357,7 +353,8 @@ struct damos_watermarks { * Total bytes that passed ops layer-handled DAMOS filters. * @qt_exceeds: Total number of times the quota of the scheme has exceeded. * @nr_snapshots: - * Total number of DAMON snapshots that the scheme has tried. + * Total number of DAMON snapshots that the scheme is completely + * tried to be applied. * * "Tried an action to a region" in this context means the DAMOS core logic * determined the region as eligible to apply the action. The access pattern @@ -396,14 +393,14 @@ struct damos_stat { * @DAMOS_FILTER_TYPE_UNMAPPED: Unmapped pages. * @DAMOS_FILTER_TYPE_ADDR: Address range. * @DAMOS_FILTER_TYPE_TARGET: Data Access Monitoring target. - * @NR_DAMOS_FILTER_TYPES: Number of filter types. + * @DAMOS_FILTER_TYPE_PROBE_HITS_WSUM: probe_hits weighted sum range. * - * All types except &DAMOS_FILTER_TYPE_ADDR and &DAMOS_FILTER_TYPE_TARGET - * are handled by the underlying &struct damon_operations as a part of scheme - * action trying, and therefore accounted as 'tried'. In contrast, - * &DAMOS_FILTER_TYPE_ADDR and &DAMOS_FILTER_TYPE_TARGET filters are handled - * by the core layer before trying of the action, and therefore not accounted - * as 'tried'. + * All types except &DAMOS_FILTER_TYPE_ADDR, &DAMOS_FILTER_TYPE_TARGET and + * &DAMOS_FILTER_TYPE_PROBE_HITS_WSUM are handled by the underlying &struct + * damon_operations as a part of scheme action trying, and therefore accounted + * as 'tried'. In contrast, &DAMOS_FILTER_TYPE_ADDR, &DAMOS_FILTER_TYPE_TARGET + * and &DAMOS_FILTER_TYPE_PROBE_HITS_WSUM filters are handled by the core layer + * before trying of the action, and therefore not accounted as 'tried'. * * Support for the operations-handled filters depends on the running * &struct damon_operations. @@ -417,7 +414,7 @@ enum damos_filter_type { DAMOS_FILTER_TYPE_UNMAPPED, DAMOS_FILTER_TYPE_ADDR, DAMOS_FILTER_TYPE_TARGET, - NR_DAMOS_FILTER_TYPES, + DAMOS_FILTER_TYPE_PROBE_HITS_WSUM, }; /** @@ -431,6 +428,8 @@ enum damos_filter_type { * &damon_ctx->adaptive_targets if @type is * DAMOS_FILTER_TYPE_TARGET. * @sz_range: Size range if @type is DAMOS_FILTER_TYPE_HUGEPAGE_SIZE. + * @range_min: Minimum value of range arguments. + * @range_max: Maximum value of range arguments. * * Before applying the &damos->action to a memory region, DAMOS checks if each * byte of the region matches to this given condition and avoid applying the @@ -447,6 +446,10 @@ struct damos_filter { struct damon_addr_range addr_range; int target_idx; struct damon_size_range sz_range; + struct { + unsigned long range_min; + unsigned long range_max; + }; }; /* private: */ /* List head for siblings. */ @@ -548,7 +551,7 @@ struct damos_migrate_dests { * * After applying the &action to each region, &stat is updated. * - * If &max_nr_snapshots is set as non-zero and &stat.nr_snapshots be same to or + * If &max_nr_snapshots is set as non-zero and &stat.nr_snapshots equals or is * greater than it, the scheme is deactivated. */ struct damos { @@ -627,6 +630,7 @@ enum damon_ops_id { * @update: Update operations-related data structures. * @prepare_access_checks: Prepare next access check of target regions. * @check_accesses: Check the accesses to target regions. + * @prep_probes: Prepare applying probes for each region. * @apply_probes: Apply probes for each region. * @get_scheme_score: Get the score of a region for a scheme. * @apply_scheme: Apply a DAMON-based operation scheme. @@ -654,6 +658,9 @@ enum damon_ops_id { * last preparation and update the number of observed accesses of each region. * It should also return max number of observed accesses that made as a result * of its update. The value will be used for regions adjustment threshold. + * @prep_probes should execute required &struct damon_prep for next &struct + * damon_probe applications to each region. It should also set + * &damon_region->sampling_addr of each region if ``set_samples`` is true. * @apply_probes should apply the data attribute probes to each region and * accordingly update the probe hits counter of the region. It should also * set &damon_region->sampling_addr of each region if ``set_samples`` is true. @@ -676,6 +683,7 @@ struct damon_operations { void (*update)(struct damon_ctx *context); void (*prepare_access_checks)(struct damon_ctx *context); unsigned int (*check_accesses)(struct damon_ctx *context); + void (*prep_probes)(struct damon_ctx *context, bool set_samples); unsigned int (*apply_probes)(struct damon_ctx *context, bool set_samples, bool return_max_wsum); int (*get_scheme_score)(struct damon_ctx *context, @@ -740,14 +748,41 @@ struct damon_intervals_goal { }; /** + * enum damon_prep_action - DAMON probing preparation action. + * + * @DAMON_PREP_SET_PGIDLE: Set the probing memory as idle page. + */ +enum damon_prep_action { + DAMON_PREP_SET_PGIDLE, +}; + +/** + * struct damon_prep - DAMON probing preparation request. + * + * @action: Action to do to the probing memory for the preparation. + */ +struct damon_prep { + enum damon_prep_action action; +/* private: */ + /* siblings list. */ + struct list_head list; +}; + +/** * enum damon_filter_type - Type of &struct damon_filter * - * @DAMON_FILTER_TYPE_ANON: Anonymous pages. - * @DAMON_FILTER_TYPE_MEMCG: Specific memcg's pages. + * @DAMON_FILTER_TYPE_ANON: Anonymous pages. + * @DAMON_FILTER_TYPE_MEMCG: Specific memcg's pages. + * @DAMON_FILTER_TYPE_PGIDLE_UNSET: Pgidle is unset. + * @DAMON_FILTER_TYPE_PGIDLE_SET: Pgidle is set. + * @DAMON_FILTER_TYPE_HUGEPAGE_SIZE: Page is part of a hugepage. */ enum damon_filter_type { DAMON_FILTER_TYPE_ANON, DAMON_FILTER_TYPE_MEMCG, + DAMON_FILTER_TYPE_PGIDLE_UNSET, + DAMON_FILTER_TYPE_PGIDLE_SET, + DAMON_FILTER_TYPE_HUGEPAGE_SIZE, }; /** @@ -757,6 +792,8 @@ enum damon_filter_type { * @matching: Whether this filter is for the type-matching ones. * @allow: Whether the @type-@matching ones should pass this filter. * @memcg_id: Memcg id of the question if @type is DAMON_FILTER_MEMCG. + * @range_min: Minimum value of range arguments. + * @range_max: Maximum value of range arguments. */ struct damon_filter { enum damon_filter_type type; @@ -764,6 +801,10 @@ struct damon_filter { bool allow; union { u64 memcg_id; + struct { + unsigned long range_min; + unsigned long range_max; + }; }; /* private: */ /* Siblings list. */ @@ -778,6 +819,8 @@ struct damon_filter { struct damon_probe { unsigned int weight; /* private: */ + /* Preparation actions to apply to each probing memory. */ + struct list_head preps; /* Filters for assessing if a given region is for this probe. */ struct list_head filters; /* Siblings list. */ @@ -787,7 +830,8 @@ struct damon_probe { /** * struct damon_attrs - Monitoring attributes for accuracy/overhead control. * - * @sample_interval: The time between access samplings. + * @sample_interval: The time between access samplings. Zero is + * accepted. * @aggr_interval: The time between monitor results aggregations. * @ops_update_interval: The time between monitoring operations updates. * @intervals_goal: Intervals auto-tuning goal. @@ -957,6 +1001,12 @@ static inline unsigned long damon_sz_region(struct damon_region *r) return r->ar.end - r->ar.start; } +#define damon_for_each_prep(p, probe) \ + list_for_each_entry(p, &(probe)->preps, list) + +#define damon_for_each_prep_safe(p, next, probe) \ + list_for_each_entry_safe(p, next, &(probe)->preps, list) + #define damon_for_each_filter(f, p) \ list_for_each_entry(f, &(p)->filters, list) @@ -1010,6 +1060,9 @@ static inline unsigned long damon_sz_region(struct damon_region *r) #ifdef CONFIG_DAMON +struct damon_prep *damon_new_prep(enum damon_prep_action action); +void damon_add_prep(struct damon_probe *p, struct damon_prep *prep); + struct damon_filter *damon_new_filter(enum damon_filter_type type, bool matching, bool allow); void damon_add_filter(struct damon_probe *probe, struct damon_filter *f); @@ -1023,7 +1076,7 @@ unsigned int damon_nr_accesses_mvsum(struct damon_region *r, struct damon_ctx *ctx); unsigned char damon_probe_hits_mvsum(int probe_idx, struct damon_region *r, struct damon_ctx *ctx); -unsigned int damon_probe_hits_wsum(struct damon_region *r, bool last, +unsigned int damon_probe_hits_wsum(struct damon_region *r, bool last, bool mv, struct damon_ctx *ctx); int damon_set_regions(struct damon_target *t, struct damon_addr_range *ranges, @@ -1037,7 +1090,7 @@ bool damos_filter_for_ops(enum damos_filter_type type); void damos_destroy_filter(struct damos_filter *f); struct damos_quota_goal *damos_new_quota_goal( - enum damos_quota_goal_metric metric, + enum damos_quota_goal_metric metric, bool complement, unsigned long target_value); void damos_add_quota_goal(struct damos_quota *q, struct damos_quota_goal *g); void damos_destroy_quota_goal(struct damos_quota_goal *goal); @@ -1054,7 +1107,7 @@ int damos_commit_quota_goals(struct damos_quota *dst, struct damos_quota *src); struct damon_target *damon_new_target(void); void damon_add_target(struct damon_ctx *ctx, struct damon_target *t); -bool damon_targets_empty(struct damon_ctx *ctx); +int damon_set_target_pid(struct damon_target *t, int pid); void damon_free_target(struct damon_target *t); void damon_destroy_target(struct damon_target *t, struct damon_ctx *ctx); unsigned int damon_nr_regions(struct damon_target *t); @@ -1065,7 +1118,6 @@ int damon_set_attrs(struct damon_ctx *ctx, struct damon_attrs *attrs); void damon_set_schemes(struct damon_ctx *ctx, struct damos **schemes, ssize_t nr_schemes); int damon_commit_ctx(struct damon_ctx *old_ctx, struct damon_ctx *new_ctx); -int damon_nr_running_ctxs(void); bool damon_is_registered_ops(enum damon_ops_id id); int damon_register_ops(struct damon_operations *ops); int damon_select_ops(struct damon_ctx *ctx, enum damon_ops_id id); diff --git a/include/linux/dax.h b/include/linux/dax.h index fe6c3ded1b50..a3af0c63c61b 100644 --- a/include/linux/dax.h +++ b/include/linux/dax.h @@ -54,7 +54,6 @@ struct dax_device *alloc_dax(void *private, const struct dax_operations *ops); void *dax_holder(struct dax_device *dax_dev); void put_dax(struct dax_device *dax_dev); void kill_dax(struct dax_device *dax_dev); -struct dax_device *dax_dev_get(dev_t devt); void dax_write_cache(struct dax_device *dax_dev, bool wc); bool dax_write_cache_enabled(struct dax_device *dax_dev); bool dax_synchronous(struct dax_device *dax_dev); @@ -155,8 +154,6 @@ int dax_writeback_mapping_range(struct address_space *mapping, struct dax_device *dax_dev, struct writeback_control *wbc); int dax_folio_reset_order(struct folio *folio); -struct page *dax_layout_busy_page(struct address_space *mapping); -struct page *dax_layout_busy_page_range(struct address_space *mapping, loff_t start, loff_t end); dax_entry_t dax_lock_folio(struct folio *folio); void dax_unlock_folio(struct folio *folio, dax_entry_t cookie); dax_entry_t dax_lock_mapping_entry(struct address_space *mapping, @@ -173,16 +170,6 @@ static inline int fs_dax_get(struct dax_device *dax_dev, void *holder, { return -EOPNOTSUPP; } -static inline struct page *dax_layout_busy_page(struct address_space *mapping) -{ - return NULL; -} - -static inline struct page *dax_layout_busy_page_range(struct address_space *mapping, pgoff_t start, pgoff_t nr_pages) -{ - return NULL; -} - static inline int dax_writeback_mapping_range(struct address_space *mapping, struct dax_device *dax_dev, struct writeback_control *wbc) { @@ -323,3 +310,5 @@ typedef int (*walk_hmem_fn)(struct device *dev, int target_nid, const struct resource *res); int walk_hmem_resources(struct device *dev, walk_hmem_fn fn); #endif + +struct dax_device *fsdev_dax_from_file(struct file *file); diff --git a/include/linux/dcache.h b/include/linux/dcache.h index 4b1ff99608e0..84b0cfe4ed0e 100644 --- a/include/linux/dcache.h +++ b/include/linux/dcache.h @@ -116,6 +116,8 @@ struct dentry { * possible! */ + /* lockdep tracking of DCACHE_PAR_LOOKUP locks */ + struct lockdep_map dep_map; struct list_head d_lru; /* LRU list */ struct hlist_node d_sib; /* child of parent list */ struct hlist_head d_children; /* our children */ @@ -236,7 +238,9 @@ enum dentry_flags { DCACHE_PAR_LOOKUP = BIT(24), /* being looked up (with parent locked shared) */ DCACHE_DENTRY_CURSOR = BIT(25), DCACHE_NORCU = BIT(26), /* No RCU delay for freeing */ - DCACHE_PERSISTENT = BIT(27) + DCACHE_PERSISTENT = BIT(27), +/* 28, 29, 30 free */ + DCACHE_PRIVATE = BIT(31) /* fs-specific flag */ }; #define DCACHE_MANAGED_DENTRY \ @@ -257,7 +261,9 @@ extern void d_delete(struct dentry *); extern struct dentry * d_alloc(struct dentry *, const struct qstr *); extern struct dentry * d_alloc_anon(struct super_block *); extern struct dentry * d_alloc_parallel(struct dentry *, const struct qstr *); +extern struct dentry * d_alloc_trylock(struct dentry *, struct qstr *); extern struct dentry * d_splice_alias(struct inode *, struct dentry *); +struct dentry *d_duplicate(struct dentry *dentry); /* weird procfs mess; *NOT* exported */ extern struct dentry * d_splice_alias_ops(struct inode *, struct dentry *, const struct dentry_operations *); @@ -553,6 +559,36 @@ static inline int simple_positive(const struct dentry *dentry) unsigned long vfs_pressure_ratio(unsigned long val); /** + * d_lookup_release - release ownership of DCACHE_PAR_LOOKUP lock + * @dentry: dentry that is locked + * + * If an in-lookup dentry is to be passed to another thread which + * will drop the in-lookup lock, then d_lookup_release() must be called + * to tell lockdep that this thread no lock holds the lock. The + * thread that receives the lock must call d_lookup_acquire() to + * acquire the lock. + */ +static inline void d_lookup_release(struct dentry *dentry) +{ + if (d_in_lookup(dentry)) + lock_map_release(&dentry->dep_map); +} + +/** + * d_lookup_acquire - acquire ownership of DCACHE_PAR_LOOKUP lock + * @dentry: dentry that is locked + * + * If an in-lookup dentry was passed to this thread, the + * d_lookup_acquire() must be called to tell lockdep that this + * thread now owns the DCACHE_PAR_LOOKUP lock. + */ +static inline void d_lookup_acquire(struct dentry *dentry) +{ + if (d_in_lookup(dentry)) + lock_map_acquire_try(&dentry->dep_map); +} + +/** * d_inode - Get the actual inode of this dentry * @dentry: The dentry to query * @@ -629,6 +665,7 @@ struct name_snapshot { union shortname_store inline_name; }; void take_dentry_name_snapshot(struct name_snapshot *, struct dentry *); +bool try_take_dentry_name_snapshot(struct name_snapshot *, struct dentry *); void release_dentry_name_snapshot(struct name_snapshot *); static inline struct dentry *d_first_child(const struct dentry *dentry) diff --git a/include/linux/device-id/apr.h b/include/linux/device-id/apr.h deleted file mode 100644 index f282608ea018..000000000000 --- a/include/linux/device-id/apr.h +++ /dev/null @@ -1,21 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef LINUX_DEVICE_ID_APR_H -#define LINUX_DEVICE_ID_APR_H - -#ifdef __KERNEL__ -#include <linux/types.h> -typedef unsigned long kernel_ulong_t; -#endif - -#define APR_NAME_SIZE 32 -#define APR_MODULE_PREFIX "apr:" - -struct apr_device_id { - char name[APR_NAME_SIZE]; - __u32 domain_id; - __u32 svc_id; - __u32 svc_version; - kernel_ulong_t driver_data; /* Data private to the driver */ -}; - -#endif /* ifndef LINUX_DEVICE_ID_APR_H */ diff --git a/include/linux/device-id/arm_smccc.h b/include/linux/device-id/arm_smccc.h new file mode 100644 index 000000000000..a8579832fc1c --- /dev/null +++ b/include/linux/device-id/arm_smccc.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __LINUX_DEVICE_ID_ARM_SMCCC_H +#define __LINUX_DEVICE_ID_ARM_SMCCC_H + +#ifdef __KERNEL__ +#include <linux/types.h> +#endif + +#define ARM_SMCCC_MODULE_PREFIX "arm_smccc:" + +/** + * struct arm_smccc_device_id - Arm SMCCC bus device identifier + * @func_id: SMCCC function identifier + */ +struct arm_smccc_device_id { + __u32 func_id; +}; + +#endif /* __LINUX_DEVICE_ID_ARM_SMCCC_H */ diff --git a/include/linux/device-id/scmi.h b/include/linux/device-id/scmi.h new file mode 100644 index 000000000000..1b4ccfa9dcc5 --- /dev/null +++ b/include/linux/device-id/scmi.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef LINUX_DEVICE_ID_SCMI_H +#define LINUX_DEVICE_ID_SCMI_H + +#ifdef __KERNEL__ +#include <linux/types.h> +#endif + +#define SCMI_NAME_SIZE 32 +#define SCMI_MODULE_PREFIX "scmi:" + +struct scmi_device_id { + __u8 protocol_id; + char name[SCMI_NAME_SIZE]; +}; + +#endif /* ifndef LINUX_DEVICE_ID_SCMI_H */ diff --git a/include/linux/device.h b/include/linux/device.h index aee79fd6b32b..429e39f647cf 100644 --- a/include/linux/device.h +++ b/include/linux/device.h @@ -83,6 +83,10 @@ int subsys_virtual_register(const struct bus_type *subsys, * information, equivalent to the kobj_type of a kobject. * If "name" is specified, the uevent will contain it in * the DEVTYPE variable. + * + * @need_parent_lock: When probing or removing a device of this type, + * driver core should lock the device's parent. Note that + * the device must hold a reference to its parent. */ struct device_type { const char *name; @@ -93,6 +97,7 @@ struct device_type { void (*release)(struct device *dev); const struct dev_pm_ops *pm; + bool need_parent_lock; }; /** @@ -129,19 +134,19 @@ struct dev_ext_attribute { void *var; }; -ssize_t device_show_ulong(struct device *dev, struct device_attribute *attr, +ssize_t device_show_ulong(struct device *dev, const struct device_attribute *attr, char *buf); -ssize_t device_store_ulong(struct device *dev, struct device_attribute *attr, +ssize_t device_store_ulong(struct device *dev, const struct device_attribute *attr, const char *buf, size_t count); -ssize_t device_show_int(struct device *dev, struct device_attribute *attr, +ssize_t device_show_int(struct device *dev, const struct device_attribute *attr, char *buf); -ssize_t device_store_int(struct device *dev, struct device_attribute *attr, +ssize_t device_store_int(struct device *dev, const struct device_attribute *attr, const char *buf, size_t count); -ssize_t device_show_bool(struct device *dev, struct device_attribute *attr, +ssize_t device_show_bool(struct device *dev, const struct device_attribute *attr, char *buf); -ssize_t device_store_bool(struct device *dev, struct device_attribute *attr, +ssize_t device_store_bool(struct device *dev, const struct device_attribute *attr, const char *buf, size_t count); -ssize_t device_show_string(struct device *dev, struct device_attribute *attr, +ssize_t device_show_string(struct device *dev, const struct device_attribute *attr, char *buf); typedef ssize_t __device_show_handler_const(struct device *dev, const struct device_attribute *attr, @@ -1121,6 +1126,7 @@ static inline void device_unlock(struct device *dev) DEFINE_GUARD(device, struct device *, device_lock(_T), device_unlock(_T)) DEFINE_GUARD_COND(device, _intr, device_lock_interruptible(_T), _RET == 0) +DEFINE_GUARD_COND(device, _try, device_trylock(_T)) static inline void device_lock_assert(struct device *dev) { diff --git a/include/linux/device/bus.h b/include/linux/device/bus.h index a38f7229b8f4..9ef7538a9a7f 100644 --- a/include/linux/device/bus.h +++ b/include/linux/device/bus.h @@ -68,8 +68,6 @@ struct fwnode_handle; * @driver_override: Set to true if this bus supports the driver_override * mechanism, which allows userspace to force a specific * driver to bind to a device via a sysfs attribute. - * @need_parent_lock: When probing or removing a device on this bus, the - * device core should lock the device's parent. * * A bus is a channel between the processor and one or more devices. For the * purposes of the device model, all devices are connected via a bus, even if @@ -110,9 +108,9 @@ struct bus_type { const struct dev_pm_ops *pm; bool driver_override; - bool need_parent_lock; }; +int __must_check companion_bus_register(const struct bus_type *bus); int __must_check bus_register(const struct bus_type *bus); void bus_unregister(const struct bus_type *bus); diff --git a/include/linux/device/devres.h b/include/linux/device/devres.h index 14ab9159bdda..7656c9ecd4da 100644 --- a/include/linux/device/devres.h +++ b/include/linux/device/devres.h @@ -62,6 +62,108 @@ static inline void *devm_kcalloc(struct device *dev, size_t n, size_t size, gfp_ { return devm_kmalloc_array(dev, n, size, flags | __GFP_ZERO); } + +/** + * __devm_alloc_objs - Device-managed allocation of objects of a given type + * @DEVM_ALLOC: which device-managed size-based allocator to use. + * @DEV: device to scope the allocation lifetime to. + * @GFP: GFP flags for the allocation. + * @TYPE: type to allocate space for. + * @COUNT: how many @TYPE objects to allocate. + * + * Device-managed counterpart of __alloc_objs(). + * @DEVM_ALLOC takes (@DEV, size, gfp). + * + * Returns: Newly allocated pointer to (first) @TYPE of @COUNT-many allocated + * @TYPE objects, or NULL on failure. + */ +#define __devm_alloc_objs(DEVM_ALLOC, DEV, GFP, TYPE, COUNT) \ +({ \ + const size_t __obj_size = size_mul(sizeof(TYPE), COUNT); \ + (TYPE *)DEVM_ALLOC(DEV, __obj_size, GFP); \ +}) + +/** + * devm_kmalloc_obj - Device-managed allocation of a single typed object + * @DEV: Device to scope the allocation lifetime to. + * @VAR_OR_TYPE: Variable or type to allocate. + * @GFP: Optional GFP flags (defaults to %GFP_KERNEL). + * + * Device-managed counterpart of kmalloc_obj(): the allocation is sized from + * @VAR_OR_TYPE and returned as a pointer to that type, not void *. + * + * Returns: newly allocated pointer to a @VAR_OR_TYPE on success, or NULL on + * failure. + */ +#define devm_kmalloc_obj(DEV, VAR_OR_TYPE, ...) \ + __devm_alloc_objs(devm_kmalloc, DEV, default_gfp(__VA_ARGS__), \ + typeof(VAR_OR_TYPE), 1) + +/** + * devm_kmalloc_objs - Device-managed allocation of an array of a typed object + * @DEV: Device to scope the allocation lifetime to. + * @VAR_OR_TYPE: Variable or type to allocate an array of. + * @COUNT: How many elements in the array. + * @GFP: Optional GFP flags (defaults to %GFP_KERNEL). + * + * Returns: newly allocated pointer to an array of @VAR_OR_TYPE on success, or + * NULL on failure (including when the total size overflows). + */ +#define devm_kmalloc_objs(DEV, VAR_OR_TYPE, COUNT, ...) \ + __devm_alloc_objs(devm_kmalloc, DEV, default_gfp(__VA_ARGS__), \ + typeof(VAR_OR_TYPE), COUNT) + +/** + * __devm_alloc_flex - Device-managed allocation of a trailing-flex-array object + * @DEVM_ALLOC: which device-managed allocator to use. + * @DEV: device to scope the allocation lifetime to. + * @GFP: GFP flags for the allocation. + * @TYPE: type of structure to allocate space for. + * @FAM: name of the flexible array member of @TYPE. + * @COUNT: how many @FAM elements to allocate space for. + * + * Device-managed counterpart of __alloc_flex(). + * @DEVM_ALLOC takes (@DEV, size, gfp). + * + * Returns: Newly allocated pointer to @TYPE with @COUNT-many trailing @FAM + * elements, or NULL on failure. + */ +#define __devm_alloc_flex(DEVM_ALLOC, DEV, GFP, TYPE, FAM, COUNT) \ +({ \ + const size_t __count = (COUNT); \ + const size_t __obj_size = struct_size_t(TYPE, FAM, __count); \ + TYPE *__obj_ptr = DEVM_ALLOC(DEV, __obj_size, GFP); \ + if (__obj_ptr) \ + __set_flex_counter(__obj_ptr->FAM, __count); \ + __obj_ptr; \ +}) + +/** + * devm_kmalloc_flex - Device-managed allocation of a flexible-array structure + * @DEV: Device to scope the allocation lifetime to. + * @VAR_OR_TYPE: Variable or type to allocate (with its flexible array). + * @FAM: The name of the flexible array member of the structure. + * @COUNT: How many flexible array member elements to allocate. + * @GFP: Optional GFP flags (defaults to %GFP_KERNEL). + * + * Returns: newly allocated pointer to @VAR_OR_TYPE on success, or NULL on + * failure. If @FAM has been annotated with __counted_by(), its counter is set + * to @COUNT. + */ +#define devm_kmalloc_flex(DEV, VAR_OR_TYPE, FAM, COUNT, ...) \ + __devm_alloc_flex(devm_kmalloc, DEV, default_gfp(__VA_ARGS__), \ + typeof(VAR_OR_TYPE), FAM, COUNT) + +/* All devm_kzalloc aliases for devm_kmalloc_(obj|objs|flex). */ +#define devm_kzalloc_obj(DEV, P, ...) \ + __devm_alloc_objs(devm_kzalloc, DEV, default_gfp(__VA_ARGS__), typeof(P), 1) +#define devm_kzalloc_objs(DEV, P, COUNT, ...) \ + __devm_alloc_objs(devm_kzalloc, DEV, default_gfp(__VA_ARGS__), \ + typeof(P), COUNT) +#define devm_kzalloc_flex(DEV, P, FAM, COUNT, ...) \ + __devm_alloc_flex(devm_kzalloc, DEV, default_gfp(__VA_ARGS__), \ + typeof(P), FAM, COUNT) + static inline __realloc_size(3, 4) void * __must_check devm_krealloc_array(struct device *dev, void *p, size_t new_n, size_t new_size, gfp_t flags) { diff --git a/include/linux/device/driver.h b/include/linux/device/driver.h index 768a1334c0a1..29fbc01ef06f 100644 --- a/include/linux/device/driver.h +++ b/include/linux/device/driver.h @@ -297,4 +297,35 @@ static int __init __driver##_init(void) \ } \ device_initcall(__driver##_init); +/** + * subsys_driver() - Helper macro for drivers that don't do anything special + * in init/exit but have to register earlier, at subsys_initcall level, when + * built in. This eliminates a lot of boilerplate. Each driver may only use + * this macro once, and calling it replaces the init/exit boilerplate. + * + * @__driver: driver name + * @__register: register function for this driver type + * @__unregister: unregister function for this driver type + * @...: Additional arguments to be passed to __register and __unregister. + * + * This is meant to be a direct parallel of module_driver() above, but with + * the init call promoted to subsys_initcall() for built-in drivers so they + * are available earlier during boot. When built as a module subsys_initcall() + * collapses to module_init() and the normal module init/exit paths are used. + * + * Use this macro to construct bus specific macros for registering drivers, + * and do not use it on its own. + */ +#define subsys_driver(__driver, __register, __unregister, ...) \ +static int __init __driver##_init(void) \ +{ \ + return __register(&(__driver), ##__VA_ARGS__); \ +} \ +subsys_initcall(__driver##_init) \ +static void __exit __driver##_exit(void) \ +{ \ + __unregister(&(__driver), ##__VA_ARGS__); \ +} \ +module_exit(__driver##_exit) + #endif /* _DEVICE_DRIVER_H_ */ diff --git a/include/linux/dibs.h b/include/linux/dibs.h index d3e0777f25ae..0c10c224bcca 100644 --- a/include/linux/dibs.h +++ b/include/linux/dibs.h @@ -261,12 +261,12 @@ struct dibs_dev_ops { * @vlan_id: deprecated, ignored if device does not support vlan * Upon return in addition the following fields will be valid: * @dmb_tok: for usage by remote and local devices and clients - * @cpu_addr: allocated buffer + * @cpu_addr: allocated, zerorized buffer * @idx: dmb index, unique per dibs device * @dma_addr: to be used by device driver,if applicable * - * Allocate a dmb buffer and register it with this device and for this - * client. + * Allocate and zerorize a dmb buffer and register it with this device + * and for this client. * Return: zero on success */ int (*register_dmb)(struct dibs_dev *dev, struct dibs_dmb *dmb, diff --git a/include/linux/dma-buf-io.h b/include/linux/dma-buf-io.h new file mode 100644 index 000000000000..d9f27650a851 --- /dev/null +++ b/include/linux/dma-buf-io.h @@ -0,0 +1,126 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __DMA_BUF_IO_H__ +#define __DMA_BUF_IO_H__ + +#include <linux/dma-buf.h> + +struct dma_buf_io_ctx; +struct dma_buf_io_map; + +struct dma_buf_io_ops { + /* + * Create a new map for the given ctx. Called with the reservation + * lock held. + */ + struct dma_buf_io_map *(*map)(struct dma_buf_io_ctx *ctx); + + /* + * Clean up device specific parts of the @map. Called with the + * reservation lock held. + */ + void (*unmap)(struct dma_buf_io_ctx *ctx, struct dma_buf_io_map *map); + + /* + * The user tries to destroy the ctx. Release all device specific + * parts of the token. + */ + void (*release)(struct dma_buf_io_ctx *); +}; + +struct dma_buf_io_map { + /* + * Counts attached requests and other users. Device specific unmapping + * is deferred until all refs are dropped. + */ + struct percpu_ref refs; + /* + * Shift for the minimum segment size of the mapping. + */ + unsigned min_seg_shift; + + struct dma_buf_io_ctx *ctx; + struct completion drained; +}; + +struct dma_buf_io_ctx { + struct dma_buf_io_map __rcu *map; + struct dma_buf *dmabuf; + enum dma_data_direction dir; + bool is_dead; + + void *dev_priv; + const struct dma_buf_io_ops *dev_ops; +}; + +int dma_buf_io_ctx_create(struct file *file, + struct dma_buf *dmabuf, + enum dma_data_direction dir, + struct dma_buf_io_ctx **ctx); +void dma_buf_io_ctx_release(struct dma_buf_io_ctx *ctx); + +struct dma_buf_io_map *dma_buf_io_create_map(struct dma_buf_io_ctx *ctx); + +/* Internal helper, see dma_buf_io_get_map() */ +static inline struct dma_buf_io_map * +__dma_buf_io_get_map(struct dma_buf_io_ctx *ctx) +{ + struct dma_buf_io_map *map; + + guard(rcu)(); + + map = rcu_dereference(ctx->map); + if (unlikely(!map || !percpu_ref_tryget_live_rcu(&map->refs))) + return NULL; + + return map; +} + +/** + * dma_buf_io_get_map - Create a new or reference an existing map. + * @ctx: [in] dma-buf I/O context + * @nowait: [in] specifies whether it's allowed to block + * + * Create a new or acquire an existing map. On success, it returns + * a referenced map, which should eventually be put down with + * dma_buf_io_map_drop(). If @nowait is set and the function would block, + * it returns -EAGAIN, and the caller is expected to retry from a context + * where blocking is allowed. + * + * Returns a map on success, or ERR_PTR encoded errno otherwise. + */ +static inline struct dma_buf_io_map * +dma_buf_io_get_map(struct dma_buf_io_ctx *ctx, bool nowait) +{ + struct dma_buf_io_map *map; + + map = __dma_buf_io_get_map(ctx); + if (likely(map)) + return map; + + if (nowait) + return ERR_PTR(-EAGAIN); + return dma_buf_io_create_map(ctx); +} + +/** + * dma_buf_io_map_drop - Drop the map. + * @map: [in] dma-buf I/O map + * + * Put down a reference to the @map acquired via dma_buf_io_get_map(). + * Should be called as soon as I/O is completed. + */ +static inline void dma_buf_io_map_drop(struct dma_buf_io_map *map) +{ + percpu_ref_put(&map->refs); +} + +/* + * Device API + */ + +void dma_buf_io_invalidate_mappings(struct dma_buf_io_ctx *ctx); +int dma_buf_io_init_map(struct dma_buf_io_ctx *ctx, struct dma_buf_io_map *map, + struct sg_table *sgt); +void dma_buf_io_detach(struct dma_buf_io_ctx *ctx); + +#endif diff --git a/include/linux/dma-fence.h b/include/linux/dma-fence.h index ffa99b930843..dd07d128adc3 100644 --- a/include/linux/dma-fence.h +++ b/include/linux/dma-fence.h @@ -507,7 +507,7 @@ dma_fence_test_signaled_flag(struct dma_fence *fence) * Returns true if the fence was already signaled, false if not. Since this * function doesn't enable signaling, it is not guaranteed to ever return * true if dma_fence_add_callback(), dma_fence_wait() or - * dma_fence_enable_sw_signaling() haven't been called before. + * dma_fence_enable_signaling() haven't been called before. * * This function requires &dma_fence.lock to be held. * diff --git a/include/linux/dma-map-ops.h b/include/linux/dma-map-ops.h index 8fae2b7deb20..afaf2c12189b 100644 --- a/include/linux/dma-map-ops.h +++ b/include/linux/dma-map-ops.h @@ -60,7 +60,7 @@ struct dma_map_ops { int (*dma_supported)(struct device *dev, u64 mask); u64 (*get_required_mask)(struct device *dev); size_t (*max_mapping_size)(struct device *dev); - size_t (*opt_mapping_size)(void); + size_t (*max_opt_mapping_size)(void); unsigned long (*get_merge_boundary)(struct device *dev); }; @@ -143,7 +143,7 @@ static inline struct page *dma_alloc_contiguous(struct device *dev, size_t size, static inline void dma_free_contiguous(struct device *dev, struct page *page, size_t size) { - __free_pages(page, get_order(size)); + free_pages_exact(page_address(page), size); } #endif /* CONFIG_DMA_CMA*/ diff --git a/include/linux/dma-mapping.h b/include/linux/dma-mapping.h index a3e880649fa4..efde0e5a58dc 100644 --- a/include/linux/dma-mapping.h +++ b/include/linux/dma-mapping.h @@ -207,7 +207,7 @@ int dma_set_coherent_mask(struct device *dev, u64 mask); u64 dma_get_required_mask(struct device *dev); bool dma_addressing_limited(struct device *dev); size_t dma_max_mapping_size(struct device *dev); -size_t dma_opt_mapping_size(struct device *dev); +size_t dma_max_opt_mapping_size(struct device *dev); unsigned long dma_get_merge_boundary(struct device *dev); struct sg_table *dma_alloc_noncontiguous(struct device *dev, size_t size, enum dma_data_direction dir, gfp_t gfp, unsigned long attrs); @@ -326,7 +326,7 @@ static inline size_t dma_max_mapping_size(struct device *dev) { return 0; } -static inline size_t dma_opt_mapping_size(struct device *dev) +static inline size_t dma_max_opt_mapping_size(struct device *dev) { return 0; } diff --git a/include/linux/dma/edma.h b/include/linux/dma/edma.h index 3c8e2ef9dee0..eead04b805e7 100644 --- a/include/linux/dma/edma.h +++ b/include/linux/dma/edma.h @@ -101,6 +101,27 @@ enum dw_edma_ch_irq_mode { DW_EDMA_CH_IRQ_REMOTE, }; +#define DW_EDMA_CH_CONFIG_NON_LL BIT(0) +#define DW_EDMA_CH_CONFIG_IRQ_MODE BIT(1) + +/** + * struct dw_edma_chan_config - dw-edma channel configuration + * @flags: fields selected by DW_EDMA_CH_CONFIG_* + * @non_ll: use HDMA non-linked-list mode + * @irq_mode: interrupt routing mode + * + * Pass this structure through dma_slave_config.peripheral_config. Before + * synchronizing a remote-routed channel, the client must stop remote + * programming and own every channel affected by the hardware quiesce: the + * entire direction for eDMA-compatible layouts, or the individual channel for + * native HDMA. + */ +struct dw_edma_chan_config { + u32 flags; + bool non_ll; + enum dw_edma_ch_irq_mode irq_mode; +}; + /** * struct dw_edma_chip - representation of DesignWare eDMA controller hardware * @dev: struct device of the eDMA controller @@ -130,7 +151,7 @@ struct dw_edma_chip { u16 ll_wr_cnt; u16 ll_rd_cnt; - /* link list address */ + /* Linked-list regions must not cross a 4 GiB address boundary. */ struct dw_edma_region ll_region_wr[HDMA_MAX_WR_CH]; struct dw_edma_region ll_region_rd[HDMA_MAX_RD_CH]; @@ -147,6 +168,7 @@ struct dw_edma_chip { struct dw_edma *dw; bool cfg_non_ll; + u32 ch_space_sz; }; /* Export to the platform drivers */ diff --git a/include/linux/dma/imx-dma.h b/include/linux/dma/imx-dma.h index 76a8de9ae151..7c678316b757 100644 --- a/include/linux/dma/imx-dma.h +++ b/include/linux/dma/imx-dma.h @@ -57,15 +57,12 @@ struct imx_dma_data { int priority; }; -static inline int imx_dma_is_ipu(struct dma_chan *chan) -{ - return !strcmp(dev_name(chan->device->dev), "ipu-core"); -} - static inline int imx_dma_is_general_purpose(struct dma_chan *chan) { - return !strcmp(chan->device->dev->driver->name, "imx-sdma") || - !strcmp(chan->device->dev->driver->name, "imx-dma"); + struct device *dev = dmaengine_get_provider_device(chan); + + return !strcmp(dev->driver->name, "imx-sdma") || + !strcmp(dev->driver->name, "imx-dma"); } /** diff --git a/include/linux/dmaengine.h b/include/linux/dmaengine.h index f322e501ff62..805668fbe437 100644 --- a/include/linux/dmaengine.h +++ b/include/linux/dmaengine.h @@ -325,7 +325,8 @@ struct dma_router { * @lock: protect between config and prepare transfer when driver have not * implemented callback device_prep_config_sg(). * @chan_id: channel ID for sysfs - * @dev: class device for sysfs + * @chan_dev: class channel device for sysfs, some device use it for per-channel + * IOMMU mapping. * @name: backlink name for sysfs * @dbg_client_name: slave name for debugfs in format: * dev_name(requester's dev):channel name, for example: "2b00000.mcasp:tx" @@ -351,7 +352,14 @@ struct dma_chan { /* sysfs */ int chan_id; - struct dma_chan_dev *dev; + union { + struct dma_chan_dev *chan_dev; + /* + * Use chan_dev; dev will be removed once all users are + * converted. + */ + struct dma_chan_dev *dev; + }; const char *name; #ifdef CONFIG_DEBUG_FS char *dbg_client_name; @@ -369,6 +377,26 @@ struct dma_chan { void *private; }; +#define DMA_CHAN_ID_STATIC BIT(30) + +/** + * dmaengine_set_static_chan_id - request an exact DMA engine channel ID + * @chan: DMA channel + * @id: channel ID, unique within the DMA device + * + * Drivers may call this after initializing @chan and before registering its + * DMA device. The dmaengine core reserves @id from the device IDA instead of + * assigning the next available ID. + */ +static inline void dmaengine_set_static_chan_id(struct dma_chan *chan, + unsigned int id) +{ + if (WARN_ON_ONCE(id >= DMA_CHAN_ID_STATIC)) + return; + + chan->chan_id = DMA_CHAN_ID_STATIC | id; +} + /** * struct dma_chan_dev - relate sysfs device node to backing channel device * @chan: driver channel device @@ -532,7 +560,7 @@ struct dma_slave_caps { static inline const char *dma_chan_name(struct dma_chan *chan) { - return dev_name(&chan->dev->device); + return dev_name(&chan->chan_dev->device); } /** @@ -981,12 +1009,6 @@ static inline int dmaengine_slave_config(struct dma_chan *chan, return -ENOSYS; } -static inline bool is_slave_direction(enum dma_transfer_direction direction) -{ - return (direction == DMA_MEM_TO_DEV) || (direction == DMA_DEV_TO_MEM) || - (direction == DMA_DEV_TO_DEV); -} - static inline struct dma_async_tx_descriptor * dmaengine_prep_config_single(struct dma_chan *chan, dma_addr_t buf, size_t len, enum dma_transfer_direction dir, @@ -1224,6 +1246,137 @@ static inline struct dma_async_tx_descriptor *dmaengine_prep_dma_memcpy( len, flags); } +/** + * dmaengine_get_max_xor - get the maximum number of XOR sources for a channel + * @chan: DMA channel + * + * Returns the maximum number of source buffers supported by the XOR engine + * behind @chan, or 0 if the channel does not support XOR operations. + */ +static inline unsigned int dmaengine_get_max_xor(struct dma_chan *chan) +{ + if (!chan || !chan->device) + return 0; + + return chan->device->max_xor; +} + +/** + * dmaengine_prep_dma_xor - prepare a DMA XOR operation + * @chan: the channel to use for this operation + * @dst: destination buffer address + * @src: array of source buffer addresses + * @src_cnt: number of source buffers + * @len: length in bytes of each source and destination buffer + * @flags: DMA engine flags (e.g. DMA_PREP_INTERRUPT) + * + * Prepare an XOR parity generation transaction. The engine computes: + * dst = src[0] XOR src[1] XOR ... XOR src[src_cnt - 1] + * + * Returns a descriptor on success, or NULL if the channel does not support + * this operation or the request could not be queued. + */ +static inline struct dma_async_tx_descriptor * +dmaengine_prep_dma_xor(struct dma_chan *chan, dma_addr_t dst, dma_addr_t *src, + unsigned int src_cnt, size_t len, unsigned long flags) +{ + if (!chan || !chan->device || !chan->device->device_prep_dma_xor) + return NULL; + + return chan->device->device_prep_dma_xor(chan, dst, src, + src_cnt, len, flags); +} + +/** + * dmaengine_prep_dma_xor_val - prepare a DMA XOR zero-sum validation operation + * @chan: the channel to use for this operation + * @src: array of source buffer addresses + * @src_cnt: number of source buffers + * @len: length in bytes of each source buffer + * @result: output flag set to SUM_CHECK_P_RESULT if the XOR of all sources + * is non-zero (i.e. parity error), cleared otherwise + * @flags: DMA engine flags (e.g. DMA_PREP_INTERRUPT) + * + * Prepare an XOR zero-sum validation transaction. The engine XORs all + * source buffers and checks whether the result is zero. The outcome is + * written to @result on completion. + * + * Returns a descriptor on success, or NULL if the channel does not support + * this operation or the request could not be queued. + */ +static inline struct dma_async_tx_descriptor * +dmaengine_prep_dma_xor_val(struct dma_chan *chan, dma_addr_t *src, + unsigned int src_cnt, size_t len, + enum sum_check_flags *result, unsigned long flags) +{ + if (!chan || !chan->device || !chan->device->device_prep_dma_xor_val) + return NULL; + + return chan->device->device_prep_dma_xor_val(chan, src, src_cnt, + len, result, flags); +} + +/** + * dmaengine_prep_dma_pq - prepare a DMA PQ (RAID-6 P+Q) operation + * @chan: the channel to use for this operation + * @dst: array of two destination addresses: dst[0] for P, dst[1] for Q + * @src: array of source buffer addresses + * @src_cnt: number of source buffers + * @scf: array of scaling coefficients, one per source buffer + * @len: length in bytes of each source and destination buffer + * @flags: DMA engine flags (e.g. DMA_PREP_INTERRUPT) + * + * Prepare a P+Q parity generation transaction. The engine computes: + * P = XOR of all source buffers + * Q = Galois field sum of (scf[i] * src[i]) over all sources + * + * Returns a descriptor on success, or NULL if the channel does not support + * this operation or the request could not be queued. + */ +static inline struct dma_async_tx_descriptor * +dmaengine_prep_dma_pq(struct dma_chan *chan, dma_addr_t *dst, dma_addr_t *src, + unsigned int src_cnt, const unsigned char *scf, + size_t len, unsigned long flags) +{ + if (!chan || !chan->device || !chan->device->device_prep_dma_pq) + return NULL; + + return chan->device->device_prep_dma_pq(chan, dst, src, + src_cnt, scf, len, flags); +} + +/** + * dmaengine_prep_dma_pq_val - prepare a DMA PQ validation operation + * @chan: the channel to use for this operation + * @pq: array of two addresses holding existing P and Q parity buffers + * @src: array of source buffer addresses + * @src_cnt: number of source buffers + * @scf: array of scaling coefficients, one per source buffer + * @len: length in bytes of each buffer + * @pqres: output flags indicating P and/or Q check results (SUM_CHECK_P_VALID, + * SUM_CHECK_Q_VALID) + * @flags: DMA engine flags (e.g. DMA_PREP_INTERRUPT) + * + * Prepare a PQ validation transaction. The engine recomputes P and Q from the + * source buffers and compares them against the existing parity stored at @pq. + * The result of each comparison is reported through @pqres. + * + * Returns a descriptor on success, or NULL if the channel does not support + * this operation or the request could not be queued. + */ +static inline struct dma_async_tx_descriptor * +dmaengine_prep_dma_pq_val(struct dma_chan *chan, dma_addr_t *pq, dma_addr_t *src, + unsigned int src_cnt, const unsigned char *scf, + size_t len, enum sum_check_flags *pqres, unsigned long flags) +{ + if (!chan || !chan->device || !chan->device->device_prep_dma_pq_val) + return NULL; + + return chan->device->device_prep_dma_pq_val(chan, pq, src, + src_cnt, scf, len, + pqres, flags); +} + static inline bool dmaengine_is_metadata_mode_supported(struct dma_chan *chan, enum dma_desc_metadata_mode mode) { @@ -1413,6 +1566,130 @@ static inline bool is_dma_fill_aligned(struct dma_device *dev, size_t off1, return dmaengine_check_align(dev->fill_align, off1, off2, len); } +/** + * dmaengine_get_copy_align - get copy alignment requirement of a DMA channel + * @chan: DMA channel + * + * Return the copy alignment requirement of @chan as a power-of-2 exponent. + */ +static inline enum dmaengine_alignment +dmaengine_get_copy_align(struct dma_chan *chan) +{ + return chan->device->copy_align; +} + +/** + * dmaengine_get_xor_align - get xor alignment requirement of a DMA channel + * @chan: DMA channel + * + * Return the xor alignment requirement of @chan as a power-of-2 exponent. + */ +static inline enum dmaengine_alignment +dmaengine_get_xor_align(struct dma_chan *chan) +{ + return chan->device->xor_align; +} + +/** + * dmaengine_get_pq_align - get pq alignment requirement of a DMA channel + * @chan: DMA channel + * + * Return the pq alignment requirement of @chan as a power-of-2 exponent. + */ +static inline enum dmaengine_alignment +dmaengine_get_pq_align(struct dma_chan *chan) +{ + return chan->device->pq_align; +} + +/** + * dmaengine_get_fill_align - get fill alignment requirement of a DMA channel + * @chan: DMA channel + * + * Return the fill alignment requirement of @chan as a power-of-2 exponent. + */ +static inline enum dmaengine_alignment +dmaengine_get_fill_align(struct dma_chan *chan) +{ + return chan->device->fill_align; +} + +/** + * dmaengine_get_cap_mask - get capability mask of a DMA channel + * @chan: DMA channel + * + * Return a pointer to the capability mask of the DMA device backing @chan. + */ +static inline const dma_cap_mask_t * +dmaengine_get_cap_mask(struct dma_chan *chan) +{ + return &chan->device->cap_mask; +} + +/** + * dmaengine_is_copy_aligned - test copy alignment + * @chan: DMA channel + * @off1: first buffer offset + * @off2: second buffer offset + * @len: transfer length + * + * Return true if off1, off2 and len satisfy the copy alignment requirement of + * the DMA channel, false otherwise. + */ +static inline bool dmaengine_is_copy_aligned(struct dma_chan *chan, size_t off1, + size_t off2, size_t len) +{ + return dmaengine_check_align(chan->device->copy_align, off1, off2, len); +} + +/** + * dmaengine_is_xor_aligned - test xor alignment + * @chan: DMA channel + * @off1: first buffer offset + * @off2: second buffer offset + * @len: transfer length + * + * Return true if off1, off2 and len satisfy the xor alignment requirement of + * the DMA channel, false otherwise. + */ +static inline bool dmaengine_is_xor_aligned(struct dma_chan *chan, size_t off1, + size_t off2, size_t len) +{ + return dmaengine_check_align(chan->device->xor_align, off1, off2, len); +} + +/** + * dmaengine_is_pq_aligned - test pq alignment + * @chan: DMA channel + * @off1: first buffer offset + * @off2: second buffer offset + * @len: transfer length + * + * Return true if off1, off2 and len satisfy the pq alignment requirement of + * the DMA channel, false otherwise. + */ +static inline bool dmaengine_is_pq_aligned(struct dma_chan *chan, size_t off1, + size_t off2, size_t len) +{ + return dmaengine_check_align(chan->device->pq_align, off1, off2, len); +} + +/** + * dmaengine_is_fill_aligned - test fill alignment + * @chan: DMA channel + * @off1: first buffer offset + * @off2: second buffer offset + * @len: transfer length + * + * Return true if off1, off2 and len satisfy the fill alignment requirement of + * the DMA channel, false otherwise. + */ +static inline bool dmaengine_is_fill_aligned(struct dma_chan *chan, size_t off1, + size_t off2, size_t len) +{ + return dmaengine_check_align(chan->device->fill_align, off1, off2, len); +} + static inline void dma_set_maxpq(struct dma_device *dma, int maxpq, int has_pq_continue) { @@ -1574,10 +1851,41 @@ __dma_has_cap(enum dma_transaction_type tx_type, dma_cap_mask_t *srcp) return test_bit(tx_type, srcp->bits); } +/** + * dmaengine_has_cap - test whether a DMA channel supports a transaction type + * @chan: DMA channel + * @tx_type: transaction type to test + * + * Return true if @chan supports @tx_type, false otherwise. + */ +static inline bool +dmaengine_has_cap(struct dma_chan *chan, enum dma_transaction_type tx_type) +{ + return dma_has_cap(tx_type, chan->device->cap_mask); +} + #define for_each_dma_cap_mask(cap, mask) \ for_each_set_bit(cap, mask.bits, DMA_TX_TYPE_END) /** + * dmaengine_prep_dma_interrupt() - Prepare a DMA interrupt descriptor. + * @chan: The channel to be used for this descriptor + * @flags: DMA engine flags + * + * Returns a descriptor for an interrupt transaction, or NULL if the + * channel does not support DMA_INTERRUPT. + */ +static inline struct dma_async_tx_descriptor * +dmaengine_prep_dma_interrupt(struct dma_chan *chan, unsigned long flags) +{ + if (!chan || !chan->device || !chan->device->device_prep_dma_interrupt || + !dma_has_cap(DMA_INTERRUPT, chan->device->cap_mask)) + return NULL; + + return chan->device->device_prep_dma_interrupt(chan, flags); +} + +/** * dma_async_issue_pending - flush pending transactions to HW * @chan: target DMA channel * @@ -1746,16 +2054,33 @@ static inline int dmaengine_desc_free(struct dma_async_tx_descriptor *desc) return desc->desc_free(desc); } +/** + * dmaengine_desc_set_callback - set the callback, callback_result and + * parameter on a descriptor + * @tx: the descriptor to set the callback on + * @cb: the plain completion callback, or NULL + * @cb_result: the result-bearing completion callback, or NULL + * @cb_param: the parameter to pass to the callback + * + * Sets the completion callbacks and their shared parameter on a DMA + * transaction descriptor. Use this helper instead of assigning the fields + * directly to ensure all three related fields are updated consistently. + * Only one of @cb and @cb_result should be non-NULL; if both are provided + * @cb_result takes precedence in drivers that support it. + */ +static inline void +dmaengine_desc_set_callback(struct dma_async_tx_descriptor *tx, + dma_async_tx_callback cb, + dma_async_tx_callback_result cb_result, + void *cb_param) +{ + tx->callback = cb; + tx->callback_result = cb_result; + tx->callback_param = cb_param; +} + /* --- DMA device --- */ -int dma_async_device_register(struct dma_device *device); -int dmaenginem_async_device_register(struct dma_device *device); -void dma_async_device_unregister(struct dma_device *device); -int dma_async_device_channel_register(struct dma_device *device, - struct dma_chan *chan, - const char *name); -void dma_async_device_channel_unregister(struct dma_device *device, - struct dma_chan *chan); void dma_run_dependencies(struct dma_async_tx_descriptor *tx); #define dma_request_channel(mask, x, y) \ __dma_request_channel(&(mask), x, y, NULL) @@ -1777,29 +2102,36 @@ static inline struct dma_chan return dma_request_channel(mask, fn, fn_param); } -static inline char * -dmaengine_get_direction_text(enum dma_transfer_direction dir) -{ - switch (dir) { - case DMA_DEV_TO_MEM: - return "DEV_TO_MEM"; - case DMA_MEM_TO_DEV: - return "MEM_TO_DEV"; - case DMA_MEM_TO_MEM: - return "MEM_TO_MEM"; - case DMA_DEV_TO_DEV: - return "DEV_TO_DEV"; - default: - return "invalid"; - } +static inline struct device *dmaengine_chan_dev(struct dma_chan *chan) +{ + return &chan->chan_dev->device; +} + +/* + * dmaengine_get_provider_device() - Get DMA engine provider device. Typical + * use is to find a DMA channel by filter. + * @chan: DMA channel + * + * Return: Pointer to the DMA engine provider device. + */ +static inline struct device *dmaengine_get_provider_device(struct dma_chan *chan) +{ + return chan->device->dev; } +/* + * dmaengine_get_dma_device() - Get DMA device used for DMA mapping. Some + * DMA engines provide per-channel iommu mapping. + * @chan: DMA channel + * + * Return: Pointer to the device used for DMA mapping. + */ static inline struct device *dmaengine_get_dma_device(struct dma_chan *chan) { - if (chan->dev->chan_dma_dev) - return &chan->dev->device; + if (chan->chan_dev->chan_dma_dev) + return dmaengine_chan_dev(chan); - return chan->device->dev; + return dmaengine_get_provider_device(chan); } #endif /* DMAENGINE_H */ diff --git a/include/linux/efi.h b/include/linux/efi.h index aa15ff88539b..cc1df1577d5f 100644 --- a/include/linux/efi.h +++ b/include/linux/efi.h @@ -631,6 +631,8 @@ typedef struct { extern unsigned long __ro_after_init efi_rng_seed; /* RNG Seed table */ +enum efi_rts_ids; + /* * All runtime access to EFI goes through this structure: */ @@ -650,6 +652,7 @@ extern struct efi { unsigned long mokvar_table; /* MOK variable config table */ unsigned long coco_secret; /* Confidential computing secret table */ unsigned long unaccepted; /* Unaccepted memory table */ + unsigned long mem_reserve; /* Persistent memory reservations */ efi_get_time_t *get_time; efi_set_time_t *set_time; @@ -666,6 +669,10 @@ extern struct efi { efi_get_next_high_mono_count_t *get_next_high_mono_count; efi_reset_system_t *reset_system; +#if IS_ENABLED(CONFIG_LKDTM) + efi_status_t (*lkdtm_rt_test)(enum efi_rts_ids id); +#endif + struct efi_memory_map memmap; unsigned long flags; } efi; @@ -957,6 +964,17 @@ extern int efi_status_to_err(efi_status_t status); #define EFI_DEV_END_INSTANCE 0x01 #define EFI_DEV_END_ENTIRE 0xFF +enum efi_hd_partition_format { + EFI_HD_PARTITION_FORMAT_MBR = 1, + EFI_HD_PARTITION_FORMAT_GPT, +}; + +enum efi_hd_signature_type { + EFI_HD_SIGNATURE_TYPE_NONE, + EFI_HD_SIGNATURE_TYPE_MBR, + EFI_HD_SIGNATURE_TYPE_GUID, +}; + struct efi_generic_dev_path { u8 type; u8 sub_type; @@ -988,6 +1006,16 @@ struct efi_rel_offset_dev_path { u64 ending_offset; } __packed; +struct efi_hd_dev_path { + struct efi_generic_dev_path header; + u32 partition_number; + u64 partition_start; + u64 partition_size; + efi_guid_t signature; + u8 partition_format; + u8 signature_type; +} __packed; + struct efi_mem_mapped_dev_path { struct efi_generic_dev_path header; u32 memory_type; @@ -1007,6 +1035,7 @@ struct efi_dev_path { struct efi_pci_dev_path pci; struct efi_vendor_dev_path vendor; struct efi_rel_offset_dev_path rel_offset; + struct efi_hd_dev_path hd; }; } __packed; @@ -1109,6 +1138,7 @@ extern void efi_call_virt_check_flags(unsigned long flags, const void *caller); extern unsigned long efi_call_virt_save_flags(void); void efi_runtime_assert_lock_held(void); +efi_status_t efi_runtime_set_enable_flag(bool enable); enum efi_secureboot_mode { efi_secureboot_mode_unset, @@ -1230,6 +1260,10 @@ enum efi_rts_ids { EFI_UPDATE_CAPSULE, EFI_QUERY_CAPSULE_CAPS, EFI_ACPI_PRM_HANDLER, +#if IS_ENABLED(CONFIG_LKDTM) + EFI_LKDTM_RUNTIME_RO_TEST, + EFI_LKDTM_RUNTIME_NX_TEST, +#endif }; union efi_rts_args; @@ -1330,7 +1364,8 @@ bool efi_config_table_is_usable(const efi_guid_t *guid, unsigned long table) return xen_efi_config_table_is_usable(guid, table); } -umode_t efi_attr_is_visible(struct kobject *kobj, struct attribute *attr, int n); +umode_t efi_attr_is_visible(struct kobject *kobj, const struct attribute *attr, + int n); int ovmf_log_probe(unsigned long ovmf_debug_log_table); diff --git a/include/linux/etherdevice.h b/include/linux/etherdevice.h index df8f88f63a70..1f833ebd71ed 100644 --- a/include/linux/etherdevice.h +++ b/include/linux/etherdevice.h @@ -38,6 +38,34 @@ int fwnode_get_mac_address(struct fwnode_handle *fwnode, char *addr); u32 eth_get_headlen(const struct net_device *dev, const void *data, u32 len); __be16 eth_type_trans(struct sk_buff *skb, struct net_device *dev); + +/** + * eth_type_trans_careful - eth_type_trans() for skbs coming from the stack + * @skb: buffer, possibly with less than ETH_HLEN bytes in its linear area + * @dev: receiving network device + * + * eth_type_trans() expects ETH_HLEN bytes in skb->head, which NIC drivers + * provide. Devices looping skbs from their transmit path back to the + * receive path (loopback, OVS internal ports) get skbs built by the stack, + * whose link-layer header might have been popped by tc or OVS actions. + * + * Sets skb->protocol on success. + * + * Return: SKB_NOT_DROPPED_YET on success, or a drop reason. In the latter + * case, the caller must free @skb. + */ +static inline enum skb_drop_reason +eth_type_trans_careful(struct sk_buff *skb, struct net_device *dev) +{ + enum skb_drop_reason reason = pskb_may_pull_reason(skb, ETH_HLEN); + + if (unlikely(reason)) + return reason; + + skb->protocol = eth_type_trans(skb, dev); + return SKB_NOT_DROPPED_YET; +} + extern const struct header_ops eth_header_ops; int eth_header(struct sk_buff *skb, struct net_device *dev, unsigned short type, diff --git a/include/linux/ethtool.h b/include/linux/ethtool.h index c4c9ce038611..c3a41fbd5426 100644 --- a/include/linux/ethtool.h +++ b/include/linux/ethtool.h @@ -1025,7 +1025,9 @@ struct kernel_ethtool_ts_info { * types should be set in @supported_coalesce_params. * Returns a negative error code or zero. * @get_ringparam: Report ring sizes - * @set_ringparam: Set ring sizes. Returns a negative error code or zero. + * @set_ringparam: Set ring sizes. The &struct ethtool_ringparam argument is + * also an output; drivers which normalize requested sizes must update it + * with the applied sizes. Returns a negative error code or zero. * @get_pause_stats: Report pause frame statistics. Drivers must not zero * statistics which they don't report. The stats structure is initialized * to ETHTOOL_STAT_NOT_SET indicating driver does not report statistics. diff --git a/include/linux/f2fs_fs.h b/include/linux/f2fs_fs.h index bb2b6cd5d507..3081702b1ddb 100644 --- a/include/linux/f2fs_fs.h +++ b/include/linux/f2fs_fs.h @@ -14,9 +14,9 @@ #define F2FS_SUPER_OFFSET 1024 /* byte-size offset */ #define F2FS_MIN_LOG_SECTOR_SIZE 9 /* 9 bits for 512 bytes */ #define F2FS_MAX_LOG_SECTOR_SIZE PAGE_SHIFT /* Max is Block Size */ -#define F2FS_LOG_SECTORS_PER_BLOCK (PAGE_SHIFT - 9) /* log number for sector/blk */ -#define F2FS_BLKSIZE PAGE_SIZE /* support only block == page */ -#define F2FS_BLKSIZE_BITS PAGE_SHIFT /* bits for F2FS_BLKSIZE */ +#define F2FS_MIN_LOG_BLOCKSIZE 12 +#define F2FS_MIN_BLKSIZE 4096UL +#define F2FS_MAX_BLKSIZE PAGE_SIZE #define F2FS_MAX_EXTENSION 64 /* # of extension entries */ #define F2FS_EXTENSION_LEN 8 /* max size of extension */ @@ -24,19 +24,25 @@ #define NEW_ADDR ((block_t)-1) /* used as block_t addresses */ #define COMPRESS_ADDR ((block_t)-2) /* used as compressed data flag */ -#define F2FS_BLKSIZE_MASK (F2FS_BLKSIZE - 1) -#define F2FS_BYTES_TO_BLK(bytes) ((unsigned long long)(bytes) >> F2FS_BLKSIZE_BITS) -#define F2FS_BLK_TO_BYTES(blk) ((unsigned long long)(blk) << F2FS_BLKSIZE_BITS) -#define F2FS_BLK_END_BYTES(blk) (F2FS_BLK_TO_BYTES(blk + 1) - 1) -#define F2FS_BLK_ALIGN(x) (F2FS_BYTES_TO_BLK((x) + F2FS_BLKSIZE - 1)) +#define F2FS_BLKSIZE(sbi) ((sbi)->blocksize) +#define F2FS_BLKSIZE_BITS(sbi) ((sbi)->log_blocksize) +#define F2FS_BLKSIZE_MASK(sbi) (F2FS_BLKSIZE(sbi) - 1) +#define F2FS_LOG_SECTORS_PER_BLOCK(sbi) (F2FS_BLKSIZE_BITS(sbi) - 9) +#define F2FS_BLKS_PER_PAGE(sbi) (PAGE_SIZE / F2FS_BLKSIZE(sbi)) +#define F2FS_BYTES_TO_BLK(sbi, bytes) \ + ((unsigned long long)(bytes) >> F2FS_BLKSIZE_BITS(sbi)) +#define F2FS_BLK_TO_BYTES(sbi, blk) \ + ((unsigned long long)(blk) << F2FS_BLKSIZE_BITS(sbi)) +#define F2FS_BLK_END_BYTES(sbi, blk) \ + (F2FS_BLK_TO_BYTES(sbi, (blk) + 1) - 1) +#define F2FS_BLK_ALIGN(sbi, bytes) \ + F2FS_BYTES_TO_BLK(sbi, (unsigned long long)(bytes) + \ + F2FS_BLKSIZE(sbi) - 1) /* 0, 1(node nid), 2(meta nid) are reserved node id */ #define F2FS_RESERVED_NODE_NUM 3 #define F2FS_ROOT_INO(sbi) ((sbi)->root_ino_num) -#define F2FS_NODE_INO(sbi) ((sbi)->node_ino_num) -#define F2FS_META_INO(sbi) ((sbi)->meta_ino_num) -#define F2FS_COMPRESS_INO(sbi) (NM_I(sbi)->max_nid) #define F2FS_MAX_QUOTAS 3 @@ -214,20 +220,27 @@ struct f2fs_checkpoint { unsigned char sit_nat_version_bitmap[]; } __packed; -#define CP_CHKSUM_OFFSET (F2FS_BLKSIZE - sizeof(__le32)) /* default chksum offset in checkpoint */ #define CP_MIN_CHKSUM_OFFSET \ (offsetof(struct f2fs_checkpoint, sit_nat_version_bitmap)) /* * For orphan inode management + * + * The number of inode entries in an orphan block depends on the filesystem + * block size. Its exact on-disk layout is: + * + * 0 blocksize - 16 blocksize + * +--------------------------+--------------------------+ + * | ino[0] ... ino[n - 1] | struct f2fs_orphan_footer | + * +--------------------------+--------------------------+ + * + * n = (blocksize - sizeof(struct f2fs_orphan_footer)) / sizeof(__le32) */ -#define F2FS_ORPHANS_PER_BLOCK ((F2FS_BLKSIZE - 4 * sizeof(__le32)) / sizeof(__le32)) - -#define GET_ORPHAN_BLOCKS(n) (((n) + F2FS_ORPHANS_PER_BLOCK - 1) / \ - F2FS_ORPHANS_PER_BLOCK) - struct f2fs_orphan_block { - __le32 ino[F2FS_ORPHANS_PER_BLOCK]; /* inode numbers */ + DECLARE_FLEX_ARRAY(__le32, ino); +} __packed; + +struct f2fs_orphan_footer { __le32 reserved; /* reserved */ __le16 blk_addr; /* block index in current CP */ __le16 blk_count; /* Number of orphan inode blocks in CP */ @@ -260,26 +273,14 @@ struct node_footer { } __packed; /* Address Pointers in an Inode */ -#define DEF_ADDRS_PER_INODE ((F2FS_BLKSIZE - OFFSET_OF_END_OF_I_EXT \ - - SIZE_OF_I_NID \ - - sizeof(struct node_footer)) / sizeof(__le32)) -#define CUR_ADDRS_PER_INODE(inode) (DEF_ADDRS_PER_INODE - \ - get_extra_isize(inode)) +#define F2FS_DEF_ADDRS_PER_INODE(blocksize) \ + (((blocksize) - OFFSET_OF_END_OF_I_EXT - SIZE_OF_I_NID - \ + sizeof(struct node_footer)) / sizeof(__le32)) #define DEF_NIDS_PER_INODE 5 /* Node IDs in an Inode */ #define ADDRS_PER_INODE(inode) addrs_per_page(inode, true) /* Address Pointers in a Direct Block */ -#define DEF_ADDRS_PER_BLOCK ((F2FS_BLKSIZE - sizeof(struct node_footer)) / sizeof(__le32)) #define ADDRS_PER_BLOCK(inode) addrs_per_page(inode, false) -/* Node IDs in an Indirect Block */ -#define NIDS_PER_BLOCK ((F2FS_BLKSIZE - sizeof(struct node_footer)) / sizeof(__le32)) - -#define ADDRS_PER_PAGE(folio, inode) (addrs_per_page(inode, IS_INODE(folio))) - -#define NODE_DIR1_BLOCK (DEF_ADDRS_PER_INODE + 1) -#define NODE_DIR2_BLOCK (DEF_ADDRS_PER_INODE + 2) -#define NODE_IND1_BLOCK (DEF_ADDRS_PER_INODE + 3) -#define NODE_IND2_BLOCK (DEF_ADDRS_PER_INODE + 4) -#define NODE_DIND_BLOCK (DEF_ADDRS_PER_INODE + 5) +#define ADDRS_PER_PAGE(folio, inode) (addrs_per_page(inode, IS_INODE(F2FS_I_SB(inode), folio))) #define F2FS_INLINE_XATTR 0x01 /* file inline xattr flag */ #define F2FS_INLINE_DATA 0x02 /* file inline data flag */ @@ -339,18 +340,26 @@ struct f2fs_inode { */ __le32 i_extra_end[0]; /* for attribute size calculation */ } __packed; - __le32 i_addr[DEF_ADDRS_PER_INODE]; /* Pointers to data blocks */ + DECLARE_FLEX_ARRAY(__le32, i_addr); /* data block pointers */ }; - __le32 i_nid[DEF_NIDS_PER_INODE]; /* direct(2), indirect(2), - double_indirect(1) node id */ + /* + * __le32 i_nid[DEF_NIDS_PER_INODE]; + * direct(2), indirect(2), double_indirect(1) node IDs + * + * It is stored immediately before the node footer at the end of the + * filesystem block. Its offset depends on the filesystem block size, so + * locate it dynamically with F2FS_INODE_NIDS(). + */ } __packed; struct direct_node { - __le32 addr[DEF_ADDRS_PER_BLOCK]; /* array of data block address */ + /* The address count depends on the filesystem block size. */ + DECLARE_FLEX_ARRAY(__le32, addr); /* array of data block address */ } __packed; struct indirect_node { - __le32 nid[NIDS_PER_BLOCK]; /* array of data block address */ + /* The node ID count depends on the filesystem block size. */ + DECLARE_FLEX_ARRAY(__le32, nid); /* array of data block address */ } __packed; enum { @@ -369,14 +378,18 @@ struct f2fs_node { struct direct_node dn; struct indirect_node in; }; - struct node_footer footer; + /* + * struct node_footer footer; + * + * It is stored at the end of the filesystem block, after the inode or + * direct/indirect node data. Its offset depends on the filesystem block + * size, so locate it dynamically with F2FS_NODE_FOOTER(). + */ } __packed; /* * For NAT entries */ -#define NAT_ENTRY_PER_BLOCK (F2FS_BLKSIZE / sizeof(struct f2fs_nat_entry)) - struct f2fs_nat_entry { __u8 version; /* latest version of cached nat entry */ __le32 ino; /* inode number */ @@ -384,7 +397,8 @@ struct f2fs_nat_entry { } __packed; struct f2fs_nat_block { - struct f2fs_nat_entry entries[NAT_ENTRY_PER_BLOCK]; + /* The entry count depends on the filesystem block size. */ + DECLARE_FLEX_ARRAY(struct f2fs_nat_entry, entries); } __packed; /* @@ -396,8 +410,6 @@ struct f2fs_nat_block { * Not allow to change this. */ #define SIT_VBLOCK_MAP_SIZE 64 -#define SIT_ENTRY_PER_BLOCK (F2FS_BLKSIZE / sizeof(struct f2fs_sit_entry)) - /* * F2FS uses 4 bytes to represent block address. As a result, supported size of * disk is 16 TB for a 4K page size and 64 TB for a 16K page size and it equals @@ -424,8 +436,13 @@ struct f2fs_sit_entry { __le64 mtime; /* segment age for cleaning */ } __packed; +/* + * The on-disk SIT block is a filesystem-block-sized array of SIT entries. + * Its entry count depends on the filesystem block size, so it must be + * calculated by the caller rather than implied by this C structure. + */ struct f2fs_sit_block { - struct f2fs_sit_entry entries[SIT_ENTRY_PER_BLOCK]; + DECLARE_FLEX_ARRAY(struct f2fs_sit_entry, entries); } __packed; /* @@ -595,15 +612,7 @@ typedef __le32 f2fs_hash_t; * dentry, when converting inline dentry we should handle this carefully. */ -/* the number of dentry in a block */ -#define NR_DENTRY_IN_BLOCK ((BITS_PER_BYTE * F2FS_BLKSIZE) / \ - ((SIZE_OF_DIR_ENTRY + F2FS_SLOT_LEN) * BITS_PER_BYTE + 1)) #define SIZE_OF_DIR_ENTRY 11 /* by byte */ -#define SIZE_OF_DENTRY_BITMAP ((NR_DENTRY_IN_BLOCK + BITS_PER_BYTE - 1) / \ - BITS_PER_BYTE) -#define SIZE_OF_RESERVED (F2FS_BLKSIZE - ((SIZE_OF_DIR_ENTRY + \ - F2FS_SLOT_LEN) * \ - NR_DENTRY_IN_BLOCK + SIZE_OF_DENTRY_BITMAP)) #define MIN_INLINE_DENTRY_SIZE 40 /* just include '.' and '..' entries */ /* One directory entry slot representing F2FS_SLOT_LEN-sized file name */ @@ -614,14 +623,21 @@ struct f2fs_dir_entry { __u8 file_type; /* file type */ } __packed; -/* Block-sized directory entry block */ -struct f2fs_dentry_block { - /* validity bitmap for directory entries in each block */ - __u8 dentry_bitmap[SIZE_OF_DENTRY_BITMAP]; - __u8 reserved[SIZE_OF_RESERVED]; - struct f2fs_dir_entry dentry[NR_DENTRY_IN_BLOCK]; - __u8 filename[NR_DENTRY_IN_BLOCK][F2FS_SLOT_LEN]; -} __packed; +/* + * A dentry block is laid out as follows, where the number of entries and all + * offsets are determined by the filesystem block size at runtime: + * + * 0 blocksize + * +--------+----------+-------------------+-----------------------+ + * | bitmap | reserved | dir_entry[entries]| filename[entries][8] | + * +--------+----------+-------------------+-----------------------+ + * + * entries = (BITS_PER_BYTE * blocksize) / + * ((SIZE_OF_DIR_ENTRY + F2FS_SLOT_LEN) * BITS_PER_BYTE + 1) + * bitmap_size = DIV_ROUND_UP(entries, BITS_PER_BYTE) + * reserved_size = blocksize - bitmap_size - + * (SIZE_OF_DIR_ENTRY + F2FS_SLOT_LEN) * entries + */ #define F2FS_DEF_PROJID 0 /* default project ID */ diff --git a/include/linux/fault-inject.h b/include/linux/fault-inject.h index 58fd14c82270..5c74748a53f3 100644 --- a/include/linux/fault-inject.h +++ b/include/linux/fault-inject.h @@ -19,6 +19,13 @@ enum fault_flags { #include <linux/ratelimit.h> /* + * Length of the debugfs directory name embedded in struct fault_attr. + * Chosen to accommodate every in-tree caller of fault_create_debugfs_attr() + * (the longest is "fail_dma_array_full", 19 chars) with generous headroom. + */ +#define FAULT_ATTR_DNAME_LEN 64 + +/* * For explanation of the elements of this struct, see * Documentation/fault-injection/fault-injection.rst */ @@ -37,7 +44,7 @@ struct fault_attr { unsigned long count; struct ratelimit_state ratelimit_state; - struct dentry *dname; + char dname[FAULT_ATTR_DNAME_LEN]; }; #define FAULT_ATTR_INITIALIZER { \ @@ -47,7 +54,6 @@ struct fault_attr { .stacktrace_depth = 32, \ .ratelimit_state = RATELIMIT_STATE_INIT_DISABLED, \ .verbose = 2, \ - .dname = NULL, \ } #define DECLARE_FAULT_ATTR(name) struct fault_attr name = FAULT_ATTR_INITIALIZER diff --git a/include/linux/fdtable.h b/include/linux/fdtable.h index c45306a9f007..ba2536c3b911 100644 --- a/include/linux/fdtable.h +++ b/include/linux/fdtable.h @@ -25,7 +25,7 @@ struct fdtable { unsigned int max_fds; - struct file __rcu **fd; /* current fd array */ + struct file __rcu **fd __counted_by_ptr(max_fds); /* current fd array */ unsigned long *close_on_exec; unsigned long *open_fds; unsigned long *full_fds_bits; @@ -100,12 +100,22 @@ static inline bool close_on_exec(unsigned int fd, const struct files_struct *fil struct task_struct; void put_files_struct(struct files_struct *fs); -int unshare_files(void); +void switch_files_struct(struct task_struct *tsk, struct files_struct *files); +int unshare_fd(unsigned long unshare_flags, struct files_struct **new_fdp); +enum fd_range_flags { + /* Leave behind all descriptors outside of the specified range. */ + FD_RANGE_EXCEPT = (1U << 0), + + /* Only select descriptors that have close-on-exec set. */ + FD_RANGE_CLOEXEC_ONLY = (1U << 1), +}; + struct fd_range { unsigned int from, to; + enum fd_range_flags flags; }; struct files_struct *dup_fd(struct files_struct *, struct fd_range *) __latent_entropy; -void do_close_on_exec(struct files_struct *); +void close_cloexec_files(struct files_struct *); int iterate_fd(struct files_struct *, unsigned, int (*)(const void *, struct file *, unsigned), const void *); diff --git a/include/linux/file.h b/include/linux/file.h index 27484b444d31..41c3c0be1064 100644 --- a/include/linux/file.h +++ b/include/linux/file.h @@ -12,6 +12,7 @@ #include <linux/errno.h> #include <linux/cleanup.h> #include <linux/err.h> +#include <linux/vfsdebug.h> struct file; @@ -129,117 +130,84 @@ extern unsigned int sysctl_nr_open_min, sysctl_nr_open_max; /* * fd_prepare: Combined fd + file allocation cleanup class. - * @err: Error code to indicate if allocation succeeded. - * @__fd: Allocated fd (may not be accessed directly) - * @__file: Allocated struct file pointer (may not be accessed directly) + * @fd: Allocated fd + * @file: Allocated struct file pointer * * Allocates an fd and a file together. On error paths, automatically cleans * up whichever resource was successfully allocated. Allows flexible file * allocation with different functions per usage. * - * Do not use directly. + * Do not declare directly, use FD_PREPARE(). */ struct fd_prepare { - s32 err; - s32 __fd; /* do not access directly */ - struct file *__file; /* do not access directly */ + int fd; + struct file *file; }; -/* Typedef for fd_prepare cleanup guards. */ -typedef struct fd_prepare class_fd_prepare_t; - -/* - * Accessors for fd_prepare class members. - * _Generic() is used for zero-cost type safety. - */ -#define fd_prepare_fd(_fdf) \ - (_Generic((_fdf), struct fd_prepare: (_fdf).__fd)) - -#define fd_prepare_file(_fdf) \ - (_Generic((_fdf), struct fd_prepare: (_fdf).__file)) - /* Do not use directly. */ -static inline void class_fd_prepare_destructor(const struct fd_prepare *fdf) +static __always_inline void __fd_prepare_cleanup(const struct fd_prepare *fdf) { - if (unlikely(fdf->__fd >= 0)) - put_unused_fd(fdf->__fd); - if (unlikely(!IS_ERR_OR_NULL(fdf->__file))) - fput(fdf->__file); + if (unlikely(fdf->fd >= 0)) { + put_unused_fd(fdf->fd); + fput(fdf->file); + } } /* Do not use directly. */ -static inline int class_fd_prepare_lock_err(const struct fd_prepare *fdf) +static __always_inline struct fd_prepare __fd_prepare(int fd, struct file *file) { - if (unlikely(fdf->err)) - return fdf->err; - if (unlikely(fdf->__fd < 0)) - return fdf->__fd; - if (unlikely(IS_ERR(fdf->__file))) - return PTR_ERR(fdf->__file); - if (unlikely(!fdf->__file)) - return -ENOMEM; - return 0; + if (fd >= 0 && IS_ERR_OR_NULL(file)) { + int err = file ? PTR_ERR(file) : -ENOMEM; + + put_unused_fd(fd); + fd = err; + file = NULL; + } + return (struct fd_prepare){ .fd = fd, .file = file }; } /* - * __FD_PREPARE_INIT - Helper to initialize fd_prepare class. - * @_fd_flags: flags for get_unused_fd_flags() - * @_file_owned: expression that returns struct file * - * - * Returns a struct fd_prepare with fd, file, and err set. - * If fd allocation fails, fd will be negative and err will be set. If - * fd succeeds but file_init_expr fails, file will be ERR_PTR and err - * will be set. The err field is the single source of truth for error - * checking. - */ -#define __FD_PREPARE_INIT(_fd_flags, _file_owned) \ - ({ \ - struct fd_prepare fdf = { \ - .__fd = get_unused_fd_flags((_fd_flags)), \ - }; \ - if (likely(fdf.__fd >= 0)) \ - fdf.__file = (_file_owned); \ - fdf.err = ACQUIRE_ERR(fd_prepare, &fdf); \ - fdf; \ - }) - -/* - * FD_PREPARE - Macro to declare and initialize an fd_prepare variable. + * FD_PREPARE - Declare and initialize an fd_prepare instance. * - * Declares and initializes an fd_prepare variable with automatic - * cleanup. No separate scope required - cleanup happens when variable - * goes out of scope. + * This allocates a new fd and only evaluates @_file_owned if the + * allocation succeeded. Cleanup happens when the variable goes out of + * scope and the guard releases whichever of the descriptor and the file + * was allocated. If fd_publish() was called the fd and file are + * published and cleanup becomes a nop. * - * @_fdf: name of struct fd_prepare variable to define + * @_fdf: name of the const struct fd_prepare pointer to define * @_fd_flags: flags for get_unused_fd_flags() * @_file_owned: struct file to take ownership of (can be expression) */ +#define __FD_PREPARE(_guard, _fdf, _fd_flags, _file_owned) \ + struct fd_prepare _guard __cleanup(__fd_prepare_cleanup) = ({ \ + int __fd = get_unused_fd_flags(_fd_flags); \ + __fd_prepare(__fd, __fd < 0 ? NULL : (_file_owned)); \ + }); \ + const struct fd_prepare *const _fdf = &_guard + #define FD_PREPARE(_fdf, _fd_flags, _file_owned) \ - CLASS_INIT(fd_prepare, _fdf, __FD_PREPARE_INIT(_fd_flags, _file_owned)) + __FD_PREPARE(__UNIQUE_ID(fd_prepare), _fdf, _fd_flags, _file_owned) /* * fd_publish - Publish prepared fd and file to the fd table. - * @_fdf: struct fd_prepare variable + * @fdf: struct fd_prepare pointer defined by FD_PREPARE() */ -#define fd_publish(_fdf) \ - ({ \ - struct fd_prepare *fdp = &(_fdf); \ - VFS_WARN_ON_ONCE(fdp->err); \ - VFS_WARN_ON_ONCE(fdp->__fd < 0); \ - VFS_WARN_ON_ONCE(IS_ERR_OR_NULL(fdp->__file)); \ - fd_install(fdp->__fd, fdp->__file); \ - retain_and_null_ptr(fdp->__file); \ - take_fd(fdp->__fd); \ - }) +static __always_inline int fd_publish(const struct fd_prepare *fdf) +{ + /* Callers only get a const view, the guard itself is writable. */ + struct fd_prepare *guard = (struct fd_prepare *)fdf; + + VFS_WARN_ON_ONCE(guard->fd < 0); + fd_install(guard->fd, guard->file); + return take_fd(guard->fd); +} /* Do not use directly. */ -#define __FD_ADD(_fdf, _fd_flags, _file_owned) \ - ({ \ - FD_PREPARE(_fdf, _fd_flags, _file_owned); \ - s32 ret = _fdf.err; \ - if (likely(!ret)) \ - ret = fd_publish(_fdf); \ - ret; \ +#define __FD_ADD(_fdf, _fd_flags, _file_owned) \ + ({ \ + FD_PREPARE(_fdf, _fd_flags, _file_owned); \ + _fdf->fd < 0 ? _fdf->fd : fd_publish(_fdf); \ }) /* diff --git a/include/linux/fileattr.h b/include/linux/fileattr.h index 58044b598016..09e32b84e02a 100644 --- a/include/linux/fileattr.h +++ b/include/linux/fileattr.h @@ -74,7 +74,7 @@ static inline bool fileattr_has_fsx(const struct file_kattr *fa) } int vfs_fileattr_get(struct dentry *dentry, struct file_kattr *fa); -int vfs_fileattr_set(struct mnt_idmap *idmap, struct dentry *dentry, +int vfs_fileattr_set(const struct mnt_idmap *idmap, struct dentry *dentry, struct file_kattr *fa); int ioctl_getflags(struct file *file, unsigned int __user *argp); int ioctl_setflags(struct file *file, unsigned int __user *argp); diff --git a/include/linux/filelock.h b/include/linux/filelock.h index ec11cc6b4c58..e909db319067 100644 --- a/include/linux/filelock.h +++ b/include/linux/filelock.h @@ -530,8 +530,8 @@ static inline bool is_delegated(struct delegated_inode *di) * * Break delegations in a non-blocking fashion. If there are * outstanding delegations and @di is set, then an extra reference - * will be taken on @inode and @di->di_inode will be populated so - * that it may be waited upon. + * will be taken on @inode and its superblock, and @di->di_inode will + * be populated so that it may be waited upon. * * Returns 0 if there is no need to wait or an error. If -EWOULDBLOCK * is returned, then @di will be populated (if non-NULL). @@ -545,16 +545,21 @@ static inline int try_break_deleg(struct inode *inode, unsigned int flags, if (ret == -EWOULDBLOCK && di) { di->di_inode = inode; ihold(inode); + /* The caller may drop its mount before break_deleg_wait(). */ + atomic_inc(&inode->i_sb->s_active); } return ret; } static inline int break_deleg_wait(struct delegated_inode *di) { + struct inode *inode = di->di_inode; + struct super_block *sb = inode->i_sb; int ret; - ret = break_deleg(di->di_inode, 0); - iput(di->di_inode); + ret = break_deleg(inode, 0); + iput(inode); + deactivate_super(sb); di->di_inode = NULL; return ret; } diff --git a/include/linux/filter.h b/include/linux/filter.h index 39decde7fc73..98254742ae37 100644 --- a/include/linux/filter.h +++ b/include/linux/filter.h @@ -98,6 +98,11 @@ struct ctl_table_header; /* BPF program can access up to 512 bytes of stack space. */ #define MAX_BPF_STACK 512 +/* + * Stack budget of a program on a JIT that lays out frames of that size. + * The interpreter and JITs without such support keep MAX_BPF_STACK. + */ +#define MAX_BPF_STACK_JIT 2048 /* Helper macros for filter block array initializers. */ @@ -249,6 +254,29 @@ static inline bool insn_is_mov_percpu_addr(const struct bpf_insn *insn) .off = 0, \ .imm = 1 }) +static inline bool bpf_is_unwind_goto(const struct bpf_insn *insn) +{ + return insn->code == (BPF_JMP | BPF_JCOND) && insn->src_reg == BPF_UNWIND_GOTO; +} + +static inline bool bpf_is_unwind_exit(const struct bpf_insn *insn) +{ + return insn->code == (BPF_JMP | BPF_EXIT) && insn->src_reg == BPF_UNWIND_EXIT; +} + +/* + * The caller of the main prog doesn't tell unwind_exit from exit. An extension + * is different: its caller is the caller of the subprog that it replaced. + * An exception callback returns from the frame of the main prog. After freplace + * or a tail call it returns to a prog that wasn't verified with the callback. + */ +static inline bool bpf_prog_can_unwind(const struct bpf_prog *prog) +{ + if (prog->aux->exception_cb) + return false; + return bpf_is_subprog(prog) || prog->type == BPF_PROG_TYPE_EXT; +} + static inline bool insn_is_zext(const struct bpf_insn *insn) { return insn->code == (BPF_ALU | BPF_MOV | BPF_X) && insn->imm == 1; @@ -1105,6 +1133,19 @@ static inline unsigned int bpf_prog_size(unsigned int proglen) offsetof(struct bpf_prog, insns[proglen])); } +bool bpf_prog_has_arena_insns(const struct bpf_prog *prog); + +/* + * Whether a JIT has to set up the arena base for @prog. A program bound to + * an arena need not touch it, libbpf binds the arena of an object to every + * program in it. The flag caches bpf_prog_has_arena_insns() for the function + * about to be JITed, see bpf_prog_jit_compile() and jit_subprogs(). + */ +static inline bool bpf_prog_uses_arena_insns(const struct bpf_prog *prog) +{ + return prog->aux->uses_arena_insns; +} + static inline bool bpf_prog_was_classic(const struct bpf_prog *prog) { /* When classic BPF programs have been loaded and the arch @@ -1237,6 +1278,9 @@ bool bpf_jit_inlines_helper_call(s32 imm); bool bpf_jit_supports_subprog_tailcalls(void); bool bpf_jit_supports_percpu_insn(void); bool bpf_jit_supports_kfunc_call(void); +bool bpf_jit_supports_callx(void); +bool bpf_jit_supports_kfunc_ret_reg_pair(void); +bool bpf_jit_supports_unwind(void); bool bpf_jit_supports_stack_args(void); bool bpf_jit_supports_arena_args(void); bool bpf_jit_supports_far_kfunc_call(void); @@ -1245,8 +1289,41 @@ bool bpf_jit_supports_ptr_xchg(void); bool bpf_jit_supports_arena(void); bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena); bool bpf_jit_supports_private_stack(void); +bool bpf_jit_supports_large_stack(void); bool bpf_jit_supports_timed_may_goto(void); bool bpf_jit_supports_fsession(void); + +struct bpf_jit_arg_abi { + /* Argument registers of the kernel convention. */ + u8 nr_arg_regs; + /* Round the register number up to an even one for 16-byte alignment. */ + bool even_reg_align; + /* Round the stack slot up to an even one for 16-byte alignment. */ + bool even_stack_align; + /* An argument may straddle the last register and the stack. */ + bool split_at_boundary; + /* A later argument may reuse a register a stack-passed one skipped. */ + bool backfill_after_stack; +}; + +const struct bpf_jit_arg_abi *bpf_jit_arg_abi(void); +u32 bpf_jit_place_args(const struct bpf_jit_arg_abi *abi, + const struct btf_func_model *fm, u8 *pos_of_slot); + +/* The JIT's scratch register, in place of an argument slot. */ +#define BPF_JIT_ARG_TMP 0xff + +/* Every argument slot moves at most once, and the scratch goes out and back. */ +#define BPF_JIT_MAX_ARG_MOVES (MAX_BPF_FUNC_ARG_SLOTS + 2) + +struct bpf_jit_arg_move { + u8 dst; + u8 src; +}; + +u32 bpf_jit_plan_arg_moves(const struct bpf_jit_arg_abi *abi, + const struct btf_func_model *fm, + struct bpf_jit_arg_move *moves); u64 bpf_arch_uaddress_limit(void); void arch_bpf_stack_walk(bool (*consume_fn)(void *cookie, u64 ip, u64 sp, u64 bp), void *cookie); u64 arch_bpf_timed_may_goto(void); @@ -1897,6 +1974,11 @@ static __always_inline long __bpf_xdp_redirect_map(struct bpf_map *map, u64 inde return XDP_REDIRECT; } +int __bpf_sock_ops_load_hdr_opt(struct bpf_sock_ops_kern *bpf_sock, + void *search_res, u32 len, u64 flags); +int __bpf_sock_ops_store_hdr_opt(struct bpf_sock_ops_kern *bpf_sock, + const void *from, u32 len, u64 flags); + #ifdef CONFIG_NET int __bpf_skb_load_bytes(const struct sk_buff *skb, u32 offset, void *to, u32 len); int __bpf_skb_store_bytes(struct sk_buff *skb, u32 offset, const void *from, diff --git a/include/linux/firewire.h b/include/linux/firewire.h index fd35a6570cd8..9c563fbdbbf0 100644 --- a/include/linux/firewire.h +++ b/include/linux/firewire.h @@ -92,17 +92,17 @@ struct fw_card { u64 reset_jiffies; struct { - int current_tlabel; - u64 tlabel_mask; - struct list_head list; + int current_tlabel __guarded_by(&lock); + u64 tlabel_mask __guarded_by(&lock); + struct list_head list __guarded_by(&lock); spinlock_t lock; } transactions; struct { - u32 hi; - u32 lo; - unsigned int cycles; - unsigned int jiffies; + u32 hi __guarded_by(&lock); + u32 lo __guarded_by(&lock); + unsigned int cycles __guarded_by(&lock); + unsigned int jiffies __guarded_by(&lock); spinlock_t lock; } split_timeout; @@ -139,7 +139,7 @@ struct fw_card { u32 broadcast_channel; struct { - __be32 buffer[(CSR_TOPOLOGY_MAP_END - CSR_TOPOLOGY_MAP) / 4]; + __be32 buffer[(CSR_TOPOLOGY_MAP_END - CSR_TOPOLOGY_MAP) / 4] __guarded_by(&lock); spinlock_t lock; } topology_map; @@ -172,10 +172,10 @@ struct fw_attribute_group { }; enum fw_device_quirk { - // See afa1282a35d3 ("firewire: core: check for 1394a compliant IRM, fix inaccessibility of Sony camcorder"). + // See 10389536742c ("firewire: core: check for 1394a compliant IRM, fix inaccessibility of Sony camcorder"). FW_DEVICE_QUIRK_IRM_IS_1394_1995_ONLY = BIT(0), - // See a509e43ff338 ("firewire: core: fix unstable I/O with Canon camcorder"). + // See 6044565af458 ("firewire: core: fix unstable I/O with Canon camcorder"). FW_DEVICE_QUIRK_IRM_IGNORES_BUS_MANAGER = BIT(1), // MOTU Audio Express transfers acknowledge packet with 0x10 for pending state. @@ -224,7 +224,7 @@ struct fw_device { struct mutex client_list_mutex; struct list_head client_list; - const u32 *config_rom; + const u32 *config_rom __counted_by_ptr(config_rom_length); size_t config_rom_length; int config_rom_retries; unsigned is_local:1; @@ -286,28 +286,87 @@ struct fw_request; typedef void (*fw_packet_callback_t)(struct fw_packet *packet, struct fw_card *card, int status); + +/** + * typedef fw_transaction_callback_t - Function to handle the result of outgoing transaction. + * @card: the interface to send the request at + * @rcode: the response code + * @data: the data for the response subaction + * @length: the length of data + * @callback_data: the data registered with this callback + * + * This callback is called from a workqueue in process context after the outgoing transaction is + * completed successfully or unsuccessfully. + * + * Among its parameters is the response code which is either one of the rcodes per IEEE 1394 or, in + * case of internal errors, the firewire-core specific %RCODE_SEND_ERROR. The other firewire-core + * specific rcodes (%RCODE_CANCELLED, %RCODE_BUSY, %RCODE_GENERATION, %RCODE_NO_ACK) denote + * transaction timeout, busy responder, stale request generation, or missing ACK respectively. + * + * Context: Process context. + */ typedef void (*fw_transaction_callback_t)(struct fw_card *card, int rcode, void *data, size_t length, void *callback_data); + +/** + * typedef fw_transaction_callback_with_tstamp_t - Function to handle the result of outgoing + * transaction with hardware time stamps. + * @card: the interface to send the request at + * @rcode: the response code + * @request_tstamp: the time stamp at which the request subaction finished + * @response_tstamp: the time stamp at which the response subaction finished + * @data: the data for the response subaction + * @length: the length of data + * @callback_data: the data registered with this callback + * + * This callback is called from a workqueue in process context after the outgoing transaction is + * completed successfully or unsuccessfully. + * + * Among its parameters is the response code which is either one of the rcodes per IEEE 1394 or, in + * case of internal errors, the firewire-core specific %RCODE_SEND_ERROR. The other firewire-core + * specific rcodes (%RCODE_CANCELLED, %RCODE_BUSY, %RCODE_GENERATION, %RCODE_NO_ACK) denote + * transaction timeout, busy responder, stale request generation, or missing ACK respectively. + * + * The timestamp consists of the low order 3 bits of second field and the full 13 bits of count + * field of isochronous cycle time register. + * + * Context: Process context. + */ typedef void (*fw_transaction_callback_with_tstamp_t)(struct fw_card *card, int rcode, u32 request_tstamp, u32 response_tstamp, void *data, size_t length, void *callback_data); +/** + * union fw_transaction_callback - A union to store the outgoing transaction callback. + * @without_tstamp: Callback without hardware timestamps. + * @with_tstamp: Callback with hardware timestamps. + */ union fw_transaction_callback { fw_transaction_callback_t without_tstamp; fw_transaction_callback_with_tstamp_t with_tstamp; }; -/* - * This callback handles an inbound request subaction. It is called in - * RCU read-side context, therefore must not sleep. +/** + * typedef fw_address_callback_t - Function to handle the request of the incoming transaction. + * @card: the card instance which receives the request + * @request: the request instance. + * @tcode: the transaction code + * @destination: the destination node ID + * @source: the source node ID + * @generation: the bus generation in which the request was sent + * @offset: the destination offset in source node. + * @data: the request content if available. + * @length: the length of data. + * @callback_data: the data registered with this function. * - * The callback should not initiate outbound request subactions directly. - * Otherwise there is a danger of recursion of inbound and outbound - * transactions from and to the local node. + * This callback handles an inbound request subaction. * - * The callback is responsible that fw_send_response() is called on the @request, except for FCP - * registers for which the core takes care of that. + * The contents of @request are opaque to the callback. The callback is responsible for calling + * fw_send_response() for @request, except for FCP registers, for which the core handles the + * response. + * + * Context: Process context. */ typedef void (*fw_address_callback_t)(struct fw_card *card, struct fw_request *request, @@ -322,26 +381,36 @@ struct fw_packet { int generation; u32 header[4]; size_t header_length; - void *payload; + void *payload __counted_by_ptr(payload_length); size_t payload_length; dma_addr_t payload_bus; bool payload_mapped; u32 timestamp; + // Used to handle the local-to-local packets in the AT request/response contexts. + struct list_head link_for_local; + /* * This callback is called when the packet transmission has completed. * For successful transmission, the status code is the ack received * from the destination. Otherwise it is one of the juju-specific * rcodes: RCODE_SEND_ERROR, _CANCELLED, _BUSY, _GENERATION, _NO_ACK. - * The callback can be called from workqueue and thus must never block. + * The callback is called from a workqueue. It is not preferable to block it so long. */ fw_packet_callback_t callback; int ack; - struct list_head link; void *driver_data; }; +/** + * struct fw_transaction - A structure to hold outgoing transaction state. + * + * All structure members are for internal use only. Once passed to __fw_send_request(), this + * structure holds the state of an outgoing transaction. It must not be freed until the callback + * reports the transaction result. + */ struct fw_transaction { + /* private: */ int node_id; /* The generation is implied; it is always the current. */ int tlabel; struct list_head link; @@ -359,20 +428,44 @@ struct fw_transaction { union fw_transaction_callback callback; bool with_tstamp; void *callback_data; + + // For some error cases. + struct work_struct error_work; + int rcode; + u32 response_timestamp; }; +/** + * struct fw_address_handler - A handler for incoming transaction requests to a registered address + * region. + * @offset: The registered offset, available after successful registration. + * @length: The length of the address region in bytes, a multiple of a quadlet. + * @address_callback: Callback invoked when a transaction request is received for the registered + * address region. + * @callback_data: Data passed to @address_callback. + * + * This structure represents a handler registered to receive incoming asynchronous requests to a + * specific region of the address space on the local node. + */ struct fw_address_handler { u64 offset; u64 length; fw_address_callback_t address_callback; void *callback_data; - // Only for core functions. + /* private: */ struct list_head link; struct kref kref; struct completion done; }; +/** + * struct fw_address_region - An address range. + * @start: The starting offset in bytes, a multiple of a quadlet. + * @end: The ending offset in bytes, a multiple of a quadlet. + * + * Represents a range of addresses in the local node's address space. + */ struct fw_address_region { u64 start; u64 end; @@ -411,9 +504,10 @@ void __fw_send_request(struct fw_card *card, struct fw_transaction *t, int tcode * A variation of __fw_send_request() to generate callback for response subaction without time * stamp. * - * The callback is invoked in the workqueue context in most cases. However, if an error is detected - * before queueing or the destination address refers to the local node, it is invoked in the - * current context instead. + * After the transaction is completed successfully or unsuccessfully, the @callback will be called + * in process context. + * + * Context: Any context. */ static inline void fw_send_request(struct fw_card *card, struct fw_transaction *t, int tcode, int destination_id, int generation, int speed, @@ -444,9 +538,10 @@ static inline void fw_send_request(struct fw_card *card, struct fw_transaction * * * A variation of __fw_send_request() to generate callback for response subaction with time stamp. * - * The callback is invoked in the workqueue context in most cases. However, if an error is detected - * before queueing or the destination address refers to the local node, it is invoked in the current - * context instead. + * After the transaction is completed successfully or unsuccessfully, the @callback will be called + * in process context. + * + * Context: Any context. */ static inline void fw_send_request_with_tstamp(struct fw_card *card, struct fw_transaction *t, int tcode, int destination_id, int generation, int speed, unsigned long long offset, diff --git a/include/linux/firmware/intel/stratix10-smc.h b/include/linux/firmware/intel/stratix10-smc.h index 366309260121..6a6fa51ad39e 100644 --- a/include/linux/firmware/intel/stratix10-smc.h +++ b/include/linux/firmware/intel/stratix10-smc.h @@ -725,6 +725,27 @@ INTEL_SIP_SMC_FAST_CALL_VAL(INTEL_SIP_SMC_FUNCID_FPGA_CONFIG_COMPLETED_WRITE) INTEL_SIP_SMC_ASYNC_VAL(INTEL_SIP_SMC_ASYNC_FUNC_ID_POLL) /** + * Request INTEL_SIP_SMC_ASYNC_POLL_ON_IRQ + * Async call used by service driver at EL1 to retrieve the transaction IDs + * of available responses from the SDM mailbox (typically after an IRQ). + * + * Call register usage: + * a0 INTEL_SIP_SMC_ASYNC_POLL_ON_IRQ + * a1-a17 not used + * + * Return status + * a0 INTEL_SIP_SMC_STATUS_OK + * a1-a4 contain a bitmap of pending transaction IDs (set bit positions) + * a5-a17 not used + * Or + * a0 INTEL_SIP_SMC_STATUS_NO_RESPONSE + * a1-a17 not used + */ +#define INTEL_SIP_SMC_ASYNC_FUNC_ID_IRQ_POLL (0xC9) +#define INTEL_SIP_SMC_ASYNC_POLL_ON_IRQ \ + INTEL_SIP_SMC_ASYNC_VAL(INTEL_SIP_SMC_ASYNC_FUNC_ID_IRQ_POLL) + +/** * Request INTEL_SIP_SMC_ASYNC_HWMON_READTEMP * Async call to request temperature * @@ -813,4 +834,27 @@ INTEL_SIP_SMC_FAST_CALL_VAL(INTEL_SIP_SMC_FUNCID_FPGA_CONFIG_COMPLETED_WRITE) #define INTEL_SIP_SMC_ASYNC_FUNC_ID_RSU_NOTIFY (0xEC) #define INTEL_SIP_SMC_ASYNC_RSU_NOTIFY \ INTEL_SIP_SMC_ASYNC_VAL(INTEL_SIP_SMC_ASYNC_FUNC_ID_RSU_NOTIFY) + +/** + * Request INTEL_SIP_SMC_SDM_REMAPPER_CONFIG + * + * Sync call to configure the SDM address remapper. On Agilex5, the remapper + * must be bypassed when the SMMU is active to avoid conflicts with IOMMU + * address translation. + * + * Call register usage: + * a0: INTEL_SIP_SMC_SDM_REMAPPER_CONFIG + * a1: INTEL_SIP_SMC_SDM_REMAPPER_ENABLE or INTEL_SIP_SMC_SDM_REMAPPER_BYPASS + * a2-7: not used + * + * Return status: + * a0: INTEL_SIP_SMC_STATUS_OK + * a1-3: not used + */ +#define INTEL_SIP_SMC_FUNCID_SDM_REMAPPER_CONFIG 513 +#define INTEL_SIP_SMC_SDM_REMAPPER_CONFIG \ + INTEL_SIP_SMC_FAST_CALL_VAL(INTEL_SIP_SMC_FUNCID_SDM_REMAPPER_CONFIG) +#define INTEL_SIP_SMC_SDM_REMAPPER_ENABLE 0 +#define INTEL_SIP_SMC_SDM_REMAPPER_BYPASS 1 + #endif diff --git a/include/linux/firmware/qcom/qcom_scm.h b/include/linux/firmware/qcom/qcom_scm.h index 5747bd191bf1..a0a6bc0229c4 100644 --- a/include/linux/firmware/qcom/qcom_scm.h +++ b/include/linux/firmware/qcom/qcom_scm.h @@ -64,35 +64,6 @@ bool qcom_scm_is_available(void); int qcom_scm_set_cold_boot_addr(void *entry); int qcom_scm_set_warm_boot_addr(void *entry); void qcom_scm_cpu_power_down(u32 flags); -int qcom_scm_set_remote_state(u32 state, u32 id); - -struct qcom_scm_pas_context { - struct device *dev; - u32 pas_id; - phys_addr_t mem_phys; - size_t mem_size; - void *ptr; - dma_addr_t phys; - ssize_t size; - bool use_tzmem; -}; - -struct qcom_scm_pas_context *devm_qcom_scm_pas_context_alloc(struct device *dev, - u32 pas_id, - phys_addr_t mem_phys, - size_t mem_size); -int qcom_scm_pas_init_image(u32 pas_id, const void *metadata, size_t size, - struct qcom_scm_pas_context *ctx); -void qcom_scm_pas_metadata_release(struct qcom_scm_pas_context *ctx); -int qcom_scm_pas_mem_setup(u32 pas_id, phys_addr_t addr, phys_addr_t size); -int qcom_scm_pas_auth_and_reset(u32 pas_id); -int qcom_scm_pas_shutdown(u32 pas_id); -bool qcom_scm_pas_supported(u32 pas_id); -struct resource_table *qcom_scm_pas_get_rsc_table(struct qcom_scm_pas_context *ctx, - void *input_rt, size_t input_rt_size, - size_t *output_rt_size); - -int qcom_scm_pas_prepare_and_auth_reset(struct qcom_scm_pas_context *ctx); int qcom_scm_io_readl(phys_addr_t addr, unsigned int *val); int qcom_scm_io_writel(phys_addr_t addr, unsigned int val); diff --git a/include/linux/firmware/xlnx-zynqmp-ufs.h b/include/linux/firmware/xlnx-zynqmp-ufs.h index d3538dd5822a..00383dd835f2 100644 --- a/include/linux/firmware/xlnx-zynqmp-ufs.h +++ b/include/linux/firmware/xlnx-zynqmp-ufs.h @@ -9,17 +9,17 @@ #define __FIRMWARE_XLNX_ZYNQMP_UFS_H__ #if IS_REACHABLE(CONFIG_ZYNQMP_FIRMWARE) -int zynqmp_pm_is_mphy_tx_rx_config_ready(bool *is_ready); -int zynqmp_pm_is_sram_init_done(bool *is_done); +int zynqmp_pm_wait_mphy_tx_rx_config_ready(u32 timeout_us); +int zynqmp_pm_wait_sram_init_done(u32 timeout_us); int zynqmp_pm_set_sram_bypass(void); int zynqmp_pm_get_ufs_calibration_values(u32 *val); #else -static inline int zynqmp_pm_is_mphy_tx_rx_config_ready(bool *is_ready) +static inline int zynqmp_pm_wait_mphy_tx_rx_config_ready(u32 timeout_us) { return -ENODEV; } -static inline int zynqmp_pm_is_sram_init_done(bool *is_done) +static inline int zynqmp_pm_wait_sram_init_done(u32 timeout_us) { return -ENODEV; } diff --git a/include/linux/firmware/xlnx-zynqmp.h b/include/linux/firmware/xlnx-zynqmp.h index f4b3df614c77..41a1cd810bda 100644 --- a/include/linux/firmware/xlnx-zynqmp.h +++ b/include/linux/firmware/xlnx-zynqmp.h @@ -277,11 +277,6 @@ enum rpu_oper_mode { PM_RPU_MODE_SPLIT = 1, }; -enum rpu_boot_mem { - PM_RPU_BOOTMEM_LOVEC = 0, - PM_RPU_BOOTMEM_HIVEC = 1, -}; - enum rpu_tcm_comb { PM_RPU_TCM_SPLIT = 0, PM_RPU_TCM_COMB = 1, diff --git a/include/linux/firmware_attributes.h b/include/linux/firmware_attributes.h new file mode 100644 index 000000000000..d4026ba1fcc2 --- /dev/null +++ b/include/linux/firmware_attributes.h @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +/* Firmware attributes class helper module */ + +#ifndef _LINUX_FIRMWARE_ATTRIBUTES_H +#define _LINUX_FIRMWARE_ATTRIBUTES_H + +#include <linux/device/class.h> + +extern const struct class firmware_attributes_class; + +#endif /* _LINUX_FIRMWARE_ATTRIBUTES_H */ diff --git a/include/linux/font.h b/include/linux/font.h index 5e1cf9830084..be1a26ff8ae9 100644 --- a/include/linux/font.h +++ b/include/linux/font.h @@ -109,6 +109,10 @@ const unsigned char *font_data_glyph_buf(font_data_t *fd, bool font_data_is_equal(font_data_t *lhs, font_data_t *rhs); int font_data_export(font_data_t *fd, struct console_font *font, unsigned int vpitch); +/* font_cursor.c */ +void font_glyph_cursor(unsigned int width, unsigned int height, + unsigned int from, unsigned int to, unsigned char *out); + /* font_rotate.c */ void font_glyph_rotate_90(const unsigned char *glyph, unsigned int width, unsigned int height, unsigned char *out); diff --git a/include/linux/fortify-string.h b/include/linux/fortify-string.h index cf841dc71fef..7e7c369e0a6c 100644 --- a/include/linux/fortify-string.h +++ b/include/linux/fortify-string.h @@ -458,10 +458,8 @@ __FORTIFY_INLINE bool fortify_memset_chk(__kernel_size_t size, * __struct_size() vs __member_size() must be captured here to avoid * evaluating argument side-effects further into the macro layers. */ -#ifndef CONFIG_KMSAN #define memset(p, c, s) __fortify_memset_chk(p, c, s, \ __struct_size(p), __member_size(p)) -#endif /* * To make sure the compiler can enforce protection against buffer overflows, diff --git a/include/linux/fs.h b/include/linux/fs.h index deb411e86661..99caef43ff3c 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -815,7 +815,9 @@ struct inode { /* 32-bit hole */ struct rw_semaphore i_rwsem; - unsigned long dirtied_when; /* jiffies of first dirtying */ + /* Timestamp for writeback list ordering and age checks (jiffies). */ + unsigned long dirtied_when; + /* Start time for deciding when lazytime updates need writeback (jiffies). */ unsigned long dirtied_time_when; struct hlist_node i_hash; @@ -1436,10 +1438,10 @@ static inline void i_gid_write(struct inode *inode, gid_t gid) * @idmap: idmap of the mount the inode was found from * @inode: inode to map * - * Return: whe inode's i_uid mapped down according to @idmap. + * Return: the inode's i_uid mapped down according to @idmap. * If the inode's i_uid has no mapping INVALID_VFSUID is returned. */ -static inline vfsuid_t i_uid_into_vfsuid(struct mnt_idmap *idmap, +static inline vfsuid_t i_uid_into_vfsuid(const struct mnt_idmap *idmap, const struct inode *inode) { return make_vfsuid(idmap, i_user_ns(inode), inode->i_uid); @@ -1456,7 +1458,7 @@ static inline vfsuid_t i_uid_into_vfsuid(struct mnt_idmap *idmap, * * Return: true if @inode's i_uid field needs to be updated, false if not. */ -static inline bool i_uid_needs_update(struct mnt_idmap *idmap, +static inline bool i_uid_needs_update(const struct mnt_idmap *idmap, const struct iattr *attr, const struct inode *inode) { @@ -1474,7 +1476,7 @@ static inline bool i_uid_needs_update(struct mnt_idmap *idmap, * Safely update @inode's i_uid field translating the vfsuid of any idmapped * mount into the filesystem kuid. */ -static inline void i_uid_update(struct mnt_idmap *idmap, +static inline void i_uid_update(const struct mnt_idmap *idmap, const struct iattr *attr, struct inode *inode) { @@ -1491,7 +1493,7 @@ static inline void i_uid_update(struct mnt_idmap *idmap, * Return: the inode's i_gid mapped down according to @idmap. * If the inode's i_gid has no mapping INVALID_VFSGID is returned. */ -static inline vfsgid_t i_gid_into_vfsgid(struct mnt_idmap *idmap, +static inline vfsgid_t i_gid_into_vfsgid(const struct mnt_idmap *idmap, const struct inode *inode) { return make_vfsgid(idmap, i_user_ns(inode), inode->i_gid); @@ -1508,7 +1510,7 @@ static inline vfsgid_t i_gid_into_vfsgid(struct mnt_idmap *idmap, * * Return: true if @inode's i_gid field needs to be updated, false if not. */ -static inline bool i_gid_needs_update(struct mnt_idmap *idmap, +static inline bool i_gid_needs_update(const struct mnt_idmap *idmap, const struct iattr *attr, const struct inode *inode) { @@ -1526,7 +1528,7 @@ static inline bool i_gid_needs_update(struct mnt_idmap *idmap, * Safely update @inode's i_gid field translating the vfsgid of any idmapped * mount into the filesystem kgid. */ -static inline void i_gid_update(struct mnt_idmap *idmap, +static inline void i_gid_update(const struct mnt_idmap *idmap, const struct iattr *attr, struct inode *inode) { @@ -1544,7 +1546,7 @@ static inline void i_gid_update(struct mnt_idmap *idmap, * an idmapped mount map the caller's fsuid according to @idmap. */ static inline void inode_fsuid_set(struct inode *inode, - struct mnt_idmap *idmap) + const struct mnt_idmap *idmap) { inode->i_uid = mapped_fsuid(idmap, i_user_ns(inode)); } @@ -1558,7 +1560,7 @@ static inline void inode_fsuid_set(struct inode *inode, * an idmapped mount map the caller's fsgid according to @idmap. */ static inline void inode_fsgid_set(struct inode *inode, - struct mnt_idmap *idmap) + const struct mnt_idmap *idmap) { inode->i_gid = mapped_fsgid(idmap, i_user_ns(inode)); } @@ -1575,7 +1577,7 @@ static inline void inode_fsgid_set(struct inode *inode, * Return: true if fsuid and fsgid is mapped, false if not. */ static inline bool fsuidgid_has_mapping(struct super_block *sb, - struct mnt_idmap *idmap) + const struct mnt_idmap *idmap) { struct user_namespace *fs_userns = sb->s_user_ns; kuid_t kuid; @@ -1755,25 +1757,25 @@ static inline bool file_write_not_started(const struct file *file) return sb_write_not_started(file_inode(file)->i_sb); } -bool inode_owner_or_capable(struct mnt_idmap *idmap, +bool inode_owner_or_capable(const struct mnt_idmap *idmap, const struct inode *inode); /* * VFS helper functions.. */ -int vfs_create(struct mnt_idmap *, struct dentry *, umode_t, +int vfs_create(const struct mnt_idmap *, struct dentry *, umode_t, struct delegated_inode *); -struct dentry *vfs_mkdir(struct mnt_idmap *, struct inode *, +struct dentry *vfs_mkdir(const struct mnt_idmap *, struct inode *, struct dentry *, umode_t, struct delegated_inode *); -int vfs_mknod(struct mnt_idmap *, struct inode *, struct dentry *, +int vfs_mknod(const struct mnt_idmap *, struct inode *, struct dentry *, umode_t, dev_t, struct delegated_inode *); -int vfs_symlink(struct mnt_idmap *, struct inode *, +int vfs_symlink(const struct mnt_idmap *, struct inode *, struct dentry *, const char *, struct delegated_inode *); -int vfs_link(struct dentry *, struct mnt_idmap *, struct inode *, +int vfs_link(struct dentry *, const struct mnt_idmap *, struct inode *, struct dentry *, struct delegated_inode *); -int vfs_rmdir(struct mnt_idmap *, struct inode *, struct dentry *, +int vfs_rmdir(const struct mnt_idmap *, struct inode *, struct dentry *, struct delegated_inode *); -int vfs_unlink(struct mnt_idmap *, struct inode *, struct dentry *, +int vfs_unlink(const struct mnt_idmap *, struct inode *, struct dentry *, struct delegated_inode *); /** @@ -1787,7 +1789,7 @@ int vfs_unlink(struct mnt_idmap *, struct inode *, struct dentry *, * @flags: rename flags */ struct renamedata { - struct mnt_idmap *mnt_idmap; + const struct mnt_idmap *mnt_idmap; struct dentry *old_parent; struct dentry *old_dentry; struct dentry *new_parent; @@ -1798,14 +1800,14 @@ struct renamedata { int vfs_rename(struct renamedata *); -static inline int vfs_whiteout(struct mnt_idmap *idmap, +static inline int vfs_whiteout(const struct mnt_idmap *idmap, struct inode *dir, struct dentry *dentry) { return vfs_mknod(idmap, dir, dentry, S_IFCHR | WHITEOUT_MODE, WHITEOUT_DEV, NULL); } -struct file *kernel_tmpfile_open(struct mnt_idmap *idmap, +struct file *kernel_tmpfile_open(const struct mnt_idmap *idmap, const struct path *parentpath, umode_t mode, int open_flag, const struct cred *cred); @@ -1830,12 +1832,12 @@ extern long compat_ptr_ioctl(struct file *file, unsigned int cmd, /* * VFS file helper functions. */ -void inode_init_owner(struct mnt_idmap *idmap, struct inode *inode, +void inode_init_owner(const struct mnt_idmap *idmap, struct inode *inode, const struct inode *dir, umode_t mode); extern bool may_open_dev(const struct path *path); -umode_t mode_strip_sgid(struct mnt_idmap *idmap, +umode_t mode_strip_sgid(const struct mnt_idmap *idmap, const struct inode *dir, umode_t mode); -bool in_group_or_capable(struct mnt_idmap *idmap, +bool in_group_or_capable(const struct mnt_idmap *idmap, const struct inode *inode, vfsgid_t vfsgid); /* @@ -1914,6 +1916,7 @@ struct dir_context { #define COPY_FILE_SPLICE (1 << 0) struct io_uring_cmd; +struct dma_buf_io_ctx; struct offset_ctx; struct file_operations { @@ -1960,6 +1963,7 @@ struct file_operations { int (*uring_cmd_iopoll)(struct io_uring_cmd *, struct io_comp_batch *, unsigned int poll_flags); int (*mmap_prepare)(struct vm_area_desc *); + int (*init_dma_buf_io_ctx)(struct file *, struct dma_buf_io_ctx *); } __randomize_layout; /* Supports async buffered reads */ @@ -1996,26 +2000,26 @@ enum fs_update_time { struct inode_operations { struct dentry * (*lookup) (struct inode *,struct dentry *, unsigned int); const char * (*get_link) (struct dentry *, struct inode *, struct delayed_call *); - int (*permission) (struct mnt_idmap *, struct inode *, int); + int (*permission) (const struct mnt_idmap *, struct inode *, int); struct posix_acl * (*get_inode_acl)(struct inode *, int, bool); int (*readlink) (struct dentry *, char __user *,int); - int (*create) (struct mnt_idmap *, struct inode *,struct dentry *, + int (*create) (const struct mnt_idmap *, struct inode *,struct dentry *, umode_t); int (*link) (struct dentry *,struct inode *,struct dentry *); int (*unlink) (struct inode *,struct dentry *); - int (*symlink) (struct mnt_idmap *, struct inode *,struct dentry *, + int (*symlink) (const struct mnt_idmap *, struct inode *,struct dentry *, const char *); - struct dentry *(*mkdir) (struct mnt_idmap *, struct inode *, + struct dentry *(*mkdir) (const struct mnt_idmap *, struct inode *, struct dentry *, umode_t); int (*rmdir) (struct inode *,struct dentry *); - int (*mknod) (struct mnt_idmap *, struct inode *,struct dentry *, + int (*mknod) (const struct mnt_idmap *, struct inode *,struct dentry *, umode_t,dev_t); - int (*rename) (struct mnt_idmap *, struct inode *, struct dentry *, + int (*rename) (const struct mnt_idmap *, struct inode *, struct dentry *, struct inode *, struct dentry *, unsigned int); - int (*setattr) (struct mnt_idmap *, struct dentry *, struct iattr *); - int (*getattr) (struct mnt_idmap *, const struct path *, + int (*setattr) (const struct mnt_idmap *, struct dentry *, struct iattr *); + int (*getattr) (const struct mnt_idmap *, const struct path *, struct kstat *, u32, unsigned int); ssize_t (*listxattr) (struct dentry *, char *, size_t); int (*fiemap)(struct inode *, struct fiemap_extent_info *, u64 start, @@ -2026,13 +2030,13 @@ struct inode_operations { int (*atomic_open)(struct inode *, struct dentry *, struct file *, unsigned open_flag, umode_t create_mode); - int (*tmpfile) (struct mnt_idmap *, struct inode *, + int (*tmpfile) (const struct mnt_idmap *, struct inode *, struct file *, umode_t); - struct posix_acl *(*get_acl)(struct mnt_idmap *, struct dentry *, + struct posix_acl *(*get_acl)(const struct mnt_idmap *, struct dentry *, int); - int (*set_acl)(struct mnt_idmap *, struct dentry *, + int (*set_acl)(const struct mnt_idmap *, struct dentry *, struct posix_acl *, int); - int (*fileattr_set)(struct mnt_idmap *idmap, + int (*fileattr_set)(const struct mnt_idmap *idmap, struct dentry *dentry, struct file_kattr *fa); int (*fileattr_get)(struct dentry *dentry, struct file_kattr *fa); struct offset_ctx *(*get_offset_ctx)(struct inode *inode); @@ -2175,7 +2179,7 @@ extern loff_t vfs_dedupe_file_range_one(struct file *src_file, loff_t src_pos, (inode)->i_rdev == WHITEOUT_DEV) #define IS_ANON_FILE(inode) ((inode)->i_flags & S_ANON_INODE) -static inline bool HAS_UNMAPPED_ID(struct mnt_idmap *idmap, +static inline bool HAS_UNMAPPED_ID(const struct mnt_idmap *idmap, struct inode *inode) { return !vfsuid_valid(i_uid_into_vfsuid(idmap, inode)) || @@ -2316,6 +2320,8 @@ struct file_system_type { struct lock_class_key i_mutex_key; struct lock_class_key invalidate_lock_key; struct lock_class_key i_mutex_dir_key; + + struct lock_class_key d_lookup_key; }; #define MODULE_ALIAS_FS(NAME) MODULE_ALIAS("fs-" NAME) @@ -2462,7 +2468,7 @@ struct filename { static_assert(offsetof(struct filename, iname) % sizeof(long) == 0); static_assert(sizeof(struct filename) % 64 == 0); -static inline struct mnt_idmap *file_mnt_idmap(const struct file *file) +static inline const struct mnt_idmap *file_mnt_idmap(const struct file *file) { return mnt_idmap(file->f_path.mnt); } @@ -2486,7 +2492,7 @@ static inline bool is_idmapped_mnt(const struct vfsmount *mnt) } int vfs_truncate(const struct path *, loff_t); -int do_truncate(struct mnt_idmap *, struct dentry *, loff_t start, +int do_truncate(const struct mnt_idmap *, struct dentry *, loff_t start, unsigned int time_attrs, struct file *filp); extern int vfs_fallocate(struct file *file, int mode, loff_t offset, loff_t len); @@ -2710,10 +2716,10 @@ static inline int bmap(struct inode *inode, sector_t *block) } #endif -int notify_change(struct mnt_idmap *, struct dentry *, +int notify_change(const struct mnt_idmap *, struct dentry *, struct iattr *, struct delegated_inode *); -int inode_permission(struct mnt_idmap *, struct inode *, int); -int generic_permission(struct mnt_idmap *, struct inode *, int); +int inode_permission(const struct mnt_idmap *, struct inode *, int); +int generic_permission(const struct mnt_idmap *, struct inode *, int); static inline int file_permission(struct file *file, int mask) { return inode_permission(file_mnt_idmap(file), @@ -2724,12 +2730,12 @@ static inline int path_permission(const struct path *path, int mask) return inode_permission(mnt_idmap(path->mnt), d_inode(path->dentry), mask); } -int __check_sticky(struct mnt_idmap *idmap, struct inode *dir, +int __check_sticky(const struct mnt_idmap *idmap, struct inode *dir, struct inode *inode); -int may_delete_dentry(struct mnt_idmap *idmap, struct inode *dir, +int may_delete_dentry(const struct mnt_idmap *idmap, struct inode *dir, struct dentry *victim, bool isdir); -int may_create_dentry(struct mnt_idmap *idmap, +int may_create_dentry(const struct mnt_idmap *idmap, struct inode *dir, struct dentry *child); static inline bool execute_ok(struct inode *inode) @@ -3048,9 +3054,9 @@ static inline struct inode *new_inode_pseudo(struct super_block *sb) } extern struct inode *new_inode(struct super_block *sb); extern void free_inode_nonrcu(struct inode *inode); -extern int setattr_should_drop_suidgid(struct mnt_idmap *, struct inode *); +extern int setattr_should_drop_suidgid(const struct mnt_idmap *, struct inode *); extern int file_remove_privs(struct file *); -int setattr_should_drop_sgid(struct mnt_idmap *idmap, +int setattr_should_drop_sgid(const struct mnt_idmap *idmap, const struct inode *inode); /* @@ -3207,7 +3213,7 @@ extern int page_symlink(struct inode *inode, const char *symname, int len); extern const struct inode_operations page_symlink_inode_operations; extern void kfree_link(void *); void fill_mg_cmtime(struct kstat *stat, u32 request_mask, struct inode *inode); -void generic_fillattr(struct mnt_idmap *, u32, struct inode *, struct kstat *); +void generic_fillattr(const struct mnt_idmap *, u32, struct inode *, struct kstat *); void generic_fill_statx_attr(struct inode *inode, struct kstat *stat); void generic_fill_statx_atomic_writes(struct kstat *stat, unsigned int unit_min, @@ -3264,9 +3270,9 @@ extern int dcache_dir_open(struct inode *, struct file *); extern int dcache_dir_close(struct inode *, struct file *); extern loff_t dcache_dir_lseek(struct file *, loff_t, int); extern int dcache_readdir(struct file *, struct dir_context *); -extern int simple_setattr(struct mnt_idmap *, struct dentry *, +extern int simple_setattr(const struct mnt_idmap *, struct dentry *, struct iattr *); -extern int simple_getattr(struct mnt_idmap *, const struct path *, +extern int simple_getattr(const struct mnt_idmap *, const struct path *, struct kstat *, u32, unsigned int); extern int simple_statfs(struct dentry *, struct kstatfs *); extern int simple_open(struct inode *inode, struct file *file); @@ -3279,7 +3285,7 @@ void simple_rename_timestamp(struct inode *old_dir, struct dentry *old_dentry, struct inode *new_dir, struct dentry *new_dentry); extern int simple_rename_exchange(struct inode *old_dir, struct dentry *old_dentry, struct inode *new_dir, struct dentry *new_dentry); -extern int simple_rename(struct mnt_idmap *, struct inode *, +extern int simple_rename(const struct mnt_idmap *, struct inode *, struct dentry *, struct inode *, struct dentry *, unsigned int); extern void simple_recursive_removal(struct dentry *, @@ -3400,11 +3406,11 @@ static inline bool generic_ci_validate_strict_name(struct inode *dir, } #endif -int may_setattr(struct mnt_idmap *idmap, struct inode *inode, +int may_setattr(const struct mnt_idmap *idmap, struct inode *inode, unsigned int ia_valid); -int setattr_prepare(struct mnt_idmap *, struct dentry *, struct iattr *); +int setattr_prepare(const struct mnt_idmap *, struct dentry *, struct iattr *); extern int inode_newsize_ok(const struct inode *, loff_t offset); -void setattr_copy(struct mnt_idmap *, struct inode *inode, +void setattr_copy(const struct mnt_idmap *, struct inode *inode, const struct iattr *attr); extern int file_update_time(struct file *file); @@ -3581,7 +3587,7 @@ static inline bool is_sxid(umode_t mode) return mode & (S_ISUID | S_ISGID); } -static inline int check_sticky(struct mnt_idmap *idmap, +static inline int check_sticky(const struct mnt_idmap *idmap, struct inode *dir, struct inode *inode) { if (!(dir->i_mode & S_ISVTX)) @@ -3656,23 +3662,6 @@ extern int vfs_fadvise(struct file *file, loff_t offset, loff_t len, extern int generic_fadvise(struct file *file, loff_t offset, loff_t len, int advice); -static inline bool vfs_empty_path(int dfd, const char __user *path) -{ - char c; - - if (dfd < 0) - return false; - - /* We now allow NULL to be used for empty path. */ - if (!path) - return true; - - if (unlikely(get_user(c, path))) - return false; - - return !c; -} - int generic_atomic_write_valid(struct kiocb *iocb, struct iov_iter *iter); static inline bool extensible_ioctl_valid(unsigned int cmd_a, diff --git a/include/linux/fs/super_types.h b/include/linux/fs/super_types.h index ecd96aeb1cee..72f4288e16c5 100644 --- a/include/linux/fs/super_types.h +++ b/include/linux/fs/super_types.h @@ -30,7 +30,6 @@ struct mount; struct mtd_info; struct quotactl_ops; struct shrinker; -struct super_dev; struct unicode_map; struct user_namespace; struct workqueue_struct; @@ -132,10 +131,18 @@ struct super_operations { void (*report_error)(const struct fserror_event *event); }; +/* A superblock's claim on a device, see fs/super.c. */ +struct super_dev { + struct hlist_node sd_hash; + struct super_block *sd_sb; + dev_t sd_dev; + unsigned int sd_claims; +}; + struct super_block { struct list_head s_list; /* Keep this first */ dev_t s_dev; /* search index; _not_ kdev_t */ - struct super_dev *s_super_dev; /* sget_fc()'s device table claim */ + struct super_dev s_super_dev; /* claim on s_dev */ unsigned char s_blocksize_bits; unsigned long s_blocksize; loff_t s_maxbytes; /* Max file size */ diff --git a/include/linux/fs_context.h b/include/linux/fs_context.h index 0d6c8a6d7be2..d1727dffecc5 100644 --- a/include/linux/fs_context.h +++ b/include/linux/fs_context.h @@ -150,6 +150,10 @@ extern int vfs_parse_fs_param_source(struct fs_context *fc, struct fs_parameter *param); extern void fc_drop_locked(struct fs_context *fc); +int get_tree_super(struct fs_context *fc, + int (*test)(struct super_block *, struct fs_context *), + int (*fill_super)(struct super_block *sb, + struct fs_context *fc)); extern int get_tree_nodev(struct fs_context *fc, int (*fill_super)(struct super_block *sb, struct fs_context *fc)); diff --git a/include/linux/fscache-cache.h b/include/linux/fscache-cache.h index 4c91a019972b..2b83ffa4510e 100644 --- a/include/linux/fscache-cache.h +++ b/include/linux/fscache-cache.h @@ -67,7 +67,7 @@ struct fscache_cache_ops { /* Change the size of a data object */ void (*resize_cookie)(struct netfs_cache_resources *cres, - loff_t new_size); + uoff_t new_size, bool shrank); /* Invalidate an object */ bool (*invalidate_cookie)(struct fscache_cookie *cookie); diff --git a/include/linux/fscache.h b/include/linux/fscache.h index 58fdb9605425..ecd14d645690 100644 --- a/include/linux/fscache.h +++ b/include/linux/fscache.h @@ -112,7 +112,7 @@ struct fscache_cookie { struct list_head proc_link; /* Link in proc list */ struct list_head commit_link; /* Link in commit queue */ struct work_struct work; /* Commit/relinq/withdraw work */ - loff_t object_size; /* Size of the netfs object */ + uoff_t object_size; /* Size of the netfs object */ unsigned long unused_at; /* Time at which unused (jiffies) */ unsigned long flags; #define FSCACHE_COOKIE_RELINQUISHED 0 /* T if cookie has been relinquished */ @@ -147,6 +147,23 @@ struct fscache_cookie { }; }; +enum fscache_extent_type { + FSCACHE_EXTENT_DATA, + FSCACHE_EXTENT_ZERO, +} __mode(byte); + +/* + * Cache occupancy information. + */ +struct fscache_occupancy { + unsigned long long query_from; /* Point to query from */ + unsigned long long query_to; /* Point to query to */ + unsigned long long cached_from[2]; /* Point at which cache extents start */ + unsigned long long cached_to[2]; /* Point at which cache extents end */ + unsigned int granularity; /* Granularity desired */ + enum fscache_extent_type cached_type[2]; /* Type of cache extent */ +}; + /* * slow-path functions for when there is actually caching available, and the * netfs does actually have a valid token @@ -163,22 +180,22 @@ extern struct fscache_cookie *__fscache_acquire_cookie( u8, const void *, size_t, const void *, size_t, - loff_t); + uoff_t); extern void __fscache_use_cookie(struct fscache_cookie *, bool); -extern void __fscache_unuse_cookie(struct fscache_cookie *, const void *, const loff_t *); +extern void __fscache_unuse_cookie(struct fscache_cookie *, const void *, const uoff_t *); extern void __fscache_relinquish_cookie(struct fscache_cookie *, bool); -extern void __fscache_resize_cookie(struct fscache_cookie *, loff_t); -extern void __fscache_invalidate(struct fscache_cookie *, const void *, loff_t, unsigned int); +void __fscache_resize_cookie(struct fscache_cookie *cookie, uoff_t new_size, bool shrank); +extern void __fscache_invalidate(struct fscache_cookie *, const void *, uoff_t, unsigned int); extern int __fscache_begin_read_operation(struct netfs_cache_resources *, struct fscache_cookie *); extern int __fscache_begin_write_operation(struct netfs_cache_resources *, struct fscache_cookie *); void __fscache_write_to_cache(struct fscache_cookie *cookie, struct address_space *mapping, - loff_t start, size_t len, loff_t i_size, + uoff_t start, size_t len, uoff_t i_size, netfs_io_terminated_t term_func, void *term_func_priv, bool using_pgpriv2, bool cond); -extern void __fscache_clear_page_bits(struct address_space *, loff_t, size_t); +extern void __fscache_clear_page_bits(struct address_space *, uoff_t, size_t); /** * fscache_acquire_volume - Register a volume as desiring caching services @@ -249,7 +266,7 @@ struct fscache_cookie *fscache_acquire_cookie(struct fscache_volume *volume, size_t index_key_len, const void *aux_data, size_t aux_data_len, - loff_t object_size) + uoff_t object_size) { if (!fscache_volume_valid(volume)) return NULL; @@ -286,7 +303,7 @@ static inline void fscache_use_cookie(struct fscache_cookie *cookie, */ static inline void fscache_unuse_cookie(struct fscache_cookie *cookie, const void *aux_data, - const loff_t *object_size) + const uoff_t *object_size) { if (fscache_cookie_valid(cookie)) __fscache_unuse_cookie(cookie, aux_data, object_size); @@ -327,7 +344,7 @@ static inline void *fscache_get_aux(struct fscache_cookie *cookie) */ static inline void fscache_update_aux(struct fscache_cookie *cookie, - const void *aux_data, const loff_t *object_size) + const void *aux_data, const uoff_t *object_size) { void *p = fscache_get_aux(cookie); @@ -343,7 +360,7 @@ extern atomic_t fscache_n_updates; static inline void __fscache_update_cookie(struct fscache_cookie *cookie, const void *aux_data, - const loff_t *object_size) + const uoff_t *object_size) { #ifdef CONFIG_FSCACHE_STATS atomic_inc(&fscache_n_updates); @@ -369,7 +386,7 @@ void __fscache_update_cookie(struct fscache_cookie *cookie, const void *aux_data */ static inline void fscache_update_cookie(struct fscache_cookie *cookie, const void *aux_data, - const loff_t *object_size) + const uoff_t *object_size) { if (fscache_cookie_enabled(cookie)) __fscache_update_cookie(cookie, aux_data, object_size); @@ -377,8 +394,9 @@ void fscache_update_cookie(struct fscache_cookie *cookie, const void *aux_data, /** * fscache_resize_cookie - Request that a cache object be resized - * @cookie: The cookie representing the cache object - * @new_size: The new size of the object (may be NULL) + * @cookie: The cookie representing the cache object (may be NULL) + * @new_size: The new size of the object + * @shrank: True if the file shrank; false if it expanded * * Request that the size of an object be changed. * @@ -386,10 +404,10 @@ void fscache_update_cookie(struct fscache_cookie *cookie, const void *aux_data, * description. */ static inline -void fscache_resize_cookie(struct fscache_cookie *cookie, loff_t new_size) +void fscache_resize_cookie(struct fscache_cookie *cookie, uoff_t new_size, bool shrank) { if (fscache_cookie_enabled(cookie)) - __fscache_resize_cookie(cookie, new_size); + __fscache_resize_cookie(cookie, new_size, shrank); } /** @@ -413,7 +431,7 @@ void fscache_resize_cookie(struct fscache_cookie *cookie, loff_t new_size) */ static inline void fscache_invalidate(struct fscache_cookie *cookie, - const void *aux_data, loff_t size, unsigned int flags) + const void *aux_data, uoff_t size, unsigned int flags) { if (fscache_cookie_enabled(cookie)) __fscache_invalidate(cookie, aux_data, size, flags); @@ -502,7 +520,7 @@ static inline void fscache_end_operation(struct netfs_cache_resources *cres) */ static inline int fscache_read(struct netfs_cache_resources *cres, - loff_t start_pos, + uoff_t start_pos, struct iov_iter *iter, enum netfs_read_from_hole read_hole, netfs_io_terminated_t term_func, @@ -561,7 +579,7 @@ int fscache_begin_write_operation(struct netfs_cache_resources *cres, */ static inline int fscache_write(struct netfs_cache_resources *cres, - loff_t start_pos, + uoff_t start_pos, struct iov_iter *iter, netfs_io_terminated_t term_func, void *term_func_priv) @@ -581,7 +599,7 @@ int fscache_write(struct netfs_cache_resources *cres, * waiting. */ static inline void fscache_clear_page_bits(struct address_space *mapping, - loff_t start, size_t len, + uoff_t start, size_t len, bool caching) { if (caching) @@ -615,7 +633,7 @@ static inline void fscache_clear_page_bits(struct address_space *mapping, */ static inline void fscache_write_to_cache(struct fscache_cookie *cookie, struct address_space *mapping, - loff_t start, size_t len, loff_t i_size, + uoff_t start, size_t len, uoff_t i_size, netfs_io_terminated_t term_func, void *term_func_priv, bool using_pgpriv2, bool caching) diff --git a/include/linux/fsl/bestcomm/gen_bd.h b/include/linux/fsl/bestcomm/gen_bd.h index aeb312a1cd00..7253d308d77a 100644 --- a/include/linux/fsl/bestcomm/gen_bd.h +++ b/include/linux/fsl/bestcomm/gen_bd.h @@ -16,10 +16,6 @@ struct bcom_gen_bd { }; -extern struct bcom_task * -bcom_gen_bd_rx_init(int queue_len, phys_addr_t fifo, - int initiator, int ipr, int maxbufsize); - extern int bcom_gen_bd_rx_reset(struct bcom_task *tsk); @@ -27,10 +23,6 @@ extern void bcom_gen_bd_rx_release(struct bcom_task *tsk); -extern struct bcom_task * -bcom_gen_bd_tx_init(int queue_len, phys_addr_t fifo, - int initiator, int ipr); - extern int bcom_gen_bd_tx_reset(struct bcom_task *tsk); diff --git a/include/linux/fsl/mc.h b/include/linux/fsl/mc.h index c25f0f7e6dd4..0b2c0eb9b605 100644 --- a/include/linux/fsl/mc.h +++ b/include/linux/fsl/mc.h @@ -336,7 +336,7 @@ struct fsl_mc_io { u16 flags; u32 portal_size; phys_addr_t portal_phys_addr; - void __iomem *portal_virt_addr; + void __iomem *portal_virt_addr __counted_by_ptr(portal_size); struct fsl_mc_device *dpmcp_dev; union { /* diff --git a/include/linux/fsnotify_backend.h b/include/linux/fsnotify_backend.h index 618eed4d6d72..a6d5c1a89cff 100644 --- a/include/linux/fsnotify_backend.h +++ b/include/linux/fsnotify_backend.h @@ -566,6 +566,7 @@ struct fsnotify_mark_connector { unsigned char prio; /* Highest priority group */ #define FSNOTIFY_CONN_FLAG_IS_WATCHED 0x01 #define FSNOTIFY_CONN_FLAG_HAS_IREF 0x02 +#define FSNOTIFY_CONN_FLAG_DESTROYING 0x04 unsigned short flags; /* flags [lock] */ union { /* Object pointer [lock] */ diff --git a/include/linux/fsverity.h b/include/linux/fsverity.h index 6c467ded9751..61fe96ccdff6 100644 --- a/include/linux/fsverity.h +++ b/include/linux/fsverity.h @@ -54,11 +54,13 @@ struct fsverity_operations { * @merkle_tree_size: total bytes the Merkle tree took up * * If desc == NULL, then enabling verity failed and the filesystem only - * must do any necessary cleanups. Else, it must also store the given - * verity descriptor to a fs-specific location associated with the inode - * and do any fs-specific actions needed to mark the inode as a verity - * inode, e.g. setting a bit in the on-disk inode. The filesystem is - * also responsible for setting the S_VERITY flag in the VFS inode. + * must do any necessary cleanups. + * + * Else, the filesystem must store the given verity descriptor to a + * fs-specific location associated with the inode and do any fs-specific + * actions needed to mark the inode as a verity inode, e.g. set a bit in + * the on-disk inode. Finally, on success (only when rollback is no + * longer possible!) it must set S_VERITY in the VFS inode. * * i_rwsem is held for write, but it may have been dropped between * ->begin_enable_verity() and ->end_enable_verity(). @@ -124,6 +126,8 @@ struct fsverity_operations { * @buf: the Merkle tree block to write * @pos: the position of the block in the Merkle tree (in bytes) * @size: the Merkle tree block size (in bytes) + * @zero_digest: the hash of the all-zeroes block + * @digest_size: size of zero_digest, in bytes * * This is only called between ->begin_enable_verity() and * ->end_enable_verity(). @@ -131,7 +135,9 @@ struct fsverity_operations { * Return: 0 on success, -errno on failure */ int (*write_merkle_tree_block)(struct file *file, const void *buf, - u64 pos, unsigned int size); + u64 pos, unsigned int size, + const u8 *zero_digest, + unsigned int digest_size); }; #ifdef CONFIG_FS_VERITY @@ -317,10 +323,14 @@ static inline int fsverity_file_open(struct inode *inode, struct file *filp) return 0; } +int fsverity_ensure_verity_info(struct inode *inode); + void fsverity_cleanup_inode(struct inode *inode); struct page *generic_read_merkle_tree_page(struct inode *inode, pgoff_t index); void generic_readahead_merkle_tree(struct inode *inode, pgoff_t index, unsigned long nr_pages); +int fsverity_pagecache_read(struct inode *inode, void *buf, size_t count, + loff_t pos); #endif /* _LINUX_FSVERITY_H */ diff --git a/include/linux/ftrace.h b/include/linux/ftrace.h index bd76a16a63af..b041d9142d2b 100644 --- a/include/linux/ftrace.h +++ b/include/linux/ftrace.h @@ -416,6 +416,7 @@ void free_ftrace_hash(struct ftrace_hash *hash); struct ftrace_func_entry *add_ftrace_hash_entry_direct(struct ftrace_hash *hash, unsigned long ip, unsigned long direct); void add_ftrace_hash_entry(struct ftrace_hash *hash, struct ftrace_func_entry *entry); +struct ftrace_func_entry *ftrace_lookup_ip(struct ftrace_hash *hash, unsigned long ip); void ftrace_hash_remove(struct ftrace_hash *hash); /* The hash used to know what functions callbacks trace */ diff --git a/include/linux/futex.h b/include/linux/futex.h index 18ed18d5cbc1..a895d8c69d66 100644 --- a/include/linux/futex.h +++ b/include/linux/futex.h @@ -23,28 +23,32 @@ struct task_struct; * 01 : Shared futex (PTHREAD_PROCESS_SHARED) * mapped on a file (reference on the underlying inode) * 10 : Shared futex (PTHREAD_PROCESS_SHARED) - * (but private mapping on an mm, and reference taken on it) + * (but private mapping on an mm) */ -#define FUT_OFF_INODE 1 /* We set bit 0 if key has a reference on inode */ -#define FUT_OFF_MMSHARED 2 /* We set bit 1 if key has a reference on mm */ +#define FUT_OFF_INODE 1 /* We set bit 0 if shared key identifies an inode */ +#define FUT_OFF_MMSHARED 2 /* We set bit 1 if shared key identifies an mm */ union futex_key { + /* For FUT_OFF_INODE */ struct { u64 i_seq; unsigned long pgoff; unsigned int offset; /* unsigned int node; */ } shared; + + /* For FUT_OFF_MMSHARED or private */ struct { union { - struct mm_struct *mm; - u64 __tmp; + struct mm_struct *mm; /* for private */ + u64 mm_seq; /* for FUT_OFF_MMSHARED */ }; unsigned long address; unsigned int offset; /* unsigned int node; */ } private; + struct { u64 ptr; unsigned long word; @@ -152,11 +156,10 @@ static inline void futex_set_vdso_cs_range(struct futex_mm_data *fd, unsigned in static inline void futex_fixup_robust_unlock(struct pt_regs *regs) { } #endif /* !CONFIG_FUTEX_ROBUST_UNLOCK */ - -#if defined(CONFIG_FUTEX_PRIVATE_HASH) || defined(CONFIG_FUTEX_ROBUST_UNLOCK) +#ifdef CONFIG_FUTEX void futex_mm_init(struct mm_struct *mm); #else static inline void futex_mm_init(struct mm_struct *mm) { } -#endif +#endif /* !CONFIG_FUTEX */ #endif /* _LINUX_FUTEX_H */ diff --git a/include/linux/futex_types.h b/include/linux/futex_types.h index d320c0571f0c..34b16f2828e7 100644 --- a/include/linux/futex_types.h +++ b/include/linux/futex_types.h @@ -85,10 +85,12 @@ struct futex_unlock_cs_ranges { }; * struct futex_mm_data - Futex related per MM data * @phash: Futex private hash related data * @unlock: Futex unlock VDSO critical sections + * @unique_id: Unique ID of the MM, for FUT_OFF_MMSHARED futexes */ struct futex_mm_data { struct futex_mm_phash phash; struct futex_unlock_cs_ranges unlock; + atomic64_t unique_id; }; #else /* CONFIG_FUTEX */ struct futex_sched_data { }; diff --git a/include/linux/generic_pt/common.h b/include/linux/generic_pt/common.h index 07ef1c8341a4..1b39b27f0cfa 100644 --- a/include/linux/generic_pt/common.h +++ b/include/linux/generic_pt/common.h @@ -213,4 +213,10 @@ enum { PT_FEAT_X86_64_AMD_ENCRYPT_TABLES = PT_FEAT_FMT_START, }; +struct pt_bcm2712 { + struct pt_common common; + u8 bigpage_lg2; + u8 superpage_lg2; +}; + #endif diff --git a/include/linux/generic_pt/iommu.h b/include/linux/generic_pt/iommu.h index dd0edd02a48a..13ebb72a67de 100644 --- a/include/linux/generic_pt/iommu.h +++ b/include/linux/generic_pt/iommu.h @@ -346,6 +346,18 @@ struct pt_iommu_x86_64_hw_info { IOMMU_FORMAT(x86_64, x86_64_pt); +struct pt_iommu_bcm2712_cfg { + struct pt_iommu_cfg common; + u8 bigpage_lg2; + u8 superpage_lg2; +}; + +struct pt_iommu_bcm2712_hw_info { + phys_addr_t pt_base; +}; + +IOMMU_FORMAT(bcm2712, bcm2712pt); + #undef IOMMU_PROTOTYPES #undef IOMMU_FORMAT #endif diff --git a/include/linux/gfp.h b/include/linux/gfp.h index 872bc53f32ec..63a7654c725f 100644 --- a/include/linux/gfp.h +++ b/include/linux/gfp.h @@ -13,10 +13,6 @@ struct vm_area_struct; struct mempolicy; -/* Helper macro to avoid gfp flags if they are the default one */ -#define __default_gfp(a,b,...) b -#define default_gfp(...) __default_gfp(,##__VA_ARGS__,GFP_KERNEL) - static inline bool gfpflags_allow_blocking(const gfp_t gfp_flags) { return !!(gfp_flags & __GFP_DIRECT_RECLAIM); @@ -329,7 +325,7 @@ void *alloc_pages_exact_noprof(size_t size, gfp_t gfp_mask) __alloc_size(1); void free_pages_exact(void *virt, size_t size); -__meminit void *alloc_pages_exact_nid_noprof(int nid, size_t size, gfp_t gfp_mask) __alloc_size(2); +void *alloc_pages_exact_nid_noprof(int nid, size_t size, gfp_t gfp_mask) __alloc_size(2); #define alloc_pages_exact_nid(...) \ alloc_hooks(alloc_pages_exact_nid_noprof(__VA_ARGS__)) diff --git a/include/linux/gfp_types.h b/include/linux/gfp_types.h index 190191411009..c1331bf0b239 100644 --- a/include/linux/gfp_types.h +++ b/include/linux/gfp_types.h @@ -244,7 +244,7 @@ enum { * definitely preferable to use the flag rather than opencode endless * loop around allocator. * Allocating pages from the buddy with __GFP_NOFAIL and order > 1 is - * not supported. Please consider using kvmalloc() instead. + * discouraged. Please consider using kvmalloc() instead if possible. */ #define __GFP_IO ((__force gfp_t)___GFP_IO) #define __GFP_FS ((__force gfp_t)___GFP_FS) @@ -382,4 +382,8 @@ enum { __GFP_NOMEMALLOC | __GFP_NOWARN) & ~__GFP_RECLAIM) #define GFP_TRANSHUGE (GFP_TRANSHUGE_LIGHT | __GFP_DIRECT_RECLAIM) +/* Helper macro to avoid gfp flags if they are the default one */ +#define __default_gfp(a,b,...) b +#define default_gfp(...) __default_gfp(,##__VA_ARGS__,GFP_KERNEL) + #endif /* __LINUX_GFP_TYPES_H */ diff --git a/include/linux/gpio/driver.h b/include/linux/gpio/driver.h index 17511434ed07..4077dc678cae 100644 --- a/include/linux/gpio/driver.h +++ b/include/linux/gpio/driver.h @@ -359,6 +359,10 @@ struct gpio_irq_chip { * @set_config: optional hook for all kinds of settings. Uses the same * packed config format as generic pinconf. Must return 0 on success and * a negative error number on failure. + * @get_config: optional hook to read back a setting. Takes the packed + * generic pinconf parameter to query and returns its bare argument in + * the same variable, like pinctrl_gpio_get_config(). Must return 0 on + * success and a negative error number on failure. * @to_irq: optional hook supporting non-static gpiod_to_irq() mappings; * implementation may not sleep * @dbg_show: optional routine to show contents in debugfs; default code @@ -434,6 +438,9 @@ struct gpio_chip { int (*set_config)(struct gpio_chip *gc, unsigned int offset, unsigned long config); + int (*get_config)(struct gpio_chip *gc, + unsigned int offset, + unsigned long *config); int (*to_irq)(struct gpio_chip *gc, unsigned int offset); @@ -708,6 +715,8 @@ int gpiochip_generic_request(struct gpio_chip *gc, unsigned int offset); void gpiochip_generic_free(struct gpio_chip *gc, unsigned int offset); int gpiochip_generic_config(struct gpio_chip *gc, unsigned int offset, unsigned long config); +int gpiochip_generic_get_config(struct gpio_chip *gc, unsigned int offset, + unsigned long *config); /** * struct gpio_pin_range - pin range controlled by a gpio chip diff --git a/include/linux/gpio/legacy.h b/include/linux/gpio/legacy.h index 557ef635935e..4001208d5d63 100644 --- a/include/linux/gpio/legacy.h +++ b/include/linux/gpio/legacy.h @@ -63,10 +63,6 @@ static inline int gpio_get_value_cansleep(unsigned gpio) { return gpiod_get_raw_value_cansleep(gpio_to_desc(gpio)); } -static inline void gpio_set_value_cansleep(unsigned gpio, int value) -{ - gpiod_set_raw_value_cansleep(gpio_to_desc(gpio), value); -} static inline int gpio_get_value(unsigned gpio) { @@ -148,12 +144,6 @@ static inline int gpio_get_value_cansleep(unsigned gpio) return 0; } -static inline void gpio_set_value_cansleep(unsigned gpio, int value) -{ - /* GPIO can never have been requested or set as output */ - WARN_ON(1); -} - static inline int gpio_to_irq(unsigned gpio) { /* GPIO can never have been requested or set as input */ diff --git a/include/linux/gpio/regmap.h b/include/linux/gpio/regmap.h index 6f52c140e50d..6fe0877717cb 100644 --- a/include/linux/gpio/regmap.h +++ b/include/linux/gpio/regmap.h @@ -46,6 +46,7 @@ enum gpio_regmap_operation { * @parent: The parent device * @regmap: The regmap used to access the registers * given, the name of the device is used + * @pm_dev: (Optional) Device to use for runtime power management. * @fwnode: (Optional) The firmware node. * If not given, the fwnode of the parent is used. * @label: (Optional) Descriptive name for GPIO controller. @@ -117,6 +118,7 @@ enum gpio_regmap_operation { struct gpio_regmap_config { struct device *parent; struct regmap *regmap; + struct device *pm_dev; struct fwnode_handle *fwnode; const char *label; diff --git a/include/linux/gpu_buddy.h b/include/linux/gpu_buddy.h index 2c36124bb696..ddc4eca84175 100644 --- a/include/linux/gpu_buddy.h +++ b/include/linux/gpu_buddy.h @@ -43,8 +43,8 @@ /** * GPU_BUDDY_CLEAR_ALLOCATION - Prefer pre-cleared (zeroed) memory * - * Attempt to allocate from the clear tree first. If insufficient clear - * memory is available, falls back to dirty memory. Useful when the + * Attempt to allocate outside dirty-tracked ranges first. If insufficient + * clear memory is available, falls back to dirty memory. Useful when the * caller needs zeroed memory and wants to avoid GPU clear operations. */ #define GPU_BUDDY_CLEAR_ALLOCATION BIT(3) @@ -53,8 +53,8 @@ * GPU_BUDDY_CLEARED - Mark returned blocks as cleared * * Used with gpu_buddy_free_list() to indicate that the memory being - * freed has been cleared (zeroed). The blocks will be placed in the - * clear tree for future GPU_BUDDY_CLEAR_ALLOCATION requests. + * freed has been cleared (zeroed). The blocks will be removed from the + * dirty tracker for future GPU_BUDDY_CLEAR_ALLOCATION requests. */ #define GPU_BUDDY_CLEARED BIT(4) @@ -67,15 +67,17 @@ */ #define GPU_BUDDY_TRIM_DISABLE BIT(5) -enum gpu_buddy_free_tree { - GPU_BUDDY_CLEAR_TREE = 0, - GPU_BUDDY_DIRTY_TREE, - GPU_BUDDY_MAX_FREE_TREES, +/* + * Clear/dirty state of a free block. Ordered so a numerically larger value + * is "more clear" (DIRTY < MIXED < CLEAR) which lets subtree_block_state be + * maintained as a simple max-augment over the per-order free tree. + */ +enum gpu_block_state { + GPU_BLOCK_DIRTY = 0, + GPU_BLOCK_MIXED = 1, + GPU_BLOCK_CLEAR = 2, }; -#define for_each_free_tree(tree) \ - for ((tree) = 0; (tree) < GPU_BUDDY_MAX_FREE_TREES; (tree)++) - /** * struct gpu_buddy_block - Block within a buddy allocator * @@ -103,6 +105,13 @@ struct gpu_buddy_block { #define GPU_BUDDY_ALLOCATED (1 << 10) #define GPU_BUDDY_FREE (2 << 10) #define GPU_BUDDY_SPLIT (3 << 10) +/* + * GPU_BUDDY_HEADER_CLEAR has two roles: + * - FREE state: set when the block's full range is cleared (dirty + * tracker confirmed no overlap). + * - ALLOCATED state: set when the block was served from cleared memory, + * informing the caller that no GPU clear pass is needed. + */ #define GPU_BUDDY_HEADER_CLEAR GENMASK_ULL(9, 9) /* Free to be used, if needed in the future */ #define GPU_BUDDY_HEADER_UNUSED GENMASK_ULL(8, 6) @@ -128,14 +137,45 @@ struct gpu_buddy_block { struct list_head link; }; /* private: */ - struct list_head tmp_link; + enum gpu_block_state subtree_block_state; unsigned int subtree_max_alignment; + struct list_head tmp_link; + bool has_clear; }; /* Order-zero must be at least SZ_4K */ #define GPU_BUDDY_MAX_ORDER (63 - 12) /** + * struct gpu_dirty_extent - a contiguous dirty address range + * + * Tracks a single contiguous address range whose memory content is known + * to be dirty. Extents are non-overlapping and stored in an augmented + * red-black tree sorted by @start. The augmented value @subtree_max_size + * allows O(log N) search for an extent of at least a given size. + */ +struct gpu_dirty_extent { +/* private: */ + struct rb_node rb; + u64 start; + u64 end; + u64 subtree_max_size; +}; + +/** + * struct gpu_dirty_tracker - tracks dirty address intervals + * + * Maintains a set of non-overlapping dirty extents as an augmented + * red-black tree. + */ +struct gpu_dirty_tracker { +/* private: */ + struct rb_root root; + /* Total bytes of dirty memory currently tracked. */ + u64 total_dirty; +}; + +/** * struct gpu_buddy - GPU binary buddy allocator * * The buddy allocator provides efficient power-of-two memory allocation @@ -152,20 +192,21 @@ struct gpu_buddy_block { * @chunk_size: Minimum allocation granularity in bytes. Must be at least SZ_4K. * @size: Total size of the address space managed by this allocator in bytes. * @avail: Total free space currently available for allocation in bytes. - * @clear_avail: Free space available in the clear tree (zeroed memory) in bytes. - * This is a subset of @avail. * @lock_dep_map: Annotates gpu_buddy API with a driver provided lock. */ struct gpu_buddy { /* private: */ + /* Tracker of dirty address ranges (decoupled from free_tree). */ + struct gpu_dirty_tracker dirty; /* - * Array of red-black trees for free block management. - * Indexed as free_trees[clear/dirty][order] where: - * - Index 0 (GPU_BUDDY_CLEAR_TREE): blocks with zeroed content - * - Index 1 (GPU_BUDDY_DIRTY_TREE): blocks with unknown content - * Each tree holds free blocks of the corresponding order. + * One RB-tree per order containing all free blocks (clear and + * dirty alike). The augment field subtree_block_state (a max over + * the subtree of each block's state) lets clear allocations + * find the right-most fully-clear or mixed block in O(log N). + * Dirty free blocks coexist here but are also indexed by the + * @dirty tracker for fast dirty allocation lookups. */ - struct rb_root **free_trees; + struct rb_root *free_tree; /* * Array of root blocks representing the top-level blocks of the * binary tree(s). Multiple roots exist when the total size is not @@ -194,7 +235,6 @@ struct gpu_buddy { u64 chunk_size; u64 size; u64 avail; - u64 clear_avail; #ifdef CONFIG_LOCKDEP struct lockdep_map *lock_dep_map; #endif @@ -226,17 +266,32 @@ struct gpu_buddy { * * Ensure driver lock is held. */ -static inline void gpu_buddy_driver_lock_held(struct gpu_buddy *mm) +static inline void gpu_buddy_driver_lock_held(const struct gpu_buddy *mm) { if (mm->lock_dep_map) lockdep_assert(lock_is_held_type(mm->lock_dep_map, 0)); } #else -static inline void gpu_buddy_driver_lock_held(struct gpu_buddy *mm) +static inline void gpu_buddy_driver_lock_held(const struct gpu_buddy *mm) { } #endif +/** + * gpu_buddy_clear_avail - free space that is clear (zeroed), in bytes + * @mm: gpu buddy allocator + * + * A subset of @mm->avail. Derived on demand as @mm->avail minus the bytes + * the dirty tracker records as dirty, so it is always consistent with the + * tracker without a cached field to keep in sync. Zero for a fresh pool, + * which is fully dirty. + */ +static inline u64 gpu_buddy_clear_avail(const struct gpu_buddy *mm) +{ + gpu_buddy_driver_lock_held(mm); + return mm->avail - mm->dirty.total_dirty; +} + static inline u64 gpu_buddy_block_offset(const struct gpu_buddy_block *block) { diff --git a/include/linux/greybus/bundle.h b/include/linux/greybus/bundle.h index df8d88424cb7..361a94d3499b 100644 --- a/include/linux/greybus/bundle.h +++ b/include/linux/greybus/bundle.h @@ -59,14 +59,9 @@ static inline int gb_pm_runtime_get_sync(struct gb_bundle *bundle) return 0; } -static inline int gb_pm_runtime_put_autosuspend(struct gb_bundle *bundle) +static inline void gb_pm_runtime_put_autosuspend(struct gb_bundle *bundle) { - int retval; - - pm_runtime_mark_last_busy(&bundle->dev); - retval = pm_runtime_put_autosuspend(&bundle->dev); - - return retval; + pm_runtime_put_autosuspend(&bundle->dev); } static inline void gb_pm_runtime_get_noresume(struct gb_bundle *bundle) @@ -82,8 +77,7 @@ static inline void gb_pm_runtime_put_noidle(struct gb_bundle *bundle) #else static inline int gb_pm_runtime_get_sync(struct gb_bundle *bundle) { return 0; } -static inline int gb_pm_runtime_put_autosuspend(struct gb_bundle *bundle) -{ return 0; } +static inline void gb_pm_runtime_put_autosuspend(struct gb_bundle *bundle) {} static inline void gb_pm_runtime_get_noresume(struct gb_bundle *bundle) {} static inline void gb_pm_runtime_put_noidle(struct gb_bundle *bundle) {} diff --git a/include/linux/hardirq.h b/include/linux/hardirq.h index 73b48dd9f135..2648c49cd8f6 100644 --- a/include/linux/hardirq.h +++ b/include/linux/hardirq.h @@ -108,7 +108,7 @@ DECLARE_PER_CPU(unsigned int, nmi_nesting); #define __preempt_count_nmi_enter() \ do { \ __preempt_count_add(HARDIRQ_OFFSET); \ - /* Maximum NMI nesting is 15. */ \ + /* NMI nesting is represented in 4 bits. */ \ BUG_ON(__this_cpu_read(nmi_nesting) >= 15); \ __this_cpu_inc(nmi_nesting); \ preempt_count_set(preempt_count() | NMI_MASK); \ diff --git a/include/linux/hdmi.h b/include/linux/hdmi.h index 8dab78e1f61b..b80a5ee63bb2 100644 --- a/include/linux/hdmi.h +++ b/include/linux/hdmi.h @@ -27,6 +27,18 @@ #include <linux/types.h> #include <linux/device.h> +enum hdmi_version { + HDMI_VERSION_UNKNOWN, + HDMI_VERSION_1_0, + HDMI_VERSION_1_1, + HDMI_VERSION_1_2, + HDMI_VERSION_1_3, + HDMI_VERSION_1_4, + HDMI_VERSION_2_0, + HDMI_VERSION_2_1, + HDMI_VERSION_2_2, +}; + enum hdmi_packet_type { HDMI_PACKET_TYPE_NULL = 0x00, HDMI_PACKET_TYPE_AUDIO_CLOCK_REGEN = 0x01, diff --git a/include/linux/hiddev.h b/include/linux/hiddev.h index 2164c03d2c72..8e9f8a33e359 100644 --- a/include/linux/hiddev.h +++ b/include/linux/hiddev.h @@ -13,6 +13,7 @@ #ifndef _HIDDEV_H #define _HIDDEV_H +#include <linux/refcount.h> #include <uapi/linux/hiddev.h> @@ -24,6 +25,7 @@ struct hiddev { int minor; int exist; int open; + refcount_t refcount; struct mutex existancelock; wait_queue_head_t wait; struct hid_device *hid; diff --git a/include/linux/hisi_acc_qm.h b/include/linux/hisi_acc_qm.h index f7570a409905..f2885d60b468 100644 --- a/include/linux/hisi_acc_qm.h +++ b/include/linux/hisi_acc_qm.h @@ -196,7 +196,7 @@ struct qm_dev_dfx { }; struct dfx_diff_registers { - u32 *regs; + u32 *regs __counted_by_ptr(reg_len); u32 reg_offset; u32 reg_len; }; @@ -332,9 +332,9 @@ struct hisi_qm_cap_record { struct hisi_qm_cap_tables { u32 qm_cap_size; - struct hisi_qm_cap_record *qm_cap_table; + struct hisi_qm_cap_record *qm_cap_table __counted_by_ptr(qm_cap_size); u32 dev_cap_size; - struct hisi_qm_cap_record *dev_cap_table; + struct hisi_qm_cap_record *dev_cap_table __counted_by_ptr(dev_cap_size); }; struct hisi_qm_list { diff --git a/include/linux/host1x.h b/include/linux/host1x.h index a7a675783136..e7ba601b2afb 100644 --- a/include/linux/host1x.h +++ b/include/linux/host1x.h @@ -237,6 +237,8 @@ void host1x_syncpt_release_vblank_reservation(struct host1x_client *client, struct dma_fence *host1x_fence_create(struct host1x_syncpt *sp, u32 threshold, bool timeout); void host1x_fence_cancel(struct dma_fence *fence); +int host1x_fence_extract(struct dma_fence *fence, struct host1x **host1x, + u32 *id, u32 *threshold); /* * host1x channel @@ -465,13 +467,26 @@ int host1x_client_resume(struct host1x_client *client); struct host1x_memory_context { struct host1x *host; + struct device *dev; /* Owning engine */ + struct pid *pid; + refcount_t ref; - struct pid *owner; + bool static_alloc; - struct device_dma_parameters dma_parms; - struct device dev; - u64 dma_mask; - u32 stream_id; + struct host1x_hw_memory_context *hw; + struct list_head entry; /* Entry in hw_memory_context's list */ + struct list_head mappings; /* List of mappings */ +}; + +struct host1x_context_mapping { + struct host1x *host; + + struct host1x_bo_mapping *mapping; + + struct host1x_bo *bo; + enum dma_data_direction direction; + + struct list_head entry; }; #ifdef CONFIG_IOMMU_API @@ -480,6 +495,12 @@ struct host1x_memory_context *host1x_memory_context_alloc(struct host1x *host1x, struct pid *pid); void host1x_memory_context_get(struct host1x_memory_context *cd); void host1x_memory_context_put(struct host1x_memory_context *cd); +int host1x_memory_context_active(struct host1x_memory_context *cd); +void host1x_memory_context_inactive(struct host1x_memory_context *cd); +struct host1x_context_mapping *host1x_memory_context_map(struct host1x_memory_context *ctx, + struct host1x_bo *bo, + enum dma_data_direction direction); +void host1x_memory_context_unmap(struct host1x_context_mapping *m); #else static inline struct host1x_memory_context *host1x_memory_context_alloc(struct host1x *host1x, struct device *dev, @@ -495,6 +516,26 @@ static inline void host1x_memory_context_get(struct host1x_memory_context *cd) static inline void host1x_memory_context_put(struct host1x_memory_context *cd) { } + +static inline int host1x_memory_context_active(struct host1x_memory_context *cd) +{ + return -ENODEV; +} + +static inline void host1x_memory_context_inactive(struct host1x_memory_context *cd) +{ +} + +static inline struct host1x_context_mapping * +host1x_memory_context_map(struct host1x_memory_context *ctx, struct host1x_bo *bo, + enum dma_data_direction direction) +{ + return ERR_PTR(-ENODEV); +} + +static inline void host1x_memory_context_unmap(struct host1x_context_mapping *m) +{ +} #endif #endif diff --git a/include/linux/hrtimer.h b/include/linux/hrtimer.h index 29072d89e5cb..acfc5f93d967 100644 --- a/include/linux/hrtimer.h +++ b/include/linux/hrtimer.h @@ -12,6 +12,7 @@ #ifndef _LINUX_HRTIMER_H #define _LINUX_HRTIMER_H +#include <linux/cleanup.h> #include <linux/hrtimer_rearm.h> #include <linux/hrtimer_types.h> #include <linux/init.h> @@ -72,7 +73,7 @@ enum hrtimer_mode { */ struct hrtimer_sleeper { struct hrtimer timer; - struct task_struct *task; + struct task_struct *__private task; }; static inline void hrtimer_set_expires(struct hrtimer *timer, ktime_t time) @@ -201,6 +202,29 @@ extern void destroy_hrtimer_on_stack(struct hrtimer *timer); static inline void destroy_hrtimer_on_stack(struct hrtimer *timer) { } #endif +/* + * The cleanup class pairs on-stack setup with debug-object teardown. Callers + * using it must not call destroy_hrtimer_on_stack() manually. + * + * This class provides debug-object teardown, not timer cancellation. Callers + * must ensure that no hrtimer operation can still access the timer before + * leaving the scope. In particular, the timer must no longer be queued and + * any running callback must have completed. + * + * Callers using hrtimer_setup_on_stack() without this class retain + * responsibility for destroying the timer before its backing storage leaves + * scope. + */ +DEFINE_CLASS(hrtimer_on_stack, struct hrtimer *, + destroy_hrtimer_on_stack(_T), + ({ + hrtimer_setup_on_stack(timer, function, clock_id, mode); + timer; + }), + struct hrtimer *timer, + enum hrtimer_restart (*function)(struct hrtimer *), + clockid_t clock_id, enum hrtimer_mode mode) + /* Basic timer operations: */ extern void hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim, u64 range_ns, const enum hrtimer_mode mode); @@ -320,6 +344,14 @@ extern int schedule_hrtimeout_range_clock(ktime_t *expires, const enum hrtimer_mode mode, clockid_t clock_id); extern int schedule_hrtimeout(ktime_t *expires, const enum hrtimer_mode mode); +static inline struct task_struct *hrtimer_sleeper_task_get(struct hrtimer_sleeper *sl) +{ + return READ_ONCE(ACCESS_PRIVATE(sl, task)); +} +static inline void hrtimer_sleeper_task_set(struct hrtimer_sleeper *sl, struct task_struct *t) +{ + WRITE_ONCE(ACCESS_PRIVATE(sl, task), t); +} /* Soft interrupt function to run the hrtimer queues: */ extern void hrtimer_run_queues(void); diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index 16c4c4caa126..5029c7241863 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -7,11 +7,11 @@ #include <linux/mm_types.h> #include <linux/mmdebug.h> #include <linux/fs.h> -#include <linux/hugetlb_inline.h> #include <linux/cgroup.h> #include <linux/page_ref.h> #include <linux/list.h> #include <linux/kref.h> +#include <linux/atomic.h> #include <linux/pgtable.h> #include <linux/gfp.h> #include <linux/userfaultfd_k.h> @@ -171,7 +171,6 @@ struct address_space *hugetlb_folio_mapping_lock_write(struct folio *folio); extern int movable_gigantic_pages __read_mostly; extern int sysctl_hugetlb_shm_group __read_mostly; -extern struct list_head huge_boot_pages[MAX_NUMNODES]; void hugetlb_bootmem_struct_page_init(void); void hugetlb_bootmem_alloc(void); @@ -253,14 +252,14 @@ extern void __hugetlb_zap_end(struct vm_area_struct *vma, static inline void hugetlb_zap_begin(struct vm_area_struct *vma, unsigned long *start, unsigned long *end) { - if (is_vm_hugetlb_page(vma)) + if (vma_is_hugetlb(vma)) __hugetlb_zap_begin(vma, start, end); } static inline void hugetlb_zap_end(struct vm_area_struct *vma, struct zap_details *details) { - if (is_vm_hugetlb_page(vma)) + if (vma_is_hugetlb(vma)) __hugetlb_zap_end(vma, details); } @@ -509,6 +508,9 @@ struct hugetlbfs_inode_info { struct inode vfs_inode; struct resv_map *resv_map; unsigned int seals; +#ifdef CONFIG_HUGETLB_PMD_PAGE_TABLE_SHARING + atomic64_t pmd_sharing_count; +#endif }; static inline struct hugetlbfs_inode_info *HUGETLBFS_I(struct inode *inode) @@ -516,6 +518,39 @@ static inline struct hugetlbfs_inode_info *HUGETLBFS_I(struct inode *inode) return container_of(inode, struct hugetlbfs_inode_info, vfs_inode); } +#ifdef CONFIG_HUGETLB_PMD_PAGE_TABLE_SHARING +static inline void hugetlbfs_pmd_sharing_init(struct inode *inode) +{ + atomic64_set(&HUGETLBFS_I(inode)->pmd_sharing_count, 0); +} + +static inline void hugetlbfs_pmd_sharing_inc(struct inode *inode) +{ + atomic64_inc(&HUGETLBFS_I(inode)->pmd_sharing_count); +} + +static inline void hugetlbfs_pmd_sharing_dec(struct inode *inode) +{ + atomic64_dec(&HUGETLBFS_I(inode)->pmd_sharing_count); +} + +static inline bool hugetlbfs_pmd_sharing_active(struct inode *inode) +{ + return atomic64_read(&HUGETLBFS_I(inode)->pmd_sharing_count) != 0; +} +#else +static inline void hugetlbfs_pmd_sharing_init(struct inode *inode) {} + +static inline void hugetlbfs_pmd_sharing_inc(struct inode *inode) {} + +static inline void hugetlbfs_pmd_sharing_dec(struct inode *inode) {} + +static inline bool hugetlbfs_pmd_sharing_active(struct inode *inode) +{ + return false; +} +#endif + extern const struct vm_operations_struct hugetlb_vm_ops; struct file *hugetlb_file_setup(const char *name, size_t size, vma_flags_t acct, int creat_flags, int page_size_log); @@ -676,10 +711,6 @@ struct hstate { char name[HSTATE_NAME_LEN]; }; -#define HUGE_BOOTMEM_HVO 0x0001 -#define HUGE_BOOTMEM_ZONES_VALID 0x0002 -#define HUGE_BOOTMEM_CMA 0x0004 - int isolate_or_dissolve_huge_folio(struct folio *folio, struct list_head *list); int replace_free_hugepage_folios(unsigned long start_pfn, unsigned long end_pfn); void wait_for_freed_hugetlb_folios(void); @@ -699,7 +730,8 @@ enum hugetlb_alloc_flag { #define HUGETLB_ALLOC_USE_GLOBAL_RESERVATIONS BIT(HUGETLB_ALLOC_USE_GLOBAL_RESERVATIONS_BIT) struct folio *hugetlb_alloc_folio(struct hstate *h, - struct mempolicy_interpreted *mpoli, u8 alloc_flags); + struct mempolicy_interpreted *mpoli, struct mm_struct *mm, + u8 alloc_flags); struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma, unsigned long addr, bool cow_from_owner); struct folio *alloc_hugetlb_folio_nodemask(struct hstate *h, int preferred_nid, diff --git a/include/linux/hugetlb_cgroup.h b/include/linux/hugetlb_cgroup.h index 16d72c8c71f6..b4e0958eabf3 100644 --- a/include/linux/hugetlb_cgroup.h +++ b/include/linux/hugetlb_cgroup.h @@ -268,4 +268,4 @@ static inline void hugetlb_cgroup_migrate(struct folio *old_folio, } #endif /* CONFIG_CGROUP_HUGETLB */ -#endif +#endif /* _LINUX_HUGETLB_CGROUP_H */ diff --git a/include/linux/hugetlb_inline.h b/include/linux/hugetlb_inline.h deleted file mode 100644 index 5c29cd3223a1..000000000000 --- a/include/linux/hugetlb_inline.h +++ /dev/null @@ -1,28 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _LINUX_HUGETLB_INLINE_H -#define _LINUX_HUGETLB_INLINE_H - -#include <linux/mm.h> - -#ifdef CONFIG_HUGETLB_PAGE - -static inline bool is_vma_hugetlb_flags(const vma_flags_t *flags) -{ - return vma_flags_test(flags, VMA_HUGETLB_BIT); -} - -#else - -static inline bool is_vma_hugetlb_flags(const vma_flags_t *flags) -{ - return false; -} - -#endif - -static inline bool is_vm_hugetlb_page(const struct vm_area_struct *vma) -{ - return is_vma_hugetlb_flags(&vma->flags); -} - -#endif diff --git a/include/linux/hwmon.h b/include/linux/hwmon.h index dd713e193d0c..a3a7d27f3b5f 100644 --- a/include/linux/hwmon.h +++ b/include/linux/hwmon.h @@ -134,6 +134,8 @@ enum hwmon_in_attributes { hwmon_in_max, hwmon_in_lcrit, hwmon_in_crit, + hwmon_in_lemergency, + hwmon_in_emergency, hwmon_in_average, hwmon_in_lowest, hwmon_in_highest, @@ -144,6 +146,8 @@ enum hwmon_in_attributes { hwmon_in_max_alarm, hwmon_in_lcrit_alarm, hwmon_in_crit_alarm, + hwmon_in_lemergency_alarm, + hwmon_in_emergency_alarm, hwmon_in_rated_min, hwmon_in_rated_max, hwmon_in_beep, @@ -156,6 +160,8 @@ enum hwmon_in_attributes { #define HWMON_I_MAX BIT(hwmon_in_max) #define HWMON_I_LCRIT BIT(hwmon_in_lcrit) #define HWMON_I_CRIT BIT(hwmon_in_crit) +#define HWMON_I_LEMERGENCY BIT(hwmon_in_lemergency) +#define HWMON_I_EMERGENCY BIT(hwmon_in_emergency) #define HWMON_I_AVERAGE BIT(hwmon_in_average) #define HWMON_I_LOWEST BIT(hwmon_in_lowest) #define HWMON_I_HIGHEST BIT(hwmon_in_highest) @@ -166,6 +172,8 @@ enum hwmon_in_attributes { #define HWMON_I_MAX_ALARM BIT(hwmon_in_max_alarm) #define HWMON_I_LCRIT_ALARM BIT(hwmon_in_lcrit_alarm) #define HWMON_I_CRIT_ALARM BIT(hwmon_in_crit_alarm) +#define HWMON_I_LEMERGENCY_ALARM BIT(hwmon_in_lemergency_alarm) +#define HWMON_I_EMERGENCY_ALARM BIT(hwmon_in_emergency_alarm) #define HWMON_I_RATED_MIN BIT(hwmon_in_rated_min) #define HWMON_I_RATED_MAX BIT(hwmon_in_rated_max) #define HWMON_I_BEEP BIT(hwmon_in_beep) @@ -178,6 +186,7 @@ enum hwmon_curr_attributes { hwmon_curr_max, hwmon_curr_lcrit, hwmon_curr_crit, + hwmon_curr_emergency, hwmon_curr_average, hwmon_curr_lowest, hwmon_curr_highest, @@ -188,6 +197,7 @@ enum hwmon_curr_attributes { hwmon_curr_max_alarm, hwmon_curr_lcrit_alarm, hwmon_curr_crit_alarm, + hwmon_curr_emergency_alarm, hwmon_curr_rated_min, hwmon_curr_rated_max, hwmon_curr_beep, @@ -199,6 +209,7 @@ enum hwmon_curr_attributes { #define HWMON_C_MAX BIT(hwmon_curr_max) #define HWMON_C_LCRIT BIT(hwmon_curr_lcrit) #define HWMON_C_CRIT BIT(hwmon_curr_crit) +#define HWMON_C_EMERGENCY BIT(hwmon_curr_emergency) #define HWMON_C_AVERAGE BIT(hwmon_curr_average) #define HWMON_C_LOWEST BIT(hwmon_curr_lowest) #define HWMON_C_HIGHEST BIT(hwmon_curr_highest) @@ -209,6 +220,7 @@ enum hwmon_curr_attributes { #define HWMON_C_MAX_ALARM BIT(hwmon_curr_max_alarm) #define HWMON_C_LCRIT_ALARM BIT(hwmon_curr_lcrit_alarm) #define HWMON_C_CRIT_ALARM BIT(hwmon_curr_crit_alarm) +#define HWMON_C_EMERGENCY_ALARM BIT(hwmon_curr_emergency_alarm) #define HWMON_C_RATED_MIN BIT(hwmon_curr_rated_min) #define HWMON_C_RATED_MAX BIT(hwmon_curr_rated_max) #define HWMON_C_BEEP BIT(hwmon_curr_beep) diff --git a/include/linux/hyperv.h b/include/linux/hyperv.h index 9e109d91aa14..e85a1a043eac 100644 --- a/include/linux/hyperv.h +++ b/include/linux/hyperv.h @@ -209,31 +209,63 @@ struct hv_ring_buffer_info { }; +/* + * The indices live in memory shared with the untrusted host, so check a + * snapshot of them before using it as an offset or to compute a byte count. + */ +static inline bool +hv_ringbuffer_indices_invalid(const struct hv_ring_buffer_info *rbi, + u32 read_loc, u32 write_loc) +{ + return read_loc >= rbi->ring_datasize || + write_loc >= rbi->ring_datasize; +} + +/* + * Byte counts for a caller-supplied snapshot of the indices, so that the + * counts and the indices the caller goes on to use describe one state of the + * ring. + */ +static inline u32 +hv_ringbuffer_avail_write(const struct hv_ring_buffer_info *rbi, + u32 read_loc, u32 write_loc) +{ + u32 dsize = rbi->ring_datasize; + + return write_loc >= read_loc ? dsize - (write_loc - read_loc) : + read_loc - write_loc; +} + +static inline u32 +hv_ringbuffer_avail_read(const struct hv_ring_buffer_info *rbi, + u32 read_loc, u32 write_loc) +{ + return rbi->ring_datasize - + hv_ringbuffer_avail_write(rbi, read_loc, write_loc); +} + static inline u32 hv_get_bytes_to_read(const struct hv_ring_buffer_info *rbi) { - u32 read_loc, write_loc, dsize, read; + u32 read_loc, write_loc; - dsize = rbi->ring_datasize; - read_loc = rbi->ring_buffer->read_index; + read_loc = READ_ONCE(rbi->ring_buffer->read_index); write_loc = READ_ONCE(rbi->ring_buffer->write_index); + if (unlikely(hv_ringbuffer_indices_invalid(rbi, read_loc, write_loc))) + return 0; - read = write_loc >= read_loc ? (write_loc - read_loc) : - (dsize - read_loc) + write_loc; - - return read; + return hv_ringbuffer_avail_read(rbi, read_loc, write_loc); } static inline u32 hv_get_bytes_to_write(const struct hv_ring_buffer_info *rbi) { - u32 read_loc, write_loc, dsize, write; + u32 read_loc, write_loc; - dsize = rbi->ring_datasize; read_loc = READ_ONCE(rbi->ring_buffer->read_index); - write_loc = rbi->ring_buffer->write_index; + write_loc = READ_ONCE(rbi->ring_buffer->write_index); + if (unlikely(hv_ringbuffer_indices_invalid(rbi, read_loc, write_loc))) + return 0; - write = write_loc >= read_loc ? dsize - (write_loc - read_loc) : - read_loc - write_loc; - return write; + return hv_ringbuffer_avail_write(rbi, read_loc, write_loc); } static inline u32 hv_get_avail_to_write_percent( @@ -1299,6 +1331,14 @@ struct hv_device { #define device_to_hv_device(d) container_of_const(d, struct hv_device, device) #define drv_to_hv_drv(d) container_of_const(d, struct hv_driver, driver) +static inline u32 hv_build_logical_dev_id_prefix(struct hv_device *hdev) +{ + return ((u32)hdev->dev_instance.b[5] << 24) | + ((u32)hdev->dev_instance.b[4] << 16) | + ((u32)hdev->dev_instance.b[7] << 8) | + (hdev->dev_instance.b[6] & 0xf8u); +} + static inline void hv_set_drvdata(struct hv_device *dev, void *data) { dev_set_drvdata(&dev->device, data); diff --git a/include/linux/i2c.h b/include/linux/i2c.h index 14ab4d3055af..6a847fc88d25 100644 --- a/include/linux/i2c.h +++ b/include/linux/i2c.h @@ -174,8 +174,23 @@ i2c_smbus_write_word_swapped(const struct i2c_client *client, } /* Returns the number of read bytes */ -s32 i2c_smbus_read_block_data(const struct i2c_client *client, - u8 command, u8 *values); +s32 __i2c_smbus_read_block_data(const struct i2c_client *client, + u8 command, u8 length, u8 *values); +/* + * This monstrosity allows to call i2c_smbus_read_block_data() with either + * 3 or 4 arguments and will be removed once all users have been switched + * to the 4 argument version. + */ +#define __i2c_smbus_read_block_data_3arg(client, cmd, values) \ + __i2c_smbus_read_block_data(client, cmd, I2C_SMBUS_BLOCK_MAX, values) +#define __i2c_smbus_read_block_data_4arg(client, cmd, length, values) \ + __i2c_smbus_read_block_data(client, cmd, length, values) +#define __i2c_smbus_read_block_data_impl(_1, _2, _3, _4, impl, ...) impl +#define i2c_smbus_read_block_data(client, cmd, varargs...) \ + __i2c_smbus_read_block_data_impl(client, cmd, varargs, \ + __i2c_smbus_read_block_data_4arg, \ + __i2c_smbus_read_block_data_3arg) \ + (client, cmd, varargs) s32 i2c_smbus_write_block_data(const struct i2c_client *client, u8 command, u8 length, const u8 *values); /* Returns the number of read bytes */ @@ -742,6 +757,9 @@ struct i2c_adapter { struct rt_mutex mux_lock; int timeout; /* in jiffies */ +#ifdef CONFIG_I2C_DYNAMIC_TIMEOUT + int user_timeout; /* I2C_TIMEOUT ioctl value in jiffies */ +#endif int retries; struct device dev; /* the adapter device */ unsigned long locked_flags; /* owned by the I2C core */ @@ -913,14 +931,24 @@ unsigned int i2c_adapter_depth(struct i2c_adapter *adapter); void i2c_parse_fw_timings(struct device *dev, struct i2c_timings *t, bool use_defaults); +#ifdef CONFIG_I2C_DYNAMIC_TIMEOUT +void i2c_update_timeout(struct i2c_adapter *adap, u32 bus_freq_hz, + size_t len, unsigned int safety_coeff, + unsigned int min_usec); +#else +static inline void i2c_update_timeout(struct i2c_adapter *adap, u32 bus_freq_hz, + size_t len, unsigned int safety_coeff, + unsigned int min_usec) {} +#endif + /* Return the functionality mask */ static inline u32 i2c_get_functionality(struct i2c_adapter *adap) { return adap->algo->functionality(adap); } -/* Return 1 if adapter supports everything we need, 0 if not. */ -static inline int i2c_check_functionality(struct i2c_adapter *adap, u32 func) +/* Return true if adapter supports everything we need, false if not. */ +static inline bool i2c_check_functionality(struct i2c_adapter *adap, u32 func) { return (func & i2c_get_functionality(adap)) == func; } diff --git a/include/linux/i3c/device.h b/include/linux/i3c/device.h index 0f065b883ee0..e30ea2cb13cb 100644 --- a/include/linux/i3c/device.h +++ b/include/linux/i3c/device.h @@ -79,6 +79,10 @@ struct i3c_xfer { enum i3c_error_code err; }; +/* For splitting 'cmd' from struct i3c_xfer */ +#define I3C_HDR_CMD_RNW BIT(7) +#define I3C_HDR_CMD_CODE GENMASK(6, 0) + /** * enum i3c_dcr - I3C DCR values * @I3C_DCR_GENERIC_DEVICE: generic I3C device diff --git a/include/linux/i3c/hub.h b/include/linux/i3c/hub.h new file mode 100644 index 000000000000..a368ea9e5ef7 --- /dev/null +++ b/include/linux/i3c/hub.h @@ -0,0 +1,92 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright 2026 NXP + * Generic hub definitions and helper interfaces. + */ +#ifndef _LINUX_I3C_HUB_H +#define _LINUX_I3C_HUB_H + +#include <linux/i3c/master.h> +#include <linux/mutex.h> + +/** + * struct i3c_hub - Generic I3C hub context + * @ops: Vendor callbacks for port connection control + * @hub_dev: I3C device representing the hub on the parent bus + * @lock: Serializes hub port routing/forwarding; its lockdep class is keyed + * per hub nesting depth in i3c_hub_init(). + */ +struct i3c_hub { + const struct i3c_hub_ops *ops; + struct i3c_device *hub_dev; + struct mutex lock; /* Serializes hub port routing. */ +}; + +struct i3c_hub_controller { + struct i3c_master_controller *parent; + struct i3c_master_controller controller; + struct i3c_hub *hub; +}; + +struct i3c_hub_ops { + void (*enable_port)(struct i3c_master_controller *controller); + void (*disable_port)(struct i3c_master_controller *controller); +}; + +/** + * i3c_hub_enable_port() - Enable hub connection for a controller + * @controller: Virtual controller representing a hub port + * + * Retrieves hub context from controller drvdata and invokes the vendor + * callback to enable the associated port connection. + */ +static inline void i3c_hub_enable_port(struct i3c_master_controller *controller) +{ + struct i3c_hub_controller *hub_controller; + struct i3c_hub *hub; + + hub_controller = dev_get_drvdata(&controller->dev); + if (!hub_controller || !hub_controller->hub) + return; + + hub = hub_controller->hub; + + if (hub && hub->ops && hub->ops->enable_port) + hub->ops->enable_port(controller); +} + +/** + * i3c_hub_disable_port() - Disable hub connection for a controller + * @controller: Virtual controller representing a hub port + * + * Retrieves hub context from controller drvdata and invokes the vendor + * callback to disable the associated port connection. + */ +static inline void i3c_hub_disable_port(struct i3c_master_controller *controller) +{ + struct i3c_hub_controller *hub_controller; + struct i3c_hub *hub; + + hub_controller = dev_get_drvdata(&controller->dev); + if (!hub_controller || !hub_controller->hub) + return; + + hub = hub_controller->hub; + + if (hub && hub->ops && hub->ops->disable_port) + hub->ops->disable_port(controller); +} + +/* + * Controller operations used by the virtual controllers created for hub + * target ports. Hub drivers pass this to i3c_master_register_fwnode(). + */ +extern const struct i3c_master_controller_ops i3c_hub_master_ops; + +void i3c_hub_init(struct i3c_hub *hub, + const struct i3c_hub_ops *ops, + struct i3c_device *hub_dev); + +int i3c_hub_reserve_parent_addrslots_from_dt(struct i3c_hub_controller *hubc, + struct device_node *node); +#endif diff --git a/include/linux/i3c/master.h b/include/linux/i3c/master.h index f7ceec2b4477..d58070a72a49 100644 --- a/include/linux/i3c/master.h +++ b/include/linux/i3c/master.h @@ -536,6 +536,8 @@ struct i3c_master_controller_ops { * Bit 0: SETDASA * Bit 1: SETAASA * All other bits are reserved. + * @instance: Zero-based instance number of the Bus Controller as defined by the + * DisCo specification I3C Target Address (_ADR) Encoding * @wq: freezable workqueue which can be used by master * drivers if they need to postpone operations that need to take place * in a thread context. Typical examples are Hot Join processing which @@ -573,6 +575,7 @@ struct i3c_master_controller { } boardinfo; struct i3c_bus bus; u8 addr_method; + u8 instance; struct workqueue_struct *wq; struct work_struct hj_work; struct work_struct reg_work; @@ -649,9 +652,18 @@ DEFINE_FREE(i3c_master_dma_unmap_single, void *, int i3c_master_reattach_i3c_dev_locked(struct i3c_dev_desc *dev, u8 old_dyn_addr); +int i3c_master_send_ccc_cmd(struct i3c_master_controller *master, + struct i3c_ccc_cmd *cmd); +bool i3c_master_supports_ccc_cmd(struct i3c_master_controller *master, + const struct i3c_ccc_cmd *cmd); int i3c_master_set_info(struct i3c_master_controller *master, const struct i3c_device_info *info); +int i3c_master_register_fwnode(struct i3c_master_controller *master, + struct device *parent, + struct fwnode_handle *fwnode, + const struct i3c_master_controller_ops *ops, + bool secondary); int i3c_master_register(struct i3c_master_controller *master, struct device *parent, const struct i3c_master_controller_ops *ops, @@ -775,4 +787,12 @@ void i3c_for_each_bus_locked(int (*fn)(struct i3c_bus *bus, void *data), int i3c_register_notifier(struct notifier_block *nb); int i3c_unregister_notifier(struct notifier_block *nb); +enum i3c_addr_slot_status +i3c_bus_get_addr_slot_status(struct i3c_bus *bus, u16 addr); + +void i3c_bus_set_addr_slot_status(struct i3c_bus *bus, u16 addr, + enum i3c_addr_slot_status status); + +void i3c_bus_maintenance_lock(struct i3c_bus *bus); +void i3c_bus_maintenance_unlock(struct i3c_bus *bus); #endif /* I3C_MASTER_H */ diff --git a/include/linux/idr.h b/include/linux/idr.h index 789e23e67444..e2a4b6298511 100644 --- a/include/linux/idr.h +++ b/include/linux/idr.h @@ -16,6 +16,7 @@ #include <linux/gfp.h> #include <linux/percpu.h> #include <linux/cleanup.h> +#include <linux/compiler.h> struct idr { struct radix_tree_root idr_rt; @@ -269,7 +270,9 @@ struct ida { #define IDA_INIT(name) { \ .xa = XARRAY_INIT(name, IDA_INIT_FLAGS) \ } -#define DEFINE_IDA(name) struct ida name = IDA_INIT(name) +#define DEFINE_IDA(name) \ + struct ida name = IDA_INIT(name); \ + ASSERT_STATIC_STORAGE(name) int ida_alloc_range(struct ida *, unsigned int min, unsigned int max, gfp_t); void ida_free(struct ida *, unsigned int id); diff --git a/include/linux/ieee80211-eht.h b/include/linux/ieee80211-eht.h index b62297a978e7..a699d108eee6 100644 --- a/include/linux/ieee80211-eht.h +++ b/include/linux/ieee80211-eht.h @@ -482,6 +482,7 @@ struct ieee80211_multi_link_elem { #define IEEE80211_MLC_BASIC_PRES_MLD_ID 0x0200 #define IEEE80211_MLC_BASIC_PRES_EXT_MLD_CAPA_OP 0x0400 #define IEEE80211_MLC_BASIC_PRES_ENH_CRIT_UPD 0x0800 +#define IEEE80211_MLC_BASIC_PRES_AGE_OF_BSS_LOAD 0x1000 #define IEEE80211_MED_SYNC_DELAY_DURATION 0x00ff #define IEEE80211_MED_SYNC_DELAY_SYNC_OFDM_ED_THRESH 0x0f00 @@ -550,6 +551,8 @@ struct ieee80211_mle_basic_common_info { } __packed; #define IEEE80211_MLC_PREQ_PRES_MLD_ID 0x0010 +#define IEEE80211_MLC_PREQ_PRES_MLD_MAC_ADDR 0x0020 +#define IEEE80211_MLC_PREQ_PRES_NEIGH_AP_MLD_MAC_ADDR 0x0040 struct ieee80211_mle_preq_common_info { u8 len; @@ -560,6 +563,7 @@ struct ieee80211_mle_preq_common_info { #define IEEE80211_MLC_RECONF_PRES_EML_CAPA 0x0020 #define IEEE80211_MLC_RECONF_PRES_MLD_CAPA_OP 0x0040 #define IEEE80211_MLC_RECONF_PRES_EXT_MLD_CAPA_OP 0x0080 +#define IEEE80211_MLC_RECONF_PRES_TARGET_AP_MLD_ADDR 0x0100 /* no fixed fields in RECONF */ @@ -766,7 +770,7 @@ static inline u16 ieee80211_mle_get_mld_capa_op(const u8 *data) #define IEEE80211_EHT_ML_EXT_MLD_CAPA_NSTR_UPDATE 0x0020 #define IEEE80211_EHT_ML_EXT_MLD_CAPA_EMLSR_ENA_ON_ONE_LINK 0x0040 #define IEEE80211_EHT_ML_EXT_MLD_CAPA_BTM_MLD_RECO_MULTI_AP 0x0080 -/* defined by UHR Draft P802.11bn_D1.3 Figure 9-1147 */ +/* defined by UHR Draft P802.11bn_D1.5 Figure 9-1168 */ #define IEEE80211_UHR_ML_EXT_MLD_CAPA_ML_PM 0x0100 /** @@ -928,11 +932,17 @@ static inline bool ieee80211_mle_size_ok(const u8 *data, size_t len) common += 2; if (control & IEEE80211_MLC_BASIC_PRES_ENH_CRIT_UPD) common += 1; + if (control & IEEE80211_MLC_BASIC_PRES_AGE_OF_BSS_LOAD) + common += 1; break; case IEEE80211_ML_CONTROL_TYPE_PREQ: common += sizeof(struct ieee80211_mle_preq_common_info); if (control & IEEE80211_MLC_PREQ_PRES_MLD_ID) common += 1; + if (control & IEEE80211_MLC_PREQ_PRES_MLD_MAC_ADDR) + common += ETH_ALEN; + if (control & IEEE80211_MLC_PREQ_PRES_NEIGH_AP_MLD_MAC_ADDR) + common += ETH_ALEN; break; case IEEE80211_ML_CONTROL_TYPE_RECONF: common += 1; @@ -944,6 +954,8 @@ static inline bool ieee80211_mle_size_ok(const u8 *data, size_t len) common += 2; if (control & IEEE80211_MLC_RECONF_PRES_EXT_MLD_CAPA_OP) common += 2; + if (control & IEEE80211_MLC_RECONF_PRES_TARGET_AP_MLD_ADDR) + common += ETH_ALEN; break; case IEEE80211_ML_CONTROL_TYPE_TDLS: common += sizeof(struct ieee80211_mle_tdls_common_info); @@ -1003,6 +1015,7 @@ enum ieee80211_mle_subelems { #define IEEE80211_MLE_STA_CONTROL_BSS_PARAM_CHANGE_CNT_PRESENT 0x0800 #define IEEE80211_MLE_STA_CONTROL_ENH_CRIT_UPD_PRESENT 0x1000 #define IEEE80211_MLE_STA_CONTROL_AP_CONDUCTED_TX_PWR_PRESENT 0x2000 +#define IEEE80211_MLE_STA_CONTROL_AGE_OF_BSS_LOAD_PRESENT 0x4000 struct ieee80211_mle_per_sta_profile { __le16 control; @@ -1054,6 +1067,9 @@ static inline bool ieee80211_mle_basic_sta_prof_size_ok(const u8 *data, if (control & IEEE80211_MLE_STA_CONTROL_AP_CONDUCTED_TX_PWR_PRESENT) info_len += 1; + if (control & IEEE80211_MLE_STA_CONTROL_AGE_OF_BSS_LOAD_PRESENT) + info_len += 1; + return prof->sta_info_len >= info_len && fixed + prof->sta_info_len - 1 <= len; } diff --git a/include/linux/ieee80211-nan.h b/include/linux/ieee80211-nan.h index 455033955e54..7815a6cf8160 100644 --- a/include/linux/ieee80211-nan.h +++ b/include/linux/ieee80211-nan.h @@ -40,6 +40,8 @@ /* NAN attributes, as defined in Wi-Fi Aware (TM) specification 4.0 Table 42 */ #define NAN_ATTR_MASTER_INDICATION 0x00 #define NAN_ATTR_CLUSTER_INFO 0x01 +#define NAN_ATTR_SERVICE_ID_LIST 0x02 +#define NAN_ATTR_SUBSCRIBE_SERVICE_ID_LIST 0x28 struct ieee80211_nan_attr { u8 attr; @@ -74,4 +76,30 @@ struct ieee80211_nan_anchor_master_info { _attr = (const struct ieee80211_nan_attr *) \ (_attr->data + le16_to_cpu(_attr->length))) +static inline bool ieee80211_is_nan_beacon(const struct ieee80211_mgmt *mgmt, + size_t len) +{ + const struct element *elem; + + /* The NAN IE is at least 6 octets */ + if (len < offsetofend(struct ieee80211_mgmt, u.beacon) + 6) + return false; + + if (!ieee80211_is_beacon(mgmt->frame_control)) + return false; + + /* NAN Cluster IDs range from 50-6F-9A-01-00-00 to 50-6F-9A-01-FF-FF */ + if (get_unaligned_be32(mgmt->bssid) != ((WLAN_OUI_WFA << 8) | 0x01)) + return false; + + elem = (const struct element *)mgmt->u.beacon.variable; + if (elem->id != WLAN_EID_VENDOR_SPECIFIC || + elem->datalen < 4 || + get_unaligned_be32(elem->data) != + (WLAN_OUI_WFA << 8 | WLAN_OUI_TYPE_WFA_NAN)) + return false; + + return true; +} + #endif /* LINUX_IEEE80211_NAN_H */ diff --git a/include/linux/ieee80211-uhr.h b/include/linux/ieee80211-uhr.h index 665d4b3a5b41..19e9260d0db8 100644 --- a/include/linux/ieee80211-uhr.h +++ b/include/linux/ieee80211-uhr.h @@ -17,32 +17,32 @@ #define IEEE80211_UHR_OPER_PARAMS_PEDCA_ENA 0x0004 #define IEEE80211_UHR_OPER_PARAMS_DBE_ENA 0x0008 #define IEEE80211_UHR_OPER_PARAMS_DBE_BW 0x0070 -#define IEEE80211_UHR_OPER_PARAMS_DUO_PRES 0x0080 -#define IEEE80211_UHR_OPER_PARAMS_DPS_PRES 0x0100 -#define IEEE80211_UHR_OPER_PARAMS_NPCA_PRES 0x0200 -#define IEEE80211_UHR_OPER_PARAMS_PEDCA_PRES 0x0400 -#define IEEE80211_UHR_OPER_PARAMS_DBE_PRES 0x0800 +#define IEEE80211_UHR_OPER_PARAMS_ELR_RX 0x0080 +#define IEEE80211_UHR_OPER_PARAMS_DUO_PRES 0x0100 +#define IEEE80211_UHR_OPER_PARAMS_DPS_PRES 0x0200 +#define IEEE80211_UHR_OPER_PARAMS_NPCA_PRES 0x0400 +#define IEEE80211_UHR_OPER_PARAMS_PEDCA_PRES 0x0800 +#define IEEE80211_UHR_OPER_PARAMS_DBE_PRES 0x1000 struct ieee80211_uhr_operation { __le16 params; - u8 basic_mcs_nss_set[4]; u8 variable[]; } __packed; -#define IEEE80211_UHR_NPCA_PARAMS_PRIMARY_CHAN_OFFS 0x0000000F -#define IEEE80211_UHR_NPCA_PARAMS_MIN_DUR_THRESH 0x000000F0 -#define IEEE80211_UHR_NPCA_PARAMS_SWITCH_DELAY 0x00003F00 -#define IEEE80211_UHR_NPCA_PARAMS_SWITCH_BACK_DELAY 0x000FC000 -#define IEEE80211_UHR_NPCA_PARAMS_INIT_QSRC 0x00300000 -#define IEEE80211_UHR_NPCA_PARAMS_MOPLEN 0x00400000 -#define IEEE80211_UHR_NPCA_PARAMS_DIS_SUBCH_BMAP_PRES 0x00800000 +#define IEEE80211_UHR_NPCA_PARAMS_PRIMARY_CHAN 0x000000FF +#define IEEE80211_UHR_NPCA_PARAMS_MIN_DUR_THRESH 0x00000F00 +#define IEEE80211_UHR_NPCA_PARAMS_SWITCH_DELAY 0x0003F000 +#define IEEE80211_UHR_NPCA_PARAMS_SWITCH_BACK_DELAY 0x00FC0000 +#define IEEE80211_UHR_NPCA_PARAMS_INIT_QSRC 0x03000000 +#define IEEE80211_UHR_NPCA_PARAMS_MOPLEN 0x04000000 +#define IEEE80211_UHR_NPCA_PARAMS_DIS_SUBCH_BMAP_PRES 0x08000000 /** * struct ieee80211_uhr_npca_info - npca operation information * * This structure is the "NPCA Operation Parameters field format" of "UHR - * Operation Element" fields as described in P802.11bn_D1.3 - * subclause 9.4.2.353. See Figure 9-aa4. + * Operation Element" fields as described in P802.11bn_D1.5 + * subclause 9.4.2.355.2, see Figure 9-aa6. * * Refer to IEEE80211_UHR_NPCA* * @params: @@ -97,7 +97,7 @@ struct ieee80211_uhr_npca_info { * struct ieee80211_uhr_dps_info - DPS operation information * * This structure is the "DPS Operation Parameter field" of "UHR - * Operation Element" fields as described in P802.11bn_D1.3 + * Operation Element" fields as described in P802.11bn_D1.5 * subclause 9.4.1.87. See Figure 9-207u. * * Refer to IEEE80211_UHR_DPS* @@ -211,8 +211,8 @@ static inline int ieee80211_uhr_dbe_bw_mhz(enum ieee80211_uhr_dbe_oper_bw bw) * struct ieee80211_uhr_dbe_info - DBE operation information * * This structure is the "DBE Operation Parameters field" of - * "UHR Operation Element" fields as described in P802.11bn_D1.3 - * subclause 9.4.2.353. See Figure 9-aa6. + * "UHR Operation Element" fields as described in P802.11bn_D1.5 + * subclause 9.4.2.355.4, see Figure 9-aa9. * * Refer to IEEE80211_UHR_DBE_OPER* * @params: @@ -234,25 +234,23 @@ struct ieee80211_uhr_dbe_info { __le16 dis_subch_bmap[]; } __packed; -#define IEEE80211_UHR_P_EDCA_ECWMIN 0x0F -#define IEEE80211_UHR_P_EDCA_ECWMAX 0xF0 -#define IEEE80211_UHR_P_EDCA_AIFSN 0x000F -#define IEEE80211_UHR_P_EDCA_CW_DS 0x0030 -#define IEEE80211_UHR_P_EDCA_PSRC_THRESHOLD 0x01C0 -#define IEEE80211_UHR_P_EDCA_QSRC_THRESHOLD 0x0600 +#define IEEE80211_UHR_P_EDCA_ECWMIN 0x0007 +#define IEEE80211_UHR_P_EDCA_ECWMAX 0x0038 +#define IEEE80211_UHR_P_EDCA_AIFSN 0x01C0 +#define IEEE80211_UHR_P_EDCA_CW_DS 0x0600 +#define IEEE80211_UHR_P_EDCA_PSRC_THRESHOLD 0x3800 +#define IEEE80211_UHR_P_EDCA_QSRC_THRESHOLD 0xC000 /** * struct ieee80211_uhr_p_edca_info - P-EDCA operation information * * This structure is the "P-EDCA Operation Parameters field" of - * "UHR Operation Element" fields as described in P802.11bn_D1.3 - * subclause 9.4.2.353. See Figure 9-aa5. + * "UHR Operation Element" fields as described in P802.11bn_D1.5 + * subclause 9.4.2.355.3, see Figure 9-aa8. * * Refer to IEEE80211_UHR_P_EDCA* - * @p_edca_ec: P-EDCA ECWmin and ECWmax. - * These fields indicate the CWmin and CWmax values used by a - * P-EDCA STA during P-EDCA contention. - * @params: AIFSN, CW DS, PSRC threshold, and QSRC threshold. + * @params: ECWmin, ECWmax, AIFSN, CW DS, PSRC threshold, and QSRC threshold. + * - CWmin/CWmax values used by a P-EDCA STA during P-EDCA contention. * - The AIFSN field indicates the AIFSN value used by a P-EDCA STA * during P-EDCA contention. * - The CW DS field indicates the value used for randomization of the @@ -266,7 +264,6 @@ struct ieee80211_uhr_dbe_info { * value 0 is reserved. */ struct ieee80211_uhr_p_edca_info { - u8 p_edca_ec; __le16 params; } __packed; @@ -302,7 +299,7 @@ static inline bool ieee80211_uhr_oper_size_ok(const u8 *data, u8 len) } } - /* P-EDCA Operation Parameters (fixed 3 bytes) */ + /* P-EDCA Operation Parameters */ if (oper->params & cpu_to_le16(IEEE80211_UHR_OPER_PARAMS_PEDCA_PRES)) { needed += sizeof(struct ieee80211_uhr_p_edca_info); if (len < needed) @@ -341,6 +338,9 @@ ieee80211_uhr_npca_info(const struct ieee80211_uhr_operation *oper) if (!(oper->params & cpu_to_le16(IEEE80211_UHR_OPER_PARAMS_NPCA_ENA))) return NULL; + if (oper->params & cpu_to_le16(IEEE80211_UHR_OPER_PARAMS_DUO_PRES)) + pos += 1; + if (oper->params & cpu_to_le16(IEEE80211_UHR_OPER_PARAMS_DPS_PRES)) pos += sizeof(struct ieee80211_uhr_dps_info); @@ -372,6 +372,9 @@ ieee80211_uhr_oper_dbe_info(const struct ieee80211_uhr_operation *oper) if (!(oper->params & cpu_to_le16(IEEE80211_UHR_OPER_PARAMS_DBE_ENA))) return NULL; + if (oper->params & cpu_to_le16(IEEE80211_UHR_OPER_PARAMS_DUO_PRES)) + pos += 1; + if (oper->params & cpu_to_le16(IEEE80211_UHR_OPER_PARAMS_DPS_PRES)) pos += sizeof(struct ieee80211_uhr_dps_info); @@ -415,28 +418,42 @@ ieee80211_uhr_oper_dbe_info(const struct ieee80211_uhr_operation *oper) #define IEEE80211_UHR_MAC_CAP2_UHR_OM_PU_TO_LOW 0xC0 #define IEEE80211_UHR_MAC_CAP3_UHR_OM_PU_TO_HIGH 0x03 -#define IEEE80211_UHR_MAC_CAP3_PARAM_UPD_ADV_NOTIF_INTV 0x1C -#define IEEE80211_UHR_MAC_CAP3_UPD_IND_TIM_INTV_LOW 0xE0 +#define IEEE80211_UHR_MAC_CAP3_PARAM_UPD_ADV_NOTIF_INTV 0x7C +#define IEEE80211_UHR_MAC_CAP3_UPD_IND_TIM_INTV_LOW 0x80 -#define IEEE80211_UHR_MAC_CAP4_UPD_IND_TIM_INTV_HIGH 0x03 -#define IEEE80211_UHR_MAC_CAP4_BOUNDED_ESS 0x04 -#define IEEE80211_UHR_MAC_CAP4_BTM_ASSURANCE 0x08 -#define IEEE80211_UHR_MAC_CAP4_CO_BF_SUPP 0x10 +#define IEEE80211_UHR_MAC_CAP4_UPD_IND_TIM_INTV_HIGH 0x0F +#define IEEE80211_UHR_MAC_CAP4_BOUNDED_ESS 0x10 +#define IEEE80211_UHR_MAC_CAP4_BTM_ASSURANCE 0x20 +#define IEEE80211_UHR_MAC_CAP4_CO_BF_SUPP 0x40 +#define IEEE80211_UHR_MAC_CAP4_CO_SR_SUPP 0x80 + +#define IEEE80211_UHR_MAC_CAP5_MAPC_ENH_MSRMT_SUPP 0x01 #define IEEE80211_UHR_MAC_CAP_DBE_MAX_BW 0x07 #define IEEE80211_UHR_MAC_CAP_DBE_EHT_MCS_MAP_160_PRES 0x08 #define IEEE80211_UHR_MAC_CAP_DBE_EHT_MCS_MAP_320_PRES 0x10 +/* struct ieee80211_uhr_cap_dbe::bwcap[].cap */ +#define IEEE80211_UHR_MAC_CAP_DBE_CAP_NUM_SND_DIMS 0x07 +#define IEEE80211_UHR_MAC_CAP_DBE_CAP_NON_OFDMA_UL_MUMIMO 0x08 +#define IEEE80211_UHR_MAC_CAP_DBE_CAP_MU_BEAMFORMER 0x10 +#define IEEE80211_UHR_MAC_CAP_DBE_CAP_BEAMFORMEE_SS 0xe0 + struct ieee80211_uhr_cap_dbe { u8 cap; + u8 max_switch_time_period; + u8 mode_change_intvl; /* present 0, 1 or 2 times depending on _PRES bits */ - struct ieee80211_eht_mcs_nss_supp_bw eht_mcs_map[]; + struct { + struct ieee80211_eht_mcs_nss_supp_bw eht_mcs_map; + u8 cap; + } __packed bwcap[]; } __packed; /** * enum ieee80211_uhr_dbe_max_supported_bw - DBE Maximum Supported Bandwidth * - * As per spec P802.11bn_D1.3 "Table 9-bb5—Encoding of the DBE Maximum + * As per spec P802.11bn_D1.5 Table 9-bb8 "Encoding of the DBE Maximum * Supported Bandwidth field". * * @IEEE80211_UHR_DBE_MAX_BW_40: Indicates 40 MHz DBE max supported bw @@ -520,10 +537,10 @@ static inline bool ieee80211_uhr_capa_size_ok(const u8 *data, u8 len, dbe = (const void *)cap->variable; if (dbe->cap & IEEE80211_UHR_MAC_CAP_DBE_EHT_MCS_MAP_160_PRES) - needed += sizeof(dbe->eht_mcs_map[0]); + needed += sizeof(dbe->bwcap[0]); if (dbe->cap & IEEE80211_UHR_MAC_CAP_DBE_EHT_MCS_MAP_320_PRES) - needed += sizeof(dbe->eht_mcs_map[0]); + needed += sizeof(dbe->bwcap[0]); } return len >= needed; diff --git a/include/linux/ieee80211.h b/include/linux/ieee80211.h index 26e674038865..66f9ebe79c61 100644 --- a/include/linux/ieee80211.h +++ b/include/linux/ieee80211.h @@ -1828,6 +1828,7 @@ enum ieee80211_eid_ext { WLAN_EID_EXT_BANDWIDTH_INDICATION = 135, WLAN_EID_EXT_KNOWN_STA_IDENTIFCATION = 136, WLAN_EID_EXT_NON_AP_STA_REG_CON = 137, + WLAN_EID_EXT_CIP_CAPA = 150, WLAN_EID_EXT_UHR_OPER = 151, WLAN_EID_EXT_UHR_CAPA = 152, WLAN_EID_EXT_MACP = 153, @@ -2421,8 +2422,7 @@ static inline bool ieee80211_is_bufferable_mmpdu(struct sk_buff *skb) /* action frame - additionally check for non-bufferable FTM */ - if (mgmt->u.action.category != WLAN_CATEGORY_PUBLIC && - mgmt->u.action.category != WLAN_CATEGORY_PROTECTED_DUAL_OF_ACTION) + if (mgmt->u.action.category != WLAN_CATEGORY_PUBLIC) return true; if (mgmt->u.action.action_code == WLAN_PUB_ACTION_FTM_REQUEST || @@ -2730,6 +2730,10 @@ struct element { u8 data[]; } __packed; +/* trigger enum comparison warnings if applicable (constant id) */ +#define element_iteration_typecheck(type, id) \ + (((type)0 == (id)) || true) + /* element iteration helpers */ #define for_each_element(_elem, _data, _datalen) \ for (_elem = (const struct element *)(_data); \ @@ -2741,11 +2745,15 @@ struct element { #define for_each_element_id(element, _id, data, datalen) \ for_each_element(element, data, datalen) \ - if (element->id == (_id)) + if (element_iteration_typecheck(enum ieee80211_eid, \ + _id) && \ + element->id == (_id)) #define for_each_element_extid(element, extid, _data, _datalen) \ for_each_element(element, _data, _datalen) \ - if (element->id == WLAN_EID_EXTENSION && \ + if (element_iteration_typecheck(enum ieee80211_eid_ext, \ + extid) && \ + element->id == WLAN_EID_EXTENSION && \ element->datalen > 0 && \ element->data[0] == (extid)) @@ -2753,10 +2761,11 @@ struct element { for_each_element(sub, (element)->data, (element)->datalen) #define for_each_subelement_id(sub, id, element) \ - for_each_element_id(sub, id, (element)->data, (element)->datalen) + for_each_element_id(sub, (u8)id, (element)->data, (element)->datalen) #define for_each_subelement_extid(sub, extid, element) \ - for_each_element_extid(sub, extid, (element)->data, (element)->datalen) + for_each_element_extid(sub, (u8)extid, (element)->data, \ + (element)->datalen) /** * for_each_element_completed - determine if element parsing consumed all data @@ -2900,4 +2909,25 @@ static inline bool ieee80211_check_tim(const struct ieee80211_tim_ie *tim, __ieee80211_check_tim(tim, tim_len, aid); } +/** + * enum ieee80211_cip_cap_fields - CIP Capabilities element fields + * @IEEE80211_CIP_CAP_MIC_PADDING: The MIC padding subfield in the CIP + * capabilities + * @IEEE80211_CIP_CAP_PROTECTED_CTRL_FRAME_ONLY: The MIC Padding For Protected + * Control Frames Only bit in CIP capabilities + */ +enum ieee80211_cip_cap_fields { + IEEE80211_CIP_CAP_MIC_PADDING = 0x0F, + IEEE80211_CIP_CAP_PROTECTED_CTRL_FRAME_ONLY = 0x10, +}; + +/** + * struct ieee80211_cip_cap - CIP Capabilities element + * + * @v: The value of the Control Integrity Protocol Capability element + */ +struct ieee80211_cip_cap { + u8 v; +} __packed; + #endif /* LINUX_IEEE80211_H */ diff --git a/include/linux/if_vlan.h b/include/linux/if_vlan.h index 4846032bf4ff..c863cdf60f01 100644 --- a/include/linux/if_vlan.h +++ b/include/linux/if_vlan.h @@ -427,8 +427,8 @@ static inline int __vlan_insert_tag(struct sk_buff *skb, * Inserts the VLAN tag into @skb as part of the payload at offset mac_len * Returns a VLAN tagged skb. This might change skb->head. * - * Following the skb_unshare() example, in case of error, the calling function - * doesn't have to worry about freeing the original skb. + * Following the skb_unclone_full() example, in case of error, the calling + * function doesn't have to worry about freeing the original skb. * * Does not change skb->protocol so this function can be used during receive. * @@ -458,8 +458,8 @@ static inline struct sk_buff *vlan_insert_inner_tag(struct sk_buff *skb, * Inserts the VLAN tag into @skb as part of the payload * Returns a VLAN tagged skb. This might change skb->head. * - * Following the skb_unshare() example, in case of error, the calling function - * doesn't have to worry about freeing the original skb. + * Following the skb_unclone_full() example, in case of error, the calling + * function doesn't have to worry about freeing the original skb. * * Does not change skb->protocol so this function can be used during receive. * @@ -480,8 +480,8 @@ static inline struct sk_buff *vlan_insert_tag(struct sk_buff *skb, * Inserts the VLAN tag into @skb as part of the payload * Returns a VLAN tagged skb. This might change skb->head. * - * Following the skb_unshare() example, in case of error, the calling function - * doesn't have to worry about freeing the original skb. + * Following the skb_unclone_full() example, in case of error, the calling + * function doesn't have to worry about freeing the original skb. * * Return: modified @skb on success, NULL on error (@skb is freed). */ @@ -524,8 +524,8 @@ static inline void __vlan_hwaccel_copy_tag(struct sk_buff *dst, const struct sk_ * * Pushes the VLAN tag from @skb->vlan_tci inside to the payload. * - * Following the skb_unshare() example, in case of error, the calling function - * doesn't have to worry about freeing the original skb. + * Following the skb_unclone_full() example, in case of error, the calling + * function doesn't have to worry about freeing the original skb. */ static inline struct sk_buff *__vlan_hwaccel_push_inside(struct sk_buff *skb) { @@ -698,7 +698,6 @@ static inline void vlan_set_encap_proto(struct sk_buff *skb, struct vlan_hdr *vhdr) { __be16 proto; - unsigned short *rawp; /* * Was a VLAN packet, grab the encapsulated protocol, which the layer @@ -711,21 +710,8 @@ static inline void vlan_set_encap_proto(struct sk_buff *skb, return; } - rawp = (unsigned short *)(vhdr + 1); - if (*rawp == 0xFFFF) - /* - * This is a magic hack to spot IPX packets. Older Novell - * breaks the protocol design and runs IPX over 802.3 without - * an 802.2 LLC layer. We look for FFFF which isn't a used - * 802.2 SSAP/DSAP. This won't work for fault tolerant netware - * but does for the rest. - */ - skb->protocol = htons(ETH_P_802_3); - else - /* - * Real 802.2 LLC - */ - skb->protocol = htons(ETH_P_802_2); + /* No ethertype: this is an 802.2 LLC frame (length field). */ + skb->protocol = htons(ETH_P_802_2); } /** diff --git a/include/linux/igmp.h b/include/linux/igmp.h index a0cf0398519f..e075611344ef 100644 --- a/include/linux/igmp.h +++ b/include/linux/igmp.h @@ -14,6 +14,7 @@ #include <linux/timer.h> #include <linux/in.h> #include <linux/ip.h> +#include <linux/net.h> #include <linux/refcount.h> #include <linux/sockptr.h> #include <uapi/linux/igmp.h> @@ -273,7 +274,7 @@ extern int ip_mc_source(int add, int omode, struct sock *sk, struct ip_mreq_source *mreqs, int ifindex); extern int ip_mc_msfilter(struct sock *sk, struct ip_msfilter *msf,int ifindex); extern int ip_mc_msfget(struct sock *sk, struct ip_msfilter *msf, - sockptr_t optval, sockptr_t optlen); + sockopt_t *opt); extern int ip_mc_gsfget(struct sock *sk, struct group_filter *gsf, sockptr_t optval, size_t offset); extern int ip_mc_sf_allow(const struct sock *sk, __be32 local, __be32 rmt, diff --git a/include/linux/iio/adc/qcom-adc5-gen3-common.h b/include/linux/iio/adc/qcom-adc5-gen3-common.h index 39cbfcbdb101..b3e2c2f50e8f 100644 --- a/include/linux/iio/adc/qcom-adc5-gen3-common.h +++ b/include/linux/iio/adc/qcom-adc5-gen3-common.h @@ -137,7 +137,7 @@ struct adc5_sdam_data { */ struct adc5_device_data { struct regmap *regmap; - struct adc5_sdam_data *base; + struct adc5_sdam_data *base __counted_by_ptr(num_sdams); int num_sdams; }; diff --git a/include/linux/iio/backend.h b/include/linux/iio/backend.h index 3f95ed1fdf9e..9c16b695a602 100644 --- a/include/linux/iio/backend.h +++ b/include/linux/iio/backend.h @@ -133,6 +133,8 @@ enum iio_backend_capabilities { * @filter_type_set: Set filter type. * @interface_data_align: Perform the data alignment process. * @num_lanes_set: Set the number of lanes enabled. + * @crc_enable: Enable direction-specific CRC handling. + * @crc_disable: Disable direction-specific CRC handling. * @ddr_enable: Enable interface DDR (Double Data Rate) mode. * @ddr_disable: Disable interface DDR (Double Data Rate) mode. * @data_stream_enable: Enable data stream. @@ -189,6 +191,8 @@ struct iio_backend_ops { enum iio_backend_filter_type type); int (*interface_data_align)(struct iio_backend *back, u32 timeout_us); int (*num_lanes_set)(struct iio_backend *back, unsigned int num_lanes); + int (*crc_enable)(struct iio_backend *back); + int (*crc_disable)(struct iio_backend *back); int (*ddr_enable)(struct iio_backend *back); int (*ddr_disable)(struct iio_backend *back); int (*data_stream_enable)(struct iio_backend *back); @@ -237,6 +241,8 @@ int iio_backend_filter_type_set(struct iio_backend *back, enum iio_backend_filter_type type); int iio_backend_interface_data_align(struct iio_backend *back, u32 timeout_us); int iio_backend_num_lanes_set(struct iio_backend *back, unsigned int num_lanes); +int iio_backend_crc_enable(struct iio_backend *back); +int iio_backend_crc_disable(struct iio_backend *back); int iio_backend_ddr_enable(struct iio_backend *back); int iio_backend_ddr_disable(struct iio_backend *back); int iio_backend_data_stream_enable(struct iio_backend *back); diff --git a/include/linux/iio/iio-gts-helper.h b/include/linux/iio/iio-gts-helper.h index 66f830ab9b49..0c4b2e9fbe1e 100644 --- a/include/linux/iio/iio-gts-helper.h +++ b/include/linux/iio/iio-gts-helper.h @@ -7,6 +7,7 @@ #ifndef __IIO_GTS_HELPER__ #define __IIO_GTS_HELPER__ +#include <linux/errno.h> #include <linux/types.h> struct device; @@ -58,14 +59,14 @@ struct iio_itime_sel_mul { struct iio_gts { u64 max_scale; - const struct iio_gain_sel_pair *hwgain_table; + const struct iio_gain_sel_pair *hwgain_table __counted_by_ptr(num_hwgain); int num_hwgain; - const struct iio_itime_sel_mul *itime_table; + const struct iio_itime_sel_mul *itime_table __counted_by_ptr(num_itime); int num_itime; - int **per_time_avail_scale_tables; - int *avail_all_scales_table; + int **per_time_avail_scale_tables __counted_by_ptr(num_itime); + int *avail_all_scales_table __counted_by_ptr(num_avail_all_scales); int num_avail_all_scales; - int *avail_time_tables; + int *avail_time_tables __counted_by_ptr(num_avail_time_tables); int num_avail_time_tables; }; diff --git a/include/linux/iio/iio-opaque.h b/include/linux/iio/iio-opaque.h index b87841a355f8..98330385e08d 100644 --- a/include/linux/iio/iio-opaque.h +++ b/include/linux/iio/iio-opaque.h @@ -73,7 +73,7 @@ struct iio_dev_opaque { #if defined(CONFIG_DEBUG_FS) struct dentry *debugfs_dentry; unsigned int cached_reg_addr; - char read_buf[20]; + char read_buf[24]; unsigned int read_buf_len; #endif }; diff --git a/include/linux/iio/iio.h b/include/linux/iio/iio.h index 711c00f67371..9b63785d5aec 100644 --- a/include/linux/iio/iio.h +++ b/include/linux/iio/iio.h @@ -264,6 +264,10 @@ struct iio_scan_type { * @ext_info: Array of extended info attributes for this channel. * The array is NULL terminated, the last element should * have its name field set to NULL. + * @parent: Optional pointer to the parent channel spec for + * hierarchical channel relationships. When set, a read-only + * "parent" sysfs attribute is created containing the + * parent channel's sysfs name prefix (e.g. "in_voltage0"). * @extend_name: Allows labeling of channel attributes with an * informative name. Note this has no effect codes etc, * unlike modifiers. @@ -309,6 +313,7 @@ struct iio_chan_spec { const struct iio_event_spec *event_spec; unsigned int num_event_specs; const struct iio_chan_spec_ext_info *ext_info; + const struct iio_chan_spec *parent; const char *extend_name; const char *datasheet_name; unsigned int modified:1; @@ -484,6 +489,10 @@ struct iio_trigger; /* forward declaration */ * @update_scan_mode: function to configure device and scan buffer when * channels have changed * @debugfs_reg_access: function to read or write register value of device + * @debugfs_reg64_access: function to read or write 64-bit register value of + * device. debugfs_reg_access() and debugfs_reg64_access() + * are mutually exclusive, but the 64-bit variant has + * precedence in case both are implemented. * @fwnode_xlate: fwnode based function pointer to obtain channel specifier index. * @hwfifo_set_watermark: function pointer to set the current hardware * fifo watermark level; see hwfifo_* entries in @@ -572,6 +581,9 @@ struct iio_info { int (*debugfs_reg_access)(struct iio_dev *indio_dev, unsigned int reg, unsigned int writeval, unsigned int *readval); + int (*debugfs_reg64_access)(struct iio_dev *indio_dev, + unsigned int reg, u64 writeval, + u64 *readval); int (*fwnode_xlate)(struct iio_dev *indio_dev, const struct fwnode_reference_args *iiospec); int (*hwfifo_set_watermark)(struct iio_dev *indio_dev, unsigned int val); @@ -648,7 +660,7 @@ struct iio_dev { struct iio_poll_func *pollfunc; struct iio_poll_func *pollfunc_event; - struct iio_chan_spec const *channels; + struct iio_chan_spec const *channels __counted_by_ptr(num_channels); int num_channels; const char *name; @@ -789,7 +801,7 @@ DEFINE_GUARD_COND(__priv__iio_dev_mode_lock, _try_direct, * * Tries to acquire the direct mode lock with cleanup ACQUIRE() semantics and * automatically releases it at the end of the scope. It most be always paired - * with IIO_DEV_ACQUIRE_ERR(), for example (notice the scope braces):: + * with IIO_DEV_ACQUIRE_FAILED(), for example (notice the scope braces):: * * switch() { * case IIO_CHAN_INFO_RAW: { @@ -1083,6 +1095,15 @@ static inline unsigned int iio_get_masklength(const struct iio_dev *indio_dev) return ACCESS_PRIVATE(indio_dev, masklength); } +/** + * iio_scan_timestamp_enabled - Is the timestamp channel in the current scan + * @indio_dev: the IIO device to check + */ +static inline bool iio_scan_timestamp_enabled(const struct iio_dev *indio_dev) +{ + return ACCESS_PRIVATE(indio_dev, scan_timestamp); +} + int iio_active_scan_mask_index(struct iio_dev *indio_dev); /** diff --git a/include/linux/input/samsung-keypad.h b/include/linux/input/samsung-keypad.h deleted file mode 100644 index ab6b97114c08..000000000000 --- a/include/linux/input/samsung-keypad.h +++ /dev/null @@ -1,39 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* - * Samsung Keypad platform data definitions - * - * Copyright (C) 2010 Samsung Electronics Co.Ltd - * Author: Joonyoung Shim <jy0922.shim@samsung.com> - */ - -#ifndef __SAMSUNG_KEYPAD_H -#define __SAMSUNG_KEYPAD_H - -#include <linux/input/matrix_keypad.h> - -#define SAMSUNG_MAX_ROWS 8 -#define SAMSUNG_MAX_COLS 8 - -/** - * struct samsung_keypad_platdata - Platform device data for Samsung Keypad. - * @keymap_data: pointer to &matrix_keymap_data. - * @rows: number of keypad row supported. - * @cols: number of keypad col supported. - * @no_autorepeat: disable key autorepeat. - * @wakeup: controls whether the device should be set up as wakeup source. - * @cfg_gpio: configure the GPIO. - * - * Initialisation data specific to either the machine or the platform - * for the device driver to use or call-back when configuring gpio. - */ -struct samsung_keypad_platdata { - const struct matrix_keymap_data *keymap_data; - unsigned int rows; - unsigned int cols; - bool no_autorepeat; - bool wakeup; - - void (*cfg_gpio)(unsigned int rows, unsigned int cols); -}; - -#endif /* __SAMSUNG_KEYPAD_H */ diff --git a/include/linux/intel_vsec.h b/include/linux/intel_vsec.h index 843cda8f8644..8686e79c4f75 100644 --- a/include/linux/intel_vsec.h +++ b/include/linux/intel_vsec.h @@ -90,13 +90,25 @@ enum intel_vsec_quirks { * @read_telem: when specified, called by client driver to access PMT * data (instead of direct copy). * * dev: device reference for the callback's use - * * guid: ID of data to acccss + * * guid: ID of data to access * * data: buffer for the data to be copied * * off: offset into the requested buffer * * count: size of buffer + * @read_reg: when specified called by client driver to read PMT state + * * dev: device reference for the callback's use + * * guid: ID of data to access + * * reg_data: register data + * * offset: offset of register to read + * @write_reg: when specified called by client driver to write PMT state + * * dev: device reference for the callback's use + * * guid: ID of data to access + * * reg_data: register data + * * offset: offset of register to write */ struct pmt_callbacks { int (*read_telem)(struct device *dev, u32 guid, u64 *data, loff_t off, u32 count); + int (*read_reg)(struct device *dev, u32 guid, u32 *reg_data, u32 offset); + int (*write_reg)(struct device *dev, u32 guid, u32 reg_data, u32 offset); }; struct vsec_feature_dependency { @@ -221,8 +233,7 @@ static inline struct intel_vsec_device *auxdev_to_ivdev(struct auxiliary_device #if IS_ENABLED(CONFIG_INTEL_VSEC) int intel_vsec_register(struct device *dev, const struct intel_vsec_platform_info *info); -int intel_vsec_set_mapping(struct oobmsm_plat_info *plat_info, - struct intel_vsec_device *vsec_dev); +int intel_vsec_set_mapping(struct oobmsm_plat_info *plat_info, struct device *dev); struct oobmsm_plat_info *intel_vsec_get_mapping(struct pci_dev *pdev); #else static inline int intel_vsec_register(struct device *dev, @@ -231,7 +242,7 @@ static inline int intel_vsec_register(struct device *dev, return -ENODEV; } static inline int intel_vsec_set_mapping(struct oobmsm_plat_info *plat_info, - struct intel_vsec_device *vsec_dev) + struct device *dev) { return -ENODEV; } diff --git a/include/linux/interrupt.h b/include/linux/interrupt.h index 3bf969ad8fe0..ee2dd5349697 100644 --- a/include/linux/interrupt.h +++ b/include/linux/interrupt.h @@ -119,6 +119,7 @@ typedef irqreturn_t (*irq_handler_t)(int, void *); * @thread_flags: flags related to @thread * @thread_mask: bitmask for keeping track of @thread activity * @dir: pointer to the proc/irq/NN/name entry + * @thread_dtor: task_work for @thread irregular exit cleanup */ struct irqaction { irq_handler_t handler; @@ -137,6 +138,7 @@ struct irqaction { unsigned long thread_mask; const char *name; struct proc_dir_entry *dir; + struct callback_head thread_dtor; } ____cacheline_internodealigned_in_smp; extern irqreturn_t no_action(int cpl, void *dev_id); @@ -573,9 +575,11 @@ enum * _ IRQ_POLL: irq_poll_cpu_dead() migrates the queue * * _ (HR)TIMER_SOFTIRQ: (hr)timers_dead_cpu() migrates the queue + * + * _ BLOCK_SOFTIRQ: blk_softirq_cpu_dead() completes the remaining requests */ -#define SOFTIRQ_HOTPLUG_SAFE_MASK (BIT(TIMER_SOFTIRQ) | BIT(IRQ_POLL_SOFTIRQ) |\ - BIT(HRTIMER_SOFTIRQ) | BIT(RCU_SOFTIRQ)) +#define SOFTIRQ_HOTPLUG_SAFE_MASK (BIT(TIMER_SOFTIRQ) | BIT(BLOCK_SOFTIRQ) |\ + BIT(IRQ_POLL_SOFTIRQ) | BIT(HRTIMER_SOFTIRQ) | BIT(RCU_SOFTIRQ)) /* map softirq index to softirq name. update 'softirq_to_name' in diff --git a/include/linux/interrupt_rc.h b/include/linux/interrupt_rc.h index a9ed937a80e7..d385ab05a2b6 100644 --- a/include/linux/interrupt_rc.h +++ b/include/linux/interrupt_rc.h @@ -11,6 +11,7 @@ * make it easier to include without hitting circular header dependencies. */ +#include <linux/debug_locks.h> #include <linux/irqflags.h> #include <linux/preempt.h> #include <linux/processor.h> @@ -65,6 +66,13 @@ static inline void local_interrupt_disable(void) local_irq_save(flags); new_count = hardirq_disable_enter(); + /* Is hardirq disable count overflow soon? */ + if (IS_ENABLED(CONFIG_DEBUG_PREEMPT) && + DEBUG_LOCKS_WARN_ON(((new_count & HARDIRQ_DISABLE_MASK) + + (10 << HARDIRQ_DISABLE_SHIFT)) >= + HARDIRQ_DISABLE_MASK)) + return; + if ((new_count & HARDIRQ_DISABLE_MASK) == HARDIRQ_DISABLE_OFFSET) _local_interrupt_save_state(flags); } @@ -73,6 +81,11 @@ static inline void local_interrupt_enable(void) { int new_count; + /* Unpaired local_interrupt_enable()? Warn and abort. */ + if (IS_ENABLED(CONFIG_DEBUG_PREEMPT) && + DEBUG_LOCKS_WARN_ON((preempt_count() & HARDIRQ_DISABLE_MASK) == 0)) + return; + new_count = hardirq_disable_exit(); if ((new_count & HARDIRQ_DISABLE_MASK) == 0) diff --git a/include/linux/io_uring.h b/include/linux/io_uring.h index d1aa4edfc2a5..505caa99c621 100644 --- a/include/linux/io_uring.h +++ b/include/linux/io_uring.h @@ -2,10 +2,42 @@ #ifndef _LINUX_IO_URING_H #define _LINUX_IO_URING_H +#include <linux/io_uring_types.h> #include <linux/sched.h> #include <linux/xarray.h> #include <uapi/linux/io_uring.h> +static inline void req_set_fail(struct io_kiocb *req) +{ + req->flags |= REQ_F_FAIL; + if (req->flags & REQ_F_CQE_SKIP) { + req->flags &= ~REQ_F_CQE_SKIP; + req->flags |= REQ_F_SKIP_LINK_CQES; + } +} + +static inline void io_req_set_res(struct io_kiocb *req, s32 res, u32 cflags) +{ + req->cqe.res = res; + req->cqe.flags = cflags; +} + +static inline u32 ctx_cqe32_flags(struct io_ring_ctx *ctx) +{ + if (ctx->flags & IORING_SETUP_CQE_MIXED) + return IORING_CQE_F_32; + return 0; +} + +static inline void io_req_set_res32(struct io_kiocb *req, s32 res, u32 cflags, + __u64 extra1, __u64 extra2) +{ + req->cqe.res = res; + req->cqe.flags = cflags | ctx_cqe32_flags(req->ctx); + req->big_cqe.extra1 = extra1; + req->big_cqe.extra2 = extra2; +} + #if defined(CONFIG_IO_URING) void __io_uring_cancel(bool cancel_all); void __io_uring_free(struct task_struct *tsk); diff --git a/include/linux/io_uring/cmd.h b/include/linux/io_uring/cmd.h index 42801f0b6456..e18d4af62e1a 100644 --- a/include/linux/io_uring/cmd.h +++ b/include/linux/io_uring/cmd.h @@ -3,6 +3,7 @@ #define _LINUX_IO_URING_CMD_H #include <uapi/linux/io_uring.h> +#include <linux/io_uring.h> #include <linux/io_uring_types.h> #include <linux/blk-mq.h> @@ -41,6 +42,24 @@ static inline void io_uring_cmd_private_sz_check(size_t cmd_sz) ((pdu_type *)&(cmd)->pdu) \ ) +static inline void io_uring_cmd_set_res(struct io_uring_cmd *cmd, s32 ret) +{ + struct io_kiocb *req = cmd_to_io_kiocb(cmd); + + if (ret < 0) + req_set_fail(req); + io_req_set_res(req, ret, 0); +} + +static inline void io_uring_cmd_set_res32(struct io_uring_cmd *cmd, s32 ret, u64 res2) +{ + struct io_kiocb *req = cmd_to_io_kiocb(cmd); + + if (ret < 0) + req_set_fail(req); + io_req_set_res32(req, ret, 0, res2, 0); +} + #if defined(CONFIG_IO_URING) int io_uring_cmd_import_fixed(u64 ubuf, unsigned long len, int rw, struct iov_iter *iter, @@ -56,11 +75,12 @@ int io_uring_cmd_import_fixed_vec(struct io_uring_cmd *ioucmd, * Completes the request, i.e. posts an io_uring CQE and deallocates @ioucmd * and the corresponding io_uring request. * + * io_uring_cmd_set_res()/io_uring_cmd_set_res32() must be called first. + * * Note: the caller should never hard code @issue_flags and is only allowed * to pass the mask provided by the core io_uring code. */ -void __io_uring_cmd_done(struct io_uring_cmd *cmd, s32 ret, u64 res2, - unsigned issue_flags, bool is_cqe32); +void __io_uring_cmd_done(struct io_uring_cmd *, unsigned issue_flags); void __io_uring_cmd_do_in_task(struct io_uring_cmd *ioucmd, io_req_tw_func_t task_work_cb, @@ -116,8 +136,8 @@ static inline int io_uring_cmd_import_fixed_vec(struct io_uring_cmd *ioucmd, { return -EOPNOTSUPP; } -static inline void __io_uring_cmd_done(struct io_uring_cmd *cmd, s32 ret, - u64 ret2, unsigned issue_flags, bool is_cqe32) +static inline void __io_uring_cmd_done(struct io_uring_cmd *cmd, + unsigned issue_flags) { } static inline void __io_uring_cmd_do_in_task(struct io_uring_cmd *ioucmd, @@ -205,13 +225,15 @@ static inline void *io_uring_cmd_ctx_handle(struct io_uring_cmd *cmd) static inline void io_uring_cmd_done(struct io_uring_cmd *ioucmd, s32 ret, unsigned issue_flags) { - return __io_uring_cmd_done(ioucmd, ret, 0, issue_flags, false); + io_uring_cmd_set_res(ioucmd, ret); + __io_uring_cmd_done(ioucmd, issue_flags); } static inline void io_uring_cmd_done32(struct io_uring_cmd *ioucmd, s32 ret, u64 res2, unsigned issue_flags) { - return __io_uring_cmd_done(ioucmd, ret, res2, issue_flags, true); + io_uring_cmd_set_res32(ioucmd, ret, res2); + __io_uring_cmd_done(ioucmd, issue_flags); } #endif /* _LINUX_IO_URING_CMD_H */ diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h index 8126af652845..1468bcb02893 100644 --- a/include/linux/io_uring_types.h +++ b/include/linux/io_uring_types.h @@ -11,6 +11,7 @@ struct iou_loop_params; struct io_uring_bpf_ops; +struct dma_buf_io_map; enum { /* @@ -161,6 +162,9 @@ struct io_uring_task { /* drains ->task_list once the task can no longer run task_work */ struct work_struct fallback_work; + /* the tctx is freed after a grace period, see io_put_task() */ + struct rcu_head rcu; + struct { /* task_work */ struct mpscq task_list; struct callback_head task_work; @@ -519,11 +523,18 @@ struct io_ring_ctx { u32 iowq_limits[2]; struct callback_head poll_wq_task_work; + /* deferred ->poll_wq wake up for local task_work */ + struct rcu_head poll_wq_rcu; + atomic_t poll_wq_rcu_pending; struct list_head defer_list; unsigned nr_drained; + /* drain request dispatched from ->defer_list, until it completes */ + struct io_kiocb *drain_req; /* protected by ->completion_lock */ unsigned nr_req_allocated; + /* pending SEND_ZC notifications, protected by ->uring_lock */ + unsigned nr_notifs; #ifdef CONFIG_NET_RX_BUSY_POLL struct list_head napi_list; /* track busy poll napi_id */ @@ -610,6 +621,7 @@ enum { REQ_F_IMPORT_BUFFER_BIT, REQ_F_SQE_COPIED_BIT, REQ_F_IOPOLL_BIT, + REQ_F_DMABUF_BIT, /* not a real bit, just to check we're not overflowing the space */ __REQ_F_LAST_BIT, @@ -705,6 +717,8 @@ enum { REQ_F_SQE_COPIED = IO_REQ_FLAG(REQ_F_SQE_COPIED_BIT), /* request must be iopolled to completion (set in ->issue()) */ REQ_F_IOPOLL = IO_REQ_FLAG(REQ_F_IOPOLL_BIT), + /* there is a dma map attached to request that needs to be dropped */ + REQ_F_DMABUF = IO_REQ_FLAG(REQ_F_DMABUF_BIT), }; struct io_tw_req { @@ -827,6 +841,7 @@ struct io_kiocb { /* custom credentials, valid IFF REQ_F_CREDS is set */ const struct cred *creds; struct io_wq_work work; + struct dma_buf_io_map *dmabuf_map; struct io_big_cqe { u64 extra1; diff --git a/include/linux/iomap.h b/include/linux/iomap.h index bc7ae6327dbf..5254430844b2 100644 --- a/include/linux/iomap.h +++ b/include/linux/iomap.h @@ -483,13 +483,35 @@ sector_t iomap_bmap(struct address_space *mapping, sector_t bno, #define IOMAP_IOEND_BOUNDARY (1U << 2) /* is direct I/O */ #define IOMAP_IOEND_DIRECT (1U << 3) +/* generate integrity (PI) information */ +#ifdef CONFIG_BLK_DEV_INTEGRITY +#define IOMAP_IOEND_INTEGRITY (1U << 4) +#else +#define IOMAP_IOEND_INTEGRITY 0 +#endif /* CONFIG_BLK_DEV_INTEGRITY */ /* * Flags that if set on either ioend prevent the merge of two ioends. * (IOMAP_IOEND_BOUNDARY also prevents merges, but only one-way) */ #define IOMAP_IOEND_NOMERGE_FLAGS \ - (IOMAP_IOEND_SHARED | IOMAP_IOEND_UNWRITTEN | IOMAP_IOEND_DIRECT) + (IOMAP_IOEND_SHARED | IOMAP_IOEND_UNWRITTEN | IOMAP_IOEND_DIRECT | \ + IOMAP_IOEND_INTEGRITY) + +/* ioend flags directly implied by iomap flags */ +static inline u16 iomap_ioend_flags(const struct iomap *iomap) +{ + unsigned int flags = 0; + + if (iomap->type == IOMAP_UNWRITTEN) + flags |= IOMAP_IOEND_UNWRITTEN; + if (iomap->flags & IOMAP_F_SHARED) + flags |= IOMAP_IOEND_SHARED; + if (iomap->flags & IOMAP_F_INTEGRITY) + flags |= IOMAP_IOEND_INTEGRITY; + + return flags; +} /* * Structure for writeback I/O completions. @@ -500,6 +522,7 @@ sector_t iomap_bmap(struct address_space *mapping, sector_t bno, struct iomap_ioend { struct list_head io_list; /* next ioend in chain */ u16 io_flags; /* IOMAP_IOEND_* */ + u32 io_bvec_offset; /* offset into first bvec */ struct inode *io_inode; /* file being written to */ size_t io_size; /* size of the extent */ atomic_t io_remaining; /* completetion defer count */ @@ -517,6 +540,13 @@ static inline struct iomap_ioend *iomap_ioend_from_bio(struct bio *bio) return container_of(bio, struct iomap_ioend, io_bio); } +#define BVEC_ITER_IOEND(_ioend) \ +{ \ + .bi_sector = (_ioend)->io_sector, \ + .bi_size = (_ioend)->io_size, \ + .bi_offset = (_ioend)->io_bvec_offset, \ +} + struct iomap_writeback_ops { /* * Performs writeback on the passed in range @@ -565,6 +595,7 @@ void iomap_finish_ioends(struct iomap_ioend *ioend, int error); void iomap_ioend_try_merge(struct iomap_ioend *ioend, struct list_head *more_ioends); void iomap_sort_ioends(struct list_head *ioend_list); +int iomap_ioend_integrity_verify(struct iomap_ioend *ioend); ssize_t iomap_add_to_ioend(struct iomap_writepage_ctx *wpc, struct folio *folio, loff_t pos, loff_t end_pos, unsigned int dirty_len); int iomap_ioend_writeback_submit(struct iomap_writepage_ctx *wpc, int error); @@ -577,6 +608,11 @@ void iomap_finish_folio_write(struct inode *inode, struct folio *folio, int iomap_writeback_folio(struct iomap_writepage_ctx *wpc, struct folio *folio); int iomap_writepages(struct iomap_writepage_ctx *wpc); +void iomap_bounce_read(struct iomap_ioend *orig_ioend, unsigned int minsize, + void (*submit_ioend)(struct iomap_ioend *ioend)); +void iomap_bounce_read_end_io(struct iomap_ioend *ioend, struct bio *orig_bio, + int error); + struct iomap_read_folio_ctx { const struct iomap_read_ops *ops; struct folio *cur_folio; @@ -657,7 +693,8 @@ struct iomap_dio_ops { #define IOMAP_DIO_PARTIAL (1 << 2) /* - * Ensure each bio is aligned to fs block size. + * Ensure every operation, including bio and hole zeroing, is aligned to fs + * block size. * * For filesystems which need to calculate/verify the checksum of each fs * block. Otherwise they may not be able to handle unaligned bios. diff --git a/include/linux/iommu-dma.h b/include/linux/iommu-dma.h index 060f6e23ab3c..fae5d50e4f27 100644 --- a/include/linux/iommu-dma.h +++ b/include/linux/iommu-dma.h @@ -39,7 +39,7 @@ int iommu_dma_get_sgtable(struct device *dev, struct sg_table *sgt, void *cpu_addr, dma_addr_t dma_addr, size_t size, unsigned long attrs); unsigned long iommu_dma_get_merge_boundary(struct device *dev); -size_t iommu_dma_opt_mapping_size(void); +size_t iommu_dma_max_opt_mapping_size(void); size_t iommu_dma_max_mapping_size(struct device *dev); void iommu_dma_free(struct device *dev, size_t size, void *cpu_addr, dma_addr_t handle, unsigned long attrs); diff --git a/include/linux/iommu.h b/include/linux/iommu.h index ac43b8b93f14..ef5043752a36 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -773,8 +773,10 @@ struct iommu_ops { * passes in the cache invalidation requests, in form * of a driver data structure. The driver must update * array->entry_num to report the number of handled - * invalidation requests. The driver data structure - * must be defined in include/uapi/linux/iommufd.h + * invalidation requests. A driver may handle fewer than + * the requested, in which case the core re-invokes the + * op for the remainder. The driver data structure must + * be defined in include/uapi/linux/iommufd.h * @iova_to_phys: translate iova to physical address * @enforce_cache_coherency: Prevent any kind of DMA from bypassing IOMMU_CACHE, * including no-snoop TLPs on PCIe or other platform @@ -898,7 +900,6 @@ int iommu_device_sysfs_add(struct iommu_device *iommu, void iommu_device_sysfs_remove(struct iommu_device *iommu); int iommu_device_link(struct iommu_device *iommu, struct device *link); void iommu_device_unlink(struct iommu_device *iommu, struct device *link); -int iommu_deferred_attach(struct device *dev, struct iommu_domain *domain); static inline struct iommu_device *dev_to_iommu_device(struct device *dev) { @@ -943,7 +944,6 @@ extern void iommu_detach_device(struct iommu_domain *domain, struct device *dev); extern struct iommu_domain *iommu_get_domain_for_dev(struct device *dev); struct iommu_domain *iommu_driver_get_domain_for_dev(struct device *dev); -extern struct iommu_domain *iommu_get_dma_domain(struct device *dev); extern int iommu_map(struct iommu_domain *domain, unsigned long iova, phys_addr_t paddr, size_t size, int prot, gfp_t gfp); int iommu_map_nosync(struct iommu_domain *domain, unsigned long iova, diff --git a/include/linux/iommufd.h b/include/linux/iommufd.h index 6e7efe83bc5d..3087f5b2def8 100644 --- a/include/linux/iommufd.h +++ b/include/linux/iommufd.h @@ -154,6 +154,8 @@ struct iommufd_hw_queue { * The @array passes in the cache invalidation requests, in * form of a driver data structure. A driver must update the * array->entry_num to report the number of handled requests. + * A driver may handle fewer than the requested entry_num, in + * which case the core re-invokes the op for the remainder. * The data structure of the array entry must be defined in * include/uapi/linux/iommufd.h * @vdevice_size: Size of the driver-defined vDEVICE structure per this vIOMMU diff --git a/include/linux/iov_iter.h b/include/linux/iov_iter.h index f9a17fbbd398..2f91884d733b 100644 --- a/include/linux/iov_iter.h +++ b/include/linux/iov_iter.h @@ -310,6 +310,8 @@ size_t iterate_and_advance2(struct iov_iter *iter, size_t len, void *priv, return iterate_folioq(iter, len, priv, priv2, step); if (iov_iter_is_xarray(iter)) return iterate_xarray(iter, len, priv, priv2, step); + if (WARN_ON_ONCE(iov_iter_is_dmabuf_map(iter))) + return 0; return iterate_discard(iter, len, priv, priv2, step); } diff --git a/include/linux/irq-entry-common.h b/include/linux/irq-entry-common.h index 0bb6c03481fa..d5a66f54d70b 100644 --- a/include/linux/irq-entry-common.h +++ b/include/linux/irq-entry-common.h @@ -346,22 +346,7 @@ typedef struct irqentry_state { * * Conditional reschedule with additional sanity checks. */ -void raw_irqentry_exit_cond_resched(void); - -#ifdef CONFIG_PREEMPT_DYNAMIC -#if defined(CONFIG_HAVE_PREEMPT_DYNAMIC_CALL) -#define irqentry_exit_cond_resched_dynamic_enabled raw_irqentry_exit_cond_resched -#define irqentry_exit_cond_resched_dynamic_disabled NULL -DECLARE_STATIC_CALL(irqentry_exit_cond_resched, raw_irqentry_exit_cond_resched); -#define irqentry_exit_cond_resched() static_call(irqentry_exit_cond_resched)() -#elif defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY) -DECLARE_STATIC_KEY_TRUE(sk_dynamic_irqentry_exit_cond_resched); -void dynamic_irqentry_exit_cond_resched(void); -#define irqentry_exit_cond_resched() dynamic_irqentry_exit_cond_resched() -#endif -#else /* CONFIG_PREEMPT_DYNAMIC */ -#define irqentry_exit_cond_resched() raw_irqentry_exit_cond_resched() -#endif /* CONFIG_PREEMPT_DYNAMIC */ +void irqentry_exit_cond_resched(void); /** * irqentry_enter_from_kernel_mode - Establish state before invoking the irq handler @@ -468,6 +453,25 @@ static inline void irqentry_exit_to_kernel_mode_preempt(struct pt_regs *regs, irqentry_exit_cond_resched(); } +/* + * The deferred hrtimer rearm is the tail of the timer interrupt, moved to the + * interrupt return path so that it runs after a possible reschedule. By now + * irq_exit_rcu() has left lockdep's hardirq context. Re-enter it for the + * rearm, so that the hrtimer base lock and the timekeeping sequence count are + * not chained onto the locks held by the interrupted context, which would add + * new lock chains for every lock stack an interrupt hits. + * + * The rearm on return to user mode needs no such annotation: no locks are + * held there, so it uses the same lock chains as the rearm in hardirq + * context. + */ +static __always_inline void irqentry_hrtimer_rearm_deferred(void) +{ + lockdep_hardirq_enter(); + hrtimer_rearm_deferred(); + lockdep_hardirq_exit(); +} + /** * irqentry_exit_to_kernel_mode_after_preempt - Establish trace, lockdep and RCU state * @regs: Pointer to current's pt_regs @@ -491,7 +495,7 @@ irqentry_exit_to_kernel_mode_after_preempt(struct pt_regs *regs, irqentry_state_ */ if (state.exit_rcu) { instrumentation_begin(); - hrtimer_rearm_deferred(); + irqentry_hrtimer_rearm_deferred(); /* Tell the tracer that IRET will enable interrupts */ trace_hardirqs_on_prepare(); lockdep_hardirqs_on_prepare(); @@ -502,7 +506,7 @@ irqentry_exit_to_kernel_mode_after_preempt(struct pt_regs *regs, irqentry_state_ } instrumentation_begin(); - hrtimer_rearm_deferred(); + irqentry_hrtimer_rearm_deferred(); /* Covers both tracing and lockdep */ trace_hardirqs_on(); instrumentation_end(); diff --git a/include/linux/irq_work.h b/include/linux/irq_work.h index c5afd053ae32..7184e1273965 100644 --- a/include/linux/irq_work.h +++ b/include/linux/irq_work.h @@ -59,6 +59,7 @@ void irq_work_sync(struct irq_work *work); #include <asm/irq_work.h> void irq_work_run(void); +void irq_work_run_cpu(unsigned int cpu); bool irq_work_needs_cpu(void); void irq_work_single(void *arg); @@ -67,6 +68,7 @@ void arch_irq_work_raise(void); #else static inline bool irq_work_needs_cpu(void) { return false; } static inline void irq_work_run(void) { } +static inline void irq_work_run_cpu(unsigned int cpu) { } static inline void irq_work_single(void *arg) { } #endif diff --git a/include/linux/irqchip/arm-gic-v5.h b/include/linux/irqchip/arm-gic-v5.h index 2c2fb39f049c..299e75bc9525 100644 --- a/include/linux/irqchip/arm-gic-v5.h +++ b/include/linux/irqchip/arm-gic-v5.h @@ -63,20 +63,34 @@ #define GICV5_OUTER_SHARE 0b10 #define GICV5_INNER_SHARE 0b11 +#define GICV5_AIDR_COMPONENT_IRS 0b00 +#define GICV5_AIDR_COMPONENT_ITS 0b01 +#define GICV5_AIDR_COMPONENT_IWB 0b10 + +#define GICV5_AIDR_ARCH_MAJ_REV_V5 0 +#define GICV5_AIDR_ARCH_MIN_REV_V0 0 + /* * IRS registers and tables structures */ #define GICV5_IRS_IDR0 0x0000 #define GICV5_IRS_IDR1 0x0004 #define GICV5_IRS_IDR2 0x0008 +#define GICV5_IRS_IDR3 0x000c +#define GICV5_IRS_IDR4 0x0010 #define GICV5_IRS_IDR5 0x0014 #define GICV5_IRS_IDR6 0x0018 #define GICV5_IRS_IDR7 0x001c +#define GICV5_IRS_IIDR 0x0040 +#define GICV5_IRS_AIDR 0x0044 #define GICV5_IRS_CR0 0x0080 #define GICV5_IRS_CR1 0x0084 #define GICV5_IRS_SYNCR 0x00c0 #define GICV5_IRS_SYNC_STATUSR 0x00c4 +#define GICV5_IRS_SPI_VMR 0x0100 #define GICV5_IRS_SPI_SELR 0x0108 +#define GICV5_IRS_SPI_DOMAINR 0x010c +#define GICV5_IRS_SPI_RESAMPLER 0x0110 #define GICV5_IRS_SPI_CFGR 0x0114 #define GICV5_IRS_SPI_STATUSR 0x0118 #define GICV5_IRS_PE_SELR 0x0140 @@ -86,11 +100,51 @@ #define GICV5_IRS_IST_CFGR 0x0190 #define GICV5_IRS_IST_STATUSR 0x0194 #define GICV5_IRS_MAP_L2_ISTR 0x01c0 - +#define GICV5_IRS_VMT_BASER 0x0200 +#define GICV5_IRS_VMT_CFGR 0x0210 +#define GICV5_IRS_VMT_STATUSR 0x0214 +#define GICV5_IRS_VPE_SELR 0x0240 +#define GICV5_IRS_VPE_DBR 0x0248 +#define GICV5_IRS_VPE_HPPIR 0x0250 +#define GICV5_IRS_VPE_CR0 0x0258 +#define GICV5_IRS_VPE_STATUSR 0x025c +#define GICV5_IRS_VM_DBR 0x0280 +#define GICV5_IRS_VM_SELR 0x0288 +#define GICV5_IRS_VM_STATUSR 0x028c +#define GICV5_IRS_VMAP_L2_VMTR 0x02c0 +#define GICV5_IRS_VMAP_VMR 0x02c8 +#define GICV5_IRS_VMAP_VISTR 0x02d0 +#define GICV5_IRS_VMAP_L2_VISTR 0x02d8 +#define GICV5_IRS_VMAP_VPER 0x02e0 +#define GICV5_IRS_SAVE_VMR 0x0300 +#define GICV5_IRS_SAVE_VM_STATUSR 0x0308 +#define GICV5_IRS_MEC_IDR 0x0340 +#define GICV5_IRS_MEC_MECID_R 0x0344 +#define GICV5_IRS_MPAM_IDR 0x0380 +#define GICV5_IRS_MPAM_PARTID_R 0x0384 +#define GICV5_IRS_SWERR_STATUSR 0x03c0 +#define GICV5_IRS_SWERR_SYNDROMER0 0x03c8 +#define GICV5_IRS_SWERR_SYNDROMER1 0x03d0 + +#define GICV5_IRS_IDR0_IRSID GENMASK(31, 16) +#define GICV5_IRS_IDR0_SWE BIT(12) +#define GICV5_IRS_IDR0_MPAM BIT(11) +#define GICV5_IRS_IDR0_MEC BIT(10) +#define GICV5_IRS_IDR0_SETLPI BIT(9) +#define GICV5_IRS_IDR0_VIRT_ONE_N BIT(8) +#define GICV5_IRS_IDR0_ONE_N BIT(7) #define GICV5_IRS_IDR0_VIRT BIT(6) +#define GICV5_IRS_IDR0_PA_RANGE GENMASK(5, 2) +#define GICV5_IRS_IDR0_INT_DOM GENMASK(1, 0) + +#define GICV5_IRS_IDR0_INT_DOM_SECURE 0b00 +#define GICV5_IRS_IDR0_INT_DOM_NON_SECURE 0b01 +#define GICV5_IRS_IDR0_INT_DOM_EL3 0b10 +#define GICV5_IRS_IDR0_INT_DOM_REALM 0b11 #define GICV5_IRS_IDR1_PRIORITY_BITS GENMASK(22, 20) #define GICV5_IRS_IDR1_IAFFID_BITS GENMASK(19, 16) +#define GICV5_IRS_IDR1_PE_CNT GENMASK(15, 0) #define GICV5_IRS_IDR1_PRIORITY_BITS_1BITS 0b000 #define GICV5_IRS_IDR1_PRIORITY_BITS_2BITS 0b001 @@ -106,13 +160,30 @@ #define GICV5_IRS_IDR2_LPI BIT(5) #define GICV5_IRS_IDR2_ID_BITS GENMASK(4, 0) +#define GICV5_IRS_IST_L2SZ_SUPPORT_4KB(r) FIELD_GET(BIT(0), (r)) +#define GICV5_IRS_IST_L2SZ_SUPPORT_16KB(r) FIELD_GET(BIT(1), (r)) +#define GICV5_IRS_IST_L2SZ_SUPPORT_64KB(r) FIELD_GET(BIT(2), (r)) + +#define GICV5_IRS_IDR3_VMT_LEVELS BIT(10) +#define GICV5_IRS_IDR3_VM_ID_BITS GENMASK(9, 5) +#define GICV5_IRS_IDR3_VMD_SZ GENMASK(4, 1) +#define GICV5_IRS_IDR3_VMD BIT(0) + +#define GICV5_IRS_IDR4_VPE_ID_BITS GENMASK(9, 6) +#define GICV5_IRS_IDR4_VPED_SZ GENMASK(5, 0) + #define GICV5_IRS_IDR5_SPI_RANGE GENMASK(24, 0) #define GICV5_IRS_IDR6_SPI_IRS_RANGE GENMASK(24, 0) #define GICV5_IRS_IDR7_SPI_BASE GENMASK(23, 0) -#define GICV5_IRS_IST_L2SZ_SUPPORT_4KB(r) FIELD_GET(BIT(11), (r)) -#define GICV5_IRS_IST_L2SZ_SUPPORT_16KB(r) FIELD_GET(BIT(12), (r)) -#define GICV5_IRS_IST_L2SZ_SUPPORT_64KB(r) FIELD_GET(BIT(13), (r)) +#define GICV5_IRS_IIDR_PRODUCT_ID GENMASK(31, 20) +#define GICV5_IRS_IIDR_VARIANT GENMASK(19, 16) +#define GICV5_IRS_IIDR_REVISION GENMASK(15, 12) +#define GICV5_IRS_IIDR_IMPLEMENTER GENMASK(11, 0) + +#define GICV5_IRS_AIDR_COMPONENT GENMASK(11, 8) +#define GICV5_IRS_AIDR_ARCHMAJORREV GENMASK(7, 4) +#define GICV5_IRS_AIDR_ARCHMINORREV GENMASK(3, 0) #define GICV5_IRS_CR0_IDLE BIT(1) #define GICV5_IRS_CR0_IRSEN BIT(0) @@ -135,21 +206,39 @@ #define GICV5_IRS_SYNC_STATUSR_IDLE BIT(0) -#define GICV5_IRS_SPI_STATUSR_V BIT(1) -#define GICV5_IRS_SPI_STATUSR_IDLE BIT(0) +#define GICV5_IRS_SPI_VMR_VIRT BIT_ULL(63) +#define GICV5_IRS_SPI_VMR_VM_ID GENMASK_ULL(15, 0) #define GICV5_IRS_SPI_SELR_ID GENMASK(23, 0) +#define GICV5_IRS_SPI_DOMAINR_DOMAIN GENMASK(1, 0) + +#define GICV5_IRS_SPI_DOMAINR_DOMAIN_SECURE 0b00 +#define GICV5_IRS_SPI_DOMAINR_DOMAIN_NON_SECURE 0b01 +#define GICV5_IRS_SPI_DOMAINR_DOMAIN_EL3 0b10 +#define GICV5_IRS_SPI_DOMAINR_DOMAIN_REALM 0b11 + +#define GICV5_IRS_SPI_RESAMPLER_ID GENMASK(23, 0) + #define GICV5_IRS_SPI_CFGR_TM BIT(0) +#define GICV5_IRS_SPI_CFGR_TM_EDGE 0b0 +#define GICV5_IRS_SPI_CFGR_TM_LEVEL 0b1 + +#define GICV5_IRS_SPI_STATUSR_V BIT(1) +#define GICV5_IRS_SPI_STATUSR_IDLE BIT(0) + #define GICV5_IRS_PE_SELR_IAFFID GENMASK(15, 0) +#define GICV5_IRS_PE_STATUSR_ONLINE BIT(2) #define GICV5_IRS_PE_STATUSR_V BIT(1) #define GICV5_IRS_PE_STATUSR_IDLE BIT(0) #define GICV5_IRS_PE_CR0_DPS BIT(0) -#define GICV5_IRS_IST_STATUSR_IDLE BIT(0) +#define GICV5_IRS_IST_BASER_ADDR_MASK GENMASK_ULL(55, 6) +#define GICV5_IRS_IST_BASER_VALID BIT_ULL(0) +#define GICV5_IRS_IST_BASER_ADDR_SHIFT 6ULL #define GICV5_IRS_IST_CFGR_STRUCTURE BIT(16) #define GICV5_IRS_IST_CFGR_ISTSZ GENMASK(8, 7) @@ -167,15 +256,111 @@ #define GICV5_IRS_IST_CFGR_L2SZ_16K 0b01 #define GICV5_IRS_IST_CFGR_L2SZ_64K 0b10 -#define GICV5_IRS_IST_BASER_ADDR_MASK GENMASK_ULL(55, 6) -#define GICV5_IRS_IST_BASER_VALID BIT_ULL(0) +#define GICV5_IRS_IST_STATUSR_IDLE BIT(0) #define GICV5_IRS_MAP_L2_ISTR_ID GENMASK(23, 0) +#define GICV5_IRS_VMT_BASER_ADDR GENMASK_ULL(55, 3) +#define GICV5_IRS_VMT_BASER_ADDR_SHIFT 3ULL +#define GICV5_IRS_VMT_BASER_VALID BIT_ULL(0) + +#define GICV5_IRS_VMT_CFGR_STRUCTURE_TWO_LEVEL 0b1 +#define GICV5_IRS_VMT_CFGR_STRUCTURE_LINEAR 0b0 + +#define GICV5_IRS_VMT_CFGR_STRUCTURE BIT(16) +#define GICV5_IRS_VMT_CFGR_VM_ID_BITS GENMASK(4, 0) + +#define GICV5_IRS_VMT_STATUSR_IDLE BIT(0) + +#define GICV5_IRS_VPE_SELR_S BIT_ULL(63) +#define GICV5_IRS_VPE_SELR_VPE_ID GENMASK_ULL(47, 32) +#define GICV5_IRS_VPE_SELR_VM_ID GENMASK_ULL(15, 0) + +#define GICV5_IRS_VPE_DBR_DBV BIT_ULL(63) +#define GICV5_IRS_VPE_DBR_REQ_DB BIT_ULL(62) +#define GICV5_IRS_VPE_DBR_DBPM GENMASK_ULL(36, 32) +#define GICV5_IRS_VPE_DBR_INTID GENMASK_ULL(23, 0) + +#define GICV5_IRS_VPE_HPPIR_HPPIV BIT_ULL(32) +#define GICV5_IRS_VPE_HPPIR_TYPE GENMASK_ULL(31, 29) +#define GICV5_IRS_VPE_HPPIR_ID GENMASK_ULL(23, 0) + +#define GICV5_IRS_VPE_CR0_DPS BIT(0) + +#define GICV5_IRS_VPE_STATUSR_V BIT(1) +#define GICV5_IRS_VPE_STATUSR_IDLE BIT(0) + +#define GICV5_IRS_VM_DBR_EN BIT_ULL(63) +#define GICV5_IRS_VM_DBR_VPE_ID GENMASK_ULL(15, 0) + +#define GICV5_IRS_VM_SELR_VM_ID GENMASK(15, 0) + +#define GICV5_IRS_VM_STATUSR_V BIT(1) +#define GICV5_IRS_VM_STATUSR_IDLE BIT(0) + +#define GICV5_IRS_VMAP_L2_VMTR_M BIT_ULL(63) +#define GICV5_IRS_VMAP_L2_VMTR_VM_ID GENMASK_ULL(15, 0) + +#define GICV5_IRS_VMAP_VMR_M BIT_ULL(63) +#define GICV5_IRS_VMAP_VMR_U BIT_ULL(62) +#define GICV5_IRS_VMAP_VMR_VM_ID GENMASK_ULL(15, 0) + +#define GICV5_IRS_VMAP_VISTR_M BIT_ULL(63) +#define GICV5_IRS_VMAP_VISTR_U BIT_ULL(62) +#define GICV5_IRS_VMAP_VISTR_VM_ID GENMASK_ULL(47, 32) +#define GICV5_IRS_VMAP_VISTR_TYPE GENMASK_ULL(31, 29) + +#define GICV5_IRS_VMAP_L2_VISTR_M BIT_ULL(63) +#define GICV5_IRS_VMAP_L2_VISTR_VM_ID GENMASK_ULL(47, 32) +#define GICV5_IRS_VMAP_L2_VISTR_TYPE GENMASK_ULL(31, 29) +#define GICV5_IRS_VMAP_L2_VISTR_ID GENMASK_ULL(23, 0) + +#define GICV5_IRS_VMAP_VPER_M BIT_ULL(63) +#define GICV5_IRS_VMAP_VPER_VM_ID GENMASK_ULL(47, 32) +#define GICV5_IRS_VMAP_VPER_VPE_ID GENMASK_ULL(15, 0) + +#define GICV5_IRS_SAVE_VMR_VM_ID GENMASK_ULL(15, 0) +#define GICV5_IRS_SAVE_VMR_Q BIT_ULL(62) +#define GICV5_IRS_SAVE_VMR_S BIT_ULL(63) + +#define GICV5_IRS_SAVE_VM_STATUSR_IDLE BIT(0) +#define GICV5_IRS_SAVE_VM_STATUSR_Q BIT(1) + +#define GICV5_IRS_MEC_IDR_MECIDSIZE GENMASK(3, 0) + +#define GICV5_IRS_MEC_MECID_R_MECID GENMASK(15, 0) + +#define GICV5_IRS_MPAM_IDR_HAS_MPAM_SP BIT(24) +#define GICV5_IRS_MPAM_IDR_PMG_MAX GENMASK(23, 16) +#define GICV5_IRS_MPAM_IDR_PARTID_MAX GENMASK(15, 0) + +#define GICV5_IRS_MPAM_PARTID_R_IDLE BIT(31) +#define GICV5_IRS_MPAM_PARTID_R_MPAM_SP GENMASK(25, 24) +#define GICV5_IRS_MPAM_PARTID_R_PMG GENMASK(23, 16) +#define GICV5_IRS_MPAM_PARTID_R_PARTID GENMASK(15, 0) + +#define GICV5_IRS_SWERR_STATUSR_IMP_EC GENMASK_ULL(31, 24) +#define GICV5_IRS_SWERR_STATUSR_EC GENMASK_ULL(23, 16) +#define GICV5_IRS_SWERR_STATUSR_OF BIT_ULL(3) +#define GICV5_IRS_SWERR_STATUSR_S1V BIT_ULL(2) +#define GICV5_IRS_SWERR_STATUSR_S0V BIT_ULL(1) +#define GICV5_IRS_SWERR_STATUSR_V BIT_ULL(0) + +#define GICV5_IRS_SWERR_SYNDROMER0_VIRTUAL BIT_ULL(63) +#define GICV5_IRS_SWERR_SYNDROMER0_TYPE GENMASK_ULL(62, 60) +#define GICV5_IRS_SWERR_SYNDROMER0_ID GENMASK_ULL(55, 32) +#define GICV5_IRS_SWERR_SYNDROMER0_VM_ID GENMASK_ULL(15, 0) + +#define GICV5_IRS_SWERR_SYNDROMER1_ADDR GENMASK_ULL(55, 3) + #define GICV5_ISTL1E_VALID BIT_ULL(0) +#define GICV5_IRS_ISTL1E_SIZE 8UL #define GICV5_ISTL1E_L2_ADDR_MASK GENMASK_ULL(55, 12) +#define GICV5_IRS_SETLPIR 0x0000 +#define GICV5_IRS_SETLPIR_ID GENMASK(23, 0) + /* * ITS registers and tables structures */ @@ -298,6 +483,42 @@ #define GICV5_GSI_IWB_WIRE GENMASK(15, 0) /* + * CoreSight identification registers - ordered by increasing offset. + */ +#define GICV5_CORESIGHT_DEVARCH 0xffbc +#define GICV5_CORESIGHT_PIDR4 0xffd0 +#define GICV5_CORESIGHT_PIDR5 0xffd4 +#define GICV5_CORESIGHT_PIDR6 0xffd8 +#define GICV5_CORESIGHT_PIDR7 0xffdc +#define GICV5_CORESIGHT_PIDR0 0xffe0 +#define GICV5_CORESIGHT_PIDR1 0xffe4 +#define GICV5_CORESIGHT_PIDR2 0xffe8 +#define GICV5_CORESIGHT_PIDR3 0xffec +#define GICV5_CORESIGHT_CIDR0 0xfff0 +#define GICV5_CORESIGHT_CIDR1 0xfff4 +#define GICV5_CORESIGHT_CIDR2 0xfff8 +#define GICV5_CORESIGHT_CIDR3 0xfffc + +#define GICV5_CORESIGHT_DEVARCH_VAL \ + (FIELD_PREP(GENMASK(31, 21), 0x23b) | \ + BIT(20) | \ + 0x5a19) + +#define GICV5_CORESIGHT_PIDR4_JEP106_CONT 0x04 +#define GICV5_CORESIGHT_PIDR5_RES0 0x00 +#define GICV5_CORESIGHT_PIDR6_RES0 0x00 +#define GICV5_CORESIGHT_PIDR7_RES0 0x00 +#define GICV5_CORESIGHT_PIDR0_PART_0 0x4b +#define GICV5_CORESIGHT_PIDR1_DES_0_PART_1 0xb0 +#define GICV5_CORESIGHT_PIDR2_DES_1 0x0b +#define GICV5_CORESIGHT_PIDR3_REVAND_CMOD 0x00 + +#define GICV5_CORESIGHT_CIDR0_VAL 0x0d +#define GICV5_CORESIGHT_CIDR1_VAL 0xf0 +#define GICV5_CORESIGHT_CIDR2_VAL 0x05 +#define GICV5_CORESIGHT_CIDR3_VAL 0xb1 + +/* * Global Data structures and functions */ struct gicv5_chip_data { @@ -332,6 +553,8 @@ struct gicv5_irs_chip_data { raw_spinlock_t spi_config_lock; }; +#define IRS_FLAGS_NON_COHERENT BIT(0) + static inline int gicv5_wait_for_op_s_atomic(void __iomem *addr, u32 offset, const char *reg_s, u32 mask, u32 *val) @@ -379,6 +602,7 @@ void __init gicv5_free_lpi_domain(void); int gicv5_irs_of_probe(struct device_node *parent); int gicv5_irs_acpi_probe(void); +struct gicv5_irs_chip_data *gicv5_irs_get_chip_data(void); void gicv5_irs_remove(void); int gicv5_irs_enable(void); void gicv5_irs_its_probe(void); @@ -387,10 +611,13 @@ int gicv5_irs_cpu_to_iaffid(int cpu_id, u16 *iaffid); struct gicv5_irs_chip_data *gicv5_irs_lookup_by_spi_id(u32 spi_id); int gicv5_spi_irq_set_type(struct irq_data *d, unsigned int type); int gicv5_irs_iste_alloc(u32 lpi); +unsigned int gicv5_irs_l2_sz(u32 l2sz); void gicv5_irs_syncr(void); /* Embedded in kvm.arch */ struct gicv5_vpe { + int db; + bool db_fired; bool resident; }; @@ -429,4 +656,15 @@ void gicv5_deinit_lpis(void); void __init gicv5_its_of_probe(struct device_node *parent); void __init gicv5_its_acpi_probe(void); + +enum gicv5_vcpu_cmd { + VMT_L2_MAP, /* Map in a L2 VMT - *may* happen on VM init */ + VMTE_MAKE_VALID, /* Make the VMTE valid */ + VMTE_MAKE_INVALID, /* Make the VMTE (et al.) invalid */ + VPE_MAKE_VALID, /* No corresponding invalid */ + SPI_VIST_MAKE_VALID, /* No corresponding invalid */ + LPI_VIST_MAKE_VALID, /* Triggered by a guest */ + LPI_VIST_MAKE_INVALID, /* Triggered by a guest */ +}; + #endif diff --git a/include/linux/irqchip/arm-vgic-info.h b/include/linux/irqchip/arm-vgic-info.h index 67d9d960273b..f05370e2debf 100644 --- a/include/linux/irqchip/arm-vgic-info.h +++ b/include/linux/irqchip/arm-vgic-info.h @@ -38,6 +38,11 @@ struct gic_kvm_info { bool has_v4_1; /* Deactivation impared, subpar stuff */ bool no_hw_deactivation; + /* GICv5 IRS base */ + struct { + void __iomem *base; + bool non_coherent; + } gicv5_irs; }; #ifdef CONFIG_KVM diff --git a/include/linux/irqchip/riscv-imsic.h b/include/linux/irqchip/riscv-imsic.h index 61af3a5bea09..b7be1aad7774 100644 --- a/include/linux/irqchip/riscv-imsic.h +++ b/include/linux/irqchip/riscv-imsic.h @@ -46,6 +46,9 @@ struct imsic_local_config { }; struct imsic_global_config { + /* Device details */ + struct fwnode_handle *fwnode; + /* * MSI Target Address Scheme * diff --git a/include/linux/jbd2.h b/include/linux/jbd2.h index 1b42fe47c26b..e3886963ba00 100644 --- a/include/linux/jbd2.h +++ b/include/linux/jbd2.h @@ -795,7 +795,7 @@ struct journal_s /** * @j_state_lock: Protect the various scalars in the journal. */ - rwlock_t j_state_lock; + rwlock_t j_state_lock ____cacheline_aligned_in_smp; /** * @j_barrier_count: diff --git a/include/linux/kdev_t.h b/include/linux/kdev_t.h index 4856706fbfeb..2dbbd47f1e68 100644 --- a/include/linux/kdev_t.h +++ b/include/linux/kdev_t.h @@ -9,7 +9,7 @@ #define MAJOR(dev) ((unsigned int) ((dev) >> MINORBITS)) #define MINOR(dev) ((unsigned int) ((dev) & MINORMASK)) -#define MKDEV(ma,mi) (((ma) << MINORBITS) | (mi)) +#define MKDEV(ma, mi) (((dev_t)(ma) << MINORBITS) | (mi)) #define print_dev_t(buffer, dev) \ sprintf((buffer), "%u:%u\n", MAJOR(dev), MINOR(dev)) diff --git a/include/linux/kernel-page-flags.h b/include/linux/kernel-page-flags.h index 196778a087c4..fe5ab6e50bd7 100644 --- a/include/linux/kernel-page-flags.h +++ b/include/linux/kernel-page-flags.h @@ -11,7 +11,6 @@ #define KPF_RESERVED 32 #define KPF_MLOCKED 33 #define KPF_OWNER_2 34 -#define KPF_PRIVATE 35 #define KPF_PRIVATE_2 36 #define KPF_OWNER_PRIVATE 37 #define KPF_ARCH 38 diff --git a/include/linux/kernel.h b/include/linux/kernel.h index 24414c79e59a..4b11d1dc0a67 100644 --- a/include/linux/kernel.h +++ b/include/linux/kernel.h @@ -43,30 +43,10 @@ struct completion; struct user; #ifdef CONFIG_PREEMPT_VOLUNTARY_BUILD - extern int __cond_resched(void); # define might_resched() __cond_resched() - -#elif defined(CONFIG_PREEMPT_DYNAMIC) && defined(CONFIG_HAVE_PREEMPT_DYNAMIC_CALL) - -extern int __cond_resched(void); - -DECLARE_STATIC_CALL(might_resched, __cond_resched); - -static __always_inline void might_resched(void) -{ - static_call_mod(might_resched)(); -} - -#elif defined(CONFIG_PREEMPT_DYNAMIC) && defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY) - -extern int dynamic_might_resched(void); -# define might_resched() dynamic_might_resched() - #else - # define might_resched() do { } while (0) - #endif /* CONFIG_PREEMPT_* */ #ifdef CONFIG_DEBUG_ATOMIC_SLEEP diff --git a/include/linux/kernel_stat.h b/include/linux/kernel_stat.h index 9ca6c2259dfe..9f0f91dc8d7d 100644 --- a/include/linux/kernel_stat.h +++ b/include/linux/kernel_stat.h @@ -40,6 +40,9 @@ struct kernel_cpustat { seqcount_t idle_sleeptime_seq; u64 idle_entrytime; u64 idle_stealtime[2]; + u64 idle_dyntick_entry; + u64 idle_tick_period; + u64 idle_tick_overlap; #endif u64 cpustat[NR_STATS]; }; @@ -111,7 +114,7 @@ static inline unsigned long kstat_cpu_irqs_sum(unsigned int cpu) #ifdef CONFIG_HAVE_VIRT_CPU_ACCOUNTING_IDLE static inline void kcpustat_dyntick_start(u64 now) { } -static inline void kcpustat_dyntick_stop(u64 now) { } +static inline void kcpustat_dyntick_stop(u64 now, u64 tick_start) { } static inline void kcpustat_irq_enter(u64 now) { } static inline void kcpustat_irq_exit(u64 now) { } static inline bool kcpustat_idle_dyntick(void) { return false; } @@ -132,7 +135,7 @@ static inline u64 kcpustat_field_iowait(int cpu) #else /* !CONFIG_HAVE_VIRT_CPU_ACCOUNTING_IDLE */ extern void kcpustat_dyntick_start(u64 now); -extern void kcpustat_dyntick_stop(u64 now); +extern void kcpustat_dyntick_stop(u64 now, u64 tick_start); extern void kcpustat_irq_enter(u64 now); extern void kcpustat_irq_exit(u64 now); extern u64 kcpustat_field_idle(int cpu); @@ -196,6 +199,17 @@ static inline void kcpustat_cpu_fetch(struct kernel_cpustat *dst, int cpu) } #endif /* !CONFIG_VIRT_CPU_ACCOUNTING_GEN */ +static inline u64 kcpustat_field_total(enum cpu_usage_stat usage, const struct cpumask *cpus) +{ + u64 total = 0; + int cpu; + + for_each_cpu(cpu, cpus) + total += kcpustat_field(usage, cpu); + + return total; +} + extern void account_user_time(struct task_struct *, u64); extern void account_guest_time(struct task_struct *, u64); extern void account_system_time(struct task_struct *, int, u64); diff --git a/include/linux/kernfs.h b/include/linux/kernfs.h index 351a5101c862..73786b567ccd 100644 --- a/include/linux/kernfs.h +++ b/include/linux/kernfs.h @@ -309,7 +309,7 @@ struct kernfs_ops { * write() is bounced through kernel buffer. If atomic_write_len * is not set, a write larger than PAGE_SIZE results in partial * operations of PAGE_SIZE chunks. If atomic_write_len is set, - * writes upto the specified size are executed atomically but + * writes up to the specified size are executed atomically but * larger ones are rejected with -E2BIG. */ size_t atomic_write_len; diff --git a/include/linux/kexec.h b/include/linux/kexec.h index 0af8ae4fdd08..bac11e940e74 100644 --- a/include/linux/kexec.h +++ b/include/linux/kexec.h @@ -259,21 +259,6 @@ int kexec_kernel_verify_pe_sig(const char *kernel, unsigned long kernel_len); extern int kexec_add_buffer(struct kexec_buf *kbuf); int kexec_locate_mem_hole(struct kexec_buf *kbuf); -#ifndef arch_kexec_locate_mem_hole -/** - * arch_kexec_locate_mem_hole - Find free memory to place the segments. - * @kbuf: Parameters for the memory search. - * - * On success, kbuf->mem will have the start address of the memory region found. - * - * Return: 0 on success, negative errno on error. - */ -static inline int arch_kexec_locate_mem_hole(struct kexec_buf *kbuf) -{ - return kexec_locate_mem_hole(kbuf); -} -#endif - #ifndef arch_kexec_apply_relocations_add /* * arch_kexec_apply_relocations_add - apply relocations of type RELA @@ -418,7 +403,7 @@ struct kimage { #endif struct { - struct kexec_segment *scratch; + struct kexec_segment *bootmem; phys_addr_t fdt; } kho; @@ -468,6 +453,7 @@ bool kexec_load_permitted(int kexec_image_type); /* flag to track if kexec reboot is in progress */ extern bool kexec_in_progress; +bool kexec_is_in_progress(void); #ifndef page_to_boot_pfn static inline unsigned long page_to_boot_pfn(struct page *page) @@ -544,6 +530,7 @@ static inline void *kimage_map_segment(struct kimage *image, int idx) { return NULL; } static inline void kimage_unmap_segment(void *buffer) { } #define kexec_in_progress false +static inline bool kexec_is_in_progress(void) { return false; } #endif /* CONFIG_KEXEC_CORE */ #ifdef CONFIG_KEXEC_SIG diff --git a/include/linux/kexec_handover.h b/include/linux/kexec_handover.h index 46de86dc343e..ef2b188c8ed0 100644 --- a/include/linux/kexec_handover.h +++ b/include/linux/kexec_handover.h @@ -36,15 +36,15 @@ int kho_retrieve_subtree(const char *name, phys_addr_t *phys, size_t *size); void kho_memory_init(void); void kho_memory_init_early(void); -void kho_populate(phys_addr_t fdt_phys, u64 fdt_len, phys_addr_t scratch_phys, - u64 scratch_len); +void kho_populate(phys_addr_t fdt_phys, u64 fdt_len, phys_addr_t bootmem_phys, + u64 bootmem_len); -bool kho_scratch_overlap(phys_addr_t phys, size_t size); +bool kho_bootmem_overlap(phys_addr_t phys, size_t size); -static inline enum migratetype kho_scratch_migratetype(unsigned long pfn, +static inline enum migratetype kho_bootmem_migratetype(unsigned long pfn, enum migratetype mt) { - if (kho_scratch_overlap(PFN_PHYS(pfn), pageblock_nr_pages << PAGE_SHIFT)) + if (kho_bootmem_overlap(PFN_PHYS(pfn), pageblock_nr_pages << PAGE_SHIFT)) return MIGRATE_CMA; return mt; } @@ -123,16 +123,16 @@ static inline void kho_memory_init(void) { } static inline void kho_memory_init_early(void) { } static inline void kho_populate(phys_addr_t fdt_phys, u64 fdt_len, - phys_addr_t scratch_phys, u64 scratch_len) + phys_addr_t bootmem_phys, u64 bootmem_len) { } -static inline bool kho_scratch_overlap(phys_addr_t phys, size_t size) +static inline bool kho_bootmem_overlap(phys_addr_t phys, size_t size) { return false; } -static inline enum migratetype kho_scratch_migratetype(unsigned long pfn, +static inline enum migratetype kho_bootmem_migratetype(unsigned long pfn, enum migratetype mt) { return mt; diff --git a/include/linux/key-type.h b/include/linux/key-type.h index bb97bd3e5af4..c4f329b6ae3f 100644 --- a/include/linux/key-type.h +++ b/include/linux/key-type.h @@ -161,6 +161,7 @@ struct key_type { const void *in, const void *in2); /* internal fields */ + atomic64_t index_id; /* Opaque keyring index ID */ struct list_head link; /* link in types list */ struct lock_class_key lock_class; /* key->sem lock class */ } __randomize_layout; diff --git a/include/linux/key.h b/include/linux/key.h index 81b8f05c6898..115db6e60254 100644 --- a/include/linux/key.h +++ b/include/linux/key.h @@ -107,6 +107,7 @@ struct keyring_name; struct key_tag { struct rcu_head rcu; + atomic64_t index_id; /* Opaque keyring index ID */ refcount_t usage; bool removed; /* T when subject removed */ }; @@ -129,6 +130,8 @@ struct keyring_index_key { struct key_type *type; struct key_tag *domain_tag; /* Domain of operation */ const char *description; + u64 type_id; /* Cached index IDs */ + u64 domain_id; }; union key_payload { @@ -251,6 +254,8 @@ struct key { struct key_type *type; /* type of key */ struct key_tag *domain_tag; /* Domain of operation */ char *description; + u64 type_id; + u64 domain_id; }; }; @@ -440,6 +445,10 @@ extern key_ref_t keyring_search(key_ref_t keyring, extern int keyring_restrict(key_ref_t keyring, const char *type, const char *restriction); +extern void key_register_bpf_keyring(struct key *keyring); +extern void key_register_dm_verity_keyring(struct key *keyring); +extern void key_register_fs_verity_keyring(struct key *keyring); + extern struct key *key_lookup(key_serial_t id); static inline key_serial_t key_serial(const struct key *key) diff --git a/include/linux/kmod.h b/include/linux/kmod.h index 9a07c3215389..b9474a62a568 100644 --- a/include/linux/kmod.h +++ b/include/linux/kmod.h @@ -2,17 +2,9 @@ #ifndef __LINUX_KMOD_H__ #define __LINUX_KMOD_H__ -/* - * include/linux/kmod.h - */ - -#include <linux/umh.h> -#include <linux/gfp.h> -#include <linux/stddef.h> +#include <linux/compiler_attributes.h> #include <linux/errno.h> -#include <linux/compiler.h> -#include <linux/workqueue.h> -#include <linux/sysctl.h> +#include <linux/types.h> #ifdef CONFIG_MODULES /* modprobe exit status on success, -ve on error. Return value diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index c844def6dd95..e7fc94e2910a 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -45,6 +45,10 @@ #include <asm/kvm_host.h> #include <linux/kvm_dirty_ring.h> +#define __KVM_DEV_STRING(name, x) __stringify(name ## _ ## x) +#define _KVM_DEV_STRING(name, x) __KVM_DEV_STRING(name, x) +#define KVM_DEV_STRING(x) _KVM_DEV_STRING(KVM_DEV_NAME, x) + #ifndef KVM_MAX_VCPU_IDS #define KVM_MAX_VCPU_IDS KVM_MAX_VCPUS #endif @@ -562,7 +566,7 @@ static inline int kvm_vcpu_exiting_guest_mode(struct kvm_vcpu *vcpu) /* * The memory barrier ensures a previous write to vcpu->requests cannot * be reordered with the read of vcpu->mode. It pairs with the general - * memory barrier following the write of vcpu->mode in VCPU RUN. + * memory barrier following the write of vcpu->mode in KVM_RUN. */ smp_mb__before_atomic(); return cmpxchg(&vcpu->mode, IN_GUEST_MODE, EXITING_GUEST_MODE); @@ -722,31 +726,14 @@ static inline int kvm_arch_vcpu_memslots_id(struct kvm_vcpu *vcpu) } #endif -#ifndef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES -static inline bool kvm_arch_has_private_mem(struct kvm *kvm) -{ - return false; -} -#endif - -#ifdef CONFIG_KVM_GUEST_MEMFD -bool kvm_arch_supports_gmem_init_shared(struct kvm *kvm); - -static inline u64 kvm_gmem_get_supported_flags(struct kvm *kvm) -{ - u64 flags = GUEST_MEMFD_FLAG_MMAP; - - if (!kvm || kvm_arch_supports_gmem_init_shared(kvm)) - flags |= GUEST_MEMFD_FLAG_INIT_SHARED; - - return flags; -} -#endif - #ifndef kvm_arch_has_readonly_mem static inline bool kvm_arch_has_readonly_mem(struct kvm *kvm) { - return IS_ENABLED(CONFIG_HAVE_KVM_READONLY_MEM); +#ifdef __KVM_HAVE_READONLY_MEM + return true; +#else + return false; +#endif } #endif @@ -784,6 +771,12 @@ struct kvm { * kvm_swap_active_memslots(). */ struct mutex slots_arch_lock; + /* + * Back-reference to the VM file for subsystems (e.g., VFIO). Holds no + * reference to avoid pinning the VM. Callers storing the file must + * take their own reference. + */ + struct file *file; struct mm_struct *mm; /* userspace tied to this vm */ unsigned long nr_memslot_pages; /* The two memslot sets - active and inactive (per address space) */ @@ -832,7 +825,7 @@ struct kvm { struct kvm_vm_stat stat; struct kvm_arch arch; refcount_t users_count; -#ifdef CONFIG_KVM_MMIO +#ifndef __KVM_NO_MMIO struct kvm_coalesced_mmio_ring *coalesced_mmio_ring; spinlock_t ring_lock; struct list_head coalesced_zones; @@ -854,6 +847,8 @@ struct kvm { gfn_t mmu_invalidate_range_start; gfn_t mmu_invalidate_range_end; + unsigned long gpc_invalidate_seq; + struct list_head devices; u64 manual_dirty_log_protect; struct dentry *debugfs_dentry; @@ -871,7 +866,7 @@ struct kvm { #ifdef CONFIG_HAVE_KVM_PM_NOTIFIER struct notifier_block pm_notifier; #endif -#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES +#ifdef CONFIG_KVM_VM_MEMORY_ATTRIBUTES /* Protected by slots_lock (for writes) and RCU (for reads) */ struct xarray mem_attr_array; #endif @@ -1080,7 +1075,18 @@ void kvm_exit(void); void kvm_get_kvm(struct kvm *kvm); bool kvm_get_kvm_safe(struct kvm *kvm); void kvm_put_kvm(struct kvm *kvm); -bool file_is_kvm(struct file *file); + +/* + * Architectures define kvm_file_to_kvm_arch to <arch> + * to get a typed, arch-namespaced helper: + * + * struct kvm *file_to_kvm_<arch>(struct file *file) + */ +#ifdef kvm_file_to_kvm_arch +#define kvm_file_to_kvm_fn CONCATENATE(file_to_kvm_, kvm_file_to_kvm_arch) +struct kvm *kvm_file_to_kvm_fn(struct file *file); +#endif /* kvm_file_to_kvm_arch */ + void kvm_put_kvm_no_destroy(struct kvm *kvm); static inline struct kvm_memslots *__kvm_memslots(struct kvm *kvm, int as_id) @@ -2464,7 +2470,7 @@ void kvm_arch_update_irqfd_routing(struct kvm_kernel_irqfd *irqfd, struct kvm_kernel_irq_routing_entry *new); #endif /* CONFIG_HAVE_KVM_IRQ_BYPASS */ -#ifdef CONFIG_HAVE_KVM_INVALID_WAKEUPS +#ifdef __KVM_HAVE_INVALID_WAKEUPS /* If we wakeup during the poll time, was it a sucessful poll? */ static inline bool vcpu_valid_wakeup(struct kvm_vcpu *vcpu) { @@ -2476,9 +2482,9 @@ static inline bool vcpu_valid_wakeup(struct kvm_vcpu *vcpu) { return true; } -#endif /* CONFIG_HAVE_KVM_INVALID_WAKEUPS */ +#endif /* __KVM_HAVE_INVALID_WAKEUPS */ -#ifdef CONFIG_HAVE_KVM_NO_POLL +#ifdef __KVM_HAVE_NO_POLL /* Callback that tells if we must not poll */ bool kvm_arch_no_poll(struct kvm_vcpu *vcpu); #else @@ -2486,18 +2492,18 @@ static inline bool kvm_arch_no_poll(struct kvm_vcpu *vcpu) { return false; } -#endif /* CONFIG_HAVE_KVM_NO_POLL */ +#endif /* __KVM_HAVE_NO_POLL */ void kvm_arch_guest_memory_reclaimed(struct kvm *kvm); -#ifdef CONFIG_HAVE_KVM_VCPU_RUN_PID_CHANGE +#ifdef __KVM_HAVE_VCPU_RUN_PID_CHANGE int kvm_arch_vcpu_run_pid_change(struct kvm_vcpu *vcpu); #else static inline int kvm_arch_vcpu_run_pid_change(struct kvm_vcpu *vcpu) { return 0; } -#endif /* CONFIG_HAVE_KVM_VCPU_RUN_PID_CHANGE */ +#endif /* __KVM_HAVE_VCPU_RUN_PID_CHANGE */ #ifdef CONFIG_VIRT_XFER_TO_GUEST_WORK static inline void kvm_handle_signal_exit(struct kvm_vcpu *vcpu) @@ -2564,48 +2570,79 @@ static inline bool kvm_memslot_is_gmem_only(const struct kvm_memory_slot *slot) return slot->flags & KVM_MEMSLOT_GMEM_ONLY; } -#ifdef CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES -static inline unsigned long kvm_get_memory_attributes(struct kvm *kvm, gfn_t gfn) +#ifdef CONFIG_KVM_VM_MEMORY_ATTRIBUTES +static inline unsigned long kvm_get_vm_memory_attributes(struct kvm *kvm, gfn_t gfn) { return xa_to_value(xa_load(&kvm->mem_attr_array, gfn)); } -bool kvm_range_has_memory_attributes(struct kvm *kvm, gfn_t start, gfn_t end, - unsigned long mask, unsigned long attrs); -bool kvm_arch_pre_set_memory_attributes(struct kvm *kvm, - struct kvm_gfn_range *range); -bool kvm_arch_post_set_memory_attributes(struct kvm *kvm, - struct kvm_gfn_range *range); +bool kvm_range_has_vm_memory_attributes(struct kvm *kvm, gfn_t start, gfn_t end, + unsigned long mask, unsigned long attrs); +bool kvm_arch_pre_set_vm_memory_attributes(struct kvm *kvm, + struct kvm_gfn_range *range); +bool kvm_arch_post_set_vm_memory_attributes(struct kvm *kvm, + struct kvm_gfn_range *range); +#endif /* CONFIG_KVM_VM_MEMORY_ATTRIBUTES */ + +#ifdef CONFIG_KVM_PRIVATE_MEMATTR_VM +extern bool gmem_in_place_conversion; +typedef bool (kvm_is_private_gfn_t)(struct kvm *kvm, gfn_t gfn); +DECLARE_STATIC_CALL(__kvm_is_private_gfn, kvm_is_private_gfn_t); -static inline bool kvm_mem_is_private(struct kvm *kvm, gfn_t gfn) +static inline bool kvm_is_private_gfn(struct kvm *kvm, gfn_t gfn) { - return kvm_get_memory_attributes(kvm, gfn) & KVM_MEMORY_ATTRIBUTE_PRIVATE; + return static_call(__kvm_is_private_gfn)(kvm, gfn); } + +#elif defined kvm_arch_has_private_mem +#define gmem_in_place_conversion true +#define kvm_is_private_gfn kvm_gmem_is_private_gfn + #else -static inline bool kvm_mem_is_private(struct kvm *kvm, gfn_t gfn) +#define gmem_in_place_conversion false + +static inline bool kvm_arch_has_private_mem(struct kvm *kvm) { return false; } -#endif /* CONFIG_KVM_GENERIC_MEMORY_ATTRIBUTES */ + +static inline bool kvm_is_private_gfn(struct kvm *kvm, gfn_t gfn) +{ + return false; +} +#endif /* kvm_arch_has_private_mem */ #ifdef CONFIG_KVM_GUEST_MEMFD +bool kvm_gmem_is_private_gfn(struct kvm *kvm, gfn_t gfn); +bool kvm_arch_supports_gmem_init_shared(struct kvm *kvm); + +static inline u64 kvm_gmem_get_supported_flags(struct kvm *kvm) +{ + u64 flags = GUEST_MEMFD_FLAG_MMAP; + + if (!kvm || kvm_arch_supports_gmem_init_shared(kvm)) + flags |= GUEST_MEMFD_FLAG_INIT_SHARED; + + return flags; +} + int kvm_gmem_get_pfn(struct kvm *kvm, struct kvm_memory_slot *slot, - gfn_t gfn, kvm_pfn_t *pfn, struct page **page, - int *max_order); + gfn_t gfn, kvm_pfn_t *pfn, int *max_order); #else static inline int kvm_gmem_get_pfn(struct kvm *kvm, struct kvm_memory_slot *slot, gfn_t gfn, - kvm_pfn_t *pfn, struct page **page, - int *max_order) + kvm_pfn_t *pfn, int *max_order) { KVM_BUG_ON(1, kvm); return -EIO; } #endif /* CONFIG_KVM_GUEST_MEMFD */ -#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT int kvm_arch_gmem_make_private(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, kvm_pfn_t nr_pages); +void kvm_arch_gmem_make_shared(kvm_pfn_t pfn, kvm_pfn_t nr_pages); +#ifndef CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT +#define kvm_arch_has_gmem_convert() false #endif #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_POPULATE @@ -2647,6 +2684,7 @@ void kvm_arch_gmem_invalidate_range(struct kvm *kvm, struct kvm_gfn_range *range #endif #ifdef CONFIG_KVM_GENERIC_PRE_FAULT_MEMORY +int kvm_arch_pre_fault_allowed(struct kvm_vcpu *vcpu); long kvm_arch_vcpu_pre_fault_memory(struct kvm_vcpu *vcpu, struct kvm_pre_fault_memory *range); #endif diff --git a/include/linux/landlock.h b/include/linux/landlock.h index 004cbd0b9298..0ee4880988ba 100644 --- a/include/linux/landlock.h +++ b/include/linux/landlock.h @@ -13,14 +13,16 @@ #include <uapi/linux/landlock.h> /* - * Access-right and scope names, shared between the audit records (get_blocker() - * in security/landlock/audit.c) and the trace events + * Access-right, scope, and permission names, shared between the audit records + * (get_blocker() in security/landlock/audit.c) and the trace events * (include/trace/events/landlock.h). A consumer defines * _LANDLOCK_NAME_ENTRY(mask, name) before expanding a list and undefines it * afterwards: audit maps each entry to a "[bit] = name" slot for O(1) lookup, * the trace events map it to a __print_flags() { mask, name } pair. The bit * value lives only in the LANDLOCK_* UAPI constant each entry references. - * Names are unprefixed; audit prepends the "fs."/"net."/"scope." category. + * Access-right and scope names are unprefixed; audit prepends the + * "fs."/"net."/"scope." category. Permission entries carry an action and a + * domain for the qualified and bare views below. */ #define _LANDLOCK_ACCESS_FS_NAMES \ _LANDLOCK_NAME_ENTRY(LANDLOCK_ACCESS_FS_EXECUTE, "execute"), \ @@ -53,4 +55,25 @@ "abstract_unix_socket"), \ _LANDLOCK_NAME_ENTRY(LANDLOCK_SCOPE_SIGNAL, "signal") +#define _LANDLOCK_PERMISSION_NAMESPACE_NAME "namespace" +#define _LANDLOCK_PERMISSION_CAPABILITY_NAME "capability" + +#define _LANDLOCK_PERMISSION_LIST(entry) \ + entry(LANDLOCK_PERMISSION_NAMESPACE_USE, "use", \ + _LANDLOCK_PERMISSION_NAMESPACE_NAME), \ + entry(LANDLOCK_PERMISSION_CAPABILITY_USE, "use", \ + _LANDLOCK_PERMISSION_CAPABILITY_NAME) + +#define _LANDLOCK_PERMISSION_QUALIFIED_ENTRY(mask, action, domain) \ + _LANDLOCK_NAME_ENTRY(mask, domain "." action) + +#define _LANDLOCK_PERMISSION_BARE_ENTRY(mask, action, ...) \ + _LANDLOCK_NAME_ENTRY(mask, action) + +#define _LANDLOCK_PERMISSION_NAMES \ + _LANDLOCK_PERMISSION_LIST(_LANDLOCK_PERMISSION_QUALIFIED_ENTRY) + +#define _LANDLOCK_PERMISSION_BLOCKER_NAMES \ + _LANDLOCK_PERMISSION_LIST(_LANDLOCK_PERMISSION_BARE_ENTRY) + #endif /* _LINUX_LANDLOCK_H */ diff --git a/include/linux/leds.h b/include/linux/leds.h index a515f075c29a..beaf23993063 100644 --- a/include/linux/leds.h +++ b/include/linux/leds.h @@ -109,6 +109,7 @@ struct led_classdev { #define LED_INIT_DEFAULT_TRIGGER BIT(23) #define LED_REJECT_NAME_CONFLICT BIT(24) #define LED_MULTI_COLOR BIT(25) +#define LED_TRIG_HW_CHANGED BIT(26) /* set_brightness_work / blink_timer flags, atomic, private. */ unsigned long work_flags; @@ -239,6 +240,11 @@ struct led_classdev { struct kernfs_node *brightness_hw_changed_kn; #endif +#ifdef CONFIG_LEDS_TRIGGERS_HW_CHANGED + bool trigger_hw_changed; + struct work_struct trigger_hw_changed_work; +#endif + /* Ensures consistent access to the LED class device */ struct mutex led_access; }; @@ -485,6 +491,7 @@ struct led_trigger { const char *name; int (*activate)(struct led_classdev *led_cdev); void (*deactivate)(struct led_classdev *led_cdev); + bool (*hw_offloaded)(struct led_classdev *led_cdev); /* Brightness set by led_trigger_event */ enum led_brightness brightness; @@ -533,6 +540,9 @@ void led_trigger_blink_oneshot(struct led_trigger *trigger, void led_trigger_set_default(struct led_classdev *led_cdev); int led_trigger_set(struct led_classdev *led_cdev, struct led_trigger *trigger); void led_trigger_remove(struct led_classdev *led_cdev); +void led_trigger_remove_hw_control(struct led_classdev *led_cdev); + +bool led_trigger_is_hw_controlled(struct led_classdev *led_cdev); static inline void led_set_trigger_data(struct led_classdev *led_cdev, void *trigger_data) @@ -584,6 +594,13 @@ static inline int led_trigger_set(struct led_classdev *led_cdev, } static inline void led_trigger_remove(struct led_classdev *led_cdev) {} +static inline void led_trigger_remove_hw_control(struct led_classdev *led_cdev) {} + +static inline bool led_trigger_is_hw_controlled(struct led_classdev *led_cdev) +{ + return false; +} + static inline void led_set_trigger_data(struct led_classdev *led_cdev) {} static inline void *led_get_trigger_data(struct led_classdev *led_cdev) { @@ -598,6 +615,13 @@ led_trigger_get_brightness(const struct led_trigger *trigger) #endif /* CONFIG_LEDS_TRIGGERS */ +#ifdef CONFIG_LEDS_TRIGGERS_HW_CHANGED +void led_trigger_notify_hw_control_changed(struct led_classdev *led_cdev, bool activate); +#else +static inline void led_trigger_notify_hw_control_changed(struct led_classdev *led_cdev, + bool activate) {} +#endif + /* Trigger specific enum */ enum led_trigger_netdev_modes { TRIGGER_NETDEV_LINK = 0, diff --git a/include/linux/libata.h b/include/linux/libata.h index 48bde275968d..3bd74d79f812 100644 --- a/include/linux/libata.h +++ b/include/linux/libata.h @@ -1147,6 +1147,7 @@ extern struct ata_host *ata_host_alloc_pinfo(struct device *dev, extern void ata_host_get(struct ata_host *host); extern void ata_host_put(struct ata_host *host); extern int ata_host_start(struct ata_host *host); +void ata_host_undo_start(struct ata_host *host); extern int ata_host_register(struct ata_host *host, const struct scsi_host_template *sht); extern int ata_host_activate(struct ata_host *host, int irq, diff --git a/include/linux/list.h b/include/linux/list.h index 77fb62f79928..e3753695e76c 100644 --- a/include/linux/list.h +++ b/include/linux/list.h @@ -1172,7 +1172,7 @@ static inline void hlist_move_list(struct hlist_head *old, { new->first = old->first; if (new->first) - new->first->pprev = &new->first; + WRITE_ONCE(new->first->pprev, &new->first); old->first = NULL; } @@ -1189,10 +1189,10 @@ static inline void hlist_splice_init(struct hlist_head *from, struct hlist_head *to) { if (to->first) - to->first->pprev = &last->next; + WRITE_ONCE(to->first->pprev, &last->next); last->next = to->first; to->first = from->first; - from->first->pprev = &to->first; + WRITE_ONCE(from->first->pprev, &to->first); from->first = NULL; } diff --git a/include/linux/livepatch.h b/include/linux/livepatch.h index ba9e3988c07c..5f74f79c22b4 100644 --- a/include/linux/livepatch.h +++ b/include/linux/livepatch.h @@ -33,6 +33,7 @@ * @kobj: kobject for sysfs resources * @node: list node for klp_object func_list * @stack_node: list node for klp_ops func_stack list + * @obj: back pointer to the owning object * @old_size: size of the old function * @new_size: size of the new function * @nop: temporary patch to use the original code again; dyn. allocated @@ -72,6 +73,7 @@ struct klp_func { struct kobject kobj; struct list_head node; struct list_head stack_node; + struct klp_object *obj; unsigned long old_size, new_size; bool nop; bool patched; @@ -86,6 +88,7 @@ struct klp_func { * @kobj: kobject for sysfs resources * @func_list: dynamic list of the function entries * @node: list node for klp_patch obj_list + * @patch: back pointer to the owning patch * @mod: kernel module associated with the patched object * (NULL for vmlinux) * @dynamic: temporary object for nop functions; dynamically allocated @@ -101,6 +104,7 @@ struct klp_object { struct kobject kobj; struct list_head func_list; struct list_head node; + struct klp_patch *patch; struct module *mod; bool dynamic; bool patched; diff --git a/include/linux/llc.h b/include/linux/llc.h index 944e9e213112..c03bb12bcc9c 100644 --- a/include/linux/llc.h +++ b/include/linux/llc.h @@ -9,9 +9,4 @@ #include <uapi/linux/llc.h> -#define LLC_SAP_DYN_START 0xC0 -#define LLC_SAP_DYN_STOP 0xDE -#define LLC_SAP_DYN_TRIES 4 - -#define llc_ui_skb_cb(__skb) ((struct sockaddr_llc *)&((__skb)->cb[0])) #endif /* __LINUX_LLC_H */ diff --git a/include/linux/lsm_audit.h b/include/linux/lsm_audit.h index 584db296e43b..5cf0b4795065 100644 --- a/include/linux/lsm_audit.h +++ b/include/linux/lsm_audit.h @@ -44,7 +44,7 @@ struct lsm_network_audit { struct lsm_ioctlop_audit { struct path path; - u16 cmd; + unsigned int cmd; }; struct lsm_ibpkey_audit { @@ -78,6 +78,7 @@ struct common_audit_data { #define LSM_AUDIT_DATA_NOTIFICATION 16 #define LSM_AUDIT_DATA_ANONINODE 17 #define LSM_AUDIT_DATA_NLMSGTYPE 18 +#define LSM_AUDIT_DATA_NS 19 union { struct path path; struct dentry *dentry; @@ -100,6 +101,10 @@ struct common_audit_data { int reason; const char *anonclass; u16 nlmsg_type; + struct { + u32 ns_type; + u64 ns_id; + } ns; } u; /* this union contains LSM specific data */ union { diff --git a/include/linux/lsm_hook_defs.h b/include/linux/lsm_hook_defs.h index 65c9609ec207..87d9d06710cb 100644 --- a/include/linux/lsm_hook_defs.h +++ b/include/linux/lsm_hook_defs.h @@ -94,7 +94,7 @@ LSM_HOOK(int, 0, path_mkdir, const struct path *dir, struct dentry *dentry, LSM_HOOK(int, 0, path_rmdir, const struct path *dir, struct dentry *dentry) LSM_HOOK(int, 0, path_mknod, const struct path *dir, struct dentry *dentry, umode_t mode, unsigned int dev) -LSM_HOOK(void, LSM_RET_VOID, path_post_mknod, struct mnt_idmap *idmap, +LSM_HOOK(void, LSM_RET_VOID, path_post_mknod, const struct mnt_idmap *idmap, struct dentry *dentry) LSM_HOOK(int, 0, path_truncate, const struct path *path) LSM_HOOK(int, 0, path_symlink, const struct path *dir, struct dentry *dentry, @@ -120,59 +120,60 @@ LSM_HOOK(int, -EOPNOTSUPP, inode_init_security, struct inode *inode, int *xattr_count) LSM_HOOK(int, 0, inode_init_security_anon, struct inode *inode, const struct qstr *name, const struct inode *context_inode) -LSM_HOOK(int, 0, inode_create, struct inode *dir, struct dentry *dentry, - umode_t mode) -LSM_HOOK(void, LSM_RET_VOID, inode_post_create_tmpfile, struct mnt_idmap *idmap, +LSM_HOOK(int, 0, inode_create, const struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, umode_t mode) +LSM_HOOK(void, LSM_RET_VOID, inode_post_create_tmpfile, const struct mnt_idmap *idmap, struct inode *inode) -LSM_HOOK(int, 0, inode_link, struct dentry *old_dentry, struct inode *dir, - struct dentry *new_dentry) +LSM_HOOK(int, 0, inode_link, const struct mnt_idmap *idmap, + struct dentry *old_dentry, struct inode *dir, struct dentry *new_dentry) LSM_HOOK(int, 0, inode_unlink, struct inode *dir, struct dentry *dentry) -LSM_HOOK(int, 0, inode_symlink, struct inode *dir, struct dentry *dentry, - const char *old_name) -LSM_HOOK(int, 0, inode_mkdir, struct inode *dir, struct dentry *dentry, - umode_t mode) +LSM_HOOK(int, 0, inode_symlink, const struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, const char *old_name) +LSM_HOOK(int, 0, inode_mkdir, const struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, umode_t mode) LSM_HOOK(int, 0, inode_rmdir, struct inode *dir, struct dentry *dentry) -LSM_HOOK(int, 0, inode_mknod, struct inode *dir, struct dentry *dentry, - umode_t mode, dev_t dev) +LSM_HOOK(int, 0, inode_mknod, const struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, umode_t mode, dev_t dev) LSM_HOOK(int, 0, inode_rename, struct inode *old_dir, struct dentry *old_dentry, struct inode *new_dir, struct dentry *new_dentry) LSM_HOOK(int, 0, inode_readlink, struct dentry *dentry) LSM_HOOK(int, 0, inode_follow_link, struct dentry *dentry, struct inode *inode, bool rcu) -LSM_HOOK(int, 0, inode_permission, struct inode *inode, int mask) -LSM_HOOK(int, 0, inode_setattr, struct mnt_idmap *idmap, struct dentry *dentry, +LSM_HOOK(int, 0, inode_permission, const struct mnt_idmap *idmap, + struct inode *inode, int mask) +LSM_HOOK(int, 0, inode_setattr, const struct mnt_idmap *idmap, struct dentry *dentry, struct iattr *attr) -LSM_HOOK(void, LSM_RET_VOID, inode_post_setattr, struct mnt_idmap *idmap, +LSM_HOOK(void, LSM_RET_VOID, inode_post_setattr, const struct mnt_idmap *idmap, struct dentry *dentry, int ia_valid) LSM_HOOK(int, 0, inode_getattr, const struct path *path) LSM_HOOK(int, 0, inode_xattr_skipcap, const char *name) -LSM_HOOK(int, 0, inode_setxattr, struct mnt_idmap *idmap, +LSM_HOOK(int, 0, inode_setxattr, const struct mnt_idmap *idmap, struct dentry *dentry, const char *name, const void *value, size_t size, int flags) LSM_HOOK(void, LSM_RET_VOID, inode_post_setxattr, struct dentry *dentry, const char *name, const void *value, size_t size, int flags) LSM_HOOK(int, 0, inode_getxattr, struct dentry *dentry, const char *name) LSM_HOOK(int, 0, inode_listxattr, struct dentry *dentry) -LSM_HOOK(int, 0, inode_removexattr, struct mnt_idmap *idmap, +LSM_HOOK(int, 0, inode_removexattr, const struct mnt_idmap *idmap, struct dentry *dentry, const char *name) LSM_HOOK(void, LSM_RET_VOID, inode_post_removexattr, struct dentry *dentry, const char *name) LSM_HOOK(int, 0, inode_file_setattr, struct dentry *dentry, struct file_kattr *fa) LSM_HOOK(int, 0, inode_file_getattr, struct dentry *dentry, struct file_kattr *fa) -LSM_HOOK(int, 0, inode_set_acl, struct mnt_idmap *idmap, +LSM_HOOK(int, 0, inode_set_acl, const struct mnt_idmap *idmap, struct dentry *dentry, const char *acl_name, struct posix_acl *kacl) LSM_HOOK(void, LSM_RET_VOID, inode_post_set_acl, struct dentry *dentry, const char *acl_name, struct posix_acl *kacl) -LSM_HOOK(int, 0, inode_get_acl, struct mnt_idmap *idmap, +LSM_HOOK(int, 0, inode_get_acl, const struct mnt_idmap *idmap, struct dentry *dentry, const char *acl_name) -LSM_HOOK(int, 0, inode_remove_acl, struct mnt_idmap *idmap, +LSM_HOOK(int, 0, inode_remove_acl, const struct mnt_idmap *idmap, struct dentry *dentry, const char *acl_name) -LSM_HOOK(void, LSM_RET_VOID, inode_post_remove_acl, struct mnt_idmap *idmap, +LSM_HOOK(void, LSM_RET_VOID, inode_post_remove_acl, const struct mnt_idmap *idmap, struct dentry *dentry, const char *acl_name) LSM_HOOK(int, 0, inode_need_killpriv, struct dentry *dentry) -LSM_HOOK(int, 0, inode_killpriv, struct mnt_idmap *idmap, +LSM_HOOK(int, 0, inode_killpriv, const struct mnt_idmap *idmap, struct dentry *dentry) -LSM_HOOK(int, -EOPNOTSUPP, inode_getsecurity, struct mnt_idmap *idmap, +LSM_HOOK(int, -EOPNOTSUPP, inode_getsecurity, const struct mnt_idmap *idmap, struct inode *inode, const char *name, void **buffer, bool alloc) LSM_HOOK(int, -EOPNOTSUPP, inode_setsecurity, struct inode *inode, const char *name, const void *value, size_t size, int flags) @@ -265,6 +266,9 @@ LSM_HOOK(int, -ENOSYS, task_prctl, int option, unsigned long arg2, LSM_HOOK(void, LSM_RET_VOID, task_to_inode, struct task_struct *p, struct inode *inode) LSM_HOOK(int, 0, userns_create, const struct cred *cred) +LSM_HOOK(int, 0, namespace_init, struct ns_common *ns) +LSM_HOOK(void, LSM_RET_VOID, namespace_free, struct ns_common *ns) +LSM_HOOK(int, 0, namespace_install, const struct nsset *nsset, struct ns_common *ns) LSM_HOOK(int, 0, ipc_permission, struct kern_ipc_perm *ipcp, short flag) LSM_HOOK(void, LSM_RET_VOID, ipc_getlsmprop, struct kern_ipc_perm *ipcp, struct lsm_prop *prop) diff --git a/include/linux/lsm_hooks.h b/include/linux/lsm_hooks.h index c4488c4a6d8a..13621e9e233e 100644 --- a/include/linux/lsm_hooks.h +++ b/include/linux/lsm_hooks.h @@ -112,6 +112,7 @@ struct lsm_blob_sizes { unsigned int lbs_ipc; unsigned int lbs_key; unsigned int lbs_msg_msg; + unsigned int lbs_ns; unsigned int lbs_perf_event; unsigned int lbs_task; unsigned int lbs_xattr_count; /* num xattr slots in new_xattrs array */ diff --git a/include/linux/mailbox/riscv-rpmi-message.h b/include/linux/mailbox/riscv-rpmi-message.h index e135c6564d0c..d5362b5821f9 100644 --- a/include/linux/mailbox/riscv-rpmi-message.h +++ b/include/linux/mailbox/riscv-rpmi-message.h @@ -93,6 +93,7 @@ static inline int rpmi_to_linux_error(int rpmi_error) /* RPMI service group IDs */ #define RPMI_SRVGRP_SYSTEM_MSI 0x00002 #define RPMI_SRVGRP_CLOCK 0x00008 +#define RPMI_SRVGRP_DEVICE_POWER 0x00009 /* RPMI clock service IDs */ enum rpmi_clock_service_id { @@ -119,6 +120,16 @@ enum rpmi_sysmsi_service_id { RPMI_SYSMSI_SRV_ID_MAX_COUNT }; +/* RPMI device power service IDs */ +enum rpmi_device_power_service_id { + RPMI_DP_SRV_ENABLE_NOTIFICATION = 0x01, + RPMI_DP_SRV_GET_NUM_DOMAINS = 0x02, + RPMI_DP_SRV_GET_ATTRS = 0x03, + RPMI_DP_SRV_SET_STATE = 0x04, + RPMI_DP_SRV_GET_STATE = 0x05, + RPMI_DP_SRV_ID_MAX_COUNT, +}; + /* RPMI Linux mailbox attribute IDs */ enum rpmi_mbox_attribute_id { RPMI_MBOX_ATTR_SPEC_VERSION, diff --git a/include/linux/maple_tree.h b/include/linux/maple_tree.h index e595ae5cd0ee..691f6012daea 100644 --- a/include/linux/maple_tree.h +++ b/include/linux/maple_tree.h @@ -9,6 +9,7 @@ */ #include <linux/kernel.h> +#include <linux/compiler.h> #include <linux/rcupdate.h> #include <linux/spinlock.h> @@ -297,7 +298,8 @@ struct maple_tree { #endif #define DEFINE_MTREE(name) \ - struct maple_tree name = MTREE_INIT(name, 0) + struct maple_tree name = MTREE_INIT(name, 0); \ + ASSERT_STATIC_STORAGE(name) #define mtree_lock(mt) spin_lock((&(mt)->ma_lock)) #define mtree_lock_nested(mas, subclass) \ @@ -507,9 +509,13 @@ struct ma_wr_state { }; #define mas_lock(mas) spin_lock(&((mas)->tree->ma_lock)) +#define mas_lock_irqsave(mas, flags) \ + spin_lock_irqsave(&((mas)->tree->ma_lock), flags) #define mas_lock_nested(mas, subclass) \ spin_lock_nested(&((mas)->tree->ma_lock), subclass) #define mas_unlock(mas) spin_unlock(&((mas)->tree->ma_lock)) +#define mas_unlock_irqrestore(mas, flags) \ + spin_unlock_irqrestore(&((mas)->tree->ma_lock), flags) /* * Special values for ma_state.node. diff --git a/include/linux/mdio-mux.h b/include/linux/mdio-mux.h index a5d58f221939..e2af9cb8e017 100644 --- a/include/linux/mdio-mux.h +++ b/include/linux/mdio-mux.h @@ -1,5 +1,5 @@ /* - * MDIO bus multiplexer framwork. + * MDIO bus multiplexer framework. * * This file is subject to the terms and conditions of the GNU General Public * License. See the file "COPYING" in the main directory of this archive diff --git a/include/linux/memblock.h b/include/linux/memblock.h index d62db9e776cf..40e6e0aa2f8d 100644 --- a/include/linux/memblock.h +++ b/include/linux/memblock.h @@ -46,11 +46,11 @@ extern unsigned long long max_possible_pfn; * @MEMBLOCK_RSRV_KERN: memory region that is reserved for kernel use, * either explictitly with memblock_reserve_kern() or via memblock * allocation APIs. All memblock allocations set this flag. - * @MEMBLOCK_KHO_SCRATCH: memory region that kexec can pass to the next - * kernel in handover mode. During early boot, we do not know about all - * memory reservations yet, so we get scratch memory from the previous - * kernel that we know is good to use. It is the only memory that - * allocations may happen from in this phase. + * @MEMBLOCK_KHO_NOPRSRV: memory region with no preservation from kexec + * handover. During early boot, we do not know about all memory preservations + * yet, so we get memory with no preservations from the previous kernel that we + * know is good to use. It is the only memory that allocations may happen from + * in this phase. * @MEMBLOCK_RSRV_HUGETLB: memory is reserved for hugetlb pages */ enum memblock_flags { @@ -61,7 +61,7 @@ enum memblock_flags { MEMBLOCK_DRIVER_MANAGED = 0x8, /* always detected via a driver */ MEMBLOCK_RSRV_NOINIT = 0x10, /* don't initialize struct pages */ MEMBLOCK_RSRV_KERN = 0x20, /* memory reserved for kernel use */ - MEMBLOCK_KHO_SCRATCH = 0x40, /* scratch memory for kexec handover */ + MEMBLOCK_KHO_NOPRSRV = 0x40, /* memory with no KHO preservations */ MEMBLOCK_RSRV_HUGETLB = 0x80, /* memory reserved for hugetlb pages */ }; @@ -158,11 +158,10 @@ int memblock_mark_nomap(phys_addr_t base, phys_addr_t size); int memblock_clear_nomap(phys_addr_t base, phys_addr_t size); int memblock_reserved_mark_noinit(phys_addr_t base, phys_addr_t size); int memblock_reserved_mark_kern(phys_addr_t base, phys_addr_t size); -int memblock_mark_kho_scratch(phys_addr_t base, phys_addr_t size); -int memblock_clear_kho_scratch(phys_addr_t base, phys_addr_t size); +int memblock_mark_kho_noprsrv(phys_addr_t base, phys_addr_t size); +int memblock_clear_kho_noprsrv(phys_addr_t base, phys_addr_t size); void memblock_free(void *ptr, size_t size); -void reset_all_zones_managed_pages(void); /* Low level functions */ void __next_mem_range(u64 *idx, int nid, enum memblock_flags flags, @@ -301,9 +300,9 @@ static inline bool memblock_is_driver_managed(struct memblock_region *m) return m->flags & MEMBLOCK_DRIVER_MANAGED; } -static inline bool memblock_is_kho_scratch(struct memblock_region *m) +static inline bool memblock_is_kho_noprsrv(struct memblock_region *m) { - return m->flags & MEMBLOCK_KHO_SCRATCH; + return m->flags & MEMBLOCK_KHO_NOPRSRV; } int memblock_search_pfn_nid(unsigned long pfn, unsigned long *start_pfn, @@ -413,9 +412,6 @@ static __always_inline phys_addr_t memblock_phys_alloc(phys_addr_t size, MEMBLOCK_ALLOC_ACCESSIBLE); } -void *memblock_alloc_exact_nid_raw(phys_addr_t size, phys_addr_t align, - phys_addr_t min_addr, phys_addr_t max_addr, - int nid); void *memblock_alloc_try_nid_raw(phys_addr_t size, phys_addr_t align, phys_addr_t min_addr, phys_addr_t max_addr, int nid); @@ -614,12 +610,12 @@ static inline void early_memtest(phys_addr_t start, phys_addr_t end) { } static inline void memtest_report_meminfo(struct seq_file *m) { } #endif -#ifdef CONFIG_MEMBLOCK_KHO_SCRATCH -void memblock_set_kho_scratch_only(void); -void memblock_clear_kho_scratch_only(void); +#ifdef CONFIG_KEXEC_HANDOVER +void memblock_set_kho_noprsrv_only(void); +void memblock_clear_kho_noprsrv_only(void); #else -static inline void memblock_set_kho_scratch_only(void) { } -static inline void memblock_clear_kho_scratch_only(void) { } +static inline void memblock_set_kho_noprsrv_only(void) { } +static inline void memblock_clear_kho_noprsrv_only(void) { } #endif #endif /* _LINUX_MEMBLOCK_H */ diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index 7d1c0ce189a8..69e1bcc67c58 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -23,6 +23,7 @@ #include <linux/writeback.h> #include <linux/page-flags.h> #include <linux/shrinker.h> +#include <linux/irq_work_types.h> struct mem_cgroup; struct obj_cgroup; @@ -66,11 +67,6 @@ struct mem_cgroup_reclaim_cookie { #define MEM_CGROUP_ID_SHIFT 16 -struct mem_cgroup_private_id { - int id; - refcount_t ref; -}; - struct memcg_vmstats_percpu; struct memcg1_events_percpu; struct memcg_vmstats; @@ -87,50 +83,48 @@ struct mem_cgroup_reclaim_iter { * per-node information in memory controller. */ struct mem_cgroup_per_node { - /* Keep the read-only fields at the start */ + /* Set when the memcg is created, then only read. */ + __cacheline_group_begin_aligned(memcg_pn_read_mostly); struct mem_cgroup *memcg; /* Back pointer, we cannot */ /* use container_of */ struct lruvec_stats_percpu __percpu *lruvec_stats_percpu; struct lruvec_stats *lruvec_stats; struct shrinker_info __rcu *shrinker_info; + struct obj_cgroup __rcu *objcg; + + __cacheline_group_end_aligned(memcg_pn_read_mostly); -#ifdef CONFIG_MEMCG_V1 /* - * Memcg-v1 only stuff in middle as buffer between read mostly fields - * and update often fields to avoid false sharing. If v1 stuff is - * not present, an explicit padding is needed. + * Keep lruvec on its own lines. Sharing them with lru_zone_size[] + * regressed, see commit f59adcf59332 ("mm: memcg: add cacheline + * padding after lruvec in mem_cgroup_per_node"). */ - - struct rb_node tree_node; /* RB tree node */ - unsigned long usage_in_excess;/* Set to the value by which */ - /* the soft limit is exceeded*/ - bool on_tree; -#else - CACHELINE_PADDING(_pad1_); -#endif - - /* Fields which get updated often at the end. */ + __cacheline_group_begin_aligned(memcg_pn_lruvec); struct lruvec lruvec; - CACHELINE_PADDING(_pad2_); - unsigned long lru_zone_size[MAX_NR_ZONES][NR_LRU_LISTS]; + __cacheline_group_end_aligned(memcg_pn_lruvec); + + /* Written on every LRU update and on every reclaim iteration. */ + __cacheline_group_begin_aligned(memcg_pn_write_hot); + long lru_zone_size[MAX_NR_ZONES][NR_LRU_LISTS]; struct mem_cgroup_reclaim_iter iter; +#ifdef CONFIG_MEMCG_NMI_SAFETY_REQUIRES_ATOMIC + /* slab stats for nmi context */ + atomic_t slab_reclaimable; + atomic_t slab_unreclaimable; +#endif + __cacheline_group_end_aligned(memcg_pn_write_hot); + /* Touched only when the memcg is reparented or freed. */ + __cacheline_group_begin_aligned(memcg_pn_cold); /* - * objcg is wiped out as a part of the objcg repaprenting process. * orig_objcg preserves a pointer (and a reference) to the original - * objcg until the end of live of memcg. + * objcg until the end of life of memcg. */ - struct obj_cgroup __rcu *objcg; struct obj_cgroup *orig_objcg; /* list of inherited objcgs, protected by objcg_lock */ struct list_head objcg_list; - -#ifdef CONFIG_MEMCG_NMI_SAFETY_REQUIRES_ATOMIC - /* slab stats for nmi context */ - atomic_t slab_reclaimable; - atomic_t slab_unreclaimable; -#endif + __cacheline_group_end_aligned(memcg_pn_cold); }; struct mem_cgroup_threshold { @@ -186,6 +180,7 @@ struct obj_cgroup { struct percpu_ref refcnt; struct mem_cgroup *memcg; atomic_t nr_charged_bytes; + refcount_t private_id_ref; union { struct list_head list; /* protected by objcg_lock */ struct rcu_head rcu; @@ -202,9 +197,6 @@ struct obj_cgroup { struct mem_cgroup { struct cgroup_subsys_state css; - /* Private memcg ID. Used to ID objects that outlive the cgroup */ - struct mem_cgroup_private_id id; - /* Accounted resources */ struct page_counter memory; /* Both v1 & v2 */ @@ -213,70 +205,59 @@ struct mem_cgroup { struct page_counter memsw; /* v1 only */ }; - /* registered local peak watchers */ - struct list_head memory_peaks; - struct list_head swap_peaks; - spinlock_t peaks_lock; - - /* Range enforcement for interrupt charges */ - struct work_struct high_work; - -#ifdef CONFIG_ZSWAP - unsigned long zswap_max; - + /* Written on the charge, reclaim and socket paths. */ + __cacheline_group_begin_aligned(memcg_write_hot); /* - * Prevent pages from this memcg from being written back from zswap to - * swap, and from being swapped out on zswap store failures. + * Hint of reclaim pressure for socket memory management. Note + * that this indicator should NOT be used in legacy cgroup mode + * where socket memory is accounted/charged separately. */ - bool zswap_writeback; + u64 socket_pressure; +#if BITS_PER_LONG < 64 + seqlock_t socket_pressure_seqlock; #endif - - /* vmpressure notifications */ - struct vmpressure vmpressure; - /* - * Should the OOM killer kill all belonging tasks, had it kill one? + * memory.events is bumped for this memcg and all its ancestors, so a + * busy child dirties every ancestor. */ - bool oom_group; - - /* memory.events and memory.events.local */ - struct cgroup_file events_file; - struct cgroup_file events_local_file; - - /* handle for "memory.swap.events" */ - struct cgroup_file swap_events_file; - - /* memory.stat */ - struct memcg_vmstats *vmstats; - - /* memory.events */ atomic_long_t memory_events[MEMCG_NR_MEMORY_EVENTS]; atomic_long_t memory_events_local[MEMCG_NR_MEMORY_EVENTS]; + /* vmpressure notifications. Written on every reclaim iteration. */ + struct vmpressure vmpressure; + #ifdef CONFIG_MEMCG_NMI_SAFETY_REQUIRES_ATOMIC /* MEMCG_KMEM for nmi context */ atomic_t kmem_stat; #endif + + /* Range enforcement for interrupt charges */ + struct irq_work high_irq_work; + struct work_struct high_work; + + __cacheline_group_end_aligned(memcg_write_hot); + /* - * Hint of reclaim pressure for socket memroy management. Note - * that this indicator should NOT be used in legacy cgroup mode - * where socket memory is accounted/charged separately. + * Off the charge and fault paths. Not write free: cgwb_domain is + * written on every writeout completion and mm_list on fork, exit and + * MGLRU aging. They are grouped here so those writes cannot land on + * a line that the fast paths read. */ - u64 socket_pressure; -#if BITS_PER_LONG < 64 - seqlock_t socket_pressure_seqlock; -#endif - int kmemcg_id; + __cacheline_group_begin_aligned(memcg_cold); + /* registered local peak watchers */ + struct list_head memory_peaks; + struct list_head swap_peaks; + spinlock_t peaks_lock; -#ifdef CONFIG_CGROUP_WRITEBACK - struct list_head cgwb_list; -#endif + /* memory.events and memory.events.local */ + struct cgroup_file events_file; + struct cgroup_file events_local_file; - /* Keep the hot per-CPU stats pointer away from memory event counters. */ - struct memcg_vmstats_percpu __percpu *vmstats_percpu - ____cacheline_aligned_in_smp; + /* handle for "memory.swap.events" */ + struct cgroup_file swap_events_file; #ifdef CONFIG_CGROUP_WRITEBACK + struct list_head cgwb_list; struct wb_domain cgwb_domain; struct memcg_cgwb_frn cgwb_frn[MEMCG_CGWB_FRN_CNT]; #endif @@ -285,16 +266,17 @@ struct mem_cgroup { /* per-memcg mm_struct list */ struct lru_gen_mm_list mm_list; #endif + __cacheline_group_end_aligned(memcg_cold); #ifdef CONFIG_MEMCG_V1 + /* v1 only. Not grouped: v1 is legacy, sorting it is not worth it. */ + /* Legacy consumer-oriented counters */ struct page_counter kmem; /* v1 only */ struct page_counter tcpmem; /* v1 only */ struct memcg1_events_percpu __percpu *events_percpu; - unsigned long soft_limit; - /* protected by memcg_oom_lock */ bool oom_lock; int under_oom; @@ -325,6 +307,44 @@ struct mem_cgroup { int swappiness; #endif /* CONFIG_MEMCG_V1 */ + /* + * Set when the memcg is created and cleared when it is offlined. + * Never written on a hot path. + */ + __cacheline_group_begin_aligned(memcg_read_mostly); + /* Read on every stat update */ + struct memcg_vmstats_percpu __percpu *vmstats_percpu; + + /* memory.stat */ + struct memcg_vmstats *vmstats; + +#ifdef CONFIG_ZSWAP + unsigned long zswap_max; +#endif + + /* The objcg holding private memcg ID. */ + struct obj_cgroup *private_id_objcg; + + /* Private memcg ID. Used to ID objects that outlive the cgroup */ + int private_id; + + int kmemcg_id; + + /* + * Should the OOM killer kill all belonging tasks, had it kill one? + */ + bool oom_group; + +#ifdef CONFIG_ZSWAP + /* + * Prevent pages from this memcg from being written back from zswap to + * swap, and from being swapped out on zswap store failures. + */ + bool zswap_writeback; +#endif + /* Not padded: nodeinfo[] is read-mostly too, let it share the line. */ + __cacheline_group_end(memcg_read_mostly); + struct mem_cgroup_per_node *nodeinfo[]; }; @@ -377,10 +397,10 @@ enum objext_flags { * * The caller must ensure that the returned memcg won't be released. */ -static inline struct mem_cgroup *obj_cgroup_memcg(struct obj_cgroup *objcg) +static inline struct mem_cgroup *obj_cgroup_memcg(const struct obj_cgroup *objcg) { lockdep_assert_once(rcu_read_lock_held() || lockdep_is_held(&cgroup_mutex)); - return READ_ONCE(objcg->memcg); + return objcg ? READ_ONCE(objcg->memcg) : NULL; } /* @@ -391,7 +411,7 @@ static inline struct mem_cgroup *obj_cgroup_memcg(struct obj_cgroup *objcg) * or NULL. This function assumes that the folio is known to have a * proper object cgroup pointer. */ -static inline struct obj_cgroup *folio_objcg(struct folio *folio) +static inline struct obj_cgroup *folio_objcg(const struct folio *folio) { unsigned long memcg_data = folio->memcg_data; @@ -429,11 +449,11 @@ static inline struct obj_cgroup *folio_objcg(struct folio *folio) * Note: The caller should hold an rcu read lock or cgroup_mutex to protect * memcg associated with a folio from being released. */ -static inline struct mem_cgroup *folio_memcg(struct folio *folio) +static inline struct mem_cgroup *folio_memcg(const struct folio *folio) { struct obj_cgroup *objcg = folio_objcg(folio); - return objcg ? obj_cgroup_memcg(objcg) : NULL; + return obj_cgroup_memcg(objcg); } /* @@ -442,7 +462,7 @@ static inline struct mem_cgroup *folio_memcg(struct folio *folio) * * Returns true if folio is charged to a memory cgroup, otherwise returns false. */ -static inline bool folio_memcg_charged(struct folio *folio) +static inline bool folio_memcg_charged(const struct folio *folio) { return folio->memcg_data != 0; } @@ -462,7 +482,7 @@ static inline bool folio_memcg_charged(struct folio *folio) * A caller should hold an rcu read lock to protect memcg associated with a * page from being released. */ -static inline struct mem_cgroup *folio_memcg_check(struct folio *folio) +static inline struct mem_cgroup *folio_memcg_check(const struct folio *folio) { /* * Because folio->memcg_data might be changed asynchronously @@ -476,17 +496,17 @@ static inline struct mem_cgroup *folio_memcg_check(struct folio *folio) objcg = (void *)(memcg_data & ~OBJEXTS_FLAGS_MASK); - return objcg ? obj_cgroup_memcg(objcg) : NULL; + return obj_cgroup_memcg(objcg); } -static inline struct mem_cgroup *page_memcg_check(struct page *page) +static inline struct mem_cgroup *page_memcg_check(const struct page *page) { if (PageTail(page)) return NULL; - return folio_memcg_check((struct folio *)page); + return folio_memcg_check((const struct folio *)page); } -static inline struct mem_cgroup *get_mem_cgroup_from_objcg(struct obj_cgroup *objcg) +static inline struct mem_cgroup *get_mem_cgroup_from_objcg(const struct obj_cgroup *objcg) { struct mem_cgroup *memcg; @@ -508,19 +528,19 @@ retry: * that the folio has an associated memory cgroup. It's not safe to call * this function against some types of folios, e.g. slab folios. */ -static inline bool folio_memcg_kmem(struct folio *folio) +static inline bool folio_memcg_kmem(const struct folio *folio) { VM_BUG_ON_PGFLAGS(PageTail(&folio->page), &folio->page); VM_BUG_ON_FOLIO(folio->memcg_data & MEMCG_DATA_OBJEXTS, folio); return folio->memcg_data & MEMCG_DATA_KMEM; } -static inline bool PageMemcgKmem(struct page *page) +static inline bool PageMemcgKmem(const struct page *page) { return folio_memcg_kmem(page_folio(page)); } -static inline bool mem_cgroup_is_root(struct mem_cgroup *memcg) +static inline bool mem_cgroup_is_root(const struct mem_cgroup *memcg) { return (memcg == root_mem_cgroup); } @@ -536,7 +556,7 @@ static inline bool mem_cgroup_is_root(struct mem_cgroup *memcg) * and do not honour sc->memcg can use this to early-return 0 in per-memcg * contexts. */ -static inline bool mem_cgroup_shrink_is_root(struct shrink_control *sc) +static inline bool mem_cgroup_shrink_is_root(const struct shrink_control *sc) { return !sc->memcg || mem_cgroup_is_root(sc->memcg); } @@ -551,8 +571,8 @@ static inline bool mem_cgroup_disabled(void) return !cgroup_subsys_enabled(memory_cgrp_subsys); } -static inline void mem_cgroup_protection(struct mem_cgroup *root, - struct mem_cgroup *memcg, +static inline void mem_cgroup_protection(const struct mem_cgroup *root, + const struct mem_cgroup *memcg, unsigned long *min, unsigned long *low, unsigned long *usage) @@ -606,8 +626,8 @@ static inline void mem_cgroup_protection(struct mem_cgroup *root, void mem_cgroup_calculate_protection(struct mem_cgroup *root, struct mem_cgroup *memcg); -static inline bool mem_cgroup_unprotected(struct mem_cgroup *target, - struct mem_cgroup *memcg) +static inline bool mem_cgroup_unprotected(const struct mem_cgroup *target, + const struct mem_cgroup *memcg) { /* * The root memcg doesn't account charges, and doesn't support @@ -618,8 +638,8 @@ static inline bool mem_cgroup_unprotected(struct mem_cgroup *target, memcg == target; } -static inline bool mem_cgroup_below_low(struct mem_cgroup *target, - struct mem_cgroup *memcg) +static inline bool mem_cgroup_below_low(const struct mem_cgroup *target, + const struct mem_cgroup *memcg) { if (mem_cgroup_unprotected(target, memcg)) return false; @@ -628,8 +648,8 @@ static inline bool mem_cgroup_below_low(struct mem_cgroup *target, page_counter_read(&memcg->memory); } -static inline bool mem_cgroup_below_min(struct mem_cgroup *target, - struct mem_cgroup *memcg) +static inline bool mem_cgroup_below_min(const struct mem_cgroup *target, + const struct mem_cgroup *memcg) { if (mem_cgroup_unprotected(target, memcg)) return false; @@ -662,7 +682,8 @@ static inline int mem_cgroup_charge(struct folio *folio, struct mm_struct *mm, return __mem_cgroup_charge(folio, mm, gfp); } -int mem_cgroup_charge_hugetlb(struct folio* folio, gfp_t gfp); +int mem_cgroup_charge_hugetlb(struct folio *folio, struct mm_struct *mm, + gfp_t gfp); int mem_cgroup_swapin_charge_folio(struct folio *folio, unsigned short id, struct mm_struct *mm, gfp_t gfp); @@ -702,7 +723,7 @@ void mem_cgroup_migrate(struct folio *old, struct folio *new); * @pgdat combination. This can be the node lruvec, if the memory * controller is disabled. */ -static inline struct lruvec *mem_cgroup_lruvec(struct mem_cgroup *memcg, +static inline struct lruvec *mem_cgroup_lruvec(const struct mem_cgroup *memcg, struct pglist_data *pgdat) { struct mem_cgroup_per_node *mz; @@ -743,7 +764,7 @@ out: * their binding is stable if the returned lruvec matches the one the caller has * locked. Useful for lock batching. */ -static inline struct lruvec *folio_lruvec(struct folio *folio) +static inline struct lruvec *folio_lruvec(const struct folio *folio) { struct mem_cgroup *memcg = folio_memcg(folio); @@ -757,11 +778,11 @@ struct mem_cgroup *get_mem_cgroup_from_mm(struct mm_struct *mm); struct mem_cgroup *get_mem_cgroup_from_current(void); -struct mem_cgroup *get_mem_cgroup_from_folio(struct folio *folio); +struct mem_cgroup *get_mem_cgroup_from_folio(const struct folio *folio); -struct lruvec *folio_lruvec_lock(struct folio *folio); -struct lruvec *folio_lruvec_lock_irq(struct folio *folio); -struct lruvec *folio_lruvec_lock_irqsave(struct folio *folio, +struct lruvec *folio_lruvec_lock(const struct folio *folio); +struct lruvec *folio_lruvec_lock_irq(const struct folio *folio); +struct lruvec *folio_lruvec_lock_irqsave(const struct folio *folio, unsigned long *flags); static inline @@ -820,16 +841,16 @@ void mem_cgroup_iter_break(struct mem_cgroup *, struct mem_cgroup *); void mem_cgroup_scan_tasks(struct mem_cgroup *memcg, int (*)(struct task_struct *, void *), void *arg); -static inline unsigned short mem_cgroup_private_id(struct mem_cgroup *memcg) +static inline unsigned short mem_cgroup_private_id(const struct mem_cgroup *memcg) { if (mem_cgroup_disabled()) return 0; - return memcg->id.id; + return memcg->private_id; } struct mem_cgroup *mem_cgroup_from_private_id(unsigned short id); -static inline u64 mem_cgroup_id(struct mem_cgroup *memcg) +static inline u64 mem_cgroup_id(const struct mem_cgroup *memcg) { return memcg ? cgroup_id(memcg->css.cgroup) : 0; } @@ -841,14 +862,14 @@ static inline struct mem_cgroup *mem_cgroup_from_seq(struct seq_file *m) return mem_cgroup_from_css(seq_css(m)); } -static inline struct mem_cgroup *lruvec_memcg(struct lruvec *lruvec) +static inline struct mem_cgroup *lruvec_memcg(const struct lruvec *lruvec) { - struct mem_cgroup_per_node *mz; + const struct mem_cgroup_per_node *mz; if (mem_cgroup_disabled()) return NULL; - mz = container_of(lruvec, struct mem_cgroup_per_node, lruvec); + mz = container_of_const(lruvec, struct mem_cgroup_per_node, lruvec); return mz->memcg; } @@ -858,13 +879,13 @@ static inline struct mem_cgroup *lruvec_memcg(struct lruvec *lruvec) * * Returns the parent memcg, or NULL if this is the root. */ -static inline struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *memcg) +static inline struct mem_cgroup *parent_mem_cgroup(const struct mem_cgroup *memcg) { return mem_cgroup_from_css(memcg->css.parent); } -static inline bool mem_cgroup_is_descendant(struct mem_cgroup *memcg, - struct mem_cgroup *root) +static inline bool mem_cgroup_is_descendant(const struct mem_cgroup *memcg, + const struct mem_cgroup *root) { if (root == memcg) return true; @@ -872,7 +893,7 @@ static inline bool mem_cgroup_is_descendant(struct mem_cgroup *memcg, } static inline bool mm_match_cgroup(struct mm_struct *mm, - struct mem_cgroup *memcg) + const struct mem_cgroup *memcg) { struct mem_cgroup *task_memcg; bool match = false; @@ -885,8 +906,8 @@ static inline bool mm_match_cgroup(struct mm_struct *mm, return match; } -struct cgroup_subsys_state *get_mem_cgroup_css_from_folio(struct folio *folio); -ino_t page_cgroup_ino(struct page *page); +struct cgroup_subsys_state *get_mem_cgroup_css_from_folio(const struct folio *folio); +ino_t page_cgroup_ino(const struct page *page); static inline bool mem_cgroup_online(struct mem_cgroup *memcg) { @@ -899,13 +920,18 @@ void mem_cgroup_update_lru_size(struct lruvec *lruvec, enum lru_list lru, int zid, long nr_pages); static inline -unsigned long mem_cgroup_get_zone_lru_size(struct lruvec *lruvec, +unsigned long mem_cgroup_get_zone_lru_size(const struct lruvec *lruvec, enum lru_list lru, int zone_idx) { - struct mem_cgroup_per_node *mz; + long val; + const struct mem_cgroup_per_node *mz; + + mz = container_of_const(lruvec, struct mem_cgroup_per_node, lruvec); + val = READ_ONCE(mz->lru_zone_size[zone_idx][lru]); + if (WARN_ON_ONCE(val < 0)) + return 0; - mz = container_of(lruvec, struct mem_cgroup_per_node, lruvec); - return READ_ONCE(mz->lru_zone_size[zone_idx][lru]); + return val; } void __mem_cgroup_handle_over_high(gfp_t gfp_mask); @@ -916,9 +942,9 @@ static inline void mem_cgroup_handle_over_high(gfp_t gfp_mask) __mem_cgroup_handle_over_high(gfp_mask); } -unsigned long mem_cgroup_get_max(struct mem_cgroup *memcg); +unsigned long mem_cgroup_get_max(const struct mem_cgroup *memcg); -void mem_cgroup_print_oom_context(struct mem_cgroup *memcg, +void mem_cgroup_print_oom_context(const struct mem_cgroup *memcg, struct task_struct *p); void mem_cgroup_print_oom_meminfo(struct mem_cgroup *memcg); @@ -931,7 +957,7 @@ void mem_cgroup_print_oom_group(struct mem_cgroup *memcg); void mod_memcg_state(struct mem_cgroup *memcg, enum memcg_stat_item idx, int val); -static inline void mod_memcg_page_state(struct page *page, +static inline void mod_memcg_page_state(const struct page *page, enum memcg_stat_item idx, int val) { struct mem_cgroup *memcg; @@ -946,17 +972,21 @@ static inline void mod_memcg_page_state(struct page *page, rcu_read_unlock(); } -unsigned long memcg_events(struct mem_cgroup *memcg, int event); -unsigned long memcg_page_state(struct mem_cgroup *memcg, int idx); -unsigned long memcg_page_state_output(struct mem_cgroup *memcg, int item); +unsigned long memcg_events(const struct mem_cgroup *memcg, int event); +unsigned long memcg_page_state(const struct mem_cgroup *memcg, int idx); +unsigned long memcg_page_state_output(const struct mem_cgroup *memcg, int item); bool memcg_stat_item_valid(int idx); bool memcg_vm_event_item_valid(enum vm_event_item idx); -unsigned long lruvec_page_state(struct lruvec *lruvec, enum node_stat_item idx); -unsigned long lruvec_page_state_monotonic(struct lruvec *lruvec, +unsigned long lruvec_page_state(const struct lruvec *lruvec, + enum node_stat_item idx); +unsigned long lruvec_page_state_monotonic(const struct lruvec *lruvec, enum node_stat_item idx); -unsigned long lruvec_page_state_local(struct lruvec *lruvec, +unsigned long lruvec_page_state_local(const struct lruvec *lruvec, enum node_stat_item idx); +void mod_memcg_lruvec_state(struct lruvec *lruvec, + enum node_stat_item idx, int val); + void mem_cgroup_flush_stats(struct mem_cgroup *memcg); void mem_cgroup_flush_stats_ratelimited(struct mem_cgroup *memcg); @@ -965,7 +995,7 @@ void mod_lruvec_kmem_state(void *p, enum node_stat_item idx, int val); void count_memcg_events(struct mem_cgroup *memcg, enum vm_event_item idx, unsigned long count); -static inline void count_memcg_folio_events(struct folio *folio, +static inline void count_memcg_folio_events(const struct folio *folio, enum vm_event_item idx, unsigned long nr) { struct mem_cgroup *memcg; @@ -1050,51 +1080,56 @@ void mem_cgroup_flush_workqueue(void); extern int mem_cgroup_init(void); #else /* CONFIG_MEMCG */ +static inline struct mem_cgroup *obj_cgroup_memcg(const struct obj_cgroup *objcg) +{ + return NULL; +} + #define MEM_CGROUP_ID_SHIFT 0 #define root_mem_cgroup (NULL) -static inline struct mem_cgroup *folio_memcg(struct folio *folio) +static inline struct mem_cgroup *folio_memcg(const struct folio *folio) { return NULL; } -static inline bool folio_memcg_charged(struct folio *folio) +static inline bool folio_memcg_charged(const struct folio *folio) { return false; } -static inline struct mem_cgroup *folio_memcg_check(struct folio *folio) +static inline struct mem_cgroup *folio_memcg_check(const struct folio *folio) { return NULL; } -static inline struct mem_cgroup *page_memcg_check(struct page *page) +static inline struct mem_cgroup *page_memcg_check(const struct page *page) { return NULL; } -static inline struct mem_cgroup *get_mem_cgroup_from_objcg(struct obj_cgroup *objcg) +static inline struct mem_cgroup *get_mem_cgroup_from_objcg(const struct obj_cgroup *objcg) { return NULL; } -static inline bool folio_memcg_kmem(struct folio *folio) +static inline bool folio_memcg_kmem(const struct folio *folio) { return false; } -static inline bool PageMemcgKmem(struct page *page) +static inline bool PageMemcgKmem(const struct page *page) { return false; } -static inline bool mem_cgroup_is_root(struct mem_cgroup *memcg) +static inline bool mem_cgroup_is_root(const struct mem_cgroup *memcg) { return true; } -static inline bool mem_cgroup_shrink_is_root(struct shrink_control *sc) +static inline bool mem_cgroup_shrink_is_root(const struct shrink_control *sc) { return true; } @@ -1119,8 +1154,8 @@ static inline void memcg_memory_event_mm(struct mm_struct *mm, { } -static inline void mem_cgroup_protection(struct mem_cgroup *root, - struct mem_cgroup *memcg, +static inline void mem_cgroup_protection(const struct mem_cgroup *root, + const struct mem_cgroup *memcg, unsigned long *min, unsigned long *low, unsigned long *usage) @@ -1133,19 +1168,19 @@ static inline void mem_cgroup_calculate_protection(struct mem_cgroup *root, { } -static inline bool mem_cgroup_unprotected(struct mem_cgroup *target, - struct mem_cgroup *memcg) +static inline bool mem_cgroup_unprotected(const struct mem_cgroup *target, + const struct mem_cgroup *memcg) { return true; } -static inline bool mem_cgroup_below_low(struct mem_cgroup *target, - struct mem_cgroup *memcg) +static inline bool mem_cgroup_below_low(const struct mem_cgroup *target, + const struct mem_cgroup *memcg) { return false; } -static inline bool mem_cgroup_below_min(struct mem_cgroup *target, - struct mem_cgroup *memcg) +static inline bool mem_cgroup_below_min(const struct mem_cgroup *target, + const struct mem_cgroup *memcg) { return false; } @@ -1156,9 +1191,10 @@ static inline int mem_cgroup_charge(struct folio *folio, return 0; } -static inline int mem_cgroup_charge_hugetlb(struct folio* folio, gfp_t gfp) +static inline int mem_cgroup_charge_hugetlb(struct folio *folio, + struct mm_struct *mm, gfp_t gfp) { - return 0; + return 0; } static inline int mem_cgroup_swapin_charge_folio(struct folio *folio, @@ -1184,25 +1220,25 @@ static inline void mem_cgroup_migrate(struct folio *old, struct folio *new) { } -static inline struct lruvec *mem_cgroup_lruvec(struct mem_cgroup *memcg, +static inline struct lruvec *mem_cgroup_lruvec(const struct mem_cgroup *memcg, struct pglist_data *pgdat) { return &pgdat->__lruvec; } -static inline struct lruvec *folio_lruvec(struct folio *folio) +static inline struct lruvec *folio_lruvec(const struct folio *folio) { struct pglist_data *pgdat = folio_pgdat(folio); return &pgdat->__lruvec; } -static inline struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *memcg) +static inline struct mem_cgroup *parent_mem_cgroup(const struct mem_cgroup *memcg) { return NULL; } static inline bool mm_match_cgroup(struct mm_struct *mm, - struct mem_cgroup *memcg) + const struct mem_cgroup *memcg) { return true; } @@ -1217,7 +1253,7 @@ static inline struct mem_cgroup *get_mem_cgroup_from_current(void) return NULL; } -static inline struct mem_cgroup *get_mem_cgroup_from_folio(struct folio *folio) +static inline struct mem_cgroup *get_mem_cgroup_from_folio(const struct folio *folio) { return NULL; } @@ -1250,7 +1286,7 @@ static inline void mem_cgroup_put(struct mem_cgroup *memcg) { } -static inline struct lruvec *folio_lruvec_lock(struct folio *folio) +static inline struct lruvec *folio_lruvec_lock(const struct folio *folio) { struct pglist_data *pgdat = folio_pgdat(folio); @@ -1259,7 +1295,7 @@ static inline struct lruvec *folio_lruvec_lock(struct folio *folio) return &pgdat->__lruvec; } -static inline struct lruvec *folio_lruvec_lock_irq(struct folio *folio) +static inline struct lruvec *folio_lruvec_lock_irq(const struct folio *folio) { struct pglist_data *pgdat = folio_pgdat(folio); @@ -1268,7 +1304,7 @@ static inline struct lruvec *folio_lruvec_lock_irq(struct folio *folio) return &pgdat->__lruvec; } -static inline struct lruvec *folio_lruvec_lock_irqsave(struct folio *folio, +static inline struct lruvec *folio_lruvec_lock_irqsave(const struct folio *folio, unsigned long *flagsp) { struct pglist_data *pgdat = folio_pgdat(folio); @@ -1296,7 +1332,7 @@ static inline void mem_cgroup_scan_tasks(struct mem_cgroup *memcg, { } -static inline unsigned short mem_cgroup_private_id(struct mem_cgroup *memcg) +static inline unsigned short mem_cgroup_private_id(const struct mem_cgroup *memcg) { return 0; } @@ -1308,7 +1344,7 @@ static inline struct mem_cgroup *mem_cgroup_from_private_id(unsigned short id) return NULL; } -static inline u64 mem_cgroup_id(struct mem_cgroup *memcg) +static inline u64 mem_cgroup_id(const struct mem_cgroup *memcg) { return 0; } @@ -1323,7 +1359,7 @@ static inline struct mem_cgroup *mem_cgroup_from_seq(struct seq_file *m) return NULL; } -static inline struct mem_cgroup *lruvec_memcg(struct lruvec *lruvec) +static inline struct mem_cgroup *lruvec_memcg(const struct lruvec *lruvec) { return NULL; } @@ -1334,19 +1370,20 @@ static inline bool mem_cgroup_online(struct mem_cgroup *memcg) } static inline -unsigned long mem_cgroup_get_zone_lru_size(struct lruvec *lruvec, +unsigned long mem_cgroup_get_zone_lru_size(const struct lruvec *lruvec, enum lru_list lru, int zone_idx) { return 0; } -static inline unsigned long mem_cgroup_get_max(struct mem_cgroup *memcg) +static inline unsigned long mem_cgroup_get_max(const struct mem_cgroup *memcg) { return 0; } static inline void -mem_cgroup_print_oom_context(struct mem_cgroup *memcg, struct task_struct *p) +mem_cgroup_print_oom_context(const struct mem_cgroup *memcg, + struct task_struct *p) { } @@ -1375,17 +1412,17 @@ static inline void mod_memcg_state(struct mem_cgroup *memcg, { } -static inline void mod_memcg_page_state(struct page *page, +static inline void mod_memcg_page_state(const struct page *page, enum memcg_stat_item idx, int val) { } -static inline unsigned long memcg_page_state(struct mem_cgroup *memcg, int idx) +static inline unsigned long memcg_page_state(const struct mem_cgroup *memcg, int idx) { return 0; } -static inline unsigned long memcg_page_state_output(struct mem_cgroup *memcg, int item) +static inline unsigned long memcg_page_state_output(const struct mem_cgroup *memcg, int item) { return 0; } @@ -1400,24 +1437,29 @@ static inline bool memcg_vm_event_item_valid(enum vm_event_item idx) return false; } -static inline unsigned long lruvec_page_state(struct lruvec *lruvec, +static inline unsigned long lruvec_page_state(const struct lruvec *lruvec, enum node_stat_item idx) { return node_page_state(lruvec_pgdat(lruvec), idx); } -static inline unsigned long lruvec_page_state_monotonic(struct lruvec *lruvec, +static inline unsigned long lruvec_page_state_monotonic(const struct lruvec *lruvec, enum node_stat_item idx) { return node_page_state_monotonic(lruvec_pgdat(lruvec), idx); } -static inline unsigned long lruvec_page_state_local(struct lruvec *lruvec, +static inline unsigned long lruvec_page_state_local(const struct lruvec *lruvec, enum node_stat_item idx) { return node_page_state(lruvec_pgdat(lruvec), idx); } +static inline void mod_memcg_lruvec_state(struct lruvec *lruvec, + enum node_stat_item idx, int val) +{ +} + static inline void mem_cgroup_flush_stats(struct mem_cgroup *memcg) { } @@ -1440,7 +1482,7 @@ static inline void count_memcg_events(struct mem_cgroup *memcg, { } -static inline void count_memcg_folio_events(struct folio *folio, +static inline void count_memcg_folio_events(const struct folio *folio, enum vm_event_item idx, unsigned long nr) { } @@ -1474,7 +1516,7 @@ static inline void mem_cgroup_flush_workqueue(void) { } static inline int mem_cgroup_init(void) { return 0; } #endif /* CONFIG_MEMCG */ -static inline struct lruvec *parent_lruvec(struct lruvec *lruvec) +static inline struct lruvec *parent_lruvec(const struct lruvec *lruvec) { struct mem_cgroup *memcg; @@ -1537,15 +1579,15 @@ static inline void lruvec_unlock_irqrestore(struct lruvec *lruvec, unsigned long } /* Test requires a stable folio->memcg binding, see folio_memcg() */ -static inline bool folio_matches_lruvec(struct folio *folio, - struct lruvec *lruvec) +static inline bool folio_matches_lruvec(const struct folio *folio, + const struct lruvec *lruvec) { return lruvec_pgdat(lruvec) == folio_pgdat(folio) && lruvec_memcg(lruvec) == folio_memcg(folio); } /* Don't lock again iff page's lruvec locked */ -static inline struct lruvec *folio_lruvec_relock_irq(struct folio *folio, +static inline struct lruvec *folio_lruvec_relock_irq(const struct folio *folio, struct lruvec *locked_lruvec) { if (locked_lruvec) { @@ -1559,7 +1601,7 @@ static inline struct lruvec *folio_lruvec_relock_irq(struct folio *folio, } /* Don't lock again iff folio's lruvec locked */ -static inline void folio_lruvec_relock_irqsave(struct folio *folio, +static inline void folio_lruvec_relock_irqsave(const struct folio *folio, struct lruvec **lruvecp, unsigned long *flags) { if (*lruvecp) { @@ -1651,7 +1693,7 @@ static inline void mem_cgroup_set_socket_pressure(struct mem_cgroup *memcg) write_sequnlock_irqrestore(&memcg->socket_pressure_seqlock, flags); } -static inline u64 mem_cgroup_get_socket_pressure(struct mem_cgroup *memcg) +static inline u64 mem_cgroup_get_socket_pressure(const struct mem_cgroup *memcg) { unsigned int seq; u64 val; @@ -1669,12 +1711,13 @@ static inline void mem_cgroup_set_socket_pressure(struct mem_cgroup *memcg) WRITE_ONCE(memcg->socket_pressure, jiffies + HZ); } -static inline u64 mem_cgroup_get_socket_pressure(struct mem_cgroup *memcg) +static inline u64 mem_cgroup_get_socket_pressure(const struct mem_cgroup *memcg) { return READ_ONCE(memcg->socket_pressure); } #endif +void shrinker_unit_cache_init(void); int alloc_shrinker_info(struct mem_cgroup *memcg); void free_shrinker_info(struct mem_cgroup *memcg); void set_shrinker_bit(struct mem_cgroup *memcg, int nid, int shrinker_id); @@ -1736,7 +1779,7 @@ void __memcg_kmem_uncharge_page(struct page *page, int order); * needs to be used outside of the local scope. */ struct obj_cgroup *current_obj_cgroup(void); -struct obj_cgroup *get_obj_cgroup_from_folio(struct folio *folio); +struct obj_cgroup *get_obj_cgroup_from_folio(const struct folio *folio); static inline struct obj_cgroup *get_obj_cgroup_from_current(void) { @@ -1782,7 +1825,7 @@ static inline void memcg_kmem_uncharge_page(struct page *page, int order) * A helper for accessing memcg's kmem_id, used for getting * corresponding LRU lists. */ -static inline int memcg_kmem_id(struct mem_cgroup *memcg) +static inline int memcg_kmem_id(const struct mem_cgroup *memcg) { return memcg ? memcg->kmemcg_id : -1; } @@ -1839,7 +1882,7 @@ static inline void __memcg_kmem_uncharge_page(struct page *page, int order) { } -static inline struct obj_cgroup *get_obj_cgroup_from_folio(struct folio *folio) +static inline struct obj_cgroup *get_obj_cgroup_from_folio(const struct folio *folio) { return NULL; } @@ -1854,7 +1897,7 @@ static inline bool memcg_kmem_online(void) return false; } -static inline int memcg_kmem_id(struct mem_cgroup *memcg) +static inline int memcg_kmem_id(const struct mem_cgroup *memcg) { return -1; } @@ -1870,7 +1913,7 @@ static inline void count_objcg_events(struct obj_cgroup *objcg, { } -static inline ino_t page_cgroup_ino(struct page *page) +static inline ino_t page_cgroup_ino(const struct page *page) { return 0; } @@ -1890,11 +1933,21 @@ static inline bool memcg_is_dying(struct mem_cgroup *memcg) } #endif /* CONFIG_MEMCG */ +#if defined(CONFIG_MEMCG) && defined(CONFIG_LRU_GEN) +void mem_cgroup_calculate_protection_path(struct mem_cgroup *root, + struct mem_cgroup *memcg); +#else +static inline void mem_cgroup_calculate_protection_path(struct mem_cgroup *root, + struct mem_cgroup *memcg) +{ +} +#endif + #if defined(CONFIG_MEMCG) && defined(CONFIG_ZSWAP) bool obj_cgroup_may_zswap(struct obj_cgroup *objcg); void obj_cgroup_charge_zswap(struct obj_cgroup *objcg, size_t size); void obj_cgroup_uncharge_zswap(struct obj_cgroup *objcg, size_t size); -bool mem_cgroup_zswap_writeback_enabled(struct mem_cgroup *memcg); +bool mem_cgroup_zswap_writeback_enabled(const struct mem_cgroup *memcg); #else static inline bool obj_cgroup_may_zswap(struct obj_cgroup *objcg) { @@ -1908,7 +1961,7 @@ static inline void obj_cgroup_uncharge_zswap(struct obj_cgroup *objcg, size_t size) { } -static inline bool mem_cgroup_zswap_writeback_enabled(struct mem_cgroup *memcg) +static inline bool mem_cgroup_zswap_writeback_enabled(const struct mem_cgroup *memcg) { /* if zswap is disabled, do not block pages going to the swapping device */ return true; @@ -1919,13 +1972,9 @@ static inline bool mem_cgroup_zswap_writeback_enabled(struct mem_cgroup *memcg) /* Cgroup v1-related declarations */ #ifdef CONFIG_MEMCG_V1 -unsigned long memcg1_soft_limit_reclaim(pg_data_t *pgdat, int order, - gfp_t gfp_mask, - unsigned long *total_scanned); - bool mem_cgroup_oom_synchronize(bool wait); -static inline bool task_in_memcg_oom(struct task_struct *p) +static inline bool task_in_memcg_oom(const struct task_struct *p) { return p->memcg_in_oom; } @@ -1943,15 +1992,7 @@ static inline void mem_cgroup_exit_user_fault(void) } #else /* CONFIG_MEMCG_V1 */ -static inline -unsigned long memcg1_soft_limit_reclaim(pg_data_t *pgdat, int order, - gfp_t gfp_mask, - unsigned long *total_scanned) -{ - return 0; -} - -static inline bool task_in_memcg_oom(struct task_struct *p) +static inline bool task_in_memcg_oom(const struct task_struct *p) { return false; } diff --git a/include/linux/mfd/88pm886.h b/include/linux/mfd/88pm886.h index 2c24dd3032ab..9e96d2cb92f5 100644 --- a/include/linux/mfd/88pm886.h +++ b/include/linux/mfd/88pm886.h @@ -2,6 +2,7 @@ #ifndef __MFD_88PM886_H #define __MFD_88PM886_H +#include <linux/bits.h> #include <linux/i2c.h> #include <linux/regmap.h> @@ -130,6 +131,12 @@ #define PM886_GPADC_INDEX_TO_BIAS_uA(i) (1 + (i) * 5) /* Battery block register definitions */ +#define PM886_REG_BATTERY_CONFIG1 0x28 +#define PM886_REG_VBUS_EN BIT(7) + +#define PM886_REG_BOOST_CONFIG1 0x6b +#define PM886_REG_BOOST_MASK GENMASK(2, 0) + #define PM886_REG_CLS_CONFIG1 0x71 struct pm886_chip { diff --git a/include/linux/mfd/cs42l43-regs.h b/include/linux/mfd/cs42l43-regs.h index 68831f113589..4c00ceae8b46 100644 --- a/include/linux/mfd/cs42l43-regs.h +++ b/include/linux/mfd/cs42l43-regs.h @@ -1183,6 +1183,7 @@ /* CS42L43B VARIANT REGISTERS */ #define CS42L43B_DEVID_VAL 0x0042A43B +#define CS42L44_DEVID_VAL 0x00042A44 #define CS42L43B_DECIM_VOL_CTRL_CH1_CH2 0x00008280 #define CS42L43B_DECIM_VOL_CTRL_CH3_CH4 0x00008284 diff --git a/include/linux/mfd/da9063/registers.h b/include/linux/mfd/da9063/registers.h index 7b8364bd08a0..376c5d330599 100644 --- a/include/linux/mfd/da9063/registers.h +++ b/include/linux/mfd/da9063/registers.h @@ -14,7 +14,7 @@ #define DA9063_I2C_PAGE_SEL_SHIFT 1 #define DA9063_EVENT_REG_NUM 4 -/* Page selection I2C or SPI always in the begining of any page. */ +/* Page selection I2C or SPI always in the beginning of any page. */ /* Page 0 : I2C access 0x000 - 0x0FF SPI access 0x000 - 0x07F */ /* Page 1 : SPI access 0x080 - 0x0FF */ /* Page 2 : I2C access 0x100 - 0x1FF SPI access 0x100 - 0x17F */ diff --git a/include/linux/mfd/da9150/core.h b/include/linux/mfd/da9150/core.h index d116d5f3ef56..369698036d1a 100644 --- a/include/linux/mfd/da9150/core.h +++ b/include/linux/mfd/da9150/core.h @@ -65,6 +65,7 @@ struct da9150 { struct regmap_irq_chip_data *regmap_irq_data; int irq; int irq_base; + bool irq_wake_enabled; }; /* Device I/O - Query Interface for FG and standard register access */ diff --git a/include/linux/mfd/db8500-prcmu.h b/include/linux/mfd/db8500-prcmu.h index c939c9a1170a..87a28c4f1d81 100644 --- a/include/linux/mfd/db8500-prcmu.h +++ b/include/linux/mfd/db8500-prcmu.h @@ -337,31 +337,31 @@ enum sia_mmdsp_stat { * @SYSCLKOK_ERR: The SYSCLK is not available in the given time * @BOOT_ERR: Romcode has not validated the XP70 self reset in the given time * @ROMCODESAVECONTEXT: The Romcode didn.t correctly save it secure context - * @VARMHIGHSPEEDVALTO_ERR: The ARM high speed supply value transfered + * @VARMHIGHSPEEDVALTO_ERR: The ARM high speed supply value transferred * through I2C has not been correctly executed in the given time - * @VARMHIGHSPEEDACCESS_ERR: The command value of VarmHighSpeedVal transfered + * @VARMHIGHSPEEDACCESS_ERR: The command value of VarmHighSpeedVal transferred * through I2C has not been correctly executed in the given time - * @VARMLOWSPEEDVALTO_ERR:The ARM low speed supply value transfered through + * @VARMLOWSPEEDVALTO_ERR:The ARM low speed supply value transferred through * I2C has not been correctly executed in the given time - * @VARMLOWSPEEDACCESS_ERR: The command value of VarmLowSpeedVal transfered + * @VARMLOWSPEEDACCESS_ERR: The command value of VarmLowSpeedVal transferred * through I2C has not been correctly executed in the given time - * @VARMRETENTIONVALTO_ERR: The ARM retention supply value transfered through + * @VARMRETENTIONVALTO_ERR: The ARM retention supply value transferred through * I2C has not been correctly executed in the given time - * @VARMRETENTIONACCESS_ERR: The command value of VarmRetentionVal transfered + * @VARMRETENTIONACCESS_ERR: The command value of VarmRetentionVal transferred * through I2C has not been correctly executed in the given time - * @VAPEHIGHSPEEDVALTO_ERR: The APE highspeed supply value transfered through + * @VAPEHIGHSPEEDVALTO_ERR: The APE highspeed supply value transferred through * I2C has not been correctly executed in the given time - * @VSAFEHPVALTO_ERR: The SAFE high power supply value transfered through I2C + * @VSAFEHPVALTO_ERR: The SAFE high power supply value transferred through I2C * has not been correctly executed in the given time - * @VMODSEL1VALTO_ERR: The MODEM sel1 supply value transfered through I2C has + * @VMODSEL1VALTO_ERR: The MODEM sel1 supply value transferred through I2C has * not been correctly executed in the given time - * @VMODSEL2VALTO_ERR: The MODEM sel2 supply value transfered through I2C has + * @VMODSEL2VALTO_ERR: The MODEM sel2 supply value transferred through I2C has * not been correctly executed in the given time - * @VARMOFFACCESS_ERR: The command value of Varm ON/OFF transfered through + * @VARMOFFACCESS_ERR: The command value of Varm ON/OFF transferred through * I2C has not been correctly executed in the given time - * @VAPEOFFACCESS_ERR: The command value of Vape ON/OFF transfered through + * @VAPEOFFACCESS_ERR: The command value of Vape ON/OFF transferred through * I2C has not been correctly executed in the given time - * @VARMRETACCES_ERR: The command value of Varm retention ON/OFF transfered + * @VARMRETACCES_ERR: The command value of Varm retention ON/OFF transferred * through I2C has not been correctly executed in the given time * @CURAPPWRSTISNOTBOOT:Generated when Arm want to do power state transition * ApBoot to ApExecute but the power current state is not Apboot @@ -376,11 +376,11 @@ enum sia_mmdsp_stat { * been correctly executed in the given time * @ARMREGU2VALTO_ERR: The ArmRegu2 value transferred through I2C has not * been correctly executed in the given time - * @VAPEREGUVALTO_ERR: The VApeRegu value transfered through I2C has not + * @VAPEREGUVALTO_ERR: The VApeRegu value transferred through I2C has not * been correctly executed in the given time - * @VSMPS3REGUVALTO_ERR: The VSmps3Regu value transfered through I2C has not + * @VSMPS3REGUVALTO_ERR: The VSmps3Regu value transferred through I2C has not * been correctly executed in the given time - * @VMODREGUVALTO_ERR: The VModemRegu value transfered through I2C has not + * @VMODREGUVALTO_ERR: The VModemRegu value transferred through I2C has not * been correctly executed in the given time */ enum mbox_to_arm_err { diff --git a/include/linux/mfd/khadas-mcu.h b/include/linux/mfd/khadas-mcu.h index a99ba2ed0e4e..acd3291061b4 100644 --- a/include/linux/mfd/khadas-mcu.h +++ b/include/linux/mfd/khadas-mcu.h @@ -70,6 +70,13 @@ #define KHADAS_MCU_WOL_INIT_START_REG 0x87 /* WO */ #define KHADAS_MCU_CMD_FAN_STATUS_CTRL_REG 0x88 /* WO */ +/* VIM4 specific registers */ +#define KHADAS_MCU_VIM4_REST_CONF_REG 0x2c /* WO - reset EEPROM */ +#define KHADAS_MCU_VIM4_LED_ON_RAM_REG 0x89 /* WO - LED volatile */ +#define KHADAS_MCU_VIM4_FAN_CTRL_REG 0x8a /* WO */ +#define KHADAS_MCU_VIM4_WDT_EN_REG 0x8b /* WO */ +#define KHADAS_MCU_VIM4_SYS_RST_REG 0x91 /* WO */ + enum { KHADAS_BOARD_VIM1 = 0x1, KHADAS_BOARD_VIM2, @@ -80,7 +87,7 @@ enum { /** * struct khadas_mcu - Khadas MCU structure - * @device: device reference used for logs + * @dev: device reference used for logs * @regmap: register map */ struct khadas_mcu { @@ -88,4 +95,14 @@ struct khadas_mcu { struct regmap *regmap; }; +/** + * enum khadas_mcu_type - Khadas MCU hardware variant + * @KHADAS_MCU_GENERIC: VIM1, VIM2, VIM3, Edge, Edge-V (shared register map) + * @KHADAS_MCU_VIM4: VIM4 (extended register map, distinct fan/LED/WDT regs) + */ +enum khadas_mcu_type { + KHADAS_MCU_GENERIC = 1, + KHADAS_MCU_VIM4, +}; + #endif /* MFD_KHADAS_MCU_H */ diff --git a/include/linux/mfd/lm3533.h b/include/linux/mfd/lm3533.h index 69059a7a2ce5..8f72dd41e8f0 100644 --- a/include/linux/mfd/lm3533.h +++ b/include/linux/mfd/lm3533.h @@ -15,9 +15,14 @@ #define LM3533_ATTR_RW(_name) \ DEVICE_ATTR(_name, S_IRUGO | S_IWUSR , show_##_name, store_##_name) +#define LM3533_MAX_CURRENT_MIN 5000 +#define LM3533_MAX_CURRENT_MAX 29800 +#define LM3533_MAX_CURRENT_STEP 800 + struct device; struct gpio_desc; struct regmap; +struct regulator; struct lm3533 { struct device *dev; @@ -25,7 +30,10 @@ struct lm3533 { struct regmap *regmap; struct gpio_desc *hwen; - int irq; + struct regulator *vin_supply; + + u32 boost_ovp; + u32 boost_freq; unsigned have_als:1; unsigned have_backlights:1; @@ -33,67 +41,18 @@ struct lm3533 { }; struct lm3533_ctrlbank { - struct lm3533 *lm3533; + struct regmap *regmap; struct device *dev; int id; }; -struct lm3533_als_platform_data { - unsigned pwm_mode:1; /* PWM input mode (default analog) */ - u8 r_select; /* 1 - 127 (ignored in PWM-mode) */ -}; - -struct lm3533_bl_platform_data { - char *name; - u16 max_current; /* 5000 - 29800 uA (800 uA step) */ - u8 default_brightness; /* 0 - 255 */ - u8 pwm; /* 0 - 0x3f */ -}; - -struct lm3533_led_platform_data { - char *name; - const char *default_trigger; - u16 max_current; /* 5000 - 29800 uA (800 uA step) */ - u8 pwm; /* 0 - 0x3f */ -}; - -enum lm3533_boost_freq { - LM3533_BOOST_FREQ_500KHZ, - LM3533_BOOST_FREQ_1000KHZ, -}; - -enum lm3533_boost_ovp { - LM3533_BOOST_OVP_16V, - LM3533_BOOST_OVP_24V, - LM3533_BOOST_OVP_32V, - LM3533_BOOST_OVP_40V, -}; - -struct lm3533_platform_data { - enum lm3533_boost_ovp boost_ovp; - enum lm3533_boost_freq boost_freq; - - struct lm3533_als_platform_data *als; - - struct lm3533_bl_platform_data *backlights; - int num_backlights; - - struct lm3533_led_platform_data *leds; - int num_leds; -}; - -extern int lm3533_ctrlbank_enable(struct lm3533_ctrlbank *cb); -extern int lm3533_ctrlbank_disable(struct lm3533_ctrlbank *cb); - -extern int lm3533_ctrlbank_set_brightness(struct lm3533_ctrlbank *cb, u8 val); -extern int lm3533_ctrlbank_get_brightness(struct lm3533_ctrlbank *cb, u8 *val); -extern int lm3533_ctrlbank_set_max_current(struct lm3533_ctrlbank *cb, - u16 imax); -extern int lm3533_ctrlbank_set_pwm(struct lm3533_ctrlbank *cb, u8 val); -extern int lm3533_ctrlbank_get_pwm(struct lm3533_ctrlbank *cb, u8 *val); +int lm3533_ctrlbank_enable(struct lm3533_ctrlbank *cb); +int lm3533_ctrlbank_disable(struct lm3533_ctrlbank *cb); -extern int lm3533_read(struct lm3533 *lm3533, u8 reg, u8 *val); -extern int lm3533_write(struct lm3533 *lm3533, u8 reg, u8 val); -extern int lm3533_update(struct lm3533 *lm3533, u8 reg, u8 val, u8 mask); +int lm3533_ctrlbank_set_brightness(struct lm3533_ctrlbank *cb, u32 val); +int lm3533_ctrlbank_get_brightness(struct lm3533_ctrlbank *cb, u32 *val); +int lm3533_ctrlbank_set_max_current(struct lm3533_ctrlbank *cb, u32 imax); +int lm3533_ctrlbank_set_pwm(struct lm3533_ctrlbank *cb, u32 val); +int lm3533_ctrlbank_get_pwm(struct lm3533_ctrlbank *cb, u32 *val); #endif /* __LINUX_MFD_LM3533_H */ diff --git a/include/linux/mfd/lp8788.h b/include/linux/mfd/lp8788.h index fd17bec2a33e..e8c50bd38a8a 100644 --- a/include/linux/mfd/lp8788.h +++ b/include/linux/mfd/lp8788.h @@ -34,7 +34,7 @@ #define LP8788_ALM_IRQ "ALARM_IRQ" enum lp8788_int_id { - /* interrup register 1 : Addr 00h */ + /* interrupt register 1 : Addr 00h */ LP8788_INT_TSDL, LP8788_INT_TSDH, LP8788_INT_UVLO, diff --git a/include/linux/mfd/max8998.h b/include/linux/mfd/max8998.h index 5473f1983e31..ad2ad24412fb 100644 --- a/include/linux/mfd/max8998.h +++ b/include/linux/mfd/max8998.h @@ -45,7 +45,7 @@ enum { /** * max8998_regulator_data - regulator data * @id: regulator id - * @initdata: regulator init data (contraints, supplies, ...) + * @initdata: regulator init data (constraints, supplies, ...) * @reg_node: DT node of regulator (unused on non-DT platforms) */ struct max8998_regulator_data { diff --git a/include/linux/mfd/motorola-cpcap.h b/include/linux/mfd/motorola-cpcap.h index 981e5777deb7..bb23363eeccd 100644 --- a/include/linux/mfd/motorola-cpcap.h +++ b/include/linux/mfd/motorola-cpcap.h @@ -25,6 +25,13 @@ #define CPCAP_REVISION_2_0 0x10 #define CPCAP_REVISION_2_1 0x11 +enum cpcap_variant { + CPCAP_DEFAULT = 1, + CPCAP_MAPPHONE, + CPCAP_MOT, + CPCAP_MAX +}; + /* CPCAP registers */ #define CPCAP_REG_INT1 0x0000 /* Interrupt 1 */ #define CPCAP_REG_INT2 0x0004 /* Interrupt 2 */ diff --git a/include/linux/mfd/mt6397/rtc.h b/include/linux/mfd/mt6397/rtc.h index 27883af44f87..f4da579ec638 100644 --- a/include/linux/mfd/mt6397/rtc.h +++ b/include/linux/mfd/mt6397/rtc.h @@ -49,6 +49,13 @@ #define RTC_AL_SEC 0x0018 +/* The high byte of four of the alarms is spare, always-on storage */ +#define RTC_AL_HOU 0x001c +#define RTC_AL_DOM 0x001e +#define RTC_AL_DOW 0x0020 +#define RTC_AL_MTH 0x0022 +#define RTC_SPARE_MASK GENMASK(15, 8) + #define RTC_AL_SEC_MASK 0x003f #define RTC_AL_MIN_MASK 0x003f #define RTC_AL_HOU_MASK 0x001f diff --git a/include/linux/mfd/palmas.h b/include/linux/mfd/palmas.h index dabcc0dea802..1d872363820a 100644 --- a/include/linux/mfd/palmas.h +++ b/include/linux/mfd/palmas.h @@ -251,7 +251,7 @@ enum tps65917_regulators { TPS65917_NUM_REGS, }; -/* External controll signal name */ +/* External control signal name */ enum { PALMAS_EXT_CONTROL_ENABLE1 = 0x1, PALMAS_EXT_CONTROL_ENABLE2 = 0x2, diff --git a/include/linux/mfd/rohm-bd71828.h b/include/linux/mfd/rohm-bd71828.h index 39fe275a8117..ea9b1a8f495f 100644 --- a/include/linux/mfd/rohm-bd71828.h +++ b/include/linux/mfd/rohm-bd71828.h @@ -186,7 +186,7 @@ enum { #define BD71828_REG_RTC_ALM2 0x63 #define BD71828_REG_RTC_START BD71828_REG_RTC_SEC -/* Charger/Battey */ +/* Charger/Battery */ #define BD71828_REG_CHG_STATE 0x65 #define BD71828_REG_CHG_FULL 0xd2 #define BD71828_REG_CHG_EN 0x6F diff --git a/include/linux/mfd/rohm-bd73800.h b/include/linux/mfd/rohm-bd73800.h new file mode 100644 index 000000000000..c6c0d453a40d --- /dev/null +++ b/include/linux/mfd/rohm-bd73800.h @@ -0,0 +1,306 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright 2026 ROHM Semiconductors. + * + * Author: Matti Vaittinen <matti.vaittinen@fi.rohmeurope.com> + */ + +#ifndef _MFD_BD73800_H +#define _MFD_BD73800_H + +#include <linux/regmap.h> + +enum { + BD73800_BUCK1 = 0, + BD73800_BUCK2, + BD73800_BUCK3, + BD73800_BUCK4, + BD73800_BUCK5, + BD73800_BUCK6, + BD73800_BUCK7, + BD73800_BUCK8, + BD73800_LDO1, + BD73800_LDO2, + BD73800_LDO3, + BD73800_LDO4, +}; + +/* + * All regulators except BUCK 5 have full 8-bit register of valid voltage + * values, including 0. + */ +#define BD73800_NUM_VOLTS (0xff + 1) +/* + * BUCK 5 has two sets of voltage ranges, both having valid voltage selectors + * from 0x0 to 0x7f + */ +#define BD73800_BUCK5_VOLTS (0x80 + 0x80) + +/* BD73800 interrupts */ +enum { + /* INT_STAT_1 register IRQs, ADC and RTC */ + BD73800_INT_ADC_ACCUM_DONE, + BD73800_INT_ADC_ACCUM_OVF, + BD73800_INT_ADC_ACCUM_VAL, + BD73800_INT_ADC_ACCUM_TW, + BD73800_INT_ADC_POW_VAL, + BD73800_INT_RTC0, + BD73800_INT_RTC1, + BD73800_INT_RTC2, + + /* BUCK reg interrupts */ + /* INT_STAT_2 IRQs */ + BD73800_INT_BUCK1_DVS_DONE, + BD73800_INT_BUCK2_DVS_DONE, + BD73800_INT_BUCK3_DVS_DONE, + BD73800_INT_BUCK4_DVS_DONE, + BD73800_INT_BUCK5_DVS_DONE, + BD73800_INT_BUCK6_DVS_DONE, + BD73800_INT_BUCK7_DVS_DONE, + BD73800_INT_BUCK8_DVS_DONE, + /* INT_STAT_3 IRQs */ + BD73800_INT_BUCK1_OCP, + BD73800_INT_BUCK2_OCP, + BD73800_INT_BUCK3_OCP, + BD73800_INT_BUCK4_OCP, + BD73800_INT_BUCK5_OCP, + BD73800_INT_BUCK6_OCP, + BD73800_INT_BUCK7_OCP, + BD73800_INT_BUCK8_OCP, + + /* INT_STAT_4 IRQs, power-button, WDG and reset */ + BD73800_INT_PBTN_LONG_PRESS, + BD73800_INT_PBTN_MID_PRESS, + /* + * The SHORT_PUSH is generated when button is first pressed (longer + * than configured time limit), and then released before the MID_PRESS + * time limit. The SHORT_PRESS is generated immediately when button is + * pressed for longer than configured limit, whether it is released or + * not. + */ + BD73800_INT_PBTN_SHORT_PUSH, + BD73800_INT_PBTN_SHORT_PRESS, + BD73800_INT_WDG, + BD73800_INT_SWRESET, + BD73800_INT_SEQ_DONE, + + /* INT_STAT_5 IRQs, GPIO */ + BD73800_INT_GPIO1, + BD73800_INT_GPIO2, + BD73800_INT_GPIO3, + BD73800_INT_GPIO4, +}; + +#define BD73800_MASK_RUN_EN BIT(2) +#define BD73800_MASK_SUSP_EN BIT(1) +#define BD73800_MASK_IDLE_EN BIT(0) +#define BD73800_MASK_VOLT GENMASK(7, 0) +#define BD73800_MASK_BUCK5_VOLT GENMASK(6, 0) +#define BD73800_MASK_RAMP_DELAY GENMASK(2, 1) +#define BD73800_BUCK5_RANGE_MASK BIT(7) + +/* BD73800 registers */ +enum { + BD73800_REG_PRODUCT_ID = 0x0, + BD73800_REG_MANUFACTURER_ID, + BD73800_REG_REVISION, + BD73800_REG_NVMVERSION, + BD73800_REG_POR_REASON, + BD73800_REG_RESET_REASON1, + BD73800_REG_RESET_REASON2, + BD73800_REG_RESET_REASON3, + BD73800_REG_POW_STATE, + BD73800_REG_WRST_SEL, + BD73800_REG_PS_CTRL_1, + BD73800_REG_PS_CTRL_2, + BD73800_REG_RCVCFG, + BD73800_REG_RCVNUM, + BD73800_REG_CRDCFG, /* 0x0f, followed by undocumented reg */ + + BD73800_REG_BUCK1_ON = 0x11, + BD73800_REG_BUCK1_MODE, + BD73800_REG_BUCK1_VOLT_RUN, + BD73800_REG_BUCK1_VOLT_IDLE, + BD73800_REG_BUCK1_VOLT_SUSP, /* 0x15, followed by undocumented reg */ + + BD73800_REG_BUCK2_ON = 0x17, + BD73800_REG_BUCK2_MODE, + BD73800_REG_BUCK2_VOLT_RUN, + BD73800_REG_BUCK2_VOLT_IDLE, + BD73800_REG_BUCK2_VOLT_SUSP, /* 0x1b */ + + BD73800_REG_BUCK3_ON = 0x1d, + BD73800_REG_BUCK3_MODE, + BD73800_REG_BUCK3_VOLT_RUN, + BD73800_REG_BUCK3_VOLT_IDLE, + BD73800_REG_BUCK3_VOLT_SUSP, /* 0x21 */ + + BD73800_REG_BUCK4_ON = 0x23, + BD73800_REG_BUCK4_MODE, + BD73800_REG_BUCK4_VOLT_RUN, + BD73800_REG_BUCK4_VOLT_IDLE, + BD73800_REG_BUCK4_VOLT_SUSP, /* 0x27 */ + + BD73800_REG_BUCK5_ON = 0x29, + BD73800_REG_BUCK5_MODE, + BD73800_REG_BUCK5_VOLT_RUN, + BD73800_REG_BUCK5_VOLT_IDLE, + BD73800_REG_BUCK5_VOLT_SUSP, /* 0x2d */ + + BD73800_REG_BUCK6_ON = 0x2f, + BD73800_REG_BUCK6_MODE, + BD73800_REG_BUCK6_VOLT_RUN, + BD73800_REG_BUCK6_VOLT_IDLE, + BD73800_REG_BUCK6_VOLT_SUSP, /* 0x33 */ + + BD73800_REG_BUCK7_ON = 0x35, + BD73800_REG_BUCK7_MODE, + BD73800_REG_BUCK7_VOLT_RUN, + BD73800_REG_BUCK7_VOLT_IDLE, + BD73800_REG_BUCK7_VOLT_SUSP, /* 0x39 */ + + BD73800_REG_BUCK8_ON = 0x3b, + BD73800_REG_BUCK8_MODE, + BD73800_REG_BUCK8_VOLT_RUN, + BD73800_REG_BUCK8_VOLT_IDLE, + BD73800_REG_BUCK8_VOLT_SUSP, /* 0x3f */ + + BD73800_REG_LDO1_ON = 0x41, + BD73800_REG_LDO1_VOLT, + BD73800_REG_LDO1_MODE, + BD73800_REG_LDO2_ON, + BD73800_REG_LDO2_VOLT, + BD73800_REG_LDO2_MODE, + BD73800_REG_LDO3_ON, + BD73800_REG_LDO3_VOLT, + BD73800_REG_LDO3_MODE, + BD73800_REG_LDO4_ON, + BD73800_REG_LDO4_VOLT, + BD73800_REG_LDO4_MODE, /* 0x4c */ + + BD73800_REG_GPO_OUT = 0x4e, + BD73800_REG_OUT32K = 0x50, + BD73800_REG_RTC_SEC, + BD73800_REG_RTC_MIN, + BD73800_REG_RTC_HOUR, + BD73800_REG_RTC_WEEK, + BD73800_REG_RTC_DAY, + BD73800_REG_RTC_MONTH, + BD73800_REG_RTC_YEAR, + BD73800_REG_RTC_ALM0_SEC, + BD73800_REG_RTC_ALM0_MIN, + BD73800_REG_RTC_ALM0_HOUR, + BD73800_REG_RTC_ALM0_WEEK, + BD73800_REG_RTC_ALM0_DAY, + BD73800_REG_RTC_ALM0_MONTH, + BD73800_REG_RTC_ALM0_YEAR, + BD73800_REG_RTC_ALM1_SEC, + BD73800_REG_RTC_ALM1_MIN, + BD73800_REG_RTC_ALM1_HOUR, + BD73800_REG_RTC_ALM1_WEEK, + BD73800_REG_RTC_ALM1_DAY, + BD73800_REG_RTC_ALM1_MONTH, + BD73800_REG_RTC_ALM1_YEAR, + BD73800_REG_RTC_ALM2, + BD73800_REG_RTC_CONF, /* 0x69 */ + + BD73800_REG_ADC_CTRL_1 = 0x6b, + BD73800_REG_ADC_CTRL_2, + BD73800_REG_ADC_ACCUM_NUM2, + BD73800_REG_ADC_ACCUM_NUM1, + BD73800_REG_ADC_ACCUM_NUM0, + BD73800_REG_ADC_ACCUM_KICK, + BD73800_REG_ADC_ACCUM_CNT2, + BD73800_REG_ADC_ACCUM_CNT1, + BD73800_REG_ADC_ACCUM_CNT0, + BD73800_REG_ADC_ACCUM_VAL2, + BD73800_REG_ADC_ACCUM_VAL1, + BD73800_REG_ADC_ACCUM_VAL0, + BD73800_REG_ADC_VOL_VAL1, + BD73800_REG_ADC_VOL_VAL0, + BD73800_REG_ADC_CUR_VAL1, + BD73800_REG_ADC_CUR_VAL0, + BD73800_REG_ADC_POW_VAL1, + BD73800_REG_ADC_POW_VAL0, + BD73800_REG_ADC_TEMP_VAL1, + BD73800_REG_ADC_TEMP_VAL0, + BD73800_REG_ADC_ACCUM_VAL_INT_TH4, + BD73800_REG_ADC_ACCUM_VAL_INT_TH3, + BD73800_REG_ADC_ACCUM_VAL_INT_TH2, + BD73800_REG_ADC_ACCUM_VAL_INT_TH1, + BD73800_REG_ADC_ACCUM_VAL_INT_TH0, + BD73800_REG_ADC_WARN_TEMP_INT_TH1, + BD73800_REG_ADC_WARN_TEMP_INT_TH0, + BD73800_REG_ADC_POW_VAL_INT_TH1, + BD73800_REG_ADC_POW_VAL_INT_TH0, /* 0x89 */ + + BD73800_REG_PBTN_CONF = 0x8b, + + BD73800_REG_INT_MAIN_EN = 0x8f, + BD73800_REG_INT_1_EN, + BD73800_REG_INT_2_EN, + BD73800_REG_INT_3_EN, + BD73800_REG_INT_4_EN, + BD73800_REG_INT_5_EN, /* 0x94 */ + + BD73800_REG_INT_MAIN_STAT = 0x96, + BD73800_REG_INT_1_STAT, + BD73800_REG_INT_2_STAT, + BD73800_REG_INT_3_STAT, + BD73800_REG_INT_4_STAT, + BD73800_REG_INT_5_STAT, /* 0x9b */ + + BD73800_REG_INT_MAIN_SRC = 0x9d, + BD73800_REG_INT_1_SRC, + BD73800_REG_INT_2_SRC, + BD73800_REG_INT_3_SRC, + BD73800_REG_INT_4_SRC, + BD73800_REG_INT_5_SRC, /* 0xa2 */ + + BD73800_REG_RST_MASK = 0xaf, + BD73800_MAX_REGISTER, +}; + +#define BD73800_REG_RTC_START BD73800_REG_RTC_SEC +#define BD73800_REG_RTC_ALM_START BD73800_REG_RTC_ALM0_SEC + +/* BD73800 IRQ register masks */ + +#define BD73800_INT_MAIN_EN_ALL GENMASK(4, 0) +#define BD73800_INT_ADC_ACCUM_DONE_MASK BIT(0) +#define BD73800_INT_ADC_ACCUM_OVF_MASK BIT(1) +#define BD73800_INT_ADC_ACCUM_VAL_MASK BIT(2) +#define BD73800_INT_ADC_ACCUM_TW_MASK BIT(3) +#define BD73800_INT_ADC_POW_VAL_MASK BIT(4) +#define BD73800_INT_RTC0_MASK BIT(5) +#define BD73800_INT_RTC1_MASK BIT(6) +#define BD73800_INT_RTC2_MASK BIT(7) +#define BD73800_INT_BUCK1_DVS_DONE_MASK BIT(0) +#define BD73800_INT_BUCK2_DVS_DONE_MASK BIT(1) +#define BD73800_INT_BUCK3_DVS_DONE_MASK BIT(2) +#define BD73800_INT_BUCK4_DVS_DONE_MASK BIT(3) +#define BD73800_INT_BUCK5_DVS_DONE_MASK BIT(4) +#define BD73800_INT_BUCK6_DVS_DONE_MASK BIT(5) +#define BD73800_INT_BUCK7_DVS_DONE_MASK BIT(6) +#define BD73800_INT_BUCK8_DVS_DONE_MASK BIT(7) +#define BD73800_INT_BUCK1_OCP_MASK BIT(0) +#define BD73800_INT_BUCK2_OCP_MASK BIT(1) +#define BD73800_INT_BUCK3_OCP_MASK BIT(2) +#define BD73800_INT_BUCK4_OCP_MASK BIT(3) +#define BD73800_INT_BUCK5_OCP_MASK BIT(4) +#define BD73800_INT_BUCK6_OCP_MASK BIT(5) +#define BD73800_INT_BUCK7_OCP_MASK BIT(6) +#define BD73800_INT_BUCK8_OCP_MASK BIT(7) +#define BD73800_INT_PBTN_LONG_PRESS_MASK BIT(0) +#define BD73800_INT_PBTN_MID_PRESS_MASK BIT(1) +#define BD73800_INT_PBTN_SHORT_PUSH_MASK BIT(2) +#define BD73800_INT_PBTN_SHORT_PRESS_MASK BIT(3) +#define BD73800_INT_WDG_MASK BIT(4) +#define BD73800_INT_SWRESET_MASK BIT(5) +#define BD73800_INT_SEQ_DONE_MASK BIT(6) +#define BD73800_INT_GPIO1_MASK BIT(0) +#define BD73800_INT_GPIO2_MASK BIT(1) +#define BD73800_INT_GPIO3_MASK BIT(2) +#define BD73800_INT_GPIO4_MASK BIT(3) + +#endif /* _MFD_BD73800_H */ diff --git a/include/linux/mfd/rohm-generic.h b/include/linux/mfd/rohm-generic.h index 0a284919a6c3..3ec87428ee97 100644 --- a/include/linux/mfd/rohm-generic.h +++ b/include/linux/mfd/rohm-generic.h @@ -17,6 +17,7 @@ enum rohm_chip_type { ROHM_CHIP_TYPE_BD71837, ROHM_CHIP_TYPE_BD71847, ROHM_CHIP_TYPE_BD72720, + ROHM_CHIP_TYPE_BD73800, ROHM_CHIP_TYPE_BD96801, ROHM_CHIP_TYPE_BD96802, ROHM_CHIP_TYPE_BD96805, diff --git a/include/linux/mfd/samsung/core.h b/include/linux/mfd/samsung/core.h index 6191f409de94..e489ca0d55ed 100644 --- a/include/linux/mfd/samsung/core.h +++ b/include/linux/mfd/samsung/core.h @@ -106,7 +106,7 @@ struct sec_platform_data { /** * sec_regulator_data - regulator data * @id: regulator id - * @initdata: regulator init data (contraints, supplies, ...) + * @initdata: regulator init data (constraints, supplies, ...) */ struct sec_regulator_data { int id; diff --git a/include/linux/mfd/si476x-core.h b/include/linux/mfd/si476x-core.h index d9e3a322134c..3e924ca4db00 100644 --- a/include/linux/mfd/si476x-core.h +++ b/include/linux/mfd/si476x-core.h @@ -94,10 +94,10 @@ enum si476x_power_state { * of the @rds_fifo_drainer. * @rds_drainer_status_lock: Lock used to guard access to the * @rds_drainer_is_working variable. - * @command: Wait queue for wainting on the command comapletion. + * @command: Wait queue for waiting on the command completion. * @cts: Clear To Send flag set upon receiving first status with CTS * set. - * @tuning: Wait queue used for wainting for tune/seek comand + * @tuning: Wait queue used for waiting for tune/seek command * completion. * @stc: Similar to @cts, but for the STC bit of the status value. * @power_up_parameters: Parameters used as argument for POWER_UP diff --git a/include/linux/mfd/twl.h b/include/linux/mfd/twl.h index b31e07fa4d51..238d9dcefc35 100644 --- a/include/linux/mfd/twl.h +++ b/include/linux/mfd/twl.h @@ -739,7 +739,7 @@ int twl4030_sih_setup(struct device *dev, int module, int irq_base); #define TWL4030_REG_VUSB3V1 19 /* TWL6030 SMPS/LDO's */ -/* EXTERNAL dc-to-dc buck convertor controllable via SR */ +/* EXTERNAL dc-to-dc buck converter controllable via SR */ #define TWL6030_REG_VDD1 30 #define TWL6030_REG_VDD2 31 #define TWL6030_REG_VDD3 32 diff --git a/include/linux/mhi.h b/include/linux/mhi.h index 8616bacd8675..65e8a397cc50 100644 --- a/include/linux/mhi.h +++ b/include/linux/mhi.h @@ -687,7 +687,7 @@ int mhi_pm_resume(struct mhi_controller *mhi_cntrl); * Resume the device irrespective of its MHI state. As per the MHI spec, devices * has to be in M3 state during resume. But some devices seem to be in a * different MHI state other than M3 but they continue working fine if allowed. - * This API is intented to be used for such devices. + * This API is intended to be used for such devices. * * Return: 0 if the resume succeeds, a negative error code otherwise */ diff --git a/include/linux/mhi_ep.h b/include/linux/mhi_ep.h index f6383a57a872..852f331f03fd 100644 --- a/include/linux/mhi_ep.h +++ b/include/linux/mhi_ep.h @@ -304,4 +304,20 @@ bool mhi_ep_queue_is_empty(struct mhi_ep_device *mhi_dev, enum dma_data_directio */ int mhi_ep_queue_skb(struct mhi_ep_device *mhi_dev, struct sk_buff *skb); +/** + * mhi_ep_queue_buf - Transfer buffer contents to host over MHI Endpoint + * @mhi_dev: Device associated with the DL channel + * @buf: Buffer to be queued. On success, ownership passes to the MHI stack; + * the caller must not free @buf until the DL transfer callback fires + * with result->buf_addr equal to @buf. On failure, the caller retains + * ownership and must free @buf. + * Note: if @len spans multiple host DL TREs, the DL transfer callback + * fires once per TRE, each time with result->buf_addr equal to @buf. + * @len: Size of the buffer + * + * Return: 0 if the buffer contents have been transferred successfully, a + * negative error code otherwise. + */ +int mhi_ep_queue_buf(struct mhi_ep_device *mhi_dev, void *buf, size_t len); + #endif diff --git a/include/linux/micrel_phy.h b/include/linux/micrel_phy.h index 9c6f9817383f..bb71b2510c2c 100644 --- a/include/linux/micrel_phy.h +++ b/include/linux/micrel_phy.h @@ -47,11 +47,6 @@ #define MICREL_PHY_FXEN BIT(1) #define MICREL_KSZ8_P1_ERRATA BIT(2) -#define MICREL_KSZ9021_EXTREG_CTRL 0xB -#define MICREL_KSZ9021_EXTREG_DATA_WRITE 0xC -#define MICREL_KSZ9021_RGMII_CLK_CTRL_PAD_SCEW 0x104 -#define MICREL_KSZ9021_RGMII_RX_DATA_PAD_SCEW 0x105 - /* Device specific MII_BMCR (Reg 0) bits */ /* 1 = HP Auto MDI/MDI-X mode, 0 = Microchip Auto MDI/MDI-X mode */ #define KSZ886X_BMCR_HP_MDIX BIT(5) diff --git a/include/linux/minmax.h b/include/linux/minmax.h index a0158db54a04..5ef4d58c0c42 100644 --- a/include/linux/minmax.h +++ b/include/linux/minmax.h @@ -38,9 +38,9 @@ * Note that 'x' is the original expression, and 'ux' is the unique variable * that contains the value. * - * We use 'ux' for pure type checking, and 'x' for when we need to look at the - * value (but without evaluating it for side effects! - * Careful to only ever evaluate it with sizeof() or __builtin_constant_p() etc). + * We use 'ux' for both the type and the value checks, so 'x' itself is only + * expanded twice: once to initialise 'ux', and once quoted in the error + * message. * * Pointers end up being checked by the normal C type rules at the actual * comparison, and these expressions only need to be careful to not cause diff --git a/include/linux/mlx5/cq.h b/include/linux/mlx5/cq.h index 9d47cdc727ad..a1c14479e462 100644 --- a/include/linux/mlx5/cq.h +++ b/include/linux/mlx5/cq.h @@ -81,16 +81,6 @@ enum { enum { MLX5_CQE_OWNER_MASK = 1, - MLX5_CQE_REQ = 0, - MLX5_CQE_RESP_WR_IMM = 1, - MLX5_CQE_RESP_SEND = 2, - MLX5_CQE_RESP_SEND_IMM = 3, - MLX5_CQE_RESP_SEND_INV = 4, - MLX5_CQE_RESIZE_CQ = 5, - MLX5_CQE_SIG_ERR = 12, - MLX5_CQE_REQ_ERR = 13, - MLX5_CQE_RESP_ERR = 14, - MLX5_CQE_INVALID = 15, }; enum { diff --git a/include/linux/mlx5/data_direct.h b/include/linux/mlx5/data_direct.h new file mode 100644 index 000000000000..4cff5ff5489d --- /dev/null +++ b/include/linux/mlx5/data_direct.h @@ -0,0 +1,47 @@ +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ +/* + * Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved + */ + +#ifndef _MLX5_DATA_DIRECT_H +#define _MLX5_DATA_DIRECT_H + +#include <linux/compiler.h> +#include <linux/list.h> + +struct device; +struct mlx5_core_dev; +struct notifier_block; +struct pci_dev; + +enum mlx5_data_direct_event { + MLX5_DATA_DIRECT_UNBIND, +}; + +struct mlx5_data_direct_dev { + struct device *device; + struct pci_dev *pdev; + char *vuid; + struct list_head list; +}; + +struct mlx5_data_direct { + struct mlx5_data_direct_dev *dev; + u32 pdn; + u32 mkey; + u32 mkey_ro; + u8 mkey_ro_valid :1; +}; + +static inline struct mlx5_data_direct_dev * +mlx5_data_direct_get_dev(struct mlx5_data_direct *dd) +{ + return dd ? READ_ONCE(dd->dev) : NULL; +} + +int mlx5_data_direct_register(struct mlx5_core_dev *mdev, + struct notifier_block *nb); +void mlx5_data_direct_unregister(struct mlx5_core_dev *mdev, + struct notifier_block *nb); + +#endif diff --git a/include/linux/mlx5/device.h b/include/linux/mlx5/device.h index 8cb321a9fb3d..37727b5ab78f 100644 --- a/include/linux/mlx5/device.h +++ b/include/linux/mlx5/device.h @@ -36,6 +36,7 @@ #include <linux/types.h> #include <rdma/ib_verbs.h> #include <linux/mlx5/mlx5_ifc.h> +#include <linux/mlx5/mlx5_ifc_macros.h> #include <linux/bitfield.h> #if defined(__LITTLE_ENDIAN) @@ -46,122 +47,6 @@ #error Host endianness not defined #endif -/* helper macros */ -#define __mlx5_nullp(typ) ((struct mlx5_ifc_##typ##_bits *)0) -#define __mlx5_bit_sz(typ, fld) sizeof(__mlx5_nullp(typ)->fld) -#define __mlx5_bit_off(typ, fld) (offsetof(struct mlx5_ifc_##typ##_bits, fld)) -#define __mlx5_16_off(typ, fld) (__mlx5_bit_off(typ, fld) / 16) -#define __mlx5_dw_off(typ, fld) (__mlx5_bit_off(typ, fld) / 32) -#define __mlx5_64_off(typ, fld) (__mlx5_bit_off(typ, fld) / 64) -#define __mlx5_16_bit_off(typ, fld) (16 - __mlx5_bit_sz(typ, fld) - (__mlx5_bit_off(typ, fld) & 0xf)) -#define __mlx5_dw_bit_off(typ, fld) (32 - __mlx5_bit_sz(typ, fld) - (__mlx5_bit_off(typ, fld) & 0x1f)) -#define __mlx5_mask(typ, fld) ((u32)((1ull << __mlx5_bit_sz(typ, fld)) - 1)) -#define __mlx5_dw_mask(typ, fld) (__mlx5_mask(typ, fld) << __mlx5_dw_bit_off(typ, fld)) -#define __mlx5_mask16(typ, fld) ((u16)((1ull << __mlx5_bit_sz(typ, fld)) - 1)) -#define __mlx5_16_mask(typ, fld) (__mlx5_mask16(typ, fld) << __mlx5_16_bit_off(typ, fld)) -#define __mlx5_st_sz_bits(typ) sizeof(struct mlx5_ifc_##typ##_bits) - -#define MLX5_FLD_SZ_BYTES(typ, fld) (__mlx5_bit_sz(typ, fld) / 8) -#define MLX5_ST_SZ_BYTES(typ) (sizeof(struct mlx5_ifc_##typ##_bits) / 8) -#define MLX5_ST_SZ_DW(typ) (sizeof(struct mlx5_ifc_##typ##_bits) / 32) -#define MLX5_ST_SZ_QW(typ) (sizeof(struct mlx5_ifc_##typ##_bits) / 64) -#define MLX5_UN_SZ_BYTES(typ) (sizeof(union mlx5_ifc_##typ##_bits) / 8) -#define MLX5_UN_SZ_DW(typ) (sizeof(union mlx5_ifc_##typ##_bits) / 32) -#define MLX5_BYTE_OFF(typ, fld) (__mlx5_bit_off(typ, fld) / 8) -#define MLX5_ADDR_OF(typ, p, fld) ((void *)((u8 *)(p) + MLX5_BYTE_OFF(typ, fld))) - -/* insert a value to a struct */ -#define MLX5_SET(typ, p, fld, v) do { \ - u32 _v = v; \ - BUILD_BUG_ON(__mlx5_st_sz_bits(typ) % 32); \ - *((__be32 *)(p) + __mlx5_dw_off(typ, fld)) = \ - cpu_to_be32((be32_to_cpu(*((__be32 *)(p) + __mlx5_dw_off(typ, fld))) & \ - (~__mlx5_dw_mask(typ, fld))) | (((_v) & __mlx5_mask(typ, fld)) \ - << __mlx5_dw_bit_off(typ, fld))); \ -} while (0) - -#define MLX5_ARRAY_SET(typ, p, fld, idx, v) do { \ - BUILD_BUG_ON(__mlx5_bit_off(typ, fld) % 32); \ - MLX5_SET(typ, p, fld[idx], v); \ -} while (0) - -#define MLX5_SET_TO_ONES(typ, p, fld) do { \ - BUILD_BUG_ON(__mlx5_st_sz_bits(typ) % 32); \ - *((__be32 *)(p) + __mlx5_dw_off(typ, fld)) = \ - cpu_to_be32((be32_to_cpu(*((__be32 *)(p) + __mlx5_dw_off(typ, fld))) & \ - (~__mlx5_dw_mask(typ, fld))) | ((__mlx5_mask(typ, fld)) \ - << __mlx5_dw_bit_off(typ, fld))); \ -} while (0) - -#define MLX5_GET(typ, p, fld) ((be32_to_cpu(*((__be32 *)(p) +\ -__mlx5_dw_off(typ, fld))) >> __mlx5_dw_bit_off(typ, fld)) & \ -__mlx5_mask(typ, fld)) - -#define MLX5_GET_PR(typ, p, fld) ({ \ - u32 ___t = MLX5_GET(typ, p, fld); \ - pr_debug(#fld " = 0x%x\n", ___t); \ - ___t; \ -}) - -#define __MLX5_SET64(typ, p, fld, v) do { \ - BUILD_BUG_ON(__mlx5_bit_sz(typ, fld) != 64); \ - *((__be64 *)(p) + __mlx5_64_off(typ, fld)) = cpu_to_be64(v); \ -} while (0) - -#define MLX5_SET64(typ, p, fld, v) do { \ - BUILD_BUG_ON(__mlx5_bit_off(typ, fld) % 64); \ - __MLX5_SET64(typ, p, fld, v); \ -} while (0) - -#define MLX5_ARRAY_SET64(typ, p, fld, idx, v) do { \ - BUILD_BUG_ON(__mlx5_bit_off(typ, fld) % 64); \ - __MLX5_SET64(typ, p, fld[idx], v); \ -} while (0) - -#define MLX5_GET64(typ, p, fld) be64_to_cpu(*((__be64 *)(p) + __mlx5_64_off(typ, fld))) - -#define MLX5_GET64_PR(typ, p, fld) ({ \ - u64 ___t = MLX5_GET64(typ, p, fld); \ - pr_debug(#fld " = 0x%llx\n", ___t); \ - ___t; \ -}) - -#define MLX5_GET16(typ, p, fld) ((be16_to_cpu(*((__be16 *)(p) +\ -__mlx5_16_off(typ, fld))) >> __mlx5_16_bit_off(typ, fld)) & \ -__mlx5_mask16(typ, fld)) - -#define MLX5_SET16(typ, p, fld, v) do { \ - u16 _v = v; \ - BUILD_BUG_ON(__mlx5_st_sz_bits(typ) % 16); \ - *((__be16 *)(p) + __mlx5_16_off(typ, fld)) = \ - cpu_to_be16((be16_to_cpu(*((__be16 *)(p) + __mlx5_16_off(typ, fld))) & \ - (~__mlx5_16_mask(typ, fld))) | (((_v) & __mlx5_mask16(typ, fld)) \ - << __mlx5_16_bit_off(typ, fld))); \ -} while (0) - -/* Big endian getters */ -#define MLX5_GET64_BE(typ, p, fld) (*((__be64 *)(p) +\ - __mlx5_64_off(typ, fld))) - -#define MLX5_GET_BE(type_t, typ, p, fld) ({ \ - type_t tmp; \ - switch (sizeof(tmp)) { \ - case sizeof(u8): \ - tmp = (__force type_t)MLX5_GET(typ, p, fld); \ - break; \ - case sizeof(u16): \ - tmp = (__force type_t)cpu_to_be16(MLX5_GET(typ, p, fld)); \ - break; \ - case sizeof(u32): \ - tmp = (__force type_t)cpu_to_be32(MLX5_GET(typ, p, fld)); \ - break; \ - case sizeof(u64): \ - tmp = (__force type_t)MLX5_GET64_BE(typ, p, fld); \ - break; \ - } \ - tmp; \ - }) - enum mlx5_inline_modes { MLX5_INLINE_MODE_NONE, MLX5_INLINE_MODE_L2, @@ -172,7 +57,7 @@ enum mlx5_inline_modes { enum { MLX5_MAX_COMMANDS = 32, MLX5_CMD_DATA_BLOCK_SIZE = 512, - MLX5_PCI_CMD_XPORT = 7, + MLX5_PCI_CMD_XPORT = MLX5_CMD_QUEUE_ENTRY_TYPE_PCIE_CMD_IF_TRANSPORT, MLX5_MKEY_BSF_OCTO_SIZE = 4, MLX5_MAX_PSVS = 4, }; @@ -308,63 +193,6 @@ enum { MLX5_EVENT_QUEUE_TYPE_DCT = 6, }; -/* mlx5 components can subscribe to any one of these events via - * mlx5_eq_notifier_register API. - */ -enum mlx5_event { - /* Special value to subscribe to any event */ - MLX5_EVENT_TYPE_NOTIFY_ANY = 0x0, - /* HW events enum start: comp events are not subscribable */ - MLX5_EVENT_TYPE_COMP = 0x0, - /* HW Async events enum start: subscribable events */ - MLX5_EVENT_TYPE_PATH_MIG = 0x01, - MLX5_EVENT_TYPE_COMM_EST = 0x02, - MLX5_EVENT_TYPE_SQ_DRAINED = 0x03, - MLX5_EVENT_TYPE_SRQ_LAST_WQE = 0x13, - MLX5_EVENT_TYPE_SRQ_RQ_LIMIT = 0x14, - - MLX5_EVENT_TYPE_CQ_ERROR = 0x04, - MLX5_EVENT_TYPE_WQ_CATAS_ERROR = 0x05, - MLX5_EVENT_TYPE_PATH_MIG_FAILED = 0x07, - MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR = 0x10, - MLX5_EVENT_TYPE_WQ_ACCESS_ERROR = 0x11, - MLX5_EVENT_TYPE_SRQ_CATAS_ERROR = 0x12, - MLX5_EVENT_TYPE_OBJECT_CHANGE = 0x27, - - MLX5_EVENT_TYPE_INTERNAL_ERROR = 0x08, - MLX5_EVENT_TYPE_PORT_CHANGE = 0x09, - MLX5_EVENT_TYPE_GPIO_EVENT = 0x15, - MLX5_EVENT_TYPE_PORT_MODULE_EVENT = 0x16, - MLX5_EVENT_TYPE_TEMP_WARN_EVENT = 0x17, - MLX5_EVENT_TYPE_XRQ_ERROR = 0x18, - MLX5_EVENT_TYPE_REMOTE_CONFIG = 0x19, - MLX5_EVENT_TYPE_GENERAL_EVENT = 0x22, - MLX5_EVENT_TYPE_MONITOR_COUNTER = 0x24, - MLX5_EVENT_TYPE_PPS_EVENT = 0x25, - - MLX5_EVENT_TYPE_DB_BF_CONGESTION = 0x1a, - MLX5_EVENT_TYPE_STALL_EVENT = 0x1b, - - MLX5_EVENT_TYPE_CMD = 0x0a, - MLX5_EVENT_TYPE_PAGE_REQUEST = 0xb, - - MLX5_EVENT_TYPE_PAGE_FAULT = 0xc, - MLX5_EVENT_TYPE_NIC_VPORT_CHANGE = 0xd, - - MLX5_EVENT_TYPE_ESW_FUNCTIONS_CHANGED = 0xe, - MLX5_EVENT_TYPE_VHCA_STATE_CHANGE = 0xf, - - MLX5_EVENT_TYPE_DCT_DRAINED = 0x1c, - MLX5_EVENT_TYPE_DCT_KEY_VIOLATION = 0x1d, - - MLX5_EVENT_TYPE_FPGA_ERROR = 0x20, - MLX5_EVENT_TYPE_FPGA_QP_ERROR = 0x21, - - MLX5_EVENT_TYPE_DEVICE_TRACER = 0x26, - - MLX5_EVENT_TYPE_MAX = 0x100, -}; - enum mlx5_driver_event { MLX5_DRIVER_EVENT_TYPE_TRAP = 0, MLX5_DRIVER_EVENT_UPLINK_NETDEV, @@ -420,22 +248,6 @@ enum { }; enum { - MLX5_OPCODE_NOP = 0x00, - MLX5_OPCODE_SEND_INVAL = 0x01, - MLX5_OPCODE_RDMA_WRITE = 0x08, - MLX5_OPCODE_RDMA_WRITE_IMM = 0x09, - MLX5_OPCODE_SEND = 0x0a, - MLX5_OPCODE_SEND_IMM = 0x0b, - MLX5_OPCODE_LSO = 0x0e, - MLX5_OPCODE_RDMA_READ = 0x10, - MLX5_OPCODE_ATOMIC_CS = 0x11, - MLX5_OPCODE_ATOMIC_FA = 0x12, - MLX5_OPCODE_ATOMIC_MASKED_CS = 0x14, - MLX5_OPCODE_ATOMIC_MASKED_FA = 0x15, - MLX5_OPCODE_BIND_MW = 0x18, - MLX5_OPCODE_CONFIG_CMD = 0x1f, - MLX5_OPCODE_ENHANCED_MPSW = 0x29, - MLX5_RECV_OPCODE_RDMA_WRITE_IMM = 0x00, MLX5_RECV_OPCODE_SEND = 0x01, MLX5_RECV_OPCODE_SEND_IMM = 0x02, @@ -443,19 +255,6 @@ enum { MLX5_CQE_OPCODE_ERROR = 0x1e, MLX5_CQE_OPCODE_RESIZE = 0x16, - - MLX5_OPCODE_SET_PSV = 0x20, - MLX5_OPCODE_GET_PSV = 0x21, - MLX5_OPCODE_CHECK_PSV = 0x22, - MLX5_OPCODE_DUMP = 0x23, - MLX5_OPCODE_RGET_PSV = 0x26, - MLX5_OPCODE_RCHECK_PSV = 0x27, - - MLX5_OPCODE_UMR = 0x25, - - MLX5_OPCODE_FLOW_TBL_ACCESS = 0x2c, - - MLX5_OPCODE_ACCESS_ASO = 0x2d, }; enum { @@ -1224,12 +1023,6 @@ enum mlx5_flex_parser_protos { /* MLX5 DEV CAPs */ -/* TODO: EAT.ME */ -enum mlx5_cap_mode { - HCA_CAP_OPMOD_GET_MAX = 0, - HCA_CAP_OPMOD_GET_CUR = 1, -}; - /* Any new cap addition must update mlx5_hca_caps_alloc() to allocate * capability memory. */ @@ -1508,26 +1301,6 @@ enum mlx5_qcam_feature_groups { MLX5_GET(psp_cap, (mdev)->caps.hca[MLX5_CAP_PSP]->cur, cap) enum { - MLX5_CMD_STAT_OK = 0x0, - MLX5_CMD_STAT_INT_ERR = 0x1, - MLX5_CMD_STAT_BAD_OP_ERR = 0x2, - MLX5_CMD_STAT_BAD_PARAM_ERR = 0x3, - MLX5_CMD_STAT_BAD_SYS_STATE_ERR = 0x4, - MLX5_CMD_STAT_BAD_RES_ERR = 0x5, - MLX5_CMD_STAT_RES_BUSY = 0x6, - MLX5_CMD_STAT_NOT_READY = 0x7, - MLX5_CMD_STAT_LIM_ERR = 0x8, - MLX5_CMD_STAT_BAD_RES_STATE_ERR = 0x9, - MLX5_CMD_STAT_IX_ERR = 0xa, - MLX5_CMD_STAT_NO_RES_ERR = 0xf, - MLX5_CMD_STAT_BAD_INP_LEN_ERR = 0x50, - MLX5_CMD_STAT_BAD_OUTP_LEN_ERR = 0x51, - MLX5_CMD_STAT_BAD_QP_STATE_ERR = 0x10, - MLX5_CMD_STAT_BAD_PKT_ERR = 0x30, - MLX5_CMD_STAT_BAD_SIZE_OUTS_CQES_ERR = 0x40, -}; - -enum { MLX5_IEEE_802_3_COUNTERS_GROUP = 0x0, MLX5_RFC_2863_COUNTERS_GROUP = 0x1, MLX5_RFC_2819_COUNTERS_GROUP = 0x2, diff --git a/include/linux/mlx5/driver.h b/include/linux/mlx5/driver.h index 83d0a83bbfbc..abb99371ff13 100644 --- a/include/linux/mlx5/driver.h +++ b/include/linux/mlx5/driver.h @@ -707,6 +707,7 @@ struct mlx5_rsvd_gids { struct mlx5_clock; struct mlx5_clock_dev_state; +struct mlx5_data_direct; struct mlx5_dm; struct mlx5_fw_tracer; struct mlx5_vxlan; @@ -801,6 +802,7 @@ struct mlx5_core_dev { /* sync write combining state */ struct mutex wc_state_lock; struct devlink *shd; + struct mlx5_data_direct *data_direct; }; struct mlx5_db { @@ -1395,4 +1397,10 @@ static inline struct net *mlx5_core_net(struct mlx5_core_dev *dev) #define MLX5_SW_IMAGE_GUID_MAX_BYTES 9 +static inline bool mlx5_data_direct_supported(struct mlx5_core_dev *mdev) +{ + return MLX5_CAP_GEN(mdev, data_direct) && + MLX5_CAP_GEN_2(mdev, query_vuid); +} + #endif /* MLX5_DRIVER_H */ diff --git a/include/linux/mlx5/eswitch.h b/include/linux/mlx5/eswitch.h index a0dd162baa78..03d3620141c8 100644 --- a/include/linux/mlx5/eswitch.h +++ b/include/linux/mlx5/eswitch.h @@ -222,6 +222,9 @@ static inline bool is_mdev_switchdev_mode(struct mlx5_core_dev *dev) /* The returned number is valid only when the dev is eswitch manager. */ static inline u16 mlx5_eswitch_manager_vport(struct mlx5_core_dev *dev) { + if (MLX5_CAP_ESW(dev, esw_manager_vport_number_valid)) + return MLX5_CAP_ESW(dev, esw_manager_vport_number); + return mlx5_core_is_ecpf_esw_manager(dev) ? MLX5_VPORT_ECPF : MLX5_VPORT_HOST_PF; } diff --git a/include/linux/mlx5/mlx5_ifc.h b/include/linux/mlx5/mlx5_ifc.h index 12b9a81e5621..697fc9387b73 100644 --- a/include/linux/mlx5/mlx5_ifc.h +++ b/include/linux/mlx5/mlx5_ifc.h @@ -929,7 +929,7 @@ struct mlx5_ifc_flow_table_nic_cap_bits { struct mlx5_ifc_port_selection_cap_bits { u8 reserved_at_0[0x10]; u8 port_select_flow_table[0x1]; - u8 reserved_at_11[0x1]; + u8 port_select_eswitch[0x1]; u8 port_select_flow_table_bypass[0x1]; u8 reserved_at_13[0xd]; @@ -1070,7 +1070,9 @@ struct mlx5_ifc_e_switch_cap_bits { u8 root_ft_on_other_esw[0x1]; u8 reserved_at_a[0x1]; u8 esw_vport_state_max_tx_speed[0x1]; - u8 reserved_at_c[0xd]; + u8 esw_vport_state_cap_tx_speed[0x1]; + u8 esw_vport_state_effective_tx_speed[0x1]; + u8 reserved_at_e[0xb]; u8 esw_functions_changed[0x1]; u8 reserved_at_1a[0x1]; u8 ecpf_vport_exists[0x1]; @@ -2008,7 +2010,9 @@ struct mlx5_ifc_cmd_hca_cap_bits { u8 reserved_at_3b0[0x2]; u8 qp_latency_sensitive_disable[0x1]; u8 log_max_stride_sz_sq[0x5]; - u8 reserved_at_3b8[0x3]; + u8 reserved_at_3b8[0x1]; + u8 multi_uplink_eswitch[0x1]; + u8 reserved_at_3ba[0x1]; u8 log_min_stride_sz_sq[0x5]; u8 hairpin[0x1]; @@ -5492,6 +5496,9 @@ struct mlx5_ifc_query_vport_state_out_bits { u8 reserved_at_70[0x8]; u8 admin_state[0x4]; u8 state[0x4]; + + u8 cap_tx_speed[0x10]; + u8 effective_tx_speed[0x10]; }; struct mlx5_ifc_array1024_auto_bits { @@ -5979,6 +5986,108 @@ struct mlx5_ifc_cqe_error_syndrome_bits { u8 syndrome[0x8]; }; +struct mlx5_ifc_wqe_ctrl_seg_bits { + u8 opmod[0x8]; + u8 wqe_index[0x10]; + u8 opcode[0x8]; + + u8 qp_or_sq[0x18]; + u8 reserved_at_38[0x2]; + u8 ds[0x6]; + + u8 signature[0x8]; + u8 reserved_at_48[0x10]; + u8 fm[0x3]; + u8 reserved_at_5b[0x1]; + u8 ce[0x2]; + u8 se[0x1]; + u8 reserved_at_5f[0x1]; + + u8 imm[0x20]; +}; + +/* Values for wqe_ctrl_seg_bits.opcode */ +enum { + MLX5_OPCODE_NOP = 0x00, + MLX5_OPCODE_SEND_INVAL = 0x01, + MLX5_OPCODE_RDMA_WRITE = 0x08, + MLX5_OPCODE_RDMA_WRITE_IMM = 0x09, + MLX5_OPCODE_SEND = 0x0a, + MLX5_OPCODE_SEND_IMM = 0x0b, + MLX5_OPCODE_LSO = 0x0e, + MLX5_OPCODE_RDMA_READ = 0x10, + MLX5_OPCODE_ATOMIC_CS = 0x11, + MLX5_OPCODE_ATOMIC_FA = 0x12, + MLX5_OPCODE_ATOMIC_MASKED_CS = 0x14, + MLX5_OPCODE_ATOMIC_MASKED_FA = 0x15, + MLX5_OPCODE_BIND_MW = 0x18, + MLX5_OPCODE_CONFIG_CMD = 0x1f, + MLX5_OPCODE_SET_PSV = 0x20, + MLX5_OPCODE_GET_PSV = 0x21, + MLX5_OPCODE_CHECK_PSV = 0x22, + MLX5_OPCODE_DUMP = 0x23, + MLX5_OPCODE_UMR = 0x25, + MLX5_OPCODE_RGET_PSV = 0x26, + MLX5_OPCODE_RCHECK_PSV = 0x27, + MLX5_OPCODE_ENHANCED_MPSW = 0x29, + MLX5_OPCODE_FLOW_TBL_ACCESS = 0x2c, + MLX5_OPCODE_ACCESS_ASO = 0x2d, +}; + +/* Values for wqe_ctrl_seg_bits.ce */ +enum { + MLX5_WQE_CE_CQE_ALWAYS = 2, +}; + +struct mlx5_ifc_wqe_raddr_seg_bits { + u8 raddr[0x40]; + + u8 rkey[0x20]; + u8 reserved_at_60[0x20]; +}; + +struct mlx5_ifc_wqe_data_seg_bits { + u8 reserved_at_0[0x1]; + u8 byte_count[0x1f]; + + u8 lkey[0x20]; + + u8 addr[0x40]; +}; + +struct mlx5_ifc_cqe64_bits { + u8 reserved_at_0[0x1a0]; + + union { + u8 reserved_at_1a0[0x20]; + struct mlx5_ifc_cqe_error_syndrome_bits error_syndrome; + }; + + u8 send_wqe_opcode[0x8]; + u8 qpn_or_dctn_or_flow_tag[0x18]; + + u8 wqe_counter[0x10]; + u8 signature[0x8]; + u8 opcode[0x4]; + u8 cqe_format[0x2]; + u8 se[0x1]; + u8 owner[0x1]; +}; + +/* Values for cqe64_bits.opcode */ +enum { + MLX5_CQE_REQ = 0, + MLX5_CQE_RESP_WR_IMM = 1, + MLX5_CQE_RESP_SEND = 2, + MLX5_CQE_RESP_SEND_IMM = 3, + MLX5_CQE_RESP_SEND_INV = 4, + MLX5_CQE_RESIZE_CQ = 5, + MLX5_CQE_SIG_ERR = 12, + MLX5_CQE_REQ_ERR = 13, + MLX5_CQE_RESP_ERR = 14, + MLX5_CQE_INVALID = 15, +}; + struct mlx5_ifc_qp_context_extension_bits { u8 reserved_at_0[0x60]; @@ -6475,6 +6584,12 @@ struct mlx5_ifc_query_hca_cap_in_bits { u8 reserved_at_60[0x20]; }; +/* Values for query_hca_cap_in_bits.op_mod */ +enum mlx5_cap_mode { + HCA_CAP_OPMOD_GET_MAX = 0, + HCA_CAP_OPMOD_GET_CUR = 1, +}; + struct mlx5_ifc_other_hca_cap_bits { u8 roce[0x1]; u8 reserved_at_1[0x27f]; @@ -7840,6 +7955,9 @@ struct mlx5_ifc_modify_vport_state_in_bits { u8 reserved_at_74[0x4]; u8 admin_state[0x4]; u8 reserved_at_7c[0x4]; + + u8 cap_tx_speed[0x10]; + u8 effective_tx_speed[0x10]; }; struct mlx5_ifc_modify_tis_out_bits { @@ -11270,6 +11388,55 @@ struct mlx5_ifc_eqe_bits { u8 owner[0x1]; }; +/* Values for eqe_bits.event_type */ +enum mlx5_event { + /* Special value to subscribe to any event */ + MLX5_EVENT_TYPE_NOTIFY_ANY = 0x0, + /* HW events enum start: comp events are not subscribable */ + MLX5_EVENT_TYPE_COMP = 0x0, + /* HW Async events enum start: subscribable events */ + MLX5_EVENT_TYPE_PATH_MIG = 0x01, + MLX5_EVENT_TYPE_COMM_EST = 0x02, + MLX5_EVENT_TYPE_SQ_DRAINED = 0x03, + MLX5_EVENT_TYPE_SRQ_LAST_WQE = 0x13, + MLX5_EVENT_TYPE_SRQ_RQ_LIMIT = 0x14, + + MLX5_EVENT_TYPE_CQ_ERROR = 0x04, + MLX5_EVENT_TYPE_WQ_CATAS_ERROR = 0x05, + MLX5_EVENT_TYPE_PATH_MIG_FAILED = 0x07, + MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR = 0x10, + MLX5_EVENT_TYPE_WQ_ACCESS_ERROR = 0x11, + MLX5_EVENT_TYPE_SRQ_CATAS_ERROR = 0x12, + MLX5_EVENT_TYPE_OBJECT_CHANGE = 0x27, + + MLX5_EVENT_TYPE_INTERNAL_ERROR = 0x08, + MLX5_EVENT_TYPE_PORT_CHANGE = 0x09, + MLX5_EVENT_TYPE_CMD = 0x0a, + MLX5_EVENT_TYPE_PAGE_REQUEST = 0x0b, + MLX5_EVENT_TYPE_PAGE_FAULT = 0x0c, + MLX5_EVENT_TYPE_NIC_VPORT_CHANGE = 0x0d, + MLX5_EVENT_TYPE_ESW_FUNCTIONS_CHANGED = 0x0e, + MLX5_EVENT_TYPE_VHCA_STATE_CHANGE = 0x0f, + MLX5_EVENT_TYPE_GPIO_EVENT = 0x15, + MLX5_EVENT_TYPE_PORT_MODULE_EVENT = 0x16, + MLX5_EVENT_TYPE_TEMP_WARN_EVENT = 0x17, + MLX5_EVENT_TYPE_XRQ_ERROR = 0x18, + MLX5_EVENT_TYPE_REMOTE_CONFIG = 0x19, + MLX5_EVENT_TYPE_DB_BF_CONGESTION = 0x1a, + MLX5_EVENT_TYPE_STALL_EVENT = 0x1b, + MLX5_EVENT_TYPE_DCT_DRAINED = 0x1c, + MLX5_EVENT_TYPE_DCT_KEY_VIOLATION = 0x1d, + MLX5_EVENT_TYPE_FPGA_ERROR = 0x20, + MLX5_EVENT_TYPE_FPGA_QP_ERROR = 0x21, + MLX5_EVENT_TYPE_GENERAL_EVENT = 0x22, + MLX5_EVENT_TYPE_MONITOR_COUNTER = 0x24, + MLX5_EVENT_TYPE_PPS_EVENT = 0x25, + MLX5_EVENT_TYPE_DEVICE_TRACER = 0x26, + + MLX5_EVENT_TYPE_MAX = 0x100, +}; + +/* Values for cmd_queue_entry_bits.type */ enum { MLX5_CMD_QUEUE_ENTRY_TYPE_PCIE_CMD_IF_TRANSPORT = 0x7, }; @@ -11312,6 +11479,27 @@ struct mlx5_ifc_cmd_out_bits { u8 command_output[0x20]; }; +/* Values for cmd_out_bits.status */ +enum { + MLX5_CMD_STAT_OK = 0x0, + MLX5_CMD_STAT_INT_ERR = 0x1, + MLX5_CMD_STAT_BAD_OP_ERR = 0x2, + MLX5_CMD_STAT_BAD_PARAM_ERR = 0x3, + MLX5_CMD_STAT_BAD_SYS_STATE_ERR = 0x4, + MLX5_CMD_STAT_BAD_RES_ERR = 0x5, + MLX5_CMD_STAT_RES_BUSY = 0x6, + MLX5_CMD_STAT_NOT_READY = 0x7, + MLX5_CMD_STAT_LIM_ERR = 0x8, + MLX5_CMD_STAT_BAD_RES_STATE_ERR = 0x9, + MLX5_CMD_STAT_IX_ERR = 0xa, + MLX5_CMD_STAT_NO_RES_ERR = 0xf, + MLX5_CMD_STAT_BAD_QP_STATE_ERR = 0x10, + MLX5_CMD_STAT_BAD_PKT_ERR = 0x30, + MLX5_CMD_STAT_BAD_SIZE_OUTS_CQES_ERR = 0x40, + MLX5_CMD_STAT_BAD_INP_LEN_ERR = 0x50, + MLX5_CMD_STAT_BAD_OUTP_LEN_ERR = 0x51, +}; + struct mlx5_ifc_cmd_in_bits { u8 opcode[0x10]; u8 reserved_at_10[0x10]; diff --git a/include/linux/mlx5/mlx5_ifc_macros.h b/include/linux/mlx5/mlx5_ifc_macros.h new file mode 100644 index 000000000000..1b9e39a50949 --- /dev/null +++ b/include/linux/mlx5/mlx5_ifc_macros.h @@ -0,0 +1,185 @@ +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ +/* + * Copyright (c) 2013-2026, Mellanox Technologies. All rights reserved. + * + * Accessor macros for mlx5 IFC structures. + * + * Extracted from device.h so that code which cannot include device.h + * (e.g. selftests) can still use the MLX5_SET/GET family directly. + */ + +#ifndef MLX5_IFC_MACROS_H +#define MLX5_IFC_MACROS_H + +/* Internal helpers -- 32-bit */ +#define __mlx5_nullp(typ) ((struct mlx5_ifc_##typ##_bits *)0) +#define __mlx5_bit_sz(typ, fld) sizeof(__mlx5_nullp(typ)->fld) +#define __mlx5_bit_off(typ, fld) (offsetof(struct mlx5_ifc_##typ##_bits, fld)) +#define __mlx5_16_off(typ, fld) (__mlx5_bit_off(typ, fld) / 16) +#define __mlx5_dw_off(typ, fld) (__mlx5_bit_off(typ, fld) / 32) +#define __mlx5_64_off(typ, fld) (__mlx5_bit_off(typ, fld) / 64) +#define __mlx5_16_bit_off(typ, fld) (16 - __mlx5_bit_sz(typ, fld) - (__mlx5_bit_off(typ, fld) & 0xf)) +#define __mlx5_dw_bit_off(typ, fld) (32 - __mlx5_bit_sz(typ, fld) - (__mlx5_bit_off(typ, fld) & 0x1f)) +#define __mlx5_mask(typ, fld) ((u32)((1ull << __mlx5_bit_sz(typ, fld)) - 1)) +#define __mlx5_dw_mask(typ, fld) (__mlx5_mask(typ, fld) << __mlx5_dw_bit_off(typ, fld)) +#define __mlx5_mask16(typ, fld) ((u16)((1ull << __mlx5_bit_sz(typ, fld)) - 1)) +#define __mlx5_16_mask(typ, fld) (__mlx5_mask16(typ, fld) << __mlx5_16_bit_off(typ, fld)) +#define __mlx5_st_sz_bits(typ) sizeof(struct mlx5_ifc_##typ##_bits) + +/* Size and address macros */ +#define MLX5_FLD_SZ_BYTES(typ, fld) (__mlx5_bit_sz(typ, fld) / 8) +#define MLX5_ST_SZ_BYTES(typ) (sizeof(struct mlx5_ifc_##typ##_bits) / 8) +#define MLX5_ST_SZ_DW(typ) (sizeof(struct mlx5_ifc_##typ##_bits) / 32) +#define MLX5_ST_SZ_QW(typ) (sizeof(struct mlx5_ifc_##typ##_bits) / 64) +#define MLX5_UN_SZ_BYTES(typ) (sizeof(union mlx5_ifc_##typ##_bits) / 8) +#define MLX5_UN_SZ_DW(typ) (sizeof(union mlx5_ifc_##typ##_bits) / 32) +#define MLX5_BYTE_OFF(typ, fld) (__mlx5_bit_off(typ, fld) / 8) +#define MLX5_ADDR_OF(typ, p, fld) ((void *)((u8 *)(p) + MLX5_BYTE_OFF(typ, fld))) + +/* insert a value to a struct */ +#define MLX5_SET(typ, p, fld, v) do { \ + u32 _v = v; \ + BUILD_BUG_ON(__mlx5_st_sz_bits(typ) % 32); \ + *((__be32 *)(p) + __mlx5_dw_off(typ, fld)) = \ + cpu_to_be32((be32_to_cpu(*((__be32 *)(p) + __mlx5_dw_off(typ, fld))) & \ + (~__mlx5_dw_mask(typ, fld))) | (((_v) & __mlx5_mask(typ, fld)) \ + << __mlx5_dw_bit_off(typ, fld))); \ +} while (0) + +#define MLX5_ARRAY_SET(typ, p, fld, idx, v) do { \ + BUILD_BUG_ON(__mlx5_bit_off(typ, fld) % 32); \ + MLX5_SET(typ, p, fld[idx], v); \ +} while (0) + +#define MLX5_SET_TO_ONES(typ, p, fld) do { \ + BUILD_BUG_ON(__mlx5_st_sz_bits(typ) % 32); \ + *((__be32 *)(p) + __mlx5_dw_off(typ, fld)) = \ + cpu_to_be32((be32_to_cpu(*((__be32 *)(p) + __mlx5_dw_off(typ, fld))) & \ + (~__mlx5_dw_mask(typ, fld))) | ((__mlx5_mask(typ, fld)) \ + << __mlx5_dw_bit_off(typ, fld))); \ +} while (0) + +#define MLX5_GET(typ, p, fld) ((be32_to_cpu(*((__be32 *)(p) +\ +__mlx5_dw_off(typ, fld))) >> __mlx5_dw_bit_off(typ, fld)) & \ +__mlx5_mask(typ, fld)) + +#define MLX5_GET_PR(typ, p, fld) ({ \ + u32 ___t = MLX5_GET(typ, p, fld); \ + pr_debug(#fld " = 0x%x\n", ___t); \ + ___t; \ +}) + +/* 64-bit field accessors */ +#define __MLX5_SET64(typ, p, fld, v) do { \ + BUILD_BUG_ON(__mlx5_bit_sz(typ, fld) != 64); \ + *((__be64 *)(p) + __mlx5_64_off(typ, fld)) = cpu_to_be64(v); \ +} while (0) + +#define MLX5_SET64(typ, p, fld, v) do { \ + BUILD_BUG_ON(__mlx5_bit_off(typ, fld) % 64); \ + __MLX5_SET64(typ, p, fld, v); \ +} while (0) + +#define MLX5_ARRAY_SET64(typ, p, fld, idx, v) do { \ + BUILD_BUG_ON(__mlx5_bit_off(typ, fld) % 64); \ + __MLX5_SET64(typ, p, fld[idx], v); \ +} while (0) + +#define MLX5_ARRAY_GET64(typ, p, fld, idx) \ + ({ \ + BUILD_BUG_ON(__mlx5_bit_off(typ, fld) % 64); \ + be64_to_cpu( \ + *((__be64 *)(p) + __mlx5_64_off(typ, fld) + (idx))); \ + }) + +#define MLX5_GET64(typ, p, fld) be64_to_cpu(*((__be64 *)(p) + __mlx5_64_off(typ, fld))) + +#define MLX5_GET64_PR(typ, p, fld) ({ \ + u64 ___t = MLX5_GET64(typ, p, fld); \ + pr_debug(#fld " = 0x%llx\n", ___t); \ + ___t; \ +}) + +/* 16-bit field accessors */ +#define MLX5_GET16(typ, p, fld) ((be16_to_cpu(*((__be16 *)(p) +\ +__mlx5_16_off(typ, fld))) >> __mlx5_16_bit_off(typ, fld)) & \ +__mlx5_mask16(typ, fld)) + +#define MLX5_SET16(typ, p, fld, v) do { \ + u16 _v = v; \ + BUILD_BUG_ON(__mlx5_st_sz_bits(typ) % 16); \ + *((__be16 *)(p) + __mlx5_16_off(typ, fld)) = \ + cpu_to_be16((be16_to_cpu(*((__be16 *)(p) + __mlx5_16_off(typ, fld))) & \ + (~__mlx5_16_mask(typ, fld))) | (((_v) & __mlx5_mask16(typ, fld)) \ + << __mlx5_16_bit_off(typ, fld))); \ +} while (0) + +/* Big endian getters */ +#define MLX5_GET64_BE(typ, p, fld) (*((__be64 *)(p) +\ + __mlx5_64_off(typ, fld))) + +#define MLX5_GET_BE(type_t, typ, p, fld) ({ \ + type_t tmp; \ + switch (sizeof(tmp)) { \ + case sizeof(u8): \ + tmp = (__force type_t)MLX5_GET(typ, p, fld); \ + break; \ + case sizeof(u16): \ + tmp = (__force type_t)cpu_to_be16(MLX5_GET(typ, p, fld)); \ + break; \ + case sizeof(u32): \ + tmp = (__force type_t)cpu_to_be32(MLX5_GET(typ, p, fld)); \ + break; \ + case sizeof(u64): \ + tmp = (__force type_t)MLX5_GET64_BE(typ, p, fld); \ + break; \ + } \ + tmp; \ + }) + +/* + * Use READ_ONCE/WRITE_ONCE for a single field that hardware may read/write + * unpredictably, mostly owner bits. All other bits in the DW must be stable. + * Usually a dma_wmb() will be required before a write and a dma_rmb() after a + * read. + */ +#define MLX5_GET_ONCE(typ, p, fld) \ + ((be32_to_cpu(READ_ONCE(*((__be32 *)(p) + __mlx5_dw_off(typ, fld)))) >> \ + __mlx5_dw_bit_off(typ, fld)) & \ + __mlx5_mask(typ, fld)) + +#define MLX5_SET_ONCE(typ, p, fld, v) \ + do { \ + u32 _v = v; \ + __be32 *_dw = (__be32 *)(p) + __mlx5_dw_off(typ, fld); \ + BUILD_BUG_ON(__mlx5_st_sz_bits(typ) % 32); \ + WRITE_ONCE(*_dw, \ + cpu_to_be32((be32_to_cpu(READ_ONCE(*_dw)) & \ + (~__mlx5_dw_mask(typ, fld))) | \ + (((_v) & __mlx5_mask(typ, fld)) \ + << __mlx5_dw_bit_off(typ, fld)))); \ + } while (0) + +/* Access MMIO registers, usually the init segment, using IFC structs. */ +#define MLX5_GET_MMIO(typ, p, fld) \ + ((ioread32be(((__be32 __iomem *)(p) + __mlx5_dw_off(typ, fld))) >> \ + __mlx5_dw_bit_off(typ, fld)) & \ + __mlx5_mask(typ, fld)) + +/* The set is not relaxed so there is an integrated dma_wmb(). */ +#define MLX5_SET_MMIO(typ, p, fld, v) \ + do { \ + u32 _v = v; \ + void __iomem *_dw = \ + ((__be32 __iomem *)(p) + __mlx5_dw_off(typ, fld)); \ + if (__mlx5_bit_sz(typ, fld) == 32) \ + iowrite32be(_v, _dw); \ + else \ + iowrite32be((ioread32be(_dw) & \ + (~__mlx5_dw_mask(typ, fld))) | \ + ((_v & __mlx5_mask(typ, fld)) \ + << __mlx5_dw_bit_off(typ, fld)), \ + _dw); \ + } while (0) + +#endif /* MLX5_IFC_MACROS_H */ diff --git a/include/linux/mm.h b/include/linux/mm.h index dd09c438fa23..9f167950926a 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -38,6 +38,7 @@ #include <linux/bitops.h> #include <linux/iommu-debug-pagealloc.h> #include <linux/kcsan-checks.h> +#include <linux/vmemmap-optimization.h> struct mempolicy; struct anon_vma; @@ -57,16 +58,6 @@ static inline unsigned long totalram_pages(void) return (unsigned long)atomic_long_read(&_totalram_pages); } -static inline void totalram_pages_inc(void) -{ - atomic_long_inc(&_totalram_pages); -} - -static inline void totalram_pages_dec(void) -{ - atomic_long_dec(&_totalram_pages); -} - static inline void totalram_pages_add(long count) { atomic_long_add(count, &_totalram_pages); @@ -587,14 +578,6 @@ enum { #define VMA_ACCESS_FLAGS mk_vma_flags(VMA_READ_BIT, VMA_WRITE_BIT, VMA_EXEC_BIT) /* - * Special vmas that are non-mergable, non-mlock()able. - */ - -#define VMA_SPECIAL_FLAGS mk_vma_flags(VMA_IO_BIT, VMA_DONTEXPAND_BIT, \ - VMA_PFNMAP_BIT, VMA_MIXEDMAP_BIT) -#define VM_SPECIAL vma_flags_to_legacy(VMA_SPECIAL_FLAGS) - -/* * Physically remapped pages are special. Tell the * rest of the world about it: * IO tells people not to look at these pages @@ -610,9 +593,6 @@ enum { #define VMA_REMAP_FLAGS mk_vma_flags(VMA_IO_BIT, VMA_PFNMAP_BIT, \ VMA_DONTEXPAND_BIT, VMA_DONTDUMP_BIT) -/* This mask prevents VMA from being scanned with khugepaged */ -#define VM_NO_KHUGEPAGED (VM_SPECIAL | VM_HUGETLB) - /* This mask defines which mm->def_flags a process can inherit its parent */ #define VM_INIT_DEF_MASK VM_NOHUGEPAGE @@ -928,7 +908,6 @@ static inline void vma_numab_state_free(struct vm_area_struct *vma) {} * These must be here rather than mmap_lock.h as dependent on vm_fault type, * declared in this header. */ -#ifdef CONFIG_PER_VMA_LOCK static inline void release_fault_lock(struct vm_fault *vmf) { if (vmf->flags & FAULT_FLAG_VMA_LOCK) @@ -944,17 +923,6 @@ static inline void assert_fault_locked(const struct vm_fault *vmf) else mmap_assert_locked(vmf->vma->vm_mm); } -#else -static inline void release_fault_lock(struct vm_fault *vmf) -{ - mmap_read_unlock(vmf->vma->vm_mm); -} - -static inline void assert_fault_locked(const struct vm_fault *vmf) -{ - mmap_assert_locked(vmf->vma->vm_mm); -} -#endif /* CONFIG_PER_VMA_LOCK */ static inline bool mm_flags_test(int flag, const struct mm_struct *mm) { @@ -1551,11 +1519,6 @@ static inline void vma_set_anonymous(struct vm_area_struct *vma) vma->vm_ops = NULL; } -static inline void vma_desc_set_anonymous(struct vm_area_desc *desc) -{ - desc->vm_ops = NULL; -} - static inline bool vma_is_anonymous(const struct vm_area_struct *vma) { return !vma->vm_ops; @@ -1640,6 +1603,183 @@ static inline bool vma_is_shared_maywrite(const struct vm_area_struct *vma) } /** + * vma_flags_is_hugetlb() - Do the specified VMA flags indicate that the + * VMA is a hugetlb mapping? + * @flags: The VMA flags to test. + * + * Returns: true if the flags indicate a hugetlb mapping, false otherwise. + */ +static inline bool vma_flags_is_hugetlb(const vma_flags_t *flags) +{ + return IS_ENABLED(CONFIG_HUGETLB_PAGE) && + vma_flags_test(flags, VMA_HUGETLB_BIT); +} + +/** + * vma_is_hugetlb() - Is @vma a hugetlb mapping? + * @vma: The VMA to test. + * + * Returns: true if @vma is a hugetlb mapping, false otherwise. + */ +static inline bool vma_is_hugetlb(const struct vm_area_struct *vma) +{ + return vma_flags_is_hugetlb(&vma->flags); +} + +/** + * vma_flags_is_mm_managed() - Do the specified VMA flags indicate that the + * contents of the VMA are managed by core mm? + * @flags: The VMA flags to test. + * + * Such a mapping is one established by a simple mmap() or brk() with no driver + * or other kernel component involved. + * + * They are rmappable, and core mm services rmap operations including reclaim, + * as well as population, CoW and merge/split (if eligible). + * + * Mappings which fail this test are those where this is not the case - + * e.g. MMIO, user-mapped kernel pages, udmabuf (pfnmap shmem), etc. + * + * Returns: true if the flags indicate an mm-managed mapping. + */ +static inline bool vma_flags_is_mm_managed(const vma_flags_t *flags) +{ + return !vma_flags_test_any(flags, VMA_PFNMAP_BIT, VMA_MIXEDMAP_BIT); +} + +/** + * vma_is_mm_managed() - Are the contents of @vma managed by core mm? + * @vma: The VMA to test. + * + * See vma_flags_is_mm_managed() for a description of this property. + * + * Returns: true if the VMA is mm-managed. + */ +static inline bool vma_is_mm_managed(const struct vm_area_struct *vma) +{ + return vma_flags_is_mm_managed(&vma->flags); +} + +/** + * vma_flags_is_fixed_mapping() - Do the specified VMA flags indicate that this + * is a fixed mapping that cannot be expanded or merged? + * @flags: The VMA flags to test. + * + * Fixed mappings are those whose size is set at the point of mmap (for + * instance, a mapping of a fixed range of memory established by a driver), + * and thus cannot be expanded or merged. + * + * Returns: true if the flags indicate a fixed mapping. + */ +static inline bool vma_flags_is_fixed_mapping(const vma_flags_t *flags) +{ + /* + * VMA_PFNMAP_BIT should imply VMA_DONTEXPAND_BIT, but some callers set + * only the former. + */ + return vma_flags_test_any(flags, VMA_PFNMAP_BIT, VMA_DONTEXPAND_BIT); +} + +/** + * vma_is_fixed_mapping() - Is this VMA a fixed mapping that cannot be + * expanded or merged? + * @vma: The VMA to test. + * + * See vma_flags_is_fixed_mapping() for a description of this property. + * + * Returns: true if the VMA maps a fixed mapping. + */ +static inline bool vma_is_fixed_mapping(const struct vm_area_struct *vma) +{ + return vma_flags_is_fixed_mapping(&vma->flags); +} + +/** + * vma_flags_is_mm_backed() - Do the specified VMA flags imply the mapping is + * backed by core mm? + * @flags: The VMA flags to test. + * + * An mm-backed mapping is one managed by core mm (see + * vma_flags_is_mm_managed()) whose contents are established by core mm's own + * fault path rather than a custom ->fault handler. + * + * Mappings not managed by core mm are populated by their owner, and fixed + * mappings may be mm-managed but have their contents established by a custom + * ->fault handler, so neither are mm-backed. hugetlb mappings are fixed, but + * core mm owns them entirely, so they are. + * + * These are the only mappings which can sensibly be merged, locked, dumped or + * be subject to uffd faulting, as all of these assume core mm and core mm + * alone manages their contents. + * + * Returns: true if the flags imply an mm-backed mapping, otherwise false. + */ +static inline bool vma_flags_is_mm_backed(const vma_flags_t *flags) +{ + /* hugetlb is a fixed mapping, but core mm owns it entirely. */ + if (vma_flags_is_hugetlb(flags)) + return true; + /* + * Mappings not managed by core mm are populated by their owner. + * + * Fixed mappings may be mm-managed, but their contents are established + * by a custom ->fault handler, so core mm cannot assume ordinary fault + * semantics over them. + */ + return vma_flags_is_mm_managed(flags) && !vma_flags_is_fixed_mapping(flags); +} + +/** + * vma_is_mm_backed() - Is @vma backed by core mm? + * @vma: The VMA to test. + * + * See vma_flags_is_mm_backed() for details. + * + * Returns: true if the VMA is mm-backed, otherwise false. + */ +static inline bool vma_is_mm_backed(const struct vm_area_struct *vma) +{ + return vma_flags_is_mm_backed(&vma->flags); +} + +/** + * vma_flags_can_merge() - Do the specified VMA flags permit the VMA to be + * merged with another? + * @flags: The VMA flags to test. + * Returns: true if the flags permit merging, false otherwise. + */ +static inline bool vma_flags_can_merge(const vma_flags_t *flags) +{ + /* + * VMA merging assumes that a VMA's flags and fields completely describe + * its state, which is only the case for mm-backed mappings. + * + * Mappings which are not mm-managed may have established state upon + * mapping not embodied in any attribute of the VMA. + * + * Additionally, private (CoW) PFN maps encode the source PFN of the + * range in vma->vm_pgoff, which may otherwise cause spurious merges. + */ + if (!vma_flags_is_mm_backed(flags)) + return false; + /* + * hugetlb mappings contain per-VMA state, so their flags and fields do + * not completely describe them. + */ + return !vma_flags_is_hugetlb(flags); +} + +/** + * vma_can_merge() - Do @vma's flags permit it to be merged with another VMA? + * @vma: The VMA to test. + * Returns: true if the flags permit merging, otherwise false. + */ +static inline bool vma_can_merge(const struct vm_area_struct *vma) +{ + return vma_flags_can_merge(&vma->flags); +} + +/** * vma_kernel_pagesize - Default page size granularity for this VMA. * @vma: The user mapping. * @@ -2075,20 +2215,21 @@ vm_fault_t finish_fault(struct vm_fault *vmf); * * A pagecache page contains an opaque `private' member, which belongs to the * page's address_space. Usually, this is the address of a circular list of - * the page's disk buffers. PG_private must be set to tell the VM to call - * into the filesystem to release these pages. + * the page's disk buffers. It tells the VM to call into the filesystem to + * release these pages. * * A folio may belong to an inode's memory mapping. In this case, * folio->mapping points to the inode, and folio->index is the file * offset of the folio, in units of PAGE_SIZE. * - * If pagecache pages are not associated with an inode, they are said to be - * anonymous pages. These may become associated with the swapcache, and in that - * case PG_swapcache is set, and page->private is an offset into the swapcache. + * If pagecache folios are not associated with an inode, they are said to be + * anonymous folios. These may become associated with the swapcache, and in that + * case PG_swapcache is set, and folio->private is an offset into the swapcache. * * In either case (swapcache or inode backed), the pagecache itself holds one - * reference to the page. Setting PG_private should also increment the - * refcount. The each user mapping also has a reference to the page. + * reference to the folio. Attaching filesystem private data via + * folio_attach_private() also increments the refcount. Each user mapping also + * has a reference to the folio. * * The pagecache pages are stored in a per-mapping radix tree, which is * rooted at mapping->i_pages, and indexed by offset. @@ -2644,12 +2785,23 @@ static inline void set_page_section(struct page *page, unsigned long section) page->flags.f |= (section & SECTIONS_MASK) << SECTIONS_PGSHIFT; } +static inline void set_page_section_from_pfn(struct page *page, + unsigned long pfn) +{ + set_page_section(page, pfn_to_section_nr(pfn)); +} + static inline unsigned long memdesc_section(const memdesc_flags_t *mdf) { ASSERT_EXCLUSIVE_BITS(mdf->f, SECTIONS_MASK << SECTIONS_PGSHIFT); return (mdf->f >> SECTIONS_PGSHIFT) & SECTIONS_MASK; } #else /* !SECTION_IN_PAGE_FLAGS */ +static inline void set_page_section_from_pfn(struct page *page, + unsigned long pfn) +{ +} + static inline unsigned long memdesc_section(const memdesc_flags_t *mdf) { return 0; @@ -2872,9 +3024,7 @@ static inline void set_page_links(struct page *page, enum zone_type zone, { set_page_zone(page, zone); set_page_node(page, node); -#ifdef SECTION_IN_PAGE_FLAGS - set_page_section(page, pfn_to_section_nr(pfn)); -#endif + set_page_section_from_pfn(page, pfn); } /** @@ -3022,9 +3172,9 @@ static inline bool folio_maybe_mapped_shared(struct folio *folio) * @folio: the folio * * Calculate the expected folio refcount, taking references from the pagecache, - * swapcache, PG_private and page table mappings into account. Useful in - * combination with folio_ref_count() to detect unexpected references (e.g., - * GUP or other temporary references). + * swapcache, private data (folio->private != NULL) and page table mappings into + * account. Useful in combination with folio_ref_count() to detect unexpected + * references (e.g., GUP or other temporary references). * * Does currently not consider references from the LRU cache. If the folio * was isolated from the LRU (which is the case during migration or split), @@ -3062,10 +3212,15 @@ static inline int folio_expected_ref_count(const struct folio *folio) ref_count += folio_test_swapcache(folio) << order; if (!folio_test_anon(folio)) { - /* One reference per page from the pagecache. */ - ref_count += !!folio->mapping << order; - /* One reference from PG_private. */ - ref_count += folio_test_private(folio); + /* + * One reference per page from the pagecache. + * Use data_race() since folio might not be locked. + */ + ref_count += !!data_race(folio->mapping) << order; + /* + * One reference from filesystem private data. + */ + ref_count += folio_has_attached_private(folio); } /* One reference per page table mapping. */ @@ -3325,10 +3480,10 @@ extern int access_process_vm(struct task_struct *tsk, unsigned long addr, extern int access_remote_vm(struct mm_struct *mm, unsigned long addr, void *buf, int len, unsigned int gup_flags); -#ifdef CONFIG_BPF_SYSCALL -extern int copy_remote_vm_str(struct task_struct *tsk, unsigned long addr, - void *buf, int len, unsigned int gup_flags); -#endif +int copy_remote_mm_str(struct mm_struct *mm, unsigned long addr, + void *buf, int len, unsigned int gup_flags); +int copy_remote_vm_str(struct task_struct *tsk, unsigned long addr, + void *buf, int len, unsigned int gup_flags); long get_user_pages_remote(struct mm_struct *mm, unsigned long start, unsigned long nr_pages, @@ -4083,12 +4238,6 @@ static inline void free_reserved_page(struct page *page) free_reserved_pages(page, 0); } -static inline void mark_page_reserved(struct page *page) -{ - SetPageReserved(page); - adjust_managed_page_count(page, -1); -} - static inline void free_reserved_ptdesc(struct ptdesc *pt) { free_reserved_page(ptdesc_page(pt)); @@ -4181,6 +4330,12 @@ void mapping_rmap_tree_insert_after(struct vm_area_struct *vma, struct address_space *mapping); void mapping_rmap_tree_remove(struct vm_area_struct *vma, struct address_space *mapping); +void mapping_rmap_tree_pre_update(struct vm_area_struct *vma, + struct address_space *mapping, + bool pgoff_unchanged); +void mapping_rmap_tree_post_update(struct vm_area_struct *vma, + struct address_space *mapping, + bool pgoff_unchanged); struct vm_area_struct * mapping_rmap_tree_iter_first(struct address_space *mapping, pgoff_t pgoff_start, pgoff_t pgoff_last); @@ -4198,6 +4353,10 @@ void anon_rmap_tree_insert(struct anon_vma_chain *avc, struct anon_vma *anon_vma); void anon_rmap_tree_remove(struct anon_vma_chain *avc, struct anon_vma *anon_vma); +void anon_rmap_tree_pre_update_vma(struct vm_area_struct *vma, + bool anon_pgoff_unchanged); +void anon_rmap_tree_post_update_vma(struct vm_area_struct *vma, + bool anon_pgoff_unchanged); struct anon_vma_chain * anon_rmap_tree_iter_first(struct anon_vma *anon_vma, pgoff_t pgoff_start, pgoff_t pgoff_last); @@ -4408,9 +4567,8 @@ static inline unsigned long vma_pages(const struct vm_area_struct *vma) * If @vma is a MAP_PRIVATE file-backed mapping, then this returns the * page offset within the file. * - * Edge cases: nommu does not abide by these, MAP_PRIVATE-/dev/zero satisfies - * vma_is_anonymous() but has file-backed page offset, and MAP_PRIVATE-pfnmap - * regions have their page offset set to the first PFN in the range. + * Edge cases: nommu does not abide by these and CoW MAP_PRIVATE-pfnmap regions + * have their page offset set to the first PFN in the range. * * Returns: The page offset of the start of @vma. */ @@ -4619,7 +4777,7 @@ static inline void mmap_action_simple_ioremap(struct vm_area_desc *desc, * @desc: The VMA descriptor for the VMA requiring kernel pags to be mapped. * @start: The virtual address from which to map them. * @pages: An array of struct page pointers describing the memory to map. - * @nr_pages: The number of entries in the @pages aray. + * @nr_pages: The number of entries in the @pages array. */ static inline void mmap_action_map_kernel_pages(struct vm_area_desc *desc, unsigned long start, struct page **pages, @@ -4627,7 +4785,7 @@ static inline void mmap_action_map_kernel_pages(struct vm_area_desc *desc, { struct mmap_action *action = &desc->action; - action->type = MMAP_MAP_KERNEL_PAGES; + action->type = MMAP_KERNEL_PAGES; action->map_kernel.start = start; action->map_kernel.pages = pages; action->map_kernel.nr_pages = nr_pages; @@ -4651,10 +4809,53 @@ static inline void mmap_action_map_kernel_pages_full(struct vm_area_desc *desc, vma_desc_pages(desc)); } +static inline +void mmap_action_map_discontig_kernel_pages(struct vm_area_desc *desc, + void *init_private, const struct discontig_kernel_page_ops *ops) +{ + struct mmap_action *action = &desc->action; + + action->type = MMAP_DISCONTIG_KERNEL_PAGES; + action->map_kernel_discontig.init_private = init_private; + action->map_kernel_discontig.ops = ops; +} + int mmap_action_prepare(struct vm_area_desc *desc); int mmap_action_complete(struct vm_area_struct *vma, struct mmap_action *action, bool is_compat); +static inline void +discontig_kernel_map_abort(struct discontig_kernel_page_state *state) +{ + state->action = DISCONTIG_KERNEL_PAGE_ABORT; +} + +static inline void +discontig_kernel_map_page(struct discontig_kernel_page_state *state, + struct page *page) +{ + struct folio *folio = page_folio(page); + unsigned long nr_pages = 1; + + if (folio_test_large(folio)) { + VM_WARN_ON_ONCE(page != folio_page(folio, 0)); + nr_pages = folio_nr_pages(folio); + } + + state->action = DISCONTIG_KERNEL_PAGE_MAP_CONTIG_PAGES; + state->__page = page; + state->__nr_pages = min(nr_pages, state->nr_pages_remain); +} + +static inline void +discontig_kernel_map_pages(struct discontig_kernel_page_state *state, + struct page **page_arr, unsigned long nr_pages) +{ + state->action = DISCONTIG_KERNEL_PAGE_MAP_PAGES; + state->__page_arr = page_arr; + state->__nr_pages = min(nr_pages, state->nr_pages_remain); +} + /* Look up the first VMA which exactly match the interval vm_start ... vm_end */ static inline struct vm_area_struct *find_exact_vma(struct mm_struct *mm, unsigned long vm_start, unsigned long vm_end) @@ -4772,9 +4973,6 @@ int remap_pfn_range(struct vm_area_struct *vma, unsigned long addr, int vm_insert_page(struct vm_area_struct *, unsigned long addr, struct page *); int vm_insert_pages(struct vm_area_struct *vma, unsigned long addr, struct page **pages, unsigned long *num); -int map_kernel_pages_prepare(struct vm_area_desc *desc); -int map_kernel_pages_complete(struct vm_area_struct *vma, - struct mmap_action *action); int vm_map_pages(struct vm_area_struct *vma, struct page **pages, unsigned long num); int vm_map_pages_zero(struct vm_area_struct *vma, struct page **pages, @@ -5140,7 +5338,6 @@ static inline void print_vma_addr(char *prefix, unsigned long rip) } #endif -unsigned long section_map_size(void); struct page * __populate_section_memmap(unsigned long pfn, unsigned long nr_pages, int nid, struct vmem_altmap *altmap, struct dev_pagemap *pgmap); @@ -5159,9 +5356,6 @@ int vmemmap_populate_hugepages(unsigned long start, unsigned long end, int node, struct vmem_altmap *altmap); int vmemmap_populate(unsigned long start, unsigned long end, int node, struct vmem_altmap *altmap); -int vmemmap_populate_hvo(unsigned long start, unsigned long end, - unsigned int order, struct zone *zone, - unsigned long headsize); void vmemmap_wrprotect_hvo(unsigned long start, unsigned long end, int node, unsigned long headsize); void vmemmap_populate_print_last(void); @@ -5196,7 +5390,6 @@ static inline void vmem_altmap_free(struct vmem_altmap *altmap, } #endif -#define VMEMMAP_RESERVE_NR 2 #ifdef CONFIG_ARCH_WANT_OPTIMIZE_DAX_VMEMMAP static inline bool __vmemmap_can_optimize(struct vmem_altmap *altmap, struct dev_pagemap *pgmap) @@ -5204,6 +5397,9 @@ static inline bool __vmemmap_can_optimize(struct vmem_altmap *altmap, unsigned long nr_pages; unsigned long nr_vmemmap_pages; + if (!IS_ENABLED(CONFIG_VMEMMAP_OPTIMIZATION)) + return false; + if (!pgmap || !is_power_of_2(sizeof(struct page))) return false; @@ -5213,7 +5409,7 @@ static inline bool __vmemmap_can_optimize(struct vmem_altmap *altmap, * For vmemmap optimization with DAX we need minimum 2 vmemmap * pages. See layout diagram in Documentation/mm/vmemmap_dedup.rst */ - return !altmap && (nr_vmemmap_pages > VMEMMAP_RESERVE_NR); + return !altmap && (nr_vmemmap_pages > VMEMMAP_OPTIMIZATION_PAGES); } /* * If we don't have an architecture override, use the generic rule @@ -5254,6 +5450,8 @@ extern const struct attribute_group memory_failure_attr_group; extern void memory_failure_queue(unsigned long pfn, int flags); void num_poisoned_pages_inc(unsigned long pfn); void num_poisoned_pages_sub(unsigned long pfn, long i); +phys_addr_t range_first_hwpoison(phys_addr_t start, unsigned long size); +phys_addr_t range_last_hwpoison(phys_addr_t start, unsigned long size); #else static inline void memory_failure_queue(unsigned long pfn, int flags) { @@ -5266,6 +5464,18 @@ static inline void num_poisoned_pages_inc(unsigned long pfn) static inline void num_poisoned_pages_sub(unsigned long pfn, long i) { } + +static inline phys_addr_t range_first_hwpoison(phys_addr_t start, + unsigned long size) +{ + return PHYS_ADDR_MAX; +} + +static inline phys_addr_t range_last_hwpoison(phys_addr_t start, + unsigned long size) +{ + return PHYS_ADDR_MAX; +} #endif #if defined(CONFIG_MEMORY_FAILURE) && defined(CONFIG_MEMORY_HOTPLUG) diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h index 621c8653d8f7..ab69b9930893 100644 --- a/include/linux/mm_inline.h +++ b/include/linux/mm_inline.h @@ -30,6 +30,17 @@ static inline int folio_is_file_lru(const struct folio *folio) return !folio_test_swapbacked(folio); } +/** + * folio_flags_is_file_lru - Should the folio be on a file LRU or anon LRU? + * @flags: The folio's flags. + * + * Just like folio_is_file_lru but take the folio flags directly instead. + */ +static inline int folio_flags_is_file_lru(const unsigned long *flags) +{ + return !test_bit(PG_swapbacked, flags); +} + static __always_inline void __update_lru_size(struct lruvec *lruvec, enum lru_list lru, enum zone_type zid, long nr_pages) @@ -142,10 +153,66 @@ static inline int lru_tier_from_refs(int refs, bool workingset) return workingset ? MAX_NR_TIERS - 1 : order_base_2(refs); } -static inline int folio_lru_refs(const struct folio *folio) +/** + * lru_set_gen_flags - Set the LRU generation number to specified folio flags. + * @flags: pointer to the folio flags + * @gen: generation number, between 0 and (MAX_NR_GENS - 1), inclusive. + */ +static inline void lru_set_gen_flags(unsigned long *flags, int gen) +{ + BUILD_BUG_ON(LRU_GEN_MASK & LRU_REFS_MASK); + VM_WARN_ON_ONCE(gen >= MAX_NR_GENS || gen < 0); + /* Store gen offset by 1, zero means the folio is off-list. */ + *flags &= ~LRU_GEN_MASK; + *flags |= (gen + 1UL) << LRU_GEN_PGOFF; +} + +/** + * lru_get_gen_flags - Return the LRU generation number from folio flags. + * @flags: folio flags + * + * Returns: A number between 0 and (MAX_NR_GENS - 1), inclusive. Returns + * -1 if the flags indicate the folio is off the list (e.g., isolated). + */ +static inline int lru_get_gen_flags(unsigned long flags) +{ + int gen = ((flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1; + + /* Exclude the legal -1 from the unsigned MAX_NR_GENS comparison */ + VM_WARN_ON_ONCE(gen != -1 && gen >= MAX_NR_GENS); + return gen; +} + +/** + * lru_set_refs_flags - Set the LRU referenced count to folio flags. + * @flags: pointer to the folio flags + * @refs: referenced / access count number, between 0 and LRU_REFS_MAX, inclusive. + * + * For MGLRU, PG_referenced holds the first ref, and the extra bits hold the + * remaining refs. For classical LRU the extra bits are not used, so it can + * also be seen as the refs count never exceeds 1. In both cases, refs == 1 + * means PG_referenced is set and the extra bits are zero, and refs == 0 means + * PG_referenced and the extra bits are all unset. + */ +static inline void lru_set_refs_flags(unsigned long *flags, unsigned int refs) { - unsigned long flags = READ_ONCE(folio->flags.f); + VM_WARN_ON_ONCE(refs > LRU_REFS_MAX); + BUILD_BUG_ON(LRU_REFS_MAX != (LRU_REFS_MASK >> LRU_REFS_PGOFF) + 1); + *flags &= ~LRU_REFS_FLAGS; + if (!refs) + return; + *flags |= (BIT(PG_referenced) | ((refs - 1UL) << LRU_REFS_PGOFF)); +} + +/** + * lru_get_refs_flags - Return LRU referenced / access count from folio flags. + * @flags: folio flags + * + * Reads the LRU referenced count set by lru_set_refs_flags(). + */ +static inline int lru_get_refs_flags(unsigned long flags) +{ if (!(flags & BIT(PG_referenced))) return 0; /* @@ -155,11 +222,24 @@ static inline int folio_lru_refs(const struct folio *folio) return ((flags & LRU_REFS_MASK) >> LRU_REFS_PGOFF) + 1; } -static inline int folio_lru_gen(const struct folio *folio) +static inline int folio_lru_refs(const struct folio *folio) { - unsigned long flags = READ_ONCE(folio->flags.f); + return lru_get_refs_flags(READ_ONCE(*const_folio_flags(folio, 0))); +} - return ((flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1; +static inline void folio_set_lru_refs(struct folio *folio, unsigned int refs) +{ + unsigned long new_flags, old_flags = READ_ONCE(*folio_flags(folio, 0)); + + do { + new_flags = old_flags; + lru_set_refs_flags(&new_flags, refs); + } while (!try_cmpxchg(folio_flags(folio, 0), &old_flags, new_flags)); +} + +static inline int folio_lru_gen(const struct folio *folio) +{ + return lru_get_gen_flags(READ_ONCE(*const_folio_flags(folio, 0))); } static inline bool lru_gen_is_active(const struct lruvec *lruvec, int gen) @@ -270,7 +350,7 @@ static inline bool lru_gen_add_folio(struct lruvec *lruvec, struct folio *folio, gen = lru_gen_from_seq(seq); flags = (gen + 1UL) << LRU_GEN_PGOFF; /* see the comment on MIN_NR_GENS about PG_active */ - set_mask_bits(&folio->flags.f, LRU_GEN_MASK | BIT(PG_active), flags); + set_mask_bits(folio_flags(folio, 0), LRU_GEN_MASK | BIT(PG_active), flags); lru_gen_update_size(lruvec, folio, -1, gen); /* for folio_rotate_reclaimable() */ @@ -295,7 +375,7 @@ static inline bool lru_gen_del_folio(struct lruvec *lruvec, struct folio *folio, /* for folio_migrate_flags() */ flags = !reclaiming && lru_gen_is_active(lruvec, gen) ? BIT(PG_active) : 0; - flags = set_mask_bits(&folio->flags.f, LRU_GEN_MASK, flags); + flags = set_mask_bits(folio_flags(folio, 0), LRU_GEN_MASK, flags); gen = ((flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1; lru_gen_update_size(lruvec, folio, gen, -1); @@ -304,11 +384,19 @@ static inline bool lru_gen_del_folio(struct lruvec *lruvec, struct folio *folio, return true; } -static inline void folio_migrate_refs(struct folio *new, const struct folio *old) +/** + * folio_migrate_lru_refs - copy the reference state to a new folio + * @new: the destination folio + * @old: the source folio + * + * Transfer the reference state to @new during migration: the MGLRU + * refs count, including PG_referenced, or just PG_referenced for the + * active/inactive LRU. + */ +static inline void folio_migrate_lru_refs(struct folio *new, const struct folio *old) { - unsigned long refs = READ_ONCE(old->flags.f) & LRU_REFS_MASK; - - set_mask_bits(&new->flags.f, LRU_REFS_MASK, refs); + BUILD_BUG_ON(LRU_REFS_MASK & BIT(PG_referenced)); + folio_set_lru_refs(new, folio_lru_refs(old)); } #else /* !CONFIG_LRU_GEN */ @@ -337,9 +425,10 @@ static inline bool lru_gen_del_folio(struct lruvec *lruvec, struct folio *folio, return false; } -static inline void folio_migrate_refs(struct folio *new, const struct folio *old) +static inline void folio_migrate_lru_refs(struct folio *new, const struct folio *old) { - + if (folio_test_referenced(old)) + folio_set_referenced(new); } #endif /* CONFIG_LRU_GEN */ diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index f3e5a2fadbe5..1ed35d258d2e 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -108,7 +108,7 @@ struct page { }; /** * @private: Mapping-private opaque data. - * Usually used for buffer_heads if PagePrivate. + * Usually used for buffer_heads. * Used for swp_entry_t if swapcache flag set. * Indicates order in the buddy system if PageBuddy * or on pcp_llist. @@ -675,7 +675,7 @@ static inline void ptdesc_pmd_pts_init(struct ptdesc *ptdesc) #define STRUCT_PAGE_MAX_SHIFT (order_base_2(sizeof(struct page))) /* - * page_private can be used on tail pages. However, PagePrivate is only + * page_private can be used on tail pages. However, it is only * checked by the VM on the head page. So page_private on the tail pages * should be used for data that's ancillary to the head page (eg attaching * buffer heads to tail pages after attaching buffer heads to the head page) @@ -759,7 +759,7 @@ static inline struct anon_vma_name *anon_vma_name_alloc(const char *name) /* * While __vma_enter_locked() is working to ensure are no read-locks held on a * VMA (either while acquiring a VMA write lock or marking a VMA detached) we - * set the VM_REFCNT_EXCLUDE_READERS_FLAG in vma->vm_refcnt to indiciate to + * set the VM_REFCNT_EXCLUDE_READERS_FLAG in vma->vm_refcnt to indicate to * vma_start_read() that the reference count should be left alone. * * See the comment describing vm_refcnt in vm_area_struct for details as to @@ -815,11 +815,45 @@ struct pfnmap_track_ctx { /* What action should be taken after an .mmap_prepare call is complete? */ enum mmap_action_type { - MMAP_NOTHING, /* Mapping is complete, no further action. */ - MMAP_REMAP_PFN, /* Remap PFN range. */ - MMAP_IO_REMAP_PFN, /* I/O remap PFN range. */ - MMAP_SIMPLE_IO_REMAP, /* I/O remap with guardrails. */ - MMAP_MAP_KERNEL_PAGES, /* Map kernel page range from array. */ + MMAP_NOTHING, + MMAP_REMAP_PFN, + MMAP_IO_REMAP_PFN, + MMAP_SIMPLE_IO_REMAP, /* I/O remap with guardrails. */ + MMAP_KERNEL_PAGES, /* Map kernel page range from array. */ + MMAP_DISCONTIG_KERNEL_PAGES, /* Map kernel discontig page range. */ +}; + +enum discontig_kernel_page_action { + DISCONTIG_KERNEL_PAGE_ABORT, + DISCONTIG_KERNEL_PAGE_MAP_PAGES, /* Array of individual pages. */ + DISCONTIG_KERNEL_PAGE_MAP_CONTIG_PAGES, /* 1..N contiguous pages. */ +}; + +struct discontig_kernel_page_state { + /* Map state. */ + const unsigned long start; /* Start address of VMA. */ + const unsigned long end; /* End address of VMA. */ + unsigned long addr; /* The current address to be mapped. */ + pgoff_t pgoff; /* The current pgoff to be mapped. */ + unsigned long nr_pages_mapped; /* The number of pages mapped. */ + unsigned long nr_pages_remain; /* The number of pages remaining. */ + + /* User-defined state. */ + void *vm_private_data; /* VMA private data. */ + void *private; /* Mapping private data. */ + + /* Users should not touch these, use discontig_kernel_map_*() helpers. */ + enum discontig_kernel_page_action action; + union { + struct page *__page; + struct page **__page_arr; + }; + unsigned long __nr_pages; +}; + +struct discontig_kernel_page_ops { + int (*init)(void *vm_private_data, void **private); + int (*get)(struct discontig_kernel_page_state *state); }; /* @@ -844,6 +878,10 @@ struct mmap_action { unsigned long nr_pages; pgoff_t pgoff; } map_kernel; + struct { + void *init_private; + const struct discontig_kernel_page_ops *ops; + } map_kernel_discontig; }; enum mmap_action_type type; @@ -950,23 +988,26 @@ struct vm_area_struct { vma_flags_t flags; }; -#ifdef CONFIG_PER_VMA_LOCK /* * Can only be written (using WRITE_ONCE()) while holding both: * - mmap_lock (in write mode) * - vm_refcnt bit at VM_REFCNT_EXCLUDE_READERS_FLAG is set + * The one exception is __mm_reset_vma_seqcounts(), which stores zero + * (an even value, which can never indicate a write lock) under the + * mmap write lock alone once the mm seqcount has wrapped, so a reader + * holding only vm_refcnt may observe the value change from odd to zero. * Can be read reliably while holding one of: * - mmap_lock (in read or write mode) - * - vm_refcnt bit at VM_REFCNT_EXCLUDE_READERS_BIT is set or vm_refcnt > 1 + * - vm_refcnt bit at VM_REFCNT_EXCLUDE_READERS_BIT is set * Can be read unreliably (using READ_ONCE()) for pessimistic bailout * while holding nothing (except RCU to keep the VMA struct allocated). * - * This sequence counter is explicitly allowed to overflow; sequence - * counter reuse can only lead to occasional unnecessary use of the - * slowpath. + * Assigned the (odd) mm seqcount on write lock and not updated on + * unlock, so a stale value would alias the mm seqcount again 2^31 write + * lock/unlock cycles later. The reset described above prevents this. */ unsigned int vm_lock_seq; -#endif + /* * Low 32-bits of anonymous page offset. * See vma_start_anon_pgoff() comment for details. @@ -1003,7 +1044,6 @@ struct vm_area_struct { #ifdef CONFIG_NUMA_BALANCING struct vma_numab_state *numab_state; /* NUMA Balancing state */ #endif -#ifdef CONFIG_PER_VMA_LOCK /* * Used to keep track of firstly, whether the VMA is attached, secondly, * if attached, how many read locks are taken, and thirdly, if the @@ -1046,7 +1086,6 @@ struct vm_area_struct { #ifdef CONFIG_DEBUG_LOCK_ALLOC struct lockdep_map vmlock_dep_map; #endif -#endif #ifdef CONFIG_64BIT /* * High 32-bits of anonymous page offset. @@ -1254,7 +1293,6 @@ struct mm_struct { * init_mm.mmlist, and are protected * by mmlist_lock */ -#ifdef CONFIG_PER_VMA_LOCK struct rcuwait vma_writer_wait; /* * This field has lock-like semantics, meaning it is sometimes @@ -1274,7 +1312,7 @@ struct mm_struct { * mmap_lock. */ seqcount_t mm_lock_seq; -#endif + struct futex_mm_data futex; unsigned long hiwater_rss; /* High-watermark of RSS usage */ @@ -1975,7 +2013,7 @@ enum { /* * MMF_HAS_PINNED: Whether this mm has pinned any pages. This can be either * replaced in the future by mm.pinned_vm when it becomes stable, or grow into - * a counter on its own. We're aggresive on this bit for now: even if the + * a counter on its own. We're aggressive on this bit for now: even if the * pinned pages were unpinned later on, we'll still keep this bit set for the * lifecycle of this mm, just for simplicity. */ diff --git a/include/linux/mmap_lock.h b/include/linux/mmap_lock.h index b8a13b8d36a4..11b3ffef3966 100644 --- a/include/linux/mmap_lock.h +++ b/include/linux/mmap_lock.h @@ -76,8 +76,6 @@ static inline void mmap_assert_write_locked(const struct mm_struct *mm) rwsem_assert_held_write(&mm->mmap_lock); } -#ifdef CONFIG_PER_VMA_LOCK - #ifdef CONFIG_LOCKDEP #define __vma_lockdep_map(vma) (&vma->vmlock_dep_map) #else @@ -156,7 +154,7 @@ static inline void vma_lock_init(struct vm_area_struct *vma, bool reset_refcnt) #endif if (reset_refcnt) refcount_set(&vma->vm_refcnt, 0); - vma->vm_lock_seq = UINT_MAX; + vma->vm_lock_seq = 0; } /* @@ -230,10 +228,15 @@ static inline void vma_refcount_put(struct vm_area_struct *vma) } /* - * Use only while holding mmap read lock which guarantees that locking will not - * fail (nobody can concurrently write-lock the vma). vma_start_read() should - * not be used in such cases because it might fail due to mm_lock_seq overflow. + * Use only while holding mmap read lock which guarantees that vma lock is not + * contended (nobody can concurrently write-lock the vma). vma_start_read() need + * not be used in such cases, as the mmap lock already excludes VMA writers so + * its sequence number checks are redundant. * This functionality is used to obtain vma read lock and drop the mmap read lock. + * + * VMA can't be detached while we are holding mmap lock, therefore in practice this + * function can fail only when there are so many readers that vm_refcnt overflows. + * The failure case is very unlikely and is already annotated as such internally. */ static inline bool vma_start_read_locked_nested(struct vm_area_struct *vma, int subclass) { @@ -249,22 +252,30 @@ static inline bool vma_start_read_locked_nested(struct vm_area_struct *vma, int } /* - * Use only while holding mmap read lock which guarantees that locking will not - * fail (nobody can concurrently write-lock the vma). vma_start_read() should - * not be used in such cases because it might fail due to mm_lock_seq overflow. + * Use only while holding mmap read lock which guarantees that vma lock is not + * contended (nobody can concurrently write-lock the vma). vma_start_read() need + * not be used in such cases, as the mmap lock already excludes VMA writers so + * its sequence number checks are redundant. * This functionality is used to obtain vma read lock and drop the mmap read lock. + * + * VMA can't be detached while we are holding mmap lock, therefore in practice this + * function can fail only when there are so many readers that vm_refcnt overflows. + * The failure case is very unlikely and is already annotated as such internally. */ static inline bool vma_start_read_locked(struct vm_area_struct *vma) { return vma_start_read_locked_nested(vma, 0); } +struct vm_area_struct *vma_start_read_unlocked(struct mm_struct *mm, + unsigned long address); + static inline void vma_end_read(struct vm_area_struct *vma) { vma_refcount_put(vma); } -static inline unsigned int __vma_raw_mm_seqnum(struct vm_area_struct *vma) +static inline unsigned int __vma_raw_mm_seqnum(const struct vm_area_struct *vma) { const struct mm_struct *mm = vma->vm_mm; @@ -279,7 +290,7 @@ static inline unsigned int __vma_raw_mm_seqnum(struct vm_area_struct *vma) * * Returns true if write-locked, otherwise false. */ -static inline bool __is_vma_write_locked(struct vm_area_struct *vma) +static inline bool __is_vma_write_locked(const struct vm_area_struct *vma) { /* * current task is holding mmap_write_lock, both vma->vm_lock_seq and @@ -297,6 +308,9 @@ int __vma_start_write(struct vm_area_struct *vma, int state); */ static inline void vma_start_write(struct vm_area_struct *vma) { + if (!IS_ENABLED(CONFIG_MMU)) + return; + if (__is_vma_write_locked(vma)) return; @@ -319,6 +333,9 @@ static inline void vma_start_write(struct vm_area_struct *vma) static inline __must_check int vma_start_write_killable(struct vm_area_struct *vma) { + if (!IS_ENABLED(CONFIG_MMU)) + return 0; + if (__is_vma_write_locked(vma)) return 0; @@ -329,8 +346,13 @@ int vma_start_write_killable(struct vm_area_struct *vma) * vma_assert_write_locked() - assert that @vma holds a VMA write lock. * @vma: The VMA to assert. */ -static inline void vma_assert_write_locked(struct vm_area_struct *vma) +static inline void vma_assert_write_locked(const struct vm_area_struct *vma) { + if (!IS_ENABLED(CONFIG_MMU)) { + mmap_assert_write_locked(vma->vm_mm); + return; + } + VM_WARN_ON_ONCE_VMA(!__is_vma_write_locked(vma), vma); } @@ -339,10 +361,15 @@ static inline void vma_assert_write_locked(struct vm_area_struct *vma) * lock and is not detached. * @vma: The VMA to assert. */ -static inline void vma_assert_locked(struct vm_area_struct *vma) +static inline void vma_assert_locked(const struct vm_area_struct *vma) { unsigned int refcnt; + if (!IS_ENABLED(CONFIG_MMU)) { + mmap_assert_locked(vma->vm_mm); + return; + } + if (IS_ENABLED(CONFIG_LOCKDEP)) { if (!lock_is_held(__vma_lockdep_map(vma))) vma_assert_write_locked(vma); @@ -385,7 +412,7 @@ static inline void vma_assert_locked(struct vm_area_struct *vma) * With lockdep disabled we may sometimes race with other threads acquiring the * mmap read lock simultaneous with our VMA read lock. */ -static inline void vma_assert_stabilised(struct vm_area_struct *vma) +static inline void vma_assert_stabilised(const struct vm_area_struct *vma) { /* * If another thread owns an mmap lock, it may go away at any time, and @@ -420,7 +447,7 @@ static inline void vma_assert_stabilised(struct vm_area_struct *vma) vma_assert_locked(vma); } -static inline bool vma_is_attached(struct vm_area_struct *vma) +static inline bool vma_is_attached(const struct vm_area_struct *vma) { return refcount_read(&vma->vm_refcnt); } @@ -432,6 +459,9 @@ static inline bool vma_is_attached(struct vm_area_struct *vma) */ static inline void vma_assert_attached(struct vm_area_struct *vma) { + if (!IS_ENABLED(CONFIG_MMU)) + return; + WARN_ON_ONCE(!vma_is_attached(vma)); } @@ -442,6 +472,9 @@ static inline void vma_assert_detached(struct vm_area_struct *vma) static inline void vma_mark_attached(struct vm_area_struct *vma) { + if (!IS_ENABLED(CONFIG_MMU)) + return; + vma_assert_write_locked(vma); vma_assert_detached(vma); refcount_set_release(&vma->vm_refcnt, 1); @@ -451,6 +484,9 @@ void __vma_exclude_readers_for_detach(struct vm_area_struct *vma); static inline void vma_mark_detached(struct vm_area_struct *vma) { + if (!IS_ENABLED(CONFIG_MMU)) + return; + vma_assert_write_locked(vma); vma_assert_attached(vma); @@ -484,54 +520,6 @@ struct vm_area_struct *lock_next_vma(struct mm_struct *mm, struct vma_iterator *iter, unsigned long address); -#else /* CONFIG_PER_VMA_LOCK */ - -static inline void mm_lock_seqcount_init(struct mm_struct *mm) {} -static inline void mm_lock_seqcount_begin(struct mm_struct *mm) {} -static inline void mm_lock_seqcount_end(struct mm_struct *mm) {} - -static inline bool mmap_lock_speculate_try_begin(struct mm_struct *mm, unsigned int *seq) -{ - return false; -} - -static inline bool mmap_lock_speculate_retry(struct mm_struct *mm, unsigned int seq) -{ - return true; -} -static inline void vma_lock_init(struct vm_area_struct *vma, bool reset_refcnt) {} -static inline void vma_end_read(struct vm_area_struct *vma) {} -static inline void vma_start_write(struct vm_area_struct *vma) {} -static inline __must_check -int vma_start_write_killable(struct vm_area_struct *vma) { return 0; } -static inline void vma_assert_write_locked(struct vm_area_struct *vma) - { mmap_assert_write_locked(vma->vm_mm); } -static inline bool vma_is_attached(struct vm_area_struct *vma) - { return true; } -static inline void vma_assert_attached(struct vm_area_struct *vma) {} -static inline void vma_assert_detached(struct vm_area_struct *vma) {} -static inline void vma_mark_attached(struct vm_area_struct *vma) {} -static inline void vma_mark_detached(struct vm_area_struct *vma) {} - -static inline struct vm_area_struct *lock_vma_under_rcu(struct mm_struct *mm, - unsigned long address) -{ - return NULL; -} - -static inline void vma_assert_locked(struct vm_area_struct *vma) -{ - mmap_assert_locked(vma->vm_mm); -} - -static inline void vma_assert_stabilised(struct vm_area_struct *vma) -{ - /* If no VMA locks, then either mmap lock suffices to stabilise. */ - mmap_assert_locked(vma->vm_mm); -} - -#endif /* CONFIG_PER_VMA_LOCK */ - static inline void vma_assert_can_modify(struct vm_area_struct *vma) { if (vma_is_attached(vma)) @@ -566,6 +554,8 @@ static inline int __must_check mmap_write_lock_killable(struct mm_struct *mm) return ret; } +void __mm_reset_vma_seqcounts(struct mm_struct *mm); + /* * Drop all currently-held per-VMA locks. * This is called from the mmap_lock implementation directly before releasing @@ -578,6 +568,16 @@ static inline void vma_end_write_all(struct mm_struct *mm) { mmap_assert_write_locked(mm); mm_lock_seqcount_end(mm); + + /* + * VMA seqcounts are not updated on unlock, so once the mm seqcount has + * wrapped, any VMA still carrying a sequence number from before the + * wrap would satisfy __is_vma_write_locked() when that number comes + * round again, and readers would not be excluded. Reset every VMA's + * seqcount so that no stale value survives the wrap. + */ + if (unlikely(!mm->mm_lock_seq.sequence)) + __mm_reset_vma_seqcounts(mm); } static inline void mmap_write_unlock(struct mm_struct *mm) diff --git a/include/linux/mmc/host.h b/include/linux/mmc/host.h index ba84f02c2a10..032c018650e0 100644 --- a/include/linux/mmc/host.h +++ b/include/linux/mmc/host.h @@ -23,6 +23,7 @@ struct mmc_ios { unsigned int clock; /* clock rate */ unsigned short vdd; unsigned int power_delay_ms; /* waiting for stable power */ + unsigned int power_off_delay_us; /* waiting for power discharge */ /* vdd stores the bit number of the selected voltage range from below. */ @@ -463,6 +464,7 @@ struct mmc_host { #define MMC_CAP2_CRYPTO 0 #endif #define MMC_CAP2_ALT_GPT_TEGRA (1 << 28) /* Host with eMMC that has GPT entry at a non-standard location */ +#define MMC_CAP2_CRYPTO_NO_REPROG (1 << 29) /* Host handles inline crypto key reprogramming */ bool uhs2_sd_tran; /* UHS-II flag for SD_TRAN state */ bool uhs2_app_cmd; /* UHS-II flag for APP command */ @@ -583,11 +585,9 @@ struct mmc_host { struct device_node; -struct mmc_host *mmc_alloc_host(int extra, struct device *); struct mmc_host *devm_mmc_alloc_host(struct device *dev, int extra); int mmc_add_host(struct mmc_host *); void mmc_remove_host(struct mmc_host *); -void mmc_free_host(struct mmc_host *); void mmc_of_parse_clk_phase(struct device *dev, struct mmc_clk_phase_map *map); int mmc_of_parse(struct mmc_host *host); @@ -753,6 +753,8 @@ static inline int mmc_card_uhs2_hd_mode(struct mmc_host *host) int mmc_sd_switch(struct mmc_card *card, bool mode, int group, u8 value, u8 *resp); int mmc_send_status(struct mmc_card *card, u32 *status); +int mmc_send_tuning_timeout(struct mmc_host *host, u32 opcode, int *cmd_error, + u32 timeout_ms); int mmc_send_tuning(struct mmc_host *host, u32 opcode, int *cmd_error); int mmc_send_abort_tuning(struct mmc_host *host, u32 opcode); int mmc_get_ext_csd(struct mmc_card *card, u8 **new_ext_csd); diff --git a/include/linux/mmc/sdio_ids.h b/include/linux/mmc/sdio_ids.h index afae93f6f245..dfffda3d8002 100644 --- a/include/linux/mmc/sdio_ids.h +++ b/include/linux/mmc/sdio_ids.h @@ -123,6 +123,8 @@ #define SDIO_VENDOR_ID_NXP 0x0471 #define SDIO_DEVICE_ID_NXP_IW61X_BASE 0x0204 #define SDIO_DEVICE_ID_NXP_IW61X 0x0205 +#define SDIO_DEVICE_ID_NXP_IW610_BASE 0x0214 +#define SDIO_DEVICE_ID_NXP_IW610 0x0215 #define SDIO_VENDOR_ID_REALTEK 0x024c #define SDIO_DEVICE_ID_REALTEK_RTW8723BS 0xb723 diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index 94f9c3ff5416..ecd1ebd49756 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -96,25 +96,6 @@ #define MAX_FOLIO_NR_PAGES (1UL << MAX_FOLIO_ORDER) -/* - * HugeTLB Vmemmap Optimization (HVO) requires struct pages of the head page to - * be naturally aligned with regard to the folio size. - * - * HVO which is only active if the size of struct page is a power of 2. - */ -#define MAX_FOLIO_VMEMMAP_ALIGN \ - (IS_ENABLED(CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP) && \ - is_power_of_2(sizeof(struct page)) ? \ - MAX_FOLIO_NR_PAGES * sizeof(struct page) : 0) - -/* - * vmemmap optimization (like HVO) is only possible for page orders that fill - * two or more pages with struct pages. - */ -#define VMEMMAP_TAIL_MIN_ORDER (ilog2(2 * PAGE_SIZE / sizeof(struct page))) -#define __NR_VMEMMAP_TAILS (MAX_FOLIO_ORDER - VMEMMAP_TAIL_MIN_ORDER + 1) -#define NR_VMEMMAP_TAILS (__NR_VMEMMAP_TAILS > 0 ? __NR_VMEMMAP_TAILS : 0) - enum migratetype { MIGRATE_UNMOVABLE, MIGRATE_MOVABLE, @@ -492,11 +473,14 @@ enum lruvec_flags { * folio->flags, masked by LRU_REFS_MASK. */ #define MAX_NR_TIERS 4U +#define LRU_TIER_MIN 0U +#define LRU_TIER_MAX (MAX_NR_TIERS - 1) #ifndef __GENERATING_BOUNDS_H #define LRU_GEN_MASK ((BIT(LRU_GEN_WIDTH) - 1) << LRU_GEN_PGOFF) #define LRU_REFS_MASK ((BIT(LRU_REFS_WIDTH) - 1) << LRU_REFS_PGOFF) +#define LRU_REFS_MAX BIT(LRU_REFS_WIDTH) /* * For folios accessed multiple times through file descriptors, @@ -635,35 +619,32 @@ struct lru_gen_mm_walk { * For each node, memcgs are divided into two generations: the old and the * young. For each generation, memcgs are randomly sharded into multiple bins * to improve scalability. For each bin, the hlist_nulls is virtually divided - * into three segments: the head, the tail and the default. + * into two segments: the tail and the default. * * An onlining memcg is added to the tail of a random bin in the old generation. * The eviction starts at the head of a random bin in the old generation. The * per-node memcg generation counter, whose reminder (mod MEMCG_NR_GENS) indexes * the old generation, is incremented when all its bins become empty. * - * There are four operations: - * 1. MEMCG_LRU_HEAD, which moves a memcg to the head of a random bin in its - * current generation (old or young) and updates its "seg" to "head"; - * 2. MEMCG_LRU_TAIL, which moves a memcg to the tail of a random bin in its + * There are three operations: + * 1. MEMCG_LRU_TAIL, which moves a memcg to the tail of a random bin in its * current generation (old or young) and updates its "seg" to "tail"; - * 3. MEMCG_LRU_OLD, which moves a memcg to the head of a random bin in the old + * 2. MEMCG_LRU_OLD, which moves a memcg to the head of a random bin in the old * generation, updates its "gen" to "old" and resets its "seg" to "default"; - * 4. MEMCG_LRU_YOUNG, which moves a memcg to the tail of a random bin in the + * 3. MEMCG_LRU_YOUNG, which moves a memcg to the tail of a random bin in the * young generation, updates its "gen" to "young" and resets its "seg" to * "default". * * The events that trigger the above operations are: - * 1. Exceeding the soft limit, which triggers MEMCG_LRU_HEAD; - * 2. The first attempt to reclaim a memcg below low, which triggers + * 1. The first attempt to reclaim a memcg below low, which triggers * MEMCG_LRU_TAIL; - * 3. The first attempt to reclaim a memcg offlined or below reclaimable size + * 2. The first attempt to reclaim a memcg offlined or below reclaimable size * threshold, which triggers MEMCG_LRU_TAIL; - * 4. The second attempt to reclaim a memcg offlined or below reclaimable size + * 3. The second attempt to reclaim a memcg offlined or below reclaimable size * threshold, which triggers MEMCG_LRU_YOUNG; - * 5. Attempting to reclaim a memcg below min, which triggers MEMCG_LRU_YOUNG; - * 6. Finishing the aging on the eviction path, which triggers MEMCG_LRU_YOUNG; - * 7. Offlining a memcg, which triggers MEMCG_LRU_OLD. + * 4. Attempting to reclaim a memcg below min, which triggers MEMCG_LRU_YOUNG; + * 5. Finishing the aging on the eviction path, which triggers MEMCG_LRU_YOUNG; + * 6. Offlining a memcg, which triggers MEMCG_LRU_OLD. * * Notes: * 1. Memcg LRU only applies to global reclaim, and the round-robin incrementing @@ -696,7 +677,6 @@ void lru_gen_exit_memcg(struct mem_cgroup *memcg); void lru_gen_online_memcg(struct mem_cgroup *memcg); void lru_gen_offline_memcg(struct mem_cgroup *memcg); void lru_gen_release_memcg(struct mem_cgroup *memcg); -void lru_gen_soft_reclaim(struct mem_cgroup *memcg, int nid); void max_lru_gen_memcg(struct mem_cgroup *memcg, int nid); bool recheck_lru_gen_max_memcg(struct mem_cgroup *memcg, int nid); void lru_gen_reparent_memcg(struct mem_cgroup *memcg, struct mem_cgroup *parent, int nid); @@ -737,10 +717,6 @@ static inline void lru_gen_release_memcg(struct mem_cgroup *memcg) { } -static inline void lru_gen_soft_reclaim(struct mem_cgroup *memcg, int nid) -{ -} - static inline void max_lru_gen_memcg(struct mem_cgroup *memcg, int nid) { } @@ -1043,6 +1019,42 @@ struct zone { * cma pages is present pages that are assigned for CMA use * (MIGRATE_CMA). * + * pages_with_online_memmap tracks pages within the zone that have + * an online memory map: present pages and memory holes whose + * memory map has been initialized and pfn_to_online_page() + * succeeds. When spanned_pages == pages_with_online_memmap, + * pfn_to_page() can be performed without further checks on any + * PFN within the zone span. + * + * Note: this counter may temporarily undercount when pages with an + * online memory map exist outside the current zone span. Such pages + * are only created during boot, when initializing the memory map of + * pages that do not fall into any zone span. The undercount itself + * can only happen after boot, during memory hotplug, when growing + * the zone to cover such pages and later shrinking it back, which + * may result in a "too small" value. This is safe: it merely + * prevents detecting a contiguous zone. + * + * Here is an example (page numbers are just for illustration + * purposes): + * after boot: + * [ zone span ] + * [ zone pages ] + * spanned=10, initialized=15, online=10 + * online == spanned -> contiguous + * + * growing after hotplug (hotplug 5): + * [ zone span ] + * [ zone pages ] [ zone pages ] + * spanned=30, initialized=20, online=15 + * online != spanned -> not contiguous + * + * shrinking after hotunplug (hotunplug 5 again): + * [ zone span ] + * [ zone pages ] + * spanned=15, initialized=15, online=10 + * online != spanned -> not contiguous although contiguous + * * So present_pages may be used by memory hotplug or memory power * management logic to figure out unmanaged pages by checking * (present_pages - managed_pages). And managed_pages should be used @@ -1067,6 +1079,7 @@ struct zone { atomic_long_t managed_pages; unsigned long spanned_pages; unsigned long present_pages; + unsigned long pages_with_online_memmap; #if defined(CONFIG_MEMORY_HOTPLUG) unsigned long present_early_pages; #endif @@ -1101,9 +1114,6 @@ struct zone { #ifdef CONFIG_UNACCEPTED_MEMORY /* Pages to be accepted. All pages on the list are MAX_PAGE_ORDER */ struct list_head unaccepted_pages; - - /* To be called once the last page in the zone is accepted */ - struct work_struct unaccepted_cleanup; #endif /* zone flags, see below */ @@ -1157,8 +1167,8 @@ struct zone { /* Zone statistics */ atomic_long_t vm_stat[NR_VM_ZONE_STAT_ITEMS]; atomic_long_t vm_numa_event[NR_VM_NUMA_EVENT_ITEMS]; -#ifdef CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP - struct page *vmemmap_tails[NR_VMEMMAP_TAILS]; +#ifdef CONFIG_VMEMMAP_OPTIMIZATION + struct page **vmemmap_tails; #endif } ____cacheline_internodealigned_in_smp; @@ -1662,7 +1672,7 @@ extern void init_currently_empty_zone(struct zone *zone, unsigned long start_pfn extern void lruvec_init(struct lruvec *lruvec); -static inline struct pglist_data *lruvec_pgdat(struct lruvec *lruvec) +static inline struct pglist_data *lruvec_pgdat(const struct lruvec *lruvec) { #ifdef CONFIG_MEMCG return lruvec->pgdat; @@ -1694,6 +1704,38 @@ static inline bool zone_is_zone_device(const struct zone *zone) } #endif +/** + * zone_is_contiguous - test whether a zone is contiguous + * @zone: the zone to test. + * + * In a contiguous zone, it is valid to call pfn_to_page() on any PFN in the + * spanned zone without requiring pfn_valid() or pfn_to_online_page() checks. + * + * Note that missing synchronization with memory offlining makes any PFN + * traversal prone to races. + * + * ZONE_DEVICE zones are always marked non-contiguous. + * + * Return: true if contiguous, otherwise false. + */ +static inline bool zone_is_contiguous(const struct zone *zone) +{ + return READ_ONCE(zone->contiguous); +} + +static inline void set_zone_contiguous(struct zone *zone) +{ + if (zone_is_zone_device(zone)) + return; + if (zone->spanned_pages == zone->pages_with_online_memmap) + WRITE_ONCE(zone->contiguous, true); +} + +static inline void clear_zone_contiguous(struct zone *zone) +{ + WRITE_ONCE(zone->contiguous, false); +} + /* * Returns true if a zone has pages managed by the buddy allocator. * All the reclaim decisions have to use this function rather than @@ -2021,19 +2063,23 @@ struct mem_section { unsigned long section_mem_map; struct mem_section_usage *usage; +#ifdef CONFIG_VMEMMAP_OPTIMIZATION + /* + * Normally, sections hold regular (order-0) pages. However, for + * sections with HVO enabled, this tracks the compound page order + * to enable deduplication of redundant vmemmap pages. + */ + unsigned int compound_page_order; +#endif #ifdef CONFIG_PAGE_EXTENSION /* * If SPARSEMEM, pgdat doesn't have page_ext pointer. We use * section. (see page_ext.h about this.) */ struct page_ext *page_ext; - unsigned long pad; #endif - /* - * WARNING: mem_section must be a power-of-2 in size for the - * calculation and use of SECTION_ROOT_MASK to make sense. - */ -}; +/* Sacrifice minor padding space for efficient lookup. */ +} __aligned(2 * sizeof(unsigned long)); #ifdef CONFIG_SPARSEMEM_EXTREME #define SECTIONS_PER_ROOT (PAGE_SIZE / sizeof (struct mem_section)) @@ -2043,7 +2089,6 @@ struct mem_section { #define SECTION_NR_TO_ROOT(sec) ((sec) / SECTIONS_PER_ROOT) #define NR_SECTION_ROOTS DIV_ROUND_UP(NR_MEM_SECTIONS, SECTIONS_PER_ROOT) -#define SECTION_ROOT_MASK (SECTIONS_PER_ROOT - 1) #ifdef CONFIG_SPARSEMEM_EXTREME extern struct mem_section **mem_section; @@ -2067,7 +2112,7 @@ static inline struct mem_section *__nr_to_section(unsigned long nr) if (!mem_section || !mem_section[root]) return NULL; #endif - return &mem_section[root][nr & SECTION_ROOT_MASK]; + return &mem_section[root][nr % SECTIONS_PER_ROOT]; } /* @@ -2083,29 +2128,21 @@ static inline struct mem_section *__nr_to_section(unsigned long nr) * accommodate SECTION_MAP_LAST_BIT. We use BUILD_BUG_ON() to ensure this. */ enum { - SECTION_MARKED_PRESENT_BIT, SECTION_HAS_MEM_MAP_BIT, SECTION_IS_ONLINE_BIT, SECTION_IS_EARLY_BIT, #ifdef CONFIG_ZONE_DEVICE SECTION_TAINT_ZONE_DEVICE_BIT, #endif -#ifdef CONFIG_SPARSEMEM_VMEMMAP_PREINIT - SECTION_IS_VMEMMAP_PREINIT_BIT, -#endif SECTION_MAP_LAST_BIT, }; -#define SECTION_MARKED_PRESENT BIT(SECTION_MARKED_PRESENT_BIT) #define SECTION_HAS_MEM_MAP BIT(SECTION_HAS_MEM_MAP_BIT) #define SECTION_IS_ONLINE BIT(SECTION_IS_ONLINE_BIT) #define SECTION_IS_EARLY BIT(SECTION_IS_EARLY_BIT) #ifdef CONFIG_ZONE_DEVICE #define SECTION_TAINT_ZONE_DEVICE BIT(SECTION_TAINT_ZONE_DEVICE_BIT) #endif -#ifdef CONFIG_SPARSEMEM_VMEMMAP_PREINIT -#define SECTION_IS_VMEMMAP_PREINIT BIT(SECTION_IS_VMEMMAP_PREINIT_BIT) -#endif #define SECTION_MAP_MASK (~(BIT(SECTION_MAP_LAST_BIT) - 1)) #define SECTION_NID_SHIFT SECTION_MAP_LAST_BIT @@ -2116,16 +2153,6 @@ static inline struct page *__section_mem_map_addr(struct mem_section *section) return (struct page *)map; } -static inline int present_section(const struct mem_section *section) -{ - return (section && (section->section_mem_map & SECTION_MARKED_PRESENT)); -} - -static inline int present_section_nr(unsigned long nr) -{ - return present_section(__nr_to_section(nr)); -} - static inline int valid_section(const struct mem_section *section) { return (section && (section->section_mem_map & SECTION_HAS_MEM_MAP)); @@ -2141,6 +2168,11 @@ static inline int valid_section_nr(unsigned long nr) return valid_section(__nr_to_section(nr)); } +static inline int early_section_nr(unsigned long nr) +{ + return early_section(__nr_to_section(nr)); +} + static inline int online_section(const struct mem_section *section) { return (section && (section->section_mem_map & SECTION_IS_ONLINE)); @@ -2153,28 +2185,20 @@ static inline int online_device_section(const struct mem_section *section) return section && ((section->section_mem_map & flags) == flags); } -#else -static inline int online_device_section(const struct mem_section *section) -{ - return 0; -} -#endif -#ifdef CONFIG_SPARSEMEM_VMEMMAP_PREINIT -static inline int preinited_vmemmap_section(const struct mem_section *section) +static inline struct zone *device_zone(int nid) { - return (section && - (section->section_mem_map & SECTION_IS_VMEMMAP_PREINIT)); + return &NODE_DATA(nid)->node_zones[ZONE_DEVICE]; } - -void sparse_vmemmap_init_nid_early(int nid); #else -static inline int preinited_vmemmap_section(const struct mem_section *section) +static inline int online_device_section(const struct mem_section *section) { return 0; } -static inline void sparse_vmemmap_init_nid_early(int nid) + +static inline struct zone *device_zone(int nid) { + return NULL; } #endif @@ -2193,7 +2217,7 @@ static inline struct mem_section *__pfn_to_section(unsigned long pfn) return __nr_to_section(pfn_to_section_nr(pfn)); } -extern unsigned long __highest_present_section_nr; +extern unsigned long __highest_used_section_nr; static inline int subsection_map_index(unsigned long pfn) { @@ -2241,9 +2265,6 @@ static inline bool pfn_section_first_valid(struct mem_section *ms, unsigned long } #endif -void sparse_init_early_section(int nid, struct page *map, unsigned long pnum, - unsigned long flags); - #ifndef CONFIG_HAVE_ARCH_PFN_VALID /** * pfn_valid - check if there is a valid memory map entry for a PFN @@ -2295,7 +2316,7 @@ static inline unsigned long first_valid_pfn(unsigned long pfn, unsigned long end rcu_read_lock_sched(); - while (nr <= __highest_present_section_nr && pfn < end_pfn) { + while (nr <= __highest_used_section_nr && pfn < end_pfn) { struct mem_section *ms = __pfn_to_section(pfn); if (valid_section(ms) && @@ -2341,27 +2362,20 @@ static inline unsigned long next_valid_pfn(unsigned long pfn, unsigned long end_ #endif -static inline int pfn_in_present_section(unsigned long pfn) -{ - if (pfn_to_section_nr(pfn) >= NR_MEM_SECTIONS) - return 0; - return present_section(__pfn_to_section(pfn)); -} - -static inline unsigned long next_present_section_nr(unsigned long section_nr) +static inline unsigned long next_early_section_nr(unsigned long section_nr) { - while (++section_nr <= __highest_present_section_nr) { - if (present_section_nr(section_nr)) + while (++section_nr <= __highest_used_section_nr) { + if (early_section_nr(section_nr)) return section_nr; } return -1; } -#define for_each_present_section_nr(start, section_nr) \ - for (section_nr = next_present_section_nr(start - 1); \ +#define for_each_early_section_nr(start, section_nr) \ + for (section_nr = next_early_section_nr(start - 1); \ section_nr != -1; \ - section_nr = next_present_section_nr(section_nr)) + section_nr = next_early_section_nr(section_nr)) /* * These are _only_ used during initialisation, therefore they @@ -2377,10 +2391,6 @@ static inline unsigned long next_present_section_nr(unsigned long section_nr) #else #define pfn_to_nid(pfn) (0) #endif - -#else -#define sparse_vmemmap_init_nid_early(_nid) do {} while (0) -#define pfn_in_present_section pfn_valid #endif /* CONFIG_SPARSEMEM */ /* diff --git a/include/linux/mnt_idmapping.h b/include/linux/mnt_idmapping.h index e71a6070a8f8..78eeef4c2996 100644 --- a/include/linux/mnt_idmapping.h +++ b/include/linux/mnt_idmapping.h @@ -8,8 +8,8 @@ struct mnt_idmap; struct user_namespace; -extern struct mnt_idmap nop_mnt_idmap; -extern struct mnt_idmap invalid_mnt_idmap; +extern const struct mnt_idmap nop_mnt_idmap; +extern const struct mnt_idmap invalid_mnt_idmap; extern struct user_namespace init_user_ns; typedef struct { @@ -121,19 +121,19 @@ static inline bool vfsgid_eq_kgid(vfsgid_t vfsgid, kgid_t kgid) int vfsgid_in_group_p(vfsgid_t vfsgid); -struct mnt_idmap *mnt_idmap_get(struct mnt_idmap *idmap); -void mnt_idmap_put(struct mnt_idmap *idmap); +const struct mnt_idmap *mnt_idmap_get(const struct mnt_idmap *idmap); +void mnt_idmap_put(const struct mnt_idmap *idmap); -vfsuid_t make_vfsuid(struct mnt_idmap *idmap, +vfsuid_t make_vfsuid(const struct mnt_idmap *idmap, struct user_namespace *fs_userns, kuid_t kuid); -vfsgid_t make_vfsgid(struct mnt_idmap *idmap, +vfsgid_t make_vfsgid(const struct mnt_idmap *idmap, struct user_namespace *fs_userns, kgid_t kgid); -kuid_t from_vfsuid(struct mnt_idmap *idmap, +kuid_t from_vfsuid(const struct mnt_idmap *idmap, struct user_namespace *fs_userns, vfsuid_t vfsuid); -kgid_t from_vfsgid(struct mnt_idmap *idmap, +kgid_t from_vfsgid(const struct mnt_idmap *idmap, struct user_namespace *fs_userns, vfsgid_t vfsgid); /** @@ -148,7 +148,7 @@ kgid_t from_vfsgid(struct mnt_idmap *idmap, * * Return: true if @vfsuid has a mapping in the filesystem, false if not. */ -static inline bool vfsuid_has_fsmapping(struct mnt_idmap *idmap, +static inline bool vfsuid_has_fsmapping(const struct mnt_idmap *idmap, struct user_namespace *fs_userns, vfsuid_t vfsuid) { @@ -186,7 +186,7 @@ static inline kuid_t vfsuid_into_kuid(vfsuid_t vfsuid) * * Return: true if @vfsgid has a mapping in the filesystem, false if not. */ -static inline bool vfsgid_has_fsmapping(struct mnt_idmap *idmap, +static inline bool vfsgid_has_fsmapping(const struct mnt_idmap *idmap, struct user_namespace *fs_userns, vfsgid_t vfsgid) { @@ -225,7 +225,7 @@ static inline kgid_t vfsgid_into_kgid(vfsgid_t vfsgid) * * Return: the caller's current fsuid mapped up according to @idmap. */ -static inline kuid_t mapped_fsuid(struct mnt_idmap *idmap, +static inline kuid_t mapped_fsuid(const struct mnt_idmap *idmap, struct user_namespace *fs_userns) { return from_vfsuid(idmap, fs_userns, VFSUIDT_INIT(current_fsuid())); @@ -244,7 +244,7 @@ static inline kuid_t mapped_fsuid(struct mnt_idmap *idmap, * * Return: the caller's current fsgid mapped up according to @idmap. */ -static inline kgid_t mapped_fsgid(struct mnt_idmap *idmap, +static inline kgid_t mapped_fsgid(const struct mnt_idmap *idmap, struct user_namespace *fs_userns) { return from_vfsgid(idmap, fs_userns, VFSGIDT_INIT(current_fsgid())); diff --git a/include/linux/mod_devicetable.h b/include/linux/mod_devicetable.h index a397213bedac..d1581bb64729 100644 --- a/include/linux/mod_devicetable.h +++ b/include/linux/mod_devicetable.h @@ -15,7 +15,7 @@ #include "device-id/acpi.h" #include "device-id/amba.h" #include "device-id/ap.h" -#include "device-id/apr.h" +#include "device-id/arm_smccc.h" #include "device-id/auxiliary.h" #include "device-id/bcma.h" #include "device-id/ccw.h" diff --git a/include/linux/module.h b/include/linux/module.h index 96cc98568eea..76f4f3596477 100644 --- a/include/linux/module.h +++ b/include/linux/module.h @@ -24,11 +24,11 @@ #include <linux/jump_label.h> #include <linux/export.h> #include <linux/rbtree_latch.h> -#include <linux/error-injection.h> #include <linux/tracepoint-defs.h> #include <linux/srcu.h> #include <linux/static_call_types.h> #include <linux/dynamic_debug.h> +#include <linux/panic.h> #include <linux/percpu.h> #include <asm/module.h> @@ -503,6 +503,10 @@ struct module { void *btf_data; void *btf_base_data; #endif +#if IS_ENABLED(CONFIG_DEBUG_INFO_BTF_INLINE) + unsigned int btf_inline_data_size; + void *btf_inline_data; +#endif #ifdef CONFIG_JUMP_LABEL struct jump_entry *jump_entries; unsigned int num_jump_entries; @@ -770,6 +774,9 @@ static inline bool is_livepatch_module(struct module *mod) void module_for_each_mod(int(*func)(struct module *mod, void *data), void *data); +void add_taint_module(struct module *mod, unsigned flag, + enum lockdep_ok lockdep_ok); + #else /* !CONFIG_MODULES... */ static inline struct module *__module_address(unsigned long addr) @@ -877,6 +884,12 @@ static inline bool module_is_coming(struct module *mod) static inline void module_for_each_mod(int(*func)(struct module *mod, void *data), void *data) { } + +static inline void add_taint_module(struct module *mod, unsigned flag, + enum lockdep_ok lockdep_ok) +{ + add_taint(flag, lockdep_ok); +} #endif /* CONFIG_MODULES */ #ifdef CONFIG_SYSFS diff --git a/include/linux/module_symbol.h b/include/linux/module_symbol.h index 574609aced99..698d3db2b37e 100644 --- a/include/linux/module_symbol.h +++ b/include/linux/module_symbol.h @@ -7,8 +7,11 @@ enum ksym_flags { KSYM_FLAG_GPL_ONLY = 1 << 0, }; -/* This ignores the intensely annoying "mapping symbols" found in ELF files. */ -static inline bool is_mapping_symbol(const char *str) +/* + * Ignore local labels (.L*, L0*) and mapping symbols ($*). These symbols are + * not useful for the kernel, for example, they should not appear in kallsyms. + */ +static inline bool is_ignored_kernel_symbol(const char *str) { if (str[0] == '.' && str[1] == 'L') return true; diff --git a/include/linux/mount.h b/include/linux/mount.h index b16ba5d45605..57860e321cf5 100644 --- a/include/linux/mount.h +++ b/include/linux/mount.h @@ -59,10 +59,10 @@ struct vfsmount { struct dentry *mnt_root; /* root of the mounted tree */ struct super_block *mnt_sb; /* pointer to superblock */ int mnt_flags; - struct mnt_idmap *mnt_idmap; + const struct mnt_idmap *mnt_idmap; } __randomize_layout; -static inline struct mnt_idmap *mnt_idmap(const struct vfsmount *mnt) +static inline const struct mnt_idmap *mnt_idmap(const struct vfsmount *mnt) { /* Pairs with smp_store_release() in do_idmap_mount(). */ return READ_ONCE(mnt->mnt_idmap); diff --git a/include/linux/mtd/bbm.h b/include/linux/mtd/bbm.h index d890805f5494..67929fdf2dff 100644 --- a/include/linux/mtd/bbm.h +++ b/include/linux/mtd/bbm.h @@ -75,7 +75,7 @@ struct nand_bbt_descr { * with NAND_BBT_CREATE. */ #define NAND_BBT_CREATE_EMPTY 0x00000400 -/* Write bbt if neccecary */ +/* Write bbt if necessary */ #define NAND_BBT_WRITE 0x00002000 /* Read and write back block contents when writing bbt */ #define NAND_BBT_SAVECONTENT 0x00004000 diff --git a/include/linux/mtd/lpc32xx_mlc.h b/include/linux/mtd/lpc32xx_mlc.h deleted file mode 100644 index 35e971be0950..000000000000 --- a/include/linux/mtd/lpc32xx_mlc.h +++ /dev/null @@ -1,17 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Platform data for LPC32xx SoC MLC NAND controller - * - * Copyright © 2012 Roland Stigge - */ - -#ifndef __LINUX_MTD_LPC32XX_MLC_H -#define __LINUX_MTD_LPC32XX_MLC_H - -#include <linux/dmaengine.h> - -struct lpc32xx_mlc_platform_data { - dma_filter_fn dma_filter; -}; - -#endif /* __LINUX_MTD_LPC32XX_MLC_H */ diff --git a/include/linux/mtd/lpc32xx_slc.h b/include/linux/mtd/lpc32xx_slc.h deleted file mode 100644 index a044b806566b..000000000000 --- a/include/linux/mtd/lpc32xx_slc.h +++ /dev/null @@ -1,17 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Platform data for LPC32xx SoC SLC NAND controller - * - * Copyright © 2012 Roland Stigge - */ - -#ifndef __LINUX_MTD_LPC32XX_SLC_H -#define __LINUX_MTD_LPC32XX_SLC_H - -#include <linux/dmaengine.h> - -struct lpc32xx_slc_platform_data { - dma_filter_fn dma_filter; -}; - -#endif /* __LINUX_MTD_LPC32XX_SLC_H */ diff --git a/include/linux/mtd/nand-qpic-common.h b/include/linux/mtd/nand-qpic-common.h index 437448995187..f31fd18d837f 100644 --- a/include/linux/mtd/nand-qpic-common.h +++ b/include/linux/mtd/nand-qpic-common.h @@ -262,7 +262,6 @@ struct bam_transaction { u32 tx_sgl_start; u32 rx_sgl_pos; u32 rx_sgl_start; - ); }; @@ -458,6 +457,12 @@ struct qcom_nandc_props { bool use_codeword_fixup; }; +struct qcom_nandc_read_stats { + __le32 flash; + __le32 buffer; + __le32 erased_cw; +}; + void qcom_free_bam_transaction(struct qcom_nand_controller *nandc); struct bam_transaction *qcom_alloc_bam_transaction(struct qcom_nand_controller *nandc); void qcom_clear_bam_transaction(struct qcom_nand_controller *nandc); @@ -483,5 +488,16 @@ int qcom_submit_descs(struct qcom_nand_controller *nandc); void qcom_clear_read_regs(struct qcom_nand_controller *nandc); void qcom_nandc_unalloc(struct qcom_nand_controller *nandc); int qcom_nandc_alloc(struct qcom_nand_controller *nandc); +int qcom_nandc_check_flash_errors(struct qcom_nand_controller *nandc, + int cw_cnt); +void qcom_nandc_config_page_read(struct qcom_nand_controller *nandc); +void qcom_nandc_config_page_write(struct qcom_nand_controller *nandc); +void qcom_nandc_config_cw_write(struct qcom_nand_controller *nandc); +void qcom_nandc_set_read_loc_first(struct qcom_nand_controller *nandc, + int reg_base, u32 cw_offset, + u32 read_size, u32 is_last_read_loc); +void qcom_nandc_set_read_loc_last(struct qcom_nand_controller *nandc, + int reg_base, u32 cw_offset, + u32 read_size, u32 is_last_read_loc); #endif diff --git a/include/linux/mtd/rawnand.h b/include/linux/mtd/rawnand.h index 5c70e7bd3ed5..43fefff02025 100644 --- a/include/linux/mtd/rawnand.h +++ b/include/linux/mtd/rawnand.h @@ -1592,7 +1592,7 @@ void nand_wait_ready(struct nand_chip *chip); /* * Free resources held by the NAND device, must be called on error after a - * sucessful nand_scan(). + * successful nand_scan(). */ void nand_cleanup(struct nand_chip *chip); diff --git a/include/linux/mtd/sh_flctl.h b/include/linux/mtd/sh_flctl.h index 78fc2d4218c8..15adf2474fa0 100644 --- a/include/linux/mtd/sh_flctl.h +++ b/include/linux/mtd/sh_flctl.h @@ -60,8 +60,8 @@ * Some hardware uses bits called PULSEx instead of FCKSEL_E and QTSEL_E * to control the clock divider used between the High-Speed Peripheral Clock * and the FLCTL internal clock. If so, use CLK_8_BIT_xxx for connecting 8 bit - * and CLK_16_BIT_xxx for connecting 16 bit bus bandwith NAND chips. For the 16 - * bit version the divider is seperate for the pulse width of high and low + * and CLK_16_BIT_xxx for connecting 16 bit bus bandwidth NAND chips. For the 16 + * bit version the divider is separate for the pulse width of high and low * signals. */ #define PULSE3 (0x1 << 27) diff --git a/include/linux/mtd/spi-nor.h b/include/linux/mtd/spi-nor.h index 4b92494827b1..b3e3c6b10186 100644 --- a/include/linux/mtd/spi-nor.h +++ b/include/linux/mtd/spi-nor.h @@ -25,6 +25,7 @@ #define SPINOR_OP_WRSR 0x01 /* Write status register 1 */ #define SPINOR_OP_RDSR2 0x3f /* Read status register 2 */ #define SPINOR_OP_WRSR2 0x3e /* Write status register 2 */ +#define SPINOR_OP_WRSR2_ALT 0x31 /* Write status register 2 (alternative) */ #define SPINOR_OP_READ 0x03 /* Read data bytes (low frequency) */ #define SPINOR_OP_READ_FAST 0x0b /* Read data bytes (high frequency) */ #define SPINOR_OP_READ_1_1_2 0x3b /* Read data bytes (Dual Output SPI) */ @@ -113,20 +114,16 @@ #define SR_E_ERR BIT(5) #define SR_P_ERR BIT(6) -#define SR1_QUAD_EN_BIT6 BIT(6) - #define SR_BP_SHIFT 2 /* Enhanced Volatile Configuration Register bits */ #define EVCR_QUAD_EN_MICRON BIT(7) /* Micron Quad I/O */ /* Status Register 2 bits. */ -#define SR2_QUAD_EN_BIT1 BIT(1) #define SR2_LB1 BIT(3) /* Security Register Lock Bit 1 */ #define SR2_LB2 BIT(4) /* Security Register Lock Bit 2 */ #define SR2_LB3 BIT(5) /* Security Register Lock Bit 3 */ #define SR2_CMP_BIT6 BIT(6) -#define SR2_QUAD_EN_BIT7 BIT(7) /* Supported SPI protocols */ #define SNOR_PROTO_INST_MASK GENMASK(23, 16) @@ -365,8 +362,6 @@ struct spi_nor_flash_parameter; * @read_dummy: the dummy needed by the read operation * @program_opcode: the program opcode * @sst_write_second: used by the SST write operation - * @flags: flag options for the current SPI NOR (SNOR_F_*) - * @cmd_ext_type: the command opcode extension type for DTR mode. * @read_proto: the SPI protocol for read operations * @write_proto: the SPI protocol for write operations * @reg_proto: the SPI protocol for read_reg/write_reg/erase operations @@ -398,6 +393,7 @@ struct spi_nor { u8 *id; const struct flash_info *info; const struct spi_nor_manufacturer *manufacturer; + const char *partname; u8 addr_nbytes; u8 erase_opcode; u8 read_opcode; @@ -407,8 +403,6 @@ struct spi_nor { enum spi_nor_protocol write_proto; enum spi_nor_protocol reg_proto; bool sst_write_second; - u32 flags; - enum spi_nor_cmd_ext cmd_ext_type; struct sfdp *sfdp; struct dentry *debugfs_root; u8 dfs_sr_cache[2]; diff --git a/include/linux/mtd/spinand.h b/include/linux/mtd/spinand.h index 5f4c00ae72a7..d7c99fcfb85e 100644 --- a/include/linux/mtd/spinand.h +++ b/include/linux/mtd/spinand.h @@ -399,7 +399,7 @@ struct spinand_devid { }; /** - * struct manufacurer_ops - SPI NAND manufacturer specific operations + * struct spinand_manufacturer_ops - SPI NAND manufacturer specific operations * @init: initialize a SPI NAND device * @cleanup: cleanup a SPI NAND device * @@ -438,6 +438,7 @@ extern const struct spinand_manufacturer fmsh_spinand_manufacturer; extern const struct spinand_manufacturer foresee_spinand_manufacturer; extern const struct spinand_manufacturer gigadevice_spinand_manufacturer; extern const struct spinand_manufacturer heyangtek_spinand_manufacturer; +extern const struct spinand_manufacturer issi_spinand_manufacturer; extern const struct spinand_manufacturer macronix_spinand_manufacturer; extern const struct spinand_manufacturer micron_spinand_manufacturer; extern const struct spinand_manufacturer paragon_spinand_manufacturer; diff --git a/include/linux/namei.h b/include/linux/namei.h index 86d657b24fc6..c4436e5c2ba6 100644 --- a/include/linux/namei.h +++ b/include/linux/namei.h @@ -32,8 +32,9 @@ enum { MAX_NESTED_LINKS = 8 }; #define LOOKUP_CREATE BIT(17) /* ... in object creation */ #define LOOKUP_EXCL BIT(18) /* ... in target must not exist */ #define LOOKUP_RENAME_TARGET BIT(19) /* ... in destination of rename() */ +#define LOOKUP_SHARED BIT(20) /* Parent lock is held shared */ -/* 4 spare bits for intent */ +/* 3 spare bits for intent */ /* Scoping flags for lookup. */ #define LOOKUP_NO_SYMLINKS BIT(24) /* No symlink crossing. */ @@ -70,24 +71,24 @@ extern struct dentry *try_lookup_noperm(struct qstr *, struct dentry *); extern struct dentry *lookup_noperm(struct qstr *, struct dentry *); extern struct dentry *lookup_noperm_unlocked(struct qstr *, struct dentry *); extern struct dentry *lookup_noperm_positive_unlocked(struct qstr *, struct dentry *); -struct dentry *lookup_one(struct mnt_idmap *, struct qstr *, struct dentry *); -struct dentry *lookup_one_unlocked(struct mnt_idmap *idmap, +struct dentry *lookup_one(const struct mnt_idmap *, struct qstr *, struct dentry *); +struct dentry *lookup_one_unlocked(const struct mnt_idmap *idmap, struct qstr *name, struct dentry *base); -struct dentry *lookup_one_positive_unlocked(struct mnt_idmap *idmap, +struct dentry *lookup_one_positive_unlocked(const struct mnt_idmap *idmap, struct qstr *name, struct dentry *base); -struct dentry *lookup_one_positive_killable(struct mnt_idmap *idmap, +struct dentry *lookup_one_positive_killable(const struct mnt_idmap *idmap, struct qstr *name, struct dentry *base); -struct dentry *start_creating(struct mnt_idmap *idmap, struct dentry *parent, +struct dentry *start_creating(const struct mnt_idmap *idmap, struct dentry *parent, struct qstr *name); -struct dentry *start_removing(struct mnt_idmap *idmap, struct dentry *parent, +struct dentry *start_removing(const struct mnt_idmap *idmap, struct dentry *parent, struct qstr *name); -struct dentry *start_creating_killable(struct mnt_idmap *idmap, +struct dentry *start_creating_killable(const struct mnt_idmap *idmap, struct dentry *parent, struct qstr *name); -struct dentry *start_removing_killable(struct mnt_idmap *idmap, +struct dentry *start_removing_killable(const struct mnt_idmap *idmap, struct dentry *parent, struct qstr *name); struct dentry *start_creating_noperm(struct dentry *parent, struct qstr *name); diff --git a/include/linux/net.h b/include/linux/net.h index 3d82966e2243..e2a866fbcfa5 100644 --- a/include/linux/net.h +++ b/include/linux/net.h @@ -47,6 +47,9 @@ typedef struct sockopt { int optlen; } sockopt_t; +int sockptr_to_sockopt(sockopt_t *opt, sockptr_t optval, sockptr_t optlen, + struct kvec *kvec); + /* * Initialize a user-backed sockopt_t from the (optval, optlen) __user pair of * a getsockopt() callback. Used by transitional __user getsockopt wrappers @@ -70,6 +73,35 @@ static inline int sockopt_init_user(sockopt_t *opt, char __user *optval, return 0; } +/* + * Grow optval to @size, for the options whose reply is sized by a count the + * caller left in optval rather than by optlen. Those write past optlen today + * and userspace relies on it. + * + * Call it before writing through opt->iter_out: it re-anchors the iterator at + * the head of optval. Only a user buffer can be longer than the optlen the + * caller declared, so a kernel-backed optval is refused with -EINVAL. + */ +static inline int sockopt_expand_out(sockopt_t *opt, size_t size) +{ + if (size <= (size_t)opt->optlen) + return 0; + + if (size > INT_MAX) + return -EINVAL; + + /* Re-anchoring reads iter_out.ubuf, so the iterator has to be a user + * buffer that nothing has written through yet. + */ + if (WARN_ON_ONCE(!iter_is_ubuf(&opt->iter_out) || + iov_iter_count(&opt->iter_out) != (size_t)opt->optlen)) + return -EINVAL; + + iov_iter_ubuf(&opt->iter_out, ITER_DEST, opt->iter_out.ubuf, size); + + return 0; +} + struct poll_table_struct; struct pipe_inode_info; struct inode; @@ -166,7 +198,7 @@ struct socket { struct file *file; struct sock *sk; - const struct proto_ops *ops; /* Might change with IPV6_ADDRFORM or MPTCP. */ + const struct proto_ops *ops; /* Might change with MPTCP. */ struct socket_wq wq; }; diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index 3cff2174dc03..c0ae442c4344 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -252,13 +252,19 @@ struct netdev_hw_addr { struct netdev_hw_addr_list { struct list_head list; - int count; + /* Only changed through the __hw_addr_count_* helpers after + * __hw_addr_init() + */ + int _count; /* Auxiliary tree for faster lookup on addition and deletion */ struct rb_root tree; + + /* Set for dev->mc only, the device the list belongs to */ + struct net_device *mc_dev; }; -#define netdev_hw_addr_list_count(l) ((l)->count) +#define netdev_hw_addr_list_count(l) ((l)->_count) #define netdev_hw_addr_list_empty(l) (netdev_hw_addr_list_count(l) == 0) #define netdev_hw_addr_list_for_each(ha, l) \ list_for_each_entry(ha, &(l)->list, list) @@ -433,6 +439,7 @@ enum { NAPI_STATE_SCHED_THREADED, /* Napi is currently scheduled in threaded mode */ NAPI_STATE_HAS_NOTIFIER, /* Napi has an IRQ notifier */ NAPI_STATE_THREADED_BUSY_POLL, /* The threaded NAPI poller will busy poll */ + NAPI_STATE_PERCPU, /* NAPI handler is run per-CPU - incompatible with threaded mode */ }; enum { @@ -448,6 +455,7 @@ enum { NAPIF_STATE_SCHED_THREADED = BIT(NAPI_STATE_SCHED_THREADED), NAPIF_STATE_HAS_NOTIFIER = BIT(NAPI_STATE_HAS_NOTIFIER), NAPIF_STATE_THREADED_BUSY_POLL = BIT(NAPI_STATE_THREADED_BUSY_POLL), + NAPIF_STATE_PERCPU = BIT(NAPI_STATE_PERCPU), }; enum gro_result { @@ -911,6 +919,7 @@ struct net_device_path { }; u8 inner_proto; + u8 encap_proto; } tun; struct { enum { @@ -1138,13 +1147,16 @@ struct netdev_net_notifier { * struct netdev_hw_addr_list *uc, * struct netdev_hw_addr_list *mc); * Async version of ndo_set_rx_mode which runs in process context - * with rtnl_lock and netdev_lock_ops(dev) held. The uc/mc parameters + * under the netdev instance lock for "ops locked" drivers, or + * rtnl_lock for all other drivers. The uc/mc parameters * are snapshots of the address lists - iterate with * netdev_hw_addr_list_for_each(ha, uc). Return 0 on success or a * negative errno to request a retry via the core backoff. * * void (*ndo_work)(struct net_device *dev, unsigned long events); * Run deferred work scheduled with netdev_work_sched(@events). + * Runs in process context under the netdev instance lock for "ops + * locked" drivers, or rtnl_lock for all other drivers. * * int (*ndo_set_mac_address)(struct net_device *dev, void *addr); * This function is called when the Media Access Control address @@ -1154,11 +1166,6 @@ struct netdev_net_notifier { * int (*ndo_validate_addr)(struct net_device *dev); * Test if Media Access Control address is valid for the device. * - * int (*ndo_do_ioctl)(struct net_device *dev, struct ifreq *ifr, int cmd); - * Old-style ioctl entry point. This is used internally by the - * ieee802154 subsystem but is no longer called by the device - * ioctl handler. - * * int (*ndo_siocbond)(struct net_device *dev, struct ifreq *ifr, int cmd); * Used by the bonding driver for its device specific ioctls: * SIOCBONDENSLAVE, SIOCBONDRELEASE, SIOCBONDSETHWADDR, SIOCBONDCHANGEACTIVE, @@ -1480,8 +1487,6 @@ struct net_device_ops { int (*ndo_set_mac_address)(struct net_device *dev, void *addr); int (*ndo_validate_addr)(struct net_device *dev); - int (*ndo_do_ioctl)(struct net_device *dev, - struct ifreq *ifr, int cmd); int (*ndo_eth_ioctl)(struct net_device *dev, struct ifreq *ifr, int cmd); int (*ndo_siocbond)(struct net_device *dev, @@ -1840,6 +1845,7 @@ enum netdev_reg_state { * drivers. Mainly used by logical interfaces, such as * bonding and tunnels * @netmem_tx: device netmem TX mode + * @pacing_offload: enable EDT pacing offload. * * @name: This is the first field of the "visible" part of this structure * (i.e. as seen by users in the "Space.c" file). It is the name @@ -2170,6 +2176,7 @@ struct net_device { unsigned long priv_flags:32; unsigned long lltx:1; unsigned long netmem_tx:2; + unsigned long pacing_offload:1; ); const struct net_device_ops *netdev_ops; const struct header_ops *header_ops; @@ -2545,6 +2552,11 @@ struct net_device { /* protected by rtnl_lock */ struct bpf_xdp_entity xdp_state[__MAX_XDP_MODE]; +#ifdef CONFIG_XDP_SOCKETS + /** @xsk_pools: assigned AF_XDP pools, ops protected */ + struct hlist_head xsk_pools; +#endif + u8 dev_addr_shadow[MAX_ADDR_LEN]; netdevice_tracker linkwatch_dev_tracker; netdevice_tracker watchdog_dev_tracker; @@ -2841,6 +2853,13 @@ static inline void *netdev_priv(const struct net_device *dev) */ static inline struct net_device *netdev_from_priv(const void *priv) { + /* + * When GCC can see that @priv came from netdev_priv(), it can + * compute an object size of 0 for the net_device and its members, + * which makes FORTIFY_SOURCE fail valid accesses through the result: + * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=127875 + */ + OPTIMIZER_HIDE_VAR(priv); return container_of(priv, struct net_device, priv); } @@ -3672,7 +3691,11 @@ struct page_pool_bh { }; DECLARE_PER_CPU(struct page_pool_bh, system_page_pool); +#ifdef CONFIG_KASAN +#define XMIT_RECURSION_LIMIT 4 +#else #define XMIT_RECURSION_LIMIT 8 +#endif #ifndef CONFIG_PREEMPT_RT static inline int dev_recursion_level(void) @@ -5163,6 +5186,7 @@ int dev_mc_sync_multiple(struct net_device *to, struct net_device *from); void dev_mc_unsync(struct net_device *to, struct net_device *from); void dev_mc_flush(struct net_device *dev); void dev_mc_init(struct net_device *dev); +int dev_mc_dump(struct sk_buff *skb, struct netlink_callback *cb); /** * __dev_mc_sync - Synchronize device's multicast list @@ -5324,7 +5348,6 @@ void netdev_lower_state_changed(struct net_device *lower_dev, void *lower_state_info); #define NETDEV_RSS_KEY_LEN 256 -extern u8 netdev_rss_key[NETDEV_RSS_KEY_LEN] __read_mostly; void netdev_rss_key_fill(void *buffer, size_t len); int skb_checksum_help(struct sk_buff *skb); @@ -5497,7 +5520,14 @@ void netif_stacked_transfer_operstate(const struct net_device *rootdev, netdev_features_t passthru_features_check(struct sk_buff *skb, struct net_device *dev, netdev_features_t features); -netdev_features_t netif_skb_features(struct sk_buff *skb); +netdev_features_t __netif_skb_features(struct sk_buff *skb, + bool check_gso_limits); + +static inline netdev_features_t netif_skb_features(struct sk_buff *skb) +{ + return __netif_skb_features(skb, true); +} + void skb_warn_bad_offload(const struct sk_buff *skb); static inline bool net_gso_ok(netdev_features_t features, int gso_type) @@ -5563,15 +5593,6 @@ netif_get_gro_max_size(const struct net_device *dev, const struct sk_buff *skb) READ_ONCE(dev->gro_ipv4_max_size); } -static inline unsigned int -netif_get_gso_max_size(const struct net_device *dev, const struct sk_buff *skb) -{ - /* pairs with WRITE_ONCE() in netif_set_gso(_ipv4)_max_size() */ - return skb->protocol == htons(ETH_P_IPV6) ? - READ_ONCE(dev->gso_max_size) : - READ_ONCE(dev->gso_ipv4_max_size); -} - static inline bool netif_is_macsec(const struct net_device *dev) { return dev->priv_flags & IFF_MACSEC; @@ -5609,12 +5630,12 @@ static inline bool netif_has_l3_rx_handler(const struct net_device *dev) static inline bool netif_is_l3_master(const struct net_device *dev) { - return dev->priv_flags & IFF_L3MDEV_MASTER; + return IS_ENABLED(CONFIG_NET_VRF) && (dev->priv_flags & IFF_L3MDEV_MASTER); } static inline bool netif_is_l3_slave(const struct net_device *dev) { - return dev->priv_flags & IFF_L3MDEV_SLAVE; + return IS_ENABLED(CONFIG_NET_VRF) && (dev->priv_flags & IFF_L3MDEV_SLAVE); } static inline int dev_sdif(const struct net_device *dev) @@ -5714,6 +5735,12 @@ static inline const char *netdev_name(const struct net_device *dev) return dev->name; } +/* Caller holds RTNL, netdev->lock or RCU */ +static inline bool netif_is_alive(const struct net_device *dev) +{ + return READ_ONCE(dev->reg_state) <= NETREG_REGISTERED; +} + static inline const char *netdev_reg_state(const struct net_device *dev) { u8 reg_state = READ_ONCE(dev->reg_state); diff --git a/include/linux/netfs.h b/include/linux/netfs.h index fe6275e55fc2..e5961450f3f1 100644 --- a/include/linux/netfs.h +++ b/include/linux/netfs.h @@ -22,6 +22,7 @@ enum netfs_sreq_ref_trace; typedef struct mempool mempool_t; +struct fscache_occupancy; struct folio_queue; /** @@ -62,8 +63,8 @@ struct netfs_inode { struct fscache_cookie *cache; #endif struct list_head wb_queue; /* Queue of processes wanting to do writeback */ - loff_t _remote_i_size; /* Size of the remote file */ - loff_t _zero_point; /* Size after which we assume there's no data + uoff_t _remote_i_size; /* Size of the remote file */ + uoff_t _zero_point; /* Size after which we assume there's no data * on the server */ spinlock_t lock; /* Lock covering wb_queue */ atomic_t io_count; /* Number of outstanding reqs */ @@ -125,6 +126,12 @@ static inline struct netfs_group *netfs_folio_group(struct folio *folio) return priv; } +enum netfs_cache_collect { + NETFS_CACHE_COLLECT_WRITE_GAP, /* Gap in collection, no state either way */ + NETFS_CACHE_COLLECT_WRITE_DATA, /* Currently collecting good writes */ + NETFS_CACHE_COLLECT_WRITE_CANCEL, /* Currently collecting cancelled writes */ +}; + /* * Stream of I/O subrequests going to a particular destination, such as the * server or the local cache. This is mainly intended for writing where we may @@ -142,7 +149,7 @@ struct netfs_io_stream { void (*issue_write)(struct netfs_io_subrequest *subreq); /* Collection tracking */ struct list_head subrequests; /* Contributory I/O operations */ - unsigned long long collected_to; /* Position we've collected results to */ + uoff_t collected_to; /* Position we've collected results to */ size_t transferred; /* The amount transferred from this stream */ unsigned short error; /* Aggregate error for the stream */ enum netfs_io_source source; /* Where to read from/write to */ @@ -152,6 +159,7 @@ struct netfs_io_stream { bool need_retry; /* T if this stream needs retrying */ bool failed; /* T if this stream failed */ bool transferred_valid; /* T is ->transferred is valid */ + enum netfs_cache_collect cache_collect; /* Current writeback cache collect state */ }; /* @@ -161,8 +169,11 @@ struct netfs_cache_resources { const struct netfs_cache_ops *ops; void *cache_priv; void *cache_priv2; - unsigned int debug_id; /* Cookie debug ID */ + uoff_t cache_i_size; /* Initial size of cache file */ + unsigned int cookie_id; /* Cache cookie debug ID */ + unsigned int object_id; /* Cache object debug ID */ unsigned int inval_counter; /* object->inval_counter at begin_op */ + unsigned int dio_size; /* DIO block size */ }; /* @@ -177,7 +188,7 @@ struct netfs_io_subrequest { struct work_struct work; struct list_head rreq_link; /* Link in rreq->subrequests */ struct iov_iter io_iter; /* Iterator for this subrequest */ - unsigned long long start; /* Where to start the I/O */ + uoff_t start; /* Where to start the I/O */ size_t len; /* Size of the I/O */ size_t transferred; /* Amount of data transferred */ refcount_t ref; @@ -196,6 +207,7 @@ struct netfs_io_subrequest { #define NETFS_SREQ_IN_PROGRESS 8 /* Unlocked when the subrequest completes */ #define NETFS_SREQ_NEED_RETRY 9 /* Set if the filesystem requests a retry */ #define NETFS_SREQ_FAILED 10 /* Set if the subreq failed unretryably */ +#define NETFS_SREQ_CANCELLED 11 /* Set if the subreq was cancelled by netfslib */ }; enum netfs_io_origin { @@ -208,7 +220,6 @@ enum netfs_io_origin { NETFS_DIO_READ, /* This is a direct I/O read */ NETFS_WRITEBACK, /* This write was triggered by writepages */ NETFS_WRITEBACK_SINGLE, /* This monolithic write was triggered by writepages */ - NETFS_WRITETHROUGH, /* This write was made by netfs_perform_write() */ NETFS_UNBUFFERED_WRITE, /* This is an unbuffered write */ NETFS_DIO_WRITE, /* This is a direct I/O write */ NETFS_PGPRIV2_COPY_TO_CACHE, /* [DEPRECATED] This is writing read data to the cache */ @@ -243,17 +254,18 @@ struct netfs_io_request { void *netfs_priv; /* Private data for the netfs */ void *netfs_priv2; /* Private data for the netfs */ struct bio_vec *direct_bv; /* DIO buffer list (when handling iovec-iter) */ - unsigned long long submitted; /* Amount submitted for I/O so far */ - unsigned long long len; /* Length of the request */ + uoff_t submitted; /* Amount submitted for I/O so far */ + uoff_t len; /* Length of the request */ size_t transferred; /* Amount to be indicated as transferred */ size_t progress_at; /* Report read progress when hit this much read */ long error; /* 0 or error that occurred */ - unsigned long long i_size; /* Size of the file */ - unsigned long long start; /* Start position */ + uoff_t i_size; /* Size of the file */ + uoff_t start; /* Start position */ atomic64_t issued_to; /* Write issuer folio cursor */ - unsigned long long collected_to; /* Point we've collected to */ - unsigned long long cleaned_to; /* Position we've cleaned folios to */ - unsigned long long abandon_to; /* Position to abandon folios to */ + uoff_t collected_to; /* Point we've collected to */ + uoff_t cache_coll_to; /* Point the cache has collected to */ + uoff_t cleaned_to; /* Position we've cleaned folios to */ + uoff_t abandon_to; /* Position to abandon folios to */ const struct folio *no_unlock_folio; /* Don't unlock this folio after read */ gfp_t gfp; /* GFP flags to use */ unsigned int direct_bv_count; /* Number of elements in direct_bv[] */ @@ -273,14 +285,18 @@ struct netfs_io_request { #define NETFS_RREQ_FAILED 3 /* The request failed */ #define NETFS_RREQ_RETRYING 4 /* Set if we're in the retry path */ #define NETFS_RREQ_SHORT_TRANSFER 5 /* Set if we have a short transfer */ -#define NETFS_RREQ_OFFLOAD_COLLECTION 8 /* Offload collection to workqueue */ -#define NETFS_RREQ_NO_UNLOCK_FOLIO 9 /* Don't unlock no_unlock_folio on completion */ +#define NETFS_RREQ_CACHE_STOP 8 /* Set to stop caching (ENOBUFS or error) */ +#define NETFS_RREQ_CACHE_ERROR 9 /* Set if we got an error from the cache */ #define NETFS_RREQ_CANCEL_CACHING 10 /* Set to cancel caching */ -#define NETFS_RREQ_UPLOAD_TO_SERVER 11 /* Need to write to the server */ -#define NETFS_RREQ_USE_IO_ITER 12 /* Use ->io_iter rather than ->i_pages */ +#define NETFS_RREQ_OFFLOAD_COLLECTION 12 /* Offload collection to workqueue */ +#define NETFS_RREQ_NO_UNLOCK_FOLIO 13 /* Don't unlock no_unlock_folio on completion */ +#define NETFS_RREQ_UPLOAD_TO_SERVER 14 /* Need to write to the server */ +#define NETFS_RREQ_USE_IO_ITER 15 /* Use ->io_iter rather than ->i_pages */ #define NETFS_RREQ_NEED_PUT_RA_REFS 17 /* Need to put the folio refs RA gave us */ +#ifdef CONFIG_NETFS_PGPRIV2 #define NETFS_RREQ_USE_PGPRIV2 31 /* [DEPRECATED] Use PG_private_2 to mark * write to cache on read */ +#endif const struct netfs_request_ops *netfs_ops; }; @@ -299,12 +315,12 @@ struct netfs_request_ops { int (*prepare_read)(struct netfs_io_subrequest *subreq); void (*issue_read)(struct netfs_io_subrequest *subreq); bool (*is_still_valid)(struct netfs_io_request *rreq); - int (*check_write_begin)(struct file *file, loff_t pos, unsigned len, + int (*check_write_begin)(struct file *file, uoff_t pos, unsigned len, struct folio **foliop, void **_fsdata); void (*done)(struct netfs_io_request *rreq); /* Modification handling */ - void (*update_i_size)(struct inode *inode, loff_t i_size); + void (*update_i_size)(struct inode *inode, uoff_t i_size); void (*post_modify)(struct inode *inode); /* Write request handling */ @@ -332,7 +348,7 @@ struct netfs_cache_ops { /* Read data from the cache */ int (*read)(struct netfs_cache_resources *cres, - loff_t start_pos, + uoff_t start_pos, struct iov_iter *iter, enum netfs_read_from_hole read_hole, netfs_io_terminated_t term_func, @@ -340,7 +356,7 @@ struct netfs_cache_ops { /* Write data to the cache */ int (*write)(struct netfs_cache_resources *cres, - loff_t start_pos, + uoff_t start_pos, struct iov_iter *iter, netfs_io_terminated_t term_func, void *term_func_priv); @@ -350,15 +366,14 @@ struct netfs_cache_ops { /* Expand readahead request */ void (*expand_readahead)(struct netfs_cache_resources *cres, - unsigned long long *_start, - unsigned long long *_len, - unsigned long long i_size); + uoff_t *_start, + uoff_t *_len, + uoff_t i_size); /* Prepare a read operation, shortening it to a cached/uncached * boundary as appropriate. */ - enum netfs_io_source (*prepare_read)(struct netfs_io_subrequest *subreq, - unsigned long long i_size); + int (*prepare_read)(struct netfs_io_subrequest *subreq); /* Prepare a write subrequest, working out if we're allowed to do it * and finding out the maximum amount of data to gather before @@ -371,15 +386,24 @@ struct netfs_cache_ops { * actually do. */ int (*prepare_write)(struct netfs_cache_resources *cres, - loff_t *_start, size_t *_len, size_t upper_len, - loff_t i_size, bool no_space_allocated_yet); + uoff_t *_start, size_t *_len, size_t upper_len, + uoff_t i_size, bool no_space_allocated_yet); /* Query the occupancy of the cache in a region, returning where the * next chunk of data starts and how long it is. */ int (*query_occupancy)(struct netfs_cache_resources *cres, - loff_t start, size_t len, size_t granularity, - loff_t *_data_start, size_t *_data_len); + struct fscache_occupancy *occ); + + /* Collect the result of buffered writeback to the cache. This + * includes copying a read to the cache. block_type is one of: + * - NETFS_CACHE_COLLECT_WRITE_DATA for a block of data + * - NETFS_CACHE_COLLECT_WRITE_GAP if a discontiguity was skipped + * - NETFS_CACHE_COLLECT_WRITE_CANCEL for a cancellation gap + */ + void (*collect_write)(struct netfs_io_request *wreq, + uoff_t start, size_t len, + enum netfs_cache_collect block_type); }; /* High-level read API. */ @@ -412,7 +436,7 @@ struct readahead_control; void netfs_readahead(struct readahead_control *); int netfs_read_folio(struct file *, struct folio *); int netfs_write_begin(struct netfs_inode *, struct file *, - struct address_space *, loff_t pos, unsigned int len, + struct address_space *, uoff_t pos, unsigned int len, struct folio **, void **fsdata); int netfs_writepages(struct address_space *mapping, struct writeback_control *wbc); @@ -490,10 +514,10 @@ static inline struct netfs_inode *netfs_inode(struct inode *inode) * cmpxchg8b without the need of the lock prefix). For SMP compiles and 64bit * archs it makes no difference if preempt is enabled or not. */ -static inline unsigned long long netfs_read_remote_i_size(const struct inode *inode) +static inline uoff_t netfs_read_remote_i_size(const struct inode *inode) { const struct netfs_inode *ictx = container_of(inode, struct netfs_inode, inode); - unsigned long long remote_i_size; + uoff_t remote_i_size; #if BITS_PER_LONG==32 && defined(CONFIG_SMP) unsigned int seq; @@ -528,7 +552,7 @@ static inline unsigned long long netfs_read_remote_i_size(const struct inode *in * spinning forever. */ static inline void netfs_write_remote_i_size(struct inode *inode, - unsigned long long remote_i_size) + uoff_t remote_i_size) { struct netfs_inode *ictx = netfs_inode(inode); @@ -565,10 +589,10 @@ static inline void netfs_write_remote_i_size(struct inode *inode, * cmpxchg8b without the need of the lock prefix). For SMP compiles and 64bit * archs it makes no difference if preempt is enabled or not. */ -static inline unsigned long long netfs_read_zero_point(const struct inode *inode) +static inline uoff_t netfs_read_zero_point(const struct inode *inode) { struct netfs_inode *ictx = container_of(inode, struct netfs_inode, inode); - unsigned long long zero_point; + uoff_t zero_point; #if BITS_PER_LONG==32 && defined(CONFIG_SMP) unsigned int seq; @@ -603,7 +627,7 @@ static inline unsigned long long netfs_read_zero_point(const struct inode *inode * forever. */ static inline void netfs_write_zero_point(struct inode *inode, - unsigned long long zero_point) + uoff_t zero_point) { struct netfs_inode *ictx = netfs_inode(inode); @@ -644,9 +668,9 @@ static inline void netfs_write_zero_point(struct inode *inode, * archs it makes no difference if preempt is enabled or not. */ static inline void netfs_read_sizes(const struct inode *inode, - unsigned long long *i_size, - unsigned long long *remote_i_size, - unsigned long long *zero_point) + uoff_t *i_size, + uoff_t *remote_i_size, + uoff_t *zero_point) { const struct netfs_inode *ictx = container_of(inode, struct netfs_inode, inode); #if BITS_PER_LONG==32 && defined(CONFIG_SMP) @@ -692,9 +716,9 @@ static inline void netfs_read_sizes(const struct inode *inode, * forever. */ static inline void netfs_write_sizes(struct inode *inode, - unsigned long long i_size, - unsigned long long remote_i_size, - unsigned long long zero_point) + uoff_t i_size, + uoff_t remote_i_size, + uoff_t zero_point) { struct netfs_inode *ictx = netfs_inode(inode); @@ -762,7 +786,7 @@ static inline void netfs_inode_init(struct netfs_inode *ctx, * Inform the netfs lib that a file got resized so that it can adjust its state. */ static inline void netfs_resize_file(struct netfs_inode *ictx, - unsigned long long new_i_size, + uoff_t new_i_size, bool changed_on_server) { #if BITS_PER_LONG==32 && defined(CONFIG_SMP) diff --git a/include/linux/netpoll.h b/include/linux/netpoll.h index 1c6b1eec5efd..ec0821a5b02d 100644 --- a/include/linux/netpoll.h +++ b/include/linux/netpoll.h @@ -16,11 +16,6 @@ #include <linux/ip.h> #include <linux/udp.h> -union inet_addr { - __be32 ip; - struct in6_addr in6; -}; - struct netpoll { struct net_device *dev; netdevice_tracker dev_tracker; diff --git a/include/linux/nfs.h b/include/linux/nfs.h index 0906a0b40c6a..8c2818db43c5 100644 --- a/include/linux/nfs.h +++ b/include/linux/nfs.h @@ -11,59 +11,8 @@ #include <linux/cred.h> #include <linux/sunrpc/auth.h> #include <linux/sunrpc/msg_prot.h> -#include <linux/string.h> -#include <linux/crc32.h> -#include <uapi/linux/nfs.h> - -/* The LOCALIO program is entirely private to Linux and is - * NOT part of the uapi. - */ -#define NFS_LOCALIO_PROGRAM 400122 -#define LOCALIOPROC_NULL 0 -#define LOCALIOPROC_UUID_IS_LOCAL 1 - -/* - * This is the kernel NFS client file handle representation - */ -#define NFS_MAXFHSIZE 128 -struct nfs_fh { - unsigned short size; - unsigned char data[NFS_MAXFHSIZE]; -}; - -/* - * Returns a zero iff the size and data fields match. - * Checks only "size" bytes in the data field. - */ -static inline int nfs_compare_fh(const struct nfs_fh *a, const struct nfs_fh *b) -{ - return a->size != b->size || memcmp(a->data, b->data, a->size) != 0; -} - -static inline void nfs_copy_fh(struct nfs_fh *target, const struct nfs_fh *source) -{ - target->size = source->size; - memcpy(target->data, source->data, source->size); -} - -enum nfs3_stable_how { - NFS_UNSTABLE = 0, - NFS_DATA_SYNC = 1, - NFS_FILE_SYNC = 2, +#include <linux/nfs_fh.h> - /* used by direct.c to mark verf as invalid */ - NFS_INVALID_STABLE_HOW = -1 -}; +#include <uapi/linux/nfs.h> -/** - * nfs_fhandle_hash - calculate the crc32 hash for the filehandle - * @fh - pointer to filehandle - * - * returns a crc32 hash for the filehandle that is compatible with - * the one displayed by "wireshark". - */ -static inline u32 nfs_fhandle_hash(const struct nfs_fh *fh) -{ - return ~crc32_le(0xFFFFFFFF, &fh->data[0], fh->size); -} #endif /* _LINUX_NFS_H */ diff --git a/include/linux/nfs3.h b/include/linux/nfs3.h index 404b8f724fc9..b6539a75edea 100644 --- a/include/linux/nfs3.h +++ b/include/linux/nfs3.h @@ -7,6 +7,49 @@ #include <uapi/linux/nfs3.h> +/* + * NFSv3 error status values. + * See RFC 1813 Section 2.5 + */ +enum { + NFS3ERR_PERM = 1, + NFS3ERR_NOENT = 2, + NFS3ERR_IO = 5, + NFS3ERR_NXIO = 6, + NFS3ERR_ACCES = 13, + NFS3ERR_EXIST = 17, + NFS3ERR_XDEV = 18, + NFS3ERR_NODEV = 19, + NFS3ERR_NOTDIR = 20, + NFS3ERR_ISDIR = 21, + NFS3ERR_INVAL = 22, + NFS3ERR_FBIG = 27, + NFS3ERR_NOSPC = 28, + NFS3ERR_ROFS = 30, + NFS3ERR_MLINK = 31, + NFS3ERR_NAMETOOLONG = 63, + NFS3ERR_NOTEMPTY = 66, + NFS3ERR_DQUOT = 69, + NFS3ERR_STALE = 70, + NFS3ERR_REMOTE = 71, + NFS3ERR_BADHANDLE = 10001, + NFS3ERR_NOT_SYNC = 10002, + NFS3ERR_BAD_COOKIE = 10003, + NFS3ERR_NOTSUPP = 10004, + NFS3ERR_TOOSMALL = 10005, + NFS3ERR_SERVERFAULT = 10006, + NFS3ERR_BADTYPE = 10007, + NFS3ERR_JUKEBOX = 10008, +}; + +enum nfs3_stable_how { + NFS_UNSTABLE = 0, + NFS_DATA_SYNC = 1, + NFS_FILE_SYNC = 2, + + /* used to mark verf as invalid */ + NFS_INVALID_STABLE_HOW = -1 +}; /* Number of 32bit words in post_op_attr */ #define NFS3_POST_OP_ATTR_WORDS 22 diff --git a/include/linux/nfs4.h b/include/linux/nfs4.h index 1a3981c26b23..41b7cdcc674f 100644 --- a/include/linux/nfs4.h +++ b/include/linux/nfs4.h @@ -263,6 +263,12 @@ enum why_no_delegation4 { /* new to v4.1 */ WND4_IS_DIR = 8, }; +enum stable_how4 { + UNSTABLE4 = 0, + DATA_SYNC4 = 1, + FILE_SYNC4 = 2, +}; + enum lock_type4 { NFS4_UNLOCK_LT = 0, NFS4_READ_LT = 1, diff --git a/include/linux/nfs_common.h b/include/linux/nfs_common.h index a541c3a02887..88dccd8564cf 100644 --- a/include/linux/nfs_common.h +++ b/include/linux/nfs_common.h @@ -6,11 +6,11 @@ #define _LINUX_NFS_COMMON_H #include <linux/errno.h> -#include <uapi/linux/nfs.h> +#include <linux/types.h> /* Mapping from NFS error code to "errno" error code. */ -int nfs_stat_to_errno(enum nfs_stat status); +int nfs_stat_to_errno(int status); int nfs4_stat_to_errno(int stat); __u32 nfs_localio_errno_to_nfs4_stat(int errno); diff --git a/include/linux/nfs_fh.h b/include/linux/nfs_fh.h new file mode 100644 index 000000000000..49dfc5ec60fe --- /dev/null +++ b/include/linux/nfs_fh.h @@ -0,0 +1,63 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * struct nfs_fh is an NFS version-agnostic data structure that + * stores an NFS file handle. It is also commonly used in NFS + * related APIs. + */ +#ifndef _LINUX_NFS_FH_H +#define _LINUX_NFS_FH_H + +#include <linux/types.h> +#include <linux/string.h> +#include <linux/crc32.h> + +/* + * The largest file handle size today is an NFSv4 file handle, + * which can be up to 128 octets long. + */ +#define NFS_MAXFHSIZE 128 +struct nfs_fh { + unsigned short size; + unsigned char data[NFS_MAXFHSIZE]; +}; + +/** + * nfs_compare_fh - Compare two NFS file handles + * @a: An NFS file handle to be compared + * @b: An NFS file handle to be compared + * + * Checks only "size" bytes in each data field. + * + * Return: %false if the two file handles are equal, otherwise %true + */ +static inline bool nfs_compare_fh(const struct nfs_fh *a, const struct nfs_fh *b) +{ + return a->size != b->size || memcmp(a->data, b->data, a->size) != 0; +} + +/** + * nfs_copy_fh - Copy an NFS file handle + * @target: Destination file handle + * @source: Source file handle + * + * Copies source->size bytes of file handle data into target. + */ +static inline void nfs_copy_fh(struct nfs_fh *target, const struct nfs_fh *source) +{ + target->size = source->size; + memcpy(target->data, source->data, source->size); +} + +/** + * nfs_fhandle_hash - Calculate the crc32 hash for the filehandle + * @fh: An NFS file handle to hash + * + * Return: a crc32 hash for the filehandle that is compatible with + * the one displayed by "wireshark" + */ +static inline u32 nfs_fhandle_hash(const struct nfs_fh *fh) +{ + return ~crc32_le(0xFFFFFFFF, &fh->data[0], fh->size); +} + +#endif /* _LINUX_NFS_FH_H */ diff --git a/include/linux/nfs_fs.h b/include/linux/nfs_fs.h index b85a73ae7919..d2c716322c6f 100644 --- a/include/linux/nfs_fs.h +++ b/include/linux/nfs_fs.h @@ -437,11 +437,11 @@ extern int nfs_refresh_inode(struct inode *, struct nfs_fattr *); extern int nfs_post_op_update_inode(struct inode *inode, struct nfs_fattr *fattr); extern int nfs_post_op_update_inode_force_wcc(struct inode *inode, struct nfs_fattr *fattr); extern int nfs_post_op_update_inode_force_wcc_locked(struct inode *inode, struct nfs_fattr *fattr); -extern int nfs_getattr(struct mnt_idmap *, const struct path *, +extern int nfs_getattr(const struct mnt_idmap *, const struct path *, struct kstat *, u32, unsigned int); extern void nfs_access_add_cache(struct inode *, struct nfs_access_entry *, const struct cred *); extern void nfs_access_set_mask(struct nfs_access_entry *, u32); -extern int nfs_permission(struct mnt_idmap *, struct inode *, int); +extern int nfs_permission(const struct mnt_idmap *, struct inode *, int); extern int nfs_open(struct inode *, struct file *); extern int nfs_attribute_cache_expired(struct inode *inode); extern int nfs_revalidate_inode(struct inode *inode, unsigned long flags); @@ -450,7 +450,7 @@ extern int nfs_clear_invalid_mapping(struct address_space *mapping); extern bool nfs_mapping_need_revalidate_inode(struct inode *inode); extern int nfs_revalidate_mapping(struct inode *inode, struct address_space *mapping); extern int nfs_revalidate_mapping_rcu(struct inode *inode); -extern int nfs_setattr(struct mnt_idmap *, struct dentry *, struct iattr *); +extern int nfs_setattr(const struct mnt_idmap *, struct dentry *, struct iattr *); extern void nfs_setattr_update_inode(struct inode *inode, struct iattr *attr, struct nfs_fattr *); extern void nfs_setsecurity(struct inode *inode, struct nfs_fattr *fattr); extern struct nfs_open_context *get_nfs_open_context(struct nfs_open_context *ctx); diff --git a/include/linux/nfs_fs_sb.h b/include/linux/nfs_fs_sb.h index 34d294774f8c..416c6f39f31d 100644 --- a/include/linux/nfs_fs_sb.h +++ b/include/linux/nfs_fs_sb.h @@ -74,6 +74,8 @@ struct nfs_client { u64 cl_clientid; /* constant */ nfs4_verifier cl_confirm; /* Clientid verifier */ unsigned long cl_state; + /* bumped on each CB_NOTIFY_DEVICEID CHANGE for this client */ + atomic_t cl_deviceid_change_epoch; spinlock_t cl_lock; @@ -101,6 +103,8 @@ struct nfs_client { /* The flags used for obtaining the clientid during EXCHANGE_ID */ u32 cl_exchange_flags; struct nfs4_session *cl_session; /* shared session */ + /* CB_NOTIFY_DEVICEID DELETE suspects, protected by cl_lock */ + struct list_head cl_deviceid_deletes; bool cl_preserve_clid; struct nfs41_server_owner *cl_serverowner; struct nfs41_server_scope *cl_serverscope; @@ -248,6 +252,10 @@ struct nfs_server { that are supported on this filesystem */ struct pnfs_layoutdriver_type *pnfs_curr_ld; /* Active layout driver */ + unsigned int lg_reply_sz; /* Learned LAYOUTGET reply + buffer size, when the layout + driver's default has proved + too small */ struct rpc_wait_queue roc_rpcwaitq; /* the following fields are protected by nfs_client->cl_lock */ diff --git a/include/linux/nfs_page.h b/include/linux/nfs_page.h index 4b9a35dbc062..c38e4b380be5 100644 --- a/include/linux/nfs_page.h +++ b/include/linux/nfs_page.h @@ -38,6 +38,7 @@ enum { PG_REMOVE, /* page group sync bit in write path */ PG_CONTENDED1, /* Is someone waiting for a lock? */ PG_CONTENDED2, /* Is someone waiting for a lock? */ + PG_PINNED, /* page is pinned by GUP */ }; struct nfs_inode; @@ -58,6 +59,7 @@ struct nfs_page { struct nfs_page *wb_this_page; /* list of reqs for this page */ struct nfs_page *wb_head; /* head pointer for req list */ unsigned short wb_nio; /* Number of I/O attempts */ + unsigned int wb_nr_pinned; /* Number of pinned pages */ }; struct nfs_pgio_mirror; @@ -125,15 +127,17 @@ struct nfs_pageio_descriptor { extern struct nfs_page *nfs_page_create_from_page(struct nfs_open_context *ctx, struct page *page, + bool pinned, unsigned int pgbase, loff_t offset, unsigned int count); extern struct nfs_page *nfs_page_create_from_folio(struct nfs_open_context *ctx, struct folio *folio, + bool pinned, unsigned int offset, unsigned int count); -extern void nfs_release_request(struct nfs_page *); - +void nfs_release_request(struct nfs_page *req); +void nfs_release_request_list(struct list_head *head); extern void nfs_pageio_init(struct nfs_pageio_descriptor *desc, struct inode *inode, diff --git a/include/linux/nfs_ssc.h b/include/linux/nfs_ssc.h index 22265b1ff080..c199ea23e7eb 100644 --- a/include/linux/nfs_ssc.h +++ b/include/linux/nfs_ssc.h @@ -2,80 +2,33 @@ /* * include/linux/nfs_ssc.h * + * NFSv4.2 server-to-server copy, NFS client side APIs + * * Author: Dai Ngo <dai.ngo@oracle.com> * * Copyright (c) 2020, Oracle and/or its affiliates. */ -#include <linux/nfs_fs.h> -#include <linux/sunrpc/svc.h> +#ifndef _LINUX_NFS_SSC_H +#define _LINUX_NFS_SSC_H -extern struct nfs_ssc_client_ops_tbl nfs_ssc_client_tbl; +#include <linux/nfs_fh.h> +#include <linux/nfs4.h> + +struct file; +struct vfsmount; -/* - * NFS_V4 - */ struct nfs4_ssc_client_ops { + struct module *owner; struct file *(*sco_open)(struct vfsmount *ss_mnt, struct nfs_fh *src_fh, nfs4_stateid *stateid); void (*sco_close)(struct file *filep); }; -/* - * NFS_FS - */ -struct nfs_ssc_client_ops { - void (*sco_sb_deactive)(struct super_block *sb); -}; - -struct nfs_ssc_client_ops_tbl { - const struct nfs4_ssc_client_ops *ssc_nfs4_ops; - const struct nfs_ssc_client_ops *ssc_nfs_ops; -}; - extern void nfs42_ssc_register_ops(void); extern void nfs42_ssc_unregister_ops(void); extern void nfs42_ssc_register(const struct nfs4_ssc_client_ops *ops); extern void nfs42_ssc_unregister(const struct nfs4_ssc_client_ops *ops); -#ifdef CONFIG_NFSD_V4_2_INTER_SSC -static inline struct file *nfs42_ssc_open(struct vfsmount *ss_mnt, - struct nfs_fh *src_fh, nfs4_stateid *stateid) -{ - if (nfs_ssc_client_tbl.ssc_nfs4_ops) - return (*nfs_ssc_client_tbl.ssc_nfs4_ops->sco_open)(ss_mnt, src_fh, stateid); - return ERR_PTR(-EIO); -} - -static inline void nfs42_ssc_close(struct file *filep) -{ - if (nfs_ssc_client_tbl.ssc_nfs4_ops) - (*nfs_ssc_client_tbl.ssc_nfs4_ops->sco_close)(filep); -} -#endif - -struct nfsd4_ssc_umount_item { - struct list_head nsui_list; - bool nsui_busy; - /* - * nsui_refcnt inited to 2, 1 on list and 1 for consumer. Entry - * is removed when refcnt drops to 1 and nsui_expire expires. - */ - refcount_t nsui_refcnt; - unsigned long nsui_expire; - struct vfsmount *nsui_vfsmount; - char nsui_ipaddr[RPC_MAX_ADDRBUFLEN + 1]; -}; - -/* - * NFS_FS - */ -extern void nfs_ssc_register(const struct nfs_ssc_client_ops *ops); -extern void nfs_ssc_unregister(const struct nfs_ssc_client_ops *ops); - -static inline void nfs_do_sb_deactive(struct super_block *sb) -{ - if (nfs_ssc_client_tbl.ssc_nfs_ops) - (*nfs_ssc_client_tbl.ssc_nfs_ops->sco_sb_deactive)(sb); -} +#endif /* _LINUX_NFS_SSC_H */ diff --git a/include/linux/nfs_xdr.h b/include/linux/nfs_xdr.h index 7ed8fdb930d6..c0e29b4dfa62 100644 --- a/include/linux/nfs_xdr.h +++ b/include/linux/nfs_xdr.h @@ -1693,6 +1693,7 @@ struct nfs_pgio_header { struct nfs_client *ds_clp; /* pNFS data server */ u32 ds_commit_idx; /* ds index if ds_clp is set */ u32 pgio_mirror_idx;/* mirror index in pgio layer */ + struct nfs4_deviceid_node *ds_dev; /* device node ref held across the I/O */ }; struct nfs_mds_commit_info { @@ -1731,6 +1732,7 @@ struct nfs_commit_data { struct nfs_open_context *context; struct pnfs_layout_segment *lseg; struct nfs_client *ds_clp; /* pNFS data server */ + struct nfs4_deviceid_node *ds_dev; /* device node ref held across the commit */ int ds_commit_index; loff_t lwb; const struct rpc_call_ops *mds_ops; diff --git a/include/linux/nfsd_ssc.h b/include/linux/nfsd_ssc.h new file mode 100644 index 000000000000..7001410f01c2 --- /dev/null +++ b/include/linux/nfsd_ssc.h @@ -0,0 +1,38 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * include/linux/nfsd_ssc.h + * + * NFSv4.2 server-to-server copy, NFS server side APIs + * + * Author: Dai Ngo <dai.ngo@oracle.com> + * + * Copyright (c) 2020, Oracle and/or its affiliates. + */ + +#ifndef _LINUX_NFSD_SSC_H +#define _LINUX_NFSD_SSC_H + +#include <linux/nfs_fh.h> +#include <linux/nfs4.h> + +struct file; +struct vfsmount; + +#if IS_ENABLED(CONFIG_NFS_V4_2_SSC_HELPER) +struct file *nfsd42_ssc_open(struct vfsmount *ss_mnt, struct nfs_fh *src_fh, + nfs4_stateid *stateid); +void nfsd42_ssc_close(struct file *filp); +#else +static inline struct file *nfsd42_ssc_open(struct vfsmount *ss_mnt, + struct nfs_fh *src_fh, + nfs4_stateid *stateid) +{ + return ERR_PTR(-EIO); +} + +static inline void nfsd42_ssc_close(struct file *filp) +{ +} +#endif + +#endif /* _LINUX_NFSD_SSC_H */ diff --git a/include/linux/nfslocalio.h b/include/linux/nfslocalio.h index 3d91043254e6..914246e3f342 100644 --- a/include/linux/nfslocalio.h +++ b/include/linux/nfslocalio.h @@ -8,14 +8,25 @@ #if IS_ENABLED(CONFIG_NFS_LOCALIO) +#include <linux/completion.h> #include <linux/module.h> #include <linux/list.h> +#include <linux/refcount.h> #include <linux/uuid.h> #include <linux/sunrpc/clnt.h> #include <linux/sunrpc/svcauth.h> -#include <linux/nfs.h> +#include <linux/nfs_fh.h> + #include <net/net_namespace.h> +/* + * The LOCALIO program is entirely private to Linux and is NOT part of + * the uapi. + */ +#define NFS_LOCALIO_PROGRAM 400122 +#define LOCALIOPROC_NULL 0 +#define LOCALIOPROC_UUID_IS_LOCAL 1 + struct nfs_client; struct nfs_file_localio; @@ -28,6 +39,11 @@ struct nfs_file_localio; typedef struct { uuid_t uuid; unsigned nfs3_localio_probe_count; + /* Owner reference plus temporary references held by nfs_common */ + refcount_t refs; + struct completion released; + /* Namespace registration order, protected by list_lock */ + u64 seq; /* this struct is over a cacheline, avoid bouncing */ spinlock_t ____cacheline_aligned lock; struct list_head list; @@ -46,6 +62,7 @@ void nfs_uuid_is_local(const uuid_t *, struct list_head *, spinlock_t *, void nfs_localio_enable_client(struct nfs_client *clp); void nfs_localio_disable_client(struct nfs_client *clp); +void nfs_localio_free_client(struct nfs_client *clp); void nfs_localio_invalidate_clients(struct list_head *nn_local_clients, spinlock_t *nn_local_clients_lock); @@ -118,6 +135,10 @@ static inline void nfs_localio_disable_client(struct nfs_client *clp) { } +static inline void nfs_localio_free_client(struct nfs_client *clp) +{ +} + #endif /* CONFIG_NFS_LOCALIO */ #endif /* __LINUX_NFSLOCALIO_H */ diff --git a/include/linux/notifier.h b/include/linux/notifier.h index 01b6c9d9956f..4eeae9741a6e 100644 --- a/include/linux/notifier.h +++ b/include/linux/notifier.h @@ -46,6 +46,7 @@ * often but notifier_blocks will seldom be removed. */ +struct device; struct notifier_block; typedef int (*notifier_fn_t)(struct notifier_block *nb, @@ -145,8 +146,14 @@ extern void srcu_init_notifier_head(struct srcu_notifier_head *nh); extern int atomic_notifier_chain_register(struct atomic_notifier_head *nh, struct notifier_block *nb); +int devm_atomic_notifier_chain_register(struct device *dev, + struct atomic_notifier_head *nh, + struct notifier_block *nb); extern int blocking_notifier_chain_register(struct blocking_notifier_head *nh, struct notifier_block *nb); +int devm_blocking_notifier_chain_register(struct device *dev, + struct blocking_notifier_head *nh, + struct notifier_block *nb); extern int raw_notifier_chain_register(struct raw_notifier_head *nh, struct notifier_block *nb); extern int srcu_notifier_chain_register(struct srcu_notifier_head *nh, diff --git a/include/linux/ns/ns_common_types.h b/include/linux/ns/ns_common_types.h index 6ed6b497831c..47a5cfe2326a 100644 --- a/include/linux/ns/ns_common_types.h +++ b/include/linux/ns/ns_common_types.h @@ -116,6 +116,9 @@ struct ns_common { struct dentry *stashed; const struct proc_ns_operations *ops; unsigned int inum; +#ifdef CONFIG_SECURITY + void *ns_security; +#endif struct ns_tree; struct rcu_head ns_rcu; }; diff --git a/include/linux/of.h b/include/linux/of.h index b920aac6b975..44d83133a65a 100644 --- a/include/linux/of.h +++ b/include/linux/of.h @@ -1837,7 +1837,6 @@ struct of_overlay_notify_data { int of_overlay_fdt_apply(const void *overlay_fdt, u32 overlay_fdt_size, int *ovcs_id, const struct device_node *target_base); int of_overlay_remove(int *ovcs_id); -int of_overlay_remove_all(void); int of_overlay_notifier_register(struct notifier_block *nb); int of_overlay_notifier_unregister(struct notifier_block *nb); @@ -1855,11 +1854,6 @@ static inline int of_overlay_remove(int *ovcs_id) return -ENOTSUPP; } -static inline int of_overlay_remove_all(void) -{ - return -ENOTSUPP; -} - static inline int of_overlay_notifier_register(struct notifier_block *nb) { return 0; diff --git a/include/linux/omap-mailbox.h b/include/linux/omap-mailbox.h index 3cc5c4ed7f5a..e45f8c3841c2 100644 --- a/include/linux/omap-mailbox.h +++ b/include/linux/omap-mailbox.h @@ -6,8 +6,9 @@ #ifndef OMAP_MAILBOX_H #define OMAP_MAILBOX_H -typedef uintptr_t mbox_msg_t; +typedef u32 mbox_msg_t; -#define omap_mbox_message(data) (u32)(mbox_msg_t)(data) +#define omap_mbox_to_message(data) ((void *)(uintptr_t)(data)) +#define omap_mbox_from_message(data) ((mbox_msg_t)(uintptr_t)(data)) #endif /* OMAP_MAILBOX_H */ diff --git a/include/linux/once_lite.h b/include/linux/once_lite.h index 236592c4eeb1..5e2b67039aaf 100644 --- a/include/linux/once_lite.h +++ b/include/linux/once_lite.h @@ -10,19 +10,21 @@ #define DO_ONCE_LITE(func, ...) \ DO_ONCE_LITE_IF(true, func, ##__VA_ARGS__) -#define __ONCE_LITE_IF(condition) \ +#define __ONCE_LITE() \ ({ \ static bool __section(".data..once") __already_done; \ - bool __ret_cond = !!(condition); \ bool __ret_once = false; \ \ - if (unlikely(__ret_cond) && unlikely(!__already_done)) {\ + if (unlikely(!__already_done)) { \ __already_done = true; \ __ret_once = true; \ } \ unlikely(__ret_once); \ }) +#define __ONCE_LITE_IF(condition) \ + (unlikely(condition) && __ONCE_LITE()) + #define DO_ONCE_LITE_IF(condition, func, ...) \ ({ \ bool __ret_do_once = !!(condition); \ diff --git a/include/linux/overflow.h b/include/linux/overflow.h index a8cb6319b4fb..d0f04436343e 100644 --- a/include/linux/overflow.h +++ b/include/linux/overflow.h @@ -418,7 +418,7 @@ static __always_inline size_t __must_check size_sub(size_t minuend, size_t subtr #define flex_array_size(p, member, count) \ __builtin_choose_expr(__is_constexpr(count), \ (count) * sizeof(*(p)->member) + __must_be_array((p)->member), \ - size_mul(count, sizeof(*(p)->member) + __must_be_array((p)->member))) + size_mul(count, sizeof(*(p)->member)) + __must_be_array((p)->member)) /** * struct_size() - Calculate size of structure with trailing flexible array. diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h index 7a863572adce..b0ddc652e76c 100644 --- a/include/linux/page-flags.h +++ b/include/linux/page-flags.h @@ -44,10 +44,6 @@ * Consequently, PG_reserved for a page mapped into user space can indicate * the zero page, the vDSO, MMIO pages or device memory. * - * The PG_private bitflag is set on pagecache pages if they contain filesystem - * specific data (which is normally at page->private). It can be used by - * private allocations for its own usage. - * * During initiation of disk I/O, PG_locked is set. This bit is set before I/O * and cleared when writeback _starts_ or when read _completes_. PG_writeback * is set before writeback starts and cleared when it finishes. @@ -105,7 +101,7 @@ enum pageflags { PG_owner_2, /* Owner use. If pagecache, fs may use */ PG_arch_1, PG_reserved, - PG_private, /* If pagecache, has fs-private data */ + PG_folio, /* Do not use: reserved for folio identification */ PG_private_2, /* If pagecache, has fs aux data */ PG_reclaim, /* To be reclaimed asap */ PG_swapbacked, /* Page is backed by RAM/swap */ @@ -208,14 +204,13 @@ enum pageflags { static __always_inline bool compound_info_has_mask(void) { /* - * Limit mask usage to HugeTLB vmemmap optimization (HVO) where it - * makes a difference. + * Limit mask usage to HVO where it makes a difference. * * The approach with mask would work in the wider set of conditions, * but it requires validating that struct pages are naturally aligned * for all orders up to the MAX_FOLIO_ORDER, which can be tricky. */ - if (!IS_ENABLED(CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP)) + if (!IS_ENABLED(CONFIG_VMEMMAP_OPTIMIZATION)) return false; return is_power_of_2(sizeof(struct page)); @@ -576,9 +571,19 @@ FOLIO_FLAG(swapbacked, FOLIO_HEAD_PAGE) /* * Private page markings that may be used by the filesystem that owns the page * for its own purposes. - * - PG_private and PG_private_2 cause release_folio() and co to be invoked + * - folio->private and PG_private_2 cause release_folio() and co to be invoked */ -PAGEFLAG(Private, private, PF_ANY) + +static __always_inline bool folio_test_private(const struct folio *folio) +{ + /* + * data_race() is added for readers without holding the folio lock. + * Only the NULL/non-NULL answer is used and both are valid while + * private is being attached or detached, so the race is benign. + */ + return data_race(folio->private); +} + FOLIO_FLAG(private_2, FOLIO_HEAD_PAGE) /* owner_2 can be set on tail pages for anon memory */ @@ -588,13 +593,12 @@ FOLIO_FLAG(owner_2, FOLIO_HEAD_PAGE) * Only test-and-set exist for PG_writeback. The unconditional operators are * risky: they bypass page accounting. */ -TESTPAGEFLAG(Writeback, writeback, PF_NO_TAIL) - TESTSCFLAG(Writeback, writeback, PF_NO_TAIL) +FOLIO_TEST_FLAG(writeback, FOLIO_HEAD_PAGE) + FOLIO_TEST_SET_FLAG(writeback, FOLIO_HEAD_PAGE) FOLIO_FLAG(mappedtodisk, FOLIO_HEAD_PAGE) /* PG_readahead is only used for reads; PG_reclaim is only for writes */ -PAGEFLAG(Reclaim, reclaim, PF_NO_TAIL) - TESTCLEARFLAG(Reclaim, reclaim, PF_NO_TAIL) +FOLIO_FLAG(reclaim, FOLIO_HEAD_PAGE) FOLIO_FLAG(readahead, FOLIO_HEAD_PAGE) FOLIO_TEST_CLEAR_FLAG(readahead, FOLIO_HEAD_PAGE) @@ -656,6 +660,7 @@ TESTSCFLAG(HWPoison, hwpoison, PF_ANY) #define __PG_HWPOISON (1UL << PG_hwpoison) #else PAGEFLAG_FALSE(HWPoison, hwpoison) +TESTSCFLAG_FALSE(HWPoison, hwpoison) #define __PG_HWPOISON 0 #endif @@ -1170,7 +1175,7 @@ static __always_inline void __ClearPageAnonExclusive(struct page *page) */ #define PAGE_FLAGS_CHECK_AT_FREE \ (1UL << PG_lru | 1UL << PG_locked | \ - 1UL << PG_private | 1UL << PG_private_2 | \ + 1UL << PG_private_2 | \ 1UL << PG_writeback | 1UL << PG_reserved | \ 1UL << PG_active | \ 1UL << PG_unevictable | __PG_MLOCKED | LRU_GEN_MASK) @@ -1194,8 +1199,31 @@ static __always_inline void __ClearPageAnonExclusive(struct page *page) (0xffUL /* order */ | 1UL << PG_has_hwpoisoned | \ 1UL << PG_large_rmappable | 1UL << PG_partially_mapped) -#define PAGE_FLAGS_PRIVATE \ - (1UL << PG_private | 1UL << PG_private_2) +/** + * folio_has_attached_private - check if the folio has private data attached + * @folio: The folio to check. + * + * Use this in code that may encounter swapcache or hugetlb folios but only + * wants to detect attached private data. + * + * Return: true if the folio has private data attached. + */ +static inline bool folio_has_attached_private(const struct folio *folio) +{ + /* + * Swapcache stores swp_entry_t in folio->swap, a union with + * folio->private, and hugetlb stores its own flags in folio->private; + * both are excluded. + * + * NOTE: For swapcache, folio->swap.val PG_swapcache are not set as + * a whole, so folio_test_swapcache() is not reliable to exclude + * swapcache. Use folio_test_swapbacked() instead, since it remains set + * when a folio is added to/removed from swapcache. + */ + + return folio_test_private(folio) && !folio_test_swapbacked(folio) && + !folio_test_hugetlb(folio); +} /** * folio_has_private - Determine if folio has private stuff * @folio: The folio to be checked @@ -1205,7 +1233,7 @@ static __always_inline void __ClearPageAnonExclusive(struct page *page) */ static inline int folio_has_private(const struct folio *folio) { - return !!(folio->flags.f & PAGE_FLAGS_PRIVATE); + return folio_has_attached_private(folio) || folio_test_private_2(folio); } #undef PF_ANY diff --git a/include/linux/page_counter.h b/include/linux/page_counter.h index d649b6bbbc87..2baf7a2b29b2 100644 --- a/include/linux/page_counter.h +++ b/include/linux/page_counter.h @@ -63,11 +63,12 @@ static inline void page_counter_init(struct page_counter *counter, counter->track_failcnt = false; } -static inline unsigned long page_counter_read(struct page_counter *counter) +static inline unsigned long page_counter_read(const struct page_counter *counter) { return atomic_long_read(&counter->usage); } +long page_counter_margin(const struct page_counter *counter); void page_counter_cancel(struct page_counter *counter, unsigned long nr_pages); void page_counter_charge(struct page_counter *counter, unsigned long nr_pages); bool page_counter_try_charge(struct page_counter *counter, diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h index 0adfa6605653..73af18a37367 100644 --- a/include/linux/pagemap.h +++ b/include/linux/pagemap.h @@ -14,7 +14,6 @@ #include <linux/gfp.h> #include <linux/bitops.h> #include <linux/hardirq.h> /* for in_interrupt() */ -#include <linux/hugetlb_inline.h> struct folio_batch; @@ -594,7 +593,6 @@ static inline void folio_attach_private(struct folio *folio, void *data) { folio_get(folio); folio->private = data; - folio_set_private(folio); } /** @@ -629,9 +627,8 @@ static inline void *folio_detach_private(struct folio *folio) { void *data = folio_get_private(folio); - if (!folio_test_private(folio)) + if (!data) return NULL; - folio_clear_private(folio); folio->private = NULL; folio_put(folio); @@ -1128,8 +1125,7 @@ static inline pgoff_t linear_anon_page_index(const struct vm_area_struct *vma, const pgoff_t pgoff = __linear_anon_page_index(vma, address); VM_WARN_ON_ONCE(!vma_is_cow_mapping(vma)); - /* Account for MAP_PRIVATE-/dev/zero which is only semi-anonymous. */ - if (vma_is_anonymous(vma) && !vma->vm_file) + if (vma_is_anonymous(vma)) VM_WARN_ON_ONCE(pgoff != linear_page_index(vma, address)); return pgoff; @@ -1416,6 +1412,7 @@ struct readahead_control { bool dropbehind; bool _workingset; unsigned long _pflags; + bool _forward; }; #define DEFINE_READAHEAD(ractl, f, r, m, i) \ @@ -1480,18 +1477,29 @@ void page_cache_async_readahead(struct address_space *mapping, page_cache_async_ra(&ractl, folio, req_count); } +/* + * Adjust readahead_control to ensure next folio comes from + * [_index, _index + _nr_pages) afterwards and reset _batch_count. + */ +static inline void __readahead_advance(struct readahead_control *rac) +{ + if (rac->_forward) + rac->_index += rac->_batch_count; + + rac->_nr_pages -= rac->_batch_count; + rac->_batch_count = 0; +} + static inline struct folio *__readahead_folio(struct readahead_control *ractl) { struct folio *folio; BUG_ON(ractl->_batch_count > ractl->_nr_pages); - ractl->_nr_pages -= ractl->_batch_count; - ractl->_index += ractl->_batch_count; + __readahead_advance(ractl); + ractl->_forward = true; - if (!ractl->_nr_pages) { - ractl->_batch_count = 0; + if (!ractl->_nr_pages) return NULL; - } folio = xa_load(&ractl->mapping->i_pages, ractl->_index); VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); @@ -1517,6 +1525,39 @@ static inline struct folio *readahead_folio(struct readahead_control *ractl) return folio; } +/** + * readahead_folio_last - Get the next folio to read, from the tail. + * @ractl: The current readahead request. + * + * Like readahead_folio(), but walks the range back-to-front. The folio is + * returned locked with its refcount dropped; the caller unlocks it once I/O + * completes. Compound folios are returned once, at their head index. + * + * Context: The folio is locked. + * Return: A pointer to the next folio, or %NULL when done. + */ +static inline struct folio *readahead_folio_last(struct readahead_control *ractl) +{ + struct folio *folio; + + /* Drop the previously returned batch from the remaining range. */ + __readahead_advance(ractl); + ractl->_forward = false; + + if (!ractl->_nr_pages) + return NULL; + + /* xa_load() follows sibling entries, so a tail index returns the head */ + folio = xa_load(&ractl->mapping->i_pages, + ractl->_index + ractl->_nr_pages - 1); + VM_WARN_ON_ONCE_FOLIO(!folio_test_locked(folio), folio); + + ractl->_batch_count = folio_nr_pages(folio); + + folio_put(folio); + return folio; +} + static inline unsigned int __readahead_batch(struct readahead_control *rac, struct page **array, unsigned int array_sz) { @@ -1525,9 +1566,8 @@ static inline unsigned int __readahead_batch(struct readahead_control *rac, struct folio *folio; BUG_ON(rac->_batch_count > rac->_nr_pages); - rac->_nr_pages -= rac->_batch_count; - rac->_index += rac->_batch_count; - rac->_batch_count = 0; + __readahead_advance(rac); + rac->_forward = true; xas_set(&xas, rac->_index); rcu_read_lock(); diff --git a/include/linux/panic.h b/include/linux/panic.h index f1dd417e54b2..17e61b61c45f 100644 --- a/include/linux/panic.h +++ b/include/linux/panic.h @@ -13,7 +13,8 @@ __printf(1, 2) void panic(const char *fmt, ...) __noreturn __cold; __printf(1, 0) void vpanic(const char *fmt, va_list args) __noreturn __cold; -void nmi_panic(struct pt_regs *regs, const char *msg); +__printf(2, 3) +void nmi_panic(struct pt_regs *regs, const char *fmt, ...); void check_panic_on_warn(const char *origin); extern void oops_enter(void); extern void oops_exit(void); @@ -110,4 +111,6 @@ extern void add_taint(unsigned flag, enum lockdep_ok); extern int test_taint(unsigned flag); extern unsigned long get_taint(void); +void arch_do_panic(void); + #endif /* _LINUX_PANIC_H */ diff --git a/include/linux/pci-epf.h b/include/linux/pci-epf.h index 704e1dc8b30a..226f3e59cf60 100644 --- a/include/linux/pci-epf.h +++ b/include/linux/pci-epf.h @@ -163,7 +163,8 @@ enum pci_epf_doorbell_type { * For MSI-backed doorbells this is the MSI message, while for * "embedded" doorbells this represents an MMIO write that asserts * an interrupt on the EP side. - * @virq: IRQ number of this doorbell message + * @virq: IRQ number of this doorbell message. Multiple messages may use the + * same IRQ; consumers must request each distinct IRQ only once. * @irq_flags: Required flags for request_irq()/request_threaded_irq(). * Callers may OR-in additional flags (e.g. IRQF_ONESHOT). * @type: Doorbell type. diff --git a/include/linux/pci.h b/include/linux/pci.h index d31a8d107b1e..aa3765a20a96 100644 --- a/include/linux/pci.h +++ b/include/linux/pci.h @@ -467,6 +467,7 @@ struct pci_dev { unsigned int is_busmaster:1; /* Is busmaster */ unsigned int no_msi:1; /* May not use MSI */ + unsigned int no_aer_anfe:1; /* Do not unmask ANFE */ unsigned int block_cfg_access:1; /* Config space access blocked */ unsigned int broken_parity_status:1; /* Generates false positive parity */ unsigned int irq_reroute_variant:2; /* Needs IRQ rerouting variant */ @@ -611,7 +612,10 @@ static inline struct pci_dev *pci_physfn(struct pci_dev *dev) struct pci_dev *pci_alloc_dev(struct pci_bus *bus); -#define to_pci_dev(n) container_of(n, struct pci_dev, dev) +static inline struct pci_dev *to_pci_dev(const struct device *dev) +{ + return container_of(dev, struct pci_dev, dev); +} #define for_each_pci_dev(d) while ((d = pci_get_device(PCI_ANY_ID, PCI_ANY_ID, d)) != NULL) #define for_each_pci_dev_reverse(d) \ while ((d = pci_get_device_reverse(PCI_ANY_ID, PCI_ANY_ID, d)) != NULL) @@ -1477,7 +1481,7 @@ int pcie_set_mps(struct pci_dev *dev, int mps); u32 pcie_bandwidth_available(struct pci_dev *dev, struct pci_dev **limiting_dev, enum pci_bus_speed *speed, enum pcie_link_width *width); -int pcie_link_speed_mbps(struct pci_dev *pdev); +int pcie_link_bandwidth_mbps(struct pci_dev *pdev); void pcie_print_link_status(struct pci_dev *dev); int pcie_reset_flr(struct pci_dev *dev, bool probe); int pcie_flr(struct pci_dev *dev); @@ -1915,12 +1919,14 @@ static inline int pci_irqd_intx_xlate(struct irq_domain *d, #ifdef CONFIG_PCIEPORTBUS extern bool pcie_ports_disabled; extern bool pcie_ports_native; +extern bool pcie_ports_dpc_native; int pcie_set_target_speed(struct pci_dev *port, enum pci_bus_speed speed_req, bool use_lt); #else #define pcie_ports_disabled true #define pcie_ports_native false +#define pcie_ports_dpc_native false static inline int pcie_set_target_speed(struct pci_dev *port, enum pci_bus_speed speed_req, diff --git a/include/linux/pci_ids.h b/include/linux/pci_ids.h index 63774ae2bf71..030c86dd6d67 100644 --- a/include/linux/pci_ids.h +++ b/include/linux/pci_ids.h @@ -889,6 +889,7 @@ #define PCI_DEVICE_ID_TI_XX12 0x8039 #define PCI_DEVICE_ID_TI_XX12_FM 0x803b #define PCI_DEVICE_ID_TI_XIO2000A 0x8231 +#define PCI_DEVICE_ID_TI_TUSB73X0 0x8241 #define PCI_DEVICE_ID_TI_1130 0xac12 #define PCI_DEVICE_ID_TI_1031 0xac13 #define PCI_DEVICE_ID_TI_1131 0xac15 @@ -2732,6 +2733,7 @@ #define PCI_DEVICE_ID_INTEL_HDA_HSW_3 0x0d0c #define PCI_DEVICE_ID_INTEL_HDA_BYT 0x0f04 #define PCI_DEVICE_ID_INTEL_SST_BYT 0x0f28 +#define PCI_DEVICE_ID_INTEL_ISP_BYT 0x0f38 #define PCI_DEVICE_ID_INTEL_8257X_SOL 0x1062 #define PCI_DEVICE_ID_INTEL_82573E_SOL 0x1085 #define PCI_DEVICE_ID_INTEL_82573L_SOL 0x108f @@ -2787,6 +2789,7 @@ #define PCI_DEVICE_ID_INTEL_VMD_201D 0x201d #define PCI_DEVICE_ID_INTEL_HDA_BSW 0x2284 #define PCI_DEVICE_ID_INTEL_SST_BSW 0x22a8 +#define PCI_DEVICE_ID_INTEL_ISP_BSW 0x22b8 #define PCI_DEVICE_ID_INTEL_DH89XXCC_LPC_MIN 0x2310 #define PCI_DEVICE_ID_INTEL_DH89XXCC_LPC_MAX 0x231f #define PCI_DEVICE_ID_INTEL_82801AA_0 0x2410 diff --git a/include/linux/pcs/pcs-xpcs-rk.h b/include/linux/pcs/pcs-xpcs-rk.h new file mode 100644 index 000000000000..e3ca3d51f7c7 --- /dev/null +++ b/include/linux/pcs/pcs-xpcs-rk.h @@ -0,0 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __LINUX_PCS_XPCS_ROCKCHIP_H +#define __LINUX_PCS_XPCS_ROCKCHIP_H + +#include <linux/device.h> +#include <linux/of.h> +#include <linux/pcs/pcs-xpcs.h> + +struct dw_xpcs *xpcs_rk_create(struct device *dev, struct device_node *pcs, int port); + +#endif /* __LINUX_PCS_XPCS_ROCKCHIP_H */ diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h index 2d5e7687875b..eb3f7bab498c 100644 --- a/include/linux/perf_event.h +++ b/include/linux/perf_event.h @@ -306,6 +306,7 @@ struct perf_event_pmu_context; #define PERF_PMU_CAP_AUX_PAUSE 0x0200 #define PERF_PMU_CAP_AUX_PREFER_LARGE 0x0400 #define PERF_PMU_CAP_MEDIATED_VPMU 0x0800 +#define PERF_PMU_CAP_SIMD_REGS 0x1000 /** * pmu::scope @@ -1467,6 +1468,7 @@ static inline u32 perf_sample_data_size(struct perf_sample_data *data, return size; } +extern u64 perf_update_xregs_size(struct perf_event *event, bool intr); extern void perf_output_sample(struct perf_output_handle *handle, struct perf_event_header *header, struct perf_sample_data *data, @@ -1518,6 +1520,27 @@ perf_event__output_id_sample(struct perf_event *event, extern void perf_log_lost_samples(struct perf_event *event, u64 lost); +static inline bool event_has_simd_regs(struct perf_event *event) +{ + struct perf_event_attr *attr = &event->attr; + + if (!(event->attr.sample_type & + (PERF_SAMPLE_REGS_INTR | PERF_SAMPLE_REGS_USER))) + return false; + + return attr->sample_simd_regs_enabled != 0; +} + +static inline bool event_has_extended_regs(struct perf_event *event) +{ + struct perf_event_attr *attr = &event->attr; + + return ((attr->sample_type & PERF_SAMPLE_REGS_USER) && + (attr->sample_regs_user & PERF_REG_EXTENDED_MASK)) || + ((attr->sample_type & PERF_SAMPLE_REGS_INTR) && + (attr->sample_regs_intr & PERF_REG_EXTENDED_MASK)); +} + static inline bool event_has_any_exclude_flag(struct perf_event *event) { struct perf_event_attr *attr = &event->attr; diff --git a/include/linux/perf_regs.h b/include/linux/perf_regs.h index f632c5725f16..09dbc2fc3859 100644 --- a/include/linux/perf_regs.h +++ b/include/linux/perf_regs.h @@ -9,6 +9,16 @@ struct perf_regs { struct pt_regs *regs; }; +u64 perf_reg_value(struct pt_regs *regs, int idx); +int perf_reg_validate(u64 mask, bool simd_enabled); +u64 perf_reg_abi(struct task_struct *task); +void perf_get_regs_user(struct perf_regs *regs_user, + struct pt_regs *regs); +int perf_simd_reg_validate(u16 vec_qwords, u64 vec_mask, + u16 pred_qwords, u32 pred_mask); +u64 perf_simd_reg_value(struct pt_regs *regs, int idx, + u16 qwords_idx, bool pred); + #ifdef CONFIG_HAVE_PERF_REGS #include <asm/perf_regs.h> @@ -16,35 +26,9 @@ struct perf_regs { #define PERF_REG_EXTENDED_MASK 0 #endif -u64 perf_reg_value(struct pt_regs *regs, int idx); -int perf_reg_validate(u64 mask); -u64 perf_reg_abi(struct task_struct *task); -void perf_get_regs_user(struct perf_regs *regs_user, - struct pt_regs *regs); #else #define PERF_REG_EXTENDED_MASK 0 -static inline u64 perf_reg_value(struct pt_regs *regs, int idx) -{ - return 0; -} - -static inline int perf_reg_validate(u64 mask) -{ - return mask ? -ENOSYS : 0; -} - -static inline u64 perf_reg_abi(struct task_struct *task) -{ - return PERF_SAMPLE_REGS_ABI_NONE; -} - -static inline void perf_get_regs_user(struct perf_regs *regs_user, - struct pt_regs *regs) -{ - regs_user->regs = task_pt_regs(current); - regs_user->abi = perf_reg_abi(current); -} #endif /* CONFIG_HAVE_PERF_REGS */ #endif /* _LINUX_PERF_REGS_H */ diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h index 8c093c119e5a..780fe849ff8b 100644 --- a/include/linux/pgtable.h +++ b/include/linux/pgtable.h @@ -490,35 +490,35 @@ static inline int pudp_set_access_flags(struct vm_area_struct *vma, #endif #ifndef ptep_get -static inline pte_t ptep_get(pte_t *ptep) +static inline pte_t ptep_get(const pte_t *ptep) { return READ_ONCE(*ptep); } #endif #ifndef pmdp_get -static inline pmd_t pmdp_get(pmd_t *pmdp) +static inline pmd_t pmdp_get(const pmd_t *pmdp) { return READ_ONCE(*pmdp); } #endif #ifndef pudp_get -static inline pud_t pudp_get(pud_t *pudp) +static inline pud_t pudp_get(const pud_t *pudp) { return READ_ONCE(*pudp); } #endif #ifndef p4dp_get -static inline p4d_t p4dp_get(p4d_t *p4dp) +static inline p4d_t p4dp_get(const p4d_t *p4dp) { return READ_ONCE(*p4dp); } #endif #ifndef pgdp_get -static inline pgd_t pgdp_get(pgd_t *pgdp) +static inline pgd_t pgdp_get(const pgd_t *pgdp) { return READ_ONCE(*pgdp); } @@ -2313,6 +2313,20 @@ static inline const char *pgtable_level_to_str(enum pgtable_level level) } } +void ptval_bytes_to_hex_str(char *buf, size_t buf_size, const void *entry, size_t entry_size); + +#define ptval_to_str(buf, val) \ + do { \ + auto __val = (val); \ + \ + ptval_bytes_to_hex_str((buf), sizeof(buf), &__val, sizeof(__val)); \ + } while (0) + +#if defined(__SIZEOF_INT128__) +#define PTVAL_STR_MAX (32 + 1) /* Max 128-bit value in hex + NUL */ +#else +#define PTVAL_STR_MAX (16 + 1) /* Max 64-bit value in hex + NUL */ +#endif #endif /* !__ASSEMBLER__ */ #if !defined(MAX_POSSIBLE_PHYSMEM_BITS) && !defined(CONFIG_64BIT) diff --git a/include/linux/phy.h b/include/linux/phy.h index 5f8d65868e0f..21afc2082b17 100644 --- a/include/linux/phy.h +++ b/include/linux/phy.h @@ -376,6 +376,24 @@ struct mii_bus { int regnum, u16 val); /** @reset: Perform a reset of the bus */ int (*reset)(struct mii_bus *bus); + /** + * @notify_phy_attach: Perform post-attach handling for MDIO bus + * drivers. Optional and independent of @notify_phy_detach. Called + * in phy_attach_direct() right before phy_resume(). Runs in process + * context, may sleep and may be called with RTNL held. Must not + * acquire or rely on RTNL. Returns 0 on success or negative errno + * on failure. Must unwind its own state on error as attachment is + * aborted. + */ + int (*notify_phy_attach)(struct phy_device *phydev); + /** + * @notify_phy_detach: Perform pre-detach handling for MDIO bus + * drivers. Optional and independent of @notify_phy_attach. Called + * in phy_detach() right after phy_suspend(). Runs in process context, + * may sleep and may be called with RTNL held. Must not acquire or + * rely on RTNL. + */ + void (*notify_phy_detach)(struct phy_device *phydev); /** @stats: Statistic counters per device on the bus */ struct mdio_bus_stats stats[PHY_MAX_ADDR]; @@ -560,6 +578,7 @@ struct phy_oatc14_sqi_capability { * * @mdio: MDIO bus this PHY is on * @drv: Pointer to the driver for this PHY instance + * @drv_owner: Driver module phy_attach_direct() took a reference on * @devlink: Create a link between phy dev and mac dev, if the external phy * used by current mac interface is managed by another mac interface. * @phyindex: Unique id across the phy's parent tree of phys to address the PHY @@ -593,7 +612,11 @@ struct phy_oatc14_sqi_capability { * @irq: IRQ number of the PHY's interrupt (-1 if none) * @phylink: Pointer to phylink instance for this PHY * @sfp_bus_attached: Flag indicating whether the SFP bus has been attached + * @has_sfp_mod_phy: Set true if downstream SFP bus's module contains a PHY * @sfp_bus: SFP bus attached to this PHY's fiber port + * @sfp_cage_port: The phy_port connected to the downstream SFP cage + * @mod_port: phy_port representing the SFP module, if it is phy-less + * @upstream_port: phy_port this PHY's MII attaches to, if any * @attached_dev: The attached enet driver's device instance ptr * @adjust_link: Callback for the enet controller to respond to changes: in the * link state. @@ -671,6 +694,7 @@ struct phy_device { /* Information about the PHY type */ /* And management functions */ const struct phy_driver *drv; + struct module *drv_owner; struct device_link *devlink; @@ -784,7 +808,11 @@ struct phy_device { /* This may be modified under the rtnl lock */ bool sfp_bus_attached; + bool has_sfp_mod_phy; struct sfp_bus *sfp_bus; + struct phy_port *sfp_cage_port; + struct phy_port *mod_port; + struct phy_port *upstream_port; struct phylink *phylink; struct net_device *attached_dev; struct mii_timestamper *mii_ts; @@ -1697,7 +1725,7 @@ static inline bool phy_can_wakeup(struct phy_device *phydev) * phy_may_wakeup() - indicate whether PHY has wakeup enabled * @phydev: The phy_device struct * - * Returns: true/false depending on the PHY driver's device_set_wakeup_enabled() + * Returns: true/false depending on the PHY driver's device_set_wakeup_enable() * setting if using the driver model, otherwise the legacy determination. */ bool phy_may_wakeup(struct phy_device *phydev); @@ -2422,10 +2450,10 @@ int phy_get_mac_termination(struct phy_device *phydev, struct device *dev, void phy_resolve_pause(unsigned long *local_adv, unsigned long *partner_adv, bool *tx_pause, bool *rx_pause); -int phy_register_fixup_for_id(const char *bus_id, - int (*run)(struct phy_device *)); -int phy_register_fixup_for_uid(u32 phy_uid, u32 phy_uid_mask, - int (*run)(struct phy_device *)); +void __init phy_register_fixup_for_id(const char *bus_id, + int (*run)(struct phy_device *)); +void __init phy_register_fixup_for_uid(u32 phy_uid, u32 phy_uid_mask, + int (*run)(struct phy_device *)); int phy_eee_tx_clock_stop_capable(struct phy_device *phydev); int phy_eee_rx_clock_stop(struct phy_device *phydev, bool clk_stop_enable); @@ -2469,6 +2497,8 @@ int __phy_hwtstamp_set(struct phy_device *phydev, struct phy_port *phy_get_sfp_port(struct phy_device *phydev); +void phy_set_upstream_port(struct phy_device *phydev, struct phy_port *port); + /** * phy_module_driver() - Helper macro for registering PHY drivers * @__phy_drivers: array of PHY drivers to register diff --git a/include/linux/phy/omap_control_phy.h b/include/linux/phy/omap_control_phy.h index aec57dd784f7..5652aa2f4915 100644 --- a/include/linux/phy/omap_control_phy.h +++ b/include/linux/phy/omap_control_phy.h @@ -12,7 +12,7 @@ enum omap_control_phy_type { OMAP_CTRL_TYPE_OTGHS = 1, /* Mailbox OTGHS_CONTROL */ OMAP_CTRL_TYPE_USB2, /* USB2_PHY, power down in CONTROL_DEV_CONF */ - OMAP_CTRL_TYPE_PIPE3, /* PIPE3 PHY, DPLL & seperate Rx/Tx power */ + OMAP_CTRL_TYPE_PIPE3, /* PIPE3 PHY, DPLL & separate Rx/Tx power */ OMAP_CTRL_TYPE_PCIE, /* RX TX control of ACSPCIE */ OMAP_CTRL_TYPE_DRA7USB2, /* USB2 PHY, power and power_aux e.g. DRA7 */ OMAP_CTRL_TYPE_AM437USB2, /* USB2 PHY, power e.g. AM437x */ diff --git a/include/linux/phy/phy-hdmi.h b/include/linux/phy/phy-hdmi.h index d4cf4430ee8f..1d4b62475079 100644 --- a/include/linux/phy/phy-hdmi.h +++ b/include/linux/phy/phy-hdmi.h @@ -19,6 +19,10 @@ enum phy_hdmi_mode { * @tmds_char_rate: HDMI TMDS Character Rate in Hertz. * @frl.rate_per_lane: HDMI FRL Rate per Lane in Gbps. * @frl.lanes: HDMI FRL lanes count. + * @frl.ffe_level: Transmitter Feed Forward Equalizer Level. + * Optional, only meaningful when set_ffe_level flag is on. + * @frl.set_ffe_level: Flag indicating whether or not to reconfigure ffe_level. + * All the other struct fields must be ignored when this is used. * * This structure is used to represent the configuration state of a HDMI phy. */ @@ -29,6 +33,8 @@ struct phy_configure_opts_hdmi { struct { u8 rate_per_lane; u8 lanes; + u8 ffe_level; + u8 set_ffe_level : 1; } frl; }; }; diff --git a/include/linux/phy/phy-thunderbolt.h b/include/linux/phy/phy-thunderbolt.h new file mode 100644 index 000000000000..5f48059814e1 --- /dev/null +++ b/include/linux/phy/phy-thunderbolt.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#ifndef __PHY_TBT_H_ +#define __PHY_TBT_H_ + +enum tbt_phy_submode { + PHY_SUBMODE_TBT3, + PHY_SUBMODE_USB4, +}; + +#endif diff --git a/include/linux/phy/phy.h b/include/linux/phy/phy.h index ea47975e288a..c480d6cb42c5 100644 --- a/include/linux/phy/phy.h +++ b/include/linux/phy/phy.h @@ -20,6 +20,7 @@ #include <linux/phy/phy-hdmi.h> #include <linux/phy/phy-lvds.h> #include <linux/phy/phy-mipi-dphy.h> +#include <linux/phy/phy-thunderbolt.h> struct phy; @@ -45,6 +46,7 @@ enum phy_mode { PHY_MODE_LVDS, PHY_MODE_DP, PHY_MODE_HDMI, + PHY_MODE_TBT, }; enum phy_media { @@ -284,6 +286,8 @@ struct phy *devm_of_phy_optional_get(struct device *dev, struct device_node *np, const char *con_id); struct phy *devm_of_phy_get_by_index(struct device *dev, struct device_node *np, int index); +struct phy *phy_get_by_of_node(struct device_node *np); +struct phy *devm_phy_get_by_of_node(struct device *dev, struct device_node *np); void of_phy_put(struct phy *phy); void phy_put(struct device *dev, struct phy *phy); void devm_phy_put(struct device *dev, struct phy *phy); @@ -493,6 +497,17 @@ static inline struct phy *devm_of_phy_get_by_index(struct device *dev, return ERR_PTR(-ENOSYS); } +static inline struct phy *phy_get_by_of_node(struct device_node *np) +{ + return ERR_PTR(-ENOSYS); +} + +static inline struct phy *devm_phy_get_by_of_node(struct device *dev, + struct device_node *np) +{ + return ERR_PTR(-ENOSYS); +} + static inline void of_phy_put(struct phy *phy) { } diff --git a/include/linux/phy_link_topology.h b/include/linux/phy_link_topology.h index 95575f68d5bc..f2e493c433ff 100644 --- a/include/linux/phy_link_topology.h +++ b/include/linux/phy_link_topology.h @@ -13,14 +13,19 @@ #include <linux/ethtool.h> #include <linux/netdevice.h> +#include <linux/rtnetlink.h> struct xarray; struct phy_device; +struct phy_port; struct sfp_bus; struct phy_link_topology { struct xarray phys; u32 next_phy_index; + + struct xarray ports; + u32 next_port_index; }; struct phy_device_node { @@ -48,6 +53,9 @@ int phy_link_topo_add_phy(struct net_device *dev, void phy_link_topo_del_phy(struct net_device *dev, struct phy_device *phy); +int phy_link_topo_add_port(struct net_device *dev, struct phy_port *port); +void phy_link_topo_del_port(struct net_device *dev, struct phy_port *port); + static inline struct phy_device * phy_link_topo_get_phy(struct net_device *dev, u32 phyindex) { @@ -64,6 +72,24 @@ phy_link_topo_get_phy(struct net_device *dev, u32 phyindex) return NULL; } +static inline struct phy_port * +phy_link_topo_get_port(struct net_device *dev, u32 port_id) +{ + struct phy_link_topology *topo = dev->link_topo; + + if (!topo) + return NULL; + + /* ports in the topo are RTNL protected, but may be accessed under + * netdev_lock for ops-locked devices. For now keep them under rtnl + * protection, as no ops-locked devices have phy_port capabilities + * yet. + */ + ASSERT_RTNL(); + + return xa_load(&topo->ports, port_id); +} + #else static inline int phy_link_topo_add_phy(struct net_device *dev, struct phy_device *phy, @@ -77,11 +103,28 @@ static inline void phy_link_topo_del_phy(struct net_device *dev, { } +static inline int phy_link_topo_add_port(struct net_device *dev, + struct phy_port *port) +{ + return 0; +} + +static inline void phy_link_topo_del_port(struct net_device *dev, + struct phy_port *port) +{ +} + static inline struct phy_device * phy_link_topo_get_phy(struct net_device *dev, u32 phyindex) { return NULL; } + +static inline struct phy_port * +phy_link_topo_get_port(struct net_device *dev, u32 port_id) +{ + return NULL; +} #endif #endif /* __PHY_LINK_TOPOLOGY_H */ diff --git a/include/linux/phy_port.h b/include/linux/phy_port.h index 0ef0f5ce4709..8f45c031cbc1 100644 --- a/include/linux/phy_port.h +++ b/include/linux/phy_port.h @@ -36,9 +36,12 @@ struct phy_port_ops { /** * struct phy_port - A representation of a network device physical interface * + * @id: Unique identifier for the port within the topology * @head: Used by the port's parent to list ports * @parent_type: The type of device this port is directly connected to * @phy: If the parent is PHY_PORT_PHYDEV, the PHY controlling that port + * @upstream_port: Indicates the MII port that feeds this port, if any, + * e.g. the SFP cage port for a SFP module port. * @ops: Callback ops implemented by the port controller * @pairs: The number of pairs this port has, 0 if not applicable * @mediums: Bitmask of the physical mediums this port provides access to @@ -52,11 +55,13 @@ struct phy_port_ops { * @is_sfp: Indicates if this port drives an SFP cage. */ struct phy_port { + u32 id; struct list_head head; enum phy_port_parent parent_type; union { struct phy_device *phy; }; + struct phy_port *upstream_port; const struct phy_port_ops *ops; diff --git a/include/linux/phylink.h b/include/linux/phylink.h index 1dda5c7ed5f1..3a88a69882a6 100644 --- a/include/linux/phylink.h +++ b/include/linux/phylink.h @@ -843,4 +843,6 @@ void phylink_replay_link_begin(struct phylink *pl); void phylink_replay_link_end(struct phylink *pl); +void phylink_update_mac_pause_capabilities(struct phylink *pl, unsigned long mac_pause); + #endif diff --git a/include/linux/pid.h b/include/linux/pid.h index ddaef0bbc8ba..185bf0cde2d1 100644 --- a/include/linux/pid.h +++ b/include/linux/pid.h @@ -2,6 +2,9 @@ #ifndef _LINUX_PID_H #define _LINUX_PID_H +#include <linux/array_size.h> +#include <linux/build_bug.h> +#include <linux/minmax.h> #include <linux/pid_types.h> #include <linux/rculist.h> #include <linux/rcupdate.h> @@ -92,6 +95,56 @@ static inline struct pid *get_pid(struct pid *pid) } extern void put_pid(struct pid *pid); + +/* + * Helpers for arrays of struct pid indexed by pid type declared with + * DECLARE_PIDS(). The array covers PIDTYPE_PID up to the pid type it + * was declared with and the helpers take that bound from the array. + */ +static inline void __get_pids(struct pid **dst, struct pid *const *src, + enum pid_type last) +{ + for (enum pid_type type = PIDTYPE_PID; type <= last; type++) + dst[type] = get_pid(src[type]); +} + +static inline void __put_pids(struct pid **pids, enum pid_type last) +{ + for (enum pid_type type = PIDTYPE_PID; type <= last; type++) { + put_pid(pids[type]); + pids[type] = NULL; + } +} + +static inline void __swap_pids(struct pid **a, struct pid **b, + enum pid_type last) +{ + for (enum pid_type type = PIDTYPE_PID; type <= last; type++) + swap(a[type], b[type]); +} + +static inline bool __pids_equal(struct pid *const *a, struct pid *const *b, + enum pid_type last) +{ + for (enum pid_type type = PIDTYPE_PID; type <= last; type++) + if (a[type] != b[type]) + return false; + return true; +} + +/* The last pid type an array declared with DECLARE_PIDS() covers. */ +#define pids_last(pids) \ + ((enum pid_type)(ARRAY_SIZE(pids) - 1 + \ + BUILD_BUG_ON_ZERO(ARRAY_SIZE(pids) > PIDTYPE_MAX))) + +#define __pids_last2(a, b) \ + (pids_last(a) + BUILD_BUG_ON_ZERO(ARRAY_SIZE(a) != ARRAY_SIZE(b))) + +#define get_pids(dst, src) __get_pids(dst, src, __pids_last2(dst, src)) +#define put_pids(pids) __put_pids(pids, pids_last(pids)) +#define swap_pids(a, b) __swap_pids(a, b, __pids_last2(a, b)) +#define pids_equal(a, b) __pids_equal(a, b, __pids_last2(a, b)) + extern struct task_struct *pid_task(struct pid *pid, enum pid_type); static inline bool pid_has_task(struct pid *pid, enum pid_type type) { @@ -187,6 +240,15 @@ static inline pid_t pid_nr(struct pid *pid) pid_t pid_nr_ns(struct pid *pid, struct pid_namespace *ns); pid_t pid_vnr(struct pid *pid); +/* + * Needs tasklist_lock if type > PIDTYPE_TGID. A lockless reader can: + * + * - race with a change_pid() that moves the task to another hlist, + * and continue to scan the wrong list. + * + * - race with a copy_process() that adds a new task to the start of hlist, + * and miss the new child. + */ #define do_each_pid_task(pid, type, task) \ do { \ if ((pid) != NULL) \ @@ -264,7 +326,7 @@ static inline pid_t task_tgid_nr(struct task_struct *tsk) */ static inline int pid_alive(const struct task_struct *p) { - return p->thread_pid != NULL; + return READ_ONCE(p->thread_pid) != NULL; } static inline pid_t task_pgrp_nr_ns(struct task_struct *tsk, struct pid_namespace *ns) diff --git a/include/linux/pid_types.h b/include/linux/pid_types.h index c2aee1d91dcf..3302690a2a28 100644 --- a/include/linux/pid_types.h +++ b/include/linux/pid_types.h @@ -2,6 +2,8 @@ #ifndef _LINUX_PID_TYPES_H #define _LINUX_PID_TYPES_H +#include <linux/build_bug.h> + enum pid_type { PIDTYPE_PID, PIDTYPE_TGID, @@ -10,6 +12,12 @@ enum pid_type { PIDTYPE_MAX, }; +struct pid; + +/* An array of struct pid indexed by pid type, PIDTYPE_PID up to @last. */ +#define DECLARE_PIDS(name, last) \ + struct pid *name[(last) + 1 + BUILD_BUG_ON_ZERO((last) >= PIDTYPE_MAX)] + struct pid_namespace; extern struct pid_namespace init_pid_ns; diff --git a/include/linux/pidfs.h b/include/linux/pidfs.h index 0abf7da9ab23..3c1e82f1a369 100644 --- a/include/linux/pidfs.h +++ b/include/linux/pidfs.h @@ -3,6 +3,7 @@ #define _LINUX_PID_FS_H #include <linux/gfp_types.h> +#include <linux/pid.h> struct coredump_params; @@ -32,5 +33,9 @@ static inline int pidfs_register_pid(struct pid *pid) } void pidfs_free_pid(struct pid *pid); +int __pidfs_register_pids(struct pid *const *pids, enum pid_type last); + +/* Register the pids of an array declared with DECLARE_PIDS(). */ +#define pidfs_register_pids(pids) __pidfs_register_pids(pids, pids_last(pids)) #endif /* _LINUX_PID_FS_H */ diff --git a/include/linux/pinctrl/consumer.h b/include/linux/pinctrl/consumer.h index 11b8f0b8da0c..81e7866bd2e3 100644 --- a/include/linux/pinctrl/consumer.h +++ b/include/linux/pinctrl/consumer.h @@ -12,6 +12,7 @@ #define __LINUX_PINCTRL_CONSUMER_H #include <linux/err.h> +#include <linux/errno.h> #include <linux/types.h> #include <linux/pinctrl/pinctrl-state.h> @@ -107,14 +108,14 @@ static inline int pinctrl_gpio_get_config(struct gpio_chip *gc, unsigned int offset, unsigned long *config) { - return 0; + return -ENOTSUPP; } static inline int pinctrl_gpio_set_config(struct gpio_chip *gc, unsigned int offset, unsigned long config) { - return 0; + return -ENOTSUPP; } static inline struct pinctrl * __must_check pinctrl_get(struct device *dev) diff --git a/include/linux/pinctrl/pinconf-generic.h b/include/linux/pinctrl/pinconf-generic.h index a5d4b2d8633a..1df63c732b99 100644 --- a/include/linux/pinctrl/pinconf-generic.h +++ b/include/linux/pinctrl/pinconf-generic.h @@ -71,6 +71,10 @@ struct pinctrl_map; * which means it will wait for signals to settle when reading inputs. The * argument gives the debounce time in usecs. Setting the * argument to zero turns debouncing off. + * @PIN_CONFIG_INPUT_DEBOUNCE_NS: this will configure the pin to debounce mode, + * which means it will wait for signals to settle when reading inputs. The + * argument gives the debounce time in nsecs. Setting the + * argument to zero turns debouncing off. * @PIN_CONFIG_INPUT_ENABLE: enable the pin's input. Note that this does not * affect the pin's ability to drive output. 1 enables input, 0 disables * input. @@ -143,6 +147,7 @@ enum pin_config_param { PIN_CONFIG_DRIVE_STRENGTH, PIN_CONFIG_DRIVE_STRENGTH_UA, PIN_CONFIG_INPUT_DEBOUNCE, + PIN_CONFIG_INPUT_DEBOUNCE_NS, PIN_CONFIG_INPUT_ENABLE, PIN_CONFIG_INPUT_SCHMITT, PIN_CONFIG_INPUT_SCHMITT_ENABLE, diff --git a/include/linux/platform_data/cros_ec_commands.h b/include/linux/platform_data/cros_ec_commands.h index 749d985e9da2..a7bd69d83522 100644 --- a/include/linux/platform_data/cros_ec_commands.h +++ b/include/linux/platform_data/cros_ec_commands.h @@ -1344,6 +1344,26 @@ enum ec_feature_code { * The EC supports UCSI PPM. */ EC_FEATURE_UCSI_PPM = 54, + /* + * The EC supports Strauss keyboard. + */ + EC_FEATURE_STRAUSS = 55, + /* + * The EC supports PoE. + */ + EC_FEATURE_POE = 56, + /* + * The EC supports a hybrid boost charger + */ + EC_FEATURE_CHARGER_HYBRID_POWER_BOOST = 57, + /* + * Support signaling new console logs via host event + */ + EC_FEATURE_CONSOLE_LOG_EVENT = 58, + /* + * The EC supports power monitoring + */ + EC_FEATURE_PWRMON = 59, }; #define EC_FEATURE_MASK_0(event_code) BIT(event_code % 32) @@ -3516,6 +3536,9 @@ enum ec_mkbp_event { /* Peripheral device charger event */ EC_MKBP_EVENT_PCHG = 12, + /* Power monitor telemetry event */ + EC_MKBP_EVENT_PWRMON = 13, + /* Number of MKBP events */ EC_MKBP_EVENT_COUNT, }; @@ -3599,6 +3622,12 @@ union __ec_align_offset1 ec_response_get_next_data_v3 { uint32_t cec_events; uint8_t cec_message[16]; + + struct __ec_todo_packed { + int64_t value; + uint32_t samples; + uint8_t channel_id; + } pwrmon_data; }; BUILD_ASSERT(sizeof(union ec_response_get_next_data_v3) == 18); @@ -6589,6 +6618,49 @@ struct ec_params_usb_pd_mux_ack { uint8_t port; /* USB-C port number */ } __ec_align1; +/** + * Power monitoring. Used to read power consumptions on rails + */ +#define EC_CMD_PWRMON 0x0608 + +enum ec_pwrmon_cmd { + EC_PWRMON_GET_CHANNEL_COUNT = 0, + EC_PWRMON_DUMP_INFO = 1, + EC_PWRMON_SET_RATE = 2, + EC_PWRMON_GET_RATE = 3, + EC_PWRMON_START = 4, + EC_PWRMON_STOP = 5, + EC_PWRMON_LATCH = 6, +}; + +struct ec_params_pwrmon { + uint8_t cmd; + union { + uint16_t set_rate; + uint8_t channel_id; + } __ec_align2; + + /* + * The following commands have no args: + * + * start, stop, latch + * + */ +} __ec_align4; + +struct pwrmon_dump_info { + uint8_t channel_id; + char channel_name[32]; +} __ec_align4; + +struct ec_response_pwrmon { + union { + uint16_t sample_rate; + uint8_t channel_count; + struct pwrmon_dump_info dump_info; + } __ec_align4; +} __ec_align4; + /*****************************************************************************/ /* * Reserve a range of host commands for board-specific, experimental, or diff --git a/include/linux/platform_data/dmtimer-omap.h b/include/linux/platform_data/dmtimer-omap.h index 726d89143842..b78ee4676b98 100644 --- a/include/linux/platform_data/dmtimer-omap.h +++ b/include/linux/platform_data/dmtimer-omap.h @@ -57,6 +57,8 @@ struct dmtimer_platform_data { u32 timer_errata; int (*get_context_loss_count)(struct device *); const struct omap_dm_timer_ops *timer_ops; + /* always-on timers can be used for clocksource/clockevent */ + bool system_timer_capable; }; #endif /* __PLATFORM_DATA_DMTIMER_OMAP_H__ */ diff --git a/include/linux/platform_data/microchip-ksz.h b/include/linux/platform_data/microchip-ksz.h index 028781ad4059..411d5e164383 100644 --- a/include/linux/platform_data/microchip-ksz.h +++ b/include/linux/platform_data/microchip-ksz.h @@ -31,9 +31,11 @@ enum ksz_chip_id { KSZ88X3_CHIP_ID = 0x8830, KSZ8864_CHIP_ID = 0x8864, KSZ8895_CHIP_ID = 0x8895, + KSZ8995XA_CHIP_ID = 0x8995, KSZ9477_CHIP_ID = 0x00947700, KSZ9896_CHIP_ID = 0x00989600, KSZ9897_CHIP_ID = 0x00989700, + KSZ9897S_CHIP_ID = 0x00989701, KSZ9893_CHIP_ID = 0x00989300, KSZ9563_CHIP_ID = 0x00956300, KSZ8567_CHIP_ID = 0x00856700, diff --git a/include/linux/platform_data/mipi-i3c-hci.h b/include/linux/platform_data/mipi-i3c-hci.h index ab7395f455f9..6f61d8a02378 100644 --- a/include/linux/platform_data/mipi-i3c-hci.h +++ b/include/linux/platform_data/mipi-i3c-hci.h @@ -3,13 +3,17 @@ #define INCLUDE_PLATFORM_DATA_MIPI_I3C_HCI_H #include <linux/compiler_types.h> +#include <linux/types.h> /** * struct mipi_i3c_hci_platform_data - Platform-dependent data for mipi_i3c_hci * @base_regs: Register set base address (to support multi-bus instances) + * @instance: Zero-based instance number of the Bus Controller as defined by the + * DisCo specification I3C Target Address (_ADR) Encoding */ struct mipi_i3c_hci_platform_data { void __iomem *base_regs; + u8 instance; }; #endif diff --git a/include/linux/platform_data/x86/asus-wmi.h b/include/linux/platform_data/x86/asus-wmi.h index b5ed8c83ace1..66c7ab35d6c8 100644 --- a/include/linux/platform_data/x86/asus-wmi.h +++ b/include/linux/platform_data/x86/asus-wmi.h @@ -54,6 +54,7 @@ #define ASUS_WMI_DEVID_LED5 0x00020015 #define ASUS_WMI_DEVID_LED6 0x00020016 #define ASUS_WMI_DEVID_MICMUTE_LED 0x00040017 +#define ASUS_WMI_DEVID_MUTE_LED 0x0004001C /* Disable Camera LED */ #define ASUS_WMI_DEVID_CAMERA_LED_NEG 0x00060078 /* 0 = on (unused) */ @@ -138,10 +139,14 @@ /* dgpu on/off */ #define ASUS_WMI_DEVID_DGPU 0x00090020 +/* dgpu on/off - alternative to ASUS_WMI_DEVID_DGPU */ +#define ASUS_WMI_DEVID_GPU_MODE 0x00090120 + #define ASUS_WMI_DEVID_APU_MEM 0x000600C1 -#define ASUS_WMI_DEVID_DGPU_BASE_TGP 0x00120099 +#define ASUS_WMI_DEVID_DGPU_POWER_STATE 0x00120097 #define ASUS_WMI_DEVID_DGPU_SET_TGP 0x00120098 +#define ASUS_WMI_DEVID_DGPU_BASE_TGP 0x00120099 /* gpu mux switch, 0 = dGPU, 1 = Optimus */ #define ASUS_WMI_DEVID_GPU_MUX 0x00090016 @@ -159,7 +164,7 @@ #define ASUS_WMI_DEVID_TUF_RGB_MODE2 0x0010005A /* TUF laptop RGB power/state */ -#define ASUS_WMI_DEVID_TUF_RGB_STATE 0x00100057 +#define ASUS_WMI_DEVID_TUF_RGB_CMD 0x00100057 /* Bootup sound control */ #define ASUS_WMI_DEVID_BOOT_SOUND 0x00130022 diff --git a/include/linux/platform_data/x86/int3472.h b/include/linux/platform_data/x86/int3472.h index a73841dfae27..b1040e36deb8 100644 --- a/include/linux/platform_data/x86/int3472.h +++ b/include/linux/platform_data/x86/int3472.h @@ -25,6 +25,8 @@ #define INT3472_GPIO_TYPE_RESET 0x00 #define INT3472_GPIO_TYPE_POWERDOWN 0x01 #define INT3472_GPIO_TYPE_STROBE 0x02 +#define INT3472_GPIO_TYPE_POWER0 0x07 +#define INT3472_GPIO_TYPE_POWER1 0x08 #define INT3472_GPIO_TYPE_POWER_ENABLE 0x0b #define INT3472_GPIO_TYPE_CLK_ENABLE 0x0c #define INT3472_GPIO_TYPE_PRIVACY_LED 0x0d diff --git a/include/linux/platform_device.h b/include/linux/platform_device.h index 3d5bbcbae730..3bd0f1f0d870 100644 --- a/include/linux/platform_device.h +++ b/include/linux/platform_device.h @@ -344,6 +344,19 @@ static inline void platform_set_drvdata(struct platform_device *pdev, #define builtin_platform_driver(__platform_driver) \ builtin_driver(__platform_driver, platform_driver_register) +/* + * subsys_platform_driver() - Helper macro for drivers that don't do anything + * special in module init/exit but need to register earlier, at + * subsys_initcall level, when built in. This eliminates a lot of + * boilerplate. Each driver may only use this macro once, and calling it + * replaces module_init() and module_exit(). This is meant to be a parallel + * of module_platform_driver() above, but with the init call promoted to + * subsys_initcall() so built-in providers are available earlier during boot. + */ +#define subsys_platform_driver(__platform_driver) \ + subsys_driver(__platform_driver, platform_driver_register, \ + platform_driver_unregister) + /* module_platform_driver_probe() - Helper macro for drivers that don't do * anything special in module init/exit. This eliminates a lot of * boilerplate. Each module may only use this macro once, and @@ -376,6 +389,23 @@ static int __init __platform_driver##_init(void) \ } \ device_initcall(__platform_driver##_init); \ +/* + * subsys_platform_driver_probe() - Helper macro for drivers that don't do + * anything special in device init and have no exit, but need to register + * earlier, at subsys_initcall level. This eliminates some boilerplate. Each + * driver may only use this macro once, and using it replaces subsys_initcall. + * This is meant to be a parallel of builtin_platform_driver_probe above, but + * with the init call promoted to subsys_initcall so the provider is available + * earlier during boot. + */ +#define subsys_platform_driver_probe(__platform_driver, __platform_probe) \ +static int __init __platform_driver##_init(void) \ +{ \ + return platform_driver_probe(&(__platform_driver), \ + __platform_probe); \ +} \ +subsys_initcall(__platform_driver##_init) \ + #define platform_create_bundle(driver, probe, res, n_res, data, size) \ __platform_create_bundle(driver, probe, res, n_res, data, size, THIS_MODULE, KBUILD_MODNAME) extern struct platform_device *__platform_create_bundle( diff --git a/include/linux/pm.h b/include/linux/pm.h index afcaaa37a812..ef3f1310e749 100644 --- a/include/linux/pm.h +++ b/include/linux/pm.h @@ -663,6 +663,99 @@ struct pm_subsys_data { #define DPM_FLAG_SMART_SUSPEND BIT(2) #define DPM_FLAG_MAY_SKIP_RESUME BIT(3) +/** + * struct dev_pm_info - Device power management information. + * + * @power_state: Legacy power state (mostly unused in modern kernels). + * @can_wakeup: Device is capable of generating wakeup signals. + * @async_suspend: Device can be suspended and resumed asynchronously. + * @in_dpm_list: Device is on the dpm_list. + * @is_prepared: Device's ->prepare() callback has run successfully. + * @is_suspended: Device is suspended during a system sleep transition. + * @is_noirq_suspended: Device's noirq suspend callback has run successfully. + * @is_late_suspended: Device's late suspend callback has run successfully. + * @no_pm: Device does not participate in power management transitions. + * @early_init: Device was initialized before standard PM initialization. + * @direct_complete: Device can skip suspend/resume callbacks and remain + * runtime-suspended during system sleep. + * @driver_flags: Driver flags (e.g. %DPM_FLAG_SMART_SUSPEND) set at probe time. + * @lock: Spinlock used for synchronizing PM state transitions and runtime PM + * operations. + * @entry: List head for device power management lists. + * @completion: Completion for synchronization during asynchronous system + * suspend/resume. + * @wakeup: Wakeup source object associated with the device. + * @work_in_progress: Asynchronous PM operation in progress. + * @wakeup_path: Device is in the wakeup path or can wake the system up. + * @syscore: Device participates in syscore power management operations. + * @no_pm_callbacks: Device has no PM callbacks; handled by parent or subsystem. + * @smart_suspend: Driver requested smart-suspend behavior. + * @must_resume: Device must be resumed during system resume. + * @may_skip_resume: Set by subsystems to indicate driver resume callbacks may + * be skipped. + * @out_band_wakeup: Out-of-band wakeup is supported. + * @strict_midlayer: Middle layer code does not want callbacks invoked via + * pm_runtime_force_suspend() / pm_runtime_force_resume(). + * @should_wakeup: Wakeup flag when system sleep is not enabled. + * @suspend_timer: High-resolution timer used for scheduling delayed runtime + * suspend and autosuspend requests. + * @timer_expires: Timer expiration time in nanoseconds monotonic time + * (runtime PM). + * @work: Work structure used for queuing up requests into pm_wq (runtime PM). + * @wait_queue: Wait queue used if any helper functions need to wait for another + * state change to complete (runtime PM). + * @wakeirq: Dedicated wakeup interrupt for the device. + * @usage_count: Device runtime PM usage counter. + * @child_count: Count of active children of the device (runtime PM). + * @disable_depth: Disable counter for runtime PM (runtime PM is enabled when + * this is 0; initial value is 1). + * @idle_notification: Set if ->runtime_idle() is being executed. + * @request_pending: Set if a work item is queued into pm_wq (runtime PM). + * @deferred_resume: Set if ->runtime_resume() should run as soon as + * ->runtime_suspend() completes. + * @needs_force_resume: Indicates the device was forced into suspend by + * pm_runtime_force_suspend() and must be resumed by + * pm_runtime_force_resume(). + * @runtime_auto: User space has allowed the driver to power manage the device + * at runtime via sysfs control attribute; also can be set by + * pm_runtime_allow() or pm_runtime_forbid(). + * @ignore_children: If set, the value of child_count is ignored for runtime + * suspend and idle decisions. + * @no_callbacks: Indicates the device does not use runtime PM callbacks. + * @irq_safe: Indicates runtime PM callbacks will be invoked with the spinlock + * held and interrupts disabled. + * @use_autosuspend: Indicates the device driver supports delayed runtime + * autosuspend. + * @timer_autosuspends: Indicates the runtime PM core should attempt an + * autosuspend rather than a normal suspend when the timer expires. + * @memalloc_noio: Indicates memory allocation during runtime PM transitions + * must avoid I/O (GFP_NOIO). + * @links_count: Number of device links that require runtime PM coordination. + * @request: Type of pending runtime PM request (valid if request_pending is + * set). + * @runtime_status: Runtime PM status of the device. + * @last_status: Last status captured before disabling runtime PM, or + * %RPM_BLOCKED / %RPM_INVALID. + * @runtime_error: Fatal error code returned by a failing callback, blocking + * helpers until cleared. + * @autosuspend_delay: Delay time in milliseconds to be used for runtime + * autosuspend. + * @last_busy: Timestamp in nanoseconds when pm_runtime_mark_last_busy() was + * last called. Used in calculating inactivity periods for autosuspend. + * @active_time: Accumulated time in nanoseconds spent in %RPM_ACTIVE state. + * @suspended_time: Accumulated time in nanoseconds spent in %RPM_SUSPENDED + * state. + * @accounting_timestamp: Timestamp in nanoseconds of the last runtime PM state + * accounting update. + * @subsys_data: Subsystem-specific power management data. + * @set_latency_tolerance: Callback for setting latency tolerance. + * @qos: Per-device PM Quality of Service (QoS) constraints. + * @detach_power_off: Indicates device should be detached from PM domain on + * power off. + * + * Device power management information stored in the "power" member of struct + * device. + */ struct dev_pm_info { pm_message_t power_state; bool can_wakeup:1; diff --git a/include/linux/pm_domain.h b/include/linux/pm_domain.h index f925614aebdb..14e0e346c610 100644 --- a/include/linux/pm_domain.h +++ b/include/linux/pm_domain.h @@ -121,6 +121,14 @@ struct dev_pm_domain_list { * powered-off until the ->sync_state() callback is * invoked. This flag informs genpd to allow a * power-off without waiting for ->sync_state(). + * + * GENPD_FLAG_POWER_UNKNOWN: Use this flag to inform genpd that its initial + * status for the PM domain is set to powered off, + * which may not correctly reflect the state of the + * HW, as it's unknown. If the PM domain becomes + * powered on during boot, genpd will prevent it + * from being powered off until the ->sync_state + * callback is invoked for it. */ #define GENPD_FLAG_PM_CLK (1U << 0) #define GENPD_FLAG_IRQ_SAFE (1U << 1) @@ -133,6 +141,7 @@ struct dev_pm_domain_list { #define GENPD_FLAG_DEV_NAME_FW (1U << 8) #define GENPD_FLAG_NO_SYNC_STATE (1U << 9) #define GENPD_FLAG_NO_STAY_ON (1U << 10) +#define GENPD_FLAG_POWER_UNKNOWN (1U << 11) enum gpd_status { GENPD_STATE_ON = 0, /* PM domain is on */ diff --git a/include/linux/pm_qos.h b/include/linux/pm_qos.h index 6cea4455f867..439a9e779d81 100644 --- a/include/linux/pm_qos.h +++ b/include/linux/pm_qos.h @@ -37,6 +37,8 @@ enum pm_qos_flags_status { #define PM_QOS_LATENCY_TOLERANCE_NO_CONSTRAINT (-1) #define PM_QOS_FLAG_NO_POWER_OFF (1 << 0) +/* latency value applies to system-wide suspend/s2idle */ +#define PM_QOS_FLAG_LATENCY_SYS (2 << 0) enum pm_qos_type { PM_QOS_UNITIALIZED, @@ -217,6 +219,12 @@ static inline s32 dev_pm_qos_raw_resume_latency(struct device *dev) PM_QOS_RESUME_LATENCY_NO_CONSTRAINT : pm_qos_read_value(&dev->power.qos->resume_latency); } + +static inline s32 dev_pm_qos_raw_flags(struct device *dev) +{ + return IS_ERR_OR_NULL(dev->power.qos) ? + 0 : READ_ONCE(dev->power.qos->flags.effective_flags); +} #else static inline enum pm_qos_flags_status __dev_pm_qos_flags(struct device *dev, s32 mask) @@ -298,6 +306,7 @@ static inline s32 dev_pm_qos_raw_resume_latency(struct device *dev) { return PM_QOS_RESUME_LATENCY_NO_CONSTRAINT; } +static inline s32 dev_pm_qos_raw_flags(struct device *dev) { return 0; } #endif static inline int freq_qos_request_active(struct freq_qos_request *req) diff --git a/include/linux/pm_runtime.h b/include/linux/pm_runtime.h index 64921b10ac74..ae68a9dc7fc5 100644 --- a/include/linux/pm_runtime.h +++ b/include/linux/pm_runtime.h @@ -137,13 +137,14 @@ static inline void pm_runtime_put_noidle(struct device *dev) * pm_runtime_suspended - Check whether or not a device is runtime-suspended. * @dev: Target device. * - * Return %true if runtime PM is enabled for @dev and its runtime PM status is - * %RPM_SUSPENDED, or %false otherwise. - * * Note that the return value of this function can only be trusted if it is * called under the runtime PM lock of @dev or under conditions in which * runtime PM cannot be either disabled or enabled for @dev and its runtime PM * status cannot change. + * + * Return: + * * %true: @dev has runtime PM enabled and its status is %RPM_SUSPENDED. + * * %false: Otherwise. */ static inline bool pm_runtime_suspended(struct device *dev) { @@ -155,13 +156,14 @@ static inline bool pm_runtime_suspended(struct device *dev) * pm_runtime_active - Check whether or not a device is runtime-active. * @dev: Target device. * - * Return %true if runtime PM is disabled for @dev or its runtime PM status is - * %RPM_ACTIVE, or %false otherwise. - * * Note that the return value of this function can only be trusted if it is * called under the runtime PM lock of @dev or under conditions in which * runtime PM cannot be either disabled or enabled for @dev and its runtime PM * status cannot change. + * + * Return: + * * %true: Runtime PM is disabled for @dev or its status is %RPM_ACTIVE. + * * %false: Otherwise. */ static inline bool pm_runtime_active(struct device *dev) { @@ -173,12 +175,13 @@ static inline bool pm_runtime_active(struct device *dev) * pm_runtime_status_suspended - Check if runtime PM status is "suspended". * @dev: Target device. * - * Return %true if the runtime PM status of @dev is %RPM_SUSPENDED, or %false - * otherwise, regardless of whether or not runtime PM has been enabled for @dev. - * * Note that the return value of this function can only be trusted if it is * called under the runtime PM lock of @dev or under conditions in which the * runtime PM status of @dev cannot change. + * + * Return: + * * %true: Runtime PM status of @dev is %RPM_SUSPENDED. + * * %false: Otherwise. */ static inline bool pm_runtime_status_suspended(struct device *dev) { @@ -189,11 +192,13 @@ static inline bool pm_runtime_status_suspended(struct device *dev) * pm_runtime_enabled - Check if runtime PM is enabled. * @dev: Target device. * - * Return %true if runtime PM is enabled for @dev or %false otherwise. - * * Note that the return value of this function can only be trusted if it is * called under the runtime PM lock of @dev or under conditions in which * runtime PM cannot be either disabled or enabled for @dev. + * + * Return: + * * %true: Runtime PM is enabled for @dev. + * * %false: Otherwise. */ static inline bool pm_runtime_enabled(struct device *dev) { @@ -205,6 +210,10 @@ static inline bool pm_runtime_enabled(struct device *dev) * @dev: Target device. * * Do not call this function outside system suspend/resume code paths. + * + * Return: + * * %true: Runtime PM enabling is blocked for @dev. + * * %false: Otherwise. */ static inline bool pm_runtime_blocked(struct device *dev) { @@ -215,8 +224,9 @@ static inline bool pm_runtime_blocked(struct device *dev) * pm_runtime_has_no_callbacks - Check if runtime PM callbacks may be present. * @dev: Target device. * - * Return %true if @dev is a special device without runtime PM callbacks or - * %false otherwise. + * Return: + * * %true: @dev is marked as having no runtime PM callbacks. + * * %false: Otherwise. */ static inline bool pm_runtime_has_no_callbacks(struct device *dev) { @@ -239,9 +249,11 @@ static inline void pm_runtime_mark_last_busy(struct device *dev) * pm_runtime_is_irq_safe - Check if runtime PM can work in interrupt context. * @dev: Target device. * - * Return %true if @dev has been marked as an "IRQ-safe" device (with respect - * to runtime PM), in which case its runtime PM callabcks can be expected to - * work correctly when invoked from interrupt handlers. + * Return: + * * %true: @dev has been marked as an "IRQ-safe" device, in which case its + * runtime PM callbacks can be expected to work correctly from interrupt + * handlers. + * * %false: Otherwise. */ static inline bool pm_runtime_is_irq_safe(struct device *dev) { @@ -340,25 +352,25 @@ static inline int pm_runtime_force_resume(struct device *dev) { return -ENXIO; } #endif /* CONFIG_PM_SLEEP */ /** - * pm_runtime_idle - Conditionally set up autosuspend of a device or suspend it. + * pm_runtime_idle - Conditionally initiate autosuspend of a device or suspend it. * @dev: Target device. * * Invoke the "idle check" callback of @dev and, depending on its return value, - * set up autosuspend of @dev or suspend it (depending on whether or not + * initiate autosuspend of @dev or suspend it (depending on whether or not * autosuspend has been enabled for it). * * Return: - * * 0: Success. - * * -EINVAL: Runtime PM error. - * * -EACCES: Runtime PM disabled. - * * -EAGAIN: Runtime PM usage counter non-zero, Runtime PM status change - * ongoing or device not in %RPM_ACTIVE state. - * * -EBUSY: Runtime PM child_count non-zero. - * * -EPERM: Device PM QoS resume latency 0. - * * -EINPROGRESS: Suspend already in progress. - * * -ENOSYS: CONFIG_PM not enabled. - * Other values and conditions for the above values are possible as returned by - * Runtime PM idle and suspend callbacks. + * * %0: Success. + * * %-EINVAL: Runtime PM error. + * * %-EACCES: Runtime PM disabled. + * * %-EAGAIN: Runtime PM usage counter non-zero, Runtime PM status change + * ongoing or device not in %RPM_ACTIVE state. + * * %-EBUSY: Runtime PM child_count non-zero. + * * %-EPERM: Device PM QoS resume latency 0. + * * %-EINPROGRESS: Suspend already in progress. + * * %-ENOSYS: %CONFIG_PM not enabled. + * * Other values and conditions for the above values are possible as returned + * by Runtime PM idle and suspend callbacks. */ static inline int pm_runtime_idle(struct device *dev) { @@ -370,17 +382,17 @@ static inline int pm_runtime_idle(struct device *dev) * @dev: Target device. * * Return: - * * 1: Success; device was already suspended. - * * 0: Success. - * * -EINVAL: Runtime PM error. - * * -EACCES: Runtime PM disabled. - * * -EAGAIN: Runtime PM usage counter non-zero or Runtime PM status change - * ongoing. - * * -EBUSY: Runtime PM child_count non-zero. - * * -EPERM: Device PM QoS resume latency 0. - * * -ENOSYS: CONFIG_PM not enabled. - * Other values and conditions for the above values are possible as returned by - * Runtime PM suspend callbacks. + * * %1: Success; device was already suspended. + * * %0: Success. + * * %-EINVAL: Runtime PM error. + * * %-EACCES: Runtime PM disabled. + * * %-EAGAIN: Runtime PM usage counter non-zero or Runtime PM status change + * ongoing. + * * %-EBUSY: Runtime PM child_count non-zero. + * * %-EPERM: Device PM QoS resume latency 0. + * * %-ENOSYS: %CONFIG_PM not enabled. + * * Other values and conditions for the above values are possible as returned + * by Runtime PM suspend callbacks. */ static inline int pm_runtime_suspend(struct device *dev) { @@ -388,26 +400,26 @@ static inline int pm_runtime_suspend(struct device *dev) } /** - * pm_runtime_autosuspend - Update the last access time and set up autosuspend + * pm_runtime_autosuspend - Update the last access time and initiate autosuspend * of a device. * @dev: Target device. * - * First update the last access time, then set up autosuspend of @dev or suspend - * it (depending on whether or not autosuspend is enabled for it) without - * engaging its "idle check" callback. + * First update the last access time, then initiate autosuspend of @dev or + * suspend it (depending on whether or not autosuspend is enabled for it) + * without engaging its "idle check" callback. * * Return: - * * 1: Success; device was already suspended. - * * 0: Success. - * * -EINVAL: Runtime PM error. - * * -EACCES: Runtime PM disabled. - * * -EAGAIN: Runtime PM usage counter non-zero or Runtime PM status change - * ongoing. - * * -EBUSY: Runtime PM child_count non-zero. - * * -EPERM: Device PM QoS resume latency 0. - * * -ENOSYS: CONFIG_PM not enabled. - * Other values and conditions for the above values are possible as returned by - * Runtime PM suspend callbacks. + * * %1: Success; device was already suspended. + * * %0: Success. + * * %-EINVAL: Runtime PM error. + * * %-EACCES: Runtime PM disabled. + * * %-EAGAIN: Runtime PM usage counter non-zero or Runtime PM status change + * ongoing. + * * %-EBUSY: Runtime PM child_count non-zero. + * * %-EPERM: Device PM QoS resume latency 0. + * * %-ENOSYS: %CONFIG_PM not enabled. + * * Other values and conditions for the above values are possible as returned + * by Runtime PM suspend callbacks. */ static inline int pm_runtime_autosuspend(struct device *dev) { @@ -418,6 +430,11 @@ static inline int pm_runtime_autosuspend(struct device *dev) /** * pm_runtime_resume - Resume a device synchronously. * @dev: Target device. + * + * Return: + * * %1: Success; @dev is already %RPM_ACTIVE. + * * %0: Success. + * * Error code on failure. */ static inline int pm_runtime_resume(struct device *dev) { @@ -425,22 +442,22 @@ static inline int pm_runtime_resume(struct device *dev) } /** - * pm_request_idle - Queue up "idle check" execution for a device. + * pm_request_idle - Request an asynchronous idle check for a device. * @dev: Target device. * - * Queue up a work item to run an equivalent of pm_runtime_idle() for @dev - * asynchronously. + * Asynchronously request the PM core to evaluate whether @dev can be idled + * or suspended, invoking its ->runtime_idle() callback if provided. * * Return: - * * 0: Success. - * * -EINVAL: Runtime PM error. - * * -EACCES: Runtime PM disabled. - * * -EAGAIN: Runtime PM usage counter non-zero, Runtime PM status change - * ongoing or device not in %RPM_ACTIVE state. - * * -EBUSY: Runtime PM child_count non-zero. - * * -EPERM: Device PM QoS resume latency 0. - * * -EINPROGRESS: Suspend already in progress. - * * -ENOSYS: CONFIG_PM not enabled. + * * %0: Success. + * * %-EINVAL: Runtime PM error. + * * %-EACCES: Runtime PM disabled. + * * %-EAGAIN: Runtime PM usage counter non-zero, Runtime PM status change + * ongoing or device not in %RPM_ACTIVE state. + * * %-EBUSY: Runtime PM child_count non-zero. + * * %-EPERM: Device PM QoS resume latency 0. + * * %-EINPROGRESS: Suspend already in progress. + * * %-ENOSYS: %CONFIG_PM not enabled. */ static inline int pm_request_idle(struct device *dev) { @@ -448,8 +465,16 @@ static inline int pm_request_idle(struct device *dev) } /** - * pm_request_resume - Queue up runtime-resume of a device. + * pm_request_resume - Request an asynchronous runtime resume for a device. * @dev: Target device. + * + * Asynchronously request the PM core to resume @dev to %RPM_ACTIVE state + * without modifying its usage counter. + * + * Return: + * * %1: Success; @dev is already %RPM_ACTIVE. + * * %0: Success. + * * Error code on failure. */ static inline int pm_request_resume(struct device *dev) { @@ -457,24 +482,23 @@ static inline int pm_request_resume(struct device *dev) } /** - * pm_request_autosuspend - Update the last access time and queue up autosuspend - * of a device. + * pm_request_autosuspend - Update access time and request delayed suspension. * @dev: Target device. * - * Update the last access time of a device and queue up a work item to run an - * equivalent pm_runtime_autosuspend() for @dev asynchronously. + * Update the last access time of @dev and asynchronously request the PM core + * to suspend it after the autosuspend delay has elapsed. * * Return: - * * 1: Success; device was already suspended. - * * 0: Success. - * * -EINVAL: Runtime PM error. - * * -EACCES: Runtime PM disabled. - * * -EAGAIN: Runtime PM usage counter non-zero or Runtime PM status change - * ongoing. - * * -EBUSY: Runtime PM child_count non-zero. - * * -EPERM: Device PM QoS resume latency 0. - * * -EINPROGRESS: Suspend already in progress. - * * -ENOSYS: CONFIG_PM not enabled. + * * %1: Success; device was already suspended. + * * %0: Success. + * * %-EINVAL: Runtime PM error. + * * %-EACCES: Runtime PM disabled. + * * %-EAGAIN: Runtime PM usage counter non-zero or Runtime PM status change + * ongoing. + * * %-EBUSY: Runtime PM child_count non-zero. + * * %-EPERM: Device PM QoS resume latency 0. + * * %-EINPROGRESS: Suspend already in progress. + * * %-ENOSYS: %CONFIG_PM not enabled. */ static inline int pm_request_autosuspend(struct device *dev) { @@ -483,11 +507,16 @@ static inline int pm_request_autosuspend(struct device *dev) } /** - * pm_runtime_get - Bump up usage counter and queue up resume of a device. + * pm_runtime_get - Increment usage counter and request asynchronous resume. * @dev: Target device. * - * Bump up the runtime PM usage counter of @dev and queue up a work item to - * carry out runtime-resume of it. + * Increment the runtime PM usage counter of @dev and, if the device is + * currently suspended, asynchronously request the PM core to resume it. + * + * Return: + * * %1: Success; @dev is already %RPM_ACTIVE. + * * %0: Success; runtime-resume was queued. + * * Error code on failure. */ static inline int pm_runtime_get(struct device *dev) { @@ -501,12 +530,15 @@ static inline int pm_runtime_get(struct device *dev) * Bump up the runtime PM usage counter of @dev and carry out runtime-resume of * it synchronously. * - * The possible return values of this function are the same as for - * pm_runtime_resume() and the runtime PM usage counter of @dev remains - * incremented in all cases, even if it returns an error code. - * Consider using pm_runtime_resume_and_get() instead of it, especially - * if its return value is checked by the caller, as this is likely to result - * in cleaner code. + * Note that the runtime PM usage counter of @dev remains incremented in all + * cases, even if it returns an error code. Consider using + * pm_runtime_resume_and_get() instead, especially if the return value is + * checked by the caller, as this is likely to result in cleaner code. + * + * Return: + * * %1: Success; @dev is already %RPM_ACTIVE. + * * %0: Success. + * * Error code on failure. */ static inline int pm_runtime_get_sync(struct device *dev) { @@ -531,8 +563,11 @@ static inline int pm_runtime_get_active(struct device *dev, int rpmflags) * @dev: Target device. * * Resume @dev synchronously and if that is successful, increment its runtime - * PM usage counter. Return 0 if the runtime PM usage counter of @dev has been - * incremented or a negative error code otherwise. + * PM usage counter. + * + * Return: + * * %0: Success; @dev is active and its usage counter has been incremented. + * * Negative error code on failure; usage counter is unchanged. */ static inline int pm_runtime_resume_and_get(struct device *dev) { @@ -540,11 +575,12 @@ static inline int pm_runtime_resume_and_get(struct device *dev) } /** - * pm_runtime_put - Drop device usage counter and queue up "idle check" if 0. + * pm_runtime_put - Drop device usage counter and request asynchronous idle check. * @dev: Target device. * - * Decrement the runtime PM usage counter of @dev and if it turns out to be - * equal to 0, queue up a work item for @dev like in pm_request_idle(). + * Decrement the runtime PM usage counter of @dev. If the counter reaches zero + * and the device has no active child dependencies, asynchronously request the + * PM core to idle or suspend the device. */ static inline void pm_runtime_put(struct device *dev) { @@ -559,16 +595,16 @@ static inline void pm_runtime_put(struct device *dev) * equal to 0, queue up a work item for @dev like in pm_request_autosuspend(). * * Return: - * * 1: Success. Usage counter dropped to zero, but device was already suspended. - * * 0: Success. - * * -EINVAL: Runtime PM error. - * * -EACCES: Runtime PM disabled. - * * -EAGAIN: Runtime PM usage counter became non-zero or Runtime PM status - * change ongoing. - * * -EBUSY: Runtime PM child_count non-zero. - * * -EPERM: Device PM QoS resume latency 0. - * * -EINPROGRESS: Suspend already in progress. - * * -ENOSYS: CONFIG_PM not enabled. + * * %1: Success. Usage counter dropped to zero, but device was already suspended. + * * %0: Success. + * * %-EINVAL: Runtime PM error. + * * %-EACCES: Runtime PM disabled. + * * %-EAGAIN: Runtime PM usage counter became non-zero or Runtime PM status + * change ongoing. + * * %-EBUSY: Runtime PM child_count non-zero. + * * %-EPERM: Device PM QoS resume latency 0. + * * %-EINPROGRESS: Suspend already in progress. + * * %-ENOSYS: %CONFIG_PM not enabled. */ static inline int __pm_runtime_put_autosuspend(struct device *dev) { @@ -576,25 +612,25 @@ static inline int __pm_runtime_put_autosuspend(struct device *dev) } /** - * pm_runtime_put_autosuspend - Update the last access time of a device, drop - * its usage counter and queue autosuspend if the usage counter becomes 0. + * pm_runtime_put_autosuspend - Update the last access time, drop usage counter + * and request autosuspend. * @dev: Target device. * - * Update the last access time of @dev, decrement runtime PM usage counter of - * @dev and if it turns out to be equal to 0, queue up a work item for @dev like - * in pm_request_autosuspend(). + * Update the last access time of @dev and decrement its runtime PM usage + * counter. If the counter drops to zero, asynchronously request the PM core to + * suspend the device once its autosuspend delay has elapsed. * * Return: - * * 1: Success. Usage counter dropped to zero, but device was already suspended. - * * 0: Success. - * * -EINVAL: Runtime PM error. - * * -EACCES: Runtime PM disabled. - * * -EAGAIN: Runtime PM usage counter became non-zero or Runtime PM status - * change ongoing. - * * -EBUSY: Runtime PM child_count non-zero. - * * -EPERM: Device PM QoS resume latency 0. - * * -EINPROGRESS: Suspend already in progress. - * * -ENOSYS: CONFIG_PM not enabled. + * * %1: Success. Usage counter dropped to zero, but device was already suspended. + * * %0: Success. + * * %-EINVAL: Runtime PM error. + * * %-EACCES: Runtime PM disabled. + * * %-EAGAIN: Runtime PM usage counter became non-zero or Runtime PM status + * change ongoing. + * * %-EBUSY: Runtime PM child_count non-zero. + * * %-EPERM: Device PM QoS resume latency 0. + * * %-EINPROGRESS: Suspend already in progress. + * * %-ENOSYS: %CONFIG_PM not enabled. */ static inline int pm_runtime_put_autosuspend(struct device *dev) { @@ -653,26 +689,28 @@ DEFINE_GUARD_COND(pm_runtime_active_auto, _try_enabled, * pm_runtime_put_sync - Drop device usage counter and run "idle check" if 0. * @dev: Target device. * - * Decrement the runtime PM usage counter of @dev and if it turns out to be - * equal to 0, invoke the "idle check" callback of @dev and, depending on its - * return value, set up autosuspend of @dev or suspend it (depending on whether - * or not autosuspend has been enabled for it). + * Decrement the runtime PM usage counter of @dev. If the counter drops to zero, + * synchronously evaluate and trigger idle/suspend handling. + * + * Note that this does not update the last access time, but it does respect + * existing autosuspend timers. If @dev uses autosuspend, consider using + * pm_runtime_put_sync_autosuspend() or pm_runtime_put_sync_suspend() instead. * * The runtime PM usage counter of @dev remains decremented in all cases, even * if it returns an error code. * * Return: - * * 1: Success. Usage counter dropped to zero, but device was already suspended. - * * 0: Success. - * * -EINVAL: Runtime PM error. - * * -EACCES: Runtime PM disabled. - * * -EAGAIN: Runtime PM usage counter became non-zero or Runtime PM status - * change ongoing. - * * -EBUSY: Runtime PM child_count non-zero. - * * -EPERM: Device PM QoS resume latency 0. - * * -ENOSYS: CONFIG_PM not enabled. - * Other values and conditions for the above values are possible as returned by - * Runtime PM suspend callbacks. + * * %1: Success. Usage counter dropped to zero, but device was already suspended. + * * %0: Success. + * * %-EINVAL: Runtime PM error. + * * %-EACCES: Runtime PM disabled. + * * %-EAGAIN: Runtime PM usage counter became non-zero or Runtime PM status + * change ongoing. + * * %-EBUSY: Runtime PM child_count non-zero. + * * %-EPERM: Device PM QoS resume latency 0. + * * %-ENOSYS: %CONFIG_PM not enabled. + * * Other values and conditions for the above values are possible as returned + * by Runtime PM suspend callbacks. */ static inline int pm_runtime_put_sync(struct device *dev) { @@ -683,24 +721,28 @@ static inline int pm_runtime_put_sync(struct device *dev) * pm_runtime_put_sync_suspend - Drop device usage counter and suspend if 0. * @dev: Target device. * - * Decrement the runtime PM usage counter of @dev and if it turns out to be - * equal to 0, carry out runtime-suspend of @dev synchronously. + * Decrement the runtime PM usage counter of @dev. If the counter drops to zero, + * suspend the device synchronously. + * + * This API differs from pm_runtime_put_sync() and + * pm_runtime_put_sync_autosuspend() in that it ignores any outstanding + * autosuspend delays. * * The runtime PM usage counter of @dev remains decremented in all cases, even * if it returns an error code. * * Return: - * * 1: Success. Usage counter dropped to zero, but device was already suspended. - * * 0: Success. - * * -EINVAL: Runtime PM error. - * * -EACCES: Runtime PM disabled. - * * -EAGAIN: Runtime PM usage counter became non-zero or Runtime PM status - * change ongoing. - * * -EBUSY: Runtime PM child_count non-zero. - * * -EPERM: Device PM QoS resume latency 0. - * * -ENOSYS: CONFIG_PM not enabled. - * Other values and conditions for the above values are possible as returned by - * Runtime PM suspend callbacks. + * * %1: Success. Usage counter dropped to zero, but device was already suspended. + * * %0: Success. + * * %-EINVAL: Runtime PM error. + * * %-EACCES: Runtime PM disabled. + * * %-EAGAIN: Runtime PM usage counter became non-zero or Runtime PM status + * change ongoing. + * * %-EBUSY: Runtime PM child_count non-zero. + * * %-EPERM: Device PM QoS resume latency 0. + * * %-ENOSYS: %CONFIG_PM not enabled. + * * Other values and conditions for the above values are possible as returned + * by Runtime PM suspend callbacks. */ static inline int pm_runtime_put_sync_suspend(struct device *dev) { @@ -712,27 +754,28 @@ static inline int pm_runtime_put_sync_suspend(struct device *dev) * drop device usage counter and autosuspend if 0. * @dev: Target device. * - * Update the last access time of @dev, decrement the runtime PM usage counter - * of @dev and if it turns out to be equal to 0, set up autosuspend of @dev or - * suspend it synchronously (depending on whether or not autosuspend has been - * enabled for it). + * Update the last access time of @dev and decrement its runtime PM usage + * counter. If the counter drops to zero, synchronously suspend the device (or + * schedule autosuspend if the delay has not elapsed). + * + * Prefer this API over pm_runtime_put_sync() for devices that use autosuspend. * * The runtime PM usage counter of @dev remains decremented in all cases, even * if it returns an error code. * * Return: - * * 1: Success. Usage counter dropped to zero, but device was already suspended. - * * 0: Success. - * * -EINVAL: Runtime PM error. - * * -EACCES: Runtime PM disabled. - * * -EAGAIN: Runtime PM usage counter became non-zero or Runtime PM status - * change ongoing. - * * -EBUSY: Runtime PM child_count non-zero. - * * -EPERM: Device PM QoS resume latency 0. - * * -EINPROGRESS: Suspend already in progress. - * * -ENOSYS: CONFIG_PM not enabled. - * Other values and conditions for the above values are possible as returned by - * Runtime PM suspend callbacks. + * * %1: Success. Usage counter dropped to zero, but device was already suspended. + * * %0: Success. + * * %-EINVAL: Runtime PM error. + * * %-EACCES: Runtime PM disabled. + * * %-EAGAIN: Runtime PM usage counter became non-zero or Runtime PM status + * change ongoing. + * * %-EBUSY: Runtime PM child_count non-zero. + * * %-EPERM: Device PM QoS resume latency 0. + * * %-EINPROGRESS: Suspend already in progress. + * * %-ENOSYS: %CONFIG_PM not enabled. + * * Other values and conditions for the above values are possible as returned + * by Runtime PM suspend callbacks. */ static inline int pm_runtime_put_sync_autosuspend(struct device *dev) { @@ -741,13 +784,22 @@ static inline int pm_runtime_put_sync_autosuspend(struct device *dev) } /** - * pm_runtime_set_active - Set runtime PM status to "active". + * pm_runtime_set_active - Set runtime PM status to "active" and clear errors. * @dev: Target device. * - * Set the runtime PM status of @dev to %RPM_ACTIVE and ensure that dependencies - * of it will be taken into account. + * Set the runtime PM status of @dev to %RPM_ACTIVE and ensure that its + * dependencies will be taken into account. Also clear the device's error + * status (@dev->power.runtime_error). + * + * It is only valid to call this function if runtime PM is disabled or if + * @dev->power.runtime_error is set. * - * It is not valid to call this function for devices with runtime PM enabled. + * This will fail if suppliers cannot be resumed, or if the parent is not in + * the correct state. + * + * Return: + * * %0: Success. + * * Error code on failure. */ static inline int pm_runtime_set_active(struct device *dev) { @@ -755,13 +807,19 @@ static inline int pm_runtime_set_active(struct device *dev) } /** - * pm_runtime_set_suspended - Set runtime PM status to "suspended". + * pm_runtime_set_suspended - Set runtime PM status to "suspended" and clear errors. * @dev: Target device. * - * Set the runtime PM status of @dev to %RPM_SUSPENDED and ensure that - * dependencies of it will be taken into account. + * Set the runtime PM status of @dev to %RPM_SUSPENDED and ensure that its + * dependencies will be taken into account. Also clear the device's error + * status (@dev->power.runtime_error). * - * It is not valid to call this function for devices with runtime PM enabled. + * It is only valid to call this function if runtime PM is disabled or if + * @dev->power.runtime_error is set. + * + * Return: + * * %0: Success. + * * Error code on failure. */ static inline int pm_runtime_set_suspended(struct device *dev) { @@ -777,9 +835,8 @@ static inline int pm_runtime_set_suspended(struct device *dev) * * If the counter is zero when this function runs and there is a pending runtime * resume request for @dev, it will be resumed. If the counter is still zero at - * that point, all of the pending runtime PM requests for @dev will be canceled - * and all runtime PM operations in progress involving it will be waited for to - * complete. + * that point, this function cancels all pending runtime PM requests for @dev + * and waits for its runtime PM operations to complete (if any). * * For each invocation of this function for @dev, there must be a matching * pm_runtime_enable() call, so that runtime PM is eventually enabled for it @@ -798,9 +855,9 @@ static inline void pm_runtime_disable(struct device *dev) * requested (or "autosuspend" will be handled as direct runtime-suspend for * it). * - * NOTE: It's important to undo this with pm_runtime_dont_use_autosuspend() - * at driver exit time unless your driver initially enabled pm_runtime - * with devm_pm_runtime_enable() (which handles it for you). + * NOTE: This is undone automatically with pm_runtime_dont_use_autosuspend() on + * driver exit regardless of whether a driver uses managed devm_* functions or + * not. */ static inline void pm_runtime_use_autosuspend(struct device *dev) { diff --git a/include/linux/posix-timers.h b/include/linux/posix-timers.h index 9a1a0c61361c..00767acbc111 100644 --- a/include/linux/posix-timers.h +++ b/include/linux/posix-timers.h @@ -66,38 +66,6 @@ struct cpu_timer { struct task_struct __rcu *handling; }; -static inline bool cpu_timer_enqueue(struct timerqueue_head *head, - struct cpu_timer *ctmr) -{ - ctmr->head = head; - return timerqueue_add(head, &ctmr->node); -} - -static inline bool cpu_timer_queued(struct cpu_timer *ctmr) -{ - return !!ctmr->head; -} - -static inline bool cpu_timer_dequeue(struct cpu_timer *ctmr) -{ - if (cpu_timer_queued(ctmr)) { - timerqueue_del(ctmr->head, &ctmr->node); - ctmr->head = NULL; - return true; - } - return false; -} - -static inline u64 cpu_timer_getexpires(struct cpu_timer *ctmr) -{ - return ctmr->node.expires; -} - -static inline void cpu_timer_setexpires(struct cpu_timer *ctmr, u64 exp) -{ - ctmr->node.expires = exp; -} - static inline void posix_cputimers_init(struct posix_cputimers *pct) { memset(pct, 0, sizeof(*pct)); @@ -224,14 +192,15 @@ struct k_itimer { } ____cacheline_aligned_in_smp; void run_posix_cpu_timers(void); -void posix_cpu_timers_exit(struct task_struct *task); -void posix_cpu_timers_exit_group(struct task_struct *task); void set_process_cpu_timer(struct task_struct *task, unsigned int clock_idx, u64 *newval, u64 *oldval); int update_rlimit_cpu(struct task_struct *task, unsigned long rlim_new); #ifdef CONFIG_POSIX_TIMERS +void posixtimer_exec(void); +void posixtimer_exit(bool group_dead); + static inline void posixtimer_putref(struct k_itimer *tmr) { if (rcuref_put(&tmr->rcuref)) @@ -259,6 +228,8 @@ static inline bool posixtimer_valid(const struct k_itimer *timer) return !(val & 0x1UL); } #else /* CONFIG_POSIX_TIMERS */ +static inline void posixtimer_exec(void) { } +static inline void posixtimer_exit(bool group_dead) { } static inline void posixtimer_sigqueue_getref(struct sigqueue *q) { } static inline void posixtimer_sigqueue_putref(struct sigqueue *q) { } #endif /* !CONFIG_POSIX_TIMERS */ diff --git a/include/linux/posix_acl.h b/include/linux/posix_acl.h index 62d497763e25..caf500bed993 100644 --- a/include/linux/posix_acl.h +++ b/include/linux/posix_acl.h @@ -74,20 +74,20 @@ extern int __posix_acl_create(struct posix_acl **, gfp_t, umode_t *); extern int __posix_acl_chmod(struct posix_acl **, gfp_t, umode_t); extern struct posix_acl *get_posix_acl(struct inode *, int); -int set_posix_acl(struct mnt_idmap *, struct dentry *, int, +int set_posix_acl(const struct mnt_idmap *, struct dentry *, int, struct posix_acl *); struct posix_acl *get_cached_acl_rcu(struct inode *inode, int type); struct posix_acl *posix_acl_clone(const struct posix_acl *acl, gfp_t flags); #ifdef CONFIG_FS_POSIX_ACL -int posix_acl_chmod(struct mnt_idmap *, struct dentry *, umode_t); +int posix_acl_chmod(const struct mnt_idmap *, struct dentry *, umode_t); extern int posix_acl_create(struct inode *, umode_t *, struct posix_acl **, struct posix_acl **); -int posix_acl_update_mode(struct mnt_idmap *, struct inode *, umode_t *, +int posix_acl_update_mode(const struct mnt_idmap *, struct inode *, umode_t *, struct posix_acl **); -int simple_set_acl(struct mnt_idmap *, struct dentry *, +int simple_set_acl(const struct mnt_idmap *, struct dentry *, struct posix_acl *, int); extern int simple_acl_create(struct inode *, struct inode *); @@ -96,7 +96,7 @@ void set_cached_acl(struct inode *inode, int type, struct posix_acl *acl); void forget_cached_acl(struct inode *inode, int type); void forget_all_cached_acls(struct inode *inode); int posix_acl_valid(struct user_namespace *, const struct posix_acl *); -int posix_acl_permission(struct mnt_idmap *, struct inode *, +int posix_acl_permission(const struct mnt_idmap *, struct inode *, const struct posix_acl *, int); static inline void cache_no_acl(struct inode *inode) @@ -105,16 +105,16 @@ static inline void cache_no_acl(struct inode *inode) inode->i_default_acl = NULL; } -int vfs_set_acl(struct mnt_idmap *idmap, struct dentry *dentry, +int vfs_set_acl(const struct mnt_idmap *idmap, struct dentry *dentry, const char *acl_name, struct posix_acl *kacl); -struct posix_acl *vfs_get_acl(struct mnt_idmap *idmap, +struct posix_acl *vfs_get_acl(const struct mnt_idmap *idmap, struct dentry *dentry, const char *acl_name); -int vfs_remove_acl(struct mnt_idmap *idmap, struct dentry *dentry, +int vfs_remove_acl(const struct mnt_idmap *idmap, struct dentry *dentry, const char *acl_name); int posix_acl_listxattr(struct inode *inode, char **buffer, ssize_t *remaining_size); #else -static inline int posix_acl_chmod(struct mnt_idmap *idmap, +static inline int posix_acl_chmod(const struct mnt_idmap *idmap, struct dentry *dentry, umode_t mode) { return 0; @@ -141,21 +141,21 @@ static inline void forget_all_cached_acls(struct inode *inode) { } -static inline int vfs_set_acl(struct mnt_idmap *idmap, +static inline int vfs_set_acl(const struct mnt_idmap *idmap, struct dentry *dentry, const char *name, struct posix_acl *acl) { return -EOPNOTSUPP; } -static inline struct posix_acl *vfs_get_acl(struct mnt_idmap *idmap, +static inline struct posix_acl *vfs_get_acl(const struct mnt_idmap *idmap, struct dentry *dentry, const char *acl_name) { return ERR_PTR(-EOPNOTSUPP); } -static inline int vfs_remove_acl(struct mnt_idmap *idmap, +static inline int vfs_remove_acl(const struct mnt_idmap *idmap, struct dentry *dentry, const char *acl_name) { return -EOPNOTSUPP; diff --git a/include/linux/power/bq27xxx_battery.h b/include/linux/power/bq27xxx_battery.h index d56e1276aafe..731ea837aac3 100644 --- a/include/linux/power/bq27xxx_battery.h +++ b/include/linux/power/bq27xxx_battery.h @@ -33,7 +33,9 @@ enum bq27xxx_chip { BQ27441, BQ27621, BQ27Z561, + BQ27Z746, BQ28Z610, + BQ28Z620, BQ34Z100, BQ78Z100, }; diff --git a/include/linux/power_supply.h b/include/linux/power_supply.h index e749d2189335..cce61137aef4 100644 --- a/include/linux/power_supply.h +++ b/include/linux/power_supply.h @@ -104,6 +104,14 @@ enum { POWER_SUPPLY_SCOPE_DEVICE, }; +enum { + POWER_SUPPLY_LOAD_SWITCH_UNKNOWN = 0, + POWER_SUPPLY_LOAD_SWITCH_ON, + POWER_SUPPLY_LOAD_SWITCH_OFF, + POWER_SUPPLY_LOAD_SWITCH_STANDBY, + POWER_SUPPLY_LOAD_SWITCH_SHIP, +}; + enum power_supply_property { /* Properties of type `int' */ POWER_SUPPLY_PROP_STATUS = 0, @@ -182,6 +190,7 @@ enum power_supply_property { POWER_SUPPLY_PROP_MANUFACTURE_DAY, POWER_SUPPLY_PROP_INTERNAL_RESISTANCE, POWER_SUPPLY_PROP_STATE_OF_HEALTH, + POWER_SUPPLY_PROP_LOAD_SWITCH, /* Properties of type `const char *' */ POWER_SUPPLY_PROP_MODEL_NAME, POWER_SUPPLY_PROP_MANUFACTURER, @@ -259,9 +268,9 @@ struct power_supply_config { struct power_supply_desc { const char *name; enum power_supply_type type; - u8 charge_behaviours; u32 charge_types; u32 usb_types; + u32 load_switches; const enum power_supply_property *properties; size_t num_properties; @@ -295,14 +304,17 @@ struct power_supply_desc { */ int (*init)(struct power_supply *psy); + /* For APM emulation, think legacy userspace. */ + int use_for_apm; + /* * Set if thermal zone should not be created for this power supply. * For example for virtual supplies forwarding calls to actual * sensors or other supplies. */ bool no_thermal; - /* For APM emulation, think legacy userspace. */ - int use_for_apm; + + u8 charge_behaviours; }; struct power_supply_ext { @@ -351,6 +363,16 @@ struct power_supply { bool removing; atomic_t use_cnt; struct power_supply_battery_info *battery_info; + /* + * Held for read while ->external_power_changed() runs, and for write by + * power_supply_unregister() when it publishes @removing, so that it + * waits for an in-flight callback to finish. Without this a driver's + * data, typically devm-allocated on its own device, can be freed while + * the callback is still using it. + * Must not be shared with extensions_sem: callbacks may read their own + * properties, which takes that one for read. + */ + struct rw_semaphore epc_sem; struct rw_semaphore extensions_sem; /* protects "extensions" */ struct list_head extensions; #ifdef CONFIG_THERMAL @@ -414,9 +436,9 @@ struct power_supply_vbat_ri_table { * @charge_voltage_max_uv: maintenance charging voltage that is usually a bit * lower than the constant_charge_voltage_max_uv. We can apply this settings * charge_current_max_ua until we get back up to this voltage. - * @safety_timer_minutes: maintenance charging safety timer, with an expiry - * time in minutes. We will only use maintenance charging in this setting - * for a certain amount of time, then we will first move to the next + * @charge_safety_timer_minutes: maintenance charging safety timer, with an + * expiry time in minutes. We will only use maintenance charging in this + * setting for a certain amount of time, then we will first move to the next * maintenance charge current and voltage pair in respective array and wait * for the next safety timer timeout, or, if we reached the last maintencance * charging setting, disable charging until we reach diff --git a/include/linux/ppp_channel.h b/include/linux/ppp_channel.h index 2f63e9a6cc88..15bc3aa4f171 100644 --- a/include/linux/ppp_channel.h +++ b/include/linux/ppp_channel.h @@ -43,6 +43,7 @@ struct ppp_channel { void *ppp; /* opaque to channel */ int speed; /* transfer rate (bytes/second) */ bool direct_xmit; /* no qdisc, xmit directly */ + bool xmit_offload; /* accepts GSO and CHECKSUM_PARTIAL skbs */ }; #ifdef __KERNEL__ diff --git a/include/linux/pps_gen_kernel.h b/include/linux/pps_gen_kernel.h index 6214c8aa2e02..c4d22b1af118 100644 --- a/include/linux/pps_gen_kernel.h +++ b/include/linux/pps_gen_kernel.h @@ -11,6 +11,7 @@ #include <linux/pps_gen.h> #include <linux/cdev.h> #include <linux/device.h> +#include <linux/mutex.h> /* * Global defines @@ -44,6 +45,7 @@ struct pps_gen_source_info { /* The main struct */ struct pps_gen_device { const struct pps_gen_source_info *info; /* PSS generator info */ + struct mutex info_lock; /* protects info */ bool enabled; /* PSS generator status */ unsigned int event; @@ -54,7 +56,7 @@ struct pps_gen_device { unsigned int id; /* PPS generator unique ID */ struct cdev cdev; - struct device *dev; + struct device dev; struct fasync_struct *async_queue; /* fasync method */ spinlock_t lock; }; diff --git a/include/linux/preempt.h b/include/linux/preempt.h index 2e689de7b29a..06ff44a4b8b6 100644 --- a/include/linux/preempt.h +++ b/include/linux/preempt.h @@ -498,21 +498,11 @@ DEFINE_LOCK_GUARD_0(preempt_notrace, preempt_disable_notrace(), preempt_enable_n #ifdef CONFIG_PREEMPT_DYNAMIC -extern bool preempt_model_none(void); -extern bool preempt_model_voluntary(void); extern bool preempt_model_full(void); extern bool preempt_model_lazy(void); #else -static inline bool preempt_model_none(void) -{ - return IS_ENABLED(CONFIG_PREEMPT_NONE); -} -static inline bool preempt_model_voluntary(void) -{ - return IS_ENABLED(CONFIG_PREEMPT_VOLUNTARY); -} static inline bool preempt_model_full(void) { return IS_ENABLED(CONFIG_PREEMPT); @@ -525,6 +515,16 @@ static inline bool preempt_model_lazy(void) #endif +static inline bool preempt_model_none(void) +{ + return IS_ENABLED(CONFIG_PREEMPT_NONE); +} + +static inline bool preempt_model_voluntary(void) +{ + return IS_ENABLED(CONFIG_PREEMPT_VOLUNTARY); +} + static inline bool preempt_model_rt(void) { return IS_ENABLED(CONFIG_PREEMPT_RT); diff --git a/include/linux/property.h b/include/linux/property.h index 907c790a3f01..9143fe4f5c05 100644 --- a/include/linux/property.h +++ b/include/linux/property.h @@ -547,6 +547,11 @@ unsigned int fwnode_graph_get_endpoint_count(const struct fwnode_handle *fwnode, for (child = fwnode_graph_get_next_endpoint(fwnode, NULL); child; \ child = fwnode_graph_get_next_endpoint(fwnode, child)) +#define fwnode_graph_for_each_endpoint_scoped(fwnode, child) \ + for (struct fwnode_handle *child __free(fwnode_handle) = \ + fwnode_graph_get_next_endpoint(fwnode, NULL); \ + child; child = fwnode_graph_get_next_endpoint(fwnode, child)) + int fwnode_graph_parse_endpoint(const struct fwnode_handle *fwnode, struct fwnode_endpoint *endpoint); diff --git a/include/linux/psp-sev.h b/include/linux/psp-sev.h index 03a79786df1d..fab62228f981 100644 --- a/include/linux/psp-sev.h +++ b/include/linux/psp-sev.h @@ -49,7 +49,7 @@ #define SEV_FW_BLOB_MAX_SIZE 0x4000 /* 16KB */ -/** +/* * SEV platform state */ enum sev_state { @@ -60,7 +60,7 @@ enum sev_state { SEV_STATE_MAX }; -/** +/* * SEV platform and guest management commands */ enum sev_cmd { @@ -157,6 +157,7 @@ enum sev_cmd { * struct sev_data_init - INIT command parameters * * @flags: processing flags + * @reserved: reserved * @tmr_address: system physical address used for SEV-ES * @tmr_len: len of tmr_address */ @@ -174,6 +175,7 @@ struct sev_data_init { * @flags: processing flags * @tmr_address: system physical address used for SEV-ES * @tmr_len: len of tmr_address + * @reserved: reserved * @nv_address: system physical address used for PSP NV storage * @nv_len: len of nv_address */ @@ -201,12 +203,13 @@ struct sev_data_pek_csr { } __packed; /** - * struct sev_data_cert_import - PEK_CERT_IMPORT command parameters + * struct sev_data_pek_cert_import - PEK_CERT_IMPORT command parameters * - * @pek_address: PEK certificate chain - * @pek_len: len of PEK certificate - * @oca_address: OCA certificate chain - * @oca_len: len of OCA certificate + * @pek_cert_address: PEK certificate chain + * @pek_cert_len: len of PEK certificate + * @reserved: reserved + * @oca_cert_address: OCA certificate chain + * @oca_cert_len: len of OCA certificate */ struct sev_data_pek_cert_import { u64 pek_cert_address; /* In */ @@ -240,8 +243,9 @@ struct sev_data_get_id { /** * struct sev_data_pdh_cert_export - PDH_CERT_EXPORT command parameters * - * @pdh_address: PDH certificate address - * @pdh_len: len of PDH certificate + * @pdh_cert_address: PDH certificate address + * @pdh_cert_len: len of PDH certificate + * @reserved: reserved * @cert_chain_address: PDH certificate chain * @cert_chain_len: len of PDH certificate chain */ @@ -304,6 +308,7 @@ struct sev_data_guest_status { * @policy: guest launch policy * @dh_cert_address: physical address of DH certificate blob * @dh_cert_len: len of DH certificate blob + * @reserved: reserved * @session_address: physical address of session parameters * @session_len: len of session parameters */ @@ -321,6 +326,7 @@ struct sev_data_launch_start { * struct sev_data_launch_update_data - LAUNCH_UPDATE_DATA command parameter * * @handle: handle of the VM to update + * @reserved: reserved * @len: len of memory to be encrypted * @address: physical address of memory region to encrypt */ @@ -335,6 +341,7 @@ struct sev_data_launch_update_data { * struct sev_data_launch_update_vmsa - LAUNCH_UPDATE_VMSA command * * @handle: handle of the VM + * @reserved: reserved * @address: physical address of memory region to encrypt * @len: len of memory region to encrypt */ @@ -349,6 +356,7 @@ struct sev_data_launch_update_vmsa { * struct sev_data_launch_measure - LAUNCH_MEASURE command parameters * * @handle: handle of the VM to process + * @reserved: reserved * @address: physical address containing the measurement blob * @len: len of measurement blob */ @@ -363,10 +371,13 @@ struct sev_data_launch_measure { * struct sev_data_launch_secret - LAUNCH_SECRET command parameters * * @handle: handle of the VM to process + * @reserved1: reserved * @hdr_address: physical address containing the packet header * @hdr_len: len of packet header + * @reserved2: reserved * @guest_address: system physical address of guest memory region * @guest_len: len of guest_paddr + * @reserved3: reserved * @trans_address: physical address of transport memory buffer * @trans_len: len of transport memory buffer */ @@ -399,10 +410,13 @@ struct sev_data_launch_finish { * @policy: policy information for the VM * @pdh_cert_address: physical address containing PDH certificate * @pdh_cert_len: len of PDH certificate + * @reserved1: reserved * @plat_certs_address: physical address containing platform certificate * @plat_certs_len: len of platform certificate + * @reserved2: reserved * @amd_certs_address: physical address containing AMD certificate * @amd_certs_len: len of AMD certificate + * @reserved3: reserved * @session_address: physical address containing Session data * @session_len: len of session data */ @@ -423,13 +437,16 @@ struct sev_data_send_start { } __packed; /** - * struct sev_data_send_update - SEND_UPDATE_DATA command + * struct sev_data_send_update_data - SEND_UPDATE_DATA command * * @handle: handle of the VM to process + * @reserved1: reserved * @hdr_address: physical address containing packet header * @hdr_len: len of packet header + * @reserved2: reserved * @guest_address: physical address of guest memory region to send * @guest_len: len of guest memory region to send + * @reserved3: reserved * @trans_address: physical address of host memory region * @trans_len: len of host memory region */ @@ -447,13 +464,15 @@ struct sev_data_send_update_data { } __packed; /** - * struct sev_data_send_update - SEND_UPDATE_VMSA command + * struct sev_data_send_update_vmsa - SEND_UPDATE_VMSA command * * @handle: handle of the VM to process * @hdr_address: physical address containing packet header * @hdr_len: len of packet header + * @reserved2: reserved * @guest_address: physical address of guest memory region to send * @guest_len: len of guest memory region to send + * @reserved3: reserved * @trans_address: physical address of host memory region * @trans_len: len of host memory region */ @@ -491,8 +510,10 @@ struct sev_data_send_cancel { * struct sev_data_receive_start - RECEIVE_START command parameters * * @handle: handle of the VM to perform receive operation + * @policy: policy information for the VM * @pdh_cert_address: system physical address containing PDH certificate blob * @pdh_cert_len: len of PDH certificate blob + * @reserved1: reserved * @session_address: system physical address containing session blob * @session_len: len of session blob */ @@ -510,10 +531,13 @@ struct sev_data_receive_start { * struct sev_data_receive_update_data - RECEIVE_UPDATE_DATA command parameters * * @handle: handle of the VM to update + * @reserved1: reserved * @hdr_address: physical address containing packet header blob * @hdr_len: len of packet header + * @reserved2: reserved * @guest_address: system physical address of guest memory region * @guest_len: len of guest memory region + * @reserved3: reserved * @trans_address: system physical address of transport buffer * @trans_len: len of transport buffer */ @@ -534,10 +558,13 @@ struct sev_data_receive_update_data { * struct sev_data_receive_update_vmsa - RECEIVE_UPDATE_VMSA command parameters * * @handle: handle of the VM to update + * @reserved1: reserved * @hdr_address: physical address containing packet header blob * @hdr_len: len of packet header + * @reserved2: reserved * @guest_address: system physical address of guest memory region * @guest_len: len of guest memory region + * @reserved3: reserved * @trans_address: system physical address of transport buffer * @trans_len: len of transport buffer */ @@ -567,6 +594,7 @@ struct sev_data_receive_finish { * struct sev_data_dbg - DBG_ENCRYPT/DBG_DECRYPT command parameters * * @handle: handle of the VM to perform debug operation + * @reserved: reserved * @src_addr: source address of data to operate on * @dst_addr: destination address of data to operate on * @len: len of data to operate on @@ -583,6 +611,7 @@ struct sev_data_dbg { * struct sev_data_attestation_report - SEV_ATTESTATION_REPORT command parameters * * @handle: handle of the VM + * @reserved: reserved * @mnonce: a random nonce that will be included in the report. * @address: physical address where the report will be copied. * @len: length of the physical buffer. @@ -781,9 +810,14 @@ struct sev_data_snp_guest_request { * * @init_rmp: indicate that the RMP should be initialized. * @list_paddr_en: indicate that list_paddr is valid + * @rapl_dis: whether RAPL is disabled + * @ciphertext_hiding_en: whether ciphertext hiding is enabled + * @tio_en: Indicates that SNP_INIT_EX initialized the RMP for SEV-TIO * @rsvd: reserved * @rsvd1: reserved * @list_paddr: system physical address of range list + * @max_snp_asid: When non-zero, enable ciphertext hiding and specify the + * maximum ASID that can be used for an SEV-SNP guest. * @rsvd2: reserved */ struct sev_data_snp_init_ex { @@ -841,7 +875,7 @@ struct sev_data_snp_shutdown_ex { } __packed; /** - * struct sev_platform_init_args + * struct sev_platform_init_args - parameters for sev_platform_init() * * @error: SEV firmware error code * @probe: True if this is being called as part of CCP module probe, which @@ -879,12 +913,12 @@ struct sev_data_snp_feature_info { } __packed; /** - * struct feature_info - FEATURE_INFO structure + * struct snp_feature_info - FEATURE_INFO structure * * @eax: output of SNP_FEATURE_INFO command * @ebx: output of SNP_FEATURE_INFO command * @ecx: output of SNP_FEATURE_INFO command - * #edx: output of SNP_FEATURE_INFO command + * @edx: output of SNP_FEATURE_INFO command */ struct snp_feature_info { u32 eax; @@ -954,7 +988,7 @@ struct sev_data_snp_verify_mitigation_dst { } __packed; /** - * struct sev_snp_tcb_version_genoa_milan + * struct sev_snp_tcb_version_genoa_milan - v1 SVN payload * * @boot_loader: SVN of PSP bootloader * @tee: SVN of PSP operating system @@ -971,7 +1005,7 @@ struct sev_snp_tcb_version_genoa_milan { }; /** - * struct sev_snp_tcb_version_turin + * struct sev_snp_tcb_version_turin - v2 SVN payload * * @fmc: SVN of FMC firmware * @boot_loader: SVN of PSP bootloader @@ -1037,9 +1071,9 @@ int sev_platform_status(struct sev_user_data_status *status, int *error); * behalf of userspace. The caller must pass a valid SEV file descriptor * so that we know that it has access to SEV device. * - * @filep - SEV device file pointer - * @cmd - command to issue - * @data - command buffer + * @filep: SEV device file pointer + * @id: command to issue + * @data: command buffer * @error: SEV command return code * * Returns: @@ -1056,8 +1090,8 @@ int sev_issue_cmd_external_user(struct file *filep, unsigned int id, /** * sev_guest_deactivate - perform SEV DEACTIVATE command * - * @deactivate: sev_data_deactivate structure to be processed - * @sev_ret: sev command return code + * @data: sev_data_deactivate structure to be processed + * @error: sev command return code * * Returns: * 0 if the sev successfully processed the command @@ -1071,8 +1105,8 @@ int sev_guest_deactivate(struct sev_data_deactivate *data, int *error); /** * sev_guest_activate - perform SEV ACTIVATE command * - * @activate: sev_data_activate structure to be processed - * @sev_ret: sev command return code + * @data: sev_data_activate structure to be processed + * @error: sev command return code * * Returns: * 0 if the sev successfully processed the command @@ -1086,7 +1120,7 @@ int sev_guest_activate(struct sev_data_activate *data, int *error); /** * sev_guest_df_flush - perform SEV DF_FLUSH command * - * @sev_ret: sev command return code + * @error: sev command return code * * Returns: * 0 if the sev successfully processed the command @@ -1100,8 +1134,8 @@ int sev_guest_df_flush(int *error); /** * sev_guest_decommission - perform SEV DECOMMISSION command * - * @decommission: sev_data_decommission structure to be processed - * @sev_ret: sev command return code + * @data: sev_data_decommission structure to be processed + * @error: sev command return code * * Returns: * 0 if the sev successfully processed the command diff --git a/include/linux/ptdump.h b/include/linux/ptdump.h index 240bd3bff18d..af18d1459b2f 100644 --- a/include/linux/ptdump.h +++ b/include/linux/ptdump.h @@ -31,9 +31,9 @@ bool ptdump_walk_pgd_level_core(struct seq_file *m, void ptdump_walk_pgd(struct ptdump_state *st, struct mm_struct *mm, pgd_t *pgd); bool ptdump_check_wx(void); -static inline void debug_checkwx(void) +static inline void pgtable_checkwx(void) { - if (IS_ENABLED(CONFIG_DEBUG_WX)) + if (IS_ENABLED(CONFIG_CHECK_WX)) ptdump_check_wx(); } diff --git a/include/linux/pwrseq/consumer.h b/include/linux/pwrseq/consumer.h index 16fad5f3e3ab..b69117ad3fa7 100644 --- a/include/linux/pwrseq/consumer.h +++ b/include/linux/pwrseq/consumer.h @@ -25,6 +25,8 @@ int pwrseq_disable(struct pwrseq_desc *desc); struct device *pwrseq_to_device(struct pwrseq_desc *desc); +bool pwrseq_is_controllable(struct pwrseq_desc *desc); + #else /* CONFIG_POWER_SEQUENCING */ static inline struct pwrseq_desc * __must_check @@ -58,6 +60,11 @@ static inline struct device *pwrseq_to_device(struct pwrseq_desc *desc) return NULL; } +static inline bool pwrseq_is_controllable(struct pwrseq_desc *desc) +{ + return false; +} + #endif /* CONFIG_POWER_SEQUENCING */ #endif /* __POWER_SEQUENCING_CONSUMER_H__ */ diff --git a/include/linux/pwrseq/provider.h b/include/linux/pwrseq/provider.h index 33b3d2c2e39d..7285ad94a221 100644 --- a/include/linux/pwrseq/provider.h +++ b/include/linux/pwrseq/provider.h @@ -6,12 +6,15 @@ #ifndef __POWER_SEQUENCING_PROVIDER_H__ #define __POWER_SEQUENCING_PROVIDER_H__ +#include <linux/types.h> + struct device; struct module; struct pwrseq_device; typedef int (*pwrseq_power_state_func)(struct pwrseq_device *); typedef int (*pwrseq_match_func)(struct pwrseq_device *, struct device *); +typedef bool (*pwrseq_is_controllable_func)(struct pwrseq_device *); #define PWRSEQ_NO_MATCH 0 #define PWRSEQ_MATCH_OK 1 @@ -43,11 +46,16 @@ struct pwrseq_unit_data { * the state lock has been released. It's useful for implementing * boot-up delays without blocking other users from powering up * using the same power sequencer. + * @is_controllable: Optional callback checking whether enabling/disabling this + * target actually controls power (for example when the + * controlling GPIO is wired up). If not provided, the + * target's power is assumed to be always controllable. */ struct pwrseq_target_data { const char *name; const struct pwrseq_unit_data *unit; pwrseq_power_state_func post_enable; + pwrseq_is_controllable_func is_controllable; }; /** diff --git a/include/linux/quotaops.h b/include/linux/quotaops.h index f9c0f9d7c9d9..0c64ca674e77 100644 --- a/include/linux/quotaops.h +++ b/include/linux/quotaops.h @@ -20,7 +20,7 @@ static inline struct quota_info *sb_dqopt(struct super_block *sb) } /* i_rwsem must being held */ -static inline bool is_quota_modification(struct mnt_idmap *idmap, +static inline bool is_quota_modification(const struct mnt_idmap *idmap, struct inode *inode, struct iattr *ia) { return ((ia->ia_valid & ATTR_SIZE) || @@ -109,7 +109,7 @@ int dquot_set_dqblk(struct super_block *sb, struct kqid id, struct qc_dqblk *di); int __dquot_transfer(struct inode *inode, struct dquot **transfer_to); -int dquot_transfer(struct mnt_idmap *idmap, struct inode *inode, +int dquot_transfer(const struct mnt_idmap *idmap, struct inode *inode, struct iattr *iattr); static inline struct mem_dqinfo *sb_dqinfo(struct super_block *sb, int type) @@ -229,7 +229,7 @@ static inline void dquot_free_inode(struct inode *inode) { } -static inline int dquot_transfer(struct mnt_idmap *idmap, +static inline int dquot_transfer(const struct mnt_idmap *idmap, struct inode *inode, struct iattr *iattr) { return 0; diff --git a/include/linux/rbtree_augmented.h b/include/linux/rbtree_augmented.h index d2fa1c41bfd2..b9fc1be86ae3 100644 --- a/include/linux/rbtree_augmented.h +++ b/include/linux/rbtree_augmented.h @@ -15,6 +15,8 @@ #include <linux/compiler.h> #include <linux/rbtree.h> #include <linux/rcupdate.h> +#include <linux/args.h> +#include <linux/minmax.h> /* * Please note - only struct rb_augment_callbacks and the prototypes for @@ -28,6 +30,7 @@ struct rb_augment_callbacks { void (*propagate)(struct rb_node *node, struct rb_node *stop); void (*copy)(struct rb_node *old, struct rb_node *new); void (*rotate)(struct rb_node *old, struct rb_node *new); + void (*merge)(struct rb_node *node, struct rb_node *new); }; extern void __rb_insert_augmented(struct rb_node *node, struct rb_root *root, @@ -60,6 +63,12 @@ rb_insert_augmented_cached(struct rb_node *node, rb_insert_augmented(node, &root->rb_root, augment); } +/* + * Insert @node into the leftmost cached augmented tree @tree. + * + * The augmented data of @node must already describe @node alone; it is + * merged into every ancestor on the way down through augment->merge(). + */ static __always_inline struct rb_node * rb_add_augmented_cached(struct rb_node *node, struct rb_root_cached *tree, bool (*less)(struct rb_node *, const struct rb_node *), @@ -71,6 +80,7 @@ rb_add_augmented_cached(struct rb_node *node, struct rb_root_cached *tree, while (*link) { parent = *link; + augment->merge(parent, node); if (less(node, parent)) { link = &parent->rb_left; } else { @@ -80,12 +90,108 @@ rb_add_augmented_cached(struct rb_node *node, struct rb_root_cached *tree, } rb_link_node(node, parent, link); - augment->propagate(parent, NULL); /* suboptimal */ rb_insert_augmented_cached(node, tree, leftmost, augment); return leftmost ? node : NULL; } +#define RB_FOR_EACH_1(what, RBNAME, RBSTRUCT, RBFIELD, x) \ + what(1, RBNAME, RBSTRUCT, RBFIELD, x) + +#define RB_FOR_EACH_2(what, RBNAME, RBSTRUCT, RBFIELD, x, ...) \ + what(2, RBNAME, RBSTRUCT, RBFIELD, x) \ + RB_FOR_EACH_1(what, RBNAME, RBSTRUCT, RBFIELD, __VA_ARGS__) + +#define RB_FOR_EACH_3(what, RBNAME, RBSTRUCT, RBFIELD, x, ...) \ + what(3, RBNAME, RBSTRUCT, RBFIELD, x) \ + RB_FOR_EACH_2(what, RBNAME, RBSTRUCT, RBFIELD, __VA_ARGS__) + +#define RB_FOR_EACH_4(what, RBNAME, RBSTRUCT, RBFIELD, x, ...) \ + what(4, RBNAME, RBSTRUCT, RBFIELD, x) \ + RB_FOR_EACH_3(what, RBNAME, RBSTRUCT, RBFIELD, __VA_ARGS__) + +#define RB_FOR_EACH_5(what, RBNAME, RBSTRUCT, RBFIELD, x, ...) \ + what(5, RBNAME, RBSTRUCT, RBFIELD, x) \ + RB_FOR_EACH_4(what, RBNAME, RBSTRUCT, RBFIELD, __VA_ARGS__) + +#define RB_FOR_EACH_6(what, RBNAME, RBSTRUCT, RBFIELD, x, ...) \ + what(6, RBNAME, RBSTRUCT, RBFIELD, x) \ + RB_FOR_EACH_5(what, RBNAME, RBSTRUCT, RBFIELD, __VA_ARGS__) + +#define RB_FOR_EACH_7(what, RBNAME, RBSTRUCT, RBFIELD, x, ...) \ + what(7, RBNAME, RBSTRUCT, RBFIELD, x) \ + RB_FOR_EACH_6(what, RBNAME, RBSTRUCT, RBFIELD, __VA_ARGS__) + +#define RB_FOR_EACH_8(what, RBNAME, RBSTRUCT, RBFIELD, x, ...) \ + what(8, RBNAME, RBSTRUCT, RBFIELD, x) \ + RB_FOR_EACH_7(what, RBNAME, RBSTRUCT, RBFIELD, __VA_ARGS__) + +#define RB_FOR_EACH(action, RBNAME, RBSTRUCT, RBFIELD, ...) \ + CONCATENATE(RB_FOR_EACH_, COUNT_ARGS(__VA_ARGS__)) \ + (action, RBNAME, RBSTRUCT, RBFIELD, __VA_ARGS__) + +/* + * One augmented field: @val is the node's own contribution (a member for + * RB_AUG(), a function of the node for RB_AUG_FUNC()), @aug the member + * holding the aggregate over the subtree, and @fold(a, b) combines two + * aggregates: min, max, a sum, ... It must be commutative and associative. + */ +#define RB_AUG_FUNC(val, aug, fold) (val(s), aug, fold) +#define RB_AUG(val, aug, fold) (s->val, aug, fold) +#define RB_UNPACK(...) __VA_ARGS__ + +#define __RB_INST(n, RBNAME, RBSTRUCT, RBFIELD, val, aug, fold) \ +static inline void \ +RBNAME ## _copy_ ## n(RBSTRUCT *old, RBSTRUCT *new) \ +{ \ + new->aug = old->aug; \ +} \ +static inline bool \ +RBNAME ## _compute_ ## n(RBSTRUCT *s, bool exit) \ +{ \ + TYPEOF_UNQUAL(s->aug) _old_aug = s->aug; \ + TYPEOF_UNQUAL(s->aug) _val = val; \ + struct rb_node *_node = &s->RBFIELD; \ + if (_node->rb_right) { \ + RBSTRUCT *_c = container_of(_node->rb_right, typeof(*s), RBFIELD); \ + _val = fold(_val, _c->aug); \ + } \ + if (_node->rb_left) { \ + RBSTRUCT *_c = container_of(_node->rb_left, typeof(*s), RBFIELD); \ + _val = fold(_val, _c->aug); \ + } \ + if (exit && _old_aug == _val) \ + return true; \ + s->aug = _val; \ + return false; \ +} +#define _RB_INST(n, RBNAME, RBSTRUCT, RBFIELD, args) \ + __RB_INST(n, RBNAME, RBSTRUCT, RBFIELD, args) +#define RB_INST(n, RBNAME, RBSTRUCT, RBFIELD, x) \ + _RB_INST(n, RBNAME, RBSTRUCT, RBFIELD, RB_UNPACK x) + +#define __RB_COPY(n, RBNAME, RBSTRUCT, RBFIELD, val, aug, fold) \ + RBNAME ## _copy_ ## n(old, new); +#define _RB_COPY(n, RBNAME, RBSTRUCT, RBFIELD, args) \ + __RB_COPY(n, RBNAME, RBSTRUCT, RBFIELD, args) +#define RB_COPY(n, RBNAME, RBSTRUCT, RBFIELD, x) \ + _RB_COPY(n, RBNAME, RBSTRUCT, RBFIELD, RB_UNPACK x) + +#define __RB_COMPUTE(n, RBNAME, RBSTRUCT, RBFIELD, val, aug, fold) \ + ret &= RBNAME ## _compute_ ## n(node, exit); +#define _RB_COMPUTE(n, RBNAME, RBSTRUCT, RBFIELD, args) \ + __RB_COMPUTE(n, RBNAME, RBSTRUCT, RBFIELD, args) +#define RB_COMPUTE(n, RBNAME, RBSTRUCT, RBFIELD, x) \ + _RB_COMPUTE(n, RBNAME, RBSTRUCT, RBFIELD, RB_UNPACK x) + +/* fold @new, about to become a descendant of @node, into @node */ +#define __RB_MERGE(n, RBNAME, RBSTRUCT, RBFIELD, val, aug, fold) \ + node->aug = fold(node->aug, new->aug); +#define _RB_MERGE(n, RBNAME, RBSTRUCT, RBFIELD, args) \ + __RB_MERGE(n, RBNAME, RBSTRUCT, RBFIELD, args) +#define RB_MERGE(n, RBNAME, RBSTRUCT, RBFIELD, x) \ + _RB_MERGE(n, RBNAME, RBSTRUCT, RBFIELD, RB_UNPACK x) + /* * Template for declaring augmented rbtree callbacks (generic multi fields) * @@ -93,18 +199,29 @@ rb_add_augmented_cached(struct rb_node *node, struct rb_root_cached *tree, * RBNAME: name of the rb_augment_callbacks structure * RBSTRUCT: struct type of the tree nodes * RBFIELD: name of struct rb_node field within RBSTRUCT - * RBCOPY: name of function that copies the RBAUGMENTED datas - * RBCOMPUTE: name of function that recomputes the RBAUGMENTED datas + * RBAUG...: list of RB_AUG() describing the augmented data */ - -#define RB_DECLARE_CALLBACKS_MULTI(RBSTATIC, RBNAME, \ - RBSTRUCT, RBFIELD, RBCOPY, RBCOMPUTE) \ +#define RB_DECLARE_CALLBACKS(RBSTATIC, RBNAME, \ + RBSTRUCT, RBFIELD, RBAUG...) \ +RB_FOR_EACH(RB_INST, RBNAME, RBSTRUCT, RBFIELD, RBAUG) \ +static inline void \ +RBNAME ## __copy(RBSTRUCT *old, RBSTRUCT *new) \ +{ \ + RB_FOR_EACH(RB_COPY, RBNAME, RBSTRUCT, RBFIELD, RBAUG); \ +} \ +static inline bool \ +RBNAME ## __compute(RBSTRUCT *node, bool exit) \ +{ \ + bool ret = true; \ + RB_FOR_EACH(RB_COMPUTE, RBNAME, RBSTRUCT, RBFIELD, RBAUG); \ + return ret; \ +} \ static inline void \ RBNAME ## _propagate(struct rb_node *rb, struct rb_node *stop) \ { \ while (rb != stop) { \ RBSTRUCT *node = rb_entry(rb, RBSTRUCT, RBFIELD); \ - if (RBCOMPUTE(node, true)) \ + if (RBNAME ## __compute(node, true)) \ break; \ rb = rb_parent(&node->RBFIELD); \ } \ @@ -114,44 +231,31 @@ RBNAME ## _copy(struct rb_node *rb_old, struct rb_node *rb_new) \ { \ RBSTRUCT *old = rb_entry(rb_old, RBSTRUCT, RBFIELD); \ RBSTRUCT *new = rb_entry(rb_new, RBSTRUCT, RBFIELD); \ - RBCOPY(new, old); \ + RBNAME ## __copy(old, new); \ } \ static void \ RBNAME ## _rotate(struct rb_node *rb_old, struct rb_node *rb_new) \ { \ RBSTRUCT *old = rb_entry(rb_old, RBSTRUCT, RBFIELD); \ RBSTRUCT *new = rb_entry(rb_new, RBSTRUCT, RBFIELD); \ - RBCOPY(new, old); \ - RBCOMPUTE(old, false); \ + RBNAME ## __copy(old, new); \ + RBNAME ## __compute(old, false); \ +} \ +static inline void \ +RBNAME ## _merge(struct rb_node *rb, struct rb_node *rb_new) \ +{ \ + RBSTRUCT *node = rb_entry(rb, RBSTRUCT, RBFIELD); \ + RBSTRUCT *new = rb_entry(rb_new, RBSTRUCT, RBFIELD); \ + RB_FOR_EACH(RB_MERGE, RBNAME, RBSTRUCT, RBFIELD, RBAUG); \ } \ RBSTATIC const struct rb_augment_callbacks RBNAME = { \ .propagate = RBNAME ## _propagate, \ .copy = RBNAME ## _copy, \ - .rotate = RBNAME ## _rotate \ + .rotate = RBNAME ## _rotate, \ + .merge = RBNAME ## _merge \ }; /* - * Template for declaring augmented rbtree callbacks (generic single field) - * - * RBSTATIC: 'static' or empty - * RBNAME: name of the rb_augment_callbacks structure - * RBSTRUCT: struct type of the tree nodes - * RBFIELD: name of struct rb_node field within RBSTRUCT - * RBAUGMENTED: name of field within RBSTRUCT holding data for subtree - * RBCOMPUTE: name of function that recomputes the RBAUGMENTED data - */ - -#define RB_DECLARE_CALLBACKS(RBSTATIC, RBNAME, \ - RBSTRUCT, RBFIELD, RBAUGMENTED, RBCOMPUTE) \ -static inline void \ -RBNAME ## _copy_single(RBSTRUCT *new, RBSTRUCT *old) \ -{ \ - new->RBAUGMENTED = old->RBAUGMENTED; \ -} \ -RB_DECLARE_CALLBACKS_MULTI(RBSTATIC, RBNAME, \ - RBSTRUCT, RBFIELD, RBNAME ## _copy_single, RBCOMPUTE) - -/* * Template for declaring augmented rbtree callbacks, * computing RBAUGMENTED scalar as max(RBCOMPUTE(node)) for all subtree nodes. * @@ -159,34 +263,15 @@ RB_DECLARE_CALLBACKS_MULTI(RBSTATIC, RBNAME, \ * RBNAME: name of the rb_augment_callbacks structure * RBSTRUCT: struct type of the tree nodes * RBFIELD: name of struct rb_node field within RBSTRUCT - * RBTYPE: type of the RBAUGMENTED field - * RBAUGMENTED: name of RBTYPE field within RBSTRUCT holding data for subtree - * RBCOMPUTE: name of function that returns the per-node RBTYPE scalar + * RBTYPE: type of the RBAUGMENTED field -- unused, assumed typeof(RBAUGMENTED) + * RBAUGMENTED: name of field within RBSTRUCT holding data for subtree + * RBVALUE: name of function that returns the per-node RBTYPE scalar */ -#define RB_DECLARE_CALLBACKS_MAX(RBSTATIC, RBNAME, RBSTRUCT, RBFIELD, \ - RBTYPE, RBAUGMENTED, RBCOMPUTE) \ -static inline bool RBNAME ## _compute_max(RBSTRUCT *node, bool exit) \ -{ \ - RBSTRUCT *child; \ - RBTYPE max = RBCOMPUTE(node); \ - if (node->RBFIELD.rb_left) { \ - child = rb_entry(node->RBFIELD.rb_left, RBSTRUCT, RBFIELD); \ - if (child->RBAUGMENTED > max) \ - max = child->RBAUGMENTED; \ - } \ - if (node->RBFIELD.rb_right) { \ - child = rb_entry(node->RBFIELD.rb_right, RBSTRUCT, RBFIELD); \ - if (child->RBAUGMENTED > max) \ - max = child->RBAUGMENTED; \ - } \ - if (exit && node->RBAUGMENTED == max) \ - return true; \ - node->RBAUGMENTED = max; \ - return false; \ -} \ -RB_DECLARE_CALLBACKS(RBSTATIC, RBNAME, \ - RBSTRUCT, RBFIELD, RBAUGMENTED, RBNAME ## _compute_max) +#define RB_DECLARE_CALLBACKS_MAX(RBSTATIC, RBNAME, RBSTRUCT, RBFIELD, \ + RBTYPE, RBAUGMENTED, RBVALUE) \ +RB_DECLARE_CALLBACKS(RBSTATIC, RBNAME, RBSTRUCT, RBFIELD, \ + RB_AUG_FUNC(RBVALUE, RBAUGMENTED, max)) #define RB_RED 0 diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h index 44c07a66edff..3f74ae6d6e1f 100644 --- a/include/linux/rcupdate.h +++ b/include/linux/rcupdate.h @@ -488,7 +488,7 @@ static __always_inline bool lockdep_assert_rcu_helper(bool c, const struct __ctx context_unsafe( \ typeof(*p) *local = (typeof(*p) *__force)(p); \ rcu_check_sparse(p, __rcu); \ - ((typeof(*p) __force __kernel *)(local)) \ + ((TYPEOF_NO_ADDRESS_SPACE(*p) __force __kernel *)(local)) \ ) /** * unrcu_pointer - mark a pointer as not being RCU protected @@ -503,7 +503,7 @@ context_unsafe( \ ({ \ typeof(*p) *local = (typeof(*p) *__force)READ_ONCE(p); \ rcu_check_sparse(p, space); \ - ((typeof(*p) __force __kernel *)(local)); \ + ((TYPEOF_NO_ADDRESS_SPACE(*p) __force __kernel *)(local)); \ }) ) #define __rcu_dereference_check(p, local, c, space) \ ({ \ @@ -511,19 +511,19 @@ context_unsafe( \ typeof(*p) *local = (typeof(*p) *__force)READ_ONCE(p); \ RCU_LOCKDEP_WARN(!(c), "suspicious rcu_dereference_check() usage"); \ rcu_check_sparse(p, space); \ - ((typeof(*p) __force __kernel *)(local)); \ + ((TYPEOF_NO_ADDRESS_SPACE(*p) __force __kernel *)(local)); \ }) #define __rcu_dereference_protected(p, local, c, space) \ ({ \ RCU_LOCKDEP_WARN(!(c), "suspicious rcu_dereference_protected() usage"); \ rcu_check_sparse(p, space); \ - ((typeof(*p) __force __kernel *)(p)); \ + ((TYPEOF_NO_ADDRESS_SPACE(*p) __force __kernel *)(p)); \ }) #define __rcu_dereference_raw(p, local) \ ({ \ /* Dependency order vs. p above. */ \ typeof(p) local = READ_ONCE(p); \ - ((typeof(*p) __force __kernel *)(local)); \ + ((TYPEOF_NO_ADDRESS_SPACE(*p) __force __kernel *)(local)); \ }) #define rcu_dereference_raw(p) __rcu_dereference_raw(p, __UNIQUE_ID(rcu)) diff --git a/include/linux/rcuref.h b/include/linux/rcuref.h index 2fb2af6d9824..01fee161b67c 100644 --- a/include/linux/rcuref.h +++ b/include/linux/rcuref.h @@ -34,7 +34,7 @@ static inline void rcuref_init(rcuref_t *ref, unsigned int cnt) * indicate that it is safe to schedule the object, protected by this reference * counter, for deconstruction. * If you want to know if the reference counter has been marked DEAD (as - * signaled by rcuref_put()) please use rcuread_is_dead(). + * signaled by rcuref_put()) please use rcuref_is_dead(). */ static inline unsigned int rcuref_read(rcuref_t *ref) { diff --git a/include/linux/rcutree.h b/include/linux/rcutree.h index 16a04202888b..e6ad4f2a475c 100644 --- a/include/linux/rcutree.h +++ b/include/linux/rcutree.h @@ -116,7 +116,7 @@ int rcutree_offline_cpu(unsigned int cpu); void rcutree_migrate_callbacks(int cpu); -/* Called from hotplug and also arm64 early secondary boot failure */ +/* Called from hotplug */ void rcutree_report_cpu_dead(void); #endif /* __LINUX_RCUTREE_H */ diff --git a/include/linux/regulator/driver.h b/include/linux/regulator/driver.h index cc6ce709ec86..55cc519103ed 100644 --- a/include/linux/regulator/driver.h +++ b/include/linux/regulator/driver.h @@ -362,7 +362,7 @@ enum regulator_type { * @off_on_delay: guard time (in uS), before re-enabling a regulator * * @poll_enabled_time: The polling interval (in uS) to use while checking that - * the regulator was actually enabled. Max upto enable_time. + * the regulator was actually enabled. Max up to enable_time. * * @of_map_mode: Maps a hardware mode defined in a DeviceTree to a standard mode */ diff --git a/include/linux/regulator/lp872x.h b/include/linux/regulator/lp872x.h deleted file mode 100644 index b62e45aa1dd3..000000000000 --- a/include/linux/regulator/lp872x.h +++ /dev/null @@ -1,86 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright 2012 Texas Instruments - * - * Author: Milo(Woogyom) Kim <milo.kim@ti.com> - */ - -#ifndef __LP872X_REGULATOR_H__ -#define __LP872X_REGULATOR_H__ - -#include <linux/regulator/machine.h> -#include <linux/platform_device.h> -#include <linux/gpio/consumer.h> - -#define LP872X_MAX_REGULATORS 9 - -#define LP8720_ENABLE_DELAY 200 -#define LP8725_ENABLE_DELAY 30000 - -enum lp872x_regulator_id { - LP8720_ID_BASE, - LP8720_ID_LDO1 = LP8720_ID_BASE, - LP8720_ID_LDO2, - LP8720_ID_LDO3, - LP8720_ID_LDO4, - LP8720_ID_LDO5, - LP8720_ID_BUCK, - - LP8725_ID_BASE, - LP8725_ID_LDO1 = LP8725_ID_BASE, - LP8725_ID_LDO2, - LP8725_ID_LDO3, - LP8725_ID_LDO4, - LP8725_ID_LDO5, - LP8725_ID_LILO1, - LP8725_ID_LILO2, - LP8725_ID_BUCK1, - LP8725_ID_BUCK2, - - LP872X_ID_MAX, -}; - -enum lp872x_dvs_sel { - SEL_V1, - SEL_V2, -}; - -/** - * lp872x_dvs - * @gpio : gpio descriptor for dvs control - * @vsel : dvs selector for buck v1 or buck v2 register - * @init_state : initial dvs pin state - */ -struct lp872x_dvs { - struct gpio_desc *gpio; - enum lp872x_dvs_sel vsel; - enum gpiod_flags init_state; -}; - -/** - * lp872x_regdata - * @id : regulator id - * @init_data : init data for each regulator - */ -struct lp872x_regulator_data { - enum lp872x_regulator_id id; - struct regulator_init_data *init_data; -}; - -/** - * lp872x_platform_data - * @general_config : the value of LP872X_GENERAL_CFG register - * @update_config : if LP872X_GENERAL_CFG register is updated, set true - * @regulator_data : platform regulator id and init data - * @dvs : dvs data for buck voltage control - * @enable_gpio : gpio descriptor for enable control - */ -struct lp872x_platform_data { - u8 general_config; - bool update_config; - struct lp872x_regulator_data regulator_data[LP872X_MAX_REGULATORS]; - struct lp872x_dvs *dvs; - struct gpio_desc *enable_gpio; -}; - -#endif diff --git a/include/linux/regulator/pca9450.h b/include/linux/regulator/pca9450.h index 0df8b3c48082..bf94df5fafe3 100644 --- a/include/linux/regulator/pca9450.h +++ b/include/linux/regulator/pca9450.h @@ -210,9 +210,11 @@ enum { #define LDO5L_EN_MASK 0xC0 #define LDO5LOUT_MASK 0x0F -#define LDO5H_EN_MASK 0xC0 #define LDO5HOUT_MASK 0x0F +/* LDO ENMODE value: ON in RUN, OFF while PMIC_STBY_REQ is asserted */ +#define LDO_ENMODE_ONREQ_STBYREQ 0x80 + /* PCA9450_REG_IRQ bits */ #define IRQ_PWRON 0x80 #define IRQ_WDOGB 0x40 diff --git a/include/linux/remoteproc.h b/include/linux/remoteproc.h index a44368737b39..00019b1895e6 100644 --- a/include/linux/remoteproc.h +++ b/include/linux/remoteproc.h @@ -15,6 +15,7 @@ #include <linux/cdev.h> #include <linux/completion.h> #include <linux/idr.h> +#include <linux/srcu.h> #include <linux/of.h> #include <linux/rsc_table.h> @@ -228,10 +229,11 @@ enum rproc_features { * @mappings: list of iommu mappings we initiated, needed on shutdown * @bootaddr: address of first instruction to boot rproc with (optional) * @rvdevs: list of remote virtio devices + * @vq_srcu: SRCU domain for the virtqueue callbacks of @rvdevs * @subdevs: list of subdevices, to following the running state * @notifyids: idr for dynamically assigning rproc-wide unique notify ids * @index: index of this rproc device - * @attach_work: workqueue for attaching rproc + * @boot_work: workqueue for booting rproc * @crash_handler: workqueue for handling a crash * @crash_handler_lock: serializes crash handler queueing and deletion * @deleting: remoteproc deletion has begun @@ -274,10 +276,11 @@ struct rproc { struct list_head mappings; u64 bootaddr; struct list_head rvdevs; + struct srcu_struct vq_srcu; struct list_head subdevs; struct idr notifyids; int index; - struct work_struct attach_work; + struct work_struct boot_work; struct work_struct crash_handler; spinlock_t crash_handler_lock; bool deleting; diff --git a/include/linux/resctrl.h b/include/linux/resctrl.h index dd09c2ce9a0f..10dfdca7f4bf 100644 --- a/include/linux/resctrl.h +++ b/include/linux/resctrl.h @@ -505,6 +505,25 @@ bool resctrl_arch_mbm_cntr_assign_enabled(struct rdt_resource *r); */ int resctrl_arch_mbm_cntr_assign_set(struct rdt_resource *r, bool enable); +/** + * resctrl_arch_preconvert_bw() - Prepare bandwidth control value for arch use. + * @r: Resource whose schema was written. + * @val: Bandwidth control value written to the schemata file by userspace. + * + * Convert the user provided bandwidth control value to an appropriate form for + * consumption by the hardware driver for resource @r. Converted value is stored + * in rdt_ctrl_domain::staged_config[] for later consumption by + * resctrl_arch_update_domains(). Is not called when MBA software controller is + * enabled. + * + * Architectures for which this pre-conversion hook is not useful should supply + * an implementation of this function that just returns @val unmodified. + * + * Return: + * The converted value. + */ +u32 resctrl_arch_preconvert_bw(const struct rdt_resource *r, u32 val); + /* * Update the ctrl_val and apply this config right now. * Must be called on one of the domain's CPUs. diff --git a/include/linux/resctrl_types.h b/include/linux/resctrl_types.h index a5f56faa18d2..e2e2cf3d9ff6 100644 --- a/include/linux/resctrl_types.h +++ b/include/linux/resctrl_types.h @@ -60,6 +60,8 @@ enum resctrl_event_id { PMT_EVENT_AUTO_C6_RES, PMT_EVENT_UNHALTED_REF_CYCLES, PMT_EVENT_UOPS_RETIRED, + PMT_EVENT_INST_RETIRED, + PMT_EVENT_PCNT, /* Must be the last */ QOS_NUM_EVENTS, diff --git a/include/linux/rhashtable-types.h b/include/linux/rhashtable-types.h index 57c11ec9dc64..3576b8f08aff 100644 --- a/include/linux/rhashtable-types.h +++ b/include/linux/rhashtable-types.h @@ -70,6 +70,12 @@ struct rhashtable_params { rht_obj_cmpfn_t obj_cmpfn; }; +struct rhashtable_lockdep_keys { + struct lock_class_key lock_key; + struct lock_class_key mutex_key; + struct lock_class_key bucket_key; +}; + /** * struct rhashtable - Hash table handle * @tbl: Bucket table @@ -97,6 +103,9 @@ struct rhashtable { #ifdef CONFIG_MEM_ALLOC_PROFILING struct alloc_tag *alloc_tag; #endif +#ifdef CONFIG_LOCKDEP + struct lock_class_key *lockdep_key; +#endif }; /** @@ -138,24 +147,77 @@ struct rhashtable_iter { int __rhashtable_init_noprof(struct rhashtable *ht, const struct rhashtable_params *params, - struct lock_class_key *key); + struct rhashtable_lockdep_keys *keys); #define rhashtable_init_noprof(ht, params) \ ({ \ - static struct lock_class_key __key; \ + static struct rhashtable_lockdep_keys __keys; \ \ - __rhashtable_init_noprof(ht, params, &__key); \ + __rhashtable_init_noprof(ht, params, &__keys); \ }) + +/** + * rhashtable_init - initialize a new hash table + * @ht: hash table to be initialized + * @params: configuration parameters + * + * Initializes a new hash table based on the provided configuration + * parameters. A table can be configured either with a variable or + * fixed length key: + * + * Configuration Example 1: Fixed length keys + * struct test_obj { + * int key; + * void * my_member; + * struct rhash_head node; + * }; + * + * struct rhashtable_params params = { + * .head_offset = offsetof(struct test_obj, node), + * .key_offset = offsetof(struct test_obj, key), + * .key_len = sizeof(int), + * .hashfn = jhash, + * }; + * + * Configuration Example 2: Variable length keys + * struct test_obj { + * [...] + * struct rhash_head node; + * }; + * + * u32 my_hash_fn(const void *data, u32 len, u32 seed) + * { + * struct test_obj *obj = data; + * + * return [... hash ...]; + * } + * + * struct rhashtable_params params = { + * .head_offset = offsetof(struct test_obj, node), + * .hashfn = jhash, + * .obj_hashfn = my_hash_fn, + * }; + */ #define rhashtable_init(...) alloc_hooks(rhashtable_init_noprof(__VA_ARGS__)) int __rhltable_init_noprof(struct rhltable *hlt, const struct rhashtable_params *params, - struct lock_class_key *key); + struct rhashtable_lockdep_keys *keys); #define rhltable_init_noprof(hlt, params) \ ({ \ - static struct lock_class_key __key; \ + static struct rhashtable_lockdep_keys __keys; \ \ - __rhltable_init_noprof(hlt, params, &__key); \ + __rhltable_init_noprof(hlt, params, &__keys); \ }) + +/** + * rhltable_init - initialize a new hash list table + * @hlt: hash list table to be initialized + * @params: configuration parameters + * + * Initializes a new hash list table. + * + * See documentation for rhashtable_init. + */ #define rhltable_init(...) alloc_hooks(rhltable_init_noprof(__VA_ARGS__)) #endif /* _LINUX_RHASHTABLE_TYPES_H */ diff --git a/include/linux/rhashtable.h b/include/linux/rhashtable.h index 57a2a29bef0e..70dac9214617 100644 --- a/include/linux/rhashtable.h +++ b/include/linux/rhashtable.h @@ -319,18 +319,6 @@ static inline struct rhash_lock_head __rcu **rht_bucket_insert( * When we write to a bucket without unlocking, we use rht_assign_locked(). */ -static inline unsigned long rht_lock(struct bucket_table *tbl, - struct rhash_lock_head __rcu **bkt) - __acquires(__bitlock(0, bkt)) -{ - unsigned long flags; - - local_irq_save(flags); - bit_spin_lock(0, (unsigned long *)bkt); - lock_map_acquire(&tbl->dep_map); - return flags; -} - static inline unsigned long rht_lock_nested(struct bucket_table *tbl, struct rhash_lock_head __rcu **bucket, unsigned int subclass) @@ -344,6 +332,13 @@ static inline unsigned long rht_lock_nested(struct bucket_table *tbl, return flags; } +static inline unsigned long rht_lock(struct bucket_table *tbl, + struct rhash_lock_head __rcu **bkt) + __acquires(__bitlock(0, bkt)) +{ + return rht_lock_nested(tbl, bkt, 0); +} + static inline void rht_unlock(struct bucket_table *tbl, struct rhash_lock_head __rcu **bkt, unsigned long flags) @@ -598,13 +593,23 @@ static inline void rht_assign_unlock(struct bucket_table *tbl, for (pos = list; pos && rht_entry(tpos, pos, member); \ pos = rcu_dereference_all(pos->next)) -static inline int rhashtable_compare(struct rhashtable_compare_arg *arg, - const void *obj) +/* + * Use constant params from inlined callers to specialize memcmp(). If params + * isn't constant, it must be equal to ht->p. + */ +static __always_inline int rhashtable_compare(struct rhashtable_compare_arg *arg, + const void *obj, + const struct rhashtable_params params) { struct rhashtable *ht = arg->ht; const char *ptr = obj; - return memcmp(ptr + ht->p.key_offset, arg->key, ht->p.key_len); + if (!__builtin_constant_p(params.key_len)) + return memcmp(ptr + ht->p.key_offset, arg->key, + ht->p.key_len); + + return memcmp(ptr + params.key_offset, arg->key, + params.key_len ? : ht->p.key_len); } /* Internal function, do not use. */ @@ -632,7 +637,7 @@ restart: rht_for_each_rcu_from(he, __rht_ptr_rcu(bkt, freq), tbl, hash) { if (params.obj_cmpfn ? params.obj_cmpfn(&arg, rht_obj(ht, he)) : - rhashtable_compare(&arg, rht_obj(ht, he))) + rhashtable_compare(&arg, rht_obj(ht, he), params)) continue; return he; } @@ -797,7 +802,7 @@ slow_path: if (!key || (params.obj_cmpfn ? params.obj_cmpfn(&arg, rht_obj(ht, head)) : - rhashtable_compare(&arg, rht_obj(ht, head)))) { + rhashtable_compare(&arg, rht_obj(ht, head), params))) { pprev = &head->next; continue; } diff --git a/include/linux/ring_buffer.h b/include/linux/ring_buffer.h index eac3e9080c3c..649b2276b557 100644 --- a/include/linux/ring_buffer.h +++ b/include/linux/ring_buffer.h @@ -99,6 +99,13 @@ struct trace_buffer *__ring_buffer_alloc_range(unsigned long size, unsigned flag void *ring_buffer_meta_scratch(struct trace_buffer *buffer, unsigned int *size); /* + * A minimum of 2 pages ensures we can always swap a page out of the ring + * buffer. This does not account for the reader page or any other page + * containing meta data used by static ring buffers. + */ +#define RING_BUFFER_MIN_NR_PAGES 2UL + +/* * Because the ring buffer is generic, if other users of the ring buffer get * traced by ftrace, it can produce lockdep warnings. We need to keep each * ring buffer's lock class separate. @@ -166,7 +173,7 @@ void ring_buffer_iter_reset(struct ring_buffer_iter *iter); int ring_buffer_iter_empty(struct ring_buffer_iter *iter); bool ring_buffer_iter_dropped(struct ring_buffer_iter *iter); -unsigned long ring_buffer_size(struct trace_buffer *buffer, int cpu); +unsigned long ring_buffer_capacity(struct trace_buffer *buffer, int cpu); unsigned long ring_buffer_max_event_size(struct trace_buffer *buffer); void ring_buffer_reset_cpu(struct trace_buffer *buffer, int cpu); @@ -287,7 +294,7 @@ static inline struct ring_buffer_desc *__first_ring_buffer_desc(struct trace_buf static inline unsigned long __calc_nr_pages_ring_buffer_desc(size_t size) { /* Takes into account the reader page */ - return max(DIV_ROUND_UP(size, PAGE_SIZE - BUF_PAGE_HDR_SIZE), 2UL) + 1; + return max(DIV_ROUND_UP(size, PAGE_SIZE - BUF_PAGE_HDR_SIZE), RING_BUFFER_MIN_NR_PAGES) + 1; } static inline size_t trace_buffer_desc_size(size_t buffer_size, unsigned int nr_cpus) diff --git a/include/linux/rmap.h b/include/linux/rmap.h index 0b332770abee..74cca0e3c726 100644 --- a/include/linux/rmap.h +++ b/include/linux/rmap.h @@ -888,7 +888,7 @@ struct page_vma_mapped_walk { static inline void page_vma_mapped_walk_done(struct page_vma_mapped_walk *pvmw) { /* HugeTLB pte is set to the relevant page table entry without pte_mapped. */ - if (pvmw->pte && !is_vm_hugetlb_page(pvmw->vma)) + if (pvmw->pte && !vma_is_hugetlb(pvmw->vma)) pte_unmap(pvmw->pte); if (pvmw->ptl) spin_unlock(pvmw->ptl); diff --git a/include/linux/rpmsg/virtio_rpmsg.h b/include/linux/rpmsg/virtio_rpmsg.h new file mode 100644 index 000000000000..0872cbca4593 --- /dev/null +++ b/include/linux/rpmsg/virtio_rpmsg.h @@ -0,0 +1,41 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) Pinecone Inc. 2019 + * Copyright (C) Xiang Xiao <xiaoxiang@pinecone.net> + * Copyright (C) Advanced Micro Devices, Inc. 2026 + */ + +#ifndef _LINUX_VIRTIO_RPMSG_H +#define _LINUX_VIRTIO_RPMSG_H + +#include <linux/types.h> +#include <linux/virtio_types.h> + +/* The feature bitmap for virtio rpmsg */ +#define VIRTIO_RPMSG_F_NS 0 /* RP supports name service notifications */ +#define VIRTIO_RPMSG_F_BUFSZ 1 /* RP advertises RX/TX buffer sizes from config space */ + +/* Version of struct virtio_rpmsg_config understood by this driver */ +#define RPMSG_VDEV_CONFIG_V1 1 + +/** + * struct virtio_rpmsg_config - config space for rpmsg virtio device + * + * @version: version of this structure, currently %RPMSG_VDEV_CONFIG_V1. + * @size: size of this structure in bytes. + * @txbuf_size: Tx buf size from remote's view. For Linux this is rx buf size. + * @rxbuf_size: Rx buf size from remote's view. For Linux this is tx buf size. + * + * This is the configuration structure shared by the device and the driver, + * read when %VIRTIO_RPMSG_F_BUFSZ is negotiated. The fields are laid out so + * the structure is naturally 32-bit aligned. + */ +struct virtio_rpmsg_config { + u8 version; + __virtio16 size; + /* The tx/rx individual buffer size (if VIRTIO_RPMSG_F_BUFSZ) */ + __virtio32 txbuf_size; + __virtio32 rxbuf_size; +} __packed; + +#endif /* _LINUX_VIRTIO_RPMSG_H */ diff --git a/include/linux/rtc.h b/include/linux/rtc.h index c09fc22819d0..169d93520dbd 100644 --- a/include/linux/rtc.h +++ b/include/linux/rtc.h @@ -13,6 +13,7 @@ #define _LINUX_RTC_H_ +#include <linux/cleanup.h> #include <linux/types.h> #include <linux/interrupt.h> #include <linux/nvmem-provider.h> @@ -167,6 +168,9 @@ struct rtc_device { #define rtc_lock(d) mutex_lock(&d->ops_lock) #define rtc_unlock(d) mutex_unlock(&d->ops_lock) +DEFINE_GUARD(rtc_ops, struct rtc_device *, mutex_lock(&_T->ops_lock), + mutex_unlock(&_T->ops_lock)) + /* useful timestamps */ #define RTC_TIMESTAMP_BEGIN_0000 -62167219200ULL /* 0000-01-01 00:00:00 */ #define RTC_TIMESTAMP_BEGIN_1900 -2208988800LL /* 1900-01-01 00:00:00 */ @@ -201,6 +205,7 @@ extern void rtc_update_irq(struct rtc_device *rtc, extern struct rtc_device *rtc_class_open(const char *name); extern void rtc_class_close(struct rtc_device *rtc); +DEFINE_FREE(rtc_class_close, struct rtc_device *, if (_T) rtc_class_close(_T)); extern int rtc_irq_set_state(struct rtc_device *rtc, int enabled); extern int rtc_irq_set_freq(struct rtc_device *rtc, int freq); diff --git a/include/linux/rtc/rtc-omap.h b/include/linux/rtc/rtc-omap.h index 9f03a329e63f..d6ec7c4daea6 100644 --- a/include/linux/rtc/rtc-omap.h +++ b/include/linux/rtc/rtc-omap.h @@ -3,5 +3,5 @@ #ifndef _LINUX_RTCOMAP_H_ #define _LINUX_RTCOMAP_H_ -int omap_rtc_power_off_program(struct device *dev); +int omap_rtc_power_off_program(void); #endif /* _LINUX_RTCOMAP_H_ */ diff --git a/include/linux/sched.h b/include/linux/sched.h index d35ae49a991f..1c93da75226e 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -554,6 +554,7 @@ struct sched_statistics { u64 nr_failed_migrations_running; u64 nr_failed_migrations_hot; u64 nr_forced_migrations; + u64 nr_migrations_cpu_non_preferred; u64 nr_wakeups; u64 nr_wakeups_sync; @@ -1172,8 +1173,12 @@ struct task_struct { /* Objective and real subjective task credentials (COW): */ const struct cred __rcu *real_cred; - /* Effective (overridable) subjective task credentials (COW): */ - const struct cred __rcu *cred; + /* + * Effective (overridable) subjective task credentials (COW). + * Only accessible for the current task and during task creation/freeing. + * This pointer is not managed by RCU! + */ + const struct cred *cred; #ifdef CONFIG_KEYS /* Cached requested key. */ @@ -1809,15 +1814,15 @@ extern struct pid __rcu *cad_pid; #define PF_USED_MATH 0x00002000 /* If unset the fpu must be initialized before use */ #define PF_USER_WORKER 0x00004000 /* Kernel thread cloned from userspace thread */ #define PF_NOFREEZE 0x00008000 /* This thread should not be frozen */ -#define PF_KCOMPACTD 0x00010000 /* I am kcompactd */ -#define PF_KSWAPD 0x00020000 /* I am kswapd */ +#define PF__HOLE__00010000 0x00010000 +#define PF__HOLE__00020000 0x00020000 #define PF_MEMALLOC_NOFS 0x00040000 /* All allocations inherit GFP_NOFS. See memalloc_nfs_save() */ #define PF_MEMALLOC_NOIO 0x00080000 /* All allocations inherit GFP_NOIO. See memalloc_noio_save() */ #define PF_LOCAL_THROTTLE 0x00100000 /* Throttle writes only against the bdi I write to, * I am cleaning dirty pages from some other bdi. */ #define PF_KTHREAD 0x00200000 /* I am a kernel thread */ #define PF_RANDOMIZE 0x00400000 /* Randomize virtual address space */ -#define PF__HOLE__00800000 0x00800000 +#define PF_NO_NOTIFY_SIGNAL 0x00800000 /* see no_notify_signal_save() */ #define PF__HOLE__01000000 0x01000000 #define PF__HOLE__02000000 0x02000000 #define PF_NO_SETAFFINITY 0x04000000 /* Userland is not allowed to meddle with cpus_mask */ @@ -2105,10 +2110,9 @@ static inline int test_tsk_thread_flag(struct task_struct *tsk, int flag) static inline void set_tsk_need_resched(struct task_struct *tsk) { - if (tracepoint_enabled(sched_set_need_resched_tp) && - !test_tsk_thread_flag(tsk, TIF_NEED_RESCHED)) + if (!test_and_set_tsk_thread_flag(tsk, TIF_NEED_RESCHED) && + tracepoint_enabled(sched_set_need_resched_tp)) __trace_set_need_resched(tsk, TIF_NEED_RESCHED); - set_tsk_thread_flag(tsk,TIF_NEED_RESCHED); } static inline void clear_tsk_need_resched(struct task_struct *tsk) @@ -2135,44 +2139,19 @@ static inline void set_need_resched_current(void) * value indicates whether a reschedule was done in fact. * cond_resched_lock() will drop the spinlock before scheduling, */ -#if !defined(CONFIG_PREEMPTION) || defined(CONFIG_PREEMPT_DYNAMIC) +#if !defined(CONFIG_PREEMPTION) extern int __cond_resched(void); -#if defined(CONFIG_PREEMPT_DYNAMIC) && defined(CONFIG_HAVE_PREEMPT_DYNAMIC_CALL) - -DECLARE_STATIC_CALL(cond_resched, __cond_resched); - -static __always_inline int _cond_resched(void) -{ - return static_call_mod(cond_resched)(); -} - -#elif defined(CONFIG_PREEMPT_DYNAMIC) && defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY) - -extern int dynamic_cond_resched(void); - -static __always_inline int _cond_resched(void) -{ - return dynamic_cond_resched(); -} - -#else /* !CONFIG_PREEMPTION */ - static inline int _cond_resched(void) { return __cond_resched(); } - -#endif /* PREEMPT_DYNAMIC && CONFIG_HAVE_PREEMPT_DYNAMIC_CALL */ - -#else /* CONFIG_PREEMPTION && !CONFIG_PREEMPT_DYNAMIC */ - +#else static inline int _cond_resched(void) { return 0; } - -#endif /* !CONFIG_PREEMPTION || CONFIG_PREEMPT_DYNAMIC */ +#endif #define cond_resched() ({ \ __might_resched(__FILE__, __LINE__, 0); \ diff --git a/include/linux/sched/cputime.h b/include/linux/sched/cputime.h index e90efaf6d26e..694126411dfe 100644 --- a/include/linux/sched/cputime.h +++ b/include/linux/sched/cputime.h @@ -182,9 +182,9 @@ extern unsigned long long task_sched_runtime(struct task_struct *task); #ifdef CONFIG_PARAVIRT -struct static_key; -extern struct static_key paravirt_steal_enabled; -extern struct static_key paravirt_steal_rq_enabled; +#include <linux/jump_label.h> +DECLARE_STATIC_KEY_FALSE(paravirt_steal_enabled); +DECLARE_STATIC_KEY_FALSE(paravirt_steal_rq_enabled); #ifdef CONFIG_HAVE_PV_STEAL_CLOCK_GEN u64 dummy_steal_clock(int cpu); diff --git a/include/linux/sched/ext.h b/include/linux/sched/ext.h index 36c04797436c..f3564581c317 100644 --- a/include/linux/sched/ext.h +++ b/include/linux/sched/ext.h @@ -91,7 +91,7 @@ struct scx_dispatch_q { struct rhash_head hash_node; struct llist_node free_node; struct scx_sched *sched; - struct scx_dsq_pcpu __percpu *pcpu; + struct scx_dsq_pcpu __percpu *pcpu_user; struct rcu_head rcu; }; @@ -104,9 +104,24 @@ enum scx_ent_flags { SCX_TASK_SUB_INIT = 1 << 4, /* task being initialized for a sub sched */ SCX_TASK_IMMED = 1 << 5, /* task is on local DSQ with %SCX_ENQ_IMMED */ SCX_TASK_PROTECTED = 1 << 6, /* slice and DSQ head position protected */ + SCX_TASK_RUN_TRACKED = 1 << 7, /* task is in an ops.running()/stopping() session */ /* - * Bits 8 to 10 are used to carry task state: + * The kernel owns the slice, currently the rescue grant. While set: + * + * - refill_task_slice_dfl() and expire_task_slice() leave the slice + * alone. + * - Reenqueue scans skip the task. + * - put_prev_task_scx() keeps a preempted task on the local DSQ. + * - Schedulers may shorten the slice, never extend it. + */ + SCX_TASK_KERNEL_SLICE = 1 << 8, /* slice owned by the kernel */ + + /* + * The multi-bit fields are packed at the top, below SCX_TASK_CURSOR, so + * that the single-bit flags stay contiguous from bit 0. + * + * Bits 25 to 27 are used to carry task state: * * NONE ops.init_task() not called yet * INIT_BEGIN ops.init_task() in flight; see sched_ext_dead() @@ -115,7 +130,7 @@ enum scx_ent_flags { * ENABLED fully initialized and in sched_ext * DEAD terminal state set by sched_ext_dead() */ - SCX_TASK_STATE_SHIFT = 8, + SCX_TASK_STATE_SHIFT = 25, SCX_TASK_STATE_BITS = 3, SCX_TASK_STATE_MASK = ((1 << SCX_TASK_STATE_BITS) - 1) << SCX_TASK_STATE_SHIFT, @@ -127,7 +142,7 @@ enum scx_ent_flags { SCX_TASK_DEAD = 5 << SCX_TASK_STATE_SHIFT, /* - * Bits 12 to 14 are used to carry reenqueue reason. In addition to + * Bits 28 to 30 are used to carry reenqueue reason. In addition to * %SCX_ENQ_REENQ flag, ops.enqueue() can also test for * %SCX_TASK_REENQ_REASON_NONE to distinguish reenqueues. * @@ -136,8 +151,9 @@ enum scx_ent_flags { * IMMED reenqueued due to failed ENQ_IMMED * PREEMPTED preempted while running * CAP sub-sched cap miss, see p->scx.reenq_reason_* + * PROXY proxy state prevented a remote DSQ transfer */ - SCX_TASK_REENQ_REASON_SHIFT = 12, + SCX_TASK_REENQ_REASON_SHIFT = 28, SCX_TASK_REENQ_REASON_BITS = 3, SCX_TASK_REENQ_REASON_MASK = ((1 << SCX_TASK_REENQ_REASON_BITS) - 1) << SCX_TASK_REENQ_REASON_SHIFT, @@ -146,6 +162,7 @@ enum scx_ent_flags { SCX_TASK_REENQ_IMMED = 2 << SCX_TASK_REENQ_REASON_SHIFT, SCX_TASK_REENQ_PREEMPTED = 3 << SCX_TASK_REENQ_REASON_SHIFT, SCX_TASK_REENQ_CAP = 4 << SCX_TASK_REENQ_REASON_SHIFT, + SCX_TASK_REENQ_PROXY = 5 << SCX_TASK_REENQ_REASON_SHIFT, /* iteration cursor, not a task */ SCX_TASK_CURSOR = 1 << 31, @@ -154,6 +171,7 @@ enum scx_ent_flags { /* scx_entity.dsq_flags */ enum scx_ent_dsq_flags { SCX_TASK_DSQ_ON_PRIQ = 1 << 0, /* task is queued on the priority queue of a dsq */ + SCX_TASK_DSQ_REENQ = 1 << 1, /* marked by scx_bpf_dsq_reenq_task() */ }; enum scx_dsq_lnode_flags { @@ -197,9 +215,19 @@ struct sched_ext_entity { u64 ddsp_slice; u64 ddsp_vtime; struct scx_dsq_list_node dsq_list; /* dispatch order */ - struct rb_node dsq_priq; /* p->scx.dsq_vtime order */ u32 dsq_seq; u32 dsq_flags; /* protected by DSQ lock */ + + /* + * Used to order tasks when dispatching to the vtime-ordered priority + * queue of a dsq. This is usually set through + * scx_bpf_dsq_insert_vtime() but can also be modified directly by the + * BPF scheduler. Modifying it while a task is queued on a dsq may + * mangle the ordering and is not recommended. Kept next to @dsq_priq + * as rbtree insertion reads both on every visited node. + */ + u64 dsq_vtime; + struct rb_node dsq_priq; /* p->scx.dsq_vtime order */ u32 flags; /* protected by rq lock */ u32 weight; u32 reenq_cnt; /* reenqueues since last run */ @@ -226,7 +254,7 @@ struct sched_ext_entity { u64 tid; struct rhash_head tid_hash_node; /* see SCX_OPS_TID_TO_TASK */ - /* BPF scheduler modifiable fields */ + /* BPF scheduler modifiable fields, along with @dsq_vtime above */ /* * Runtime budget in nsecs - how long the task may hold its cpu. Owned @@ -242,15 +270,6 @@ struct sched_ext_entity { u64 slice; /* - * Used to order tasks when dispatching to the vtime-ordered priority - * queue of a dsq. This is usually set through - * scx_bpf_dsq_insert_vtime() but can also be modified directly by the - * BPF scheduler. Modifying it while a task is queued on a dsq may - * mangle the ordering and is not recommended. - */ - u64 dsq_vtime; - - /* * Out-of-band slice request from scx_bpf_task_set_slice() when the * caller does not hold the rq lock, applied under the rq lock at the * next slice consideration. One atomic64 packs the pending flag, the @@ -279,6 +298,14 @@ struct sched_ext_entity { */ bool disallow; /* reject switching into SCX */ + /* + * If set, depletion of this task's slice at the scheduler tick requests + * lazy instead of immediate rescheduling. Initialized from + * %SCX_OPS_LAZY_RESCHED immediately before ops.enable() and may be + * modified afterwards with scx_bpf_task_set_lazy_resched(). + */ + bool lazy_resched; + /* cold fields */ #ifdef CONFIG_EXT_GROUP_SCHED struct cgroup *cgrp_moving_from; @@ -324,7 +351,7 @@ struct scx_task_group { u64 bw_period_us; u64 bw_quota_us; u64 bw_burst_us; - bool idle; + bool sched_idle; #endif }; diff --git a/include/linux/sched/signal.h b/include/linux/sched/signal.h index d45a5476b97d..0adb51ab1542 100644 --- a/include/linux/sched/signal.h +++ b/include/linux/sched/signal.h @@ -2,6 +2,7 @@ #ifndef _LINUX_SCHED_SIGNAL_H #define _LINUX_SCHED_SIGNAL_H +#include <linux/cleanup.h> #include <linux/rculist.h> #include <linux/signal.h> #include <linux/sched.h> @@ -79,9 +80,9 @@ struct core_thread { }; struct core_state { - atomic_t nr_threads; - struct core_thread dumper; - struct completion startup; + /* Threads the dumper still waits for. */ + atomic_t threads_remaining; + struct core_thread *tasks; }; /* @@ -130,8 +131,8 @@ struct signal_struct { * process will inherit a flag if they should look for a * child_subreaper process at exit. */ - unsigned int is_child_subreaper:1; - unsigned int has_child_subreaper:1; + bool is_child_subreaper; + bool has_child_subreaper; unsigned int autoreap:1; #ifdef CONFIG_POSIX_TIMERS @@ -384,14 +385,36 @@ static inline int task_sigpending(struct task_struct *p) return unlikely(test_tsk_thread_flag(p,TIF_SIGPENDING)); } +/* Prevent TIF_NOTIFY_SIGNAL from interrupting this task. */ +static inline unsigned int no_notify_signal_save(void) +{ + unsigned int flags = current->flags; + + current->flags |= PF_NO_NOTIFY_SIGNAL; + return flags; +} + +/* Restore the previous PF_NO_NOTIFY_SIGNAL state. */ +static inline void no_notify_signal_restore(unsigned int flags) +{ + current_restore_flags(flags, PF_NO_NOTIFY_SIGNAL); +} + +DEFINE_LOCK_GUARD_0(no_notify_signal, + _T->flags = no_notify_signal_save(), + no_notify_signal_restore(_T->flags), + unsigned int flags) + static inline int signal_pending(struct task_struct *p) { /* * TIF_NOTIFY_SIGNAL isn't really a signal, but it requires the same * behavior in terms of ensuring that we break out of wait loops - * so that notify signal callbacks can be processed. + * so that notify signal callbacks can be processed. Not for a task + * that asked not to be interrupted by it, see no_notify_signal_save(). */ - if (unlikely(test_tsk_thread_flag(p, TIF_NOTIFY_SIGNAL))) + if (unlikely(test_tsk_thread_flag(p, TIF_NOTIFY_SIGNAL)) && + likely(!(READ_ONCE(p->flags) & PF_NO_NOTIFY_SIGNAL))) return 1; return task_sigpending(p); } @@ -674,6 +697,24 @@ struct pid *task_pid_type(struct task_struct *task, enum pid_type type) return pid; } +/* Fill @pids with the pid types of @task up to @last, without references. */ +static inline void __task_pids(struct pid **pids, enum pid_type last, + struct task_struct *task) +{ + for (enum pid_type type = PIDTYPE_PID; type <= last; type++) + pids[type] = task_pid_type(task, type); +} + +static inline void __get_task_pids(struct pid **pids, enum pid_type last, + struct task_struct *task) +{ + for (enum pid_type type = PIDTYPE_PID; type <= last; type++) + pids[type] = get_pid(task_pid_type(task, type)); +} + +#define task_pids(pids, task) __task_pids(pids, pids_last(pids), task) +#define get_task_pids(pids, task) __get_task_pids(pids, pids_last(pids), task) + static inline struct pid *task_tgid(struct task_struct *task) { return task->signal->pids[PIDTYPE_TGID]; diff --git a/include/linux/sched/stat.h b/include/linux/sched/stat.h index 0108a38bb64d..4810b14d0372 100644 --- a/include/linux/sched/stat.h +++ b/include/linux/sched/stat.h @@ -20,7 +20,7 @@ extern int nr_processes(void); extern unsigned int nr_running(void); extern bool single_task_running(void); extern unsigned int nr_iowait(void); -extern unsigned int nr_iowait_cpu(int cpu); +extern bool cpu_has_iowait(int cpu); static inline int sched_info_on(void) { diff --git a/include/linux/sched/task.h b/include/linux/sched/task.h index e0c1ca8c6a18..90ed5bc3c7af 100644 --- a/include/linux/sched/task.h +++ b/include/linux/sched/task.h @@ -94,7 +94,6 @@ static inline void exit_thread(struct task_struct *tsk) extern __noreturn void do_group_exit(int); extern void exit_files(struct task_struct *); -extern void exit_itimers(struct task_struct *); extern pid_t kernel_clone(struct kernel_clone_args *kargs); struct task_struct *copy_process(struct pid *pid, int trace, int node, diff --git a/include/linux/sched/user.h b/include/linux/sched/user.h index 8d7e5521f7cd..ec262b605437 100644 --- a/include/linux/sched/user.h +++ b/include/linux/sched/user.h @@ -23,11 +23,13 @@ struct user_struct { struct hlist_node uidhash_node; kuid_t uid; -#if defined(CONFIG_PERF_EVENTS) || defined(CONFIG_BPF_SYSCALL) || \ - defined(CONFIG_NET) || defined(CONFIG_IO_URING) || \ +#if defined(CONFIG_NET) || defined(CONFIG_IO_URING) || \ defined(CONFIG_VFIO_PCI_ZDEV_KVM) || IS_ENABLED(CONFIG_IOMMUFD) || \ defined(CONFIG_SECRETMEM) - atomic_long_t locked_vm; + atomic_long_t locked_vm; /* pinned pages charged to RLIMIT_MEMLOCK */ +#endif +#ifdef CONFIG_PERF_EVENTS + atomic_long_t perf_mlock; /* perf buffer pages within perf_event_mlock_kb */ #endif #ifdef CONFIG_WATCH_QUEUE atomic_t nr_watches; /* The number of watches this user currently has */ diff --git a/include/linux/scmi_protocol.h b/include/linux/scmi_protocol.h index 5ab73b1ab9aa..3f7cc35eb861 100644 --- a/include/linux/scmi_protocol.h +++ b/include/linux/scmi_protocol.h @@ -9,6 +9,7 @@ #define _LINUX_SCMI_PROTOCOL_H #include <linux/bitfield.h> +#include <linux/device-id/scmi.h> #include <linux/device.h> #include <linux/notifier.h> #include <linux/types.h> @@ -528,22 +529,25 @@ struct scmi_sensor_proto_ops { u32 sensor_id, u32 sensor_config); }; +struct scmi_reset_domain_info { + char name[SCMI_MAX_STR_SIZE]; + u32 latency_us; +}; + /** * struct scmi_reset_proto_ops - represents the various operations provided * by SCMI Reset Protocol * * @num_domains_get: get the count of reset domains provided by SCMI - * @name_get: gets the name of a reset domain - * @latency_get: gets the reset latency for the specified reset domain + * @info_get: gets the information of the specified reset domain * @reset: resets the specified reset domain * @assert: explicitly assert reset signal of the specified reset domain * @deassert: explicitly deassert reset signal of the specified reset domain */ struct scmi_reset_proto_ops { int (*num_domains_get)(const struct scmi_protocol_handle *ph); - const char *(*name_get)(const struct scmi_protocol_handle *ph, - u32 domain); - int (*latency_get)(const struct scmi_protocol_handle *ph, u32 domain); + const struct scmi_reset_domain_info __must_check *(*info_get) + (const struct scmi_protocol_handle *ph, u32 domain); int (*reset)(const struct scmi_protocol_handle *ph, u32 domain); int (*assert)(const struct scmi_protocol_handle *ph, u32 domain); int (*deassert)(const struct scmi_protocol_handle *ph, u32 domain); @@ -930,6 +934,7 @@ enum scmi_std_protocol { SCMI_PROTOCOL_VOLTAGE = 0x17, SCMI_PROTOCOL_POWERCAP = 0x18, SCMI_PROTOCOL_PINCTRL = 0x19, + SCMI_PROTOCOL_TELEMETRY = 0x1B, }; enum scmi_system_events { @@ -951,11 +956,6 @@ struct scmi_device { #define to_scmi_dev(d) container_of_const(d, struct scmi_device, dev) -struct scmi_device_id { - u8 protocol_id; - const char *name; -}; - struct scmi_driver { const char *name; int (*probe)(struct scmi_device *sdev); diff --git a/include/linux/seccomp.h b/include/linux/seccomp.h index fcb3eb9825e5..3b6cc376fff5 100644 --- a/include/linux/seccomp.h +++ b/include/linux/seccomp.h @@ -10,7 +10,8 @@ SECCOMP_FILTER_FLAG_SPEC_ALLOW | \ SECCOMP_FILTER_FLAG_NEW_LISTENER | \ SECCOMP_FILTER_FLAG_TSYNC_ESRCH | \ - SECCOMP_FILTER_FLAG_WAIT_KILLABLE_RECV) + SECCOMP_FILTER_FLAG_WAIT_KILLABLE_RECV | \ + SECCOMP_FILTER_FLAG_RESTART_BEFORE_RECV) /* sizeof() the first published struct seccomp_notif_addfd */ #define SECCOMP_NOTIFY_ADDFD_SIZE_VER0 24 diff --git a/include/linux/security.h b/include/linux/security.h index 153e9043058f..40831fab5470 100644 --- a/include/linux/security.h +++ b/include/linux/security.h @@ -67,6 +67,7 @@ enum fs_value_type; struct watch; struct watch_notification; struct lsm_ctx; +struct nsset; /* Default (no) options for the capable function */ #define CAP_OPT_NONE 0x0 @@ -80,6 +81,7 @@ struct lsm_ctx; struct ctl_table; struct audit_krule; +struct ns_common; struct user_namespace; struct timezone; @@ -185,11 +187,11 @@ extern int cap_capset(struct cred *new, const struct cred *old, extern int cap_bprm_creds_from_file(struct linux_binprm *bprm, const struct file *file); int cap_inode_setxattr(struct dentry *dentry, const char *name, const void *value, size_t size, int flags); -int cap_inode_removexattr(struct mnt_idmap *idmap, +int cap_inode_removexattr(const struct mnt_idmap *idmap, struct dentry *dentry, const char *name); int cap_inode_need_killpriv(struct dentry *dentry); -int cap_inode_killpriv(struct mnt_idmap *idmap, struct dentry *dentry); -int cap_inode_getsecurity(struct mnt_idmap *idmap, +int cap_inode_killpriv(const struct mnt_idmap *idmap, struct dentry *dentry); +int cap_inode_getsecurity(const struct mnt_idmap *idmap, struct inode *inode, const char *name, void **buffer, bool alloc); extern int cap_mmap_addr(unsigned long addr); @@ -404,49 +406,53 @@ int security_inode_init_security(struct inode *inode, struct inode *dir, int security_inode_init_security_anon(struct inode *inode, const struct qstr *name, const struct inode *context_inode); -int security_inode_create(struct inode *dir, struct dentry *dentry, umode_t mode); -void security_inode_post_create_tmpfile(struct mnt_idmap *idmap, +int security_inode_create(const struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, umode_t mode); +void security_inode_post_create_tmpfile(const struct mnt_idmap *idmap, struct inode *inode); -int security_inode_link(struct dentry *old_dentry, struct inode *dir, - struct dentry *new_dentry); +int security_inode_link(const struct mnt_idmap *idmap, struct dentry *old_dentry, + struct inode *dir, struct dentry *new_dentry); int security_inode_unlink(struct inode *dir, struct dentry *dentry); -int security_inode_symlink(struct inode *dir, struct dentry *dentry, - const char *old_name); -int security_inode_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode); +int security_inode_symlink(const struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, const char *old_name); +int security_inode_mkdir(const struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, umode_t mode); int security_inode_rmdir(struct inode *dir, struct dentry *dentry); -int security_inode_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, dev_t dev); +int security_inode_mknod(const struct mnt_idmap *idmap, struct inode *dir, + struct dentry *dentry, umode_t mode, dev_t dev); int security_inode_rename(struct inode *old_dir, struct dentry *old_dentry, struct inode *new_dir, struct dentry *new_dentry, unsigned int flags); int security_inode_readlink(struct dentry *dentry); int security_inode_follow_link(struct dentry *dentry, struct inode *inode, bool rcu); -int security_inode_permission(struct inode *inode, int mask); -int security_inode_setattr(struct mnt_idmap *idmap, +int security_inode_permission(const struct mnt_idmap *idmap, + struct inode *inode, int mask); +int security_inode_setattr(const struct mnt_idmap *idmap, struct dentry *dentry, struct iattr *attr); -void security_inode_post_setattr(struct mnt_idmap *idmap, struct dentry *dentry, +void security_inode_post_setattr(const struct mnt_idmap *idmap, struct dentry *dentry, int ia_valid); int security_inode_getattr(const struct path *path); -int security_inode_setxattr(struct mnt_idmap *idmap, +int security_inode_setxattr(const struct mnt_idmap *idmap, struct dentry *dentry, const char *name, const void *value, size_t size, int flags); -int security_inode_set_acl(struct mnt_idmap *idmap, +int security_inode_set_acl(const struct mnt_idmap *idmap, struct dentry *dentry, const char *acl_name, struct posix_acl *kacl); void security_inode_post_set_acl(struct dentry *dentry, const char *acl_name, struct posix_acl *kacl); -int security_inode_get_acl(struct mnt_idmap *idmap, +int security_inode_get_acl(const struct mnt_idmap *idmap, struct dentry *dentry, const char *acl_name); -int security_inode_remove_acl(struct mnt_idmap *idmap, +int security_inode_remove_acl(const struct mnt_idmap *idmap, struct dentry *dentry, const char *acl_name); -void security_inode_post_remove_acl(struct mnt_idmap *idmap, +void security_inode_post_remove_acl(const struct mnt_idmap *idmap, struct dentry *dentry, const char *acl_name); void security_inode_post_setxattr(struct dentry *dentry, const char *name, const void *value, size_t size, int flags); int security_inode_getxattr(struct dentry *dentry, const char *name); int security_inode_listxattr(struct dentry *dentry); -int security_inode_removexattr(struct mnt_idmap *idmap, +int security_inode_removexattr(const struct mnt_idmap *idmap, struct dentry *dentry, const char *name); void security_inode_post_removexattr(struct dentry *dentry, const char *name); int security_inode_file_setattr(struct dentry *dentry, @@ -454,8 +460,8 @@ int security_inode_file_setattr(struct dentry *dentry, int security_inode_file_getattr(struct dentry *dentry, struct file_kattr *fa); int security_inode_need_killpriv(struct dentry *dentry); -int security_inode_killpriv(struct mnt_idmap *idmap, struct dentry *dentry); -int security_inode_getsecurity(struct mnt_idmap *idmap, +int security_inode_killpriv(const struct mnt_idmap *idmap, struct dentry *dentry); +int security_inode_getsecurity(const struct mnt_idmap *idmap, struct inode *inode, const char *name, void **buffer, bool alloc); int security_inode_setsecurity(struct inode *inode, const char *name, const void *value, size_t size, int flags); @@ -540,6 +546,9 @@ int security_task_prctl(int option, unsigned long arg2, unsigned long arg3, unsigned long arg4, unsigned long arg5); void security_task_to_inode(struct task_struct *p, struct inode *inode); int security_create_user_ns(const struct cred *cred); +int security_namespace_init(struct ns_common *ns); +void security_namespace_free(struct ns_common *ns); +int security_namespace_install(const struct nsset *nsset, struct ns_common *ns); int security_ipc_permission(struct kern_ipc_perm *ipcp, short flag); void security_ipc_getlsmprop(struct kern_ipc_perm *ipcp, struct lsm_prop *prop); int security_msg_msg_alloc(struct msg_msg *msg); @@ -902,20 +911,22 @@ static inline int security_inode_init_security_anon(struct inode *inode, return 0; } -static inline int security_inode_create(struct inode *dir, - struct dentry *dentry, - umode_t mode) +static inline int security_inode_create(const struct mnt_idmap *idmap, + struct inode *dir, + struct dentry *dentry, + umode_t mode) { return 0; } static inline void -security_inode_post_create_tmpfile(struct mnt_idmap *idmap, struct inode *inode) +security_inode_post_create_tmpfile(const struct mnt_idmap *idmap, struct inode *inode) { } -static inline int security_inode_link(struct dentry *old_dentry, - struct inode *dir, - struct dentry *new_dentry) +static inline int security_inode_link(const struct mnt_idmap *idmap, + struct dentry *old_dentry, + struct inode *dir, + struct dentry *new_dentry) { return 0; } @@ -926,16 +937,18 @@ static inline int security_inode_unlink(struct inode *dir, return 0; } -static inline int security_inode_symlink(struct inode *dir, - struct dentry *dentry, - const char *old_name) +static inline int security_inode_symlink(const struct mnt_idmap *idmap, + struct inode *dir, + struct dentry *dentry, + const char *old_name) { return 0; } -static inline int security_inode_mkdir(struct inode *dir, - struct dentry *dentry, - int mode) +static inline int security_inode_mkdir(const struct mnt_idmap *idmap, + struct inode *dir, + struct dentry *dentry, + int mode) { return 0; } @@ -946,9 +959,10 @@ static inline int security_inode_rmdir(struct inode *dir, return 0; } -static inline int security_inode_mknod(struct inode *dir, - struct dentry *dentry, - int mode, dev_t dev) +static inline int security_inode_mknod(const struct mnt_idmap *idmap, + struct inode *dir, + struct dentry *dentry, + int mode, dev_t dev) { return 0; } @@ -974,12 +988,13 @@ static inline int security_inode_follow_link(struct dentry *dentry, return 0; } -static inline int security_inode_permission(struct inode *inode, int mask) +static inline int security_inode_permission(const struct mnt_idmap *idmap, + struct inode *inode, int mask) { return 0; } -static inline int security_inode_setattr(struct mnt_idmap *idmap, +static inline int security_inode_setattr(const struct mnt_idmap *idmap, struct dentry *dentry, struct iattr *attr) { @@ -987,7 +1002,7 @@ static inline int security_inode_setattr(struct mnt_idmap *idmap, } static inline void -security_inode_post_setattr(struct mnt_idmap *idmap, struct dentry *dentry, +security_inode_post_setattr(const struct mnt_idmap *idmap, struct dentry *dentry, int ia_valid) { } @@ -996,14 +1011,14 @@ static inline int security_inode_getattr(const struct path *path) return 0; } -static inline int security_inode_setxattr(struct mnt_idmap *idmap, +static inline int security_inode_setxattr(const struct mnt_idmap *idmap, struct dentry *dentry, const char *name, const void *value, size_t size, int flags) { return cap_inode_setxattr(dentry, name, value, size, flags); } -static inline int security_inode_set_acl(struct mnt_idmap *idmap, +static inline int security_inode_set_acl(const struct mnt_idmap *idmap, struct dentry *dentry, const char *acl_name, struct posix_acl *kacl) @@ -1016,21 +1031,21 @@ static inline void security_inode_post_set_acl(struct dentry *dentry, struct posix_acl *kacl) { } -static inline int security_inode_get_acl(struct mnt_idmap *idmap, +static inline int security_inode_get_acl(const struct mnt_idmap *idmap, struct dentry *dentry, const char *acl_name) { return 0; } -static inline int security_inode_remove_acl(struct mnt_idmap *idmap, +static inline int security_inode_remove_acl(const struct mnt_idmap *idmap, struct dentry *dentry, const char *acl_name) { return 0; } -static inline void security_inode_post_remove_acl(struct mnt_idmap *idmap, +static inline void security_inode_post_remove_acl(const struct mnt_idmap *idmap, struct dentry *dentry, const char *acl_name) { } @@ -1050,7 +1065,7 @@ static inline int security_inode_listxattr(struct dentry *dentry) return 0; } -static inline int security_inode_removexattr(struct mnt_idmap *idmap, +static inline int security_inode_removexattr(const struct mnt_idmap *idmap, struct dentry *dentry, const char *name) { @@ -1078,13 +1093,13 @@ static inline int security_inode_need_killpriv(struct dentry *dentry) return cap_inode_need_killpriv(dentry); } -static inline int security_inode_killpriv(struct mnt_idmap *idmap, +static inline int security_inode_killpriv(const struct mnt_idmap *idmap, struct dentry *dentry) { return cap_inode_killpriv(idmap, dentry); } -static inline int security_inode_getsecurity(struct mnt_idmap *idmap, +static inline int security_inode_getsecurity(const struct mnt_idmap *idmap, struct inode *inode, const char *name, void **buffer, bool alloc) @@ -1431,6 +1446,21 @@ static inline int security_create_user_ns(const struct cred *cred) return 0; } +static inline int security_namespace_init(struct ns_common *ns) +{ + return 0; +} + +static inline void security_namespace_free(struct ns_common *ns) +{ +} + +static inline int security_namespace_install(const struct nsset *nsset, + struct ns_common *ns) +{ + return 0; +} + static inline int security_ipc_permission(struct kern_ipc_perm *ipcp, short flag) { @@ -2085,7 +2115,7 @@ int security_path_mkdir(const struct path *dir, struct dentry *dentry, umode_t m int security_path_rmdir(const struct path *dir, struct dentry *dentry); int security_path_mknod(const struct path *dir, struct dentry *dentry, umode_t mode, unsigned int dev); -void security_path_post_mknod(struct mnt_idmap *idmap, struct dentry *dentry); +void security_path_post_mknod(const struct mnt_idmap *idmap, struct dentry *dentry); int security_path_truncate(const struct path *path); int security_path_symlink(const struct path *dir, struct dentry *dentry, const char *old_name); @@ -2120,7 +2150,7 @@ static inline int security_path_mknod(const struct path *dir, struct dentry *den return 0; } -static inline void security_path_post_mknod(struct mnt_idmap *idmap, +static inline void security_path_post_mknod(const struct mnt_idmap *idmap, struct dentry *dentry) { } diff --git a/include/linux/seq_buf.h b/include/linux/seq_buf.h index 9f2839e73f8a..f006f2654e4c 100644 --- a/include/linux/seq_buf.h +++ b/include/linux/seq_buf.h @@ -5,6 +5,7 @@ #include <linux/bug.h> #include <linux/minmax.h> #include <linux/seq_file.h> +#include <linux/string.h> #include <linux/types.h> /* @@ -30,6 +31,10 @@ struct seq_buf { .size = SIZE, \ } +/** + * seq_buf_clear - reset the seq_buf to be read / appended from the beginning + * @s: the seq_buf handle + */ static inline void seq_buf_clear(struct seq_buf *s) { s->len = 0; @@ -37,6 +42,14 @@ static inline void seq_buf_clear(struct seq_buf *s) s->buffer[0] = '\0'; } +/** + * seq_buf_init - initialize a seq_buf + * @s: the seq_buf handle + * @buf: pointer to the buffer + * @size: total size of @buf + * + * The contents of the buffer are ignored. + */ static inline void seq_buf_init(struct seq_buf *s, char *buf, unsigned int size) { @@ -55,12 +68,46 @@ seq_buf_has_overflowed(struct seq_buf *s) return s->len > s->size; } +/* + * Mark @s as overflowed, which discards the length of what it holds. The + * bytes up to its last one are the string from then on, as that is where + * seq_buf_str() terminates it, so clear whatever was not written: a writer + * sets len to how much it filled, and anything past that was never adopted. + */ static inline void seq_buf_set_overflow(struct seq_buf *s) { + if (s->len < s->size) + memset(s->buffer + s->len, 0, s->size - s->len); + s->len = s->size + 1; } +/** + * seq_buf_init_append - initialize a seq_buf over a buffer that may + * already hold NUL-terminated content + * @s: the seq_buf handle + * @buf: pointer to the (possibly non-empty) buffer + * @size: total size of @buf + * + * Unlike seq_buf_init(), which always starts @buf at len=0, this + * preserves whatever NUL-terminated content @buf already holds and + * positions @s to append after it. Useful for converting code that used + * to append to an existing buffer with strlcat()/scnprintf() and friends. + * + * If @buf holds no NUL within @size, @s starts out overflowed, as + * strlcat() treats such a buffer as already truncated. + */ +static inline void +seq_buf_init_append(struct seq_buf *s, char *buf, unsigned int size) +{ + s->buffer = buf; + s->size = size; + s->len = strnlen(buf, size); + if (s->len == size) + seq_buf_set_overflow(s); +} + /* * How much buffer is left on the seq_buf? */ @@ -79,6 +126,27 @@ static inline unsigned int seq_buf_used(struct seq_buf *s) return min(s->len, s->size); } +/* + * NUL-terminate the buffer in @s: directly after the data when there is + * room for it, otherwise in the last byte of the buffer. @s->size must not + * be zero. + * + * Returns: the offset of the NUL. + */ +static inline size_t __seq_buf_terminate(struct seq_buf *s) +{ + size_t end; + + if (seq_buf_buffer_left(s)) + end = s->len; + else + end = s->size - 1; + + s->buffer[end] = 0; + + return end; +} + /** * seq_buf_str - get NUL-terminated C string from seq_buf * @s: the seq_buf handle @@ -88,7 +156,14 @@ static inline unsigned int seq_buf_used(struct seq_buf *s) * * Note, if this is called when the buffer has overflowed, then * the last byte of the buffer is zeroed, and the len will still - * point passed it. + * point passed it. The same happens when the buffer is exactly + * full: the NUL takes the place of the last byte written, which is + * lost, though seq_buf_used() still counts it. + * + * A zero-sized seq_buf has nowhere to put a NUL, so the empty string + * is returned instead of writing to @s->buffer. Any other seq_buf + * returns @s->buffer, even when it holds an empty string, so callers + * always get their own buffer back. * * After this function is called, s->buffer is safe to use * in string operations. @@ -97,18 +172,68 @@ static inline unsigned int seq_buf_used(struct seq_buf *s) */ static inline const char *seq_buf_str(struct seq_buf *s) { - if (WARN_ON(s->size == 0)) + if (s->size == 0) return ""; - if (seq_buf_buffer_left(s)) - s->buffer[s->len] = 0; - else - s->buffer[s->size - 1] = 0; + __seq_buf_terminate(s); return s->buffer; } /** + * seq_buf_strlen - get the length of the NUL-terminated C string in seq_buf + * @s: the seq_buf handle + * + * This makes sure that the buffer in @s is NUL-terminated, exactly as + * seq_buf_str() does, and returns the length of the resulting string + * without walking it. Unlike seq_buf_used(), this does not count the byte + * given up to the NUL when the buffer is full or has overflowed. When the + * buffer is exactly full, that byte is the last one written, and calling + * either function loses it. + * + * A zero-sized seq_buf holds no string, so 0 is returned without writing + * to @s->buffer, matching what seq_buf_str() returns for one. + * + * After this function is called, s->buffer is safe to use + * in string operations. + * + * Returns: the offset of the NUL that terminates @s->buffer. That is the + * length of the string unless an earlier NUL is in the way, either one the + * data written to @s carried itself, or one seq_buf_set_overflow() left + * behind when it cleared what no writer had claimed. + */ +static inline size_t seq_buf_strlen(struct seq_buf *s) +{ + if (s->size == 0) + return 0; + + return __seq_buf_terminate(s); +} + +/** + * seq_buf_terminate - NUL-terminate the string in a seq_buf + * @s: the seq_buf handle + * + * Terminate @s->buffer exactly as seq_buf_str() and seq_buf_strlen() do, + * for callers that want neither the pointer nor the length and only need + * the buffer to be safe to read as a C string. A zero-sized seq_buf has + * nowhere to put a NUL and is left untouched. + * + * Nothing is returned on purpose: a caller that wants the length should + * use seq_buf_strlen(), which says so. + * + * After this function is called, s->buffer is safe to use + * in string operations. + */ +static inline void seq_buf_terminate(struct seq_buf *s) +{ + if (s->size == 0) + return; + + __seq_buf_terminate(s); +} + +/** * seq_buf_get_buf - get buffer to write arbitrary data to * @s: the seq_buf handle * @bufp: the beginning of the buffer is stored here @@ -155,11 +280,11 @@ static inline void seq_buf_commit(struct seq_buf *s, int num) * * Removes the last written character to the seq_buf @s. * - * Returns the last character or -1 if it is empty. + * Returns the last character, or -1 if @s is empty or has overflowed. */ static inline int seq_buf_pop(struct seq_buf *s) { - if (!s->len) + if (!s->len || seq_buf_has_overflowed(s)) return -1; s->len--; diff --git a/include/linux/serial_8250.h b/include/linux/serial_8250.h index eeb868e93e15..d91c0477be9d 100644 --- a/include/linux/serial_8250.h +++ b/include/linux/serial_8250.h @@ -186,10 +186,10 @@ static inline struct uart_8250_port *up_to_u8250p(struct uart_port *up) return container_of(up, struct uart_8250_port, port); } -int serial8250_register_8250_port(const struct uart_8250_port *); -void serial8250_unregister_port(int line); -void serial8250_suspend_port(int line); -void serial8250_resume_port(int line); +struct uart_8250_port *serial8250_register_8250_port(const struct uart_8250_port *); +void serial8250_unregister_port(struct uart_8250_port *); +void serial8250_suspend_port(struct uart_8250_port *); +void serial8250_resume_port(struct uart_8250_port *); int early_serial_setup(struct uart_port *port); int early_serial8250_setup(struct earlycon_device *device, const char *options); diff --git a/include/linux/serial_core.h b/include/linux/serial_core.h index 904760876ca8..4eeb8253752a 100644 --- a/include/linux/serial_core.h +++ b/include/linux/serial_core.h @@ -1140,8 +1140,6 @@ int uart_add_one_port(struct uart_driver *reg, struct uart_port *port); void uart_remove_one_port(struct uart_driver *reg, struct uart_port *port); int uart_read_port_properties(struct uart_port *port); int uart_read_and_validate_port_properties(struct uart_port *port); -bool uart_match_port(const struct uart_port *port1, - const struct uart_port *port2); /* * Power Management diff --git a/include/linux/set_memory.h b/include/linux/set_memory.h index 3030d9245f5a..27c32fbabfec 100644 --- a/include/linux/set_memory.h +++ b/include/linux/set_memory.h @@ -5,16 +5,92 @@ #ifndef _LINUX_SET_MEMORY_H_ #define _LINUX_SET_MEMORY_H_ +/** + * DOC: Kernel page table permissions + * + * The set_memory() and set_direct_map() APIs update permissions of existing + * kernel mappings. + * + * The set_memory() functions operate on a range of kernel virtual addresses, + * the set_direct_map() functions operate on the direct map. + * + * The updates are not atomic: when a call fails, an arbitrary prefix of the + * range may have been updated already and there is no automatic rollback. + * A caller must restore the required permissions before reusing or freeing + * the memory. + * + * When an architecture does not implement these APIs they succeed without + * doing anything, so a return value of 0 does not mean that the permissions + * were actually changed. + * + * Callers that depend on the permissions being applied must ensure that the + * architecture supports the required operation for the target addresses. The + * Kconfig symbols alone do not guarantee this. + * + * See Documentation/mm/kernel-page-tables.rst for the details and for the + * differences between the architecture implementations. + */ + #ifdef CONFIG_ARCH_HAS_SET_MEMORY #include <asm/set_memory.h> #else +/** + * set_memory_ro - make a kernel mapping read-only + * @addr: page aligned start of the kernel virtual address range + * @numpages: number of pages in the range + * + * Flushes the TLB for the range. + * + * Return: 0 on success, negative error code on failure. + */ static inline int __must_check set_memory_ro(unsigned long addr, int numpages) { return 0; } + +/** + * set_memory_rw - make a kernel mapping writable + * @addr: page aligned start of the kernel virtual address range + * @numpages: number of pages in the range + * + * Flushes the TLB for the range. + * + * Return: 0 on success, negative error code on failure. + */ static inline int __must_check set_memory_rw(unsigned long addr, int numpages) { return 0; } + +/** + * set_memory_x - make a kernel mapping executable + * @addr: page aligned start of the kernel virtual address range + * @numpages: number of pages in the range + * + * Flushes the TLB for the range. + * + * Return: 0 on success, negative error code on failure. + */ static inline int __must_check set_memory_x(unsigned long addr, int numpages) { return 0; } + +/** + * set_memory_nx - make a kernel mapping non-executable + * @addr: page aligned start of the kernel virtual address range + * @numpages: number of pages in the range + * + * Flushes the TLB for the range. + * + * Return: 0 on success, negative error code on failure. + */ static inline int __must_check set_memory_nx(unsigned long addr, int numpages) { return 0; } #endif #ifndef set_memory_rox +/** + * set_memory_rox - make a kernel mapping read-only and executable + * @addr: page aligned start of the kernel virtual address range + * @numpages: number of pages in the range + * + * A failure may leave the range read-only but not executable. + * + * Flushes the TLB for the range. + * + * Return: 0 on success, negative error code on failure. + */ static inline int set_memory_rox(unsigned long addr, int numpages) { int ret = set_memory_ro(addr, numpages); @@ -25,17 +101,35 @@ static inline int set_memory_rox(unsigned long addr, int numpages) #endif #ifndef CONFIG_ARCH_HAS_SET_DIRECT_MAP -static inline int set_direct_map_invalid_noflush(struct page *page) -{ - return 0; -} -static inline int set_direct_map_default_noflush(struct page *page) +/** + * set_direct_map_invalid_noflush - remove pages from the direct map + * @page: first page to update + * @nr: number of pages to update + * + * Makes the direct mapping of @nr pages starting at @page not present. + * The caller is responsible for any required TLB flushing. + * + * Return: 0 on success, negative error code on failure. + */ +static inline int set_direct_map_invalid_noflush(struct page *page, + unsigned int nr) { return 0; } -static inline int set_direct_map_valid_noflush(struct page *page, - unsigned nr, bool valid) +/** + * set_direct_map_default_noflush - restore the direct map of pages + * @page: first page to update + * @nr: number of pages to update + * + * Restores the default kernel permissions of the direct mapping of @nr + * pages starting at @page. + * The caller is responsible for any required TLB flushing. + * + * Return: 0 on success, negative error code on failure. + */ +static inline int set_direct_map_default_noflush(struct page *page, + unsigned int nr) { return 0; } @@ -50,6 +144,18 @@ static inline bool kernel_page_present(struct page *page) * boot time. Let them overrive this query. */ #ifndef can_set_direct_map +/** + * can_set_direct_map - check if the direct map can be modified + * + * Available with CONFIG_ARCH_HAS_SET_DIRECT_MAP. Architectures may override + * this to report whether direct map updates are enabled at runtime. + * Even though the generic implementation returns true this does not guarantee + * that every address can be updated. + * + * See Documentation/mm/kernel-page-tables.rst for the details + * + * Return: true unless the architecture reports direct map updates disabled. + */ static inline bool can_set_direct_map(void) { return true; diff --git a/include/linux/shmem_fs.h b/include/linux/shmem_fs.h index 5663dff53186..a7c7a96a7cbf 100644 --- a/include/linux/shmem_fs.h +++ b/include/linux/shmem_fs.h @@ -11,6 +11,7 @@ #include <linux/fs_parser.h> #include <linux/userfaultfd_k.h> #include <linux/bits.h> +#include <linux/list_lru.h> /* inode in-kernel data */ @@ -54,6 +55,11 @@ struct shmem_inode_info { struct dquot __rcu *i_dquot[MAXQUOTAS]; #endif struct inode vfs_inode; + +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + struct obj_cgroup *shrinklist_objcg; + int shrinklist_nid; +#endif }; #define SHMEM_FL_USER_VISIBLE (FS_FL_USER_VISIBLE | FS_CASEFOLD_FL) @@ -83,9 +89,9 @@ struct shmem_sb_info { ino_t next_ino; /* The next per-sb inode number to use */ ino_t __percpu *ino_batch; /* The next per-cpu inode number to use */ struct mempolicy *mpol; /* default memory policy for mappings */ - spinlock_t shrinklist_lock; /* Protects shrinklist */ - struct list_head shrinklist; /* List of shinkable inodes */ - unsigned long shrinklist_len; /* Length of shrinklist */ +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + struct list_lru shrinklist; /* List of shrinkable inodes */ +#endif struct shmem_quota_limits qlimits; /* Default quota limits */ struct simple_xattr_cache xa_cache; }; diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index a3ff380d43f0..6787fcf6ac3e 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -358,6 +358,12 @@ struct sk_buff; */ #define GSO_BY_FRAGS 0xFFFF +/* All frags in an skb must either all be struct page-backed or all belong to + * the same memory provider instance (e.g. the same devmem binding or io_uring + * area). Mixing page and memory-provider frags, or mixing frags from different + * memory providers within a single skb (including via coalescing), is not + * allowed. + */ typedef struct skb_frag { netmem_ref netmem; unsigned int len; @@ -633,9 +639,9 @@ struct skb_shared_info { * * Transport layers send out clones of payload skbs they hold for * retransmissions. To allow lower layers of the stack to prepend their headers - * we split &skb_shared_info.dataref into two halves. - * The lower 16 bits count the overall number of references. - * The higher 16 bits indicate how many of the references are payload-only. + * we split &skb_shared_info.dataref into two parts. + * The lower 30 bits count the overall number of references. + * Bit 30 indicates whether one of the references is payload-only. * skb_header_cloned() checks if skb is allowed to add / write the headers. * * The creator of the skb (e.g. TCP) marks its skb as &sk_buff.nohdr @@ -655,7 +661,7 @@ struct skb_shared_info { * Having multiple payload-only skbs with different lengths of hdr_len is not * possible. The payload-only skbs should never leave their owner. */ -#define SKB_DATAREF_SHIFT 16 +#define SKB_DATAREF_SHIFT 30 #define SKB_DATAREF_MASK ((1 << SKB_DATAREF_SHIFT) - 1) @@ -1036,8 +1042,6 @@ struct sk_buff { __u16 tc_index; /* traffic control index */ #endif - u16 alloc_cpu; - union { __wsum csum; struct { @@ -1088,6 +1092,8 @@ struct sk_buff { ); /* end headers group */ + u16 alloc_cpu; + /* These elements must be at the end, see alloc_skb() for details. */ sk_buff_data_t tail; sk_buff_data_t end; @@ -2125,14 +2131,14 @@ static inline struct sk_buff *skb_share_check(struct sk_buff *skb, gfp_t pri) } /* - * Copy shared buffers into a new sk_buff. We effectively do COW on + * Copy cloned buffers into a new sk_buff. We effectively do COW on * packets to handle cases where we have a local reader and forward * and a couple of other messy ones. The normal one is tcpdumping * a packet that's being forwarded. */ /** - * skb_unshare - make a copy of a shared buffer + * skb_unclone_full - make a private copy of a cloned buffer * @skb: buffer to check * @pri: priority for memory allocation * @@ -2142,10 +2148,13 @@ static inline struct sk_buff *skb_share_check(struct sk_buff *skb, gfp_t pri) * the original buffer is returned. When called with a spinlock held or * from interrupt state @pri must be %GFP_ATOMIC * + * Unlike skb_unclone(), the paged data is copied as well. A buffer with + * skb->users > 1 that is not a clone is returned as is, use + * skb_share_check() for that case. + * * %NULL is returned on a memory allocation failure. */ -static inline struct sk_buff *skb_unshare(struct sk_buff *skb, - gfp_t pri) +static inline struct sk_buff *skb_unclone_full(struct sk_buff *skb, gfp_t pri) { might_sleep_if(gfpflags_allow_blocking(pri)); if (skb_cloned(skb)) { @@ -3816,9 +3825,12 @@ static inline dma_addr_t __skb_frag_dma_map(struct device *dev, size_t offset, size_t size, enum dma_data_direction dir) { + dma_addr_t addr; + if (skb_frag_is_net_iov(frag)) { - return netmem_to_net_iov(frag->netmem)->desc.dma_addr + - offset + frag->offset; + addr = netmem_dma_addr_decode( + netmem_get_dma_addr(frag->netmem)); + return addr + offset + frag->offset; } return dma_map_page(dev, skb_frag_page(frag), skb_frag_off(frag) + offset, size, dir); diff --git a/include/linux/skbuff_ref.h b/include/linux/skbuff_ref.h index 05c8486bafac..b6f6da556ee9 100644 --- a/include/linux/skbuff_ref.h +++ b/include/linux/skbuff_ref.h @@ -9,66 +9,78 @@ #include <linux/skbuff.h> +bool napi_pp_get_page(netmem_ref netmem); + /** - * __skb_frag_ref - take an addition reference on a paged fragment. - * @frag: the paged fragment + * skb_netmem_ref - acquire a fragment reference on a netmem + * @netmem: netmem to reference + * @recycle: whether page_pool recycling is enabled (e.g. skb->pp_recycle) * - * Takes an additional reference on the paged fragment @frag. + * Acquires pp_ref_count via napi_pp_get_page() if @recycle is true and + * @netmem belongs to a page_pool; otherwise acquires a non-pp backing + * reference via netmem_inc_ref(). Counterpart to skb_netmem_unref(). */ -static __always_inline void __skb_frag_ref(skb_frag_t *frag) +static __always_inline void skb_netmem_ref(netmem_ref netmem, bool recycle) { - get_netmem(skb_frag_netmem(frag)); +#ifdef CONFIG_PAGE_POOL + if (recycle && napi_pp_get_page(netmem)) + return; +#endif + netmem_inc_ref(netmem); } /** - * skb_frag_ref - take an addition reference on a paged fragment of an skb. + * skb_frag_ref - acquire a reference on a paged fragment of an skb * @skb: the buffer - * @f: the fragment offset. + * @f: the fragment offset * - * Takes an additional reference on the @f'th paged fragment of @skb. + * Acquires a reference on the @f'th fragment of @skb via skb_netmem_ref() + * (pp_ref_count if skb->pp_recycle and page_pool-backed, else non-pp + * backing refcount). Counterpart to skb_frag_unref(). */ static __always_inline void skb_frag_ref(struct sk_buff *skb, int f) { - __skb_frag_ref(&skb_shinfo(skb)->frags[f]); + skb_netmem_ref(skb_frag_netmem(&skb_shinfo(skb)->frags[f]), + skb->pp_recycle); } bool napi_pp_put_page(netmem_ref netmem); -static __always_inline void skb_page_unref(netmem_ref netmem, bool recycle) +/** + * skb_netmem_unref - release a fragment reference on a netmem + * @netmem: netmem to unreference + * @recycle: whether page_pool recycling is enabled (e.g. skb->pp_recycle) + * + * Drops pp_ref_count via napi_pp_put_page() if @recycle is true and @netmem + * belongs to a page_pool; otherwise drops a non-pp backing reference via + * netmem_dec_ref(). Counterpart to skb_netmem_ref(). + */ +static __always_inline void skb_netmem_unref(netmem_ref netmem, bool recycle) { #ifdef CONFIG_PAGE_POOL if (recycle && napi_pp_put_page(netmem)) return; #endif - put_netmem(netmem); -} - -/** - * __skb_frag_unref - release a reference on a paged fragment. - * @frag: the paged fragment - * @recycle: recycle the page if allocated via page_pool - * - * Releases a reference on the paged fragment @frag - * or recycles the page via the page_pool API. - */ -static __always_inline void __skb_frag_unref(skb_frag_t *frag, bool recycle) -{ - skb_page_unref(skb_frag_netmem(frag), recycle); + netmem_dec_ref(netmem); } /** - * skb_frag_unref - release a reference on a paged fragment of an skb. + * skb_frag_unref - release a reference on a paged fragment of an skb * @skb: the buffer * @f: the fragment offset * - * Releases a reference on the @f'th paged fragment of @skb. + * Drops a reference on the @f'th fragment of @skb via skb_netmem_unref() + * (pp_ref_count if skb->pp_recycle and page_pool-backed, else non-pp + * backing refcount), skipping managed zerocopy fragments. Counterpart to + * skb_frag_ref(). */ static __always_inline void skb_frag_unref(struct sk_buff *skb, int f) { struct skb_shared_info *shinfo = skb_shinfo(skb); if (!skb_zcopy_managed(skb)) - __skb_frag_unref(&shinfo->frags[f], skb->pp_recycle); + skb_netmem_unref(skb_frag_netmem(&shinfo->frags[f]), + skb->pp_recycle); } #endif /* _LINUX_SKBUFF_REF_H */ diff --git a/include/linux/slimbus.h b/include/linux/slimbus.h index ca6f1da4bdf3..1a2d3aa0e033 100644 --- a/include/linux/slimbus.h +++ b/include/linux/slimbus.h @@ -98,7 +98,7 @@ struct slim_driver { * @start_offset: Specifies starting offset in information/value element map * @rbuf: buffer to read the values * @wbuf: buffer to write - * @num_bytes: upto 16. This ensures that the message will fit the slicesize + * @num_bytes: up to 16. This ensures that the message will fit the slicesize * per SLIMbus spec * @comp: completion for asynchronous operations, valid only if TID is * required for transaction, like REQUEST operations. diff --git a/include/linux/soc/marvell/octeontx2/asm.h b/include/linux/soc/marvell/octeontx2/asm.h index d683251a0b40..8b572d3d741c 100644 --- a/include/linux/soc/marvell/octeontx2/asm.h +++ b/include/linux/soc/marvell/octeontx2/asm.h @@ -17,7 +17,7 @@ #define otx2_lmt_flush(ioaddr) \ ({ \ u64 result = 0; \ - __asm__ volatile(".cpu generic+lse\n" \ + __asm__ volatile(".arch_extension lse\n" \ "ldeor xzr, %x[rf], [%[rs]]" \ : [rf]"=r" (result) \ : [rs]"r" (ioaddr)); \ @@ -30,7 +30,7 @@ */ #define cn10k_lmt_flush(val, addr) \ ({ \ - __asm__ volatile(".cpu generic+lse\n" \ + __asm__ volatile(".arch_extension lse\n" \ "steorl %x[rf],[%[rs]]" \ : [rf] "+r"(val) \ : [rs] "r"(addr)); \ @@ -40,7 +40,7 @@ static inline u64 otx2_atomic64_fetch_add(u64 incr, u64 *ptr) { u64 result; - asm volatile (".cpu generic+lse\n" + asm volatile (".arch_extension lse\n" "ldadda %x[i], %x[r], [%[b]]" : [r] "=r" (result), "+m" (*ptr) : [i] "r" (incr), [b] "r" (ptr) diff --git a/include/linux/soc/qcom/apr.h b/include/linux/soc/qcom/apr.h index 909e84f84e0c..2e4231092088 100644 --- a/include/linux/soc/qcom/apr.h +++ b/include/linux/soc/qcom/apr.h @@ -5,7 +5,6 @@ #include <linux/spinlock.h> #include <linux/device.h> -#include <linux/device-id/apr.h> #include <dt-bindings/soc/qcom,apr.h> #include <dt-bindings/soc/qcom,gpr.h> @@ -135,6 +134,8 @@ struct pkt_router_svc { typedef struct pkt_router_svc gpr_port_t; +#define APR_NAME_SIZE 32 + struct apr_device { struct device dev; uint16_t svc_id; @@ -158,7 +159,6 @@ struct apr_driver { const struct apr_resp_pkt *d); gpr_port_cb gpr_callback; struct device_driver driver; - const struct apr_device_id *id_table; }; typedef struct apr_driver gpr_driver_t; diff --git a/include/linux/soc/qcom/geni-se.h b/include/linux/soc/qcom/geni-se.h index 5f18d281e6a4..b3372a3147a0 100644 --- a/include/linux/soc/qcom/geni-se.h +++ b/include/linux/soc/qcom/geni-se.h @@ -347,12 +347,18 @@ struct geni_se { #define QUP_SE_VERSION_2_5 0x20050000 /* - * QUP Core 2X clock votes used by GENI clients through the "qup-core" ICC - * path. Values are in Bps and must be converted with Bps_to_icc() before - * setting avg_bw. + * ICC bandwidth values in Bps for the GENI_TO_CORE ("qup-core") path. + * Convert them with Bps_to_icc() before setting avg_bw. The QUP ICC provider + * maps each threshold to the Core 2X rate named by the macro suffix. + * + * These values are core clock votes, not GENI transfer bandwidths. */ #define CORE_2X_19_2_MHZ 9600000 #define CORE_2X_50_MHZ 25000000 +#define CORE_2X_100_MHZ 50000000 +#define CORE_2X_150_MHZ 75000000 +#define CORE_2X_200_MHZ 100000000 +#define CORE_2X_236_MHZ 118000000 #define GENI_DEFAULT_BW Bps_to_icc(1000) @@ -568,7 +574,7 @@ void geni_se_tx_dma_unprep(struct geni_se *se, dma_addr_t iova, size_t len); void geni_se_rx_dma_unprep(struct geni_se *se, dma_addr_t iova, size_t len); -int geni_icc_get(struct geni_se *se, const char *icc_ddr); +int geni_icc_get(struct geni_se *se); int geni_icc_set_bw(struct geni_se *se); int geni_icc_set_bw_ab(struct geni_se *se, u32 core_ab, u32 cfg_ab, u32 ddr_ab); diff --git a/include/linux/soc/qcom/llcc-qcom.h b/include/linux/soc/qcom/llcc-qcom.h index f3ed63e475ab..713cb0221b14 100644 --- a/include/linux/soc/qcom/llcc-qcom.h +++ b/include/linux/soc/qcom/llcc-qcom.h @@ -74,6 +74,7 @@ #define LLCC_CAMSRTIP 73 #define LLCC_CAMRTRF 74 #define LLCC_CAMSRTRF 75 +#define LLCC_GPU_LITTLE 80 #define LLCC_OOBM_NS 81 #define LLCC_OOBM_S 82 #define LLCC_VIDEO_APV 83 @@ -85,7 +86,12 @@ #define LLCC_CAM_OFE_STROV 93 #define LLCC_CPUSS_HEU 94 #define LLCC_PCIE_TCU 97 +#define LLCC_GPUHTW_LITTLE 98 #define LLCC_MDM_PNG_FIXED 100 +#define LLCC_PPE_RXDESC 101 +#define LLCC_PPE_RXFILL 102 +#define LLCC_WLAN_5G 103 +#define LLCC_WLAN_6G 104 /** * struct llcc_slice_desc - Cache slice descriptor diff --git a/include/linux/soc/qcom/qmi.h b/include/linux/soc/qcom/qmi.h index b9dcb437a0be..683d27cee413 100644 --- a/include/linux/soc/qcom/qmi.h +++ b/include/linux/soc/qcom/qmi.h @@ -96,6 +96,7 @@ struct qmi_elem_info { * Enumerate the IDs of the QMI services */ #define QMI_SERVICE_ID_TEST 0x0f /* 15 */ +#define QMI_SERVICE_ID_TMD 0x18 /* 24 */ #define QMI_SERVICE_ID_SSCTL 0x2b /* 43 */ #define QMI_SERVICE_ID_IPA 0x31 /* 49 */ #define QMI_SERVICE_ID_SERVREG_LOC 0x40 /* 64 */ diff --git a/include/linux/soc/qcom/qmi_tmd.h b/include/linux/soc/qcom/qmi_tmd.h new file mode 100644 index 000000000000..452f51f23731 --- /dev/null +++ b/include/linux/soc/qcom/qmi_tmd.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2025, Linaro Limited + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + * + * QMI Thermal Mitigation Device (TMD) library header. + */ + +#ifndef __QMI_TMD_H__ +#define __QMI_TMD_H__ + +struct device; +struct qmi_tmd_client; + +#if IS_ENABLED(CONFIG_QCOM_QMI_TMD) +struct qmi_tmd_client *qmi_tmd_init(struct device *dev, + unsigned int instance_id, + const char * const *tmd_names, + int num_tmds); + +void qmi_tmd_exit(struct qmi_tmd_client *tmd_cli); +#else +static inline struct qmi_tmd_client *qmi_tmd_init(struct device *dev, + unsigned int instance_id, + const char * const *tmd_names, + int num_tmds) +{ + return NULL; +} + +static inline void qmi_tmd_exit(struct qmi_tmd_client *tmd_cli) +{ +} +#endif + +#endif /* __QMI_TMD_H__ */ diff --git a/include/linux/soc/qcom/tc9563.h b/include/linux/soc/qcom/tc9563.h new file mode 100644 index 000000000000..086f37a40d80 --- /dev/null +++ b/include/linux/soc/qcom/tc9563.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2026 Qualcomm Innovation Center, Inc. All rights reserved. + * Author: Lorenzo Bianconi <lorenzo.bianconi@oss.qualcomm.com> + */ + +#ifndef __QCOM_TC9563_H +#define __QCOM_TC9563_H + +#define TC9563_GPIO_DEV_NAME "tc9563-gpio" + +#define TC9563_GPIO_IN0_OFFSET 0x801200 +#define TC9563_GPIO_EN0_OFFSET 0x801208 +#define TC9563_GPIO_OUT0_OFFSET 0x801210 + +#define TC9563_GPIO_CONFIG TC9563_GPIO_EN0_OFFSET +#define TC9563_RESET_GPIO TC9563_GPIO_OUT0_OFFSET + +#endif /* __QCOM_TC9563_H */ diff --git a/include/linux/soc/qcom/ubwc.h b/include/linux/soc/qcom/ubwc.h index f3a70360b177..b09ae05cbeae 100644 --- a/include/linux/soc/qcom/ubwc.h +++ b/include/linux/soc/qcom/ubwc.h @@ -36,6 +36,7 @@ struct qcom_ubwc_cfg_data { #define UBWC_4_3 0x40030000 #define UBWC_5_0 0x50000000 #define UBWC_6_0 0x60000000 +#define UBWC_7_0 0x70000000 #if IS_ENABLED(CONFIG_QCOM_UBWC_CONFIG) const struct qcom_ubwc_cfg_data *qcom_ubwc_config_get_data(void); @@ -108,6 +109,8 @@ static inline u32 qcom_ubwc_swizzle(const struct qcom_ubwc_cfg_data *cfg) static inline u32 qcom_ubwc_version_tag(const struct qcom_ubwc_cfg_data *cfg) { + if (cfg->ubwc_enc_version >= UBWC_7_0) + return 6; if (cfg->ubwc_enc_version >= UBWC_6_0) return 5; if (cfg->ubwc_enc_version >= UBWC_5_0) diff --git a/include/linux/soc/ti/k3-ringacc.h b/include/linux/soc/ti/k3-ringacc.h index 39b022b92598..7b7fc237c08f 100644 --- a/include/linux/soc/ti/k3-ringacc.h +++ b/include/linux/soc/ti/k3-ringacc.h @@ -57,13 +57,13 @@ struct k3_ringacc; struct k3_ring; /** - * enum k3_ring_cfg - RA ring configuration structure + * struct k3_ring_cfg - RA ring configuration structure * * @size: Ring size, number of elements * @elm_size: Ring element size * @mode: Ring operational mode * @flags: Ring configuration flags. Possible values: - * @K3_RINGACC_RING_SHARED: when set allows to request the same ring + * %K3_RINGACC_RING_SHARED: when set allows to request the same ring * few times. It's usable when the same ring is used as Free Host PD ring * for different flows, for example. * Note: Locking should be done by consumer if required @@ -88,11 +88,10 @@ struct k3_ring_cfg { /** * of_k3_ringacc_get_by_phandle - find a RA by phandle property * @np: device node - * @propname: property name containing phandle on RA node + * @property: property name containing phandle on RA node * - * Returns pointer on the RA - struct k3_ringacc - * or -ENODEV if not found, - * or -EPROBE_DEFER if not yet registered + * Return: Pointer to the RA, or ERR_PTR(-ENODEV) if the phandle cannot + * be resolved, or ERR_PTR(-EPROBE_DEFER) if the RA is not yet registered. */ struct k3_ringacc *of_k3_ringacc_get_by_phandle(struct device_node *np, const char *property); @@ -103,10 +102,8 @@ struct k3_ringacc *of_k3_ringacc_get_by_phandle(struct device_node *np, * k3_ringacc_request_ring - request ring from ringacc * @ringacc: pointer on ringacc * @id: ring id or K3_RINGACC_RING_ID_ANY for any general purpose ring - * @flags: - * @K3_RINGACC_RING_USE_PROXY: if set - proxy will be allocated and - * used to access ring memory. Sopported only for rings in - * Message/Credentials/Queue mode. + * @flags: Set %K3_RINGACC_RING_USE_PROXY to allocate a proxy for + * accessing ring memory in Message mode. * * Returns pointer on the Ring - struct k3_ring * or NULL in case of failure. @@ -126,8 +123,9 @@ int k3_ringacc_request_rings_pair(struct k3_ringacc *ringacc, */ void k3_ringacc_ring_reset(struct k3_ring *ring); /** - * k3_ringacc_ring_reset - ring reset for DMA rings + * k3_ringacc_ring_reset_dma - ring reset for DMA rings * @ring: pointer on Ring + * @occ: occupancy used by the DMA reset quirk, or zero to read it from hardware * * Resets ring internal state ((hw)occ, (hw)idx). Should be used for rings * which are read by K3 UDMA, like TX or Free Host PD rings. @@ -217,8 +215,8 @@ int k3_ringacc_ring_push(struct k3_ring *ring, void *elem); * @ring: pointer on ring * @elem: pointer on ring element buffer * - * Push one ring element from the ring head. Size of the ring element is - * determined by ring configuration &struct k3_ring_cfg elm_size.. + * Pop one ring element from the ring head. Size of the ring element is + * determined by ring configuration &struct k3_ring_cfg elm_size. * * Returns 0 on success, errno otherwise. */ @@ -242,7 +240,7 @@ int k3_ringacc_ring_push_head(struct k3_ring *ring, void *elem); * @ring: pointer on ring * @elem: pointer on ring element buffer * - * Push one ring element from the ring tail. Size of the ring element is + * Pop one ring element from the ring tail. Size of the ring element is * determined by ring configuration &struct k3_ring_cfg elm_size. * * Returns 0 on success, errno otherwise. @@ -256,7 +254,10 @@ u32 k3_ringacc_get_tisci_dev_id(struct k3_ring *ring); struct ti_sci_handle; /** - * struct struct k3_ringacc_init_data - Initialization data for DMA rings + * struct k3_ringacc_init_data - Initialization data for DMA rings + * @tisci: TI SCI firmware handle + * @tisci_dev_id: TI SCI device ID of the DMA controller + * @num_rings: Total number of available rings */ struct k3_ringacc_init_data { const struct ti_sci_handle *tisci; diff --git a/include/linux/socket.h b/include/linux/socket.h index 5a5eb1250103..b17fdb7d38a7 100644 --- a/include/linux/socket.h +++ b/include/linux/socket.h @@ -189,10 +189,11 @@ static inline size_t msg_data_left(const struct msghdr *msg) /* "Socket"-level control message types: */ -#define SCM_RIGHTS 0x01 /* rw: access rights (array of int) */ -#define SCM_CREDENTIALS 0x02 /* rw: struct ucred */ -#define SCM_SECURITY 0x03 /* rw: security label */ -#define SCM_PIDFD 0x04 /* ro: pidfd (int) */ +#define SCM_RIGHTS 0x01 /* rw: access rights (array of int) */ +#define SCM_CREDENTIALS 0x02 /* rw: struct ucred */ +#define SCM_SECURITY 0x03 /* rw: security label */ +#define SCM_PIDFD 0x04 /* ro: pidfd (int) */ +#define SCM_PIDFD_THREAD 0x05 /* ro: thread pidfd (int) */ struct ucred { __u32 pid; diff --git a/include/linux/soundwire/sdw.h b/include/linux/soundwire/sdw.h index 707ca6bd1e23..f710e5932b4b 100644 --- a/include/linux/soundwire/sdw.h +++ b/include/linux/soundwire/sdw.h @@ -618,6 +618,7 @@ struct sdw_bus_params { * @update_status: Update Slave status * @bus_config: Update the bus config for Slave * @port_prep: Prepare the port with parameters + * @get_clk_stop_mode: Get the clock stop mode of the Slave * @clk_stop: handle imp-def sequences before and after prepare and de-prepare */ struct sdw_slave_ops { @@ -631,6 +632,7 @@ struct sdw_slave_ops { int (*port_prep)(struct sdw_slave *slave, struct sdw_prepare_ch *prepare_ch, enum sdw_port_prep_ops pre_ops); + enum sdw_clk_stop_mode (*get_clk_stop_mode)(struct sdw_slave *slave); int (*clk_stop)(struct sdw_slave *slave, enum sdw_clk_stop_mode mode, enum sdw_clk_stop_type type); @@ -649,6 +651,7 @@ struct sdw_slave_ops { * @node: node for bus list * @port_ready: Port ready completion flag for each Slave port * @m_port_map: static Master port map for each Slave port + * @clk_stop_mode: The clock stop mode of the Slave * @dev_num: Current Device Number, values can be 0 or dev_num_sticky * @dev_num_sticky: one-time static Device Number assigned by Bus * @probed: boolean tracking driver state @@ -683,6 +686,7 @@ struct sdw_slave { struct list_head node; struct completion port_ready[SDW_MAX_PORTS]; unsigned int m_port_map[SDW_MAX_PORTS]; + enum sdw_clk_stop_mode clk_stop_mode; u16 dev_num; u16 dev_num_sticky; bool probed; diff --git a/include/linux/soundwire/sdw_intel.h b/include/linux/soundwire/sdw_intel.h index 9c9435009537..9710f2dc04e2 100644 --- a/include/linux/soundwire/sdw_intel.h +++ b/include/linux/soundwire/sdw_intel.h @@ -424,7 +424,7 @@ struct sdw_intel_hw_ops { int (*start_bus)(struct sdw_intel *sdw); int (*start_bus_after_reset)(struct sdw_intel *sdw); int (*start_bus_after_clock_stop)(struct sdw_intel *sdw); - int (*stop_bus)(struct sdw_intel *sdw, bool clock_stop); + int (*stop_bus)(struct sdw_intel *sdw, bool clock_stop, bool wake_enable); int (*link_power_up)(struct sdw_intel *sdw); int (*link_power_down)(struct sdw_intel *sdw); diff --git a/include/linux/spinlock_types_raw.h b/include/linux/spinlock_types_raw.h index e5644ab2161f..942c229c90bb 100644 --- a/include/linux/spinlock_types_raw.h +++ b/include/linux/spinlock_types_raw.h @@ -11,6 +11,10 @@ #include <linux/lockdep_types.h> +/* + * Keep in sync with rust/kernel/sync/lock/spinlock.rs + */ + context_lock_struct(raw_spinlock) { arch_spinlock_t raw_lock; #ifdef CONFIG_DEBUG_SPINLOCK diff --git a/include/linux/splice.h b/include/linux/splice.h index 9dec4861d09f..0e6c955dc6ff 100644 --- a/include/linux/splice.h +++ b/include/linux/splice.h @@ -79,8 +79,8 @@ ssize_t add_to_pipe(struct pipe_inode_info *pipe, struct pipe_buffer *buf); ssize_t vfs_splice_read(struct file *in, loff_t *ppos, struct pipe_inode_info *pipe, size_t len, unsigned int flags); -ssize_t splice_direct_to_actor(struct file *file, struct splice_desc *sd, - splice_direct_actor *actor); +ssize_t vfs_splice_to_actor(struct file *file, loff_t pos, size_t count, + splice_direct_actor *actor, void *private); ssize_t do_splice(struct file *in, loff_t *off_in, struct file *out, loff_t *off_out, size_t len, unsigned int flags); ssize_t do_splice_direct(struct file *in, loff_t *ppos, struct file *out, diff --git a/include/linux/spmi.h b/include/linux/spmi.h index 4eb9564a7fb3..4daebc980501 100644 --- a/include/linux/spmi.h +++ b/include/linux/spmi.h @@ -70,6 +70,22 @@ int spmi_device_add(struct spmi_device *sdev); void spmi_device_remove(struct spmi_device *sdev); /** + * struct spmi_subdevice - Basic representation of an SPMI sub-device + * @sdev: Sub-device representation of an SPMI device + * @devid: Platform Device ID of an SPMI sub-device + */ +struct spmi_subdevice { + struct spmi_device sdev; + unsigned int devid; +}; + +struct spmi_subdevice *spmi_subdevice_alloc_and_add(struct spmi_device *sparent); +void spmi_subdevice_remove(struct spmi_subdevice *sdev); + +struct spmi_subdevice *devm_spmi_subdevice_alloc_and_add(struct device *dev, + struct spmi_device *sparent); + +/** * struct spmi_controller - interface to the SPMI master controller * @dev: Driver model representation of the device. * @nr: board-specific number identifier for this controller/bus @@ -173,6 +189,7 @@ static inline void spmi_driver_unregister(struct spmi_driver *sdrv) struct device_node; struct spmi_device *spmi_find_device_by_of_node(struct device_node *np); +struct spmi_device *spmi_get_parent_spmi_device(struct device *dev); int spmi_register_read(struct spmi_device *sdev, u8 addr, u8 *buf); int spmi_ext_register_read(struct spmi_device *sdev, u8 addr, u8 *buf, size_t len); diff --git a/include/linux/srcu.h b/include/linux/srcu.h index 7d9bc06df98d..1a8a465a5650 100644 --- a/include/linux/srcu.h +++ b/include/linux/srcu.h @@ -36,6 +36,8 @@ static inline int __init_srcu_struct(struct srcu_struct *ssp, const char *name, int __init_srcu_struct_fast(struct srcu_struct *ssp, const char *name, struct lock_class_key *key); int __init_srcu_struct_fast_updown(struct srcu_struct *ssp, const char *name, struct lock_class_key *key); +int __init_srcu_struct_atomic(struct srcu_struct *ssp, const char *name, + struct lock_class_key *key); #endif // #ifndef CONFIG_TINY_SRCU #define init_srcu_struct_fast(ssp) \ @@ -52,6 +54,13 @@ int __init_srcu_struct_fast_updown(struct srcu_struct *ssp, const char *name, __init_srcu_struct_fast_updown((ssp), #ssp, &__srcu_key); \ }) +#define init_srcu_struct_atomic(ssp) \ +({ \ + static struct lock_class_key __srcu_key; \ + \ + __init_srcu_struct_atomic((ssp), #ssp, &__srcu_key); \ +}) + #define __SRCU_DEP_MAP_INIT(srcu_name) .dep_map = { .name = #srcu_name }, #else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */ @@ -64,6 +73,7 @@ static inline int __init_srcu_struct(struct srcu_struct *ssp, const char *name, #ifndef CONFIG_TINY_SRCU int init_srcu_struct_fast(struct srcu_struct *ssp); int init_srcu_struct_fast_updown(struct srcu_struct *ssp); +int init_srcu_struct_atomic(struct srcu_struct *ssp); #endif // #ifndef CONFIG_TINY_SRCU #define __SRCU_DEP_MAP_INIT(srcu_name) @@ -77,14 +87,18 @@ int init_srcu_struct_fast_updown(struct srcu_struct *ssp); }) /* Values for SRCU Tree srcu_data ->srcu_reader_flavor, but also used by rcutorture. */ -#define SRCU_READ_FLAVOR_NORMAL 0x1 // srcu_read_lock(). -#define SRCU_READ_FLAVOR_NMI 0x2 // srcu_read_lock_nmisafe(). -// 0x4 // SRCU-lite is no longer with us. -#define SRCU_READ_FLAVOR_FAST 0x4 // srcu_read_lock_fast(), also NMI-safe. -#define SRCU_READ_FLAVOR_FAST_UPDOWN 0x8 // srcu_read_lock_fast_updown(). +#define SRCU_READ_FLAVOR_NORMAL 0x01 // srcu_read_lock(). +#define SRCU_READ_FLAVOR_NMI 0x02 // srcu_read_lock_nmisafe(). +// 0x04 // SRCU-lite is no longer with us. +#define SRCU_READ_FLAVOR_FAST 0x04 // srcu_read_lock_fast(), also NMI-safe. +#define SRCU_READ_FLAVOR_FAST_UPDOWN 0x08 // srcu_read_lock_fast_updown(). +#define SRCU_READ_FLAVOR_ATOMIC 0x10 // srcu_read_lock_atomic(). #define SRCU_READ_FLAVOR_ALL (SRCU_READ_FLAVOR_NORMAL | SRCU_READ_FLAVOR_NMI | \ - SRCU_READ_FLAVOR_FAST | SRCU_READ_FLAVOR_FAST_UPDOWN) + SRCU_READ_FLAVOR_FAST | SRCU_READ_FLAVOR_FAST_UPDOWN | \ + SRCU_READ_FLAVOR_ATOMIC) // All of the above. +#define SRCU_READ_FLAVOR_PREDEF (SRCU_READ_FLAVOR_FAST | SRCU_READ_FLAVOR_ATOMIC) + // Flavors special DEFINE_SRCU() flavors. #define SRCU_READ_FLAVOR_SLOWGP (SRCU_READ_FLAVOR_FAST | SRCU_READ_FLAVOR_FAST_UPDOWN) // Flavors requiring synchronize_rcu() // instead of smp_mb(). @@ -102,6 +116,7 @@ void call_srcu(struct srcu_struct *ssp, struct rcu_head *head, void (*func)(struct rcu_head *head)); void cleanup_srcu_struct(struct srcu_struct *ssp); void synchronize_srcu(struct srcu_struct *ssp); +void synchronize_srcu_atomic(struct srcu_struct *ssp); #define SRCU_GET_STATE_COMPLETED 0x1 @@ -307,6 +322,45 @@ static inline int srcu_read_lock(struct srcu_struct *ssp) } /** + * srcu_read_lock_atomic - register a new reader promising an atomic section + * @ssp: srcu_struct in which to register the new reader. + * + * As srcu_read_lock(), but the caller promises that the read-side + * critical section never sleeps and never blocks on anything which + * may itself depend on memory allocation to make progress. Preemption + * is disabled for the duration, which both enforces that promise (any + * sleepable call in the section will splat on every configuration) + * and bounds the section so that the update side may spin rather + * than sleep when waiting for readers: see synchronize_srcu_atomic(). + * + * The lock and matching srcu_read_unlock_atomic() must be invoked on + * the same CPU, from the same context; passing the return value to + * another task is not permitted for this flavor. + */ +static inline int srcu_read_lock_atomic(struct srcu_struct *ssp) + __acquires_shared(ssp) +{ + int retval; + + preempt_disable(); + /* + * Arm might_sleep() to catch even a *potentially* sleeping call + * in the section, not just an actual schedule: the atomic-domain + * promise must hold on every path, contended or not. In hardirq, + * softirq, or NMI the annotation would land on the interrupted + * task, and can also result in data races against that task's + * own non_block_start()/non_block_end() invocations; it is also + * redundant there, so skip it. + */ + if (in_task()) + non_block_start(); + srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_ATOMIC); + retval = __srcu_read_lock(ssp); + srcu_lock_acquire(&ssp->dep_map); + return retval; +} + +/** * srcu_read_lock_fast - register a new reader for an SRCU-protected structure. * @ssp: srcu_struct in which to register the new reader. * @@ -499,6 +553,23 @@ static inline void srcu_read_unlock(struct srcu_struct *ssp, int idx) } /** + * srcu_read_unlock_atomic - unregister an atomic-section reader + * @ssp: srcu_struct from which to unregister the old reader. + * @idx: return value from corresponding srcu_read_lock_atomic(). + */ +static inline void srcu_read_unlock_atomic(struct srcu_struct *ssp, int idx) + __releases_shared(ssp) +{ + WARN_ON_ONCE(idx & ~0x1); + srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_ATOMIC); + srcu_lock_release(&ssp->dep_map); + __srcu_read_unlock(ssp, idx); + if (in_task()) + non_block_end(); + preempt_enable(); +} + +/** * srcu_read_unlock_fast - unregister a old reader from an SRCU-protected structure. * @ssp: srcu_struct in which to unregister the old reader. * @scp: return value from corresponding srcu_read_lock_fast(). diff --git a/include/linux/srcutiny.h b/include/linux/srcutiny.h index fbcf13bc12d1..47a368f945e3 100644 --- a/include/linux/srcutiny.h +++ b/include/linux/srcutiny.h @@ -12,12 +12,14 @@ #define _LINUX_SRCU_TINY_H #include <linux/irq_work_types.h> +#include <linux/llist.h> #include <linux/swait.h> struct srcu_struct { short srcu_lock_nesting[2]; /* srcu_read_lock() nesting depth. */ u8 srcu_gp_running; /* GP workqueue running? */ u8 srcu_gp_waiting; /* GP waiting for readers? */ + u8 srcu_atomic_gp_flag; /* Serialize atomic GP work.*/ unsigned long srcu_idx; /* Current reader array element in bit 0x2. */ unsigned long srcu_idx_max; /* Furthest future srcu_idx request. */ struct swait_queue_head srcu_wq; @@ -26,6 +28,8 @@ struct srcu_struct { struct rcu_head **srcu_cb_tail; /* Pending callbacks: Tail. */ struct work_struct srcu_work; /* For driving grace periods. */ struct irq_work srcu_irq_work; /* Defer schedule_work() to irq work. */ + struct llist_head defer_cbs; /* Callbacks deferred on re-entry. */ + struct irq_work defer_iw; /* Re-issues defer_cbs later. */ #ifdef CONFIG_DEBUG_LOCK_ALLOC struct lockdep_map dep_map; #endif /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */ @@ -33,6 +37,7 @@ struct srcu_struct { void srcu_drive_gp(struct work_struct *wp); void srcu_tiny_irq_work(struct irq_work *irq_work); +void srcu_defer_drain(struct irq_work *irq_work); #define __SRCU_STRUCT_INIT(name, __ignored, ___ignored, ____ignored) \ { \ @@ -40,6 +45,9 @@ void srcu_tiny_irq_work(struct irq_work *irq_work); .srcu_cb_tail = &name.srcu_cb_head, \ .srcu_work = __WORK_INITIALIZER(name.srcu_work, srcu_drive_gp), \ .srcu_irq_work = { .func = srcu_tiny_irq_work }, \ + .defer_cbs = LLIST_HEAD_INIT(name.defer_cbs), \ + .defer_iw = { .node = { .u_flags = IRQ_WORK_HARD_IRQ }, \ + .func = srcu_defer_drain }, \ __SRCU_DEP_MAP_INIT(name) \ } @@ -57,15 +65,20 @@ void srcu_tiny_irq_work(struct irq_work *irq_work); #define DEFINE_SRCU_FAST_UPDOWN(name) DEFINE_SRCU(name) #define DEFINE_STATIC_SRCU_FAST_UPDOWN(name) \ static struct srcu_struct name = __SRCU_STRUCT_INIT(name, name, name, name) +#define DEFINE_SRCU_ATOMIC(name) DEFINE_SRCU(name) +#define DEFINE_STATIC_SRCU_ATOMIC(name) \ + static struct srcu_struct name = __SRCU_STRUCT_INIT(name, name, name, name) // Dummy structure for srcu_notifier_head. struct srcu_usage { }; #define __SRCU_USAGE_INIT(name) { } #define __init_srcu_struct_fast __init_srcu_struct #define __init_srcu_struct_fast_updown __init_srcu_struct +#define __init_srcu_struct_atomic __init_srcu_struct #ifndef CONFIG_DEBUG_LOCK_ALLOC #define init_srcu_struct_fast init_srcu_struct #define init_srcu_struct_fast_updown init_srcu_struct +#define init_srcu_struct_atomic init_srcu_struct #endif // #ifndef CONFIG_DEBUG_LOCK_ALLOC void synchronize_srcu(struct srcu_struct *ssp); @@ -131,10 +144,7 @@ static inline void synchronize_srcu_expedited(struct srcu_struct *ssp) synchronize_srcu(ssp); } -static inline void srcu_barrier(struct srcu_struct *ssp) -{ - synchronize_srcu(ssp); -} +void srcu_barrier(struct srcu_struct *ssp); static inline void srcu_expedite_current(struct srcu_struct *ssp) { } #define srcu_check_read_flavor(ssp, read_flavor) do { } while (0) diff --git a/include/linux/srcutree.h b/include/linux/srcutree.h index 75e54e4f963f..93b76e543894 100644 --- a/include/linux/srcutree.h +++ b/include/linux/srcutree.h @@ -13,6 +13,8 @@ #include <linux/rcu_node_tree.h> #include <linux/completion.h> +#include <linux/irq_work_types.h> +#include <linux/llist.h> struct srcu_node; struct srcu_struct; @@ -41,6 +43,8 @@ struct srcu_data { bool srcu_cblist_invoking; /* Invoking these CBs? */ struct timer_list delay_work; /* Delay for CB invoking */ struct work_struct work; /* Context for CB invoking. */ + struct llist_head defer_cbs; /* Callbacks deferred on re-entry. */ + struct llist_node defer_link; /* Links onto the per-CPU deferral drain list */ struct rcu_head srcu_barrier_head; /* For srcu_barrier() use. */ struct rcu_head srcu_ec_head; /* For srcu_expedite_current() use. */ int srcu_ec_state; /* State for srcu_expedite_current(). */ @@ -76,6 +80,7 @@ struct srcu_usage { struct mutex srcu_cb_mutex; /* Serialize CB preparation. */ raw_spinlock_t __private lock; /* Protect counters and size state. */ struct mutex srcu_gp_mutex; /* Serialize GP work. */ + atomic_t srcu_atomic_gp_flag; /* Serialize atomic GP work. */ unsigned long srcu_gp_seq; /* Grace-period seq #. */ unsigned long srcu_gp_seq_needed; /* Latest gp_seq needed. */ unsigned long srcu_gp_seq_needed_exp; /* Furthest future exp GP. */ @@ -209,6 +214,12 @@ struct srcu_struct { * instead of smp_mb(), and given that the first (for example) * srcu_read_lock_fast() might race with the first synchronize_srcu(), * this different must be specified at initialization time. + * + * If you use any of the DEFINE_SRCU() functions within a module, the + * module-entry code will invoke init_srcu_struct() and the module-exit + * code will invoke cleanup_srcu_struct(). This means that if your module + * passes the resulting srcu_struct structure to call_srcu(), you will + * need to also pass this structure to srcu_barrier() prior to module exit. */ #ifdef MODULE # define __DEFINE_SRCU(name, fast, is_static) \ @@ -225,14 +236,16 @@ struct srcu_struct { is_static struct srcu_struct name = \ __SRCU_STRUCT_INIT(name, name##_srcu_usage, name##_srcu_data, fast) #endif -#define DEFINE_SRCU(name) __DEFINE_SRCU(name, 0, /* not static */) +#define DEFINE_SRCU(name) __DEFINE_SRCU(name, 0, /* !static */) #define DEFINE_STATIC_SRCU(name) __DEFINE_SRCU(name, 0, static) -#define DEFINE_SRCU_FAST(name) __DEFINE_SRCU(name, SRCU_READ_FLAVOR_FAST, /* not static */) +#define DEFINE_SRCU_FAST(name) __DEFINE_SRCU(name, SRCU_READ_FLAVOR_FAST, /* !static */) #define DEFINE_STATIC_SRCU_FAST(name) __DEFINE_SRCU(name, SRCU_READ_FLAVOR_FAST, static) #define DEFINE_SRCU_FAST_UPDOWN(name) __DEFINE_SRCU(name, SRCU_READ_FLAVOR_FAST_UPDOWN, \ - /* not static */) + /* !static */) #define DEFINE_STATIC_SRCU_FAST_UPDOWN(name) \ __DEFINE_SRCU(name, SRCU_READ_FLAVOR_FAST_UPDOWN, static) +#define DEFINE_SRCU_ATOMIC(name) __DEFINE_SRCU(name, SRCU_READ_FLAVOR_ATOMIC, /* !static */) +#define DEFINE_STATIC_SRCU_ATOMIC(name) __DEFINE_SRCU(name, SRCU_READ_FLAVOR_ATOMIC, static) int __srcu_read_lock(struct srcu_struct *ssp) __acquires_shared(ssp); void synchronize_srcu_expedited(struct srcu_struct *ssp); diff --git a/include/linux/stmmac.h b/include/linux/stmmac.h index 4430b967abde..00be2df63d22 100644 --- a/include/linux/stmmac.h +++ b/include/linux/stmmac.h @@ -242,6 +242,8 @@ struct plat_stmmacenet_data { * that phylink uses. */ phy_interface_t phy_interface; + bool has_internal_tx_delay; + bool has_internal_rx_delay; struct stmmac_mdio_bus_data *mdio_bus_data; struct device_node *phy_node; struct device_node *mdio_node; diff --git a/include/linux/string.h b/include/linux/string.h index 5702daca4326..6cb5cdd01158 100644 --- a/include/linux/string.h +++ b/include/linux/string.h @@ -278,6 +278,19 @@ static inline void memcpy_flushcache(void *dst, const void *src, size_t cnt) } #endif +#ifndef memcpy_nontemporal +/* + * memcpy_nontemporal() requests a non-temporal copy when the + * architecture has a suitable backend. Architectures without a + * specialized backend fall back to memcpy(). Keep this as a + * function-like macro so the compiler can still see the original + * memcpy() call site and preserve the usual FORTIFY coverage when + * object sizes remain visible there, while keeping the API void. + */ +#define memcpy_nontemporal(dst, src, len) \ + ((void)memcpy(dst, src, len)) +#endif + void *memchr_inv(const void *s, int c, size_t n); char *strreplace(char *str, char old, char new); diff --git a/include/linux/string_choices.h b/include/linux/string_choices.h index ee84087d4b26..a2eba454004f 100644 --- a/include/linux/string_choices.h +++ b/include/linux/string_choices.h @@ -65,6 +65,12 @@ static inline const char *str_read_write(bool v) } #define str_write_read(v) str_read_write(!(v)) +static inline const char *str_supported_unsupported(bool v) +{ + return v ? "supported" : "unsupported"; +} +#define str_unsupported_supported(v) str_supported_unsupported(!(v)) + static inline const char *str_true_false(bool v) { return v ? "true" : "false"; diff --git a/include/linux/sunrpc/clnt.h b/include/linux/sunrpc/clnt.h index 3c2b8c355ab3..30344c0d6a9d 100644 --- a/include/linux/sunrpc/clnt.h +++ b/include/linux/sunrpc/clnt.h @@ -199,7 +199,8 @@ struct rpc_xprt *rpc_task_get_xprt(struct rpc_clnt *clnt, int rpcb_create_local(struct net *); void rpcb_put_local(struct net *); -int rpcb_register(struct net *, u32, u32, int, unsigned short); +int rpcb_register(struct net *net, u32 prog, u32 vers, int prot, + unsigned short port); int rpcb_v4_register(struct net *net, const u32 program, const u32 version, const struct sockaddr *address, diff --git a/include/linux/sunrpc/svc.h b/include/linux/sunrpc/svc.h index 2db1b9ec5658..cfb5ab17dabd 100644 --- a/include/linux/sunrpc/svc.h +++ b/include/linux/sunrpc/svc.h @@ -78,6 +78,7 @@ struct svc_serv { unsigned int sv_max_payload; /* datagram payload size */ unsigned int sv_max_mesg; /* max_payload + 1 page for overheads */ unsigned int sv_xdrsize; /* XDR buffer size */ + atomic_t sv_rpcb_failures; /* unanswered rpcbind calls */ struct list_head sv_permsocks; /* all permanent sockets */ struct list_head sv_tempsocks; /* all temporary sockets */ int sv_tmpcnt; /* count of temporary "valid" sockets */ @@ -86,6 +87,8 @@ struct svc_serv { char * sv_name; /* service name */ bool sv_is_pooled; /* is this a pooled service? */ + /* Caller registers with rpcbind itself. Set before svc_bind(). */ + bool sv_no_rpcbind; struct svc_pool * sv_pools; /* array of thread pools */ int (*sv_threadfn)(void *data); @@ -451,6 +454,7 @@ int sunrpc_set_pool_mode(const char *val); int sunrpc_get_pool_mode(char *val, size_t size); void svc_rpcb_cleanup(struct svc_serv *serv, struct net *net); int svc_bind(struct svc_serv *serv, struct net *net); +unsigned int svc_rpcb_failure_count(struct svc_serv *serv); struct svc_serv *svc_create(struct svc_program *, unsigned int, int (*threadfn)(void *data)); bool svc_rqst_replace_page(struct svc_rqst *rqstp, @@ -471,8 +475,9 @@ unsigned int svc_serv_maxthreads(const struct svc_serv *serv); int svc_pool_stats_open(struct svc_info *si, struct file *file); void svc_process(struct svc_rqst *rqstp); void svc_process_bc(struct rpc_rqst *req, struct svc_rqst *rqstp); -int svc_register(const struct svc_serv *, struct net *, const int, - const unsigned short, const unsigned short); +int svc_register(struct svc_serv *serv, struct net *net, + const int family, const unsigned short proto, + const unsigned short port); void svc_wake_up(struct svc_serv *); void svc_reserve(struct svc_rqst *rqstp, int space); @@ -529,6 +534,8 @@ static inline void svc_reserve_auth(struct svc_rqst *rqstp, int space) * svcxdr_init_decode - Prepare an xdr_stream for Call decoding * @rqstp: controlling server RPC transaction context * + * The stream records @rqstp, so a codec running on it reaches the + * transaction with svcxdr_rqst(). */ static inline void svcxdr_init_decode(struct svc_rqst *rqstp) { @@ -541,12 +548,15 @@ static inline void svcxdr_init_decode(struct svc_rqst *rqstp) xdr_init_decode(xdr, buf, argv->iov_base, NULL); xdr_set_scratch_folio(xdr, rqstp->rq_scratch_folio); + xdr->xdrgen_ctx = rqstp; } /** * svcxdr_init_encode - Prepare an xdr_stream for svc Reply encoding * @rqstp: controlling server RPC transaction context * + * The stream records @rqstp, so a codec running on it reaches the + * transaction with svcxdr_rqst(). */ static inline void svcxdr_init_encode(struct svc_rqst *rqstp) { @@ -564,6 +574,19 @@ static inline void svcxdr_init_encode(struct svc_rqst *rqstp) xdr->page_ptr = buf->pages - 1; buf->buflen = PAGE_SIZE * (rqstp->rq_page_end - buf->pages); xdr->rqst = NULL; + xdr->xdrgen_ctx = rqstp; +} + +/** + * svcxdr_rqst - Retrieve the transaction bound to an xdr_stream + * @xdr: stream to query + * + * Return: the controlling svc_rqst when @xdr was initialized by + * svcxdr_init_decode() or svcxdr_init_encode(), otherwise NULL. + */ +static inline struct svc_rqst *svcxdr_rqst(struct xdr_stream *xdr) +{ + return xdr->xdrgen_ctx; } /** @@ -588,6 +611,74 @@ static inline void svcxdr_encode_opaque_pages(struct svc_rqst *rqstp, } /** + * svcxdr_encode_opaque_payload - Encode a page-resident opaque data item + * @xdr: xdr_stream to be updated + * @len: number of octets of content in the data item + * + * Context: Process context. @xdr must have been initialized by + * svcxdr_init_encode() and still be positioned in the reply + * head. + * + * Return: + * %true: Success + * %false: The reply head cannot hold the length prefix and XDR + * padding, or the transport could not accommodate the result payload + */ +static inline bool svcxdr_encode_opaque_payload(struct xdr_stream *xdr, u32 len) +{ + struct svc_rqst *rqstp = svcxdr_rqst(xdr); + struct xdr_buf *buf = xdr->buf; + + /* + * The length prefix and any pad word must land in the reply + * head. A prefix that spills into the first Reply page lands + * on the payload itself, and xdr_write_pages() BUGs when the + * pad word does not fit. + */ + if ((xdr->end - xdr->p) * XDR_UNIT < XDR_UNIT + xdr_pad_size(len)) + return false; + if (xdr_stream_encode_u32(xdr, len) < 0) + return false; + svcxdr_encode_opaque_pages(rqstp, xdr, buf->pages, buf->page_base, len); + if (svc_encode_result_payload(rqstp, buf->head->iov_len, len) < 0) + return false; + return true; +} + +/** + * svcxdr_decode_opaque_payload - Decode a page-resident opaque data item + * @xdr: xdr_stream to be decoded + * @payload: on success, describes the octets of the data item's content + * @maxlen: largest data item length the caller will accept, or zero for + * no limit + * + * A bulk payload such as the content of an NFS WRITE request resides in + * the pages of the Receive buffer. Rather than copy it, set @payload to + * describe the item's content in place. + * + * Context: Process context. @xdr must have been initialized by + * svcxdr_init_decode(). + * + * Return: + * %true: @payload describes the item in place and @xdr has advanced + * past it + * %false: a bounds error occurred, or the length prefix exceeds + * @maxlen; @payload is undefined + */ +static inline bool svcxdr_decode_opaque_payload(struct xdr_stream *xdr, + struct xdr_buf *payload, + u32 maxlen) +{ + u32 len; + + if (xdr_stream_decode_u32(xdr, &len) < 0) + return false; + if (maxlen && len > maxlen) + return false; + return xdr_stream_subsegment(xdr, payload, len); +} + +/** * svcxdr_set_auth_slack - * @rqstp: RPC transaction * @slack: buffer space to reserve for the transaction's security flavor diff --git a/include/linux/sunrpc/svc_rdma.h b/include/linux/sunrpc/svc_rdma.h index 76aa5ec4ab40..5ccb13bf906f 100644 --- a/include/linux/sunrpc/svc_rdma.h +++ b/include/linux/sunrpc/svc_rdma.h @@ -142,7 +142,7 @@ enum { #define RPCSVC_MAXPAYLOAD_RDMA RPCSVC_MAXPAYLOAD /** - * svc_rdma_send_cid_init - Initialize a Receive Queue completion ID + * svc_rdma_recv_cid_init - Initialize a Receive Queue completion ID * @rdma: controlling transport * @cid: completion ID to initialize */ diff --git a/include/linux/sunrpc/svc_xprt.h b/include/linux/sunrpc/svc_xprt.h index da2a2531e110..2af222f3ea2c 100644 --- a/include/linux/sunrpc/svc_xprt.h +++ b/include/linux/sunrpc/svc_xprt.h @@ -37,6 +37,9 @@ struct svc_xprt_class { struct list_head xcl_list; u32 xcl_max_payload; int xcl_ident; + u32 xcl_flags; +/* Set only on classes whose xpo_has_wspace() reads xpt_reserved */ +#define SVC_XPRT_FLAG_WSPACE_RESERVE BIT(0) }; /* @@ -59,7 +62,7 @@ struct svc_xprt { unsigned long xpt_flags; struct svc_serv *xpt_server; /* service for transport */ - atomic_t xpt_reserved; /* space on outq that is rsvd */ + atomic_t xpt_reserved; /* outq space rsvd, UDP only */ atomic_t xpt_nr_rqsts; /* Number of requests */ struct mutex xpt_mutex; /* to serialize sending data */ spinlock_t xpt_lock; /* protects sk_deferred diff --git a/include/linux/sunrpc/xdr.h b/include/linux/sunrpc/xdr.h index b102b4f21e6b..2b33fc64be17 100644 --- a/include/linux/sunrpc/xdr.h +++ b/include/linux/sunrpc/xdr.h @@ -244,6 +244,7 @@ struct xdr_stream { unsigned int nwords; /* Remaining decode buffer length */ struct rpc_rqst *rqst; /* For debugging */ + void *xdrgen_ctx; /* Context for generated codecs */ }; /* diff --git a/include/linux/sunrpc/xdrgen/_defs.h b/include/linux/sunrpc/xdrgen/_defs.h index 8f3776ef3229..d767206feea4 100644 --- a/include/linux/sunrpc/xdrgen/_defs.h +++ b/include/linux/sunrpc/xdrgen/_defs.h @@ -5,11 +5,17 @@ * This header defines XDR data type primitives specified in * Section 4 of RFC 4506, used by RPC programs implemented * in the Linux kernel. + * + * A generated definition can also embed a struct xdr_buf, which + * carries a page-resident payload captured by reference, so this + * header pulls in the kernel's XDR types as well. */ #ifndef _SUNRPC_XDRGEN__DEFS_H_ #define _SUNRPC_XDRGEN__DEFS_H_ +#include <linux/sunrpc/xdr.h> + #define TRUE (true) #define FALSE (false) @@ -23,6 +29,34 @@ typedef struct { u8 *data; } opaque; +/* + * Cursor a hook-driven aggregate codec hands to its begin/item/end + * hooks, one element at a time, in place of a materialized C array. + * The generated framing owns it. @xdr is the RPC layer's stream and + * @ctx its xdrgen_ctx, the svc_rqst on the server. @index is the + * current element; @member_id selects among a type's marked members. + * For a counted array @count is the wire length: a decoder fills it + * before the begin hook runs, an encoder's begin hook sets it. The + * hook contract is under "Pragma aggregate" in + * tools/net/sunrpc/xdrgen/README. + * + * An optional-data list ("type *name") carries no count. Its encoder + * pulls elements until the item hook returns false. False ends the + * list rather than failing the encode: the framing writes the + * value-follows terminator, so a producer that stops early leaves a + * valid truncated list on the wire. A producer that stops for a + * mid-list error records it in the application state @ctx reaches, + * and either returns false from the end hook, which fails the + * encode, or reports it in the procedure status. + */ +struct xdrgen_aggregate_cursor { + struct xdr_stream *xdr; + u32 index; + u32 count; + unsigned int member_id; + void *ctx; +}; + #define XDR_void (0) #define XDR_bool (1) #define XDR_short (1) diff --git a/include/linux/sunrpc/xdrgen/nfs2.h b/include/linux/sunrpc/xdrgen/nfs2.h new file mode 100644 index 000000000000..1d26de39ac69 --- /dev/null +++ b/include/linux/sunrpc/xdrgen/nfs2.h @@ -0,0 +1,298 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Generated by xdrgen. Manual edits will be lost. */ +/* XDR specification file: ../../Documentation/sunrpc/xdr/nfs2.x */ +/* XDR specification modification time: Tue Sep 8 11:26:28 2026 */ + +#ifndef _LINUX_XDRGEN_NFS2_DEF_H +#define _LINUX_XDRGEN_NFS2_DEF_H + +#include <linux/types.h> +#include <linux/sunrpc/xdrgen/_defs.h> + +enum { NFS_MAXDATA = 8192 }; + +enum { NFS_MAXPATHLEN = 1024 }; + +enum { NFS_MAXNAMLEN = 255 }; + +enum { NFS_COOKIESIZE = 4 }; + +enum { NFS_FHSIZE = 32 }; + +enum nfsstat { + NFS_OK = 0, + NFSERR_PERM = 1, + NFSERR_NOENT = 2, + NFSERR_IO = 5, + NFSERR_NXIO = 6, + NFSERR_ACCES = 13, + NFSERR_EXIST = 17, + NFSERR_NODEV = 19, + NFSERR_NOTDIR = 20, + NFSERR_ISDIR = 21, + NFSERR_FBIG = 27, + NFSERR_NOSPC = 28, + NFSERR_ROFS = 30, + NFSERR_NAMETOOLONG = 63, + NFSERR_NOTEMPTY = 66, + NFSERR_DQUOT = 69, + NFSERR_STALE = 70, + NFSERR_WFLUSH = 99, +}; + +typedef __be32 nfsstat; + +enum ftype { + NFNON = 0, + NFREG = 1, + NFDIR = 2, + NFBLK = 3, + NFCHR = 4, + NFLNK = 5, +}; + +typedef enum ftype ftype; + +typedef u8 fhandle[NFS_FHSIZE]; + +typedef u8 nfscookie[NFS_COOKIESIZE]; + +struct timeval { + u32 seconds; + u32 useconds; +}; + +struct fattr { + ftype type; + u32 mode; + u32 nlink; + u32 uid; + u32 gid; + u32 size; + u32 blocksize; + u32 rdev; + u32 blocks; + u32 fsid; + u32 fileid; + struct timeval atime; + struct timeval mtime; + struct timeval ctime; +}; + +struct sattr { + u32 mode; + u32 uid; + u32 gid; + u32 size; + struct timeval atime; + struct timeval mtime; +}; + +typedef string filename; + +typedef string path; + +struct attrstat { + nfsstat status; + union { + struct fattr attributes; + } u; +}; + +struct diropargs { + fhandle dir; + filename name; +}; + +struct diropok { + fhandle file; + struct fattr attributes; +}; + +struct diropres { + nfsstat status; + union { + struct diropok diropok; + } u; +}; + +struct sattrargs { + fhandle file; + struct sattr attributes; +}; + +struct readlinkres { + nfsstat status; + union { + path data; + } u; +}; + +struct readargs { + fhandle file; + u32 offset; + u32 count; + u32 totalcount; +}; + +struct readresok { + struct fattr attributes; + opaque data; +}; + +struct readres { + nfsstat status; + union { + struct readresok readresok; + } u; +}; + +struct writeargs { + fhandle file; + u32 beginoffset; + u32 offset; + u32 totalcount; + struct xdr_buf data; +}; + +struct createargs { + struct diropargs where; + struct sattr attributes; +}; + +struct renameargs { + struct diropargs from; + struct diropargs to; +}; + +struct linkargs { + fhandle from; + struct diropargs to; +}; + +struct symlinkargs { + struct diropargs from; + struct xdr_buf to; + struct sattr attributes; +}; + +struct readdirargs { + fhandle dir; + nfscookie cookie; + u32 count; +}; + +struct entry { + u32 fileid; + filename name; + nfscookie cookie; +}; + +struct readdirok { + struct entry *entries; + bool eof; +}; + +struct readdirres { + nfsstat status; + union { + struct readdirok readdirok; + } u; +}; + +struct info { + u32 tsize; + u32 bsize; + u32 blocks; + u32 bfree; + u32 bavail; +}; + +struct statfsres { + nfsstat status; + union { + struct info info; + } u; +}; + +enum { + NFSPROC_NULL = 0, + NFSPROC_GETATTR = 1, + NFSPROC_SETATTR = 2, + NFSPROC_ROOT = 3, + NFSPROC_LOOKUP = 4, + NFSPROC_READLINK = 5, + NFSPROC_READ = 6, + NFSPROC_WRITECACHE = 7, + NFSPROC_WRITE = 8, + NFSPROC_CREATE = 9, + NFSPROC_REMOVE = 10, + NFSPROC_RENAME = 11, + NFSPROC_LINK = 12, + NFSPROC_SYMLINK = 13, + NFSPROC_MKDIR = 14, + NFSPROC_RMDIR = 15, + NFSPROC_READDIR = 16, + NFSPROC_STATFS = 17, +}; + +#ifndef NFS_PROGRAM +#define NFS_PROGRAM (100003) +#endif + +#define NFS2_nfsstat_sz (XDR_int) +#define NFS2_ftype_sz (XDR_int) +#define NFS2_fhandle_sz (XDR_QUADLEN(NFS_FHSIZE)) +#define NFS2_nfscookie_sz (XDR_QUADLEN(NFS_COOKIESIZE)) +#define NFS2_timeval_sz \ + (XDR_unsigned_int + XDR_unsigned_int) +#define NFS2_fattr_sz \ + (NFS2_ftype_sz + XDR_unsigned_int + XDR_unsigned_int + XDR_unsigned_int + XDR_unsigned_int + XDR_unsigned_int + XDR_unsigned_int + XDR_unsigned_int + XDR_unsigned_int + XDR_unsigned_int + XDR_unsigned_int + NFS2_timeval_sz + NFS2_timeval_sz + NFS2_timeval_sz) +#define NFS2_sattr_sz \ + (XDR_unsigned_int + XDR_unsigned_int + XDR_unsigned_int + XDR_unsigned_int + NFS2_timeval_sz + NFS2_timeval_sz) +#define NFS2_filename_sz (XDR_unsigned_int + XDR_QUADLEN(NFS_MAXNAMLEN)) +#define NFS2_path_sz (XDR_unsigned_int + XDR_QUADLEN(NFS_MAXPATHLEN)) +#define NFS2_attrstat_sz \ + (NFS2_nfsstat_sz + NFS2_fattr_sz) +#define NFS2_diropargs_sz \ + (NFS2_fhandle_sz + NFS2_filename_sz) +#define NFS2_diropok_sz \ + (NFS2_fhandle_sz + NFS2_fattr_sz) +#define NFS2_diropres_sz \ + (NFS2_nfsstat_sz + NFS2_diropok_sz) +#define NFS2_sattrargs_sz \ + (NFS2_fhandle_sz + NFS2_sattr_sz) +#define NFS2_readlinkres_sz \ + (NFS2_nfsstat_sz + NFS2_path_sz) +#define NFS2_readargs_sz \ + (NFS2_fhandle_sz + XDR_unsigned_int + XDR_unsigned_int + XDR_unsigned_int) +#define NFS2_readresok_sz \ + (NFS2_fattr_sz + XDR_unsigned_int + XDR_QUADLEN(NFS_MAXDATA)) +#define NFS2_readres_sz \ + (NFS2_nfsstat_sz + NFS2_readresok_sz) +#define NFS2_writeargs_sz \ + (NFS2_fhandle_sz + XDR_unsigned_int + XDR_unsigned_int + XDR_unsigned_int + XDR_unsigned_int + XDR_QUADLEN(NFS_MAXDATA)) +#define NFS2_createargs_sz \ + (NFS2_diropargs_sz + NFS2_sattr_sz) +#define NFS2_renameargs_sz \ + (NFS2_diropargs_sz + NFS2_diropargs_sz) +#define NFS2_linkargs_sz \ + (NFS2_fhandle_sz + NFS2_diropargs_sz) +#define NFS2_symlinkargs_sz \ + (NFS2_diropargs_sz + NFS2_path_sz + NFS2_sattr_sz) +#define NFS2_readdirargs_sz \ + (NFS2_fhandle_sz + NFS2_nfscookie_sz + XDR_unsigned_int) +#define NFS2_entry_sz \ + (XDR_bool + XDR_unsigned_int + NFS2_filename_sz + NFS2_nfscookie_sz) +#define NFS2_readdirok_sz \ + (XDR_bool + XDR_bool) +#define NFS2_readdirres_sz \ + (NFS2_nfsstat_sz + NFS2_readdirok_sz) +#define NFS2_info_sz \ + (XDR_unsigned_int + XDR_unsigned_int + XDR_unsigned_int + XDR_unsigned_int + XDR_unsigned_int) +#define NFS2_statfsres_sz \ + (NFS2_nfsstat_sz + NFS2_info_sz) +#define NFS2_MAX_ARGS_SZ \ + (NFS2_writeargs_sz) + +#endif /* _LINUX_XDRGEN_NFS2_DEF_H */ diff --git a/include/linux/sunrpc/xprt.h b/include/linux/sunrpc/xprt.h index a82045804d34..005730623f3b 100644 --- a/include/linux/sunrpc/xprt.h +++ b/include/linux/sunrpc/xprt.h @@ -145,6 +145,7 @@ struct xprtsec_parms { enum xprtsec_policies policy; /* authentication material */ + key_serial_t keyring_serial; key_serial_t cert_serial; key_serial_t privkey_serial; }; diff --git a/include/linux/suspend.h b/include/linux/suspend.h index b02876f1ae38..a3815027773c 100644 --- a/include/linux/suspend.h +++ b/include/linux/suspend.h @@ -401,6 +401,7 @@ int hibernate_quiet_exec(int (*func)(void *data), void *data); int hibernate_resume_nonboot_cpu_disable(void); int arch_hibernation_header_save(void *addr, unsigned int max_size); int arch_hibernation_header_restore(void *addr); +bool arch_hibernation_available(void); #else /* CONFIG_HIBERNATION */ static inline void register_nosave_region(unsigned long b, unsigned long e) {} diff --git a/include/linux/swap.h b/include/linux/swap.h index 5658a1634b85..4f686709d71f 100644 --- a/include/linux/swap.h +++ b/include/linux/swap.h @@ -25,12 +25,6 @@ #define SWAP_FLAGS_VALID (SWAP_FLAG_PRIO_MASK | SWAP_FLAG_PREFER | \ SWAP_FLAG_DISCARD | SWAP_FLAG_DISCARD_ONCE | \ SWAP_FLAG_DISCARD_PAGES) - -static inline int current_is_kswapd(void) -{ - return current->flags & PF_KSWAPD; -} - /* * MAX_SWAPFILES defines the maximum number of swaptypes: things which can * be swapped to. The swap type and the offset into that swap type are @@ -246,7 +240,7 @@ struct swap_info_struct { struct plist_node list; /* entry in swap_active_head */ signed char type; /* strange name for an index */ unsigned int max; /* size of this swap device */ - struct swap_cluster_info *cluster_info; /* cluster info. Only for SSD */ + struct swap_cluster_info *cluster_info; /* array, one entry per cluster */ struct list_head free_clusters; /* free clusters list */ struct list_head full_clusters; /* full clusters list */ struct list_head nonfull_clusters[SWAP_NR_ORDERS]; @@ -278,15 +272,39 @@ struct swap_info_struct { const struct swap_ops *ops; }; -static inline swp_entry_t page_swap_entry(struct page *page) +/** + * folio_swap_entry - Return the swap entry at a page index within a folio. + * @folio: The folio. + * @idx: The index of the page within the folio. + * + * A folio in the swap cache occupies folio_nr_pages() contiguous swap + * entries starting at folio->swap. The caller must ensure the folio is + * in the swap cache and that @idx is within the folio. + */ +static inline +swp_entry_t folio_swap_entry(const struct folio *folio, unsigned long idx) { - struct folio *folio = page_folio(page); swp_entry_t entry = folio->swap; - entry.val += folio_page_idx(folio, page); + VM_WARN_ON_ONCE_FOLIO(idx >= folio_nr_pages(folio), folio); + entry.val += idx; return entry; } +/** + * folio_page_swap_entry - Return the swap entry of a page in a folio. + * @folio: The folio containing @page. + * @page: A page within @folio. + * + * The caller must ensure the folio is in the swap cache and that @page + * is part of @folio. + */ +static inline swp_entry_t folio_page_swap_entry(const struct folio *folio, + const struct page *page) +{ + return folio_swap_entry(folio, folio_page_idx(folio, page)); +} + /* linux/mm/page_alloc.c */ extern unsigned long totalreserve_pages; @@ -318,6 +336,9 @@ static inline bool lru_cache_disabled(void) extern unsigned long shrink_all_memory(unsigned long nr_pages); long remove_mapping(struct address_space *mapping, struct folio *folio); +long remove_mapping_set_shadow(struct address_space *mapping, + struct folio *folio, + struct mem_cgroup *target_memcg); #if defined(CONFIG_SYSFS) && defined(CONFIG_NUMA) extern int reclaim_register_node(struct node *node); @@ -339,6 +360,7 @@ void check_move_unevictable_folios(struct folio_batch *fbatch); extern void __meminit kswapd_run(int nid); extern void __meminit kswapd_stop(int nid); +bool current_is_kswapd(void); #ifdef CONFIG_SWAP int add_swap_extent(struct swap_info_struct *sis, unsigned long start_page, @@ -373,6 +395,8 @@ static inline long get_nr_swap_pages(void) extern void si_swapinfo(struct sysinfo *); extern int pin_hibernation_swap_type(dev_t device, sector_t offset); extern void unpin_hibernation_swap_type(int type); +extern int repin_hibernation_swap_type(int old_type, dev_t device, + sector_t offset); extern int find_hibernation_swap_type(dev_t device, sector_t offset); int find_first_swap(dev_t *device); extern unsigned int count_swap_pages(int, int); @@ -402,6 +426,8 @@ void swap_put_entries_direct(swp_entry_t entry, int nr); */ bool folio_free_swap(struct folio *folio); +void swap_writeback_dropbehind_folio(struct folio *folio); + /* Allocate / free (hibernation) exclusive entries */ swp_entry_t swap_alloc_hibernation_slot(int type); void swap_free_hibernation_slot(swp_entry_t entry); @@ -412,6 +438,7 @@ static inline void put_swap_device(struct swap_info_struct *si) } #else /* CONFIG_SWAP */ +static inline void swap_writeback_dropbehind_folio(struct folio *folio) {} static inline struct swap_info_struct *get_swap_device(swp_entry_t entry) { return NULL; @@ -508,8 +535,9 @@ static inline void mem_cgroup_uncharge_swap(unsigned short id, unsigned int nr_p __mem_cgroup_uncharge_swap(id, nr_pages); } -extern long mem_cgroup_get_nr_swap_pages(struct mem_cgroup *memcg); -extern bool mem_cgroup_swap_full(struct folio *folio); +long mem_cgroup_get_folio_swap_margin(const struct folio *folio); +long mem_cgroup_get_nr_swap_pages(const struct mem_cgroup *memcg); +bool mem_cgroup_swap_full(const struct folio *folio); #else static inline int mem_cgroup_try_charge_swap(struct folio *folio) { @@ -521,12 +549,17 @@ static inline void mem_cgroup_uncharge_swap(unsigned short id, { } -static inline long mem_cgroup_get_nr_swap_pages(struct mem_cgroup *memcg) +static inline long mem_cgroup_get_folio_swap_margin(const struct folio *folio) +{ + return PAGE_COUNTER_MAX; +} + +static inline long mem_cgroup_get_nr_swap_pages(const struct mem_cgroup *memcg) { return get_nr_swap_pages(); } -static inline bool mem_cgroup_swap_full(struct folio *folio) +static inline bool mem_cgroup_swap_full(const struct folio *folio) { return vm_swap_full(); } diff --git a/include/linux/swapops.h b/include/linux/swapops.h index e7d0d529f3e0..603f9e3f909c 100644 --- a/include/linux/swapops.h +++ b/include/linux/swapops.h @@ -261,11 +261,6 @@ static inline swp_entry_t make_hwpoison_entry(struct page *page) return swp_entry(SWP_HWPOISON, page_to_pfn(page)); } -static inline int is_hwpoison_entry(swp_entry_t entry) -{ - return swp_type(entry) == SWP_HWPOISON; -} - #else static inline swp_entry_t make_hwpoison_entry(struct page *page) @@ -273,10 +268,6 @@ static inline swp_entry_t make_hwpoison_entry(struct page *page) return swp_entry(0, 0); } -static inline int is_hwpoison_entry(swp_entry_t swp) -{ - return 0; -} #endif typedef unsigned long pte_marker; diff --git a/include/linux/swiotlb.h b/include/linux/swiotlb.h index 2a421241b980..5f24edf47b68 100644 --- a/include/linux/swiotlb.h +++ b/include/linux/swiotlb.h @@ -38,7 +38,7 @@ struct scatterlist; #define IO_TLB_DEFAULT_SIZE (64UL << 20) #endif -unsigned long swiotlb_size_or_default(void); +unsigned long swiotlb_default_pool_size(void); void __init swiotlb_init_remap(bool addressing_limit, unsigned int flags, int (*remap)(void *tlb, unsigned long nslabs)); int swiotlb_init_late(size_t size, gfp_t gfp_mask, diff --git a/include/linux/syscalls.h b/include/linux/syscalls.h index 8413b624ad47..552d9710a0d5 100644 --- a/include/linux/syscalls.h +++ b/include/linux/syscalls.h @@ -92,6 +92,7 @@ struct file_attr; #include <linux/quota.h> #include <linux/key.h> #include <linux/personality.h> +#include <linux/error-injection.h> #include <trace/syscall.h> #ifdef CONFIG_ARCH_HAS_SYSCALL_WRAPPER diff --git a/include/linux/tcp.h b/include/linux/tcp.h index 6a8c77719322..d18f7bc1e5d0 100644 --- a/include/linux/tcp.h +++ b/include/linux/tcp.h @@ -233,6 +233,13 @@ struct tcp_sock { is_sack_reneg:1, /* in recovery from loss with SACK reneg? */ is_cwnd_limited:1,/* forward progress limited by snd_cwnd? */ recvmsg_inq : 1;/* Indicate # of bytes in queue upon recvmsg */ +#ifdef CONFIG_BPF + u32 bpf_tcp_ops_flags; +#define BPF_TCP_OPS_TEST_FLAG(TP, ARG) \ + (READ_ONCE((TP)->bpf_tcp_ops_flags) & BPF_TCP_OPS_FLAG_ ## ARG) +#else +#define BPF_TCP_OPS_TEST_FLAG(TP, ARG) (0) +#endif __cacheline_group_end(tcp_sock_read_txrx); /* RX read-mostly hotpath cache lines */ @@ -269,6 +276,9 @@ struct tcp_sock { */ u32 snd_sml; /* Last byte of the most recently transmitted small packet */ u8 chrono_type; /* current chronograph type */ + u8 tcp_nospace; /* mirrors SOCK_NOSPACE, must be set whenever + * SOCK_NOSPACE is set. + */ u32 chrono_start; /* Start time in jiffies of a TCP chrono */ u32 chrono_stat[3]; /* Time in jiffies for chrono_stat stats */ u32 write_seq; /* Tail(+1) of data held in tcp send buffer */ diff --git a/include/linux/thermal.h b/include/linux/thermal.h index 083b4f533933..306ad17aed89 100644 --- a/include/linux/thermal.h +++ b/include/linux/thermal.h @@ -293,8 +293,9 @@ struct device *thermal_zone_device(struct thermal_zone_device *tzd); void thermal_zone_device_update(struct thermal_zone_device *, enum thermal_notify_event); -struct thermal_cooling_device *thermal_cooling_device_register(const char *, - void *, const struct thermal_cooling_device_ops *); +struct thermal_cooling_device *thermal_cooling_device_create( + struct device *parent, const char *type, void *devdata, + const struct thermal_cooling_device_ops *ops); struct thermal_cooling_device * devm_thermal_cooling_device_register(struct device *dev, const char *type, void *devdata, @@ -340,9 +341,9 @@ static inline void thermal_zone_device_update(struct thermal_zone_device *tz, enum thermal_notify_event event) { } -static inline struct thermal_cooling_device * -thermal_cooling_device_register(const char *type, void *devdata, - const struct thermal_cooling_device_ops *ops) +static inline struct thermal_cooling_device *thermal_cooling_device_create( + struct device *parent, const char *type, void *devdata, + const struct thermal_cooling_device_ops *ops) { return ERR_PTR(-ENODEV); } static inline struct thermal_cooling_device * @@ -391,4 +392,12 @@ static inline void thermal_pm_prepare(void) {} static inline void thermal_pm_complete(void) {} #endif /* CONFIG_THERMAL */ +static inline struct thermal_cooling_device *thermal_cooling_device_register( + const char *type, void *devdata, + const struct thermal_cooling_device_ops *ops) +{ + return thermal_cooling_device_create(NULL, type, devdata, ops); +} + + #endif /* __THERMAL_H__ */ diff --git a/include/linux/thunderbolt.h b/include/linux/thunderbolt.h index b62dfa52b149..57502da29080 100644 --- a/include/linux/thunderbolt.h +++ b/include/linux/thunderbolt.h @@ -370,7 +370,7 @@ int tb_xdomain_request(struct tb_xdomain *xd, const void *request, * @uuid: XDomain messages with this UUID are dispatched to this handler * @callback: Callback called with the XDomain message. Returning %1 * here tells the XDomain core that the message was handled - * by this handler and should not be forwared to other + * by this handler and should not be forwarded to other * handlers. * @data: Data passed with the callback * @list: Handlers are linked using this @@ -507,6 +507,7 @@ void tb_service_properties_changed(struct tb_service *svc); * @iobase: MMIO space of the NHI * @tx_rings: All Tx rings available on this host controller * @rx_rings: All Rx rings available on this host controller + * @interrupt_mask: Shadow copy of the ring interrupt mask register * @going_away: The host controller device is about to disappear so when * this flag is set, avoid touching the hardware anymore. * @iommu_dma_protection: An IOMMU will isolate external-facing ports. @@ -528,6 +529,7 @@ struct tb_nhi { void __iomem *iobase; struct tb_ring **tx_rings; struct tb_ring **rx_rings; + u32 *interrupt_mask; bool going_away; bool iommu_dma_protection; struct work_struct interrupt_work; @@ -553,6 +555,8 @@ struct tb_nhi { * @work: Interrupt work structure * @is_tx: Is the ring Tx or Rx * @running: Is the ring running + * @notify_pending: Controller has not been notified about the posted + * descriptors yet * @irq: MSI-X irq number if the ring uses MSI-X. %0 otherwise. * @vector: MSI-X vector number the ring uses (only set if @irq is > 0) * @flags: Ring specific flags @@ -582,6 +586,7 @@ struct tb_ring { struct work_struct work; bool is_tx:1; bool running:1; + bool notify_pending:1; int irq; u8 vector; unsigned int flags; @@ -672,7 +677,8 @@ bool tb_ring_flush(struct tb_ring *ring, unsigned int timeout_msec); void tb_ring_stop(struct tb_ring *ring); void tb_ring_free(struct tb_ring *ring); -int __tb_ring_enqueue(struct tb_ring *ring, struct ring_frame *frame); +int __tb_ring_enqueue(struct tb_ring *ring, struct ring_frame *frame, bool more); +void tb_ring_notify(struct tb_ring *ring); /** * tb_ring_rx() - enqueue a frame on an RX ring @@ -693,7 +699,24 @@ int __tb_ring_enqueue(struct tb_ring *ring, struct ring_frame *frame); static inline int tb_ring_rx(struct tb_ring *ring, struct ring_frame *frame) { WARN_ON(ring->is_tx); - return __tb_ring_enqueue(ring, frame); + return __tb_ring_enqueue(ring, frame, false); +} + +/** + * tb_ring_rx_more() - enqueue a frame on an RX ring without notifying + * @ring: Ring to enqueue the frame + * @frame: Frame to enqueue + * + * Same as tb_ring_rx() but does not notify the controller about the + * enqueued frame. The caller must call tb_ring_notify() once it is done + * enqueuing frames. + * + * Return: %-ESHUTDOWN if tb_ring_stop() has been called, %0 otherwise. + */ +static inline int tb_ring_rx_more(struct tb_ring *ring, struct ring_frame *frame) +{ + WARN_ON(ring->is_tx); + return __tb_ring_enqueue(ring, frame, true); } /** @@ -714,11 +737,27 @@ static inline int tb_ring_rx(struct tb_ring *ring, struct ring_frame *frame) static inline int tb_ring_tx(struct tb_ring *ring, struct ring_frame *frame) { WARN_ON(!ring->is_tx); - return __tb_ring_enqueue(ring, frame); + return __tb_ring_enqueue(ring, frame, false); +} + +/** + * tb_ring_tx_more() - enqueue a frame on a TX ring without notifying + * @ring: Ring to enqueue the frame + * @frame: Frame to enqueue + * + * Same as tb_ring_rx_more() but for TX ring. + * + * Return: %-ESHUTDOWN if tb_ring_stop() has been called, %0 otherwise. + */ +static inline int tb_ring_tx_more(struct tb_ring *ring, struct ring_frame *frame) +{ + WARN_ON(!ring->is_tx); + return __tb_ring_enqueue(ring, frame, true); } /* Used only when the ring is in polling mode */ struct ring_frame *tb_ring_poll(struct tb_ring *ring); +bool tb_ring_poll_pending(struct tb_ring *ring); void tb_ring_poll_complete(struct tb_ring *ring); int tb_ring_throttling(struct tb_ring *ring, unsigned int interval_nsec); diff --git a/include/linux/tnum.h b/include/linux/tnum.h index ca2cfec8de08..16682248f963 100644 --- a/include/linux/tnum.h +++ b/include/linux/tnum.h @@ -134,4 +134,6 @@ static inline bool tnum_subreg_is_const(struct tnum a) /* Returns the smallest member of t larger than z */ u64 tnum_step(struct tnum t, u64 z); +u32 tnum_alignment(struct tnum a); + #endif /* _LINUX_TNUM_H */ diff --git a/include/linux/trace_events.h b/include/linux/trace_events.h index 5cbd09c8be8d..321f2f3a6c8e 100644 --- a/include/linux/trace_events.h +++ b/include/linux/trace_events.h @@ -872,6 +872,7 @@ enum { FILTER_COMM, FILTER_CPU, FILTER_STACKTRACE, + FILTER_WITHIN, }; extern int trace_event_raw_init(struct trace_event_call *call); @@ -882,7 +883,7 @@ extern int trace_add_event_call(struct trace_event_call *call); extern int trace_remove_event_call(struct trace_event_call *call); extern int trace_event_get_offsets(struct trace_event_call *call); -int ftrace_set_clr_event(struct trace_array *tr, char *buf, int set); +int ftrace_set_clr_event(struct trace_array *tr, const char *buf, int set, bool handle_filter); int trace_set_clr_event(const char *system, const char *event, int set); int trace_array_set_clr_event(struct trace_array *tr, const char *system, const char *event, bool enable); diff --git a/include/linux/trace_remote_event.h b/include/linux/trace_remote_event.h index c8ae1e1f5e72..e4cc2d4497bc 100644 --- a/include/linux/trace_remote_event.h +++ b/include/linux/trace_remote_event.h @@ -3,6 +3,8 @@ #ifndef _LINUX_TRACE_REMOTE_EVENTS_H #define _LINUX_TRACE_REMOTE_EVENTS_H +#include <linux/types.h> + struct trace_remote; struct trace_event_fields; struct trace_seq; diff --git a/include/linux/trace_seq.h b/include/linux/trace_seq.h index 697d619aafdc..7174ebf3f015 100644 --- a/include/linux/trace_seq.h +++ b/include/linux/trace_seq.h @@ -86,7 +86,10 @@ static inline bool trace_seq_has_overflowed(struct trace_seq *s) * * Removes the last written character to the trace_seq @s. * - * Returns the last character or -1 if it is empty. + * Returns the last character, or -1 if the underlying seq_buf is empty or + * has overflowed. Note that only that buffer is consulted: a @s marked + * full by a write that did not fit, which trace_seq_has_overflowed() + * reports as overflowed, still pops the last character written. */ static inline int trace_seq_pop(struct trace_seq *s) { diff --git a/include/linux/uaccess.h b/include/linux/uaccess.h index eddbbb65ccc4..67cdb0a4f583 100644 --- a/include/linux/uaccess.h +++ b/include/linux/uaccess.h @@ -357,7 +357,7 @@ extern __must_check int check_zeroed_user(const void __user *from, size_t size); * struct). * * @ksize is just sizeof(*dst), and @usize should've been passed by userspace. - * The recommended usage is something like the following: + * The recommended usage is something like the following:: * * SYSCALL_DEFINE2(foobar, const struct foo __user *, uarg, size_t, usize) * { @@ -447,7 +447,7 @@ copy_struct_from_user(void *dst, size_t ksize, const void __user *src, * about data it does not. * * @ksize is just sizeof(*src), and @usize should've been passed by userspace. - * The recommended usage is something like the following: + * The recommended usage is something like the following:: * * SYSCALL_DEFINE2(foobar, struct foo __user *, uarg, size_t, usize) * { @@ -529,7 +529,7 @@ __copy_struct_generic_bounce_buffer(void *dst, size_t dstsize, memcpy(dst, src, size); } -/** +/* * This is like copy_struct_from_user(), but the * src buffer was already copied into a kernel * bounce buffer, so it will never return -EFAULT. @@ -553,7 +553,7 @@ copy_struct_from_bounce_buffer(void *dst, size_t dstsize, return 0; } -/** +/* * This is like copy_struct_to_user(), but the * dst buffer is a kernel bounce buffer instead * of a direct userspace buffer, so it will never return -EFAULT. @@ -770,7 +770,7 @@ static __always_inline void __scoped_user_rw_access_end(const void *p) * If the user access function inside the scope requires a fault label, it * can use @elbl or a different label outside the scope, which requires * that user access which is implemented with ASM GOTO has been properly - * wrapped. See unsafe_get_user() for reference. + * wrapped. See unsafe_get_user() for reference:: * * scoped_user_rw_access(ptr, efault) { * unsafe_get_user(rval, &ptr->rval, efault); diff --git a/include/linux/ucs2_string.h b/include/linux/ucs2_string.h index c499ae809c7d..74f23ca5a967 100644 --- a/include/linux/ucs2_string.h +++ b/include/linux/ucs2_string.h @@ -14,7 +14,16 @@ ssize_t ucs2_strscpy(ucs2_char_t *dst, const ucs2_char_t *src, size_t count); int ucs2_strncmp(const ucs2_char_t *a, const ucs2_char_t *b, size_t len); unsigned long ucs2_utf8size(const ucs2_char_t *src); +unsigned long +ucs2_as_utf8_l(u8 *dest, const ucs2_char_t *src, unsigned long limit, + unsigned long maxlength); + +static inline unsigned long ucs2_as_utf8(u8 *dest, const ucs2_char_t *src, - unsigned long maxlength); + unsigned long maxlength) +{ + return ucs2_as_utf8_l(dest, src, ucs2_strnlen(src, maxlength), + maxlength); +} #endif /* _LINUX_UCS2_STRING_H_ */ diff --git a/include/linux/uidgid.h b/include/linux/uidgid.h index 2dc767e08f54..02403629b49f 100644 --- a/include/linux/uidgid.h +++ b/include/linux/uidgid.h @@ -130,9 +130,9 @@ static inline bool kgid_has_mapping(struct user_namespace *ns, kgid_t gid) return from_kgid(ns, gid) != (gid_t) -1; } -u32 map_id_down(struct uid_gid_map *map, u32 id); -u32 map_id_up(struct uid_gid_map *map, u32 id); -u32 map_id_range_up(struct uid_gid_map *map, u32 id, u32 count); +u32 map_id_down(const struct uid_gid_map *map, u32 id); +u32 map_id_up(const struct uid_gid_map *map, u32 id); +u32 map_id_range_up(const struct uid_gid_map *map, u32 id, u32 count); #else @@ -182,17 +182,17 @@ static inline bool kgid_has_mapping(struct user_namespace *ns, kgid_t gid) return gid_valid(gid); } -static inline u32 map_id_down(struct uid_gid_map *map, u32 id) +static inline u32 map_id_down(const struct uid_gid_map *map, u32 id) { return id; } -static inline u32 map_id_range_up(struct uid_gid_map *map, u32 id, u32 count) +static inline u32 map_id_range_up(const struct uid_gid_map *map, u32 id, u32 count) { return id; } -static inline u32 map_id_up(struct uid_gid_map *map, u32 id) +static inline u32 map_id_up(const struct uid_gid_map *map, u32 id) { return id; } diff --git a/include/linux/uio.h b/include/linux/uio.h index fe2e985d74d2..638c1116e912 100644 --- a/include/linux/uio.h +++ b/include/linux/uio.h @@ -12,6 +12,7 @@ struct page; struct folio_queue; +struct dma_buf_io_map; typedef unsigned int __bitwise iov_iter_extraction_t; @@ -29,6 +30,7 @@ enum iter_type { ITER_FOLIOQ, ITER_XARRAY, ITER_DISCARD, + ITER_DMABUF_MAP, }; #define ITER_SOURCE 1 // == WRITE @@ -71,6 +73,7 @@ struct iov_iter { const struct folio_queue *folioq; struct xarray *xarray; void __user *ubuf; + struct dma_buf_io_map *dmabuf_map; }; size_t count; }; @@ -155,6 +158,11 @@ static inline bool iov_iter_is_xarray(const struct iov_iter *i) return iov_iter_type(i) == ITER_XARRAY; } +static inline bool iov_iter_is_dmabuf_map(const struct iov_iter *i) +{ + return iov_iter_type(i) == ITER_DMABUF_MAP; +} + static inline unsigned char iov_iter_rw(const struct iov_iter *i) { return i->data_source ? WRITE : READ; @@ -300,6 +308,9 @@ void iov_iter_folio_queue(struct iov_iter *i, unsigned int direction, unsigned int first_slot, unsigned int offset, size_t count); void iov_iter_xarray(struct iov_iter *i, unsigned int direction, struct xarray *xarray, loff_t start, size_t count); +void iov_iter_dmabuf_map(struct iov_iter *i, unsigned int direction, + struct dma_buf_io_map *map, + loff_t off, size_t count); ssize_t iov_iter_get_pages2(struct iov_iter *i, struct page **pages, size_t maxsize, unsigned maxpages, size_t *start); ssize_t iov_iter_get_pages_alloc2(struct iov_iter *i, struct page ***pages, diff --git a/include/linux/ulpi/driver.h b/include/linux/ulpi/driver.h index c668d9c8d876..c899197e0aa8 100644 --- a/include/linux/ulpi/driver.h +++ b/include/linux/ulpi/driver.h @@ -51,8 +51,8 @@ struct ulpi_driver { /* * use a macro to avoid include chaining to get THIS_MODULE */ -#define ulpi_register_driver(drv) __ulpi_register_driver(drv, THIS_MODULE) -int __ulpi_register_driver(struct ulpi_driver *drv, struct module *module); +#define ulpi_register_driver(drv) __ulpi_register_driver(drv, THIS_MODULE, KBUILD_MODNAME) +int __ulpi_register_driver(struct ulpi_driver *drv, struct module *module, const char *mod_name); void ulpi_unregister_driver(struct ulpi_driver *drv); #define module_ulpi_driver(__ulpi_driver) \ diff --git a/include/linux/unicode.h b/include/linux/unicode.h index 64fa44fe180c..3a8cb603eb93 100644 --- a/include/linux/unicode.h +++ b/include/linux/unicode.h @@ -53,6 +53,7 @@ struct unicode_map { unsigned int version; const struct utf8data *ntab[UTF8_NMAX]; const struct utf8data_table *tables; + struct rcu_head rcu; }; int utf8_validate(const struct unicode_map *um, const struct qstr *str); diff --git a/include/linux/usb.h b/include/linux/usb.h index 49ab8dbb885f..825f03b29067 100644 --- a/include/linux/usb.h +++ b/include/linux/usb.h @@ -1369,8 +1369,10 @@ extern void usb_deregister(struct usb_driver *); module_driver(__usb_driver, usb_register, \ usb_deregister) -extern int usb_register_device_driver(struct usb_device_driver *, - struct module *); +#define usb_register_device_driver(new_udriver) \ + __usb_register_device_driver(new_udriver, THIS_MODULE, KBUILD_MODNAME) +extern int __usb_register_device_driver(struct usb_device_driver *, + struct module *, const char *); extern void usb_deregister_device_driver(struct usb_device_driver *); extern int usb_register_dev(struct usb_interface *intf, @@ -1606,7 +1608,12 @@ typedef void (*usb_complete_t)(struct urb *); * * Completion Callbacks: * - * The completion callback is made in_interrupt(), and one of the first + * The completion callback can occur in any context: process, interrupt, + * BH, or anything else. Also, interrupts may or may not be disabled on + * the local CPU when the callback runs. A completion handler routine + * must not depend on any particular properties of its environment. + * + * One of the first * things that a completion handler should do is check the status field. * The status field is provided for all URBs. It is used to report * unlinked URBs, and status for all non-ISO transfers. It should not diff --git a/include/linux/usb/rndis_host.h b/include/linux/usb/rndis_host.h index 489cfb1d00f6..3eaf57a9b282 100644 --- a/include/linux/usb/rndis_host.h +++ b/include/linux/usb/rndis_host.h @@ -182,8 +182,9 @@ struct rndis_keepalive_c { /* IN (optionally OUT) */ #define FLAG_RNDIS_PHYM_WIRELESS 0x0002 /* Flags for driver_info::data */ -#define RNDIS_DRIVER_DATA_POLL_STATUS 1 /* poll status before control */ -#define RNDIS_DRIVER_DATA_DST_MAC_FIXUP 2 /* device ignores configured MAC address */ +#define RNDIS_DRIVER_DATA_POLL_STATUS 1 /* poll status before control */ +#define RNDIS_DRIVER_DATA_DST_MAC_FIXUP 2 /* device ignores configured MAC address */ +#define RNDIS_DRIVER_DATA_RX_AGGREGATION 4 /* needs RX limit > 2048 */ extern void rndis_status(struct usbnet *dev, struct urb *urb); extern int diff --git a/include/linux/usb/serial.h b/include/linux/usb/serial.h index ab0a32a90629..5866f19d3c7b 100644 --- a/include/linux/usb/serial.h +++ b/include/linux/usb/serial.h @@ -305,9 +305,17 @@ struct usb_serial_driver { void (*write_bulk_callback)(struct urb *urb); /* Called by the generic read bulk callback */ void (*process_read_urb)(struct urb *urb); + /* + * Called with port->lock held before preparing another write URB. + * Drivers returning false must restart the generic write path when + * writes may proceed again. + */ + bool (*write_ready)(struct usb_serial_port *port); /* Called by the generic write implementation */ int (*prepare_write_buffer)(struct usb_serial_port *port, void *dest, size_t size); + /* Called with port->lock held to roll back a failed write submission */ + void (*write_rollback)(struct usb_serial_port *port, int count); }; #define to_usb_serial_driver(d) \ container_of(d, struct usb_serial_driver, driver) diff --git a/include/linux/usb/typec_altmode.h b/include/linux/usb/typec_altmode.h index ef21ead551be..92309b569c3a 100644 --- a/include/linux/usb/typec_altmode.h +++ b/include/linux/usb/typec_altmode.h @@ -227,9 +227,9 @@ struct typec_altmode_driver { * handle all SVID specific communication. */ #define typec_altmode_register_driver(drv) \ - __typec_altmode_register_driver(drv, THIS_MODULE) + __typec_altmode_register_driver(drv, THIS_MODULE, KBUILD_MODNAME) int __typec_altmode_register_driver(struct typec_altmode_driver *drv, - struct module *module); + struct module *module, const char *mod_name); /** * typec_altmode_unregister_driver - unregisters a USB Type-C alternate mode * device driver diff --git a/include/linux/user_namespace.h b/include/linux/user_namespace.h index e38d9e60569f..91232053775d 100644 --- a/include/linux/user_namespace.h +++ b/include/linux/user_namespace.h @@ -29,8 +29,8 @@ struct uid_gid_map { /* 64 bytes -- 1 cache line */ u32 nr_extents; }; struct { - struct uid_gid_extent *forward; - struct uid_gid_extent *reverse; + struct uid_gid_extent *forward __counted_by_ptr(nr_extents); + struct uid_gid_extent *reverse __counted_by_ptr(nr_extents); }; }; }; @@ -207,6 +207,13 @@ extern bool in_userns(const struct user_namespace *ancestor, const struct user_namespace *child); extern bool current_in_userns(const struct user_namespace *target_ns); struct ns_common *ns_get_owner(struct ns_common *ns); + +#if IS_ENABLED(CONFIG_KUNIT) +extern int uid_gid_map_insert_extent(struct uid_gid_map *map, + struct uid_gid_extent *extent); +extern int uid_gid_map_sort(struct uid_gid_map *map); +#endif /* CONFIG_KUNIT */ + #else static inline struct user_namespace *get_user_ns(struct user_namespace *ns) diff --git a/include/linux/userfaultfd_k.h b/include/linux/userfaultfd_k.h index a4351cffc60c..a14b8a9ffb7b 100644 --- a/include/linux/userfaultfd_k.h +++ b/include/linux/userfaultfd_k.h @@ -18,7 +18,6 @@ #include <linux/swap.h> #include <linux/leafops.h> #include <asm-generic/pgtable_uffd.h> -#include <linux/hugetlb_inline.h> /* The set of all possible UFFD-related VM flags. */ #define __VM_UFFD_FLAGS (VM_UFFD_MISSING | VM_UFFD_MINOR | \ diff --git a/include/linux/vdso_time.h b/include/linux/vdso_time.h new file mode 100644 index 000000000000..6d81ee3a3c9c --- /dev/null +++ b/include/linux/vdso_time.h @@ -0,0 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LINUX_VDSO_TIME_H +#define _LINUX_VDSO_TIME_H + +unsigned long vdso_time_update_begin(void); +void vdso_time_update_end(unsigned long flags); + +#endif /* _LINUX_VDSO_TIME_H */ diff --git a/include/linux/vfio.h b/include/linux/vfio.h index 45f08986359e..0cc91c6f96d2 100644 --- a/include/linux/vfio.h +++ b/include/linux/vfio.h @@ -54,7 +54,7 @@ struct vfio_device { struct list_head dev_set_list; unsigned int migration_flags; u8 precopy_info_v2; - struct kvm *kvm; + struct file *kvm; /* Members below here are private, not for driver use */ unsigned int index; @@ -66,7 +66,6 @@ struct vfio_device { unsigned int open_count; struct completion comp; struct iommufd_access *iommufd_access; - void (*put_kvm)(struct kvm *kvm); struct inode *inode; #if IS_ENABLED(CONFIG_IOMMUFD) struct iommufd_device *iommufd_device; @@ -378,7 +377,7 @@ static inline bool vfio_file_has_dev(struct file *file, struct vfio_device *devi #endif bool vfio_file_is_valid(struct file *file); bool vfio_file_enforced_coherent(struct file *file); -void vfio_file_set_kvm(struct file *file, struct kvm *kvm); +void vfio_file_set_kvm(struct file *vfio_file, struct file *kvm); #define VFIO_PIN_PAGES_MAX_ENTRIES (PAGE_SIZE/sizeof(unsigned long)) diff --git a/include/linux/vfio_pci_core.h b/include/linux/vfio_pci_core.h index 9a1674c152aa..2f3fa2787737 100644 --- a/include/linux/vfio_pci_core.h +++ b/include/linux/vfio_pci_core.h @@ -8,6 +8,7 @@ * Author: Tom Lyon, pugs@cisco.com */ +#include <linux/device.h> #include <linux/mutex.h> #include <linux/pci.h> #include <linux/vfio.h> @@ -16,6 +17,7 @@ #include <linux/types.h> #include <linux/uuid.h> #include <linux/notifier.h> +#include <linux/pm.h> #ifndef VFIO_PCI_CORE_H #define VFIO_PCI_CORE_H @@ -168,7 +170,6 @@ int vfio_pci_core_init_dev(struct vfio_device *core_vdev); void vfio_pci_core_release_dev(struct vfio_device *core_vdev); int vfio_pci_core_register_device(struct vfio_pci_core_device *vdev); void vfio_pci_core_unregister_device(struct vfio_pci_core_device *vdev); -extern const struct pci_error_handlers vfio_pci_core_err_handlers; int vfio_pci_core_sriov_configure(struct vfio_pci_core_device *vdev, int nr_virtfn); long vfio_pci_core_ioctl(struct vfio_device *core_vdev, unsigned int cmd, @@ -193,8 +194,6 @@ int vfio_pci_core_match_token_uuid(struct vfio_device *core_vdev, int vfio_pci_core_enable(struct vfio_pci_core_device *vdev); void vfio_pci_core_disable(struct vfio_pci_core_device *vdev); void vfio_pci_core_finish_enable(struct vfio_pci_core_device *vdev); -pci_ers_result_t vfio_pci_core_aer_err_detected(struct pci_dev *pdev, - pci_channel_state_t state); ssize_t vfio_pci_core_do_io_rw(struct vfio_pci_core_device *vdev, bool test_mem, void __iomem *io, char __user *buf, loff_t off, size_t count, size_t x_start, @@ -260,4 +259,40 @@ vfio_pci_core_get_iomap(struct vfio_pci_core_device *vdev, unsigned int bar) int vfio_pci_dma_buf_iommufd_map(struct dma_buf_attachment *attachment, struct phys_vec *phys); +int vfio_pci_core_runtime_suspend(struct vfio_pci_core_device *vdev); +int vfio_pci_core_runtime_resume(struct vfio_pci_core_device *vdev); +pci_ers_result_t vfio_pci_core_aer_err_detected(struct vfio_pci_core_device *vdev); + +/* + * Per-driver PM/AER trampolines. driver_data points to the driver's private + * structure, which embeds a struct vfio_pci_core_device. + * + * @name: prefix for <name>_pm_ops and <name>_aer_err_detected + * @drv_type: type of the driver's private structure, including struct + * @member: member containing the struct vfio_pci_core_device + */ +#define VFIO_PCI_CORE_DEFINE_CALLBACKS(name, drv_type, member) \ +static int __maybe_unused name##_pm_suspend(struct device *dev) \ +{ \ + drv_type *drvdata = dev_get_drvdata(dev); \ + \ + return vfio_pci_core_runtime_suspend(&drvdata->member); \ +} \ +static int __maybe_unused name##_pm_resume(struct device *dev) \ +{ \ + drv_type *drvdata = dev_get_drvdata(dev); \ + \ + return vfio_pci_core_runtime_resume(&drvdata->member); \ +} \ +static const struct dev_pm_ops name##_pm_ops = { \ + SET_RUNTIME_PM_OPS(name##_pm_suspend, name##_pm_resume, NULL) \ +}; \ +static pci_ers_result_t name##_aer_err_detected(struct pci_dev *pdev, \ + pci_channel_state_t state) \ +{ \ + drv_type *drvdata = pci_get_drvdata(pdev); \ + \ + return vfio_pci_core_aer_err_detected(&drvdata->member); \ +} + #endif /* VFIO_PCI_CORE_H */ diff --git a/include/linux/vga_switcheroo.h b/include/linux/vga_switcheroo.h index 7e6ac0114d55..d7eee65664ab 100644 --- a/include/linux/vga_switcheroo.h +++ b/include/linux/vga_switcheroo.h @@ -31,8 +31,12 @@ #ifndef _LINUX_VGA_SWITCHEROO_H_ #define _LINUX_VGA_SWITCHEROO_H_ -#include <linux/fb.h> +#include <linux/errno.h> +#include <linux/types.h> +struct device; +struct dev_pm_domain; +struct fb_info; struct pci_dev; /** @@ -127,23 +131,28 @@ struct vga_switcheroo_handler { * @set_gpu_state: do the equivalent of suspend/resume for the card. * Mandatory. This should not cut power to the discrete GPU, * which is the job of the handler - * @reprobe: poll outputs. - * Optional. This gets called after waking the GPU and switching - * the outputs to it * @can_switch: check if the device is in a position to switch now. * Mandatory. The client should return false if a user space process * has one of its device files open + * @pre_switch: prepare switch + * Optional. This gets called before switching the outputs to the + * GPU. Allows drivers to prepare for the switch. + * @post_switch: completes switch + * Optional. This gets called after waking the GPU and switching + * the outputs to it. Allows drivers to poll the switched outputs. * @gpu_bound: notify the client id to audio client when the GPU is bound. * * Client callbacks. A client can be either a GPU or an audio device on a GPU. - * The @set_gpu_state and @can_switch methods are mandatory, @reprobe may be - * set to NULL. For audio clients, the @reprobe member is bogus. - * OTOH, @gpu_bound is only for audio clients, and not used for GPU clients. + * The @set_gpu_state and @can_switch methods are mandatory, @pre_switch and + * @post_switch may be set to NULL. For audio clients, the @pre_switch and + * @post_switch members are bogus. OTOH, @gpu_bound is only for audio clients, + * and not used for GPU clients. */ struct vga_switcheroo_client_ops { void (*set_gpu_state)(struct pci_dev *dev, enum vga_switcheroo_state); - void (*reprobe)(struct pci_dev *dev); bool (*can_switch)(struct pci_dev *dev); + void (*pre_switch)(struct pci_dev *dev); + void (*post_switch)(struct pci_dev *dev); void (*gpu_bound)(struct pci_dev *dev, enum vga_switcheroo_client_id); }; @@ -156,9 +165,6 @@ int vga_switcheroo_register_audio_client(struct pci_dev *pdev, const struct vga_switcheroo_client_ops *ops, struct pci_dev *vga_dev); -void vga_switcheroo_client_fb_set(struct pci_dev *dev, - struct fb_info *info); - int vga_switcheroo_register_handler(const struct vga_switcheroo_handler *handler, enum vga_switcheroo_handler_flags_t handler_flags); void vga_switcheroo_unregister_handler(void); @@ -178,7 +184,6 @@ void vga_switcheroo_fini_domain_pm_ops(struct device *dev); static inline void vga_switcheroo_unregister_client(struct pci_dev *dev) {} static inline int vga_switcheroo_register_client(struct pci_dev *dev, const struct vga_switcheroo_client_ops *ops, bool driver_power_control) { return 0; } -static inline void vga_switcheroo_client_fb_set(struct pci_dev *dev, struct fb_info *info) {} static inline int vga_switcheroo_register_handler(const struct vga_switcheroo_handler *handler, enum vga_switcheroo_handler_flags_t handler_flags) { return 0; } static inline int vga_switcheroo_register_audio_client(struct pci_dev *pdev, diff --git a/include/linux/vgaarb.h b/include/linux/vgaarb.h index 97129a1bbb7d..71a364669eaf 100644 --- a/include/linux/vgaarb.h +++ b/include/linux/vgaarb.h @@ -33,7 +33,8 @@ struct pci_dev *vga_default_device(void); void vga_set_default_device(struct pci_dev *pdev); int vga_remove_vgacon(struct pci_dev *pdev); int vga_client_register(struct pci_dev *pdev, - unsigned int (*set_decode)(struct pci_dev *pdev, bool state)); + unsigned int (*set_decode)(void *data, bool state), + void *data); #else /* CONFIG_VGA_ARB */ static inline void vga_set_legacy_decoding(struct pci_dev *pdev, unsigned int decodes) @@ -59,7 +60,8 @@ static inline int vga_remove_vgacon(struct pci_dev *pdev) return 0; } static inline int vga_client_register(struct pci_dev *pdev, - unsigned int (*set_decode)(struct pci_dev *pdev, bool state)) + unsigned int (*set_decode)(void *data, bool state), + void *data) { return 0; } @@ -97,7 +99,7 @@ static inline int vga_get_uninterruptible(struct pci_dev *pdev, static inline void vga_client_unregister(struct pci_dev *pdev) { - vga_client_register(pdev, NULL); + vga_client_register(pdev, NULL, NULL); } #endif /* LINUX_VGA_H */ diff --git a/include/linux/vmalloc.h b/include/linux/vmalloc.h index aed121d729b0..034a693777ca 100644 --- a/include/linux/vmalloc.h +++ b/include/linux/vmalloc.h @@ -3,6 +3,7 @@ #define _LINUX_VMALLOC_H #include <linux/alloc_tag.h> +#include <linux/cleanup.h> #include <linux/sched.h> #include <linux/spinlock.h> #include <linux/init.h> @@ -38,6 +39,7 @@ struct iov_iter; /* in uio.h */ #define VM_DEFER_KMEMLEAK 0 #endif #define VM_SPARSE 0x00001000 /* sparse vm_area. not all pages are present. */ +#define VM_REQUIRE_HUGE_VMAP 0x00002000 /* huge page mapping or nothing */ /* bits [20..32] reserved for arch specific ioremap internals */ @@ -214,6 +216,8 @@ void *__must_check vrealloc_node_align_noprof(const void *p, size_t size, extern void vfree(const void *addr); extern void vfree_atomic(const void *addr); +DEFINE_FREE(vfree, void *, if (!IS_ERR_OR_NULL(_T)) vfree(_T)) + extern void *vmap(struct page **pages, unsigned int count, unsigned long flags, pgprot_t prot); void *vmap_pfn(unsigned long *pfns, unsigned int count, pgprot_t prot); diff --git a/include/linux/vmemmap-optimization.h b/include/linux/vmemmap-optimization.h new file mode 100644 index 000000000000..fa9e9abd6656 --- /dev/null +++ b/include/linux/vmemmap-optimization.h @@ -0,0 +1,115 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * vmemmap-optimization.h + * + * Generic vmemmap optimization declarations. + * + * Author: Muchun Song <songmuchun@bytedance.com> + */ +#ifndef _LINUX_VMEMMAP_OPTIMIZATION_H +#define _LINUX_VMEMMAP_OPTIMIZATION_H + +#include <linux/align.h> +#include <linux/log2.h> +#include <linux/mmdebug.h> +#include <linux/mmzone.h> + +/* + * HugeTLB Vmemmap Optimization (HVO) requires struct pages of the head page to + * be naturally aligned with regard to the folio size. + * + * HVO which is only active if the size of struct page is a power of 2. + */ +#define MAX_FOLIO_VMEMMAP_ALIGN \ + (IS_ENABLED(CONFIG_VMEMMAP_OPTIMIZATION) && \ + is_power_of_2(sizeof(struct page)) ? \ + MAX_FOLIO_NR_PAGES * sizeof(struct page) : 0) + +/* The number of retained vmemmap pages with HVO enabled. */ +#define VMEMMAP_OPTIMIZATION_PAGES 1 +#define VMEMMAP_OPTIMIZATION_NR_STRUCT_PAGES \ + (VMEMMAP_OPTIMIZATION_PAGES * PAGE_SIZE / sizeof(struct page)) +#define VMEMMAP_OPTIMIZATION_MIN_ORDER (ilog2(VMEMMAP_OPTIMIZATION_NR_STRUCT_PAGES) + 1) + +#ifdef CONFIG_VMEMMAP_OPTIMIZATION +static inline unsigned int section_compound_order(const struct mem_section *section) +{ + return section->compound_page_order; +} + +static inline void section_set_compound_order(struct mem_section *section, + unsigned int order) +{ + VM_WARN_ON(section_compound_order(section) && order && + section_compound_order(section) != order); + section->compound_page_order = order; +} + +static inline void section_set_compound_order_range(unsigned long pfn, + unsigned long nr_pages, unsigned int order) +{ + unsigned long section_nr = pfn_to_section_nr(pfn); + + if (!IS_ALIGNED(pfn | nr_pages, PAGES_PER_SECTION)) + return; + + for (unsigned long i = 0; i < nr_pages / PAGES_PER_SECTION; i++) + section_set_compound_order(__nr_to_section(section_nr + i), order); +} + +static inline unsigned int pfn_to_section_compound_order(unsigned long pfn) +{ + return section_compound_order(__pfn_to_section(pfn)); +} + +struct page *vmemmap_shared_tail_page(unsigned int order, struct zone *zone); +#else +static inline unsigned int section_compound_order(const struct mem_section *section) +{ + return 0; +} + +static inline void section_set_compound_order(struct mem_section *section, + unsigned int order) +{ +} + +static inline void section_set_compound_order_range(unsigned long pfn, + unsigned long nr_pages, unsigned int order) +{ +} + +static inline unsigned int pfn_to_section_compound_order(unsigned long pfn) +{ + return 0; +} + +static inline struct page *vmemmap_shared_tail_page(unsigned int order, + struct zone *zone) +{ + return NULL; +} +#endif /* CONFIG_VMEMMAP_OPTIMIZATION */ + +static inline bool vmemmap_optimizable_pfn(unsigned long pfn) +{ + const unsigned int order = pfn_to_section_compound_order(pfn); + const unsigned long nr_pages = 1UL << order; + + if (!is_power_of_2(sizeof(struct page))) + return false; + + return (pfn & (nr_pages - 1)) >= VMEMMAP_OPTIMIZATION_NR_STRUCT_PAGES; +} + +static inline bool vmemmap_optimizable_order(unsigned int order) +{ + if (!IS_ENABLED(CONFIG_VMEMMAP_OPTIMIZATION)) + return false; + + if (!is_power_of_2(sizeof(struct page))) + return false; + + return order >= VMEMMAP_OPTIMIZATION_MIN_ORDER; +} +#endif /* _LINUX_VMEMMAP_OPTIMIZATION_H */ diff --git a/include/linux/wait.h b/include/linux/wait.h index 7e215330199c..c2af98b0074d 100644 --- a/include/linux/wait.h +++ b/include/linux/wait.h @@ -556,7 +556,7 @@ do { \ } \ \ __ret = ___wait_event(wq_head, condition, state, 0, 0, \ - if (!__t.task) { \ + if (!hrtimer_sleeper_task_get(&__t)) { \ __ret = -ETIME; \ break; \ } \ diff --git a/include/linux/wait_bit.h b/include/linux/wait_bit.h index 553d7b23e3ad..af077ed4caf6 100644 --- a/include/linux/wait_bit.h +++ b/include/linux/wait_bit.h @@ -433,6 +433,32 @@ do { \ }) /** + * wait_var_event_state - wait for a variable to be updated and notified + * @var: the address of variable being waited on + * @condition: the condition to wait for + * @state: the task state to sleep in, %TASK_UNINTERRUPTIBLE etc. + * + * Wait for a @condition to be true, only re-checking when a wake up is + * received for the given @var (an arbitrary kernel address which need + * not be directly related to the given condition, but usually is). + * + * Returns 0 if the condition became true, or %-ERESTARTSYS if a signal + * arrived which @state allows to interrupt. + * + * The condition should normally use smp_load_acquire() or a similarly + * ordered access to ensure that any changes to memory made before the + * condition became true will be visible after the wait completes. + */ +#define wait_var_event_state(var, condition, state) \ +({ \ + int __ret = 0; \ + might_sleep(); \ + if (!(condition)) \ + __ret = ___wait_var_event(var, condition, (state), 0, 0, schedule()); \ + __ret; \ +}) + +/** * wait_var_event_any_lock - wait for a variable to be updated under a lock * @var: the address of the variable being waited on * @condition: condition to wait for diff --git a/include/linux/watch_queue.h b/include/linux/watch_queue.h index 429c7b6afead..b87e1babb5f0 100644 --- a/include/linux/watch_queue.h +++ b/include/linux/watch_queue.h @@ -52,10 +52,7 @@ struct watch_queue { * Representation of a watch on an object. */ struct watch { - union { - struct rcu_head rcu; - u32 info_id; /* ID to be OR'd in to info field */ - }; + struct rcu_head rcu; struct watch_queue __rcu *queue; /* Queue to post events to */ struct hlist_node queue_node; /* Link in queue->watches */ struct watch_list __rcu *watch_list; @@ -64,6 +61,7 @@ struct watch { void *private; /* Private data for the watched object */ u64 id; /* Internal identifier */ struct kref usage; /* Object usage count */ + u32 info_id; /* ID to be OR'd in to info field */ }; /* diff --git a/include/linux/wmi.h b/include/linux/wmi.h index defcb624a7e2..3dd223f2ff18 100644 --- a/include/linux/wmi.h +++ b/include/linux/wmi.h @@ -13,6 +13,7 @@ #include <linux/acpi.h> #include <linux/device-id/wmi.h> #include <linux/types.h> +#include <linux/uuid.h> /** * struct wmi_device - WMI device structure @@ -83,6 +84,8 @@ acpi_status wmidev_block_set(struct wmi_device *wdev, u8 instance, const struct u8 wmidev_instance_count(struct wmi_device *wdev); +bool wmidev_exists(const guid_t *guid); + /** * struct wmi_driver - WMI driver structure * @driver: Driver model structure diff --git a/include/linux/wwan.h b/include/linux/wwan.h index 1e0e2cb53579..23a8a8ee473d 100644 --- a/include/linux/wwan.h +++ b/include/linux/wwan.h @@ -57,6 +57,14 @@ struct wwan_port; * @tx_blocking: Optional blocking routine that sends WWAN port protocol data * to the device. * @tx_poll: Optional routine that sets additional TX poll flags. + * @dtr_rts: Optional routine that updates the modem control lines to match + * @mdmbits, the TIOCM_* output bits (DTR, RTS, OUT1, OUT2, LOOP) + * currently set for the port. Called when the port is first opened + * and ->start() has succeeded, on last close if HUPCL is set, when + * the port is removed while still open, and when TIOCMSET, TIOCMBIS + * or TIOCMBIC changes DTR or RTS. Only called for WWAN_PORT_AT ports, + * as only the AT (DUN) port carries the modem's virtual RS-232 + * control signals. * * The wwan_port_ops structure contains a list of low-level operations * that control a WWAN port device. All functions are mandatory unless specified. @@ -70,6 +78,7 @@ struct wwan_port_ops { int (*tx_blocking)(struct wwan_port *port, struct sk_buff *skb); __poll_t (*tx_poll)(struct wwan_port *port, struct file *filp, poll_table *wait); + void (*dtr_rts)(struct wwan_port *port, unsigned int mdmbits); }; /** struct wwan_port_caps - The WWAN port capbilities diff --git a/include/linux/xattr.h b/include/linux/xattr.h index 54ac3cbc133f..4cc4257de084 100644 --- a/include/linux/xattr.h +++ b/include/linux/xattr.h @@ -47,7 +47,7 @@ struct xattr_handler { struct inode *inode, const char *name, void *buffer, size_t size); int (*set)(const struct xattr_handler *, - struct mnt_idmap *idmap, struct dentry *dentry, + const struct mnt_idmap *idmap, struct dentry *dentry, struct inode *inode, const char *name, const void *buffer, size_t size, int flags); }; @@ -77,25 +77,25 @@ struct xattr { }; ssize_t __vfs_getxattr(struct dentry *, struct inode *, const char *, void *, size_t); -ssize_t vfs_getxattr(struct mnt_idmap *, struct dentry *, const char *, +ssize_t vfs_getxattr(const struct mnt_idmap *, struct dentry *, const char *, void *, size_t); ssize_t vfs_listxattr(struct dentry *d, char *list, size_t size); -int __vfs_setxattr(struct mnt_idmap *, struct dentry *, struct inode *, +int __vfs_setxattr(const struct mnt_idmap *, struct dentry *, struct inode *, const char *, const void *, size_t, int); -int __vfs_setxattr_noperm(struct mnt_idmap *, struct dentry *, +int __vfs_setxattr_noperm(const struct mnt_idmap *, struct dentry *, const char *, const void *, size_t, int); -int __vfs_setxattr_locked(struct mnt_idmap *, struct dentry *, +int __vfs_setxattr_locked(const struct mnt_idmap *, struct dentry *, const char *, const void *, size_t, int, struct delegated_inode *); -int vfs_setxattr(struct mnt_idmap *, struct dentry *, const char *, +int vfs_setxattr(const struct mnt_idmap *, struct dentry *, const char *, const void *, size_t, int); -int __vfs_removexattr(struct mnt_idmap *, struct dentry *, const char *); -int __vfs_removexattr_locked(struct mnt_idmap *, struct dentry *, +int __vfs_removexattr(const struct mnt_idmap *, struct dentry *, const char *); +int __vfs_removexattr_locked(const struct mnt_idmap *, struct dentry *, const char *, struct delegated_inode *); -int vfs_removexattr(struct mnt_idmap *, struct dentry *, const char *); +int vfs_removexattr(const struct mnt_idmap *, struct dentry *, const char *); ssize_t generic_listxattr(struct dentry *dentry, char *buffer, size_t buffer_size); -int vfs_getxattr_alloc(struct mnt_idmap *idmap, +int vfs_getxattr_alloc(const struct mnt_idmap *idmap, struct dentry *dentry, const char *name, char **xattr_value, size_t size, gfp_t flags); diff --git a/include/linux/zsmalloc.h b/include/linux/zsmalloc.h index 478410c880b1..5b7298a5026b 100644 --- a/include/linux/zsmalloc.h +++ b/include/linux/zsmalloc.h @@ -40,10 +40,6 @@ unsigned int zs_lookup_class_index(struct zs_pool *pool, unsigned int size); void zs_pool_stats(struct zs_pool *pool, struct zs_pool_stats *stats); -void *zs_obj_read_begin(struct zs_pool *pool, unsigned long handle, - size_t mem_len, void *local_copy); -void zs_obj_read_end(struct zs_pool *pool, unsigned long handle, - size_t mem_len, void *handle_mem); void zs_obj_read_sg_begin(struct zs_pool *pool, unsigned long handle, struct scatterlist *sg, size_t mem_len); void zs_obj_read_sg_end(struct zs_pool *pool, unsigned long handle); diff --git a/include/linux/zswap.h b/include/linux/zswap.h index 30c193a1207e..df6cafbe95dc 100644 --- a/include/linux/zswap.h +++ b/include/linux/zswap.h @@ -27,7 +27,7 @@ struct zswap_lruvec_state { unsigned long zswap_total_pages(void); bool zswap_store(struct folio *folio); int zswap_load(struct folio *folio); -void zswap_invalidate(swp_entry_t swp); +void zswap_invalidate(int type, pgoff_t offset, unsigned long nr_entries); int zswap_swapon(int type, unsigned long nr_pages); void zswap_swapoff(int type); void zswap_memcg_offline_cleanup(struct mem_cgroup *memcg); @@ -49,7 +49,11 @@ static inline int zswap_load(struct folio *folio) return -ENOENT; } -static inline void zswap_invalidate(swp_entry_t swp) {} +static inline void zswap_invalidate(int type, pgoff_t offset, + unsigned long nr_entries) +{ +} + static inline int zswap_swapon(int type, unsigned long nr_pages) { return 0; |
