diff options
Diffstat (limited to 'include/linux')
892 files changed, 29562 insertions, 15650 deletions
diff --git a/include/linux/acpi.h b/include/linux/acpi.h index 4d2f0bed7a06..ddacac812094 100644 --- a/include/linux/acpi.h +++ b/include/linux/acpi.h @@ -13,7 +13,7 @@ #include <linux/ioport.h> /* for struct resource */ #include <linux/resource_ext.h> #include <linux/device.h> -#include <linux/mod_devicetable.h> +#include <linux/device-id/acpi.h> #include <linux/property.h> #include <linux/uuid.h> #include <linux/node.h> @@ -94,6 +94,12 @@ static inline void acpi_preset_companion(struct device *dev, ACPI_COMPANION_SET(dev, acpi_find_child_device(parent, addr, false)); } +static inline void acpi_device_clear_deps(struct device *dev) +{ + if (has_acpi_companion(dev)) + acpi_dev_clear_dependencies(ACPI_COMPANION(dev)); +} + static inline const char *acpi_dev_name(struct acpi_device *adev) { return dev_name(&adev->dev); @@ -315,6 +321,19 @@ static inline int acpi_processor_evaluate_cst(acpi_handle handle, u32 cpu, } #endif +#ifdef CONFIG_ACPI_PROCESSOR_IDLE +int acpi_processor_extract_lpi_info(acpi_handle pr_handle, + struct acpi_processor_power *pr_power, + bool strict); +#else +static inline int acpi_processor_extract_lpi_info(acpi_handle pr_handle, + struct acpi_processor_power *pr_power, + bool strict) +{ + return -ENODEV; +} +#endif + #ifdef CONFIG_ACPI_HOTPLUG_CPU /* Arch dependent functions for cpu hotplug support */ int acpi_map_cpu(acpi_handle handle, phys_cpuid_t physid, u32 acpi_id, @@ -324,6 +343,17 @@ int acpi_unmap_cpu(int cpu); acpi_handle acpi_get_processor_handle(int cpu); +/** + * acpi_get_cpu_uid() - Get ACPI Processor UID of from MADT table + * @cpu: Logical CPU number (0-based) + * @uid: Pointer to store ACPI Processor UID + * + * Return: 0 on success (ACPI Processor ID stored in *uid); + * -EINVAL if CPU number is invalid or out of range; + * -ENODEV if ACPI Processor UID for the CPU is not found. + */ +int acpi_get_cpu_uid(unsigned int cpu, u32 *uid); + #ifdef CONFIG_ACPI_HOTPLUG_IOAPIC int acpi_get_ioapic_id(acpi_handle handle, u32 gsi_base, u64 *phys_addr); #endif @@ -349,9 +379,10 @@ int acpi_gsi_to_irq (u32 gsi, unsigned int *irq); int acpi_isa_irq_to_gsi (unsigned isa_irq, u32 *gsi); typedef struct fwnode_handle *(*acpi_gsi_domain_disp_fn)(u32); +typedef acpi_handle (*acpi_gsi_handle_disp_fn)(u32); void acpi_set_irq_model(enum acpi_irq_model_id model, - acpi_gsi_domain_disp_fn fn); + acpi_gsi_domain_disp_fn fn, acpi_gsi_handle_disp_fn gsi_dep_fn); acpi_gsi_domain_disp_fn acpi_get_gsi_dispatcher(void); void acpi_set_gsi_to_irq_fallback(u32 (*)(u32)); @@ -361,6 +392,8 @@ struct irq_domain *acpi_irq_create_hierarchy(unsigned int flags, const struct irq_domain_ops *ops, void *host_data); +u32 acpi_irq_add_auto_dep(acpi_handle handle); + #ifdef CONFIG_X86_IO_APIC extern int acpi_get_override_irq(u32 gsi, int *trigger, int *polarity); #else @@ -437,6 +470,7 @@ extern char *wmi_get_acpi_device_uid(const char *guid); #define ACPI_VIDEO_OUTPUT_SWITCHING_DMI_VIDEO 0x0800 extern char acpi_video_backlight_string[]; +extern bool acpi_dev_is_video_device(struct acpi_device *adev); extern long acpi_is_video_device(acpi_handle handle); extern void acpi_osi_setup(char *str); @@ -787,10 +821,20 @@ int acpi_get_local_u64_address(acpi_handle handle, u64 *addr); int acpi_get_local_address(acpi_handle handle, u32 *addr); const char *acpi_get_subsystem_id(acpi_handle handle); +struct device *acpi_bus_find_device_by_name(const char *name); + #ifdef CONFIG_ACPI_MRRM int acpi_mrrm_max_mem_region(void); #endif +#define ACPI_CMOS_RTC_IDS \ + { "PNP0B00", }, \ + { "PNP0B01", }, \ + { "PNP0B02", }, \ + { "", } + +extern bool cmos_rtc_platform_device_present; + #else /* !CONFIG_ACPI */ #define acpi_disabled 1 @@ -887,6 +931,8 @@ static inline void acpi_preset_companion(struct device *dev, { } +static inline void acpi_device_clear_deps(struct device *dev) {} + static inline const char *acpi_dev_name(struct acpi_device *adev) { return NULL; @@ -940,6 +986,12 @@ static inline int acpi_table_parse(char *id, return -ENODEV; } +static inline int acpi_get_cpu_uid(unsigned int cpu, u32 *uid) +{ + *uid = cpu; + return 0; +} + static inline int acpi_nvs_register(__u64 start, __u64 size) { return 0; @@ -1081,6 +1133,11 @@ static inline const char *acpi_get_subsystem_id(acpi_handle handle) return ERR_PTR(-ENODEV); } +static inline struct device *acpi_bus_find_device_by_name(const char *name) +{ + return NULL; +} + static inline int acpi_register_wakeup_handler(int wake_irq, bool (*wakeup)(void *context), void *context) { @@ -1116,6 +1173,8 @@ static inline int acpi_mrrm_max_mem_region(void) return 1; } +#define cmos_rtc_platform_device_present false + #endif /* !CONFIG_ACPI */ #ifdef CONFIG_ACPI_HMAT diff --git a/include/linux/acpi_iort.h b/include/linux/acpi_iort.h index 17bb3374f4ca..931eaa7bbf6a 100644 --- a/include/linux/acpi_iort.h +++ b/include/linux/acpi_iort.h @@ -27,7 +27,8 @@ int iort_register_domain_token(int trans_id, phys_addr_t base, struct fwnode_handle *fw_node); void iort_deregister_domain_token(int trans_id); struct fwnode_handle *iort_find_domain_token(int trans_id); -struct fwnode_handle *iort_iwb_handle(u32 iwb_id); +acpi_handle iort_iwb_handle(u32 iwb_id); +struct fwnode_handle *iort_iwb_handle_fwnode(u32 iwb_id); #ifdef CONFIG_ACPI_IORT u32 iort_msi_map_id(struct device *dev, u32 id); diff --git a/include/linux/acpi_pmtmr.h b/include/linux/acpi_pmtmr.h index 0ded9220d379..cceed294d0b5 100644 --- a/include/linux/acpi_pmtmr.h +++ b/include/linux/acpi_pmtmr.h @@ -27,15 +27,17 @@ static inline u32 acpi_pm_read_early(void) } /** - * Register callback for suspend and resume event + * acpi_pmtmr_register_suspend_resume_callback - Register callback for + * suspend and resume event * - * @cb Callback triggered on suspend and resume - * @data Data passed with the callback + * @cb: Callback triggered on suspend and resume + * @data: Data passed with the callback */ void acpi_pmtmr_register_suspend_resume_callback(void (*cb)(void *data, bool suspend), void *data); /** - * Remove registered callback for suspend and resume event + * acpi_pmtmr_unregister_suspend_resume_callback - Remove registered callback + * for suspend and resume event */ void acpi_pmtmr_unregister_suspend_resume_callback(void); diff --git a/include/linux/adi-axi-common.h b/include/linux/adi-axi-common.h index 37962ba530df..e7ba393061ee 100644 --- a/include/linux/adi-axi-common.h +++ b/include/linux/adi-axi-common.h @@ -51,6 +51,7 @@ enum adi_axi_fpga_technology { ADI_AXI_FPGA_TECH_SERIES7, ADI_AXI_FPGA_TECH_ULTRASCALE, ADI_AXI_FPGA_TECH_ULTRASCALE_PLUS, + ADI_AXI_FPGA_TECH_VERSAL, }; enum adi_axi_fpga_family { @@ -71,6 +72,7 @@ enum adi_axi_fpga_speed_grade { ADI_AXI_FPGA_SPEED_2 = 20, ADI_AXI_FPGA_SPEED_2L = 21, ADI_AXI_FPGA_SPEED_2LV = 22, + ADI_AXI_FPGA_SPEED_2MP = 23, ADI_AXI_FPGA_SPEED_3 = 30, }; diff --git a/include/linux/alarmtimer.h b/include/linux/alarmtimer.h index 3ffa5341dce2..2014288ca2f4 100644 --- a/include/linux/alarmtimer.h +++ b/include/linux/alarmtimer.h @@ -42,11 +42,14 @@ struct alarm { void *data; }; +static __always_inline ktime_t alarm_get_expires(struct alarm *alarm) +{ + return alarm->node.expires; +} + void alarm_init(struct alarm *alarm, enum alarmtimer_type type, void (*function)(struct alarm *, ktime_t)); -void alarm_start(struct alarm *alarm, ktime_t start); -void alarm_start_relative(struct alarm *alarm, ktime_t start); -void alarm_restart(struct alarm *alarm); +bool alarm_start_timer(struct alarm *alarm, ktime_t expires, bool relative); int alarm_try_to_cancel(struct alarm *alarm); int alarm_cancel(struct alarm *alarm); diff --git a/include/linux/alloc_tag.h b/include/linux/alloc_tag.h index d40ac39bfbe8..7f2d80a59792 100644 --- a/include/linux/alloc_tag.h +++ b/include/linux/alloc_tag.h @@ -134,6 +134,8 @@ static inline bool mem_alloc_profiling_enabled(void) &mem_alloc_profiling_key); } +bool mem_alloc_profiling_permanently_disabled(void); + static inline struct alloc_tag_counters alloc_tag_read(struct alloc_tag *tag) { struct alloc_tag_counters v = { 0, 0 }; @@ -163,9 +165,11 @@ static inline void alloc_tag_sub_check(union codetag_ref *ref) { WARN_ONCE(ref && !ref->ct, "alloc_tag was not set\n"); } +void alloc_tag_add_early_pfn(unsigned long pfn, unsigned int alloc_flags); #else static inline void alloc_tag_add_check(union codetag_ref *ref, struct alloc_tag *tag) {} static inline void alloc_tag_sub_check(union codetag_ref *ref) {} +static inline void alloc_tag_add_early_pfn(unsigned long pfn, unsigned int alloc_flags) {} #endif /* Caller should verify both ref and tag to be valid */ @@ -237,6 +241,7 @@ static inline bool alloc_tag_is_inaccurate(struct alloc_tag *tag) #define DEFINE_ALLOC_TAG(_alloc_tag) static inline bool mem_alloc_profiling_enabled(void) { return false; } +static inline bool mem_alloc_profiling_permanently_disabled(void) { return true; } static inline void alloc_tag_add(union codetag_ref *ref, struct alloc_tag *tag, size_t bytes) {} static inline void alloc_tag_sub(union codetag_ref *ref, size_t bytes) {} diff --git a/include/linux/amba/bus.h b/include/linux/amba/bus.h index 9946276aff73..80a74cd2da7e 100644 --- a/include/linux/amba/bus.h +++ b/include/linux/amba/bus.h @@ -13,7 +13,7 @@ #include <linux/clk.h> #include <linux/device.h> -#include <linux/mod_devicetable.h> +#include <linux/device-id/amba.h> #include <linux/err.h> #include <linux/resource.h> #include <linux/regulator/consumer.h> @@ -71,11 +71,6 @@ struct amba_device { unsigned int cid; struct amba_cs_uci_id uci; unsigned int irq[AMBA_NR_IRQS]; - /* - * Driver name to force a match. Do not set directly, because core - * frees it. Use driver_set_override() to set or clear it. - */ - const char *driver_override; }; struct amba_driver { diff --git a/include/linux/amd-pmf-io.h b/include/linux/amd-pmf-io.h index 55198d2875cc..271472458516 100644 --- a/include/linux/amd-pmf-io.h +++ b/include/linux/amd-pmf-io.h @@ -19,11 +19,13 @@ * @MT_HPD: Message ID to know the Human presence info from MP2 FW * @MT_ALS: Message ID to know the Ambient light info from MP2 FW * @MT_SRA: Message ID to know the SRA data from MP2 FW + * @MT_OP_MODE: Message ID to know the operating-mode (tablet/laptop) info */ enum sfh_message_type { MT_HPD, MT_ALS, MT_SRA, + MT_OP_MODE, }; /** @@ -44,21 +46,24 @@ enum sfh_hpd_info { * @user_present: Populates the user presence information * @platform_type: Operating modes (clamshell, flat, tent, etc.) * @laptop_placement: Device states (ontable, onlap, outbag) + * @op_mode: Operating-mode field (see enum sfh_dev_mode); used for tablet detection */ struct amd_sfh_info { u32 ambient_light; u8 user_present; u32 platform_type; u32 laptop_placement; + u32 op_mode; }; -enum laptop_placement { - LP_UNKNOWN = 0, - ON_TABLE, - ON_LAP_MOTION, - IN_BAG, - OUT_OF_BAG, - LP_UNDEFINED, +/** + * enum sfh_dev_mode - SFH operating-mode field (sfh_op_mode.mode, bits 0-2) + * @SFH_MODE_LAPTOP: Device is in laptop/clamshell posture + * @SFH_MODE_TABLET: Device is in tablet posture + */ +enum sfh_dev_mode { + SFH_MODE_LAPTOP = 1, + SFH_MODE_TABLET = 3, }; /** @@ -69,6 +74,7 @@ enum laptop_placement { * @mpnpuclk_freq: MPNPU [MHz] * @npu_reads: NPU read bandwidth [MB/sec] * @npu_writes: NPU write bandwidth [MB/sec] + * @npu_temp: NPU temperature [C] */ struct amd_pmf_npu_metrics { u16 npuclk_freq; @@ -77,6 +83,7 @@ struct amd_pmf_npu_metrics { u16 mpnpuclk_freq; u16 npu_reads; u16 npu_writes; + u16 npu_temp; }; int amd_get_sfh_info(struct amd_sfh_info *sfh_info, enum sfh_message_type op); diff --git a/include/linux/arm-smccc.h b/include/linux/arm-smccc.h index 50b47eba7d01..4de81848fe2e 100644 --- a/include/linux/arm-smccc.h +++ b/include/linux/arm-smccc.h @@ -90,6 +90,11 @@ ARM_SMCCC_SMC_32, \ 0, 2) +#define ARM_SMCCC_ARCH_SOC_ID64 \ + ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL, \ + ARM_SMCCC_SMC_64, \ + 0, 2) + #define ARM_SMCCC_ARCH_WORKAROUND_1 \ ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL, \ ARM_SMCCC_SMC_32, \ @@ -105,6 +110,12 @@ ARM_SMCCC_SMC_32, \ 0, 0x3fff) +/* C1-Pro erratum 4193714: SME DVMSync early acknowledgement */ +#define ARM_SMCCC_CPU_WORKAROUND_4193714 \ + ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL, \ + ARM_SMCCC_SMC_32, \ + ARM_SMCCC_OWNER_CPU, 0x10) + #define ARM_SMCCC_VENDOR_HYP_CALL_UID_FUNC_ID \ ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL, \ ARM_SMCCC_SMC_32, \ diff --git a/include/linux/arm_ffa.h b/include/linux/arm_ffa.h index 81e603839c4a..e71d83ee0aef 100644 --- a/include/linux/arm_ffa.h +++ b/include/linux/arm_ffa.h @@ -173,7 +173,7 @@ struct ffa_partition_info; #if IS_REACHABLE(CONFIG_ARM_FFA_TRANSPORT) struct ffa_device * ffa_device_register(const struct ffa_partition_info *part_info, - const struct ffa_ops *ops); + const struct ffa_ops *ops, struct device *parent); void ffa_device_unregister(struct ffa_device *ffa_dev); int ffa_driver_register(struct ffa_driver *driver, struct module *owner, const char *mod_name); @@ -184,7 +184,7 @@ bool ffa_device_is_valid(struct ffa_device *ffa_dev); #else static inline struct ffa_device * ffa_device_register(const struct ffa_partition_info *part_info, - const struct ffa_ops *ops) + const struct ffa_ops *ops, struct device *parent) { return NULL; } @@ -421,6 +421,13 @@ struct ffa_mem_region { #define FFA_EMAD_HAS_IMPDEF_FIELD(version) ((version) >= FFA_VERSION_1_2) #define FFA_MEM_REGION_HAS_EP_MEM_OFFSET(version) ((version) > FFA_VERSION_1_0) +/* The layout changed from FFA_VERSION_1_0 and the region includes an + * ep_mem_offset. + */ +#define FFA_MEM_REGION_SZ(version) (!FFA_MEM_REGION_HAS_EP_MEM_OFFSET((version)) ?\ + offsetof(struct ffa_mem_region, ep_mem_offset) :\ + sizeof(struct ffa_mem_region)) + static inline u32 ffa_emad_size_get(u32 ffa_version) { u32 sz; @@ -445,7 +452,7 @@ ffa_mem_desc_offset(struct ffa_mem_region *buf, int count, u32 ffa_version) if (!FFA_MEM_REGION_HAS_EP_MEM_OFFSET(ffa_version)) offset += offsetof(struct ffa_mem_region, ep_mem_offset); else - offset += sizeof(struct ffa_mem_region); + offset += buf->ep_mem_offset; return offset; } diff --git a/include/linux/arm_mpam.h b/include/linux/arm_mpam.h index 7f00c5285a32..f92a36187a52 100644 --- a/include/linux/arm_mpam.h +++ b/include/linux/arm_mpam.h @@ -5,6 +5,7 @@ #define __LINUX_ARM_MPAM_H #include <linux/acpi.h> +#include <linux/resctrl_types.h> #include <linux/types.h> struct mpam_msc; @@ -49,6 +50,37 @@ static inline int mpam_ris_create(struct mpam_msc *msc, u8 ris_idx, } #endif +bool resctrl_arch_alloc_capable(void); +bool resctrl_arch_mon_capable(void); + +void resctrl_arch_set_cpu_default_closid(int cpu, u32 closid); +void resctrl_arch_set_closid_rmid(struct task_struct *tsk, u32 closid, u32 rmid); +void resctrl_arch_set_cpu_default_closid_rmid(int cpu, u32 closid, u32 rmid); +void resctrl_arch_sched_in(struct task_struct *tsk); +bool resctrl_arch_match_closid(struct task_struct *tsk, u32 closid); +bool resctrl_arch_match_rmid(struct task_struct *tsk, u32 closid, u32 rmid); +u32 resctrl_arch_rmid_idx_encode(u32 closid, u32 rmid); +void resctrl_arch_rmid_idx_decode(u32 idx, u32 *closid, u32 *rmid); +u32 resctrl_arch_system_num_rmid_idx(void); + +struct rdt_resource; +void *resctrl_arch_mon_ctx_alloc(struct rdt_resource *r, enum resctrl_event_id evtid); +void resctrl_arch_mon_ctx_free(struct rdt_resource *r, enum resctrl_event_id evtid, void *ctx); + +/* + * The CPU configuration for MPAM is cheap to write, and is only written if it + * has changed. No need for fine grained enables. + */ +static inline void resctrl_arch_enable_mon(void) { } +static inline void resctrl_arch_disable_mon(void) { } +static inline void resctrl_arch_enable_alloc(void) { } +static inline void resctrl_arch_disable_alloc(void) { } + +static inline unsigned int resctrl_arch_round_mon_val(unsigned int val) +{ + return val; +} + /** * mpam_register_requestor() - Register a requestor with the MPAM driver * @partid_max: The maximum PARTID value the requestor can generate. diff --git a/include/linux/arm_sdei.h b/include/linux/arm_sdei.h index f652a5028b59..b9dc21c241be 100644 --- a/include/linux/arm_sdei.h +++ b/include/linux/arm_sdei.h @@ -37,6 +37,15 @@ int sdei_event_unregister(u32 event_num); int sdei_event_enable(u32 event_num); int sdei_event_disable(u32 event_num); +/* + * Signal the software-signalled event (event 0) to another PE, NMI-like. + * @mpidr is the target's MPIDR affinity. + */ +int sdei_event_signal(u32 event_num, u64 mpidr); + +/* Was SDEI firmware probed and usable? */ +bool sdei_is_present(void); + /* GHES register/unregister helpers */ int sdei_register_ghes(struct ghes *ghes, sdei_event_callback *normal_cb, sdei_event_callback *critical_cb); diff --git a/include/linux/ata.h b/include/linux/ata.h index 8fd48bcb2a46..c4fac5f8d480 100644 --- a/include/linux/ata.h +++ b/include/linux/ata.h @@ -75,6 +75,7 @@ enum { ATA_ID_HW_CONFIG = 93, ATA_ID_SPG = 98, ATA_ID_LBA_CAPACITY_2 = 100, + ATA_ID_MAX_PAGES_PER_DSM = 105, ATA_ID_SECTOR_SIZE = 106, ATA_ID_WWN = 108, ATA_ID_LOGICAL_SECTOR_SIZE = 117, /* and 118 */ @@ -288,6 +289,10 @@ enum { ATA_CMD_SANITIZE_DEVICE = 0xB4, ATA_CMD_ZAC_MGMT_IN = 0x4A, ATA_CMD_ZAC_MGMT_OUT = 0x9F, + ATA_CMD_GET_PHYS_ELEMENT_STATUS = 0x12, + ATA_CMD_REMOVE_ELEMENT_AND_TRUNCATE = 0x7c, + ATA_CMD_RESTORE_ELEMENTS_AND_REBUILD = 0x7d, + ATA_CMD_REMOVE_ELEMENT_AND_MODIFY_ZONES = 0x7e, /* marked obsolete in the ATA/ATAPI-7 spec */ ATA_CMD_RESTORE = 0x10, @@ -762,8 +767,7 @@ static inline bool ata_id_sense_reporting_enabled(const u16 *id) return id[ATA_ID_COMMAND_SET_4] & BIT(6); } -/** - * +/* * Word: 206 - SCT Command Transport * 15:12 - Vendor Specific * 11:6 - Reserved @@ -810,8 +814,9 @@ static inline bool ata_id_sct_supported(const u16 *id) * * The practical impact of this is that ata_id_major_version cannot * reliably report on drives below ATA3. + * + * Returns: major version of ATA drive level or %0 if unknown */ - static inline unsigned int ata_id_major_version(const u16 *id) { unsigned int mver; @@ -928,6 +933,18 @@ static inline bool ata_id_has_trim(const u16 *id) return false; } +static inline u16 ata_id_dsm_max_pages(const u16 *id) +{ + /* + * IDENTIFY DEVICE word 105: MAX PAGES PER DSM COMMAND. Maximum number + * of 512-byte pages of LBA Range Entries the device accepts in a + * single DATA SET MANAGEMENT command. Zero means the device does not + * specify a limit. The field is reserved unless TRIM is supported, so + * callers must gate on ata_id_has_trim(). + */ + return id[ATA_ID_MAX_PAGES_PER_DSM]; +} + static inline bool ata_id_has_zero_after_trim(const u16 *id) { /* DSM supported, deterministic read, and read zero after trim set */ diff --git a/include/linux/atalk.h b/include/linux/atalk.h deleted file mode 100644 index a55bfc6567d0..000000000000 --- a/include/linux/atalk.h +++ /dev/null @@ -1,186 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef __LINUX_ATALK_H__ -#define __LINUX_ATALK_H__ - - -#include <net/sock.h> -#include <uapi/linux/atalk.h> - -struct atalk_route { - struct net_device *dev; - struct atalk_addr target; - struct atalk_addr gateway; - int flags; - struct atalk_route *next; -}; - -/** - * struct atalk_iface - AppleTalk Interface - * @dev - Network device associated with this interface - * @address - Our address - * @status - What are we doing? - * @nets - Associated direct netrange - * @next - next element in the list of interfaces - */ -struct atalk_iface { - struct net_device *dev; - struct atalk_addr address; - int status; -#define ATIF_PROBE 1 /* Probing for an address */ -#define ATIF_PROBE_FAIL 2 /* Probe collided */ - struct atalk_netrange nets; - struct atalk_iface *next; -}; - -struct atalk_sock { - /* struct sock has to be the first member of atalk_sock */ - struct sock sk; - __be16 dest_net; - __be16 src_net; - unsigned char dest_node; - unsigned char src_node; - unsigned char dest_port; - unsigned char src_port; -}; - -static inline struct atalk_sock *at_sk(struct sock *sk) -{ - return (struct atalk_sock *)sk; -} - -struct ddpehdr { - __be16 deh_len_hops; /* lower 10 bits are length, next 4 - hops */ - __be16 deh_sum; - __be16 deh_dnet; - __be16 deh_snet; - __u8 deh_dnode; - __u8 deh_snode; - __u8 deh_dport; - __u8 deh_sport; - /* And netatalk apps expect to stick the type in themselves */ -}; - -static __inline__ struct ddpehdr *ddp_hdr(struct sk_buff *skb) -{ - return (struct ddpehdr *)skb_transport_header(skb); -} - -/* AppleTalk AARP headers */ -struct elapaarp { - __be16 hw_type; -#define AARP_HW_TYPE_ETHERNET 1 -#define AARP_HW_TYPE_TOKENRING 2 - __be16 pa_type; - __u8 hw_len; - __u8 pa_len; -#define AARP_PA_ALEN 4 - __be16 function; -#define AARP_REQUEST 1 -#define AARP_REPLY 2 -#define AARP_PROBE 3 - __u8 hw_src[ETH_ALEN]; - __u8 pa_src_zero; - __be16 pa_src_net; - __u8 pa_src_node; - __u8 hw_dst[ETH_ALEN]; - __u8 pa_dst_zero; - __be16 pa_dst_net; - __u8 pa_dst_node; -} __attribute__ ((packed)); - -static __inline__ struct elapaarp *aarp_hdr(struct sk_buff *skb) -{ - return (struct elapaarp *)skb_transport_header(skb); -} - -/* Not specified - how long till we drop a resolved entry */ -#define AARP_EXPIRY_TIME (5 * 60 * HZ) -/* Size of hash table */ -#define AARP_HASH_SIZE 16 -/* Fast retransmission timer when resolving */ -#define AARP_TICK_TIME (HZ / 5) -/* Send 10 requests then give up (2 seconds) */ -#define AARP_RETRANSMIT_LIMIT 10 -/* - * Some value bigger than total retransmit time + a bit for last reply to - * appear and to stop continual requests - */ -#define AARP_RESOLVE_TIME (10 * HZ) - -extern struct datalink_proto *ddp_dl, *aarp_dl; -extern int aarp_proto_init(void); - -/* Inter module exports */ - -/* Give a device find its atif control structure */ -#if IS_ENABLED(CONFIG_ATALK) -static inline struct atalk_iface *atalk_find_dev(struct net_device *dev) -{ - return dev->atalk_ptr; -} -#endif - -extern struct atalk_addr *atalk_find_dev_addr(struct net_device *dev); -extern struct net_device *atrtr_get_dev(struct atalk_addr *sa); -extern int aarp_send_ddp(struct net_device *dev, - struct sk_buff *skb, - struct atalk_addr *sa, void *hwaddr); -extern void aarp_device_down(struct net_device *dev); -extern void aarp_probe_network(struct atalk_iface *atif); -extern int aarp_proxy_probe_network(struct atalk_iface *atif, - struct atalk_addr *sa); -extern void aarp_proxy_remove(struct net_device *dev, - struct atalk_addr *sa); - -extern void aarp_cleanup_module(void); - -extern struct hlist_head atalk_sockets; -extern rwlock_t atalk_sockets_lock; - -extern struct atalk_route *atalk_routes; -extern rwlock_t atalk_routes_lock; - -extern struct atalk_iface *atalk_interfaces; -extern rwlock_t atalk_interfaces_lock; - -extern struct atalk_route atrtr_default; - -struct aarp_iter_state { - int bucket; - struct aarp_entry **table; -}; - -extern const struct seq_operations aarp_seq_ops; - -extern int sysctl_aarp_expiry_time; -extern int sysctl_aarp_tick_time; -extern int sysctl_aarp_retransmit_limit; -extern int sysctl_aarp_resolve_time; - -#ifdef CONFIG_SYSCTL -extern int atalk_register_sysctl(void); -extern void atalk_unregister_sysctl(void); -#else -static inline int atalk_register_sysctl(void) -{ - return 0; -} -static inline void atalk_unregister_sysctl(void) -{ -} -#endif - -#ifdef CONFIG_PROC_FS -extern int atalk_proc_init(void); -extern void atalk_proc_exit(void); -#else -static inline int atalk_proc_init(void) -{ - return 0; -} -static inline void atalk_proc_exit(void) -{ -} -#endif /* CONFIG_PROC_FS */ - -#endif /* __LINUX_ATALK_H__ */ diff --git a/include/linux/atm_tcp.h b/include/linux/atm_tcp.h deleted file mode 100644 index 2558439d849b..000000000000 --- a/include/linux/atm_tcp.h +++ /dev/null @@ -1,24 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* atm_tcp.h - Driver-specific declarations of the ATMTCP driver (for use by - driver-specific utilities) */ - -/* Written 1997-2000 by Werner Almesberger, EPFL LRC/ICA */ - -#ifndef LINUX_ATM_TCP_H -#define LINUX_ATM_TCP_H - -#include <uapi/linux/atm_tcp.h> - -struct atm_vcc; -struct module; - -struct atm_tcp_ops { - int (*attach)(struct atm_vcc *vcc,int itf); - int (*create_persistent)(int itf); - int (*remove_persistent)(int itf); - struct module *owner; -}; - -extern struct atm_tcp_ops atm_tcp_ops; - -#endif diff --git a/include/linux/atmdev.h b/include/linux/atmdev.h index 70807c679f1a..92c2e20e7416 100644 --- a/include/linux/atmdev.h +++ b/include/linux/atmdev.h @@ -51,33 +51,11 @@ enum { driver, cleared by anybody. */ ATM_VF_PARTIAL, /* resources are bound to PVC (partial PVC setup), controlled by socket layer */ - ATM_VF_REGIS, /* registered with demon, controlled by SVC - socket layer */ - ATM_VF_BOUND, /* local SAP is set, controlled by SVC socket - layer */ - ATM_VF_RELEASED, /* demon has indicated/requested release, - controlled by SVC socket layer */ ATM_VF_HASQOS, /* QOS parameters have been set */ - ATM_VF_LISTEN, /* socket is used for listening */ - ATM_VF_META, /* SVC socket isn't used for normal data - traffic and doesn't depend on signaling - to be available */ - ATM_VF_SESSION, /* VCC is p2mp session control descriptor */ - ATM_VF_HASSAP, /* SAP has been set */ - ATM_VF_CLOSE, /* asynchronous close - treat like VF_RELEASED*/ - ATM_VF_WAITING, /* waiting for reply from sigd */ - ATM_VF_IS_CLIP, /* in use by CLIP protocol */ + ATM_VF_CLOSE, /* asynchronous close - VC is being torn down */ }; -#define ATM_VF2VS(flags) \ - (test_bit(ATM_VF_READY,&(flags)) ? ATM_VS_CONNECTED : \ - test_bit(ATM_VF_RELEASED,&(flags)) ? ATM_VS_CLOSING : \ - test_bit(ATM_VF_LISTEN,&(flags)) ? ATM_VS_LISTEN : \ - test_bit(ATM_VF_REGIS,&(flags)) ? ATM_VS_INUSE : \ - test_bit(ATM_VF_BOUND,&(flags)) ? ATM_VS_BOUND : ATM_VS_IDLE) - - enum { ATM_DF_REMOVED, /* device was removed from atm_devs list */ }; @@ -100,26 +78,16 @@ struct atm_vcc { unsigned long atm_options; /* ATM layer options */ struct atm_dev *dev; /* device back pointer */ struct atm_qos qos; /* QOS */ - struct atm_sap sap; /* SAP */ void (*release_cb)(struct atm_vcc *vcc); /* release_sock callback */ void (*push)(struct atm_vcc *vcc,struct sk_buff *skb); void (*pop)(struct atm_vcc *vcc,struct sk_buff *skb); /* optional */ - int (*push_oam)(struct atm_vcc *vcc,void *cell); int (*send)(struct atm_vcc *vcc,struct sk_buff *skb); void *dev_data; /* per-device data */ void *proto_data; /* per-protocol data */ struct k_atm_aal_stats *stats; /* pointer to AAL stats group */ struct module *owner; /* owner of ->push function */ - /* SVC part --- may move later ------------------------------------- */ - short itf; /* interface number */ - struct sockaddr_atmsvc local; - struct sockaddr_atmsvc remote; - /* Multipoint part ------------------------------------------------- */ - struct atm_vcc *session; /* session VCC descriptor */ - /* Other stuff ----------------------------------------------------- */ - void *user_back; /* user backlink - not touched by */ - /* native ATM stack. Currently used */ - /* by CLIP and sch_atm. */ + void *user_back; /* user backlink - not touched by the */ + /* native ATM stack, used by sch_atm */ }; static inline struct atm_vcc *atm_sk(struct sock *sk) @@ -137,31 +105,19 @@ static inline struct sock *sk_atm(struct atm_vcc *vcc) return (struct sock *)vcc; } -struct atm_dev_addr { - struct sockaddr_atmsvc addr; /* ATM address */ - struct list_head entry; /* next address */ -}; - -enum atm_addr_type_t { ATM_ADDR_LOCAL, ATM_ADDR_LECS }; - struct atm_dev { const struct atmdev_ops *ops; /* device operations; NULL if unused */ - const struct atmphy_ops *phy; /* PHY operations, may be undefined */ - /* (NULL) */ const char *type; /* device type name */ int number; /* device index */ void *dev_data; /* per-device data */ void *phy_data; /* private PHY data */ unsigned long flags; /* device flags (ATM_DF_*) */ - struct list_head local; /* local ATM addresses */ - struct list_head lecs; /* LECS ATM addresses learned via ILMI */ unsigned char esi[ESI_LEN]; /* ESI ("MAC" addr) */ struct atm_cirange ci_range; /* VPI/VCI range */ struct k_atm_dev_stats stats; /* statistics */ char signal; /* signal status (ATM_PHY_SIG_*) */ int link_rate; /* link rate (default: OC3) */ refcount_t refcnt; /* reference count */ - spinlock_t lock; /* protect internal members */ #ifdef CONFIG_PROC_FS struct proc_dir_entry *proc_entry; /* proc entry */ char *proc_name; /* proc entry name */ @@ -170,12 +126,6 @@ struct atm_dev { struct list_head dev_list; /* linkage */ }; - -/* OF: send_Oam Flags */ - -#define ATM_OF_IMMED 1 /* Attempt immediate delivery */ -#define ATM_OF_INRATE 2 /* Attempt in-rate delivery */ - struct atmdev_ops { /* only send is required */ void (*dev_close)(struct atm_dev *dev); int (*open)(struct atm_vcc *vcc); @@ -185,25 +135,11 @@ struct atmdev_ops { /* only send is required */ int (*compat_ioctl)(struct atm_dev *dev,unsigned int cmd, void __user *arg); #endif - int (*pre_send)(struct atm_vcc *vcc, struct sk_buff *skb); int (*send)(struct atm_vcc *vcc,struct sk_buff *skb); - int (*send_bh)(struct atm_vcc *vcc, struct sk_buff *skb); - int (*send_oam)(struct atm_vcc *vcc,void *cell,int flags); - void (*phy_put)(struct atm_dev *dev,unsigned char value, - unsigned long addr); - unsigned char (*phy_get)(struct atm_dev *dev,unsigned long addr); - int (*change_qos)(struct atm_vcc *vcc,struct atm_qos *qos,int flags); int (*proc_read)(struct atm_dev *dev,loff_t *pos,char *page); struct module *owner; }; -struct atmphy_ops { - int (*start)(struct atm_dev *dev); - int (*ioctl)(struct atm_dev *dev,unsigned int cmd,void __user *arg); - void (*interrupt)(struct atm_dev *dev); - int (*stop)(struct atm_dev *dev); -}; - struct atm_skb_data { struct atm_vcc *vcc; /* ATM VCC */ unsigned long atm_options; /* ATM layer options */ @@ -293,9 +229,6 @@ static inline void atm_dev_put(struct atm_dev *dev) int atm_charge(struct atm_vcc *vcc,int truesize); -struct sk_buff *atm_alloc_charge(struct atm_vcc *vcc,int pdu_size, - gfp_t gfp_flags); -int atm_pcr_goal(const struct atm_trafprm *tp); void vcc_release_async(struct atm_vcc *vcc, int reply); @@ -309,17 +242,19 @@ struct atm_ioctl { /** * register_atm_ioctl - register handler for ioctl operations + * @ioctl: ioctl handler to register * * Special (non-device) handlers of ioctl's should * register here. If you're a normal device, you should * set .ioctl in your atmdev_ops instead. */ -void register_atm_ioctl(struct atm_ioctl *); +void register_atm_ioctl(struct atm_ioctl *ioctl); /** * deregister_atm_ioctl - remove the ioctl handler + * @ioctl: ioctl handler to deregister */ -void deregister_atm_ioctl(struct atm_ioctl *); +void deregister_atm_ioctl(struct atm_ioctl *ioctl); /* register_atmdevice_notifier - register atm_dev notify events diff --git a/include/linux/audit.h b/include/linux/audit.h index b642b5faca65..45abb3722d30 100644 --- a/include/linux/audit.h +++ b/include/linux/audit.h @@ -15,7 +15,16 @@ #include <uapi/linux/audit.h> #include <uapi/linux/fanotify.h> -#define AUDIT_INO_UNSET ((unsigned long)-1) +#define AUDIT_STATUS_ALL (AUDIT_STATUS_ENABLED | \ + AUDIT_STATUS_FAILURE | \ + AUDIT_STATUS_PID | \ + AUDIT_STATUS_RATE_LIMIT | \ + AUDIT_STATUS_BACKLOG_LIMIT | \ + AUDIT_STATUS_BACKLOG_WAIT_TIME | \ + AUDIT_STATUS_LOST | \ + AUDIT_STATUS_BACKLOG_WAIT_TIME_ACTUAL) + +#define AUDIT_INO_UNSET ((u64)-1) #define AUDIT_DEV_UNSET ((dev_t)-1) struct audit_sig_info { @@ -124,8 +133,8 @@ enum audit_nfcfgop { AUDIT_NFT_OP_INVALID, }; -extern int __init audit_register_class(int class, unsigned *list); -extern int audit_classify_syscall(int abi, unsigned syscall); +extern int __init audit_register_class(int class, unsigned int *list); +extern int audit_classify_syscall(int abi, unsigned int syscall); extern int audit_classify_arch(int arch); /* audit_names->type values */ diff --git a/include/linux/audit_arch.h b/include/linux/audit_arch.h index 2b8153791e6a..a35069a6c15d 100644 --- a/include/linux/audit_arch.h +++ b/include/linux/audit_arch.h @@ -21,13 +21,13 @@ enum auditsc_class_t { AUDITSC_NVALS /* count */ }; -extern int audit_classify_compat_syscall(int abi, unsigned syscall); +extern int audit_classify_compat_syscall(int abi, unsigned int syscall); /* only for compat system calls */ -extern unsigned compat_write_class[]; -extern unsigned compat_read_class[]; -extern unsigned compat_dir_class[]; -extern unsigned compat_chattr_class[]; -extern unsigned compat_signal_class[]; +extern unsigned int compat_write_class[]; +extern unsigned int compat_read_class[]; +extern unsigned int compat_dir_class[]; +extern unsigned int compat_chattr_class[]; +extern unsigned int compat_signal_class[]; #endif diff --git a/include/linux/auxiliary_bus.h b/include/linux/auxiliary_bus.h index 4086afd0cc6b..de0ecd0fb05a 100644 --- a/include/linux/auxiliary_bus.h +++ b/include/linux/auxiliary_bus.h @@ -9,7 +9,7 @@ #define _AUXILIARY_BUS_H_ #include <linux/device.h> -#include <linux/mod_devicetable.h> +#include <linux/device-id/auxiliary.h> /** * DOC: DEVICE_LIFESPAN @@ -62,6 +62,9 @@ * @sysfs.irqs: irqs xarray contains irq indices which are used by the device, * @sysfs.lock: Synchronize irq sysfs creation, * @sysfs.irq_dir_exists: whether "irqs" directory exists, + * @registration_data_rust: private data owned by the registering (parent) + * driver; valid for as long as the device is + * registered with the driver core, * * An auxiliary_device represents a part of its parent device's functionality. * It is given a name that, combined with the registering drivers @@ -148,6 +151,7 @@ struct auxiliary_device { struct mutex lock; /* Synchronize irq sysfs creation */ bool irq_dir_exists; } sysfs; + void *registration_data_rust; }; /** @@ -271,6 +275,8 @@ struct auxiliary_device *__devm_auxiliary_device_create(struct device *dev, __devm_auxiliary_device_create(dev, KBUILD_MODNAME, devname, \ platform_data, 0) +bool dev_is_auxiliary(struct device *dev); + /** * module_auxiliary_driver() - Helper macro for registering an auxiliary driver * @__auxiliary_driver: auxiliary driver struct diff --git a/include/linux/backing-dev-defs.h b/include/linux/backing-dev-defs.h index c88fd4d37d1f..4f1084937315 100644 --- a/include/linux/backing-dev-defs.h +++ b/include/linux/backing-dev-defs.h @@ -26,6 +26,7 @@ enum wb_state { WB_writeback_running, /* Writeback is in progress */ WB_has_dirty_io, /* Dirty inodes on ->b_{dirty|io|more_io} */ WB_start_all, /* nr_pages == 0 (all) work pending */ + WB_start_dontcache, /* dontcache writeback pending */ }; enum wb_stat_item { @@ -33,6 +34,7 @@ enum wb_stat_item { WB_WRITEBACK, WB_DIRTIED, WB_WRITTEN, + WB_DONTCACHE_DIRTY, NR_WB_STAT_ITEMS }; @@ -55,6 +57,7 @@ enum wb_reason { */ WB_REASON_FORKER_THREAD, WB_REASON_FOREIGN_FLUSH, + WB_REASON_DONTCACHE, WB_REASON_MAX, }; @@ -237,7 +240,7 @@ static inline void wb_get(struct bdi_writeback *wb) } /** - * wb_put - decrement a wb's refcount + * wb_put_many - decrement a wb's refcount * @wb: bdi_writeback to put * @nr: number of references to put */ diff --git a/include/linux/backing-dev.h b/include/linux/backing-dev.h index 0c8342747cab..c2284466e7aa 100644 --- a/include/linux/backing-dev.h +++ b/include/linux/backing-dev.h @@ -76,8 +76,6 @@ static inline s64 wb_stat_sum(struct bdi_writeback *wb, enum wb_stat_item item) return percpu_counter_sum_positive(&wb->stat[item]); } -extern void wb_writeout_inc(struct bdi_writeback *wb); - /* * maximal error of a stat counter. */ @@ -136,6 +134,19 @@ static inline bool mapping_can_writeback(struct address_space *mapping) return inode_to_bdi(mapping->host)->capabilities & BDI_CAP_WRITEBACK; } +/* Must not be used by file systems that support cgroup writeback */ +static inline int bdi_wb_dirty_exceeded(struct backing_dev_info *bdi) +{ + return bdi->wb.dirty_exceeded; +} + +/* Must not be used by file systems that support cgroup writeback */ +static inline void bdi_wb_stat_mod(struct inode *inode, enum wb_stat_item item, + s64 amount) +{ + wb_stat_mod(&inode_to_bdi(inode)->wb, item, amount); +} + #ifdef CONFIG_CGROUP_WRITEBACK struct bdi_writeback *wb_get_lookup(struct backing_dev_info *bdi, diff --git a/include/linux/backing-file.h b/include/linux/backing-file.h index 1476a6ed1bfd..c939cd222730 100644 --- a/include/linux/backing-file.h +++ b/include/linux/backing-file.h @@ -18,10 +18,10 @@ struct backing_file_ctx { void (*end_write)(struct kiocb *iocb, ssize_t); }; -struct file *backing_file_open(const struct path *user_path, int flags, +struct file *backing_file_open(const struct file *user_file, int flags, const struct path *real_path, const struct cred *cred); -struct file *backing_tmpfile_open(const struct path *user_path, int flags, +struct file *backing_tmpfile_open(const struct file *user_file, int flags, const struct path *real_parentpath, umode_t mode, const struct cred *cred); ssize_t backing_file_read_iter(struct file *file, struct iov_iter *iter, diff --git a/include/linux/bcma/bcma.h b/include/linux/bcma/bcma.h index 60b94b944e9f..17fc50190014 100644 --- a/include/linux/bcma/bcma.h +++ b/include/linux/bcma/bcma.h @@ -3,7 +3,7 @@ #define LINUX_BCMA_H_ #include <linux/pci.h> -#include <linux/mod_devicetable.h> +#include <linux/device-id/bcma.h> #include <linux/bcma/bcma_driver_arm_c9.h> #include <linux/bcma/bcma_driver_chipcommon.h> @@ -486,4 +486,6 @@ extern u32 bcma_core_dma_translation(struct bcma_device *core); extern unsigned int bcma_core_irq(struct bcma_device *core, int num); +extern const struct software_node bcma_gpio_swnode; + #endif /* LINUX_BCMA_H_ */ diff --git a/include/linux/binfmt_misc.h b/include/linux/binfmt_misc.h new file mode 100644 index 000000000000..8045b10dd3e5 --- /dev/null +++ b/include/linux/binfmt_misc.h @@ -0,0 +1,113 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LINUX_BINFMT_MISC_H +#define _LINUX_BINFMT_MISC_H + +#include <linux/types.h> + +struct bpf_prog; +struct file; +struct linux_binprm; +struct ucounts; +struct user_namespace; + +#define BINFMT_MISC_OPS_NAME_MAX 16 + +/* Longest name a 'B' entry can bind an interpreter under. */ +#define BINFMT_MISC_INTERP_NAME_MAX 32 + +/* Most interpreters one entry can bind. */ +#define BINFMT_MISC_INTERP_MAX 100 + +/** + * struct binfmt_misc_interp - an interpreter an entry was registered with + * @list: link in the entry's list, in registration order + * @file: the file, opened at registration and never resolved again + * @ucounts: the UCOUNT_BINFMT_MISC_INTERPRETERS charge the binding took + * @path: the path it was registered under, used as the name the interpreter + * runs under; stored after @name in the same allocation + * @name: the name the load program selects it by; empty for the fixed + * interpreter of a static 'F' entry + * + * Owned by the entry and living exactly as long as it does. The list head + * is handed to the handler's load program for the duration of one exec, + * which picks one with bpf_binprm_select_interp(). + */ +struct binfmt_misc_interp { + struct list_head list; + struct file *file; + struct ucounts *ucounts; + const char *path; + char name[]; +}; + +const struct binfmt_misc_interp * +binfmt_misc_find_interp(const struct list_head *interps, const char *name); + +/** + * enum bpf_binprm_flags - per-exec invocation flags a load program can request + * @BPF_BINPRM_PRESERVE_ARGV0: keep the caller's argv[0] (like the 'P' flag) + * @BPF_BINPRM_CREDENTIALS: compute credentials from the binary; implies execfd + * (like the 'C' flag) + * @BPF_BINPRM_EXECFD: pass the binary via AT_EXECFD (like the 'O' flag) + * @BPF_BINPRM_TRANSPARENT: leave argv untouched, the interpreter takes the + * binary from AT_EXECFD (like the 'T' flag); implies + * execfd, excludes preserve-argv0 + * @BPF_BINPRM_LOADER: substitute the interpreter for the binary's PT_INTERP + * and run the binary as a native exec (like the 'L' + * flag); excludes every other flag + * + * Set from a load program with bpf_binprm_set_flags(). Unlike a static entry, + * a bpf handler chooses these per exec rather than once at registration. + */ +enum bpf_binprm_flags { + BPF_BINPRM_PRESERVE_ARGV0 = (1ULL << 0), + BPF_BINPRM_CREDENTIALS = (1ULL << 1), + BPF_BINPRM_EXECFD = (1ULL << 2), + BPF_BINPRM_TRANSPARENT = (1ULL << 3), + BPF_BINPRM_LOADER = (1ULL << 4), +}; + +/** + * struct binfmt_misc_ops - bpf-backed binary type handler + * @match: decide whether the handler applies to @bprm; consulted from the + * entry lookup walk like static magic and extension matching, in + * registration order with first-match-wins semantics; sleepable, + * so it can read the binary to decide, but the verifier rejects + * the interpreter selection kfuncs in it + * @load: select an interpreter for the matched @bprm via + * bpf_binprm_set_interp(), or one the entry bound via + * bpf_binprm_select_interp(), and return zero; a match is + * committed, so a failure fails the exec instead of falling + * through to later entries; -ENOEXEC does not fail the exec but + * moves on to the remaining binary formats + * @name: name that 'B' entries reference the handler by + */ +struct binfmt_misc_ops { + bool (*match)(struct linux_binprm *bprm); + int (*load)(struct linux_binprm *bprm); + char name[BINFMT_MISC_OPS_NAME_MAX]; +}; + +#ifdef CONFIG_BINFMT_MISC_BPF +const struct binfmt_misc_ops *binfmt_misc_get_ops(struct user_namespace *user_ns, + const char *name); +void binfmt_misc_put_ops(const struct binfmt_misc_ops *ops); +bool bpf_prog_is_binfmt_misc_ops(const struct bpf_prog *prog); +#else +static inline const struct binfmt_misc_ops * +binfmt_misc_get_ops(struct user_namespace *user_ns, const char *name) +{ + return NULL; +} + +static inline void binfmt_misc_put_ops(const struct binfmt_misc_ops *ops) +{ +} + +static inline bool bpf_prog_is_binfmt_misc_ops(const struct bpf_prog *prog) +{ + return false; +} +#endif /* CONFIG_BINFMT_MISC_BPF */ + +#endif /* _LINUX_BINFMT_MISC_H */ diff --git a/include/linux/binfmts.h b/include/linux/binfmts.h index 65abd5ab8836..f686a37f7a0a 100644 --- a/include/linux/binfmts.h +++ b/include/linux/binfmts.h @@ -12,6 +12,16 @@ struct coredump_params; #define CORENAME_MAX_SIZE 128 +/* Interpreter selection staged by a bpf binfmt_misc handler. */ +struct binfmt_misc_bpf { + /* interpreters the matched entry bound, selectable by name */ + const struct list_head *bpf_interps; + const char *bpf_interp; /* interpreter selected by a bpf handler */ + struct file *bpf_interp_file; /* the bound interpreter it selected */ + const char *bpf_interp_arg; /* interpreter argument from a bpf handler */ + u64 bpf_flags; /* enum bpf_binprm_flags from a bpf handler */ +}; + /* * This structure is used to hold the arguments that are used when loading binaries. */ @@ -25,6 +35,9 @@ struct linux_binprm { struct page *page[MAX_ARG_PAGES]; #endif struct mm_struct *mm; + struct mm_struct *old_mm; /* replaced address space, freed by setup_new_exec() */ + /* user_ns published to task->exec_state at execve, narrowed by would_dump(). */ + struct user_namespace *user_ns; unsigned long p; /* current top of mem */ unsigned int /* Should an execfd be passed to userspace? */ @@ -52,6 +65,7 @@ struct linux_binprm { is_check:1; struct file *executable; /* Executable to pass to the interpreter */ struct file *interpreter; + struct file *loader; struct file *file; struct cred *cred; /* new credentials */ int unsafe; /* how unsafe this exec is (mask of LSM_UNSAFE_*) */ @@ -62,6 +76,7 @@ struct linux_binprm { of the time same as filename, but could be different for binfmt_{misc,script} */ const char *fdpath; /* generated filename for execveat */ + struct binfmt_misc_bpf; /* bpf handler interpreter selection */ unsigned interp_flags; int execfd; /* File descriptor of the executable */ unsigned long exec; @@ -82,6 +97,28 @@ struct linux_binprm { #define BINPRM_FLAGS_PRESERVE_ARGV0_BIT 3 #define BINPRM_FLAGS_PRESERVE_ARGV0 (1 << BINPRM_FLAGS_PRESERVE_ARGV0_BIT) +/* binfmt_misc dispatched to the interpreter transparently */ +#define BINPRM_FLAGS_TRANSPARENT_INTERP_BIT 4 +#define BINPRM_FLAGS_TRANSPARENT_INTERP (1 << BINPRM_FLAGS_TRANSPARENT_INTERP_BIT) + +/** + * bprm_at_flags - the AT_FLAGS this invocation implies + * @bprm: binary that is being executed + * + * Tell the program on the receiving end which dispatch contract it got. + * + * Return: the AT_FLAGS value for this exec + */ +static inline unsigned long bprm_at_flags(const struct linux_binprm *bprm) +{ + /* Transparency preserves the whole argv, argv[0] included. */ + if (bprm->interp_flags & BINPRM_FLAGS_TRANSPARENT_INTERP) + return AT_FLAGS_TRANSPARENT_INTERP; + if (bprm->interp_flags & BINPRM_FLAGS_PRESERVE_ARGV0) + return AT_FLAGS_PRESERVE_ARGV0; + return 0; +} + /* * This structure defines the functions that are used to load the binary formats that * linux accepts. @@ -98,8 +135,8 @@ struct linux_binfmt { #if IS_ENABLED(CONFIG_BINFMT_MISC) struct binfmt_misc { - struct list_head entries; - rwlock_t entries_lock; + struct hlist_head entries; + spinlock_t entries_lock; bool enabled; } __randomize_layout; @@ -126,6 +163,8 @@ extern int begin_new_exec(struct linux_binprm * bprm); extern void setup_new_exec(struct linux_binprm * bprm); extern void finalize_exec(struct linux_binprm *bprm); extern void would_dump(struct linux_binprm *, struct file *); +struct file *bprm_open_interpreter(struct linux_binprm *bprm, const char *path); +void bprm_drop_loader(struct linux_binprm *bprm); extern int suid_dumpable; diff --git a/include/linux/bio-integrity.h b/include/linux/bio-integrity.h index 21e4652dcfd2..0ea2a8bf7efb 100644 --- a/include/linux/bio-integrity.h +++ b/include/linux/bio-integrity.h @@ -17,6 +17,9 @@ enum bip_flags { BIP_MEMPOOL = 1 << 15, /* buffer backed by mempool */ }; +/* flags that require generate/verify action. */ +#define BIP_CHECK_FLAGS (BIP_CHECK_GUARD | BIP_CHECK_REFTAG | BIP_CHECK_APPTAG) + struct bio_integrity_payload { struct bvec_iter bip_iter; @@ -78,7 +81,7 @@ int bio_integrity_add_page(struct bio *bio, struct page *page, unsigned int len, int bio_integrity_map_user(struct bio *bio, struct iov_iter *iter); int bio_integrity_map_iter(struct bio *bio, struct uio_meta *meta); void bio_integrity_unmap_user(struct bio *bio); -bool bio_integrity_prep(struct bio *bio); +void bio_integrity_prep(struct bio *bio, unsigned int action); void bio_integrity_advance(struct bio *bio, unsigned int bytes_done); void bio_integrity_trim(struct bio *bio); int bio_integrity_clone(struct bio *bio, struct bio *bio_src, gfp_t gfp_mask); @@ -104,9 +107,8 @@ static inline void bio_integrity_unmap_user(struct bio *bio) { } -static inline bool bio_integrity_prep(struct bio *bio) +static inline void bio_integrity_prep(struct bio *bio, unsigned int action) { - return true; } static inline int bio_integrity_clone(struct bio *bio, struct bio *bio_src, @@ -142,7 +144,14 @@ static inline int bio_integrity_add_page(struct bio *bio, struct page *page, } #endif /* CONFIG_BLK_DEV_INTEGRITY */ -void bio_integrity_alloc_buf(struct bio *bio, bool zero_buffer); +void bio_integrity_alloc_buf(struct bio *bio, gfp_t gfp, bool zero_buffer); void bio_integrity_free_buf(struct bio_integrity_payload *bip); +void bio_integrity_setup_default(struct bio *bio); + +unsigned int fs_bio_integrity_alloc(struct bio *bio); +void fs_bio_integrity_free(struct bio *bio); +void fs_bio_integrity_generate(struct bio *bio); +int fs_bio_integrity_verify(struct bio *bio, sector_t sector, + unsigned int size); #endif /* _LINUX_BIO_INTEGRITY_H */ diff --git a/include/linux/bio.h b/include/linux/bio.h index 36a3f2275ecd..bb3235497e67 100644 --- a/include/linux/bio.h +++ b/include/linux/bio.h @@ -283,7 +283,7 @@ static inline void bio_first_folio(struct folio_iter *fi, struct bio *bio, return; } - fi->folio = page_folio(bvec->bv_page); + fi->folio = bvec_folio(bvec); fi->offset = bvec->bv_offset + PAGE_SIZE * folio_page_idx(fi->folio, bvec->bv_page); fi->_seg_count = bvec->bv_len; @@ -347,11 +347,9 @@ enum { }; extern int bioset_init(struct bio_set *, unsigned int, unsigned int, int flags); extern void bioset_exit(struct bio_set *); -extern int biovec_init_pool(mempool_t *pool, int pool_entries); struct bio *bio_alloc_bioset(struct block_device *bdev, unsigned short nr_vecs, - blk_opf_t opf, gfp_t gfp_mask, - struct bio_set *bs); + blk_opf_t opf, gfp_t gfp, struct bio_set *bs); struct bio *bio_kmalloc(unsigned short nr_vecs, gfp_t gfp_mask); extern void bio_put(struct bio *); @@ -370,19 +368,68 @@ static inline struct bio *bio_alloc(struct block_device *bdev, void submit_bio(struct bio *bio); +/** + * bio_in_atomic - check if the current context is unsafe for bio completion + * + * Return: %true in atomic contexts (e.g. hard/soft IRQ, preempt-disabled); + * %false when a bio can be safely completed in the current context. + */ +static inline bool bio_in_atomic(void) +{ + if (IS_ENABLED(CONFIG_PREEMPTION) && rcu_preempt_depth()) + return true; + if (!IS_ENABLED(CONFIG_PREEMPT_COUNT)) + return true; + return !preemptible(); +} + +void __bio_complete_in_task(struct bio *bio); + +/** + * bio_complete_in_task - ensure a bio is completed in preemptible task context + * @bio: bio to complete + * + * If called from non-task context, offload the bio completion to a worker + * thread and return %true. Else return %false and do nothing. + * + * Uses BIO_COMPLETE_IN_TASK as a sentinel: if set, the bio was already + * deferred and we are running in the worker — return %false so the + * callback proceeds instead of re-deferring. + */ +static inline bool bio_complete_in_task(struct bio *bio) +{ + if (bio_flagged(bio, BIO_COMPLETE_IN_TASK)) + return false; + if (!bio_in_atomic()) + return false; + bio_set_flag(bio, BIO_COMPLETE_IN_TASK); + __bio_complete_in_task(bio); + return true; +} + extern void bio_endio(struct bio *); -static inline void bio_io_error(struct bio *bio) +/** + * bio_endio_status - end I/O on a bio with a specific status + * @bio: bio + * @status: status to set + * + * Set @bio->bi_status to @status and call bio_endio(). + **/ +static inline void bio_endio_status(struct bio *bio, blk_status_t status) { - bio->bi_status = BLK_STS_IOERR; + bio->bi_status = status; bio_endio(bio); } +static inline void bio_io_error(struct bio *bio) +{ + bio_endio_status(bio, BLK_STS_IOERR); +} + static inline void bio_wouldblock_error(struct bio *bio) { - bio_set_flag(bio, BIO_QUIET); - bio->bi_status = BLK_STS_AGAIN; - bio_endio(bio); + bio_endio_status(bio, BLK_STS_AGAIN); } /* @@ -433,6 +480,8 @@ extern void bio_uninit(struct bio *); void bio_reset(struct bio *bio, struct block_device *bdev, blk_opf_t opf); void bio_reuse(struct bio *bio, blk_opf_t opf); void bio_chain(struct bio *, struct bio *); +void bio_await(struct bio *bio, void *priv, + void (*submit)(struct bio *bio, void *priv)); int __must_check bio_add_page(struct bio *bio, struct page *page, unsigned len, unsigned off); @@ -467,27 +516,21 @@ int bdev_rw_virt(struct block_device *bdev, sector_t sector, void *data, size_t len, enum req_op op); int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter, - unsigned len_align_mask); + unsigned mem_align_mask, unsigned len_align_mask); -void bio_iov_bvec_set(struct bio *bio, const struct iov_iter *iter); +bool bio_iov_iter_set(struct bio *bio, const struct iov_iter *iter); void __bio_release_pages(struct bio *bio, bool mark_dirty); extern void bio_set_pages_dirty(struct bio *bio); extern void bio_check_pages_dirty(struct bio *bio); -int bio_iov_iter_bounce(struct bio *bio, struct iov_iter *iter); +int bio_iov_iter_bounce(struct bio *bio, struct iov_iter *iter, size_t maxlen, + size_t minsize); void bio_iov_iter_unbounce(struct bio *bio, bool is_error, bool mark_dirty); -extern void bio_copy_data_iter(struct bio *dst, struct bvec_iter *dst_iter, - struct bio *src, struct bvec_iter *src_iter); extern void bio_copy_data(struct bio *dst, struct bio *src); extern void bio_free_pages(struct bio *bio); +void zero_fill_bio(struct bio *bio); void guard_bio_eod(struct bio *bio); -void zero_fill_bio_iter(struct bio *bio, struct bvec_iter iter); - -static inline void zero_fill_bio(struct bio *bio) -{ - zero_fill_bio_iter(bio, bio->bi_iter); -} static inline void bio_release_pages(struct bio *bio, bool mark_dirty) { @@ -701,20 +744,6 @@ static inline bool bioset_initialized(struct bio_set *bs) return bs->bio_slab != NULL; } -/* - * Mark a bio as polled. Note that for async polled IO, the caller must - * expect -EWOULDBLOCK if we cannot allocate a request (or other resources). - * We cannot block waiting for requests on polled IO, as those completions - * must be found by the caller. This is different than IRQ driven IO, where - * it's safe to wait for IO to complete. - */ -static inline void bio_set_polled(struct bio *bio, struct kiocb *kiocb) -{ - bio->bi_opf |= REQ_POLLED; - if (kiocb->ki_flags & IOCB_NOWAIT) - bio->bi_opf |= REQ_NOWAIT; -} - static inline void bio_clear_polled(struct bio *bio) { bio->bi_opf &= ~REQ_POLLED; diff --git a/include/linux/bitfield.h b/include/linux/bitfield.h index 54aeeef1f0ec..14f86e455a67 100644 --- a/include/linux/bitfield.h +++ b/include/linux/bitfield.h @@ -44,7 +44,7 @@ * FIELD_MODIFY(REG_FIELD_C, ®, c); */ -#define __bf_shf(x) (__builtin_ffsll(x) - 1) +#define __bf_shf __builtin_ctzll #define __scalar_type_to_unsigned_cases(type) \ unsigned type: (unsigned type)0, \ @@ -179,6 +179,22 @@ }) /** + * FIELD_GET_SIGNED() - extract a signed bitfield element + * @mask: shifted mask defining the field's length and position + * @reg: value of entire bitfield + * + * Returns the sign-extended field specified by @_mask from the + * bitfield passed in as @reg by masking and shifting it down. + */ +#define FIELD_GET_SIGNED(mask, reg) \ + ({ \ + __BF_FIELD_CHECK(mask, reg, 0U, "FIELD_GET_SIGNED: "); \ + ((__signed_scalar_typeof(mask)) \ + (((long long)(reg) << __builtin_clzll(mask)) >> \ + (__builtin_clzll(mask) + __builtin_ctzll(mask)))); \ + }) + +/** * FIELD_MODIFY() - modify a bitfield element * @_mask: shifted mask defining the field's length and position * @_reg_p: pointer to the memory that should be updated diff --git a/include/linux/bitmap-str.h b/include/linux/bitmap-str.h index 53d3e1b32d3d..abe7a69a846f 100644 --- a/include/linux/bitmap-str.h +++ b/include/linux/bitmap-str.h @@ -5,7 +5,6 @@ #include <linux/types.h> int bitmap_parse_user(const char __user *ubuf, unsigned int ulen, unsigned long *dst, int nbits); -int bitmap_print_to_pagebuf(bool list, char *buf, const unsigned long *maskp, int nmaskbits); int bitmap_print_bitmask_to_buf(char *buf, const unsigned long *maskp, int nmaskbits, loff_t off, size_t count); int bitmap_print_list_to_buf(char *buf, const unsigned long *maskp, int nmaskbits, diff --git a/include/linux/bitmap.h b/include/linux/bitmap.h index b0395e4ccf90..7df1573a409c 100644 --- a/include/linux/bitmap.h +++ b/include/linux/bitmap.h @@ -2,7 +2,7 @@ #ifndef __LINUX_BITMAP_H #define __LINUX_BITMAP_H -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #include <linux/align.h> #include <linux/bitops.h> @@ -46,6 +46,7 @@ struct device; * bitmap_and(dst, src1, src2, nbits) *dst = *src1 & *src2 * bitmap_or(dst, src1, src2, nbits) *dst = *src1 | *src2 * bitmap_weighted_or(dst, src1, src2, nbits) *dst = *src1 | *src2. Returns Hamming Weight of dst + * bitmap_weighted_xor(dst, src1, src2, nbits) *dst = *src1 ^ *src2. Returns Hamming Weight of dst * bitmap_xor(dst, src1, src2, nbits) *dst = *src1 ^ *src2 * bitmap_andnot(dst, src1, src2, nbits) *dst = *src1 & ~(*src2) * bitmap_complement(dst, src, nbits) *dst = ~(*src) @@ -57,6 +58,7 @@ struct device; * bitmap_weight(src, nbits) Hamming Weight: number set bits * bitmap_weight_and(src1, src2, nbits) Hamming Weight of and'ed bitmap * bitmap_weight_andnot(src1, src2, nbits) Hamming Weight of andnot'ed bitmap + * bitmap_weight_from(src, start, end) Hamming Weight starting from @start * bitmap_set(dst, pos, nbits) Set specified bit area * bitmap_clear(dst, pos, nbits) Clear specified bit area * bitmap_find_next_zero_area(buf, len, pos, n, mask) Find bit free area @@ -168,6 +170,8 @@ void __bitmap_or(unsigned long *dst, const unsigned long *bitmap1, const unsigned long *bitmap2, unsigned int nbits); unsigned int __bitmap_weighted_or(unsigned long *dst, const unsigned long *bitmap1, const unsigned long *bitmap2, unsigned int nbits); +unsigned int __bitmap_weighted_xor(unsigned long *dst, const unsigned long *bitmap1, + const unsigned long *bitmap2, unsigned int nbits); void __bitmap_xor(unsigned long *dst, const unsigned long *bitmap1, const unsigned long *bitmap2, unsigned int nbits); bool __bitmap_andnot(unsigned long *dst, const unsigned long *bitmap1, @@ -205,6 +209,9 @@ unsigned long bitmap_find_next_zero_area_off(unsigned long *map, * The @align_mask should be one less than a power of 2; the effect is that * the bit offset of all zero areas this function finds is multiples of that * power of 2. A @align_mask of 0 means no alignment is required. + * + * Return: The bit offset of the found area or a value >= @size + * if no area is found. */ static __always_inline unsigned long bitmap_find_next_zero_area(unsigned long *map, @@ -353,6 +360,18 @@ unsigned int bitmap_weighted_or(unsigned long *dst, const unsigned long *src1, } static __always_inline +unsigned int bitmap_weighted_xor(unsigned long *dst, const unsigned long *src1, + const unsigned long *src2, unsigned int nbits) +{ + if (small_const_nbits(nbits)) { + *dst = *src1 ^ *src2; + return hweight_long(*dst & BITMAP_LAST_WORD_MASK(nbits)); + } else { + return __bitmap_weighted_xor(dst, src1, src2, nbits); + } +} + +static __always_inline void bitmap_xor(unsigned long *dst, const unsigned long *src1, const unsigned long *src2, unsigned int nbits) { @@ -479,6 +498,38 @@ unsigned long bitmap_weight_andnot(const unsigned long *src1, return __bitmap_weight_andnot(src1, src2, nbits); } +/** + * bitmap_weight_from - Hamming weight for a memory region + * @bitmap: The base address + * @start: The bitnumber to starts weighting + * @end: the bitmap size in bits + * + * Returns the number of set bits in the region. If @start >= @end, + * return >= end. + */ +static __always_inline +unsigned long bitmap_weight_from(const unsigned long *bitmap, + unsigned int start, unsigned int end) +{ + unsigned long w; + + if (unlikely(start >= end)) + return end; + + if (small_const_nbits(end)) + return hweight_long(*bitmap & GENMASK(end - 1, start)); + + bitmap += start / BITS_PER_LONG; + /* Opencode round_down() to not include math.h */ + end -= start & ~(BITS_PER_LONG - 1); + start %= BITS_PER_LONG; + w = bitmap_weight(bitmap, end); + if (start) + w -= hweight_long(*bitmap & BITMAP_LAST_WORD_MASK(start)); + + return w; +} + static __always_inline void bitmap_set(unsigned long *map, unsigned int start, unsigned int nbits) { @@ -649,14 +700,6 @@ void bitmap_gather(unsigned long *dst, const unsigned long *src, __assign_bit(n++, dst, test_bit(bit, src)); } -static __always_inline -void bitmap_next_set_region(unsigned long *bitmap, unsigned int *rs, - unsigned int *re, unsigned int end) -{ - *rs = find_next_bit(bitmap, end, *rs); - *re = find_next_zero_bit(bitmap, end, *rs + 1); -} - /** * bitmap_release_region - release allocated bitmap region * @bitmap: array of unsigned longs corresponding to the bitmap @@ -846,6 +889,6 @@ void bitmap_write(unsigned long *map, unsigned long value, #define bitmap_set_value8(map, value, start) \ bitmap_write(map, value, start, BITS_PER_BYTE) -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ #endif /* __LINUX_BITMAP_H */ diff --git a/include/linux/bitops.h b/include/linux/bitops.h index ea7898cc5903..657eab2725ce 100644 --- a/include/linux/bitops.h +++ b/include/linux/bitops.h @@ -179,9 +179,11 @@ static inline __u8 ror8(__u8 word, unsigned int shift) /** * sign_extend32 - sign extend a 32-bit value using specified bit as sign-bit * @value: value to sign extend - * @index: 0 based bit index (0<=index<32) to sign bit + * @index: 0 based bit index (0 <= index < 32) to sign bit * * This is safe to use for 16- and 8-bit types as well. + * + * Return: 32-bit sign extended value */ static __always_inline __s32 sign_extend32(__u32 value, int index) { @@ -192,7 +194,11 @@ static __always_inline __s32 sign_extend32(__u32 value, int index) /** * sign_extend64 - sign extend a 64-bit value using specified bit as sign-bit * @value: value to sign extend - * @index: 0 based bit index (0<=index<64) to sign bit + * @index: 0 based bit index (0 <= index < 64) to sign bit + * + * This is safe to use for 32-, 16- and 8-bit types as well. + * + * Return: 64-bit sign extended value */ static __always_inline __s64 sign_extend64(__u64 value, int index) { @@ -230,7 +236,7 @@ static inline int get_count_order_long(unsigned long l) /** * parity8 - get the parity of an u8 value - * @value: the value to be examined + * @val: the value to be examined * * Determine the parity of the u8 argument. * diff --git a/include/linux/bitrev.h b/include/linux/bitrev.h index d35b8ec1c485..11620a70e776 100644 --- a/include/linux/bitrev.h +++ b/include/linux/bitrev.h @@ -12,22 +12,10 @@ #define __bitrev8 __arch_bitrev8 #else -extern u8 const byte_rev_table[256]; -static inline u8 __bitrev8(u8 byte) -{ - return byte_rev_table[byte]; -} - -static inline u16 __bitrev16(u16 x) -{ - return (__bitrev8(x & 0xff) << 8) | __bitrev8(x >> 8); -} - -static inline u32 __bitrev32(u32 x) -{ - return (__bitrev16(x & 0xffff) << 16) | __bitrev16(x >> 16); -} - +#include <asm-generic/bitops/__bitrev.h> +#define __bitrev32 generic___bitrev32 +#define __bitrev16 generic___bitrev16 +#define __bitrev8 generic___bitrev8 #endif /* CONFIG_HAVE_ARCH_BITREVERSE */ #define __bitrev8x4(x) (__bitrev32(swab32(x))) diff --git a/include/linux/bits.h b/include/linux/bits.h index a40cc861b3a7..b7509f15718e 100644 --- a/include/linux/bits.h +++ b/include/linux/bits.h @@ -17,7 +17,7 @@ * position @h. For example * GENMASK_ULL(39, 21) gives us the 64bit vector 0x000000ffffe00000. */ -#if !defined(__ASSEMBLY__) +#if !defined(__ASSEMBLER__) /* * Missing asm support @@ -75,7 +75,7 @@ #define BIT_U32(nr) BIT_TYPE(u32, nr) #define BIT_U64(nr) BIT_TYPE(u64, nr) -#else /* defined(__ASSEMBLY__) */ +#else /* defined(__ASSEMBLER__) */ /* * BUILD_BUG_ON_ZERO is not available in h files included from asm files, @@ -84,6 +84,6 @@ #define GENMASK(h, l) __GENMASK(h, l) #define GENMASK_ULL(h, l) __GENMASK_ULL(h, l) -#endif /* !defined(__ASSEMBLY__) */ +#endif /* !defined(__ASSEMBLER__) */ #endif /* __LINUX_BITS_H */ diff --git a/include/linux/blk-cgroup.h b/include/linux/blk-cgroup.h index dd5841a42c33..58abde49f8c5 100644 --- a/include/linux/blk-cgroup.h +++ b/include/linux/blk-cgroup.h @@ -14,6 +14,8 @@ * Nauman Rafique <nauman@google.com> */ +#include <linux/atomic.h> +#include <linux/compiler.h> #include <linux/types.h> struct bio; @@ -24,10 +26,29 @@ struct gendisk; #ifdef CONFIG_BLK_CGROUP extern struct cgroup_subsys_state * const blkcg_root_css; +extern atomic_t blkcg_nr_congested; void blkcg_schedule_throttle(struct gendisk *disk, bool use_memdelay); void blkcg_maybe_throttle_current(void); -bool blk_cgroup_congested(void); +bool __blk_cgroup_congested(void); + +/** + * blk_cgroup_congested - is the current task in a throttled blkcg? + * + * Called from mm hot paths where the answer is almost always false, so keep + * that case to a load and a branch and only walk the hierarchy out of line + * when something in the system really is throttled. + * + * Return: %true if the current task's blkcg or any of its ancestors is + * throttled, %false otherwise. + */ +static inline bool blk_cgroup_congested(void) +{ + if (likely(!atomic_read(&blkcg_nr_congested))) + return false; + return __blk_cgroup_congested(); +} + void blkcg_pin_online(struct cgroup_subsys_state *blkcg_css); void blkcg_unpin_online(struct cgroup_subsys_state *blkcg_css); struct list_head *blkcg_get_cgwb_list(struct cgroup_subsys_state *css); diff --git a/include/linux/blk-crypto.h b/include/linux/blk-crypto.h index f7c3cb4a342f..938ff536838c 100644 --- a/include/linux/blk-crypto.h +++ b/include/linux/blk-crypto.h @@ -68,6 +68,15 @@ enum blk_crypto_key_type { */ #define BLK_CRYPTO_SW_SECRET_SIZE 32 +/* Flags for blk_crypto_config::flags: */ + +/* + * If set, inline encryption hardware will be used if available. + * If unset, CPU-based encryption will always be used (requires + * CONFIG_BLK_INLINE_ENCRYPTION_FALLBACK) + */ +#define BLK_CRYPTO_CFG_ALLOW_HW (1 << 0) + /** * struct blk_crypto_config - an inline encryption key's crypto configuration * @crypto_mode: encryption algorithm this key is for @@ -77,12 +86,14 @@ enum blk_crypto_key_type { * filesystem block size or the disk sector size. * @dun_bytes: the maximum number of bytes of DUN used when using this key * @key_type: the type of this key -- either raw or hardware-wrapped + * @flags: BLK_CRYPTO_CFG_* flags */ struct blk_crypto_config { enum blk_crypto_mode_num crypto_mode; unsigned int data_unit_size; unsigned int dun_bytes; enum blk_crypto_key_type key_type; + int flags; }; /** @@ -150,7 +161,7 @@ int blk_crypto_init_key(struct blk_crypto_key *blk_key, enum blk_crypto_key_type key_type, enum blk_crypto_mode_num crypto_mode, unsigned int dun_bytes, - unsigned int data_unit_size); + unsigned int data_unit_size, int flags); int blk_crypto_start_using_key(struct block_device *bdev, const struct blk_crypto_key *key); @@ -160,8 +171,6 @@ void blk_crypto_evict_key(struct block_device *bdev, bool blk_crypto_config_supported_natively(struct block_device *bdev, const struct blk_crypto_config *cfg); -bool blk_crypto_config_supported(struct block_device *bdev, - const struct blk_crypto_config *cfg); int blk_crypto_derive_sw_secret(struct block_device *bdev, const u8 *eph_key, size_t eph_key_size, diff --git a/include/linux/blk-integrity.h b/include/linux/blk-integrity.h index c15b1ac62765..b1b530613c34 100644 --- a/include/linux/blk-integrity.h +++ b/include/linux/blk-integrity.h @@ -8,17 +8,13 @@ struct request; -/* - * Maximum contiguous integrity buffer allocation. - */ -#define BLK_INTEGRITY_MAX_SIZE SZ_2M - enum blk_integrity_flags { BLK_INTEGRITY_NOVERIFY = 1 << 0, BLK_INTEGRITY_NOGENERATE = 1 << 1, BLK_INTEGRITY_DEVICE_CAPABLE = 1 << 2, BLK_INTEGRITY_REF_TAG = 1 << 3, BLK_INTEGRITY_STACKED = 1 << 4, + BLK_SPLIT_INTERVAL_CAPABLE = 1 << 5, }; const char *blk_integrity_profile_name(struct blk_integrity *bi); @@ -180,4 +176,27 @@ static inline struct bio_vec rq_integrity_vec(struct request *rq) } #endif /* CONFIG_BLK_DEV_INTEGRITY */ +enum bio_integrity_action { + BI_ACT_BUFFER = (1u << 0), /* allocate buffer */ + BI_ACT_CHECK = (1u << 1), /* generate / verify PI */ + BI_ACT_ZERO = (1u << 2), /* zero buffer */ +}; + +/** + * bio_integrity_action - return the integrity action needed for a bio + * @bio: bio to operate on + * + * Returns the mask of integrity actions (BI_ACT_*) that need to be performed + * for @bio. + */ +unsigned int __bio_integrity_action(struct bio *bio); +static inline unsigned int bio_integrity_action(struct bio *bio) +{ + if (!blk_get_integrity(bio->bi_bdev->bd_disk)) + return 0; + if (bio_integrity(bio)) + return 0; + return __bio_integrity_action(bio); +} + #endif /* _LINUX_BLK_INTEGRITY_H */ diff --git a/include/linux/blk-mq.h b/include/linux/blk-mq.h index 18a2388ba581..af878597afb8 100644 --- a/include/linux/blk-mq.h +++ b/include/linux/blk-mq.h @@ -428,7 +428,7 @@ struct blk_mq_hw_ctx { struct blk_mq_tags *sched_tags; /** @numa_node: NUMA node the storage adapter has been connected to. */ - unsigned int numa_node; + int numa_node; /** @queue_num: Index of this hardware queue. */ unsigned int queue_num; @@ -653,7 +653,7 @@ struct blk_mq_ops { * flush request. */ int (*init_request)(struct blk_mq_tag_set *set, struct request *, - unsigned int, unsigned int); + unsigned int, int); /** * @exit_request: Ditto for exit/teardown. */ @@ -1104,6 +1104,7 @@ struct req_iterator { /* * blk_rq_pos() : the current sector * blk_rq_bytes() : bytes left in the entire request + * blk_rq_has_data() : whether the request carries data * blk_rq_cur_bytes() : bytes left in the current segment * blk_rq_sectors() : sectors left in the entire request * blk_rq_cur_sectors() : sectors left in the current segment @@ -1119,6 +1120,14 @@ static inline unsigned int blk_rq_bytes(const struct request *rq) return rq->__data_len; } +static inline bool blk_rq_has_data(const struct request *rq) +{ + return blk_rq_bytes(rq) && + req_op(rq) != REQ_OP_DISCARD && + req_op(rq) != REQ_OP_SECURE_ERASE && + req_op(rq) != REQ_OP_WRITE_ZEROES; +} + static inline int blk_rq_cur_bytes(const struct request *rq) { if (!rq->bio) @@ -1243,4 +1252,44 @@ static inline int blk_rq_map_sg(struct request *rq, struct scatterlist *sglist) } void blk_dump_rq_flags(struct request *, char *); +/** + * blk_rq_passthrough_stats - check if this request should account stats + * @rq: request to check + * @q: the queue accumulating the stats + * + * Note, @q does not necessarily need to be the request_queue that provides + * @rq. + * + * Return: true if stats should be accounted. + */ +static inline bool blk_rq_passthrough_stats(struct request *rq, + struct request_queue *q) +{ + struct bio *bio = rq->bio; + + if (!blk_queue_passthrough_stat(q)) + return false; + + /* Requests without a bio do not transfer data. */ + if (!bio) + return false; + + /* + * Stats are accumulated in the bdev, so must have one attached to a + * bio to track stats. Most drivers do not set the bdev for passthrough + * requests, but nvme is one that will set it. + */ + if (!bio->bi_bdev) + return false; + + /* + * We don't know what a passthrough command does, but we know the + * payload size and data direction. Ensuring the size is aligned to the + * block size filters out most commands with payloads that don't + * represent sector access. + */ + if (blk_rq_bytes(rq) & (bdev_logical_block_size(bio->bi_bdev) - 1)) + return false; + return true; +} #endif /* BLK_MQ_H */ diff --git a/include/linux/blk_plug.h b/include/linux/blk_plug.h new file mode 100644 index 000000000000..2ac1265662ad --- /dev/null +++ b/include/linux/blk_plug.h @@ -0,0 +1,95 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LINUX_BLK_PLUG_H +#define _LINUX_BLK_PLUG_H + +#include <linux/sched.h> + +struct blk_plug_cb; +typedef void (*blk_plug_cb_fn)(struct blk_plug_cb *cb, bool from_schedule); + +struct rq_list { + struct request *head; + struct request *tail; +}; + +#ifdef CONFIG_BLOCK +/* + * blk_plug permits building a queue of related requests by holding the I/O + * fragments for a short period. This allows merging of sequential requests + * into single larger request. As the requests are moved from a per-task list to + * the device's request_queue in a batch, this results in improved scalability + * as the lock contention for request_queue lock is reduced. + * + * It is ok not to disable preemption when adding the request to the plug list + * or when attempting a merge. For details, please see schedule() where + * blk_flush_plug() is called. + */ +struct blk_plug { + struct rq_list mq_list; /* blk-mq requests */ + + /* if ios_left is > 1, we can batch tag/rq allocations */ + struct rq_list cached_rqs; + u64 cur_ktime; + unsigned short nr_ios; + + unsigned short rq_count; + + bool multiple_queues; + bool has_elevator; + + struct list_head cb_list; /* md requires an unplug callback */ +}; + +void blk_start_plug(struct blk_plug *); +void blk_start_plug_nr_ios(struct blk_plug *, unsigned short); +void blk_finish_plug(struct blk_plug *); + +void __blk_flush_plug(struct blk_plug *plug, bool from_schedule); +static inline void blk_flush_plug(struct blk_plug *plug, bool async) +{ + if (plug) + __blk_flush_plug(plug, async); +} + +static __always_inline void blk_plug_invalidate_ts(void) +{ + if (unlikely(current->flags & PF_BLOCK_TS)) { + current->plug->cur_ktime = 0; + current->flags &= ~PF_BLOCK_TS; + } +} + +struct blk_plug_cb { + struct list_head list; + blk_plug_cb_fn callback; + void *data; +}; + +struct blk_plug_cb *blk_check_plugged(blk_plug_cb_fn unplug, void *data, + int size); +#else /* CONFIG_BLOCK */ +struct blk_plug { +}; + +static inline void blk_start_plug(struct blk_plug *plug) +{ +} + +static inline void blk_start_plug_nr_ios(struct blk_plug *plug, + unsigned short nr_ios) +{ +} + +static inline void blk_finish_plug(struct blk_plug *plug) +{ +} + +static inline void blk_flush_plug(struct blk_plug *plug, bool async) +{ +} + +static inline void blk_plug_invalidate_ts(void) +{ +} +#endif /* CONFIG_BLOCK */ +#endif /* _LINUX_BLK_PLUG_H */ diff --git a/include/linux/blk_types.h b/include/linux/blk_types.h index 8808ee76e73c..98e21b4cbf32 100644 --- a/include/linux/blk_types.h +++ b/include/linux/blk_types.h @@ -66,7 +66,7 @@ struct block_device { int bd_holders; struct kobject *bd_holder_dir; - atomic_t bd_fsfreeze_count; /* number of freeze requests */ + atomic_t bd_fsfreeze_count; /* >0 freeze requests, <0 freeze deniers */ struct mutex bd_fsfreeze_mutex; /* serialize freeze/thaw */ struct partition_meta_info *bd_meta_info; @@ -322,6 +322,7 @@ enum { BIO_REMAPPED, BIO_ZONE_WRITE_PLUGGING, /* bio handled through zone write plugging */ BIO_EMULATES_ZONE_APPEND, /* bio emulates a zone append operation */ + BIO_COMPLETE_IN_TASK, /* complete bi_end_io() in task context */ BIO_FLAG_LAST }; diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index d463b9b5a0a5..4f7905c3412b 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -7,12 +7,14 @@ #include <linux/types.h> #include <linux/blk_types.h> +#include <linux/blk_plug.h> #include <linux/device.h> #include <linux/list.h> #include <linux/llist.h> #include <linux/minmax.h> #include <linux/timer.h> #include <linux/workqueue.h> +#include <linux/completion.h> #include <linux/wait.h> #include <linux/bio.h> #include <linux/gfp.h> @@ -20,7 +22,6 @@ #include <linux/rcupdate.h> #include <linux/percpu-refcount.h> #include <linux/blkzoned.h> -#include <linux/sched.h> #include <linux/sbitmap.h> #include <linux/uuid.h> #include <linux/xarray.h> @@ -38,6 +39,7 @@ struct blk_flush_queue; struct kiocb; struct pr_ops; struct rq_qos; +struct hd_geometry; struct blk_report_zones_args; struct blk_queue_stats; struct blk_stat_callback; @@ -124,8 +126,6 @@ struct blk_integrity { unsigned char pi_tuple_size; }; -typedef unsigned int __bitwise blk_mode_t; - /* open for reading */ #define BLK_OPEN_READ ((__force blk_mode_t)(1 << 0)) /* open for writing */ @@ -174,6 +174,7 @@ struct gendisk { #define GD_SUPPRESS_PART_SCAN 5 #define GD_OWNS_QUEUE 6 #define GD_ZONE_APPEND_USED 7 +#define GD_ERROR_INJECT 8 struct mutex open_mutex; /* open/close mutex */ unsigned open_partitions; /* number of open partitions */ @@ -200,10 +201,14 @@ struct gendisk { u8 __rcu *zones_cond; unsigned int zone_wplugs_hash_bits; atomic_t nr_zone_wplugs; - spinlock_t zone_wplugs_lock; + spinlock_t zone_wplugs_hash_lock; struct mempool *zone_wplugs_pool; struct hlist_head *zone_wplugs_hash; struct workqueue_struct *zone_wplugs_wq; + spinlock_t zone_wplugs_list_lock; + struct list_head zone_wplugs_list; + struct task_struct *zone_wplugs_worker; + struct completion zone_wplugs_worker_bio_done; #endif /* CONFIG_BLK_DEV_ZONED */ #if IS_ENABLED(CONFIG_CDROM) @@ -221,6 +226,11 @@ struct gendisk { */ struct blk_independent_access_ranges *ia_ranges; +#ifdef CONFIG_BLK_ERROR_INJECTION + struct mutex error_injection_lock; + struct list_head error_injection_list; +#endif + struct mutex rqos_state_mutex; /* rqos state change mutex */ }; @@ -502,7 +512,7 @@ struct request_queue { /* hw dispatch queues */ unsigned int nr_hw_queues; - struct blk_mq_hw_ctx * __rcu *queue_hw_ctx; + struct blk_mq_hw_ctx * __rcu *queue_hw_ctx __counted_by_ptr(nr_hw_queues); struct percpu_ref q_usage_counter; struct lock_class_key io_lock_cls_key; @@ -668,6 +678,7 @@ enum { QUEUE_FLAG_NO_ELV_SWITCH, /* can't switch elevator any more */ QUEUE_FLAG_QOS_ENABLED, /* qos is enabled */ QUEUE_FLAG_BIO_ISSUE_TIME, /* record bio->issue_time_ns */ + QUEUE_FLAG_ZONED_QD1_WRITES, /* Limit zoned devices writes to QD=1 */ QUEUE_FLAG_MAX }; @@ -707,6 +718,8 @@ void blk_queue_flag_clear(unsigned int flag, struct request_queue *q); test_bit(QUEUE_FLAG_DISABLE_WBT_DEF, &(q)->queue_flags) #define blk_queue_no_elv_switch(q) \ test_bit(QUEUE_FLAG_NO_ELV_SWITCH, &(q)->queue_flags) +#define blk_queue_zoned_qd1_writes(q) \ + test_bit(QUEUE_FLAG_ZONED_QD1_WRITES, &(q)->queue_flags) extern void blk_set_pm_only(struct request_queue *q); extern void blk_clear_pm_only(struct request_queue *q); @@ -1031,7 +1044,6 @@ extern const char *blk_op_str(enum req_op op); int blk_status_to_errno(blk_status_t status); blk_status_t errno_to_blk_status(int errno); -const char *blk_status_to_str(blk_status_t status); /* only poll the hardware once, don't continue until a completion was found */ #define BLK_POLL_ONESHOT (1 << 0) @@ -1084,15 +1096,17 @@ static inline unsigned int blk_boundary_sectors_left(sector_t offset, */ static inline struct queue_limits queue_limits_start_update(struct request_queue *q) + __acquires(&q->limits_lock) { mutex_lock(&q->limits_lock); return q->limits; } int queue_limits_commit_update_frozen(struct request_queue *q, - struct queue_limits *lim); + struct queue_limits *lim) __releases(&q->limits_lock); int queue_limits_commit_update(struct request_queue *q, - struct queue_limits *lim); -int queue_limits_set(struct request_queue *q, struct queue_limits *lim); + struct queue_limits *lim) __releases(&q->limits_lock); +int queue_limits_set(struct request_queue *q, struct queue_limits *lim) + __must_not_hold(&q->limits_lock); int blk_validate_limits(struct queue_limits *lim); /** @@ -1104,6 +1118,7 @@ int blk_validate_limits(struct queue_limits *lim); * starting update. */ static inline void queue_limits_cancel_update(struct request_queue *q) + __releases(&q->limits_lock) { mutex_unlock(&q->limits_lock); } @@ -1152,98 +1167,10 @@ extern void blk_put_queue(struct request_queue *); void blk_mark_disk_dead(struct gendisk *disk); -struct rq_list { - struct request *head; - struct request *tail; -}; - #ifdef CONFIG_BLOCK -/* - * blk_plug permits building a queue of related requests by holding the I/O - * fragments for a short period. This allows merging of sequential requests - * into single larger request. As the requests are moved from a per-task list to - * the device's request_queue in a batch, this results in improved scalability - * as the lock contention for request_queue lock is reduced. - * - * It is ok not to disable preemption when adding the request to the plug list - * or when attempting a merge. For details, please see schedule() where - * blk_flush_plug() is called. - */ -struct blk_plug { - struct rq_list mq_list; /* blk-mq requests */ - - /* if ios_left is > 1, we can batch tag/rq allocations */ - struct rq_list cached_rqs; - u64 cur_ktime; - unsigned short nr_ios; - - unsigned short rq_count; - - bool multiple_queues; - bool has_elevator; - - struct list_head cb_list; /* md requires an unplug callback */ -}; - -struct blk_plug_cb; -typedef void (*blk_plug_cb_fn)(struct blk_plug_cb *, bool); -struct blk_plug_cb { - struct list_head list; - blk_plug_cb_fn callback; - void *data; -}; -extern struct blk_plug_cb *blk_check_plugged(blk_plug_cb_fn unplug, - void *data, int size); -extern void blk_start_plug(struct blk_plug *); -extern void blk_start_plug_nr_ios(struct blk_plug *, unsigned short); -extern void blk_finish_plug(struct blk_plug *); - -void __blk_flush_plug(struct blk_plug *plug, bool from_schedule); -static inline void blk_flush_plug(struct blk_plug *plug, bool async) -{ - if (plug) - __blk_flush_plug(plug, async); -} - -/* - * tsk == current here - */ -static inline void blk_plug_invalidate_ts(struct task_struct *tsk) -{ - struct blk_plug *plug = tsk->plug; - - if (plug) - plug->cur_ktime = 0; - current->flags &= ~PF_BLOCK_TS; -} - int blkdev_issue_flush(struct block_device *bdev); long nr_blockdev_pages(void); #else /* CONFIG_BLOCK */ -struct blk_plug { -}; - -static inline void blk_start_plug_nr_ios(struct blk_plug *plug, - unsigned short nr_ios) -{ -} - -static inline void blk_start_plug(struct blk_plug *plug) -{ -} - -static inline void blk_finish_plug(struct blk_plug *plug) -{ -} - -static inline void blk_flush_plug(struct blk_plug *plug, bool async) -{ -} - -static inline void blk_plug_invalidate_ts(struct task_struct *tsk) -{ -} - static inline int blkdev_issue_flush(struct block_device *bdev) { return 0; @@ -1467,24 +1394,23 @@ static inline bool bdev_rot(struct block_device *bdev) return blk_queue_rot(bdev_get_queue(bdev)); } -static inline bool bdev_nonrot(struct block_device *bdev) +static inline bool bdev_synchronous(struct block_device *bdev) { - return !bdev_rot(bdev); + return bdev->bd_disk->queue->limits.features & BLK_FEAT_SYNCHRONOUS; } -static inline bool bdev_synchronous(struct block_device *bdev) +static inline bool bdev_has_integrity_csum(struct block_device *bdev) { - return bdev->bd_disk->queue->limits.features & BLK_FEAT_SYNCHRONOUS; + struct queue_limits *lim = bdev_limits(bdev); + + return IS_ENABLED(CONFIG_BLK_DEV_INTEGRITY) && + lim->integrity.csum_type != BLK_INTEGRITY_CSUM_NONE; } static inline bool bdev_stable_writes(struct block_device *bdev) { - struct request_queue *q = bdev_get_queue(bdev); - - if (IS_ENABLED(CONFIG_BLK_DEV_INTEGRITY) && - q->limits.integrity.csum_type != BLK_INTEGRITY_CSUM_NONE) - return true; - return q->limits.features & BLK_FEAT_STABLE_WRITES; + return bdev_has_integrity_csum(bdev) || + (bdev_limits(bdev)->features & BLK_FEAT_STABLE_WRITES); } static inline bool blk_queue_write_cache(struct request_queue *q) @@ -1736,32 +1662,29 @@ void blkdev_show(struct seq_file *seqf, off_t offset); #endif struct blk_holder_ops { - void (*mark_dead)(struct block_device *bdev, bool surprise); + void (*mark_dead)(struct block_device *bdev, bool surprise) + __releases(&bdev->bd_holder_lock); /* * Sync the file system mounted on the block device. */ - void (*sync)(struct block_device *bdev); + void (*sync)(struct block_device *bdev) + __releases(&bdev->bd_holder_lock); /* * Freeze the file system mounted on the block device. */ - int (*freeze)(struct block_device *bdev); + int (*freeze)(struct block_device *bdev) + __releases(&bdev->bd_holder_lock); /* * Thaw the file system mounted on the block device. */ - int (*thaw)(struct block_device *bdev); + int (*thaw)(struct block_device *bdev) + __releases(&bdev->bd_holder_lock); }; /* - * For filesystems using @fs_holder_ops, the @holder argument passed to - * helpers used to open and claim block devices via - * bd_prepare_to_claim() must point to a superblock. - */ -extern const struct blk_holder_ops fs_holder_ops; - -/* * Return the correct open flags for blkdev_get_by_* for super block flags * as stored in sb->s_flags. */ @@ -1821,7 +1744,10 @@ static inline int early_lookup_bdev(const char *pathname, dev_t *dev) int bdev_freeze(struct block_device *bdev); int bdev_thaw(struct block_device *bdev); +int bdev_deny_freeze(struct block_device *bdev); +void bdev_allow_freeze(struct block_device *bdev); void bdev_fput(struct file *bdev_file); +void bdev_yield_claim(struct file *bdev_file); struct io_comp_batch { struct rq_list req_list; @@ -1877,6 +1803,24 @@ static inline int bio_split_rw_at(struct bio *bio, return bio_split_io_at(bio, lim, segs, max_bytes, lim->dma_alignment); } +/* + * Maximum contiguous integrity buffer allocation. + */ +#define BLK_INTEGRITY_MAX_SIZE SZ_2M + +/* + * Maximum size of I/O that needs a block layer integrity buffer. Limited + * by the number of intervals for which we can fit the integrity buffer into + * the buffer size. Because the buffer is a single segment it is also limited + * by the maximum segment size. + */ +static inline unsigned int max_integrity_io_size(struct queue_limits *lim) +{ + return min_t(unsigned int, lim->max_segment_size, + (BLK_INTEGRITY_MAX_SIZE / lim->integrity.metadata_size) << + lim->integrity.interval_exp); +} + #define DEFINE_IO_COMP_BATCH(name) struct io_comp_batch name = { } #endif /* _LINUX_BLKDEV_H */ diff --git a/include/linux/bnge/hsi.h b/include/linux/bnge/hsi.h index 8ea13d5407ee..1f7bd96415a5 100644 --- a/include/linux/bnge/hsi.h +++ b/include/linux/bnge/hsi.h @@ -8317,8 +8317,7 @@ struct hwrm_ring_alloc_output { __le16 req_type; __le16 seq_id; __le16 resp_len; - __le16 ring_id; - __le16 logical_ring_id; + __le32 ring_id; u8 push_buffer_index; #define RING_ALLOC_RESP_PUSH_BUFFER_INDEX_PING_BUFFER 0x0UL #define RING_ALLOC_RESP_PUSH_BUFFER_INDEX_PONG_BUFFER 0x1UL @@ -8345,10 +8344,10 @@ struct hwrm_ring_free_input { u8 flags; #define RING_FREE_REQ_FLAGS_VIRTIO_RING_VALID 0x1UL #define RING_FREE_REQ_FLAGS_LAST RING_FREE_REQ_FLAGS_VIRTIO_RING_VALID - __le16 ring_id; + __le16 unused_1; __le32 prod_idx; __le32 opaque; - __le32 unused_1; + __le32 ring_id; }; /* hwrm_ring_free_output (size:128b/16B) */ diff --git a/include/linux/bnxt/hsi.h b/include/linux/bnxt/hsi.h index 74a6bf278d88..2f17dbd02185 100644 --- a/include/linux/bnxt/hsi.h +++ b/include/linux/bnxt/hsi.h @@ -191,6 +191,8 @@ struct cmd_nums { #define HWRM_PORT_EVENTS_LOG 0x67UL #define HWRM_VNIC_RSS_COS_LB_CTX_ALLOC 0x70UL #define HWRM_VNIC_RSS_COS_LB_CTX_FREE 0x71UL + #define HWRM_SCH_GRP_CFG 0x73UL + #define HWRM_SCH_GRP_QCFG 0x74UL #define HWRM_QUEUE_MPLS_QCAPS 0x80UL #define HWRM_QUEUE_MPLSTC2PRI_QCFG 0x81UL #define HWRM_QUEUE_MPLSTC2PRI_CFG 0x82UL @@ -4911,6 +4913,29 @@ struct hwrm_port_phy_qcfg_output { u8 valid; }; +/* hwrm_port_events_log_input (size:256b/32B) */ +struct hwrm_port_events_log_input { + __le16 req_type; + __le16 cmpl_ring; + __le16 seq_id; + __le16 target_id; + __le64 resp_addr; + __le64 host_dest_addr; + __le32 host_dest_addr_len; + u8 unused_0[4]; +}; + +/* hwrm_port_events_log_output (size:128b/16B) */ +struct hwrm_port_events_log_output { + __le16 error_code; + __le16 req_type; + __le16 seq_id; + __le16 resp_len; + __le16 data_len; + u8 unused_0[5]; + u8 valid; +}; + /* hwrm_port_mac_cfg_input (size:448b/56B) */ struct hwrm_port_mac_cfg_input { __le16 req_type; @@ -5418,6 +5443,40 @@ struct port_stats_ecn { __le64 mark_cnt_cos7; }; +/* port_stats_ext_pfc_adv (size:1536b/192B) */ +struct port_stats_ext_pfc_adv { + __le64 pfc_min_duration_time[8]; + __le64 pfc_max_duration_time[8]; + __le64 pfc_weighted_duration_time[8]; +}; + +/* hwrm_port_qstats_ext_pfc_adv_input (size:320b/40B) */ +struct hwrm_port_qstats_ext_pfc_adv_input { + __le16 req_type; + __le16 cmpl_ring; + __le16 seq_id; + __le16 target_id; + __le64 resp_addr; + __le16 port_id; + __le16 pfc_adv_stat_size; + u8 flags; + #define PORT_QSTATS_EXT_PFC_ADV_REQ_FLAGS_COUNTER_MASK 0x1UL + u8 unused_0[3]; + __le64 tx_pfc_adv_stat_host_addr; + __le64 rx_pfc_adv_stat_host_addr; +}; + +/* hwrm_port_qstats_ext_pfc_adv_output (size:128b/16B) */ +struct hwrm_port_qstats_ext_pfc_adv_output { + __le16 error_code; + __le16 req_type; + __le16 seq_id; + __le16 resp_len; + __le16 pfc_adv_stat_size; + u8 unused_0[5]; + u8 valid; +}; + /* hwrm_port_clr_stats_input (size:192b/24B) */ struct hwrm_port_clr_stats_input { __le16 req_type; @@ -6095,6 +6154,61 @@ struct hwrm_port_led_qcaps_output { u8 valid; }; +/* hwrm_port_prbs_test_input (size:384b/48B) */ +struct hwrm_port_prbs_test_input { + __le16 req_type; + __le16 cmpl_ring; + __le16 seq_id; + __le16 target_id; + __le64 resp_addr; + __le64 resp_data_addr; + __le16 data_len; + __le16 flags; + #define PORT_PRBS_TEST_REQ_FLAGS_INTERNAL 0x1UL + __le32 unused_1; + __le16 port_id; + __le16 poly; + #define PORT_PRBS_TEST_REQ_POLY_PRBS7 0x0UL + #define PORT_PRBS_TEST_REQ_POLY_PRBS9 0x1UL + #define PORT_PRBS_TEST_REQ_POLY_PRBS11 0x2UL + #define PORT_PRBS_TEST_REQ_POLY_PRBS15 0x3UL + #define PORT_PRBS_TEST_REQ_POLY_PRBS23 0x4UL + #define PORT_PRBS_TEST_REQ_POLY_PRBS31 0x5UL + #define PORT_PRBS_TEST_REQ_POLY_PRBS58 0x6UL + #define PORT_PRBS_TEST_REQ_POLY_PRBS49 0x7UL + #define PORT_PRBS_TEST_REQ_POLY_PRBS10 0x8UL + #define PORT_PRBS_TEST_REQ_POLY_PRBS20 0x9UL + #define PORT_PRBS_TEST_REQ_POLY_PRBS13 0xaUL + #define PORT_PRBS_TEST_REQ_POLY_INVALID 0xffUL + #define PORT_PRBS_TEST_REQ_POLY_LAST PORT_PRBS_TEST_REQ_POLY_INVALID + __le16 prbs_config; + #define PORT_PRBS_TEST_REQ_PRBS_CONFIG_START_STOP 0x1UL + #define PORT_PRBS_TEST_REQ_PRBS_CONFIG_TX_LANE_MAP_VALID 0x2UL + #define PORT_PRBS_TEST_REQ_PRBS_CONFIG_RX_LANE_MAP_VALID 0x4UL + #define PORT_PRBS_TEST_REQ_PRBS_CONFIG_FEC_STAT_T0_T7 0x8UL + #define PORT_PRBS_TEST_REQ_PRBS_CONFIG_FEC_STAT_T8_T15 0x10UL + #define PORT_PRBS_TEST_REQ_PRBS_CONFIG_T_CODE 0x20UL + __le16 timeout; + __le32 tx_lane_map; + __le32 rx_lane_map; +}; + +/* hwrm_port_prbs_test_output (size:128b/16B) */ +struct hwrm_port_prbs_test_output { + __le16 error_code; + __le16 req_type; + __le16 seq_id; + __le16 resp_len; + __le16 total_data_len; + u8 ber_format; + #define PORT_PRBS_TEST_RESP_BER_FORMAT_PRBS 0x0UL + #define PORT_PRBS_TEST_RESP_BER_FORMAT_FEC 0x1UL + #define PORT_PRBS_TEST_RESP_BER_FORMAT_LAST PORT_PRBS_TEST_RESP_BER_FORMAT_FEC + u8 unused_0; + u8 unused_1[3]; + u8 valid; +}; + /* hwrm_port_phy_fdrstat_input (size:192b/24B) */ struct hwrm_port_phy_fdrstat_input { __le16 req_type; @@ -6147,6 +6261,54 @@ struct hwrm_port_phy_fdrstat_cmd_err { u8 unused_0[7]; }; +/* hwrm_port_dsc_dump_input (size:320b/40B) */ +struct hwrm_port_dsc_dump_input { + __le16 req_type; + __le16 cmpl_ring; + __le16 seq_id; + __le16 target_id; + __le64 resp_addr; + __le64 resp_data_addr; + __le16 data_len; + __le16 unused_0; + __le32 data_offset; + __le16 port_id; + __le16 diag_level; + #define PORT_DSC_DUMP_REQ_DIAG_LEVEL_SRDS_DIAG_LANE 0x0UL + #define PORT_DSC_DUMP_REQ_DIAG_LEVEL_SRDS_DIAG_CORE 0x1UL + #define PORT_DSC_DUMP_REQ_DIAG_LEVEL_SRDS_DIAG_EVENT 0x2UL + #define PORT_DSC_DUMP_REQ_DIAG_LEVEL_SRDS_DIAG_EYE 0x3UL + #define PORT_DSC_DUMP_REQ_DIAG_LEVEL_SRDS_DIAG_REG_CORE 0x4UL + #define PORT_DSC_DUMP_REQ_DIAG_LEVEL_SRDS_DIAG_REG_LANE 0x5UL + #define PORT_DSC_DUMP_REQ_DIAG_LEVEL_SRDS_DIAG_UC_CORE 0x6UL + #define PORT_DSC_DUMP_REQ_DIAG_LEVEL_SRDS_DIAG_UC_LANE 0x7UL + #define PORT_DSC_DUMP_REQ_DIAG_LEVEL_SRDS_DIAG_LANE_DEBUG 0x8UL + #define PORT_DSC_DUMP_REQ_DIAG_LEVEL_SRDS_DIAG_BER_VERT 0x9UL + #define PORT_DSC_DUMP_REQ_DIAG_LEVEL_SRDS_DIAG_BER_HORZ 0xaUL + #define PORT_DSC_DUMP_REQ_DIAG_LEVEL_SRDS_DIAG_EVENT_SAFE 0xbUL + #define PORT_DSC_DUMP_REQ_DIAG_LEVEL_SRDS_DIAG_TIMESTAMP 0xcUL + #define PORT_DSC_DUMP_REQ_DIAG_LEVEL_LAST PORT_DSC_DUMP_REQ_DIAG_LEVEL_SRDS_DIAG_TIMESTAMP + __le16 lane_number; + __le16 dsc_dump_config; + #define PORT_DSC_DUMP_REQ_DSC_DUMP_CONFIG_START_RETRIEVE 0x1UL + #define PORT_DSC_DUMP_REQ_DSC_DUMP_CONFIG_BIG_BUFFER 0x2UL + #define PORT_DSC_DUMP_REQ_DSC_DUMP_CONFIG_DEFER_CLOSE 0x4UL +}; + +/* hwrm_port_dsc_dump_output (size:128b/16B) */ +struct hwrm_port_dsc_dump_output { + __le16 error_code; + __le16 req_type; + __le16 seq_id; + __le16 resp_len; + __le16 total_data_len; + __le16 total_data_len_high; + u8 unused_1[2]; + u8 flags; + #define PORT_DSC_DUMP_RESP_FLAGS_BIG_BUFFER 0x1UL + u8 valid; +}; + /* hwrm_port_mac_qcaps_input (size:192b/24B) */ struct hwrm_port_mac_qcaps_input { __le16 req_type; @@ -9559,6 +9721,62 @@ struct hwrm_stat_generic_qstats_output { u8 valid; }; +/* hwrm_stat_query_roce_stats_input (size:256b/32B) */ +struct hwrm_stat_query_roce_stats_input { + __le16 req_type; + __le16 cmpl_ring; + __le16 seq_id; + __le16 target_id; + __le64 resp_addr; + __le16 roce_stat_size; + u8 flags; + #define STAT_QUERY_ROCE_STATS_REQ_FLAGS_PORT_AGGREGATED 0x1UL + u8 port_id; + u8 unused_0[4]; + __le64 roce_stat_host_addr; +}; + +/* hwrm_stat_query_roce_stats_output (size:128b/16B) */ +struct hwrm_stat_query_roce_stats_output { + __le16 error_code; + __le16 req_type; + __le16 seq_id; + __le16 resp_len; + __le16 roce_stat_size; + u8 flags; + #define STAT_QUERY_ROCE_STATS_RESP_FLAGS_PORT_AGGREGATED 0x1UL + u8 unused_0[4]; + u8 valid; +}; + +/* hwrm_stat_query_roce_stats_ext_input (size:256b/32B) */ +struct hwrm_stat_query_roce_stats_ext_input { + __le16 req_type; + __le16 cmpl_ring; + __le16 seq_id; + __le16 target_id; + __le64 resp_addr; + __le16 roce_stat_size; + u8 flags; + #define STAT_QUERY_ROCE_STATS_EXT_REQ_FLAGS_PORT_AGGREGATED 0x1UL + u8 port_id; + u8 unused_0[4]; + __le64 roce_stat_host_addr; +}; + +/* hwrm_stat_query_roce_stats_ext_output (size:128b/16B) */ +struct hwrm_stat_query_roce_stats_ext_output { + __le16 error_code; + __le16 req_type; + __le16 seq_id; + __le16 resp_len; + __le16 roce_stat_size; + u8 flags; + #define STAT_QUERY_ROCE_STATS_EXT_RESP_FLAGS_PORT_AGGREGATED 0x1UL + u8 unused_0[4]; + u8 valid; +}; + /* generic_sw_hw_stats (size:1472b/184B) */ struct generic_sw_hw_stats { __le64 pcie_statistics_tx_tlp; @@ -10207,6 +10425,57 @@ struct hwrm_dbg_read_direct_output { u8 valid; }; +/* hwrm_dbg_read_indirect_input (size:640b/80B) */ +struct hwrm_dbg_read_indirect_input { + __le16 req_type; + __le16 cmpl_ring; + __le16 seq_id; + __le16 target_id; + __le64 resp_addr; + __le64 host_dest_addr; + __le32 host_dest_addr_len; + u8 indirect_access_type; + #define DBG_READ_INDIRECT_REQ_INDIRECT_ACCESS_TYPE_TE_MGMT_FILTERS_L2 0x0UL + #define DBG_READ_INDIRECT_REQ_INDIRECT_ACCESS_TYPE_TE_MGMT_FILTERS_L3L4 0x1UL + #define DBG_READ_INDIRECT_REQ_INDIRECT_ACCESS_TYPE_RE_MGMT_FILTERS_L2 0x2UL + #define DBG_READ_INDIRECT_REQ_INDIRECT_ACCESS_TYPE_RE_MGMT_FILTERS_L3L4 0x3UL + #define DBG_READ_INDIRECT_REQ_INDIRECT_ACCESS_TYPE_STAT_CTXS 0x4UL + #define DBG_READ_INDIRECT_REQ_INDIRECT_ACCESS_TYPE_CFA_TX_L2_TCAM 0x5UL + #define DBG_READ_INDIRECT_REQ_INDIRECT_ACCESS_TYPE_CFA_RX_L2_TCAM 0x6UL + #define DBG_READ_INDIRECT_REQ_INDIRECT_ACCESS_TYPE_CFA_TX_IPV6_SUBNET_TCAM 0x7UL + #define DBG_READ_INDIRECT_REQ_INDIRECT_ACCESS_TYPE_CFA_RX_IPV6_SUBNET_TCAM 0x8UL + #define DBG_READ_INDIRECT_REQ_INDIRECT_ACCESS_TYPE_CFA_TX_SRC_PROPERTIES_TCAM 0x9UL + #define DBG_READ_INDIRECT_REQ_INDIRECT_ACCESS_TYPE_CFA_RX_SRC_PROPERTIES_TCAM 0xaUL + #define DBG_READ_INDIRECT_REQ_INDIRECT_ACCESS_TYPE_CFA_VEB_LOOKUP_TCAM 0xbUL + #define DBG_READ_INDIRECT_REQ_INDIRECT_ACCESS_TYPE_CFA_TX_PROFILE_LOOKUP_TCAM 0xcUL + #define DBG_READ_INDIRECT_REQ_INDIRECT_ACCESS_TYPE_CFA_RX_PROFILE_LOOKUP_TCAM 0xdUL + #define DBG_READ_INDIRECT_REQ_INDIRECT_ACCESS_TYPE_CFA_TX_LOOKUP_TCAM 0xeUL + #define DBG_READ_INDIRECT_REQ_INDIRECT_ACCESS_TYPE_CFA_RX_LOOKUP_TCAM 0xfUL + #define DBG_READ_INDIRECT_REQ_INDIRECT_ACCESS_TYPE_MHB 0x10UL + #define DBG_READ_INDIRECT_REQ_INDIRECT_ACCESS_TYPE_PCIE_GBL 0x11UL + #define DBG_READ_INDIRECT_REQ_INDIRECT_ACCESS_TYPE_MULTI_HOST_SOC 0x12UL + #define DBG_READ_INDIRECT_REQ_INDIRECT_ACCESS_TYPE_PCIE_PRIVATE 0x13UL + #define DBG_READ_INDIRECT_REQ_INDIRECT_ACCESS_TYPE_HOST_DMA 0x14UL + #define DBG_READ_INDIRECT_REQ_INDIRECT_ACCESS_TYPE_SOC_ELOG 0x15UL + #define DBG_READ_INDIRECT_REQ_INDIRECT_ACCESS_TYPE_CTX 0x16UL + #define DBG_READ_INDIRECT_REQ_INDIRECT_ACCESS_TYPE_STATS 0x17UL + #define DBG_READ_INDIRECT_REQ_INDIRECT_ACCESS_TYPE_LAST DBG_READ_INDIRECT_REQ_INDIRECT_ACCESS_TYPE_STATS + u8 unused_0[3]; + __le32 start_index; + __le32 num_of_entries; + __le32 opaque[10]; +}; + +/* hwrm_dbg_read_indirect_output (size:128b/16B) */ +struct hwrm_dbg_read_indirect_output { + __le16 error_code; + __le16 req_type; + __le16 seq_id; + __le16 resp_len; + u8 unused_0[7]; + u8 valid; +}; + /* hwrm_dbg_qcaps_input (size:192b/24B) */ struct hwrm_dbg_qcaps_input { __le16 req_type; @@ -10518,6 +10787,154 @@ struct hwrm_dbg_log_buffer_flush_output { u8 valid; }; +/* hwrm_dbg_serdes_test_input (size:320b/40B) */ +struct hwrm_dbg_serdes_test_input { + __le16 req_type; + __le16 cmpl_ring; + __le16 seq_id; + __le16 target_id; + __le64 resp_addr; + __le64 resp_data_addr; + __le32 resp_data_offset; + __le16 data_len; + u8 flags; + #define DBG_SERDES_TEST_REQ_FLAGS_UNUSED_TEST_MASK 0x7UL + #define DBG_SERDES_TEST_REQ_FLAGS_UNUSED_TEST_SFT 0 + #define DBG_SERDES_TEST_REQ_FLAGS_EYE_PROJECTION 0x8UL + #define DBG_SERDES_TEST_REQ_FLAGS_PCIE_SERDES_TEST 0x10UL + #define DBG_SERDES_TEST_REQ_FLAGS_ETHERNET_SERDES_TEST 0x20UL + u8 options; + #define DBG_SERDES_TEST_REQ_OPTIONS_LANE_NO_MASK 0xfUL + #define DBG_SERDES_TEST_REQ_OPTIONS_LANE_NO_SFT 0 + #define DBG_SERDES_TEST_REQ_OPTIONS_DIRECTION 0x10UL + #define DBG_SERDES_TEST_REQ_OPTIONS_DIRECTION_HORIZONTAL (0x0UL << 4) + #define DBG_SERDES_TEST_REQ_OPTIONS_DIRECTION_VERTICAL (0x1UL << 4) + #define DBG_SERDES_TEST_REQ_OPTIONS_DIRECTION_LAST DBG_SERDES_TEST_REQ_OPTIONS_DIRECTION_VERTICAL + #define DBG_SERDES_TEST_REQ_OPTIONS_PROJ_TYPE 0x20UL + #define DBG_SERDES_TEST_REQ_OPTIONS_PROJ_TYPE_LEFT_TOP (0x0UL << 5) + #define DBG_SERDES_TEST_REQ_OPTIONS_PROJ_TYPE_RIGHT_BOTTOM (0x1UL << 5) + #define DBG_SERDES_TEST_REQ_OPTIONS_PROJ_TYPE_LAST DBG_SERDES_TEST_REQ_OPTIONS_PROJ_TYPE_RIGHT_BOTTOM + #define DBG_SERDES_TEST_REQ_OPTIONS_RSVD_MASK 0xc0UL + #define DBG_SERDES_TEST_REQ_OPTIONS_RSVD_SFT 6 + u8 targetBER; + #define DBG_SERDES_TEST_REQ_TARGETBER_BER_1E8 0x0UL + #define DBG_SERDES_TEST_REQ_TARGETBER_BER_1E9 0x1UL + #define DBG_SERDES_TEST_REQ_TARGETBER_BER_1E10 0x2UL + #define DBG_SERDES_TEST_REQ_TARGETBER_BER_1E11 0x3UL + #define DBG_SERDES_TEST_REQ_TARGETBER_BER_1E12 0x4UL + #define DBG_SERDES_TEST_REQ_TARGETBER_LAST DBG_SERDES_TEST_REQ_TARGETBER_BER_1E12 + u8 action; + #define DBG_SERDES_TEST_REQ_ACTION_SYNCHRONOUS 0x0UL + #define DBG_SERDES_TEST_REQ_ACTION_START 0x1UL + #define DBG_SERDES_TEST_REQ_ACTION_PROGRESS 0x2UL + #define DBG_SERDES_TEST_REQ_ACTION_STOP 0x3UL + #define DBG_SERDES_TEST_REQ_ACTION_LAST DBG_SERDES_TEST_REQ_ACTION_STOP + u8 unused[6]; +}; + +/* hwrm_dbg_serdes_test_output (size:192b/24B) */ +struct hwrm_dbg_serdes_test_output { + __le16 error_code; + __le16 req_type; + __le16 seq_id; + __le16 resp_len; + __le16 total_data_len; + __le16 copied_data_len; + __le16 progress_percent; + __le16 timeout; + u8 flags; + #define DBG_SERDES_TEST_RESP_FLAGS_BIT_COUNT_TYPE 0x1UL + #define DBG_SERDES_TEST_RESP_FLAGS_BIT_COUNT_TYPE_BIT_COUNT_TOTAL (0x0UL << 0) + #define DBG_SERDES_TEST_RESP_FLAGS_BIT_COUNT_TYPE_BIT_COUNT_POW2 (0x1UL << 0) + #define DBG_SERDES_TEST_RESP_FLAGS_BIT_COUNT_TYPE_LAST DBG_SERDES_TEST_RESP_FLAGS_BIT_COUNT_TYPE_BIT_COUNT_POW2 + #define DBG_SERDES_TEST_RESP_FLAGS_RSVD_MASK 0xfeUL + #define DBG_SERDES_TEST_RESP_FLAGS_RSVD_SFT 1 + u8 unused_0; + __le16 hdr_size; + u8 unused_1[3]; + u8 valid; +}; + +/* hwrm_dbg_ptrace_input (size:320b/40B) */ +struct hwrm_dbg_ptrace_input { + __le16 req_type; + __le16 cmpl_ring; + __le16 seq_id; + __le16 target_id; + __le64 resp_addr; + __le32 pdi_cmd_buf_addr[2]; + __le32 pdi_resp_buf_addr[2]; + __le32 pdi_req_buf_len; + __le16 seq_no; + __le16 flags; + #define DBG_PTRACE_REQ_FLAGS_SELECT_IN 0x1UL + #define DBG_PTRACE_REQ_FLAGS_SELECT_OUT 0x2UL + #define DBG_PTRACE_REQ_FLAGS_GLOBAL_START 0x4UL + #define DBG_PTRACE_REQ_FLAGS_GLOBAL_STOP 0x8UL +}; + +/* hwrm_dbg_ptrace_output (size:128b/16B) */ +struct hwrm_dbg_ptrace_output { + __le16 error_code; + __le16 req_type; + __le16 seq_id; + __le16 resp_len; + __le16 flags; + #define DBG_PTRACE_RESP_FLAGS_MORE 0x1UL + __le16 data_len; + u8 unused_0[3]; + u8 valid; +}; + +/* hwrm_dbg_token_cfg_input (size:256b/32B) */ +struct hwrm_dbg_token_cfg_input { + __le16 req_type; + __le16 cmpl_ring; + __le16 seq_id; + __le16 target_id; + __le64 resp_addr; + u8 flags; + #define DBG_TOKEN_CFG_REQ_FLAGS_ENABLE 0x1UL + u8 unused_0[3]; + __le32 dbg_token_len; + __le64 host_src_addr; +}; + +/* hwrm_dbg_token_cfg_output (size:128b/16B) */ +struct hwrm_dbg_token_cfg_output { + __le16 error_code; + __le16 req_type; + __le16 seq_id; + __le16 resp_len; + u8 unused_0[7]; + u8 valid; +}; + +/* hwrm_nvm_raw_write_blk_input (size:320b/40B) */ +struct hwrm_nvm_raw_write_blk_input { + __le16 req_type; + __le16 cmpl_ring; + __le16 seq_id; + __le16 target_id; + __le64 resp_addr; + __le64 host_src_addr; + __le32 dest_addr; + __le32 len; + u8 flags; + #define NVM_RAW_WRITE_BLK_REQ_FLAGS_SECURITY_SOC_NVM 0x1UL + u8 unused_0[7]; +}; + +/* hwrm_nvm_raw_write_blk_output (size:128b/16B) */ +struct hwrm_nvm_raw_write_blk_output { + __le16 error_code; + __le16 req_type; + __le16 seq_id; + __le16 resp_len; + u8 unused_0[7]; + u8 valid; +}; + /* hwrm_nvm_read_input (size:320b/40B) */ struct hwrm_nvm_read_input { __le16 req_type; @@ -10543,6 +10960,31 @@ struct hwrm_nvm_read_output { u8 valid; }; +/* hwrm_nvm_raw_dump_input (size:320b/40B) */ +struct hwrm_nvm_raw_dump_input { + __le16 req_type; + __le16 cmpl_ring; + __le16 seq_id; + __le16 target_id; + __le64 resp_addr; + __le64 host_dest_addr; + __le32 offset; + __le32 len; + u8 flags; + #define NVM_RAW_DUMP_REQ_FLAGS_SECURITY_SOC_NVM 0x1UL + u8 unused_0[7]; +}; + +/* hwrm_nvm_raw_dump_output (size:128b/16B) */ +struct hwrm_nvm_raw_dump_output { + __le16 error_code; + __le16 req_type; + __le16 seq_id; + __le16 resp_len; + u8 unused_0[7]; + u8 valid; +}; + /* hwrm_nvm_get_dir_entries_input (size:192b/24B) */ struct hwrm_nvm_get_dir_entries_input { __le16 req_type; @@ -11166,6 +11608,149 @@ struct hwrm_selftest_irq_output { u8 valid; }; +/* hwrm_selftest_retrieve_serdes_data_input (size:320b/40B) */ +struct hwrm_selftest_retrieve_serdes_data_input { + __le16 req_type; + __le16 cmpl_ring; + __le16 seq_id; + __le16 target_id; + __le64 resp_addr; + __le64 resp_data_addr; + __le32 resp_data_offset; + __le16 data_len; + u8 flags; + #define SELFTEST_RETRIEVE_SERDES_DATA_REQ_FLAGS_UNUSED_TEST_MASK 0x7UL + #define SELFTEST_RETRIEVE_SERDES_DATA_REQ_FLAGS_UNUSED_TEST_SFT 0 + #define SELFTEST_RETRIEVE_SERDES_DATA_REQ_FLAGS_EYE_PROJECTION 0x8UL + #define SELFTEST_RETRIEVE_SERDES_DATA_REQ_FLAGS_PCIE_SERDES_TEST 0x10UL + #define SELFTEST_RETRIEVE_SERDES_DATA_REQ_FLAGS_ETHERNET_SERDES_TEST 0x20UL + u8 options; + #define SELFTEST_RETRIEVE_SERDES_DATA_REQ_OPTIONS_PCIE_LANE_NO_MASK 0xfUL + #define SELFTEST_RETRIEVE_SERDES_DATA_REQ_OPTIONS_PCIE_LANE_NO_SFT 0 + #define SELFTEST_RETRIEVE_SERDES_DATA_REQ_OPTIONS_DIRECTION 0x10UL + #define SELFTEST_RETRIEVE_SERDES_DATA_REQ_OPTIONS_DIRECTION_HORIZONTAL (0x0UL << 4) + #define SELFTEST_RETRIEVE_SERDES_DATA_REQ_OPTIONS_DIRECTION_VERTICAL (0x1UL << 4) + #define SELFTEST_RETRIEVE_SERDES_DATA_REQ_OPTIONS_DIRECTION_LAST SELFTEST_RETRIEVE_SERDES_DATA_REQ_OPTIONS_DIRECTION_VERTICAL + #define SELFTEST_RETRIEVE_SERDES_DATA_REQ_OPTIONS_PROJ_TYPE 0x20UL + #define SELFTEST_RETRIEVE_SERDES_DATA_REQ_OPTIONS_PROJ_TYPE_LEFT_TOP (0x0UL << 5) + #define SELFTEST_RETRIEVE_SERDES_DATA_REQ_OPTIONS_PROJ_TYPE_RIGHT_BOTTOM (0x1UL << 5) + #define SELFTEST_RETRIEVE_SERDES_DATA_REQ_OPTIONS_PROJ_TYPE_LAST SELFTEST_RETRIEVE_SERDES_DATA_REQ_OPTIONS_PROJ_TYPE_RIGHT_BOTTOM + #define SELFTEST_RETRIEVE_SERDES_DATA_REQ_OPTIONS_RSVD_MASK 0xc0UL + #define SELFTEST_RETRIEVE_SERDES_DATA_REQ_OPTIONS_RSVD_SFT 6 + u8 targetBER; + #define SELFTEST_RETRIEVE_SERDES_DATA_REQ_TARGETBER_BER_1E8 0x0UL + #define SELFTEST_RETRIEVE_SERDES_DATA_REQ_TARGETBER_BER_1E9 0x1UL + #define SELFTEST_RETRIEVE_SERDES_DATA_REQ_TARGETBER_BER_1E10 0x2UL + #define SELFTEST_RETRIEVE_SERDES_DATA_REQ_TARGETBER_BER_1E11 0x3UL + #define SELFTEST_RETRIEVE_SERDES_DATA_REQ_TARGETBER_BER_1E12 0x4UL + #define SELFTEST_RETRIEVE_SERDES_DATA_REQ_TARGETBER_LAST SELFTEST_RETRIEVE_SERDES_DATA_REQ_TARGETBER_BER_1E12 + u8 action; + #define SELFTEST_RETRIEVE_SERDES_DATA_REQ_ACTION_SYNCHRONOUS 0x0UL + #define SELFTEST_RETRIEVE_SERDES_DATA_REQ_ACTION_START 0x1UL + #define SELFTEST_RETRIEVE_SERDES_DATA_REQ_ACTION_PROGRESS 0x2UL + #define SELFTEST_RETRIEVE_SERDES_DATA_REQ_ACTION_STOP 0x3UL + #define SELFTEST_RETRIEVE_SERDES_DATA_REQ_ACTION_LAST SELFTEST_RETRIEVE_SERDES_DATA_REQ_ACTION_STOP + u8 unused[6]; +}; + +/* hwrm_selftest_retrieve_serdes_data_output (size:192b/24B) */ +struct hwrm_selftest_retrieve_serdes_data_output { + __le16 error_code; + __le16 req_type; + __le16 seq_id; + __le16 resp_len; + __le16 total_data_len; + __le16 copied_data_len; + __le16 progress_percent; + __le16 timeout; + u8 flags; + #define SELFTEST_RETRIEVE_SERDES_DATA_RESP_FLAGS_BIT_COUNT_TYPE 0x1UL + #define SELFTEST_RETRIEVE_SERDES_DATA_RESP_FLAGS_BIT_COUNT_TYPE_BIT_COUNT_TOTAL (0x0UL << 0) + #define SELFTEST_RETRIEVE_SERDES_DATA_RESP_FLAGS_BIT_COUNT_TYPE_BIT_COUNT_POW2 (0x1UL << 0) + #define SELFTEST_RETRIEVE_SERDES_DATA_RESP_FLAGS_BIT_COUNT_TYPE_LAST SELFTEST_RETRIEVE_SERDES_DATA_RESP_FLAGS_BIT_COUNT_TYPE_BIT_COUNT_POW2 + #define SELFTEST_RETRIEVE_SERDES_DATA_RESP_FLAGS_RSVD_MASK 0xfeUL + #define SELFTEST_RETRIEVE_SERDES_DATA_RESP_FLAGS_RSVD_SFT 1 + u8 unused_0; + __le16 hdr_size; + u8 unused_1[3]; + u8 valid; +}; + +/* hwrm_sch_grp_cfg_input (size:704b/88B) */ +struct hwrm_sch_grp_cfg_input { + __le16 req_type; + __le16 cmpl_ring; + __le16 seq_id; + __le16 target_id; + __le64 resp_addr; + __le16 sch_grp_id; + __le16 num_fids; + __le32 enables; + #define SCH_GRP_CFG_REQ_ENABLES_QUEUE_ID0_VALID 0x1UL + #define SCH_GRP_CFG_REQ_ENABLES_QUEUE_ID1_VALID 0x2UL + #define SCH_GRP_CFG_REQ_ENABLES_QUEUE_ID2_VALID 0x4UL + #define SCH_GRP_CFG_REQ_ENABLES_QUEUEID3_VALID 0x8UL + #define SCH_GRP_CFG_REQ_ENABLES_QUEUE_ID4_VALID 0x10UL + #define SCH_GRP_CFG_REQ_ENABLES_QUEUE_ID5_VALID 0x20UL + #define SCH_GRP_CFG_REQ_ENABLES_QUEUE_ID6_VALID 0x40UL + #define SCH_GRP_CFG_REQ_ENABLES_QUEUE_ID7_VALID 0x80UL + #define SCH_GRP_CFG_REQ_ENABLES_MAX_BW 0x100UL + #define SCH_GRP_CFG_REQ_ENABLES_FID_MAP 0x200UL + __le64 fid_table_addr; + __le32 max_bw; + u8 unused_0[4]; + u8 queue_id[8]; + u8 queue_tsa_assign[8]; + #define SCH_GRP_CFG_REQ_QUEUE_TSA_ASSIGN_SP 0x0UL + #define SCH_GRP_CFG_REQ_QUEUE_TSA_ASSIGN_ETS 0x1UL + #define SCH_GRP_CFG_REQ_QUEUE_TSA_ASSIGN_LAST SCH_GRP_CFG_REQ_QUEUE_TSA_ASSIGN_ETS + __le32 queue_min_bw_percent[8]; +}; + +/* hwrm_sch_grp_cfg_output (size:128b/16B) */ +struct hwrm_sch_grp_cfg_output { + __le16 error_code; + __le16 req_type; + __le16 seq_id; + __le16 resp_len; + u8 unused_0[7]; + u8 valid; +}; + +/* hwrm_sch_grp_qcfg_input (size:256b/32B) */ +struct hwrm_sch_grp_qcfg_input { + __le16 req_type; + __le16 cmpl_ring; + __le16 seq_id; + __le16 target_id; + __le64 resp_addr; + __le16 sch_grp_id; + __le16 fid_table_len; + u8 unused_0[4]; + __le64 fid_table_addr; +}; + +/* hwrm_sch_grp_qcfg_output (size:576b/72B) */ +struct hwrm_sch_grp_qcfg_output { + __le16 error_code; + __le16 req_type; + __le16 seq_id; + __le16 resp_len; + __le32 max_bw; + u8 unused_0[2]; + __le16 num_fids; + u8 queue_id[8]; + #define SCH_GRP_QCFG_RESP_QUEUE_ID_INVALID_QUEUE_ID 0xffUL + #define SCH_GRP_QCFG_RESP_QUEUE_ID_LAST SCH_GRP_QCFG_RESP_QUEUE_ID_INVALID_QUEUE_ID + u8 queue_tsa_assign[8]; + #define SCH_GRP_QCFG_RESP_QUEUE_TSA_ASSIGN_SP 0x0UL + #define SCH_GRP_QCFG_RESP_QUEUE_TSA_ASSIGN_ETS 0x1UL + #define SCH_GRP_QCFG_RESP_QUEUE_TSA_ASSIGN_LAST SCH_GRP_QCFG_RESP_QUEUE_TSA_ASSIGN_ETS + __le32 queue_min_bw_percent[8]; + u8 unused_1[7]; + u8 valid; +}; + /* dbc_dbc (size:64b/8B) */ struct dbc_dbc { __le32 index; diff --git a/include/linux/bnxt/ulp.h b/include/linux/bnxt/ulp.h new file mode 100644 index 000000000000..0851ad3394b0 --- /dev/null +++ b/include/linux/bnxt/ulp.h @@ -0,0 +1,144 @@ +/* Broadcom NetXtreme-C/E network driver. + * + * Copyright (c) 2016-2018 Broadcom Limited + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation. + */ + +#ifndef BNXT_ULP_H +#define BNXT_ULP_H + +#include <linux/auxiliary_bus.h> + +#define BNXT_MIN_ROCE_CP_RINGS 2 +#define BNXT_MIN_ROCE_STAT_CTXS 1 + +#define BNXT_MAX_ROCE_MSIX_VF 2 +#define BNXT_MAX_ROCE_MSIX_NPAR_PF 5 +#define BNXT_MAX_ROCE_MSIX 64 + +struct hwrm_async_event_cmpl; +struct bnxt; + +enum bnxt_auxdev_type { + BNXT_AUXDEV_RDMA = 0, + BNXT_AUXDEV_FWCTL, + __BNXT_AUXDEV_MAX +}; + +struct bnxt_aux_priv { + struct auxiliary_device aux_dev; + struct bnxt_en_dev *edev; + int id; +}; + +struct bnxt_msix_entry { + u32 vector; + u32 ring_idx; + u32 db_offset; +}; + +struct bnxt_ulp_ops { + /* async_notifier() cannot sleep (in BH context) */ + void (*ulp_async_notifier)(void *, struct hwrm_async_event_cmpl *); + void (*ulp_irq_stop)(void *, bool); + void (*ulp_irq_restart)(void *, struct bnxt_msix_entry *); +}; + +struct bnxt_fw_msg { + void *msg; + int msg_len; + void *resp; + int resp_max_len; + int timeout; +}; + +struct bnxt_ulp { + void *handle; + struct bnxt_ulp_ops __rcu *ulp_ops; + unsigned long *async_events_bmap; + u16 max_async_event_id; + u16 msix_requested; +}; + +struct bnxt_en_dev { + struct net_device *net; + struct pci_dev *pdev; + struct bnxt_msix_entry msix_entries[BNXT_MAX_ROCE_MSIX]; + u32 flags; + #define BNXT_EN_FLAG_ROCEV1_CAP 0x1 + #define BNXT_EN_FLAG_ROCEV2_CAP 0x2 + #define BNXT_EN_FLAG_ROCE_CAP (BNXT_EN_FLAG_ROCEV1_CAP | \ + BNXT_EN_FLAG_ROCEV2_CAP) + #define BNXT_EN_FLAG_ULP_STOPPED 0x8 + #define BNXT_EN_FLAG_VF 0x10 +#define BNXT_EN_VF(edev) ((edev)->flags & BNXT_EN_FLAG_VF) + #define BNXT_EN_FLAG_ROCE_VF_RES_MGMT 0x20 + #define BNXT_EN_FLAG_SW_RES_LMT 0x40 +#define BNXT_EN_SW_RES_LMT(edev) ((edev)->flags & BNXT_EN_FLAG_SW_RES_LMT) + + struct bnxt_ulp *ulp_tbl; + int l2_db_size; /* Doorbell BAR size in + * bytes mapped by L2 + * driver. + */ + int l2_db_size_nc; /* Doorbell BAR size in + * bytes mapped as non- + * cacheable. + */ + int l2_db_offset; /* Doorbell offset in + * bytes within + * l2_db_size_nc. + */ + u16 chip_num; + u16 hw_ring_stats_size; + u16 pf_port_id; + unsigned long en_state; /* Could be checked in + * RoCE driver suspend + * mode only. Will be + * updated in resume. + */ + void __iomem *bar0; + + u16 ulp_num_msix_vec; + u16 ulp_num_ctxs; + + /* serialize ulp operations */ + struct mutex en_dev_lock; +}; + +static inline bool bnxt_ulp_registered(struct bnxt_en_dev *edev) +{ + if (edev && rcu_access_pointer(edev->ulp_tbl->ulp_ops)) + return true; + return false; +} + +int bnxt_get_ulp_msix_num(struct bnxt *bp); +int bnxt_get_ulp_msix_num_in_use(struct bnxt *bp); +void bnxt_set_ulp_msix_num(struct bnxt *bp, int num); +int bnxt_get_ulp_stat_ctxs(struct bnxt *bp); +void bnxt_set_ulp_stat_ctxs(struct bnxt *bp, int num_ctxs); +int bnxt_get_ulp_stat_ctxs_in_use(struct bnxt *bp); +void bnxt_set_dflt_ulp_stat_ctxs(struct bnxt *bp); +void bnxt_ulp_stop(struct bnxt *bp); +void bnxt_ulp_start(struct bnxt *bp); +void bnxt_ulp_sriov_cfg(struct bnxt *bp, int num_vfs); +void bnxt_ulp_irq_stop(struct bnxt *bp); +void bnxt_ulp_irq_restart(struct bnxt *bp, int err); +void bnxt_ulp_async_events(struct bnxt *bp, struct hwrm_async_event_cmpl *cmpl); +void bnxt_aux_devices_uninit(struct bnxt *bp); +void bnxt_aux_devices_del(struct bnxt *bp); +void bnxt_aux_devices_add(struct bnxt *bp); +void bnxt_aux_devices_init(struct bnxt *bp); +int bnxt_register_dev(struct bnxt_en_dev *edev, struct bnxt_ulp_ops *ulp_ops, + void *handle); +void bnxt_unregister_dev(struct bnxt_en_dev *edev); +int bnxt_send_msg(struct bnxt_en_dev *edev, struct bnxt_fw_msg *fw_msg); +void bnxt_register_async_events(struct bnxt_en_dev *edev, + unsigned long *events_bmap, u16 max_id); +int bnxt_auxdev_id_alloc(struct bnxt *bp); +void bnxt_auxdev_id_free(struct bnxt *bp, int id); +#endif diff --git a/include/linux/bootconfig.h b/include/linux/bootconfig.h index 25df9260d206..deda507500da 100644 --- a/include/linux/bootconfig.h +++ b/include/linux/bootconfig.h @@ -36,9 +36,9 @@ bool __init cmdline_has_extra_options(void); * The checksum will be used with the BOOTCONFIG_MAGIC and the size for * embedding the bootconfig in the initrd image. */ -static inline __init uint32_t xbc_calc_checksum(void *data, uint32_t size) +static inline __init uint32_t xbc_calc_checksum(const void *data, uint32_t size) { - unsigned char *p = data; + const unsigned char *p = data; uint32_t ret = 0; while (size--) @@ -66,7 +66,7 @@ struct xbc_node { /* Node tree access raw APIs */ struct xbc_node * __init xbc_root_node(void); -int __init xbc_node_index(struct xbc_node *node); +uint16_t __init xbc_node_index(struct xbc_node *node); struct xbc_node * __init xbc_node_get_parent(struct xbc_node *node); struct xbc_node * __init xbc_node_get_child(struct xbc_node *node); struct xbc_node * __init xbc_node_get_next(struct xbc_node *node); @@ -265,6 +265,9 @@ static inline struct xbc_node * __init xbc_node_get_subkey(struct xbc_node *node int __init xbc_node_compose_key_after(struct xbc_node *root, struct xbc_node *node, char *buf, size_t size); +/* Render key/value pairs under @root as a flat cmdline string */ +int __init xbc_snprint_cmdline(char *buf, size_t size, struct xbc_node *root); + /** * xbc_node_compose_key() - Compose full key string of the XBC node * @node: An XBC node. @@ -305,4 +308,18 @@ static inline const char *xbc_get_embedded_bootconfig(size_t *size) } #endif +/* Bootconfig opt-in detection, shared by setup_arch() and setup_boot_config() */ +#ifdef CONFIG_BOOT_CONFIG +bool __init bootconfig_cmdline_requested(const char *boot_cmdline, int *end_offset); +#endif + +/* Build-time-rendered bootconfig cmdline prepended in setup_arch() */ +#ifdef CONFIG_CMDLINE_FROM_BOOTCONFIG +void __init xbc_prepend_embedded_cmdline(char *dst, size_t size); +bool __init xbc_embedded_cmdline_applied(void); +#else +static inline void xbc_prepend_embedded_cmdline(char *dst, size_t size) { } +static inline bool xbc_embedded_cmdline_applied(void) { return false; } +#endif + #endif diff --git a/include/linux/bootmem_info.h b/include/linux/bootmem_info.h deleted file mode 100644 index 4c506e76a808..000000000000 --- a/include/linux/bootmem_info.h +++ /dev/null @@ -1,94 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef __LINUX_BOOTMEM_INFO_H -#define __LINUX_BOOTMEM_INFO_H - -#include <linux/mm.h> -#include <linux/kmemleak.h> - -/* - * Types for free bootmem stored in the low bits of page->private. - */ -enum bootmem_type { - MEMORY_HOTPLUG_MIN_BOOTMEM_TYPE = 1, - SECTION_INFO = MEMORY_HOTPLUG_MIN_BOOTMEM_TYPE, - MIX_SECTION_INFO, - NODE_INFO, - MEMORY_HOTPLUG_MAX_BOOTMEM_TYPE = NODE_INFO, -}; - -#ifdef CONFIG_HAVE_BOOTMEM_INFO_NODE -void __init register_page_bootmem_info_node(struct pglist_data *pgdat); -void register_page_bootmem_memmap(unsigned long section_nr, struct page *map, - unsigned long nr_pages); - -void get_page_bootmem(unsigned long info, struct page *page, - enum bootmem_type type); -void put_page_bootmem(struct page *page); - -static inline enum bootmem_type bootmem_type(const struct page *page) -{ - return (unsigned long)page->private & 0xf; -} - -static inline unsigned long bootmem_info(const struct page *page) -{ - return (unsigned long)page->private >> 4; -} - -/* - * Any memory allocated via the memblock allocator and not via the - * buddy will be marked reserved already in the memmap. For those - * pages, we can call this function to free it to buddy allocator. - */ -static inline void free_bootmem_page(struct page *page) -{ - enum bootmem_type type = bootmem_type(page); - - /* - * The reserve_bootmem_region sets the reserved flag on bootmem - * pages. - */ - VM_BUG_ON_PAGE(page_ref_count(page) != 2, page); - - if (type == SECTION_INFO || type == MIX_SECTION_INFO) - put_page_bootmem(page); - else - VM_BUG_ON_PAGE(1, page); -} -#else -static inline void register_page_bootmem_info_node(struct pglist_data *pgdat) -{ -} - -static inline void register_page_bootmem_memmap(unsigned long section_nr, - struct page *map, unsigned long nr_pages) -{ -} - -static inline void put_page_bootmem(struct page *page) -{ -} - -static inline enum bootmem_type bootmem_type(const struct page *page) -{ - return SECTION_INFO; -} - -static inline unsigned long bootmem_info(const struct page *page) -{ - return 0; -} - -static inline void get_page_bootmem(unsigned long info, struct page *page, - enum bootmem_type type) -{ -} - -static inline void free_bootmem_page(struct page *page) -{ - kmemleak_free_part_phys(PFN_PHYS(page_to_pfn(page)), PAGE_SIZE); - free_reserved_page(page); -} -#endif - -#endif /* __LINUX_BOOTMEM_INFO_H */ diff --git a/include/linux/bpf-cgroup.h b/include/linux/bpf-cgroup.h index 2f535331f926..4d0cc65976a1 100644 --- a/include/linux/bpf-cgroup.h +++ b/include/linux/bpf-cgroup.h @@ -184,7 +184,7 @@ static inline bool cgroup_bpf_sock_enabled(struct sock *sk, struct bpf_prog_array *array; array = rcu_access_pointer(cgrp->bpf.effective[type]); - return array != &bpf_empty_prog_array.hdr; + return array != &bpf_empty_prog_array; } /* Wrappers for __cgroup_bpf_run_filter_skb() guarded by cgroup_bpf_enabled. */ @@ -421,7 +421,7 @@ int cgroup_bpf_prog_detach(const union bpf_attr *attr, enum bpf_prog_type ptype); int cgroup_bpf_link_attach(const union bpf_attr *attr, struct bpf_prog *prog); int cgroup_bpf_prog_query(const union bpf_attr *attr, - union bpf_attr __user *uattr); + union bpf_attr __user *uattr, u32 uattr_size); const struct bpf_func_proto * cgroup_common_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog); @@ -452,7 +452,8 @@ static inline int cgroup_bpf_link_attach(const union bpf_attr *attr, } static inline int cgroup_bpf_prog_query(const union bpf_attr *attr, - union bpf_attr __user *uattr) + union bpf_attr __user *uattr, + u32 uattr_size) { return -EINVAL; } diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 05b34a6355b0..e57af902560c 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -6,6 +6,7 @@ #include <uapi/linux/bpf.h> #include <uapi/linux/filter.h> +#include <linux/bpf_defs.h> #include <crypto/sha2.h> #include <linux/workqueue.h> @@ -31,6 +32,9 @@ #include <linux/static_call.h> #include <linux/memcontrol.h> #include <linux/cfi.h> +#include <linux/xattr.h> +#include <linux/key.h> +#include <linux/ftrace.h> #include <asm/rqspinlock.h> struct bpf_verifier_env; @@ -110,7 +114,7 @@ struct bpf_map_ops { long (*map_pop_elem)(struct bpf_map *map, void *value); long (*map_peek_elem)(struct bpf_map *map, void *value); void *(*map_lookup_percpu_elem)(struct bpf_map *map, void *key, u32 cpu); - int (*map_get_hash)(struct bpf_map *map, u32 hash_buf_size, void *hash_buf); + int (*map_get_hash)(struct bpf_map *map); /* funcs called by prog_array and perf_event_array map */ void *(*map_fd_get_ptr)(struct bpf_map *map, struct file *map_file, @@ -141,6 +145,7 @@ struct bpf_map_ops { int (*map_direct_value_meta)(const struct bpf_map *map, u64 imm, u32 *off); int (*map_mmap)(struct bpf_map *map, struct vm_area_struct *vma); + vm_fault_t (*map_mmap_fault)(struct bpf_map *map, struct vm_fault *vmf); __poll_t (*map_poll)(struct bpf_map *map, struct file *filp, struct poll_table_struct *pts); unsigned long (*map_get_unmapped_area)(struct file *filep, unsigned long addr, @@ -488,6 +493,35 @@ static inline bool btf_record_has_field(const struct btf_record *rec, enum btf_f return rec->field_mask & type; } +static inline bool btf_field_is_nmi_safe(enum btf_field_type type) +{ + switch (type) { + case BPF_SPIN_LOCK: + case BPF_RES_SPIN_LOCK: + case BPF_TIMER: + case BPF_WORKQUEUE: + case BPF_TASK_WORK: + case BPF_KPTR_UNREF: + case BPF_REFCOUNT: + return true; + default: + return false; + } +} + +static inline bool btf_record_has_nmi_unsafe_fields(const struct btf_record *rec) +{ + int i; + + if (IS_ERR_OR_NULL(rec)) + return false; + for (i = 0; i < rec->cnt; i++) { + if (!btf_field_is_nmi_safe(rec->fields[i].type)) + return true; + } + return false; +} + static inline void bpf_obj_init(const struct btf_record *rec, void *obj) { int i; @@ -536,7 +570,7 @@ static inline void bpf_obj_memcpy(struct btf_record *rec, if (IS_ERR_OR_NULL(rec)) { if (long_memcpy) - bpf_long_memcpy(dst, src, round_up(size, 8)); + bpf_long_memcpy(dst, src, size); else memcpy(dst, src, size); return; @@ -559,7 +593,7 @@ static inline void copy_map_value(struct bpf_map *map, void *dst, void *src) static inline void copy_map_value_long(struct bpf_map *map, void *dst, void *src) { - bpf_obj_memcpy(map->record, dst, src, map->value_size, true); + bpf_obj_memcpy(map->record, dst, src, round_up(map->value_size, 8), true); } static inline void bpf_obj_swap_uptrs(const struct btf_record *rec, void *dst, void *src) @@ -617,6 +651,8 @@ void bpf_rb_root_free(const struct btf_field *field, void *rb_root, struct bpf_spin_lock *spin_lock); u64 bpf_arena_get_kern_vm_start(struct bpf_arena *arena); u64 bpf_arena_get_user_vm_start(struct bpf_arena *arena); +u64 bpf_arena_map_kern_vm_start(struct bpf_map *map); +struct bpf_map *bpf_prog_arena(struct bpf_prog *prog); int bpf_obj_name_cpy(char *dst, const char *src, unsigned int size); struct bpf_offload_dev; @@ -678,6 +714,8 @@ int bpf_dynptr_from_file_sleepable(struct file *file, u32 flags, void *bpf_arena_alloc_pages_non_sleepable(void *p__map, void *addr__ign, u32 page_cnt, int node_id, u64 flags); void bpf_arena_free_pages_non_sleepable(void *p__map, void *ptr__ign, u32 page_cnt); +void *bpf_arena_alloc_pages_sleepable(void *p__map, void *addr__ign, u32 page_cnt, int node_id, + u64 flags); #else static inline void *bpf_arena_alloc_pages_non_sleepable(void *p__map, void *addr__ign, u32 page_cnt, int node_id, u64 flags) @@ -688,6 +726,12 @@ static inline void *bpf_arena_alloc_pages_non_sleepable(void *p__map, void *addr static inline void bpf_arena_free_pages_non_sleepable(void *p__map, void *ptr__ign, u32 page_cnt) { } + +static inline void *bpf_arena_alloc_pages_sleepable(void *p__map, void *addr__ign, u32 page_cnt, + int node_id, u64 flags) +{ + return NULL; +} #endif extern const struct bpf_map_ops bpf_map_offload_ops; @@ -845,11 +889,12 @@ enum bpf_arg_type { ARG_PTR_TO_MEM, /* pointer to valid memory (stack, packet, map value) */ ARG_PTR_TO_ARENA, - ARG_CONST_SIZE, /* number of bytes accessed from memory */ - ARG_CONST_SIZE_OR_ZERO, /* number of bytes accessed from memory or 0 */ + ARG_MEM_SIZE, /* number of bytes accessed from memory */ + ARG_MEM_SIZE_OR_ZERO, /* number of bytes accessed from memory or 0 */ ARG_PTR_TO_CTX, /* pointer to context */ ARG_ANYTHING, /* any (initialized) argument is ok */ + ARG_SCALAR, /* scalar argument */ ARG_PTR_TO_SPIN_LOCK, /* pointer to bpf_spin_lock */ ARG_PTR_TO_SOCK_COMMON, /* pointer to sock_common */ ARG_PTR_TO_SOCKET, /* pointer to bpf_sock (fullsock) */ @@ -917,6 +962,21 @@ enum bpf_return_type { }; static_assert(__BPF_RET_TYPE_MAX <= BPF_BASE_TYPE_LIMIT); +/* The longest tracepoint has 12 args. + * See include/trace/bpf_probe.h + * + * Also reuse this macro for maximum number of arguments a BPF function + * or a kfunc can have. Args 1-5 are passed in registers, args 6-12 via + * stack arg slots. The JIT may map some stack arg slots to registers based + * on the native calling convention (e.g., arg 6 to R9 on x86-64). + */ +#define MAX_BPF_FUNC_ARGS 12 + +/* The maximum number of arguments passed through registers + * a single function may have. + */ +#define MAX_BPF_FUNC_REG_ARGS 5 + /* eBPF function prototype used by verifier to allow BPF_CALLs from eBPF programs * to in-kernel helper functions and for adjusting imm32 field in BPF_CALL * instructions after verifying @@ -941,7 +1001,7 @@ struct bpf_func_proto { enum bpf_arg_type arg4_type; enum bpf_arg_type arg5_type; }; - enum bpf_arg_type arg_type[5]; + enum bpf_arg_type arg_type[MAX_BPF_FUNC_ARGS]; }; union { struct { @@ -951,7 +1011,7 @@ struct bpf_func_proto { u32 *arg4_btf_id; u32 *arg5_btf_id; }; - u32 *arg_btf_id[5]; + u32 *arg_btf_id[MAX_BPF_FUNC_ARGS]; struct { size_t arg1_size; size_t arg2_size; @@ -959,7 +1019,7 @@ struct bpf_func_proto { size_t arg4_size; size_t arg5_size; }; - size_t arg_size[5]; + size_t arg_size[MAX_BPF_FUNC_ARGS]; }; int *ret_btf_id; /* return value btf_id */ bool (*allowed)(const struct bpf_prog *prog); @@ -1051,7 +1111,7 @@ struct bpf_insn_access_aux { struct { struct btf *btf; u32 btf_id; - u32 ref_obj_id; + u32 ref_id; }; }; struct bpf_verifier_log *log; /* for verbose logs */ @@ -1074,21 +1134,6 @@ static inline bool bpf_pseudo_func(const struct bpf_insn *insn) return bpf_is_ldimm64(insn) && insn->src_reg == BPF_PSEUDO_FUNC; } -/* Given a BPF_ATOMIC instruction @atomic_insn, return true if it is an - * atomic load or store, and false if it is a read-modify-write instruction. - */ -static inline bool -bpf_atomic_is_load_store(const struct bpf_insn *atomic_insn) -{ - switch (atomic_insn->imm) { - case BPF_LOAD_ACQ: - case BPF_STORE_REL: - return true; - default: - return false; - } -} - struct bpf_prog_ops { int (*test_run)(struct bpf_prog *prog, const union bpf_attr *kattr, union bpf_attr __user *uattr); @@ -1149,22 +1194,15 @@ struct bpf_prog_offload { u32 jited_len; }; -/* The longest tracepoint has 12 args. - * See include/trace/bpf_probe.h - */ -#define MAX_BPF_FUNC_ARGS 12 - -/* The maximum number of arguments passed through registers - * a single function may have. - */ -#define MAX_BPF_FUNC_REG_ARGS 5 - -/* The argument is a structure or a union. */ -#define BTF_FMODEL_STRUCT_ARG BIT(0) - /* The argument is signed. */ #define BTF_FMODEL_SIGNED_ARG BIT(1) +/* The argument is an arena pointer. */ +#define BTF_FMODEL_ARENA_ARG BIT(2) + +/* The argument is nullable. */ +#define BTF_FMODEL_NULLABLE_ARG BIT(3) + struct btf_func_model { u8 ret_size; u8 ret_flags; @@ -1233,11 +1271,20 @@ enum { #define BPF_TRAMP_COOKIE_INDEX_SHIFT 8 #define BPF_TRAMP_IS_RETURN_SHIFT 63 -struct bpf_tramp_links { - struct bpf_tramp_link *links[BPF_MAX_TRAMP_LINKS]; - int nr_links; +struct bpf_tramp_nodes { + struct bpf_tramp_node *nodes[BPF_MAX_TRAMP_LINKS]; + int nr_nodes; }; +/* + * The arena base against which a struct_ops trampoline converts the + * arguments marked with BTF_FMODEL_ARENA_ARG while saving them into the BPF + * ctx, ctx[arg] = (u32)(kaddr - kern_vm_start). Zero when the trampoline + * converts nothing. + */ +u64 bpf_tramp_arena_base(const struct btf_func_model *m, + struct bpf_tramp_nodes *tnodes, u32 flags); + struct bpf_tramp_run_ctx; /* Different use cases for BPF trampoline: @@ -1263,13 +1310,13 @@ struct bpf_tramp_run_ctx; struct bpf_tramp_image; int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *image_end, const struct btf_func_model *m, u32 flags, - struct bpf_tramp_links *tlinks, + struct bpf_tramp_nodes *tnodes, void *func_addr); void *arch_alloc_bpf_trampoline(unsigned int size); void arch_free_bpf_trampoline(void *image, unsigned int size); int __must_check arch_protect_bpf_trampoline(void *image, unsigned int size); int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, - struct bpf_tramp_links *tlinks, void *func_addr); + struct bpf_tramp_nodes *tnodes, void *func_addr); u64 notrace __bpf_prog_enter_sleepable_recur(struct bpf_prog *prog, struct bpf_tramp_run_ctx *run_ctx); @@ -1335,8 +1382,6 @@ struct bpf_trampoline { /* hlist for trampoline_ip_table */ struct hlist_node hlist_ip; struct ftrace_ops *fops; - /* serializes access to fields of this trampoline */ - struct mutex mutex; refcount_t refcnt; u32 flags; u64 key; @@ -1357,6 +1402,11 @@ struct bpf_trampoline { int progs_cnt[BPF_TRAMP_MAX]; /* Executable image of trampoline */ struct bpf_tramp_image *cur_image; + /* Used as temporary old image storage for multi_attach */ + struct { + struct bpf_tramp_image *old_image; + u32 old_flags; + } multi_attach; }; struct bpf_attach_target_info { @@ -1454,11 +1504,13 @@ static inline int bpf_dynptr_check_off_len(const struct bpf_dynptr_kern *ptr, u6 return 0; } +struct bpf_tracing_multi_link; + #ifdef CONFIG_BPF_JIT -int bpf_trampoline_link_prog(struct bpf_tramp_link *link, +int bpf_trampoline_link_prog(struct bpf_tramp_node *node, struct bpf_trampoline *tr, struct bpf_prog *tgt_prog); -int bpf_trampoline_unlink_prog(struct bpf_tramp_link *link, +int bpf_trampoline_unlink_prog(struct bpf_tramp_node *node, struct bpf_trampoline *tr, struct bpf_prog *tgt_prog); struct bpf_trampoline *bpf_trampoline_get(u64 key, @@ -1466,6 +1518,12 @@ struct bpf_trampoline *bpf_trampoline_get(u64 key, void bpf_trampoline_put(struct bpf_trampoline *tr); int arch_prepare_bpf_dispatcher(void *image, void *buf, s64 *funcs, int num_funcs); +int bpf_trampoline_multi_attach(struct bpf_prog *prog, u32 *ids, + struct bpf_tracing_multi_link *link); +void bpf_trampoline_multi_detach(struct bpf_prog *prog, + struct bpf_tracing_multi_link *link); +void bpf_trampoline_set_flags(struct bpf_trampoline *tr, u32 flags); + /* * When the architecture supports STATIC_CALL replace the bpf_dispatcher_fn * indirection with a direct call to the bpf program. If the architecture does @@ -1541,14 +1599,17 @@ bool bpf_has_frame_pointer(unsigned long ip); int bpf_jit_charge_modmem(u32 size); void bpf_jit_uncharge_modmem(u32 size); bool bpf_prog_has_trampoline(const struct bpf_prog *prog); +bool bpf_insn_is_indirect_target(const struct bpf_verifier_env *env, const struct bpf_prog *prog, + int insn_idx); +u16 bpf_out_stack_arg_cnt(const struct bpf_verifier_env *env, const struct bpf_prog *prog); #else -static inline int bpf_trampoline_link_prog(struct bpf_tramp_link *link, +static inline int bpf_trampoline_link_prog(struct bpf_tramp_node *node, struct bpf_trampoline *tr, struct bpf_prog *tgt_prog) { return -ENOTSUPP; } -static inline int bpf_trampoline_unlink_prog(struct bpf_tramp_link *link, +static inline int bpf_trampoline_unlink_prog(struct bpf_tramp_node *node, struct bpf_trampoline *tr, struct bpf_prog *tgt_prog) { @@ -1575,6 +1636,16 @@ static inline bool bpf_prog_has_trampoline(const struct bpf_prog *prog) { return false; } +static inline int bpf_trampoline_multi_attach(struct bpf_prog *prog, u32 *ids, + struct bpf_tracing_multi_link *link) +{ + return -ENOTSUPP; +} +static inline void bpf_trampoline_multi_detach(struct bpf_prog *prog, + struct bpf_tracing_multi_link *link) +{ +} +static inline void bpf_trampoline_set_flags(struct bpf_trampoline *tr, u32 flags) {} #endif struct bpf_func_info_aux { @@ -1612,7 +1683,7 @@ struct bpf_ctx_arg_aux { enum bpf_reg_type reg_type; struct btf *btf; u32 btf_id; - u32 ref_obj_id; + u32 ref_id; bool refcounted; }; @@ -1654,6 +1725,19 @@ struct bpf_stream_stage { int len; }; +enum bpf_sig_verdict { + BPF_SIG_UNSIGNED = 0, + BPF_SIG_VERIFIED, +}; + +enum bpf_sig_keyring { + BPF_SIG_KEYRING_NONE = 0, + BPF_SIG_KEYRING_BUILTIN, + BPF_SIG_KEYRING_SECONDARY, + BPF_SIG_KEYRING_PLATFORM, + BPF_SIG_KEYRING_USER, +}; + struct bpf_prog_aux { atomic64_t refcnt; u32 used_map_cnt; @@ -1696,6 +1780,11 @@ struct bpf_prog_aux { bool changes_pkt_data; bool might_sleep; bool kprobe_write_ctx; + struct { + s32 keyring_serial; + u8 keyring_type; + u8 verdict; + } sig; u64 prog_array_member_cnt; /* counts how many times as member of prog_array */ struct mutex ext_mutex; /* mutex for is_extended and prog_array_member_cnt */ struct bpf_arena *arena; @@ -1728,6 +1817,7 @@ struct bpf_prog_aux { struct bpf_map *cgroup_storage[MAX_BPF_CGROUP_STORAGE_TYPE]; char name[BPF_OBJ_NAME_LEN]; u64 (*bpf_exception_cb)(u64 cookie, u64 sp, u64 bp, u64, u64); + u16 stack_arg_sp_adjust; #ifdef CONFIG_SECURITY void *security; #endif @@ -1774,8 +1864,9 @@ struct bpf_prog_aux { struct bpf_prog { u16 pages; /* Number of allocated pages */ - u16 jited:1, /* Is our filter JIT'ed? */ + u32 jited:1, /* Is our filter JIT'ed? */ jit_requested:1,/* archs need to JIT the prog */ + jit_required:1, /* program strictly requires JIT compiler */ gpl_compatible:1, /* Is filter GPL compatible? */ cb_access:1, /* Is control block accessed? */ dst_needed:1, /* Do we need dst entry? */ @@ -1854,6 +1945,10 @@ struct bpf_link_ops { * target hook is sleepable, we'll go through tasks trace RCU GP and * then "classic" RCU GP; this need for chaining tasks trace and * classic RCU GPs is designated by setting bpf_link->sleepable flag + * + * For non-sleepable tracepoint links we go through SRCU gp instead, + * since RCU is not used in that case. Sleepable tracepoints still + * follow the scheme above. */ void (*dealloc_deferred)(struct bpf_link *link); int (*detach)(struct bpf_link *link); @@ -1867,12 +1962,17 @@ struct bpf_link_ops { __poll_t (*poll)(struct file *file, struct poll_table_struct *pts); }; -struct bpf_tramp_link { - struct bpf_link link; +struct bpf_tramp_node { + struct bpf_link *link; struct hlist_node tramp_hlist; u64 cookie; }; +struct bpf_tramp_link { + struct bpf_link link; + struct bpf_tramp_node node; +}; + struct bpf_shim_tramp_link { struct bpf_tramp_link link; struct bpf_trampoline *trampoline; @@ -1880,13 +1980,31 @@ struct bpf_shim_tramp_link { struct bpf_tracing_link { struct bpf_tramp_link link; + struct bpf_tramp_node fexit; struct bpf_trampoline *trampoline; struct bpf_prog *tgt_prog; }; -struct bpf_fsession_link { - struct bpf_tracing_link link; - struct bpf_tramp_link fexit; +struct bpf_tracing_multi_node { + struct bpf_tramp_node node; + struct bpf_trampoline *trampoline; + struct ftrace_func_entry entry; +}; + +struct bpf_tracing_multi_data { + struct ftrace_hash *unreg; + struct ftrace_hash *modify; + struct ftrace_hash *reg; + struct ftrace_func_entry *entry; +}; + +struct bpf_tracing_multi_link { + struct bpf_link link; + struct bpf_tracing_multi_data data; + u64 *cookies; + struct bpf_tramp_node *fexits; + int nodes_cnt; + struct bpf_tracing_multi_node nodes[] __counted_by(nodes_cnt); }; struct bpf_raw_tp_link { @@ -1912,6 +2030,8 @@ struct bpf_mount_opts { u64 delegate_maps; u64 delegate_progs; u64 delegate_attachs; + + struct simple_xattr_cache xa_cache; }; struct bpf_token { @@ -2070,6 +2190,18 @@ static inline void bpf_prog_put_recursion_context(struct bpf_prog *prog) #endif } +static inline bool is_tracing_multi(enum bpf_attach_type type) +{ + return type == BPF_TRACE_FENTRY_MULTI || type == BPF_TRACE_FEXIT_MULTI || + type == BPF_TRACE_FSESSION_MULTI; +} + +static inline bool is_struct_ops_tramp(const struct bpf_tramp_nodes *fentry_nodes) +{ + return fentry_nodes->nr_nodes == 1 && + fentry_nodes->nodes[0]->link->type == BPF_LINK_TYPE_STRUCT_OPS; +} + #if defined(CONFIG_BPF_JIT) && defined(CONFIG_BPF_SYSCALL) /* This macro helps developer to register a struct_ops type and generate * type information correctly. Developers should use this macro to register @@ -2090,8 +2222,8 @@ void bpf_struct_ops_put(const void *kdata); int bpf_struct_ops_supported(const struct bpf_struct_ops *st_ops, u32 moff); int bpf_struct_ops_map_sys_lookup_elem(struct bpf_map *map, void *key, void *value); -int bpf_struct_ops_prepare_trampoline(struct bpf_tramp_links *tlinks, - struct bpf_tramp_link *link, +int bpf_struct_ops_prepare_trampoline(struct bpf_tramp_nodes *tnodes, + struct bpf_tramp_node *node, const struct btf_func_model *model, void *stub_func, void **image, u32 *image_off, @@ -2116,6 +2248,9 @@ int bpf_prog_assoc_struct_ops(struct bpf_prog *prog, struct bpf_map *map); void bpf_prog_disassoc_struct_ops(struct bpf_prog *prog); void *bpf_prog_get_assoc_struct_ops(const struct bpf_prog_aux *aux); u32 bpf_struct_ops_id(const void *kdata); +int bpf_struct_ops_for_each_prog(const void *kdata, + int (*cb)(struct bpf_prog *prog, void *data), + void *data); #ifdef CONFIG_NET /* Define it here to avoid the use of forward declaration */ @@ -2183,31 +2318,33 @@ static inline void bpf_struct_ops_desc_release(struct bpf_struct_ops_desc *st_op #endif -static inline int bpf_fsession_cnt(struct bpf_tramp_links *links) +static inline int bpf_fsession_cnt(struct bpf_tramp_nodes *nodes) { - struct bpf_tramp_links fentries = links[BPF_TRAMP_FENTRY]; + struct bpf_tramp_nodes fentries = nodes[BPF_TRAMP_FENTRY]; int cnt = 0; - for (int i = 0; i < links[BPF_TRAMP_FENTRY].nr_links; i++) { - if (fentries.links[i]->link.prog->expected_attach_type == BPF_TRACE_FSESSION) + for (int i = 0; i < nodes[BPF_TRAMP_FENTRY].nr_nodes; i++) { + if (fentries.nodes[i]->link->prog->expected_attach_type == BPF_TRACE_FSESSION) + cnt++; + if (fentries.nodes[i]->link->prog->expected_attach_type == BPF_TRACE_FSESSION_MULTI) cnt++; } return cnt; } -static inline bool bpf_prog_calls_session_cookie(struct bpf_tramp_link *link) +static inline bool bpf_prog_calls_session_cookie(struct bpf_tramp_node *node) { - return link->link.prog->call_session_cookie; + return node->link->prog->call_session_cookie; } -static inline int bpf_fsession_cookie_cnt(struct bpf_tramp_links *links) +static inline int bpf_fsession_cookie_cnt(struct bpf_tramp_nodes *nodes) { - struct bpf_tramp_links fentries = links[BPF_TRAMP_FENTRY]; + struct bpf_tramp_nodes fentries = nodes[BPF_TRAMP_FENTRY]; int cnt = 0; - for (int i = 0; i < links[BPF_TRAMP_FENTRY].nr_links; i++) { - if (bpf_prog_calls_session_cookie(fentries.links[i])) + for (int i = 0; i < nodes[BPF_TRAMP_FENTRY].nr_nodes; i++) { + if (bpf_prog_calls_session_cookie(fentries.nodes[i])) cnt++; } @@ -2365,18 +2502,13 @@ struct bpf_prog_array { struct bpf_prog_array_item items[]; }; -struct bpf_empty_prog_array { - struct bpf_prog_array hdr; - struct bpf_prog *null_prog; -}; - /* to avoid allocating empty bpf_prog_array for cgroups that * don't have bpf program attached use one global 'bpf_empty_prog_array' * It will not be modified the caller of bpf_prog_array_alloc() * (since caller requested prog_cnt == 0) * that pointer should be 'freed' by bpf_prog_array_free() */ -extern struct bpf_empty_prog_array bpf_empty_prog_array; +extern struct bpf_prog_array bpf_empty_prog_array; struct bpf_prog_array *bpf_prog_array_alloc(u32 prog_cnt, gfp_t flags); void bpf_prog_array_free(struct bpf_prog_array *progs); @@ -2594,6 +2726,7 @@ bool btf_record_equal(const struct btf_record *rec_a, const struct btf_record *r void bpf_obj_free_timer(const struct btf_record *rec, void *obj); void bpf_obj_free_workqueue(const struct btf_record *rec, void *obj); void bpf_obj_free_task_work(const struct btf_record *rec, void *obj); +void bpf_obj_cancel_fields(struct bpf_map *map, void *obj); void bpf_obj_free_fields(const struct btf_record *rec, void *obj); void __bpf_obj_drop_impl(void *p, const struct btf_record *rec, bool percpu); @@ -2760,6 +2893,9 @@ void bpf_link_init(struct bpf_link *link, enum bpf_link_type type, void bpf_link_init_sleepable(struct bpf_link *link, enum bpf_link_type type, const struct bpf_link_ops *ops, struct bpf_prog *prog, enum bpf_attach_type attach_type, bool sleepable); +void bpf_tramp_link_init(struct bpf_tramp_link *link, enum bpf_link_type type, + const struct bpf_link_ops *ops, struct bpf_prog *prog, + enum bpf_attach_type attach_type, u64 cookie); int bpf_link_prime(struct bpf_link *link, struct bpf_link_primer *primer); int bpf_link_settle(struct bpf_link_primer *primer); void bpf_link_cleanup(struct bpf_link_primer *primer); @@ -2913,10 +3049,18 @@ int bpf_check_uarg_tail_zero(bpfptr_t uaddr, size_t expected_size, size_t actual_size); /* verify correctness of eBPF program */ -int bpf_check(struct bpf_prog **fp, union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size); +struct bpf_log_attr; +int bpf_check(struct bpf_prog **fp, union bpf_attr *attr, bpfptr_t uattr, + struct bpf_log_attr *attr_log); #ifndef CONFIG_BPF_JIT_ALWAYS_ON -void bpf_patch_call_args(struct bpf_insn *insn, u32 stack_depth); +int bpf_patch_call_args(struct bpf_insn *insn, u32 stack_depth); +s32 bpf_call_args_imm(s16 idx); +#else +static inline s32 bpf_call_args_imm(s16 idx) +{ + return 0; +} #endif struct btf *bpf_get_btf_vmlinux(void); @@ -3008,7 +3152,7 @@ int btf_struct_access(struct bpf_verifier_log *log, bool btf_struct_ids_match(struct bpf_verifier_log *log, const struct btf *btf, u32 id, int off, const struct btf *need_btf, u32 need_type_id, - bool strict); + bool strict, bool walk_flex_arrays); int btf_distill_func_proto(struct bpf_verifier_log *log, struct btf *btf, @@ -3032,7 +3176,6 @@ const struct bpf_func_proto *bpf_base_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog); void bpf_task_storage_free(struct task_struct *task); void bpf_cgrp_storage_free(struct cgroup *cgroup); -bool bpf_prog_has_kfunc_call(const struct bpf_prog *prog); const struct btf_func_model * bpf_jit_find_kfunc_model(const struct bpf_prog *prog, const struct bpf_insn *insn); @@ -3078,6 +3221,56 @@ void bpf_dynptr_set_null(struct bpf_dynptr_kern *ptr); void bpf_dynptr_set_rdonly(struct bpf_dynptr_kern *ptr); void bpf_prog_report_arena_violation(bool write, unsigned long addr, unsigned long fault_ip); +static __always_inline u32 +bpf_prog_run_array_sleepable(const struct bpf_prog_array *array, + const void *ctx, bpf_prog_run_fn run_prog) +{ + const struct bpf_prog_array_item *item; + struct bpf_prog *prog; + struct bpf_run_ctx *old_run_ctx; + struct bpf_trace_run_ctx run_ctx; + u32 ret = 1; + + if (unlikely(!array)) + return ret; + + migrate_disable(); + + run_ctx.is_uprobe = false; + + old_run_ctx = bpf_set_run_ctx(&run_ctx.run_ctx); + item = &array->items[0]; + while ((prog = READ_ONCE(item->prog))) { + /* Skip dummy_bpf_prog placeholder (len == 0) */ + if (unlikely(!prog->len)) { + item++; + continue; + } + + if (unlikely(!bpf_prog_get_recursion_context(prog))) { + bpf_prog_inc_misses_counter(prog); + bpf_prog_put_recursion_context(prog); + item++; + continue; + } + + run_ctx.bpf_cookie = item->bpf_cookie; + + if (!prog->sleepable) { + guard(rcu)(); + ret &= run_prog(prog, ctx); + } else { + ret &= run_prog(prog, ctx); + } + + bpf_prog_put_recursion_context(prog); + item++; + } + bpf_reset_run_ctx(old_run_ctx); + migrate_enable(); + return ret; +} + #else /* !CONFIG_BPF_SYSCALL */ static inline struct bpf_prog *bpf_prog_get(u32 ufd) { @@ -3125,6 +3318,12 @@ static inline void bpf_link_init_sleepable(struct bpf_link *link, enum bpf_link_ { } +static inline void bpf_tramp_link_init(struct bpf_tramp_link *link, enum bpf_link_type type, + const struct bpf_link_ops *ops, struct bpf_prog *prog, + enum bpf_attach_type attach_type, u64 cookie) +{ +} + static inline int bpf_link_prime(struct bpf_link *link, struct bpf_link_primer *primer) { @@ -3315,11 +3514,6 @@ static inline void bpf_task_storage_free(struct task_struct *task) { } -static inline bool bpf_prog_has_kfunc_call(const struct bpf_prog *prog) -{ - return false; -} - static inline const struct btf_func_model * bpf_jit_find_kfunc_model(const struct bpf_prog *prog, const struct bpf_insn *insn) @@ -3616,15 +3810,25 @@ static inline int bpf_fd_reuseport_array_update_elem(struct bpf_map *map, #endif /* CONFIG_BPF_SYSCALL */ #endif /* defined(CONFIG_INET) && defined(CONFIG_BPF_SYSCALL) */ -#if defined(CONFIG_KEYS) && defined(CONFIG_BPF_SYSCALL) +#ifdef CONFIG_KEYS +struct bpf_key { + struct key *key; + bool has_ref; +}; +#endif /* CONFIG_KEYS */ +#if defined(CONFIG_KEYS) && defined(CONFIG_BPF_SYSCALL) struct bpf_key *bpf_lookup_user_key(s32 serial, u64 flags); struct bpf_key *bpf_lookup_system_key(u64 id); void bpf_key_put(struct bpf_key *bkey); -int bpf_verify_pkcs7_signature(struct bpf_dynptr *data_p, - struct bpf_dynptr *sig_p, +int bpf_verify_pkcs7_signature(const struct bpf_dynptr *data_p, + const struct bpf_dynptr *sig_p, struct bpf_key *trusted_keyring); +static inline s32 bpf_key_serial(const struct bpf_key *key) +{ + return key->has_ref ? key->key->serial : 0; +} #else static inline struct bpf_key *bpf_lookup_user_key(u32 serial, u64 flags) { @@ -3640,12 +3844,17 @@ static inline void bpf_key_put(struct bpf_key *bkey) { } -static inline int bpf_verify_pkcs7_signature(struct bpf_dynptr *data_p, - struct bpf_dynptr *sig_p, +static inline int bpf_verify_pkcs7_signature(const struct bpf_dynptr *data_p, + const struct bpf_dynptr *sig_p, struct bpf_key *trusted_keyring) { return -EOPNOTSUPP; } + +static inline s32 bpf_key_serial(const struct bpf_key *key) +{ + return 0; +} #endif /* defined(CONFIG_KEYS) && defined(CONFIG_BPF_SYSCALL) */ /* verifier prototypes for helper functions called from eBPF programs */ @@ -3724,6 +3933,7 @@ extern const struct bpf_func_proto bpf_for_each_map_elem_proto; extern const struct bpf_func_proto bpf_btf_find_by_name_kind_proto; extern const struct bpf_func_proto bpf_sk_setsockopt_proto; extern const struct bpf_func_proto bpf_sk_getsockopt_proto; +extern const struct bpf_func_proto bpf_sk_setsockopt_nodelay_proto; extern const struct bpf_func_proto bpf_unlocked_sk_setsockopt_proto; extern const struct bpf_func_proto bpf_unlocked_sk_getsockopt_proto; extern const struct bpf_func_proto bpf_find_vma_proto; @@ -3920,15 +4130,6 @@ static inline void bpf_cgroup_atype_get(u32 attach_btf_id, int cgroup_atype) {} static inline void bpf_cgroup_atype_put(int cgroup_atype) {} #endif /* CONFIG_BPF_LSM */ -struct key; - -#ifdef CONFIG_KEYS -struct bpf_key { - struct key *key; - bool has_ref; -}; -#endif /* CONFIG_KEYS */ - static inline bool type_is_alloc(u32 type) { return type & MEM_ALLOC; @@ -3946,6 +4147,17 @@ static inline bool bpf_is_subprog(const struct bpf_prog *prog) return prog->aux->func_idx != 0; } +const struct bpf_line_info *bpf_find_linfo(const struct bpf_prog *prog, u32 insn_off); +struct bpf_linfo_source { + const char *file; + const char *line; + u32 file_name_off; + int line_num; + int line_col; +}; + +void bpf_get_linfo_source(struct btf *btf, const struct bpf_line_info *linfo, + struct bpf_linfo_source *src); int bpf_prog_get_file_line(struct bpf_prog *prog, unsigned long ip, const char **filep, const char **linep, int *nump); struct bpf_prog *bpf_prog_find_from_stack(void); @@ -3965,7 +4177,7 @@ bpf_prog_update_insn_ptrs(struct bpf_prog *prog, u32 *offsets, void *image) } #endif -static inline bool bpf_map_supports_cpu_flags(enum bpf_map_type map_type) +static inline bool bpf_map_is_percpu_map(enum bpf_map_type map_type) { switch (map_type) { case BPF_MAP_TYPE_PERCPU_ARRAY: @@ -3992,13 +4204,13 @@ static inline int bpf_map_check_op_flags(struct bpf_map *map, u64 flags, u64 all return -EINVAL; if (flags & (BPF_F_CPU | BPF_F_ALL_CPUS)) { - if (!bpf_map_supports_cpu_flags(map->map_type)) + if (!bpf_map_is_percpu_map(map->map_type)) return -EINVAL; if ((flags & BPF_F_CPU) && (flags & BPF_F_ALL_CPUS)) return -EINVAL; cpu = flags >> 32; - if ((flags & BPF_F_CPU) && cpu >= num_possible_cpus()) + if ((flags & BPF_F_CPU) && (cpu >= nr_cpu_ids || !cpu_possible(cpu))) return -ERANGE; } diff --git a/include/linux/bpf_defs.h b/include/linux/bpf_defs.h new file mode 100644 index 000000000000..2185cd3966d4 --- /dev/null +++ b/include/linux/bpf_defs.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Subset of bpf.h declarations, split out so files that need only these + * declarations can avoid bpf.h's full include cost. + */ +#ifndef _LINUX_BPF_DEFS_H +#define _LINUX_BPF_DEFS_H + +#ifdef CONFIG_BPF_SYSCALL +bool bpf_arena_handle_page_fault(unsigned long addr, bool is_write, unsigned long fault_ip); +#else +static inline bool bpf_arena_handle_page_fault(unsigned long addr, bool is_write, + unsigned long fault_ip) +{ + return false; +} +#endif + +#endif /* _LINUX_BPF_DEFS_H */ diff --git a/include/linux/bpf_ksock.h b/include/linux/bpf_ksock.h new file mode 100644 index 000000000000..cb387fb75e43 --- /dev/null +++ b/include/linux/bpf_ksock.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright (c) 2026 Isovalent */ + +#ifndef _BPF_KSOCK_H +#define _BPF_KSOCK_H + +#include <linux/types.h> +#include <linux/in.h> +#include <linux/in6.h> + +/** + * struct bpf_ksock_create_opts - BPF kernel socket creation parameters + * @family: Address family: AF_INET or AF_INET6. + * @type: Socket type: only SOCK_DGRAM supported for now. + * @protocol: Protocol number (e.g. IPPROTO_UDP), or 0 for the default protocol + * of the given type. + * @reserved: Must be zero. Reserved for future use. + */ +struct bpf_ksock_create_opts { + __u8 family; + __u8 type; + __u8 protocol; + __u8 reserved; +}; + +/** + * union bpf_ksock_addr - IPv4 or IPv6 socket address + * @sin: IPv4 socket address. + * @sin6: IPv6 socket address. + */ +union bpf_ksock_addr { + struct sockaddr_in sin; + struct sockaddr_in6 sin6; +}; + +#endif /* _BPF_KSOCK_H */ diff --git a/include/linux/bpf_local_storage.h b/include/linux/bpf_local_storage.h index 8157e8da61d4..9e4f5c45c974 100644 --- a/include/linux/bpf_local_storage.h +++ b/include/linux/bpf_local_storage.h @@ -54,7 +54,6 @@ struct bpf_local_storage_map { u32 bucket_log; u16 elem_size; u16 cache_idx; - bool use_kmalloc_nolock; }; struct bpf_local_storage_data { @@ -86,8 +85,7 @@ struct bpf_local_storage_elem { */ }; atomic_t state; - bool use_kmalloc_nolock; - /* 3 bytes hole */ + /* 4 bytes hole */ /* The data is stored in another cacheline to minimize * the number of cachelines access during a cache hit. */ @@ -104,7 +102,6 @@ struct bpf_local_storage { rqspinlock_t lock; /* Protect adding/removing from the "list" */ u64 mem_charge; /* Copy of mem charged to owner. Protected by "lock" */ refcount_t owner_refcnt;/* Used to pin owner when map_free is uncharging */ - bool use_kmalloc_nolock; }; /* U16_MAX is much more than enough for sk local storage @@ -137,8 +134,7 @@ int bpf_local_storage_map_alloc_check(union bpf_attr *attr); struct bpf_map * bpf_local_storage_map_alloc(union bpf_attr *attr, - struct bpf_local_storage_cache *cache, - bool use_kmalloc_nolock); + struct bpf_local_storage_cache *cache); void __bpf_local_storage_insert_cache(struct bpf_local_storage *local_storage, struct bpf_local_storage_map *smap, @@ -192,7 +188,7 @@ int bpf_selem_link_map(struct bpf_local_storage_map *smap, struct bpf_local_storage_elem * bpf_selem_alloc(struct bpf_local_storage_map *smap, void *owner, void *value, - bool swap_uptrs, gfp_t gfp_flags); + bool swap_uptrs); void bpf_selem_free(struct bpf_local_storage_elem *selem, bool reuse_now); @@ -200,12 +196,11 @@ void bpf_selem_free(struct bpf_local_storage_elem *selem, int bpf_local_storage_alloc(void *owner, struct bpf_local_storage_map *smap, - struct bpf_local_storage_elem *first_selem, - gfp_t gfp_flags); + struct bpf_local_storage_elem *first_selem); struct bpf_local_storage_data * bpf_local_storage_update(void *owner, struct bpf_local_storage_map *smap, - void *value, u64 map_flags, bool swap_uptrs, gfp_t gfp_flags); + void *value, u64 map_flags, bool swap_uptrs); u64 bpf_local_storage_map_mem_usage(const struct bpf_map *map); diff --git a/include/linux/bpf_lsm.h b/include/linux/bpf_lsm.h index 643809cc78c3..dda272d78f01 100644 --- a/include/linux/bpf_lsm.h +++ b/include/linux/bpf_lsm.h @@ -14,6 +14,8 @@ #ifdef CONFIG_BPF_LSM +extern bool bpf_lsm_initialized __ro_after_init; + #define LSM_HOOK(RET, DEFAULT, NAME, ...) \ RET bpf_lsm_##NAME(__VA_ARGS__); #include <linux/lsm_hook_defs.h> @@ -52,9 +54,12 @@ int bpf_set_dentry_xattr_locked(struct dentry *dentry, const char *name__str, const struct bpf_dynptr *value_p, int flags); int bpf_remove_dentry_xattr_locked(struct dentry *dentry, const char *name__str); bool bpf_lsm_has_d_inode_locked(const struct bpf_prog *prog); +bool bpf_lsm_hook_returns_errno(u32 btf_id); #else /* !CONFIG_BPF_LSM */ +#define bpf_lsm_initialized false + static inline bool bpf_lsm_is_sleepable_hook(u32 btf_id) { return false; @@ -104,6 +109,11 @@ static inline bool bpf_lsm_has_d_inode_locked(const struct bpf_prog *prog) { return false; } + +static inline bool bpf_lsm_hook_returns_errno(u32 btf_id) +{ + return true; +} #endif /* CONFIG_BPF_LSM */ #endif /* _LINUX_BPF_LSM_H */ diff --git a/include/linux/bpf_types.h b/include/linux/bpf_types.h index b13de31e163f..e5906829aa6f 100644 --- a/include/linux/bpf_types.h +++ b/include/linux/bpf_types.h @@ -134,6 +134,7 @@ BPF_MAP_TYPE(BPF_MAP_TYPE_BLOOM_FILTER, bloom_filter_map_ops) BPF_MAP_TYPE(BPF_MAP_TYPE_USER_RINGBUF, user_ringbuf_map_ops) BPF_MAP_TYPE(BPF_MAP_TYPE_ARENA, arena_map_ops) BPF_MAP_TYPE(BPF_MAP_TYPE_INSN_ARRAY, insn_array_map_ops) +BPF_MAP_TYPE(BPF_MAP_TYPE_RHASH, rhtab_map_ops) BPF_LINK_TYPE(BPF_LINK_TYPE_RAW_TRACEPOINT, raw_tracepoint) BPF_LINK_TYPE(BPF_LINK_TYPE_TRACING, tracing) @@ -155,3 +156,4 @@ BPF_LINK_TYPE(BPF_LINK_TYPE_PERF_EVENT, perf) BPF_LINK_TYPE(BPF_LINK_TYPE_KPROBE_MULTI, kprobe_multi) BPF_LINK_TYPE(BPF_LINK_TYPE_STRUCT_OPS, struct_ops) BPF_LINK_TYPE(BPF_LINK_TYPE_UPROBE_MULTI, uprobe_multi) +BPF_LINK_TYPE(BPF_LINK_TYPE_TRACING_MULTI, tracing_multi) diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index ef8e45a362d9..36b65797877d 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -8,13 +8,14 @@ #include <linux/btf.h> /* for struct btf and btf_id() */ #include <linux/filter.h> /* for MAX_BPF_STACK */ #include <linux/tnum.h> +#include <linux/cnum.h> /* Maximum variable offset umax_value permitted when resolving memory accesses. * In practice this is far bigger than any realistic pointer offset; this limit * ensures that umax_value + (int)off + (int)size cannot overflow a u64. */ #define BPF_MAX_VAR_OFF (1 << 29) -/* Maximum variable size permitted for ARG_CONST_SIZE[_OR_ZERO]. This ensures +/* Maximum variable size permitted for ARG_MEM_SIZE[_OR_ZERO]. This ensures * that converting umax_value to int cannot overflow. */ #define BPF_MAX_VAR_SIZ (1 << 29) @@ -38,10 +39,9 @@ struct bpf_reg_state { /* Ordering of fields matters. See states_equal() */ enum bpf_reg_type type; /* - * Fixed part of pointer offset, pointer types only. - * Or constant delta between "linked" scalars with the same ID. + * Constant delta between "linked" scalars with the same ID. */ - s32 off; + s32 delta; union { /* valid when type == PTR_TO_PACKET */ int range; @@ -66,7 +66,6 @@ struct bpf_reg_state { struct { /* for PTR_TO_MEM | PTR_TO_MEM_OR_NULL */ u32 mem_size; - u32 dynptr_id; /* for dynptr slices */ }; /* For dynptr stack slots */ @@ -121,14 +120,8 @@ struct bpf_reg_state { * These refer to the same value as var_off, not necessarily the actual * contents of the register. */ - s64 smin_value; /* minimum possible (s64)value */ - s64 smax_value; /* maximum possible (s64)value */ - u64 umin_value; /* minimum possible (u64)value */ - u64 umax_value; /* maximum possible (u64)value */ - s32 s32_min_value; /* minimum possible (s32)value */ - s32 s32_max_value; /* maximum possible (s32)value */ - u32 u32_min_value; /* minimum possible (u32)value */ - u32 u32_max_value; /* maximum possible (u32)value */ + struct cnum64 r64; /* 64-bit range as circular number */ + struct cnum32 r32; /* 32-bit range as circular number */ /* For PTR_TO_PACKET, used to find other pointers with the same variable * offset, so they can share range knowledge. * For PTR_TO_MAP_VALUE_OR_NULL this is used to share which map value we @@ -146,54 +139,22 @@ struct bpf_reg_state { * Upper bit of ID is used to remember relationship between "linked" * registers. Example: * r1 = r2; both will have r1->id == r2->id == N - * r1 += 10; r1->id == N | BPF_ADD_CONST and r1->off == 10 + * r1 += 10; r1->id == N | BPF_ADD_CONST and r1->delta == 10 * r3 = r2; both will have r3->id == r2->id == N - * w3 += 10; r3->id == N | BPF_ADD_CONST32 and r3->off == 10 + * w3 += 10; r3->id == N | BPF_ADD_CONST32 and r3->delta == 10 */ #define BPF_ADD_CONST64 (1U << 31) #define BPF_ADD_CONST32 (1U << 30) #define BPF_ADD_CONST (BPF_ADD_CONST64 | BPF_ADD_CONST32) u32 id; - /* PTR_TO_SOCKET and PTR_TO_TCP_SOCK could be a ptr returned - * from a pointer-cast helper, bpf_sk_fullsock() and - * bpf_tcp_sock(). - * - * Consider the following where "sk" is a reference counted - * pointer returned from "sk = bpf_sk_lookup_tcp();": - * - * 1: sk = bpf_sk_lookup_tcp(); - * 2: if (!sk) { return 0; } - * 3: fullsock = bpf_sk_fullsock(sk); - * 4: if (!fullsock) { bpf_sk_release(sk); return 0; } - * 5: tp = bpf_tcp_sock(fullsock); - * 6: if (!tp) { bpf_sk_release(sk); return 0; } - * 7: bpf_sk_release(sk); - * 8: snd_cwnd = tp->snd_cwnd; // verifier will complain - * - * After bpf_sk_release(sk) at line 7, both "fullsock" ptr and - * "tp" ptr should be invalidated also. In order to do that, - * the reg holding "fullsock" and "sk" need to remember - * the original refcounted ptr id (i.e. sk_reg->id) in ref_obj_id - * such that the verifier can reset all regs which have - * ref_obj_id matching the sk_reg->id. - * - * sk_reg->ref_obj_id is set to sk_reg->id at line 1. - * sk_reg->id will stay as NULL-marking purpose only. - * After NULL-marking is done, sk_reg->id can be reset to 0. - * - * After "fullsock = bpf_sk_fullsock(sk);" at line 3, - * fullsock_reg->ref_obj_id is set to sk_reg->ref_obj_id. - * - * After "tp = bpf_tcp_sock(fullsock);" at line 5, - * tp_reg->ref_obj_id is set to fullsock_reg->ref_obj_id - * which is the same as sk_reg->ref_obj_id. - * - * From the verifier perspective, if sk, fullsock and tp - * are not NULL, they are the same ptr with different - * reg->type. In particular, bpf_sk_release(tp) is also - * allowed and has the same effect as bpf_sk_release(sk). + /* + * Tracks the parent object this register was derived from. + * Used for cascading invalidation: when the parent object is + * released or invalidated, all registers with matching parent_id + * are also invalidated. For example, a slice from bpf_dynptr_data() + * gets parent_id set to the dynptr's id. */ - u32 ref_obj_id; + u32 parent_id; /* Inside the callee two registers can be both PTR_TO_STACK like * R1=fp-8 and R2=fp-8, but one of them points to this function stack * while another to the caller's stack. To differentiate them 'frameno' @@ -201,15 +162,70 @@ struct bpf_reg_state { * pointing to bpf_func_state. */ u32 frameno; - /* Tracks subreg definition. The stored value is the insn_idx of the - * writing insn. This is safe because subreg_def is used before any insn - * patching which only happens after main verification finished. - */ - s32 subreg_def; /* if (!precise && SCALAR_VALUE) min/max/tnum don't affect safety */ bool precise; }; +static inline s64 reg_smin(const struct bpf_reg_state *reg) +{ + return cnum64_smin(reg->r64); +} + +static inline s64 reg_smax(const struct bpf_reg_state *reg) +{ + return cnum64_smax(reg->r64); +} + +static inline u64 reg_umin(const struct bpf_reg_state *reg) +{ + return cnum64_umin(reg->r64); +} + +static inline u64 reg_umax(const struct bpf_reg_state *reg) +{ + return cnum64_umax(reg->r64); +} + +static inline s32 reg_s32_min(const struct bpf_reg_state *reg) +{ + return cnum32_smin(reg->r32); +} + +static inline s32 reg_s32_max(const struct bpf_reg_state *reg) +{ + return cnum32_smax(reg->r32); +} + +static inline u32 reg_u32_min(const struct bpf_reg_state *reg) +{ + return cnum32_umin(reg->r32); +} + +static inline u32 reg_u32_max(const struct bpf_reg_state *reg) +{ + return cnum32_umax(reg->r32); +} + +static inline void reg_set_srange32(struct bpf_reg_state *reg, s32 smin, s32 smax) +{ + reg->r32 = cnum32_from_srange(smin, smax); +} + +static inline void reg_set_urange32(struct bpf_reg_state *reg, u32 umin, u32 umax) +{ + reg->r32 = cnum32_from_urange(umin, umax); +} + +static inline void reg_set_srange64(struct bpf_reg_state *reg, s64 smin, s64 smax) +{ + reg->r64 = cnum64_from_srange(smin, smax); +} + +static inline void reg_set_urange64(struct bpf_reg_state *reg, u64 umin, u64 umax) +{ + reg->r64 = cnum64_from_urange(umin, umax); +} + enum bpf_stack_slot_type { STACK_INVALID, /* nothing was stored in this stack slot */ STACK_SPILL, /* register spilled into stack */ @@ -221,14 +237,67 @@ enum bpf_stack_slot_type { STACK_DYNPTR, STACK_ITER, STACK_IRQ_FLAG, + STACK_POISON, }; #define BPF_REG_SIZE 8 /* size of eBPF register in bytes */ +/* 4-byte stack slot granularity for liveness analysis */ +#define BPF_HALF_REG_SIZE 4 +#define STACK_SLOT_SZ 4 +#define STACK_SLOTS (MAX_BPF_STACK / BPF_HALF_REG_SIZE) /* 128 */ + +typedef struct { + u64 v[2]; +} spis_t; + +#define SPIS_ZERO ((spis_t){}) +#define SPIS_ALL ((spis_t){{ U64_MAX, U64_MAX }}) + +static inline bool spis_is_zero(spis_t s) +{ + return s.v[0] == 0 && s.v[1] == 0; +} + +static inline bool spis_equal(spis_t a, spis_t b) +{ + return a.v[0] == b.v[0] && a.v[1] == b.v[1]; +} + +static inline spis_t spis_or(spis_t a, spis_t b) +{ + return (spis_t){{ a.v[0] | b.v[0], a.v[1] | b.v[1] }}; +} + +static inline spis_t spis_and(spis_t a, spis_t b) +{ + return (spis_t){{ a.v[0] & b.v[0], a.v[1] & b.v[1] }}; +} + +static inline spis_t spis_not(spis_t s) +{ + return (spis_t){{ ~s.v[0], ~s.v[1] }}; +} + +static inline bool spis_test_bit(spis_t s, u32 slot) +{ + return s.v[slot / 64] & BIT_ULL(slot % 64); +} + +static inline void spis_or_range(spis_t *mask, u32 lo, u32 hi) +{ + u32 w; + + for (w = lo; w <= hi && w < STACK_SLOTS; w++) + mask->v[w / 64] |= BIT_ULL(w % 64); +} + #define BPF_REGMASK_ARGS ((1 << BPF_REG_1) | (1 << BPF_REG_2) | \ (1 << BPF_REG_3) | (1 << BPF_REG_4) | \ (1 << BPF_REG_5)) +#define BPF_MAIN_FUNC (-1) + #define BPF_DYNPTR_SIZE sizeof(struct bpf_dynptr_kern) #define BPF_DYNPTR_NR_SLOTS (BPF_DYNPTR_SIZE / BPF_REG_SIZE) @@ -257,15 +326,20 @@ struct bpf_reference_state { * is used purely to inform the user of a reference leak. */ int insn_idx; - /* Use to keep track of the source object of a lock, to ensure - * it matches on unlock. - */ - void *ptr; + union { + /* For REF_TYPE_PTR */ + int parent_id; + /* Use to keep track of the source object of a lock, to ensure + * it matches on unlock. + */ + void *ptr; + }; }; struct bpf_retval_range { s32 minval; s32 maxval; + bool return_32bit; }; /* state of the program: @@ -280,6 +354,11 @@ struct bpf_func_state { * 0 = main function, 1 = first callee. */ u32 frameno; + /* + * Unique diagnostic identity for this function invocation. Frame depth is + * reused after returns, while this ID is preserved across state clones. + */ + u32 diag_frame_id; /* subprog number == index within subprog_info * zero == main subprog */ @@ -294,6 +373,7 @@ struct bpf_func_state { bool in_callback_fn; bool in_async_callback_fn; bool in_exception_callback_fn; + bool no_stack_arg_load; /* For callback calling functions that limit number of possible * callback executions (e.g. bpf_loop) keeps track of current * simulated iteration number. @@ -305,6 +385,8 @@ struct bpf_func_state { * | number of simulations is tracked in frame N */ u32 callback_depth; + /* Instructions processed in this frame and callees on the current path. */ + u32 insns_subtotal; /* The following fields should be last. See copy_func_state() */ /* The state of the stack. Each element of the array describes BPF_REG_SIZE @@ -319,46 +401,49 @@ struct bpf_func_state { * `stack`. allocated_stack is always a multiple of BPF_REG_SIZE. */ int allocated_stack; + + u16 out_stack_arg_cnt; /* Number of outgoing on-stack argument slots */ + struct bpf_reg_state *stack_arg_regs; /* Outgoing on-stack arguments */ }; -#define MAX_CALL_FRAMES 8 +#define MAX_CALL_FRAMES 16 -/* instruction history flags, used in bpf_jmp_history_entry.flags field */ +/* instruction history flags, used in bpf_jmp_history_entry.flags field. + * Frame number and SPI are stored in dedicated fields of bpf_jmp_history_entry. + */ enum { - /* instruction references stack slot through PTR_TO_STACK register; - * we also store stack's frame number in lower 3 bits (MAX_CALL_FRAMES is 8) - * and accessed stack slot's index in next 6 bits (MAX_BPF_STACK is 512, - * 8 bytes per slot, so slot index (spi) is [0, 63]) - */ - INSN_F_FRAMENO_MASK = 0x7, /* 3 bits */ - - INSN_F_SPI_MASK = 0x3f, /* 6 bits */ - INSN_F_SPI_SHIFT = 3, /* shifted 3 bits to the left */ + INSN_F_STACK_ACCESS = BIT(0), - INSN_F_STACK_ACCESS = BIT(9), + INSN_F_DST_REG_STACK = BIT(1), /* dst_reg is PTR_TO_STACK */ + INSN_F_SRC_REG_STACK = BIT(2), /* src_reg is PTR_TO_STACK */ - INSN_F_DST_REG_STACK = BIT(10), /* dst_reg is PTR_TO_STACK */ - INSN_F_SRC_REG_STACK = BIT(11), /* src_reg is PTR_TO_STACK */ - /* total 12 bits are used now. */ + INSN_F_STACK_ARG_ACCESS = BIT(3), }; -static_assert(INSN_F_FRAMENO_MASK + 1 >= MAX_CALL_FRAMES); -static_assert(INSN_F_SPI_MASK + 1 >= MAX_BPF_STACK / 8); - struct bpf_jmp_history_entry { - u32 idx; /* insn idx can't be bigger than 1 million */ + u32 idx : 20; + u32 frame : 4; /* stack access frame number */ + u32 spi : 6; /* stack slot index (0..63) */ + u32 : 2; u32 prev_idx : 20; /* special INSN_F_xxx flags */ - u32 flags : 12; - /* additional registers that need precision tracking when this - * jump is backtracked, vector of six 10-bit records + u32 flags : 4; + u32 : 8; + /* + * additional registers that need precision tracking when this + * jump is backtracked, vector of five 11-bit records */ u64 linked_regs; }; -/* Maximum number of register states that can exist at once */ -#define BPF_ID_MAP_SIZE ((MAX_BPF_REG + MAX_BPF_STACK / BPF_REG_SIZE) * MAX_CALL_FRAMES) +static_assert(MAX_CALL_FRAMES <= (1 << 4)); +static_assert(MAX_BPF_STACK / 8 <= (1 << 6)); + +/* Maximum number of bpf_reg_state objects that can exist at once */ +#define MAX_STACK_ARG_SLOTS (MAX_BPF_FUNC_ARGS - MAX_BPF_FUNC_REG_ARGS) +#define BPF_ID_MAP_SIZE ((MAX_BPF_REG + MAX_BPF_STACK / BPF_REG_SIZE + \ + MAX_STACK_ARG_SLOTS) * MAX_CALL_FRAMES) struct bpf_verifier_state { /* call stack tracking */ struct bpf_func_state *frame[MAX_CALL_FRAMES]; @@ -424,7 +509,6 @@ struct bpf_verifier_state { bool speculative; bool in_sleepable; - bool cleaned; /* first and last insn idx of this verifier state */ u32 first_insn_idx; @@ -445,10 +529,23 @@ struct bpf_verifier_state { u32 may_goto_depth; }; -#define bpf_get_spilled_reg(slot, frame, mask) \ - (((slot < frame->allocated_stack / BPF_REG_SIZE) && \ - ((1 << frame->stack[slot].slot_type[BPF_REG_SIZE - 1]) & (mask))) \ - ? &frame->stack[slot].spilled_ptr : NULL) +static inline struct bpf_reg_state * +bpf_get_spilled_reg(int slot, struct bpf_func_state *frame, u32 mask) +{ + if (slot < frame->allocated_stack / BPF_REG_SIZE && + (1 << frame->stack[slot].slot_type[BPF_REG_SIZE - 1]) & mask) + return &frame->stack[slot].spilled_ptr; + return NULL; +} + +static inline struct bpf_reg_state * +bpf_get_spilled_stack_arg(int slot, struct bpf_func_state *frame) +{ + if (slot < frame->out_stack_arg_cnt && + frame->stack_arg_regs[slot].type != NOT_INIT) + return &frame->stack_arg_regs[slot]; + return NULL; +} /* Iterate over 'frame', setting 'reg' to either NULL or a spilled register. */ #define bpf_for_each_spilled_reg(iter, frame, reg, mask) \ @@ -456,7 +553,13 @@ struct bpf_verifier_state { iter < frame->allocated_stack / BPF_REG_SIZE; \ iter++, reg = bpf_get_spilled_reg(iter, frame, mask)) -#define bpf_for_each_reg_in_vstate_mask(__vst, __state, __reg, __mask, __expr) \ +/* Iterate over 'frame', setting 'reg' to either NULL or a spilled stack arg. */ +#define bpf_for_each_spilled_stack_arg(iter, frame, reg) \ + for (iter = 0, reg = bpf_get_spilled_stack_arg(iter, frame); \ + iter < frame->out_stack_arg_cnt; \ + iter++, reg = bpf_get_spilled_stack_arg(iter, frame)) + +#define bpf_for_each_reg_in_vstate_mask(__vst, __state, __reg, __stack, __mask, __expr) \ ({ \ struct bpf_verifier_state *___vstate = __vst; \ int ___i, ___j; \ @@ -464,6 +567,7 @@ struct bpf_verifier_state { struct bpf_reg_state *___regs; \ __state = ___vstate->frame[___i]; \ ___regs = __state->regs; \ + __stack = NULL; \ for (___j = 0; ___j < MAX_BPF_REG; ___j++) { \ __reg = &___regs[___j]; \ (void)(__expr); \ @@ -471,14 +575,27 @@ struct bpf_verifier_state { bpf_for_each_spilled_reg(___j, __state, __reg, __mask) { \ if (!__reg) \ continue; \ + __stack = &__state->stack[___j]; \ (void)(__expr); \ } \ + __stack = NULL; \ + bpf_for_each_spilled_stack_arg(___j, __state, __reg) { \ + if (!__reg) \ + continue; \ + (void)(__expr); \ + } \ } \ + (void)__stack; \ }) /* Invoke __expr over regsiters in __vst, setting __state and __reg */ -#define bpf_for_each_reg_in_vstate(__vst, __state, __reg, __expr) \ - bpf_for_each_reg_in_vstate_mask(__vst, __state, __reg, 1 << STACK_SPILL, __expr) +#define bpf_for_each_reg_in_vstate(__vst, __state, __reg, __expr) \ + ({ \ + struct bpf_stack_state * ___stack; \ + (void)___stack; \ + bpf_for_each_reg_in_vstate_mask(__vst, __state, __reg, ___stack,\ + 1 << STACK_SPILL, __expr); \ + }) /* linked list of verifier states used to prune search */ struct bpf_verifier_state_list { @@ -578,16 +695,19 @@ struct bpf_insn_aux_data { /* below fields are initialized once */ unsigned int orig_idx; /* original instruction index */ - bool jmp_point; - bool prune_point; + u32 jmp_point:1; + u32 prune_point:1; /* ensure we check state equivalence and save state checkpoint and * this instruction, regardless of any heuristics */ - bool force_checkpoint; + u32 force_checkpoint:1; /* true if instruction is a call to a helper function that * accepts callback function as a parameter. */ - bool calls_callback; + u32 calls_callback:1; + u32 indirect_target:1; /* if it is an indirect jump target */ + /* true if some jump or call instruction targets this instruction */ + u32 jump_target:1; /* * CFG strongly connected component this instruction belongs to, * zero if it is a singleton SCC. @@ -595,6 +715,18 @@ struct bpf_insn_aux_data { u32 scc; /* registers alive before this instruction. */ u16 live_regs_before; + /* + * Bitmask of R0-R9 that hold known values at this instruction. + * const_reg_mask: scalar constants that fit in 32 bits. + * const_reg_map_mask: map pointers, val is map_index into used_maps[]. + * const_reg_subprog_mask: subprog pointers, val is subprog number. + * const_reg_vals[i] holds the 32-bit value for register i. + * Populated by compute_const_regs() pre-pass. + */ + u16 const_reg_mask; + u16 const_reg_map_mask; + u16 const_reg_subprog_mask; + u32 const_reg_vals[10]; }; #define MAX_USED_MAPS 64 /* max number of maps accessed by one eBPF program */ @@ -635,6 +767,22 @@ static inline bool bpf_verifier_log_needed(const struct bpf_verifier_log *log) return log && log->level; } +struct bpf_log_attr { + char __user *ubuf; + u32 size; + u32 level; + u32 offsetof_true_size; + bpfptr_t uattr; +}; + +int bpf_log_attr_init(struct bpf_log_attr *log, u64 log_buf, u32 log_size, u32 log_level, + u32 offsetof_log_true_size, bpfptr_t uattr, struct bpf_common_attr *common, + bpfptr_t uattr_common, u32 size_common); +struct bpf_verifier_log *bpf_log_attr_create_vlog(struct bpf_log_attr *attr_log, + struct bpf_common_attr *common, bpfptr_t uattr, + u32 size); +int bpf_log_attr_finalize(struct bpf_log_attr *attr, struct bpf_verifier_log *log); + #define BPF_MAX_SUBPROGS 256 struct bpf_subprog_arg_info { @@ -652,18 +800,21 @@ enum priv_stack_mode { }; struct bpf_subprog_info { - /* 'start' has to be the first field otherwise find_subprog() won't work */ + const char *name; /* name extracted from BTF */ u32 start; /* insn idx of function entry point */ u32 linfo_idx; /* The idx to the main_prog->aux->linfo */ u32 postorder_start; /* The idx to the env->cfg.insn_postorder */ u32 exit_idx; /* Index of one of the BPF_EXIT instructions in this subprogram */ u16 stack_depth; /* max. stack depth used by this function */ u16 stack_extra; + u32 insns_total; + u32 insns_self; /* offsets in range [stack_depth .. fastcall_stack_off) * are used for bpf_fastcall spills and fills. */ s16 fastcall_stack_off; bool has_tail_call: 1; + bool might_throw: 1; bool tail_call_reachable: 1; bool has_ld_abs: 1; bool is_cb: 1; @@ -674,12 +825,22 @@ struct bpf_subprog_info { bool keep_fastcall_stack: 1; bool changes_pkt_data: 1; bool might_sleep: 1; - u8 arg_cnt:3; + u8 arg_cnt:4; enum priv_stack_mode priv_stack_mode; - struct bpf_subprog_arg_info args[MAX_BPF_FUNC_REG_ARGS]; + struct bpf_subprog_arg_info args[MAX_BPF_FUNC_ARGS]; + u16 stack_arg_cnt; /* incoming + max outgoing */ + u16 max_out_stack_arg_cnt; }; +static inline u16 bpf_in_stack_arg_cnt(const struct bpf_subprog_info *sub) +{ + if (sub->arg_cnt > MAX_BPF_FUNC_REG_ARGS) + return sub->arg_cnt - MAX_BPF_FUNC_REG_ARGS; + return 0; +} + +struct bpf_diag; struct bpf_verifier_env; struct backtrack_state { @@ -687,6 +848,7 @@ struct backtrack_state { u32 frame; u32 reg_masks[MAX_CALL_FRAMES]; u64 stack_masks[MAX_CALL_FRAMES]; + u8 stack_arg_masks[MAX_CALL_FRAMES]; }; struct bpf_id_pair { @@ -742,6 +904,14 @@ struct bpf_scc_info { struct bpf_liveness; +struct bpf_fd_array { + union { + struct bpf_map *map; + struct btf *btf; + unsigned long val; + }; +}; + /* single container for all structs * one verifier_env per bpf_check() call */ @@ -783,10 +953,15 @@ struct bpf_verifier_env { bool bypass_spec_v4; bool seen_direct_write; bool seen_exception; + bool signature; + u32 insn_aux_data_len; struct bpf_insn_aux_data *insn_aux_data; /* array of per-insn state */ const struct bpf_line_info *prev_linfo; struct bpf_verifier_log log; + struct bpf_diag *diag; struct bpf_subprog_info subprog_info[BPF_MAX_SUBPROGS + 2]; /* max + 2 for the fake and exception subprogs */ + /* subprog indices sorted in topological order: leaves first, callers last */ + int subprog_topo_order[BPF_MAX_SUBPROGS + 2]; union { struct bpf_idmap idmap_scratch; struct bpf_idset idset_scratch; @@ -805,12 +980,16 @@ struct bpf_verifier_env { } cfg; struct backtrack_state bt; struct bpf_jmp_history_entry *cur_hist_ent; + /* Per-callsite copy of parent's converged at_stack_in for cross-frame fills. */ + struct arg_track **callsite_at_stack; u32 pass_cnt; /* number of times do_check() was called */ u32 subprog_cnt; /* number of instructions analyzed by the verifier */ u32 prev_insn_processed, insn_processed; /* number of jmps, calls, exits analyzed so far */ u32 prev_jmps_processed, jmps_processed; + /* maximum combined stack depth */ + u32 max_stack_depth; /* total verification time */ u64 verification_time; /* maximum number of verifier states kept in 'branching' instructions */ @@ -827,7 +1006,19 @@ struct bpf_verifier_env { u32 free_list_size; u32 explored_states_size; u32 num_backedges; - bpfptr_t fd_array; + /* + * The program's fd_array comes in two shapes, told apart by whether + * the caller passed fd_array_cnt. They are mutually exclusive: + * - continuous (fd_array_cnt given): ->fd_array holds every entry + * resolved to its object up front, indexed by fd_array position, + * with ->fd_array_cnt slots; ->fd_array_raw is unused. + * - sparse (no fd_array_cnt): ->fd_array is NULL, and entries are + * read from ->fd_array_raw (the caller's fd_array) and resolved + * on the spot at each reference. + */ + struct bpf_fd_array *fd_array; + u32 fd_array_cnt; + bpfptr_t fd_array_raw; /* bit mask to keep track of whether a register has been accessed * since the last time the function state was printed @@ -837,11 +1028,14 @@ struct bpf_verifier_env { u64 scratched_stack_slots; u64 prev_log_pos, prev_insn_print_pos; /* buffer used to temporary hold constants as scalar registers */ - struct bpf_reg_state fake_reg[2]; + struct bpf_reg_state fake_reg[1]; + /* buffers used to save updated reg states while simulating branches */ + struct bpf_reg_state true_reg1, true_reg2, false_reg1, false_reg2; /* buffer used to generate temporary string representations, * e.g., in reg_type_str() to generate reg_type string */ char tmp_str_buf[TMP_STR_BUF_LEN]; + char tmp_arg_name[32]; struct bpf_insn insn_buf[INSN_BUF_SIZE]; struct bpf_insn epilogue_buf[INSN_BUF_SIZE]; struct bpf_scc_callchain callchain_buf; @@ -863,6 +1057,30 @@ static inline struct bpf_subprog_info *subprog_info(struct bpf_verifier_env *env return &env->subprog_info[subprog]; } +struct bpf_call_summary { + u8 num_params; + bool is_void; + bool fastcall; +}; + +static inline bool bpf_helper_call(const struct bpf_insn *insn) +{ + return insn->code == (BPF_JMP | BPF_CALL) && + insn->src_reg == 0; +} + +static inline bool bpf_pseudo_call(const struct bpf_insn *insn) +{ + return insn->code == (BPF_JMP | BPF_CALL) && + insn->src_reg == BPF_PSEUDO_CALL; +} + +static inline bool bpf_pseudo_kfunc_call(const struct bpf_insn *insn) +{ + return insn->code == (BPF_JMP | BPF_CALL) && + insn->src_reg == BPF_PSEUDO_KFUNC_CALL; +} + __printf(2, 0) void bpf_verifier_vlog(struct bpf_verifier_log *log, const char *fmt, va_list args); __printf(2, 3) void bpf_verifier_log_write(struct bpf_verifier_env *env, @@ -891,6 +1109,51 @@ __printf(3, 4) void verbose_linfo(struct bpf_verifier_env *env, bpf_log(&env->log, "verifier bug: " fmt "\n", ##args); \ }) +static inline void mark_prune_point(struct bpf_verifier_env *env, int idx) +{ + env->insn_aux_data[idx].prune_point = true; +} + +static inline bool bpf_is_prune_point(struct bpf_verifier_env *env, int insn_idx) +{ + return env->insn_aux_data[insn_idx].prune_point; +} + +static inline void mark_force_checkpoint(struct bpf_verifier_env *env, int idx) +{ + env->insn_aux_data[idx].force_checkpoint = true; +} + +static inline bool bpf_is_force_checkpoint(struct bpf_verifier_env *env, int insn_idx) +{ + return env->insn_aux_data[insn_idx].force_checkpoint; +} + +static inline void mark_calls_callback(struct bpf_verifier_env *env, int idx) +{ + env->insn_aux_data[idx].calls_callback = true; +} + +static inline bool bpf_calls_callback(struct bpf_verifier_env *env, int insn_idx) +{ + return env->insn_aux_data[insn_idx].calls_callback; +} + +static inline void mark_jmp_point(struct bpf_verifier_env *env, int idx) +{ + env->insn_aux_data[idx].jmp_point = true; +} + +static inline void mark_jump_target(struct bpf_verifier_env *env, int idx) +{ + env->insn_aux_data[idx].jump_target = true; +} + +static inline bool bpf_is_jump_target(struct bpf_verifier_env *env, int insn_idx) +{ + return env->insn_aux_data[insn_idx].jump_target; +} + static inline struct bpf_func_state *cur_func(struct bpf_verifier_env *env) { struct bpf_verifier_state *cur = env->cur_state; @@ -932,6 +1195,11 @@ static inline void bpf_trampoline_unpack_key(u64 key, u32 *obj_id, u32 *btf_id) *btf_id = key & 0x7FFFFFFF; } +int bpf_prepare_btf_info(struct bpf_verifier_env *env, + const union bpf_attr *attr, bpfptr_t uattr); +int bpf_check_btf_info(struct bpf_verifier_env *env, + const union bpf_attr *attr, bpfptr_t uattr); + int bpf_check_attach_target(struct bpf_verifier_log *log, const struct bpf_prog *prog, const struct bpf_prog *tgt_prog, @@ -941,6 +1209,103 @@ void bpf_free_kfunc_btf_tab(struct bpf_kfunc_btf_tab *tab); int mark_chain_precision(struct bpf_verifier_env *env, int regno); +int bpf_is_state_visited(struct bpf_verifier_env *env, int insn_idx); +int bpf_update_branch_counts(struct bpf_verifier_env *env, struct bpf_verifier_state *st); + +void bpf_clear_jmp_history(struct bpf_verifier_state *state); +int bpf_copy_verifier_state(struct bpf_verifier_state *dst_state, + const struct bpf_verifier_state *src); +struct list_head *bpf_explored_state(struct bpf_verifier_env *env, int idx); +void bpf_free_verifier_state(struct bpf_verifier_state *state, bool free_self); +void bpf_free_backedges(struct bpf_scc_visit *visit); +int bpf_push_jmp_history(struct bpf_verifier_env *env, struct bpf_verifier_state *cur, + int insn_flags, int spi, int frame, u64 linked_regs); +void bpf_bt_sync_linked_regs(struct backtrack_state *bt, struct bpf_jmp_history_entry *hist); +void bpf_mark_reg_not_init(const struct bpf_verifier_env *env, + struct bpf_reg_state *reg); +void bpf_mark_reg_unknown_imprecise(struct bpf_reg_state *reg); +void bpf_mark_all_scalars_precise(struct bpf_verifier_env *env, + struct bpf_verifier_state *st); +void bpf_clear_singular_ids(struct bpf_verifier_env *env, struct bpf_verifier_state *st); +int bpf_mark_chain_precision(struct bpf_verifier_env *env, + struct bpf_verifier_state *starting_state, + int regno, bool *changed); + +static inline int bpf_get_spi(s32 off) +{ + return (-off - 1) / BPF_REG_SIZE; +} + +static inline struct bpf_func_state *bpf_func(struct bpf_verifier_env *env, + const struct bpf_reg_state *reg) +{ + struct bpf_verifier_state *cur = env->cur_state; + + return cur->frame[reg->frameno]; +} + +/* Return IP for a given frame in a call stack */ +static inline u32 bpf_frame_insn_idx(struct bpf_verifier_state *st, u32 frame) +{ + return frame == st->curframe + ? st->insn_idx + : st->frame[frame + 1]->callsite; +} + +static inline bool bpf_is_jmp_point(struct bpf_verifier_env *env, int insn_idx) +{ + return env->insn_aux_data[insn_idx].jmp_point; +} + +static inline bool bpf_is_spilled_reg(const struct bpf_stack_state *stack) +{ + return stack->slot_type[BPF_REG_SIZE - 1] == STACK_SPILL; +} + +static inline bool bpf_is_spilled_scalar_reg(const struct bpf_stack_state *stack) +{ + return bpf_is_spilled_reg(stack) && stack->spilled_ptr.type == SCALAR_VALUE; +} + +static inline bool bpf_register_is_null(struct bpf_reg_state *reg) +{ + return reg->type == SCALAR_VALUE && tnum_equals_const(reg->var_off, 0); +} + +static inline void bpf_bt_set_frame_reg(struct backtrack_state *bt, u32 frame, u32 reg) +{ + bt->reg_masks[frame] |= 1 << reg; +} + +static inline void bpf_bt_set_frame_slot(struct backtrack_state *bt, u32 frame, u32 slot) +{ + bt->stack_masks[frame] |= 1ull << slot; +} + +static inline void bpf_bt_set_frame_slot_mask(struct backtrack_state *bt, u32 frame, u64 mask) +{ + bt->stack_masks[frame] |= mask; +} + +static inline void bt_set_frame_stack_arg_slot(struct backtrack_state *bt, u32 frame, u32 slot) +{ + bt->stack_arg_masks[frame] |= 1 << slot; +} + +static inline bool bt_is_frame_reg_set(struct backtrack_state *bt, u32 frame, u32 reg) +{ + return bt->reg_masks[frame] & (1 << reg); +} + +static inline bool bt_is_frame_slot_set(struct backtrack_state *bt, u32 frame, u32 slot) +{ + return bt->stack_masks[frame] & (1ull << slot); +} + +bool bpf_map_is_rdonly(const struct bpf_map *map); +int bpf_map_direct_read(struct bpf_map *map, int off, int size, u64 *val, + bool is_ldsx); + #define BPF_BASE_TYPE_MASK GENMASK(BPF_BASE_TYPE_BITS - 1, 0) /* extract base type from bpf_{arg, return, reg}_type. */ @@ -955,7 +1320,37 @@ static inline u32 type_flag(u32 type) return type & ~BPF_BASE_TYPE_MASK; } -/* only use after check_attach_btf_id() */ +static inline bool bpf_is_ptr_to_mem_or_btf_id(enum bpf_reg_type type) +{ + switch (base_type(type)) { + case PTR_TO_MEM: + case PTR_TO_BTF_ID: + return true; + default: + return false; + } +} + +static inline bool bpf_may_fault_on_deref(enum bpf_reg_type type) +{ + /* + * The pointer types which must not be dereferenced without fault + * protection, that is, the ones bpf_convert_ctx_accesses() has to + * turn a BPF_LDX into a BPF_PROBE_MEM one for. + */ + return type == PTR_TO_BTF_ID || (type_flag(type) & PTR_UNTRUSTED); +} + +static inline bool bpf_prog_has_arena_ctx_arg(const struct bpf_prog *prog) +{ + int i; + + for (i = 0; i < prog->aux->ctx_arg_info_size; i++) + if (base_type(prog->aux->ctx_arg_info[i].reg_type) == PTR_TO_ARENA) + return true; + return false; +} + static inline enum bpf_prog_type resolve_prog_type(const struct bpf_prog *prog) { return (prog->type == BPF_PROG_TYPE_EXT && prog->aux->saved_dst_prog_type) ? @@ -986,7 +1381,9 @@ static inline bool bpf_type_has_unsafe_modifiers(u32 type) static inline bool type_is_ptr_alloc_obj(u32 type) { - return base_type(type) == PTR_TO_BTF_ID && type_flag(type) & MEM_ALLOC; + return base_type(type) == PTR_TO_BTF_ID && + type_flag(type) & MEM_ALLOC && + !(type_flag(type) & PTR_UNTRUSTED); } static inline bool type_is_non_owning_ref(u32 type) @@ -994,6 +1391,18 @@ static inline bool type_is_non_owning_ref(u32 type) return type_is_ptr_alloc_obj(type) && type_flag(type) & NON_OWN_REF; } +static inline bool type_is_map_ptr(enum bpf_reg_type type) +{ + switch (base_type(type)) { + case CONST_PTR_TO_MAP: + case PTR_TO_MAP_KEY: + case PTR_TO_MAP_VALUE: + return true; + default: + return false; + } +} + static inline bool type_is_pkt_pointer(enum bpf_reg_type type) { type = base_type(type); @@ -1077,22 +1486,245 @@ void print_verifier_state(struct bpf_verifier_env *env, const struct bpf_verifie u32 frameno, bool print_all); void print_insn_state(struct bpf_verifier_env *env, const struct bpf_verifier_state *vstate, u32 frameno); +u32 bpf_vlog_alignment(u32 pos); +const char *bpf_disasm_kfunc_name(void *data, const struct bpf_insn *insn); struct bpf_subprog_info *bpf_find_containing_subprog(struct bpf_verifier_env *env, int off); +const char *bpf_subprog_name(const struct bpf_verifier_env *env, int subprog); int bpf_jmp_offset(struct bpf_insn *insn); struct bpf_iarray *bpf_insn_successors(struct bpf_verifier_env *env, u32 idx); void bpf_fmt_stack_mask(char *buf, ssize_t buf_sz, u64 stack_mask); -bool bpf_calls_callback(struct bpf_verifier_env *env, int insn_idx); +bool bpf_subprog_is_global(const struct bpf_verifier_env *env, int subprog); + +int bpf_find_subprog(struct bpf_verifier_env *env, int off); +bool bpf_is_throw_kfunc(struct bpf_insn *insn); +int bpf_compute_const_regs(struct bpf_verifier_env *env); +int bpf_prune_dead_branches(struct bpf_verifier_env *env); +int bpf_check_cfg(struct bpf_verifier_env *env); +int bpf_compute_postorder(struct bpf_verifier_env *env); +int bpf_compute_scc(struct bpf_verifier_env *env); + +struct bpf_map_desc { + struct bpf_map *ptr; + int uid; +}; + +/* The last initialized dynptr; Populated by process_dynptr_func() */ +struct bpf_dynptr_desc { + enum bpf_dynptr_type type; + u32 id; + u32 parent_id; +}; + +/* + * The last seen rereferenced object; Updated by update_ref_obj() when a register refers to a + * referenced object. Used when the helper or kfunc is casting a referenced object, returning + * allocated memory derived from referenced object or creating a dynptr with a referenced + * object as parent. + */ +struct ref_obj_desc { + u32 id; + u32 parent_id; + u8 cnt; +}; + +/* + * A memory argument a call fills in. The verifier allows the stack to be uninitialized if + * the range is a known constant. Stack slots are marked as STACK_MISC by check_mem_access(). + */ +struct arg_raw_mem_desc { + u8 regno; + int size; +}; + +/* Size of PTR_TO_MEM returned, taken from a constant allocation-size argument */ +struct ret_mem_desc { + u32 size; + bool found; +}; + +/* A constant scalar argument; Populated by process_const_arg() */ +struct arg_constant_desc { + u64 value; + bool found; +}; + +struct bpf_call_arg_meta { + /* Common */ + struct btf *btf; + u32 func_id; + const struct bpf_func_proto *fn; + u8 release_regno; + u32 ret_btf_id; + u32 subprogno; + struct bpf_map_desc map; + struct bpf_dynptr_desc dynptr; + struct ref_obj_desc ref_obj; + struct ret_mem_desc ret_mem; + + /* Only set by kfunc */ + bool r0_rdonly; + u32 kfunc_flags; + const struct btf_type *func_proto; + const char *func_name; + struct arg_constant_desc arg_constant; + + /* arg_{btf,btf_id,owning_ref} are used by kfunc-specific handling, + * generally to pass info about user-defined local kptr types to later + * verification logic + * bpf_obj_drop/bpf_percpu_obj_drop + * Record the local kptr type to be drop'd + * bpf_refcount_acquire (via KF_ARG_PTR_TO_REFCOUNTED_KPTR arg type) + * Record the local kptr type to be refcount_incr'd and use + * arg_owning_ref to determine whether refcount_acquire should be + * fallible + */ + struct btf *arg_btf; + u32 arg_btf_id; + bool arg_owning_ref; + bool arg_prog; + + struct { + struct btf_field *field; + } arg_list_head; + struct { + struct btf_field *field; + } arg_rbtree_root; + struct { + u8 spi; + u8 frameno; + } iter; + + /* Only set by helper */ + u64 msize_max_value; + s64 const_map_key; + struct btf *ret_btf; + struct btf_field *kptr_field; + struct arg_raw_mem_desc arg_raw_mem; +}; + +int bpf_get_helper_proto(struct bpf_verifier_env *env, int func_id, + const struct bpf_func_proto **ptr); +int bpf_fetch_kfunc_arg_meta(struct bpf_verifier_env *env, s32 func_id, + s16 offset, struct bpf_call_arg_meta *meta); +bool bpf_is_async_callback_calling_insn(struct bpf_insn *insn); +bool bpf_is_sync_callback_calling_insn(struct bpf_insn *insn); +static inline bool bpf_is_iter_next_kfunc(struct bpf_call_arg_meta *meta) +{ + return meta->kfunc_flags & KF_ITER_NEXT; +} + +static inline bool bpf_is_kfunc_sleepable(struct bpf_call_arg_meta *meta) +{ + return meta->kfunc_flags & KF_SLEEPABLE; +} +bool bpf_is_kfunc_pkt_changing(struct bpf_call_arg_meta *meta); +struct bpf_iarray *bpf_iarray_realloc(struct bpf_iarray *old, size_t n_elem); +int bpf_copy_insn_array_uniq(struct bpf_map *map, u32 start, u32 end, u32 *off); +bool bpf_insn_is_cond_jump(u8 code); +bool bpf_is_may_goto_insn(struct bpf_insn *insn); + +void bpf_verbose_insn(struct bpf_verifier_env *env, struct bpf_insn *insn); +bool bpf_get_call_summary(struct bpf_verifier_env *env, struct bpf_insn *call, + struct bpf_call_summary *cs); +s64 bpf_helper_stack_access_bytes(struct bpf_verifier_env *env, + struct bpf_insn *insn, int arg, + int insn_idx); +s64 bpf_kfunc_stack_access_bytes(struct bpf_verifier_env *env, + struct bpf_insn *insn, int arg, + int insn_idx); +int bpf_compute_subprog_arg_access(struct bpf_verifier_env *env); int bpf_stack_liveness_init(struct bpf_verifier_env *env); void bpf_stack_liveness_free(struct bpf_verifier_env *env); -int bpf_update_live_stack(struct bpf_verifier_env *env); -int bpf_mark_stack_read(struct bpf_verifier_env *env, u32 frameno, u32 insn_idx, u64 mask); -void bpf_mark_stack_write(struct bpf_verifier_env *env, u32 frameno, u64 mask); -int bpf_reset_stack_write_marks(struct bpf_verifier_env *env, u32 insn_idx); -int bpf_commit_stack_write_marks(struct bpf_verifier_env *env); int bpf_live_stack_query_init(struct bpf_verifier_env *env, struct bpf_verifier_state *st); bool bpf_stack_slot_alive(struct bpf_verifier_env *env, u32 frameno, u32 spi); -void bpf_reset_live_stack_callchain(struct bpf_verifier_env *env); +int bpf_compute_live_registers(struct bpf_verifier_env *env); + +#define BPF_MAP_KEY_POISON (1ULL << 63) +#define BPF_MAP_KEY_SEEN (1ULL << 62) + +static inline bool bpf_map_ptr_poisoned(const struct bpf_insn_aux_data *aux) +{ + return aux->map_ptr_state.poison; +} + +static inline bool bpf_map_ptr_unpriv(const struct bpf_insn_aux_data *aux) +{ + return aux->map_ptr_state.unpriv; +} + +static inline bool bpf_map_key_poisoned(const struct bpf_insn_aux_data *aux) +{ + return aux->map_key_state & BPF_MAP_KEY_POISON; +} + +static inline bool bpf_map_key_unseen(const struct bpf_insn_aux_data *aux) +{ + return !(aux->map_key_state & BPF_MAP_KEY_SEEN); +} + +static inline u64 bpf_map_key_immediate(const struct bpf_insn_aux_data *aux) +{ + return aux->map_key_state & ~(BPF_MAP_KEY_SEEN | BPF_MAP_KEY_POISON); +} + +#define MAX_PACKET_OFF 0xffff +#define CALLER_SAVED_REGS 6 + +enum bpf_reg_arg_type { + SRC_OP, /* register is used as source operand */ + DST_OP, /* register is used as destination operand */ + DST_OP_NO_MARK /* same as above, check only, don't mark */ +}; + +#define MAX_KFUNC_DESCS 256 + +struct bpf_kfunc_desc { + struct btf_func_model func_model; + struct bpf_func_proto proto; + u32 func_id; + s32 imm; + u16 offset; + unsigned long addr; +}; + +struct bpf_kfunc_desc_tab { + u32 nr_descs; + /* Sorted by func_id (BTF ID) and offset (fd_array offset) during + * verification. JITs do lookups by bpf_insn, where func_id may not be + * available, therefore at the end of verification do_misc_fixups() + * sorts this by imm and offset. + * + * Grown one entry at a time by bpf_add_kfunc_call(). + */ + struct bpf_kfunc_desc descs[]; +}; + +/* Functions exported from verifier.c, used by fixups.c */ +void bpf_clear_insn_aux_data(struct bpf_verifier_env *env, int start, int len); +void bpf_mark_subprog_exc_cb(struct bpf_verifier_env *env, int subprog); +bool bpf_allow_tail_call_in_subprogs(struct bpf_verifier_env *env); +bool bpf_verifier_inlines_helper_call(struct bpf_verifier_env *env, s32 imm); +int bpf_add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, u16 offset); +int bpf_fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, + struct bpf_insn *insn_buf, int insn_idx, int *cnt); + +/* Functions exported from verifier.c, used by trampoline.c */ +int bpf_check_attach_btf_id_multi(struct btf *btf, struct bpf_prog *prog, u32 btf_id, + struct bpf_attach_target_info *tgt_info); + +/* Functions in fixups.c, called from bpf_check() */ +int bpf_remove_fastcall_spills_fills(struct bpf_verifier_env *env); +int bpf_optimize_bpf_loop(struct bpf_verifier_env *env); +void bpf_opt_hard_wire_dead_code_branches(struct bpf_verifier_env *env); +int bpf_opt_remove_dead_code(struct bpf_verifier_env *env); +int bpf_opt_remove_nops(struct bpf_verifier_env *env); +int bpf_opt_subreg_zext_lo32_rnd_hi32(struct bpf_verifier_env *env, const union bpf_attr *attr); +int bpf_convert_ctx_accesses(struct bpf_verifier_env *env); +int bpf_jit_subprogs(struct bpf_verifier_env *env); +int bpf_fixup_call_args(struct bpf_verifier_env *env); +int bpf_do_misc_fixups(struct bpf_verifier_env *env); +int bpf_insn_def32(struct bpf_prog *prog, struct bpf_insn *insn); #endif /* _LINUX_BPF_VERIFIER_H */ diff --git a/include/linux/brcmphy.h b/include/linux/brcmphy.h index 115a964f3006..174687c4c80a 100644 --- a/include/linux/brcmphy.h +++ b/include/linux/brcmphy.h @@ -266,6 +266,9 @@ #define BCM54XX_TOP_MISC_IDDQ_SD (1 << 2) #define BCM54XX_TOP_MISC_IDDQ_SR (1 << 3) +#define BCM54XX_TOP_MISC_MII_BUF_CNTL0 (MII_BCM54XX_EXP_SEL_TOP + 0x00) +#define BCM54XX_MII_BUF_CNTL0_AUTOGREEEN_EN BIT(0) + #define BCM54XX_TOP_MISC_LED_CTL (MII_BCM54XX_EXP_SEL_TOP + 0x0C) #define BCM54XX_LED4_SEL_INTR BIT(1) diff --git a/include/linux/bsg.h b/include/linux/bsg.h index ee2df73edf83..162730bfc2d8 100644 --- a/include/linux/bsg.h +++ b/include/linux/bsg.h @@ -7,13 +7,17 @@ struct bsg_device; struct device; struct request_queue; +struct io_uring_cmd; typedef int (bsg_sg_io_fn)(struct request_queue *, struct sg_io_v4 *hdr, bool open_for_write, unsigned int timeout); +typedef int (bsg_uring_cmd_fn)(struct request_queue *q, struct io_uring_cmd *ioucmd, + unsigned int issue_flags, bool open_for_write); + struct bsg_device *bsg_register_queue(struct request_queue *q, struct device *parent, const char *name, - bsg_sg_io_fn *sg_io_fn); + bsg_sg_io_fn *sg_io_fn, bsg_uring_cmd_fn *uring_cmd_fn); void bsg_unregister_queue(struct bsg_device *bcd); #endif /* _LINUX_BSG_H */ diff --git a/include/linux/btf.h b/include/linux/btf.h index 48108471c5b1..89d5a5c4f117 100644 --- a/include/linux/btf.h +++ b/include/linux/btf.h @@ -79,6 +79,7 @@ #define KF_ARENA_ARG1 (1 << 14) /* kfunc takes an arena pointer as its first argument */ #define KF_ARENA_ARG2 (1 << 15) /* kfunc takes an arena pointer as its second argument */ #define KF_IMPLICIT_ARGS (1 << 16) /* kfunc has implicit arguments supplied by the verifier */ +#define KF_SPINLOCK_SAFE (1 << 17) /* kfunc is allowed inside bpf_spin_lock-ed region */ /* * Tag marking a kernel function as a kfunc. This is meant to minimize the @@ -145,7 +146,8 @@ const char *btf_get_name(const struct btf *btf); void btf_get(struct btf *btf); void btf_put(struct btf *btf); const struct btf_header *btf_header(const struct btf *btf); -int btf_new_fd(const union bpf_attr *attr, bpfptr_t uattr, u32 uattr_sz); +struct bpf_log_attr; +int btf_new_fd(const union bpf_attr *attr, bpfptr_t uattr, struct bpf_log_attr *attr_log); struct btf *btf_get_by_fd(int fd); int btf_get_info_by_fd(const struct btf *btf, const union bpf_attr *attr, @@ -212,6 +214,7 @@ int btf_type_seq_show_flags(const struct btf *btf, u32 type_id, void *obj, */ int btf_type_snprintf_show(const struct btf *btf, u32 type_id, void *obj, char *buf, int len, u64 flags); +int btf_type_name_to_buf(const struct btf *btf, u32 type_id, char *buf, int len); int btf_get_fd_by_id(u32 id); u32 btf_obj_id(const struct btf *btf); @@ -234,6 +237,8 @@ int btf_check_and_fixup_fields(const struct btf *btf, struct btf_record *rec); bool btf_type_is_void(const struct btf_type *t); s32 btf_find_by_name_kind(const struct btf *btf, const char *name, u8 kind); s32 bpf_find_btf_id(const char *name, u32 kind, struct btf **btf_p); +struct btf *btf_get_module_btf(const struct module *module); +__u32 btf_relocate_id(const struct btf *btf, __u32 id); const struct btf_type *btf_type_skip_modifiers(const struct btf *btf, u32 id, u32 *res_id); const struct btf_type *btf_type_resolve_ptr(const struct btf *btf, @@ -415,12 +420,12 @@ static inline bool btf_type_is_array(const struct btf_type *t) return BTF_INFO_KIND(t->info) == BTF_KIND_ARRAY; } -static inline u16 btf_type_vlen(const struct btf_type *t) +static inline u32 btf_type_vlen(const struct btf_type *t) { return BTF_INFO_VLEN(t->info); } -static inline u16 btf_vlen(const struct btf_type *t) +static inline u32 btf_vlen(const struct btf_type *t) { return btf_type_vlen(t); } @@ -577,6 +582,7 @@ const char *btf_str_by_offset(const struct btf *btf, u32 offset); struct btf *btf_parse_vmlinux(void); struct btf *bpf_prog_get_target_btf(const struct bpf_prog *prog); u32 *btf_kfunc_flags(const struct btf *btf, u32 kfunc_btf_id, const struct bpf_prog *prog); +int btf_kfunc_check_flag(const struct btf *btf, u32 kfunc_btf_id, u32 flag); bool btf_kfunc_is_allowed(const struct btf *btf, u32 kfunc_btf_id, const struct bpf_prog *prog); u32 *btf_kfunc_is_modify_return(const struct btf *btf, u32 kfunc_btf_id, const struct bpf_prog *prog); diff --git a/include/linux/btf_ids.h b/include/linux/btf_ids.h index 139bdececdcf..8b5a9ee92513 100644 --- a/include/linux/btf_ids.h +++ b/include/linux/btf_ids.h @@ -217,7 +217,7 @@ BTF_SET8_END(name) #else -#define BTF_ID_LIST(name) static u32 __maybe_unused name[64]; +#define BTF_ID_LIST(name) static u32 __maybe_unused name[128]; #define BTF_ID(prefix, name) #define BTF_ID_FLAGS(prefix, name, ...) #define BTF_ID_UNUSED @@ -284,5 +284,6 @@ extern u32 bpf_cgroup_btf_id[]; extern u32 bpf_local_storage_map_btf_id[]; extern u32 btf_bpf_map_id[]; extern u32 bpf_kmem_cache_btf_id[]; +extern u32 bpf_multi_func_btf_id[]; #endif diff --git a/include/linux/buffer_head.h b/include/linux/buffer_head.h index b16b88bfbc3e..fd2c7115c054 100644 --- a/include/linux/buffer_head.h +++ b/include/linux/buffer_head.h @@ -46,7 +46,6 @@ enum bh_state_bits { struct page; struct buffer_head; struct address_space; -typedef void (bh_end_io_t)(struct buffer_head *bh, int uptodate); /* * Historically, a buffer_head was used to map a single block @@ -55,7 +54,7 @@ typedef void (bh_end_io_t)(struct buffer_head *bh, int uptodate); * is the bio, and buffer_heads are used for extracting block * mappings (via a get_block_t call), for tracking state within * a folio (via a folio_mapping) and for wrapping bio submission - * for backward compatibility reasons (e.g. submit_bh). + * for backward compatibility reasons (e.g. bh_submit). */ struct buffer_head { unsigned long b_state; /* buffer state bitmap (see above) */ @@ -70,11 +69,10 @@ struct buffer_head { char *b_data; /* pointer to data within the page */ struct block_device *b_bdev; - bh_end_io_t *b_end_io; /* I/O completion */ - void *b_private; /* reserved for b_end_io */ + void *b_private; /* reserved for bio_end_io */ struct list_head b_assoc_buffers; /* associated with another mapping */ - struct address_space *b_assoc_map; /* mapping this buffer is - associated with */ + struct mapping_metadata_bhs *b_mmb; /* head of the list of metadata bhs + * this buffer is associated with */ atomic_t b_count; /* users using this buffer_head */ spinlock_t b_uptodate_lock; /* Used by the first bh in a page, to * serialise IO completion of other @@ -203,14 +201,15 @@ struct buffer_head *alloc_page_buffers(struct page *page, unsigned long size); struct buffer_head *create_empty_buffers(struct folio *folio, unsigned long blocksize, unsigned long b_state); void end_buffer_read_sync(struct buffer_head *bh, int uptodate); -void end_buffer_write_sync(struct buffer_head *bh, int uptodate); - -/* Things to do with buffers at mapping->private_list */ -void mark_buffer_dirty_inode(struct buffer_head *bh, struct inode *inode); -int generic_buffers_fsync_noflush(struct file *file, loff_t start, loff_t end, - bool datasync); -int generic_buffers_fsync(struct file *file, loff_t start, loff_t end, - bool datasync); +bool bio_endio_bh(struct bio *bio, struct buffer_head **bhp); + +/* Completion routines suitable for passing to bh_submit() */ +void bh_end_read(struct bio *bio); +void bh_end_write(struct bio *bio); +void bh_end_async_write(struct bio *bio); + +/* Things to do with metadata buffers list */ +void mmb_mark_buffer_dirty(struct buffer_head *bh, struct mapping_metadata_bhs *mmb); void clean_bdev_aliases(struct block_device *bdev, sector_t block, sector_t len); static inline void clean_bdev_bh_alias(struct buffer_head *bh) @@ -218,7 +217,6 @@ static inline void clean_bdev_bh_alias(struct buffer_head *bh) clean_bdev_aliases(bh->b_bdev, bh->b_blocknr, 1); } -void mark_buffer_async_write(struct buffer_head *bh); void __wait_on_buffer(struct buffer_head *); wait_queue_head_t *bh_waitq_head(struct buffer_head *bh); struct buffer_head *__find_get_block(struct block_device *bdev, sector_t block, @@ -239,7 +237,7 @@ void __lock_buffer(struct buffer_head *bh); int sync_dirty_buffer(struct buffer_head *bh); int __sync_dirty_buffer(struct buffer_head *bh, blk_opf_t op_flags); void write_dirty_buffer(struct buffer_head *bh, blk_opf_t op_flags); -void submit_bh(blk_opf_t, struct buffer_head *); +void bh_submit(struct buffer_head *, blk_opf_t, bio_end_io_t); void write_boundary_block(struct block_device *bdev, sector_t bblock, unsigned blocksize); int bh_uptodate_or_lock(struct buffer_head *bh); @@ -515,10 +513,10 @@ bool block_dirty_folio(struct address_space *mapping, struct folio *folio); void buffer_init(void); bool try_to_free_buffers(struct folio *folio); -int inode_has_buffers(struct inode *inode); -void invalidate_inode_buffers(struct inode *inode); -int remove_inode_buffers(struct inode *inode); -int sync_mapping_buffers(struct address_space *mapping); +void mmb_init(struct mapping_metadata_bhs *mmb, struct address_space *mapping); +bool mmb_has_buffers(struct mapping_metadata_bhs *mmb); +void mmb_invalidate(struct mapping_metadata_bhs *mmb); +int mmb_sync(struct mapping_metadata_bhs *mmb); void invalidate_bh_lrus(void); void invalidate_bh_lrus_cpu(void); bool has_bh_in_lru(int cpu, void *dummy); @@ -528,10 +526,7 @@ extern int buffer_heads_over_limit; static inline void buffer_init(void) {} static inline bool try_to_free_buffers(struct folio *folio) { return true; } -static inline int inode_has_buffers(struct inode *inode) { return 0; } -static inline void invalidate_inode_buffers(struct inode *inode) {} -static inline int remove_inode_buffers(struct inode *inode) { return 1; } -static inline int sync_mapping_buffers(struct address_space *mapping) { return 0; } +static inline int mmb_sync(struct mapping_metadata_bhs *mmb) { return 0; } static inline void invalidate_bh_lrus(void) {} static inline void invalidate_bh_lrus_cpu(void) {} static inline bool has_bh_in_lru(int cpu, void *dummy) { return false; } diff --git a/include/linux/bus/stm32_firewall.h b/include/linux/bus/stm32_firewall.h new file mode 100644 index 000000000000..e5fac85fe346 --- /dev/null +++ b/include/linux/bus/stm32_firewall.h @@ -0,0 +1,83 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2023, STMicroelectronics - All Rights Reserved + */ + +#ifndef _STM32_FIREWALL_H +#define _STM32_FIREWALL_H + +#include <linux/kernel.h> +#include <linux/list.h> +#include <linux/of.h> +#include <linux/platform_device.h> +#include <linux/types.h> + +/** + * STM32_PERIPHERAL_FIREWALL: This type of firewall protects peripherals + * STM32_MEMORY_FIREWALL: This type of firewall protects memories/subsets of memory + * zones + * STM32_NOTYPE_FIREWALL: Undefined firewall type + */ + +#define STM32_PERIPHERAL_FIREWALL BIT(1) +#define STM32_MEMORY_FIREWALL BIT(2) +#define STM32_NOTYPE_FIREWALL BIT(3) + +/** + * struct stm32_firewall_controller - Information on firewall controller supplying services + * + * @name: Name of the firewall controller + * @dev: Device reference of the firewall controller + * @mmio: Base address of the firewall controller + * @entry: List entry of the firewall controller list + * @type: Type of firewall + * @max_entries: Number of entries covered by the firewall + * @grant_access: Callback used to grant access for a device access against a + * firewall controller + * @release_access: Callback used to release resources taken by a device when access was + * granted + * @grant_memory_range_access: Callback used to grant access for a device to a given memory region + */ +struct stm32_firewall_controller { + const char *name; + struct device *dev; + void __iomem *mmio; + struct list_head entry; + unsigned int type; + unsigned int max_entries; + + int (*grant_access)(struct stm32_firewall_controller *ctrl, u32 id); + void (*release_access)(struct stm32_firewall_controller *ctrl, u32 id); + int (*grant_memory_range_access)(struct stm32_firewall_controller *ctrl, phys_addr_t paddr, + size_t size); +}; + +/** + * stm32_firewall_controller_register - Register a firewall controller to the STM32 firewall + * framework + * @firewall_controller: Firewall controller to register + * + * Returns 0 in case of success or -ENODEV if no controller was given. + */ +int stm32_firewall_controller_register(struct stm32_firewall_controller *firewall_controller); + +/** + * stm32_firewall_controller_unregister - Unregister a firewall controller from the STM32 + * firewall framework + * @firewall_controller: Firewall controller to unregister + */ +void stm32_firewall_controller_unregister(struct stm32_firewall_controller *firewall_controller); + +/** + * stm32_firewall_populate_bus - Populate device tree nodes that have a correct firewall + * configuration. This is used at boot-time only, as a sanity check + * between device tree and firewalls hardware configurations to + * prevent a kernel crash when a device driver is not granted access + * + * @firewall_controller: Firewall controller which nodes will be populated or not + * + * Returns 0 in case of success or appropriate errno code if error occurred. + */ +int stm32_firewall_populate_bus(struct stm32_firewall_controller *firewall_controller); + +#endif /* _STM32_FIREWALL_H */ diff --git a/include/linux/bus/stm32_firewall_device.h b/include/linux/bus/stm32_firewall_device.h index eaa7a3f54450..6c878f3ca86f 100644 --- a/include/linux/bus/stm32_firewall_device.h +++ b/include/linux/bus/stm32_firewall_device.h @@ -112,6 +112,25 @@ int stm32_firewall_grant_access_by_id(struct stm32_firewall *firewall, u32 subsy */ void stm32_firewall_release_access_by_id(struct stm32_firewall *firewall, u32 subsystem_id); +/** + * stm32_firewall_get_grant_all_access - Allocate and get all the firewall(s) associated to given + * device. Then, try to grant access rights for each element. + * This function is basically a helper function that wraps + * both stm32_firewall_get_firewall() and + * stm32_firewall_grant_access() on all firewall references of + * a device along with the allocation of the array. + * Realease access using stm32_firewall_release_access* APIs + * when done. + * + * @dev: Device performing the checks + * @firewall: Pointer to the array of firewall references to be allocated + * @nb_firewall: Number of allocated elements in @firewall + * + * Returns 0 on success, or appropriate errno code if error occurred. + */ +int stm32_firewall_get_grant_all_access(struct device *dev, struct stm32_firewall **firewall, + int *nb_firewall); + #else /* CONFIG_STM32_FIREWALL */ static inline int stm32_firewall_get_firewall(struct device_node *np, @@ -141,5 +160,12 @@ static inline void stm32_firewall_release_access_by_id(struct stm32_firewall *fi { } +static inline int stm32_firewall_get_grant_all_access(struct device *dev, + struct stm32_firewall **firewall, + int *nb_firewall) +{ + return -ENODEV; +} + #endif /* CONFIG_STM32_FIREWALL */ #endif /* STM32_FIREWALL_DEVICE_H */ diff --git a/include/linux/bvec.h b/include/linux/bvec.h index 06fb60471aaf..fc566ee1c1ff 100644 --- a/include/linux/bvec.h +++ b/include/linux/bvec.h @@ -74,6 +74,21 @@ static inline void bvec_set_virt(struct bio_vec *bv, void *vaddr, bvec_set_page(bv, virt_to_page(vaddr), len, offset_in_page(vaddr)); } +/** + * bvec_folio - Return the first folio referenced by this bvec + * @bv: bvec to access + * + * A bvec can contain non-folio memory, so this should only be called by + * the creator of the bvec; drivers have no business looking at the owner + * of the memory. It may not even be the right interface for the caller + * to use as a bvec can span multiple folios. You may be better off using + * something like bio_for_each_folio_all() which iterates over all folios. + */ +static inline struct folio *bvec_folio(const struct bio_vec *bv) +{ + return page_folio(bv->bv_page); +} + struct bvec_iter { /* * Current device address in 512 byte sectors. Only updated by the bio @@ -95,7 +110,7 @@ struct bvec_iter { /* * Current offset in the bvec entry pointed to by `bi_idx`. */ - unsigned int bi_bvec_done; + unsigned int bi_offset; } __packed __aligned(4); struct bvec_iter_all { @@ -104,51 +119,78 @@ struct bvec_iter_all { unsigned done; }; -/* - * various member access, note that bio_data should of course not be used - * on highmem page vectors - */ -#define __bvec_iter_bvec(bvec, iter) (&(bvec)[(iter).bi_idx]) +static __always_inline const struct bio_vec * +__bvec_iter_bvec(const struct bio_vec *bvecs, const struct bvec_iter iter) +{ + return bvecs + iter.bi_idx; +} /* multi-page (mp_bvec) helpers */ -#define mp_bvec_iter_page(bvec, iter) \ - (__bvec_iter_bvec((bvec), (iter))->bv_page) +static __always_inline struct page * +mp_bvec_iter_page(const struct bio_vec *bvecs, const struct bvec_iter iter) +{ + return __bvec_iter_bvec(bvecs, iter)->bv_page; +} -#define mp_bvec_iter_len(bvec, iter) \ - min((iter).bi_size, \ - __bvec_iter_bvec((bvec), (iter))->bv_len - (iter).bi_bvec_done) +static __always_inline unsigned int +mp_bvec_iter_len(const struct bio_vec *bvecs, const struct bvec_iter iter) +{ + return min(__bvec_iter_bvec(bvecs, iter)->bv_len - iter.bi_offset, + iter.bi_size); +} -#define mp_bvec_iter_offset(bvec, iter) \ - (__bvec_iter_bvec((bvec), (iter))->bv_offset + (iter).bi_bvec_done) +static __always_inline unsigned int +mp_bvec_iter_offset(const struct bio_vec *bvecs, const struct bvec_iter iter) +{ + return __bvec_iter_bvec(bvecs, iter)->bv_offset + iter.bi_offset; +} -#define mp_bvec_iter_page_idx(bvec, iter) \ - (mp_bvec_iter_offset((bvec), (iter)) / PAGE_SIZE) +static __always_inline unsigned int +mp_bvec_iter_page_idx(const struct bio_vec *bvecs, const struct bvec_iter iter) +{ + return mp_bvec_iter_offset(bvecs, iter) / PAGE_SIZE; +} -#define mp_bvec_iter_bvec(bvec, iter) \ -((struct bio_vec) { \ - .bv_page = mp_bvec_iter_page((bvec), (iter)), \ - .bv_len = mp_bvec_iter_len((bvec), (iter)), \ - .bv_offset = mp_bvec_iter_offset((bvec), (iter)), \ -}) +static __always_inline struct bio_vec +mp_bvec_iter_bvec(const struct bio_vec *bvecs, const struct bvec_iter iter) +{ + return (struct bio_vec) { + .bv_page = mp_bvec_iter_page(bvecs, iter), + .bv_len = mp_bvec_iter_len(bvecs, iter), + .bv_offset = mp_bvec_iter_offset(bvecs, iter), + }; +} /* For building single-page bvec in flight */ - #define bvec_iter_offset(bvec, iter) \ - (mp_bvec_iter_offset((bvec), (iter)) % PAGE_SIZE) +static __always_inline unsigned int +bvec_iter_offset(const struct bio_vec *bvecs, const struct bvec_iter iter) +{ + return mp_bvec_iter_offset(bvecs, iter) % PAGE_SIZE; +} -#define bvec_iter_len(bvec, iter) \ - min_t(unsigned, mp_bvec_iter_len((bvec), (iter)), \ - PAGE_SIZE - bvec_iter_offset((bvec), (iter))) +static __always_inline unsigned int +bvec_iter_len(const struct bio_vec *bvecs, const struct bvec_iter iter) +{ + return min(mp_bvec_iter_len(bvecs, iter), + PAGE_SIZE - bvec_iter_offset(bvecs, iter)); +} -#define bvec_iter_page(bvec, iter) \ - (mp_bvec_iter_page((bvec), (iter)) + \ - mp_bvec_iter_page_idx((bvec), (iter))) +static __always_inline struct page * +bvec_iter_page(const struct bio_vec *bvecs, const struct bvec_iter iter) +{ + return mp_bvec_iter_page(bvecs, iter) + + mp_bvec_iter_page_idx(bvecs, iter); +} -#define bvec_iter_bvec(bvec, iter) \ -((struct bio_vec) { \ - .bv_page = bvec_iter_page((bvec), (iter)), \ - .bv_len = bvec_iter_len((bvec), (iter)), \ - .bv_offset = bvec_iter_offset((bvec), (iter)), \ -}) +static __always_inline struct bio_vec +bvec_iter_bvec(const struct bio_vec *bvecs, const struct bvec_iter iter) +{ + return (struct bio_vec) { + .bv_page = bvec_iter_page(bvecs, iter), + .bv_len = bvec_iter_len(bvecs, iter), + .bv_offset = bvec_iter_offset(bvecs, iter), + }; +} static inline bool bvec_iter_advance(const struct bio_vec *bv, struct bvec_iter *iter, unsigned bytes) @@ -162,7 +204,7 @@ static inline bool bvec_iter_advance(const struct bio_vec *bv, } iter->bi_size -= bytes; - bytes += iter->bi_bvec_done; + bytes += iter->bi_offset; while (bytes && bytes >= bv[idx].bv_len) { bytes -= bv[idx].bv_len; @@ -170,7 +212,7 @@ static inline bool bvec_iter_advance(const struct bio_vec *bv, } iter->bi_idx = idx; - iter->bi_bvec_done = bytes; + iter->bi_offset = bytes; return true; } @@ -181,13 +223,13 @@ static inline bool bvec_iter_advance(const struct bio_vec *bv, static inline void bvec_iter_advance_single(const struct bio_vec *bv, struct bvec_iter *iter, unsigned int bytes) { - unsigned int done = iter->bi_bvec_done + bytes; + unsigned int done = iter->bi_offset + bytes; if (done == bv[iter->bi_idx].bv_len) { done = 0; iter->bi_idx++; } - iter->bi_bvec_done = done; + iter->bi_offset = done; iter->bi_size -= bytes; } @@ -203,15 +245,6 @@ static inline void bvec_iter_advance_single(const struct bio_vec *bv, ((bvl = mp_bvec_iter_bvec((bio_vec), (iter))), 1); \ bvec_iter_advance_single((bio_vec), &(iter), (bvl).bv_len)) -/* for iterating one bio from start to end */ -#define BVEC_ITER_ALL_INIT (struct bvec_iter) \ -{ \ - .bi_sector = 0, \ - .bi_size = UINT_MAX, \ - .bi_idx = 0, \ - .bi_bvec_done = 0, \ -} - static inline struct bio_vec *bvec_init_iter_all(struct bvec_iter_all *iter_all) { iter_all->done = 0; @@ -256,6 +289,7 @@ static inline void *bvec_kmap_local(struct bio_vec *bvec) /** * memcpy_from_bvec - copy data from a bvec + * @to: Kernel virtual address to copy to. * @bvec: bvec to copy from * * Must be called on single-page bvecs only. @@ -268,6 +302,7 @@ static inline void memcpy_from_bvec(char *to, struct bio_vec *bvec) /** * memcpy_to_bvec - copy data to a bvec * @bvec: bvec to copy to + * @from: Kernel virtual address to copy from. * * Must be called on single-page bvecs only. */ diff --git a/include/linux/cacheinfo.h b/include/linux/cacheinfo.h index c8f4f0a0b874..fc879ac4cc4f 100644 --- a/include/linux/cacheinfo.h +++ b/include/linux/cacheinfo.h @@ -89,6 +89,7 @@ int populate_cache_leaves(unsigned int cpu); int cache_setup_acpi(unsigned int cpu); bool last_level_cache_is_valid(unsigned int cpu); bool last_level_cache_is_shared(unsigned int cpu_x, unsigned int cpu_y); +struct cacheinfo *get_cpu_cacheinfo_llc(unsigned int cpu); int fetch_cache_info(unsigned int cpu); int detect_cache_attributes(unsigned int cpu); #ifndef CONFIG_ACPI_PPTT diff --git a/include/linux/call_once.h b/include/linux/call_once.h index 13cd6469e7e5..1625a9d6ff5b 100644 --- a/include/linux/call_once.h +++ b/include/linux/call_once.h @@ -36,7 +36,7 @@ do { \ * it returns a zero or positive value, mark @once as completed. Return * the value returned by @cb * - * If @once has completed succesfully before, return 0. + * If @once has completed successfully before, return 0. * * The call to @cb is implicitly surrounded by a mutex, though for * efficiency the * function avoids taking it after the first call. diff --git a/include/linux/can/dev/peak_canfd.h b/include/linux/can/dev/peak_canfd.h index d3788a3d0942..056e0efa649f 100644 --- a/include/linux/can/dev/peak_canfd.h +++ b/include/linux/can/dev/peak_canfd.h @@ -3,7 +3,7 @@ * CAN driver for PEAK System micro-CAN based adapters * * Copyright (C) 2003-2025 PEAK System-Technik GmbH - * Author: Stéphane Grosjean <stephane.grosjean@hms-networks.com> + * Author: Stéphane Grosjean <s.grosjean@peak-system.fr> */ #ifndef PUCAN_H #define PUCAN_H diff --git a/include/linux/capability.h b/include/linux/capability.h index 37db92b3d6f8..f8532d92fcad 100644 --- a/include/linux/capability.h +++ b/include/linux/capability.h @@ -145,6 +145,7 @@ extern bool has_capability_noaudit(struct task_struct *t, int cap); extern bool has_ns_capability_noaudit(struct task_struct *t, struct user_namespace *ns, int cap); extern bool capable(int cap); +bool capable_noaudit(int cap); extern bool ns_capable(struct user_namespace *ns, int cap); extern bool ns_capable_noaudit(struct user_namespace *ns, int cap); extern bool ns_capable_setid(struct user_namespace *ns, int cap); @@ -167,6 +168,10 @@ static inline bool capable(int cap) { return true; } +static inline bool capable_noaudit(int cap) +{ + return true; +} static inline bool ns_capable(struct user_namespace *ns, int cap) { return true; diff --git a/include/linux/cdrom.h b/include/linux/cdrom.h index b907e6c2307d..260d7968cf72 100644 --- a/include/linux/cdrom.h +++ b/include/linux/cdrom.h @@ -108,6 +108,7 @@ int cdrom_ioctl(struct cdrom_device_info *cdi, struct block_device *bdev, extern unsigned int cdrom_check_events(struct cdrom_device_info *cdi, unsigned int clearing); +extern void cdrom_probe_write_features(struct cdrom_device_info *cdi); extern int register_cdrom(struct gendisk *disk, struct cdrom_device_info *cdi); extern void unregister_cdrom(struct cdrom_device_info *cdi); diff --git a/include/linux/cdx/cdx_bus.h b/include/linux/cdx/cdx_bus.h index b1ba97f6c9ad..715b026ad95b 100644 --- a/include/linux/cdx/cdx_bus.h +++ b/include/linux/cdx/cdx_bus.h @@ -11,7 +11,7 @@ #include <linux/device.h> #include <linux/list.h> -#include <linux/mod_devicetable.h> +#include <linux/device-id/cdx.h> #include <linux/msi.h> #define MAX_CDX_DEV_RESOURCES 4 @@ -137,9 +137,6 @@ struct cdx_controller { * @enabled: is this bus enabled * @msi_dev_id: MSI Device ID associated with CDX device * @num_msi: Number of MSI's supported by the device - * @driver_override: driver name to force a match; do not set directly, - * because core frees it; use driver_set_override() to - * set or clear it. * @irqchip_lock: lock to synchronize irq/msi configuration * @msi_write_pending: MSI write pending for this device */ @@ -165,7 +162,6 @@ struct cdx_device { bool enabled; u32 msi_dev_id; u32 num_msi; - const char *driver_override; struct mutex irqchip_lock; bool msi_write_pending; }; diff --git a/include/linux/ceph/libceph.h b/include/linux/ceph/libceph.h index 63e0e2aa1ce9..691e1bdece49 100644 --- a/include/linux/ceph/libceph.h +++ b/include/linux/ceph/libceph.h @@ -313,8 +313,6 @@ int ceph_wait_for_latest_osdmap(struct ceph_client *client, /* pagevec.c */ extern void ceph_release_page_vector(struct page **pages, int num_pages); -extern void ceph_put_page_vector(struct page **pages, int num_pages, - bool dirty); extern struct page **ceph_alloc_page_vector(int num_pages, gfp_t flags); extern void ceph_copy_from_page_vector(struct page **pages, void *data, diff --git a/include/linux/cfi.h b/include/linux/cfi.h index 1fd22ea6eba4..0f220d29225c 100644 --- a/include/linux/cfi.h +++ b/include/linux/cfi.h @@ -9,6 +9,7 @@ #include <linux/bug.h> #include <linux/module.h> +#include <linux/uaccess.h> #include <asm/cfi.h> #ifdef CONFIG_CFI diff --git a/include/linux/cgroup-defs.h b/include/linux/cgroup-defs.h index bb92f5c169ca..3754d697854b 100644 --- a/include/linux/cgroup-defs.h +++ b/include/linux/cgroup-defs.h @@ -17,6 +17,7 @@ #include <linux/refcount.h> #include <linux/percpu-refcount.h> #include <linux/percpu-rwsem.h> +#include <linux/sched.h> #include <linux/u64_stats_sync.h> #include <linux/workqueue.h> #include <linux/bpf-cgroup-defs.h> @@ -167,6 +168,7 @@ struct cgroup_file { struct kernfs_node *kn; unsigned long notified_at; struct timer_list notify_timer; + spinlock_t lock; }; /* @@ -252,6 +254,18 @@ struct cgroup_subsys_state { int nr_descendants; /* + * Hierarchical populated state. For cgroup->self, nr_populated_csets + * counts populated csets linked via cgrp_cset_link. + * nr_populated_children counts immediate-child csses whose own + * populated state is nonzero. Protected by css_set_lock. + */ + int nr_populated_csets; + int nr_populated_children; + + /* deferred kill_css_finish() queued by css_update_populated() */ + struct work_struct kill_finish_work; + + /* * A singly-linked list of css structures to be rstat flushed. * This is a scratch field to be used exclusively by * css_rstat_flush(). @@ -502,23 +516,21 @@ struct cgroup { int max_descendants; /* - * Each non-empty css_set associated with this cgroup contributes - * one to nr_populated_csets. The counter is zero iff this cgroup - * doesn't have any tasks. - * - * All children which have non-zero nr_populated_csets and/or - * nr_populated_children of their own contribute one to either - * nr_populated_domain_children or nr_populated_threaded_children - * depending on their type. Each counter is zero iff all cgroups - * of the type in the subtree proper don't have any tasks. + * Domain/threaded split of self.nr_populated_children: each counts + * immediate-child cgroups whose subtree is populated and sums to + * self.nr_populated_children. Kept as separate fields to allow readers + * like cgroup_can_be_thread_root() unlocked access. Protected by + * css_set_lock; updated by css_update_populated(). */ - int nr_populated_csets; int nr_populated_domain_children; int nr_populated_threaded_children; int nr_threaded_children; /* # of live threaded child cgroups */ - /* sequence number for cgroup.kill, serialized by css_set_lock. */ + /* + * Sequence number for cgroup.kill. Incremented with both cgroup_mutex + * and css_set_lock held. Readers hold either one. + */ unsigned int kill_seq; struct kernfs_node *kn; /* cgroup kernfs entry */ @@ -624,6 +636,9 @@ struct cgroup { #ifdef CONFIG_BPF_SYSCALL struct bpf_local_storage __rcu *bpf_cgrp_storage; #endif +#ifdef CONFIG_EXT_SUB_SCHED + struct scx_sched __rcu *scx_sched; +#endif /* All ancestors including self */ union { @@ -884,7 +899,7 @@ static inline void cgroup_threadgroup_change_begin(struct task_struct *tsk) * cgroup_threadgroup_change_end - threadgroup exclusion for cgroups * @tsk: target task * - * Counterpart of cgroup_threadcgroup_change_begin(). + * Counterpart of cgroup_threadgroup_change_begin(). */ static inline void cgroup_threadgroup_change_end(struct task_struct *tsk) { diff --git a/include/linux/cgroup.h b/include/linux/cgroup.h index bc892e3b37ee..5dfa915a630e 100644 --- a/include/linux/cgroup.h +++ b/include/linux/cgroup.h @@ -42,9 +42,18 @@ struct kernel_clone_args; #ifdef CONFIG_CGROUPS +/* + * To avoid confusing the compiler (and generating warnings) with code + * that attempts to access what would be a 0-element array (i.e. sized + * to a potentially empty array when CGROUP_SUBSYS_COUNT == 0), this + * constant expression can be added. + */ +#define CGROUP_HAS_SUBSYS_CONFIG (CGROUP_SUBSYS_COUNT > 0) + enum css_task_iter_flags { CSS_TASK_ITER_PROCS = (1U << 0), /* walk only threadgroup leaders */ CSS_TASK_ITER_THREADED = (1U << 1), /* walk all threaded css_sets in the domain */ + CSS_TASK_ITER_WITH_DEAD = (1U << 2), /* include exiting tasks */ CSS_TASK_ITER_SKIPPED = (1U << 16), /* internal flags */ }; @@ -73,11 +82,38 @@ enum cgroup_lifetime_events { CGROUP_LIFETIME_OFFLINE, }; +/* + * Events on cgroup_task_notifier, data is struct cgroup_task_migrate_ctx. + * MIGRATING fires per task before the migration commits and an error return + * from the chain fails the migration, in which case tasks that were already + * notified receive MIGRATE_CANCELED. MIGRATED fires per task after the + * migration is committed and can't fail. Only migrations that change a task's + * dfl cgroup are reported. + */ +enum cgroup_task_events { + CGROUP_TASK_MIGRATING, + CGROUP_TASK_MIGRATED, + CGROUP_TASK_MIGRATE_CANCELED, +}; + +/* + * @src_dcgrp and @dst_dcgrp are @task's dfl cgroups before and after the + * migration. @src_dcgrp is NULL for CGROUP_TASK_MIGRATED as per-task sources + * are not tracked past the commit point. + */ +struct cgroup_task_migrate_ctx { + struct task_struct *task; + struct cgroup *src_dcgrp; + struct cgroup *dst_dcgrp; +}; + extern struct file_system_type cgroup_fs_type; extern struct cgroup_root cgrp_dfl_root; extern struct css_set init_css_set; +extern struct mutex cgroup_mutex; extern spinlock_t css_set_lock; extern struct blocking_notifier_head cgroup_lifetime_notifier; +extern struct blocking_notifier_head cgroup_task_notifier; #define SUBSYS(_x) extern struct cgroup_subsys _x ## _cgrp_subsys; #include <linux/cgroup_subsys.h> @@ -103,6 +139,8 @@ extern struct blocking_notifier_head cgroup_lifetime_notifier; #define cgroup_subsys_on_dfl(ss) \ static_branch_likely(&ss ## _on_dfl_key) +bool cgroup_on_dfl(const struct cgroup *cgrp); + bool css_has_online_children(struct cgroup_subsys_state *css); struct cgroup_subsys_state *css_from_id(int id, struct cgroup_subsys *ss); struct cgroup_subsys_state *cgroup_e_css(struct cgroup *cgroup, @@ -274,6 +312,32 @@ void css_task_iter_end(struct css_task_iter *it); for ((pos) = css_next_descendant_post(NULL, (css)); (pos); \ (pos) = css_next_descendant_post((pos), (css))) +/* iterate over child cgrps, lock should be held throughout iteration */ +#define cgroup_for_each_live_child(child, cgrp) \ + list_for_each_entry((child), &(cgrp)->self.children, self.sibling) \ + if (({ lockdep_assert_held(&cgroup_mutex); \ + cgroup_is_dead(child); })) \ + ; \ + else + +/* walk live descendants in pre order */ +#define cgroup_for_each_live_descendant_pre(dsct, d_css, cgrp) \ + css_for_each_descendant_pre((d_css), cgroup_css((cgrp), NULL)) \ + if (({ lockdep_assert_held(&cgroup_mutex); \ + (dsct) = (d_css)->cgroup; \ + cgroup_is_dead(dsct); })) \ + ; \ + else + +/* walk live descendants in postorder */ +#define cgroup_for_each_live_descendant_post(dsct, d_css, cgrp) \ + css_for_each_descendant_post((d_css), cgroup_css((cgrp), NULL)) \ + if (({ lockdep_assert_held(&cgroup_mutex); \ + (dsct) = (d_css)->cgroup; \ + cgroup_is_dead(dsct); })) \ + ; \ + else + /** * cgroup_taskset_for_each - iterate cgroup_taskset * @task: the loop cursor @@ -337,6 +401,27 @@ static inline u64 cgroup_id(const struct cgroup *cgrp) } /** + * cgroup_css - obtain a cgroup's css for the specified subsystem + * @cgrp: the cgroup of interest + * @ss: the subsystem of interest (%NULL returns @cgrp->self) + * + * Return @cgrp's css (cgroup_subsys_state) associated with @ss. This + * function must be called either under cgroup_mutex or rcu_read_lock() and + * the caller is responsible for pinning the returned css if it wants to + * keep accessing it outside the said locks. This function may return + * %NULL if @cgrp doesn't have @subsys_id enabled. + */ +static inline struct cgroup_subsys_state *cgroup_css(struct cgroup *cgrp, + struct cgroup_subsys *ss) +{ + if (CGROUP_HAS_SUBSYS_CONFIG && ss) + return rcu_dereference_check(cgrp->subsys[ss->id], + lockdep_is_held(&cgroup_mutex)); + else + return &cgrp->self; +} + +/** * css_is_dying - test whether the specified css is dying * @css: target css * @@ -372,6 +457,11 @@ static inline bool css_is_self(struct cgroup_subsys_state *css) return false; } +static inline bool cgroup_is_dead(const struct cgroup *cgrp) +{ + return !(cgrp->self.flags & CSS_ONLINE); +} + static inline void cgroup_get(struct cgroup *cgrp) { css_get(&cgrp->self); @@ -387,8 +477,6 @@ static inline void cgroup_put(struct cgroup *cgrp) css_put(&cgrp->self); } -extern struct mutex cgroup_mutex; - static inline void cgroup_lock(void) { mutex_lock(&cgroup_mutex); @@ -418,7 +506,7 @@ static inline void cgroup_unlock(void) rcu_read_lock_sched_held() || \ lockdep_is_held(&cgroup_mutex) || \ lockdep_is_held(&css_set_lock) || \ - ((task)->flags & PF_EXITING) || (__c)) + (data_race((task)->flags) & PF_EXITING) || (__c)) #else #define task_css_set_check(task, __c) \ rcu_dereference((task)->cgroups) @@ -562,6 +650,27 @@ static inline struct cgroup *cgroup_ancestor(struct cgroup *cgrp, } /** + * cgroup_common_ancestor - find common ancestor of two cgroups + * @a: first cgroup to find common ancestor of + * @b: second cgroup to find common ancestor of + * + * Find the first cgroup that is an ancestor of both @a and @b, if it exists + * and return a pointer to it. If such a cgroup doesn't exist, return NULL. + * + * This function is safe to call as long as both @a and @b are accessible. + */ +static inline struct cgroup *cgroup_common_ancestor(struct cgroup *a, + struct cgroup *b) +{ + int level; + + for (level = min(a->level, b->level); level >= 0; level--) + if (a->ancestors[level] == b->ancestors[level]) + return a->ancestors[level]; + return NULL; +} + +/** * task_under_cgroup_hierarchy - test task's membership of cgroup ancestry * @task: the task to be tested * @ancestor: possible ancestor of @task's cgroup @@ -578,11 +687,32 @@ static inline bool task_under_cgroup_hierarchy(struct task_struct *task, return cgroup_is_descendant(cset->dfl_cgrp, ancestor); } -/* no synchronization, the result can only be used as a hint */ +/* + * Populated counters: writes happen under css_set_lock. The accessors below + * may read unlocked. What an unpopulated result means depends on context: + * + * - No lock held. Just a snapshot. May race with concurrent updates and is + * useful only as a hint. + * + * - cgroup_mutex held. Migration into the cgroup is blocked, so an observed + * !populated stays !populated until cgroup_mutex is dropped. + * + * - CSS_DYING set. The css can no longer be repopulated, so !populated is + * sticky once observed. + */ +static inline bool cgroup_has_tasks(struct cgroup *cgrp) +{ + return READ_ONCE(cgrp->self.nr_populated_csets); +} + +static inline bool css_is_populated(struct cgroup_subsys_state *css) +{ + return READ_ONCE(css->nr_populated_csets) || READ_ONCE(css->nr_populated_children); +} + static inline bool cgroup_is_populated(struct cgroup *cgrp) { - return cgrp->nr_populated_csets + cgrp->nr_populated_domain_children + - cgrp->nr_populated_threaded_children; + return css_is_populated(&cgrp->self); } /* returns ino associated with a cgroup */ @@ -715,6 +845,7 @@ static inline void cgroup_path_from_kernfs_id(u64 id, char *buf, size_t buflen) /* * cgroup scalable recursive statistics. */ +void __css_rstat_updated(struct cgroup_subsys_state *css, int cpu); void css_rstat_updated(struct cgroup_subsys_state *css, int cpu); void css_rstat_flush(struct cgroup_subsys_state *css); diff --git a/include/linux/cgroup_dmem.h b/include/linux/cgroup_dmem.h index dd4869f1d736..c6cad9501770 100644 --- a/include/linux/cgroup_dmem.h +++ b/include/linux/cgroup_dmem.h @@ -14,8 +14,40 @@ struct dmem_cgroup_pool_state; /* Opaque definition of a cgroup region, used internally */ struct dmem_cgroup_region; +/** + * struct dmem_cgroup_ops - Operations for a dmem cgroup region. + * @reclaim: Optional callback invoked when dmem.max is set below the current + * usage of a pool. The driver should attempt to free at least + * @target_bytes from @pool. May be called multiple times if usage + * remains above the limit after returning. + * + * Return: 0 if some progress was made (even if less than + * @target_bytes was freed), -ENOSPC if no progress could be made + * (the caller will retry up to a bounded number of times), or + * another negative error code if a fatal error occurred (stops + * further reclaim attempts immediately). + */ +struct dmem_cgroup_ops { + int (*reclaim)(struct dmem_cgroup_pool_state *pool, + u64 target_bytes, void *priv); +}; + +/** + * struct dmem_cgroup_init - Initialization parameters for a dmem cgroup region. + * @size: Size of the region in bytes. + * @ops: Optional operations for this region. May be NULL. + * @reclaim_priv: Opaque pointer passed to @ops->reclaim. May be NULL. + */ +struct dmem_cgroup_init { + u64 size; + const struct dmem_cgroup_ops *ops; + void *reclaim_priv; +}; + #if IS_ENABLED(CONFIG_CGROUP_DMEM) -struct dmem_cgroup_region *dmem_cgroup_register_region(u64 size, const char *name_fmt, ...) __printf(2,3); +struct dmem_cgroup_region * +dmem_cgroup_register_region(const struct dmem_cgroup_init *init, + const char *name_fmt, ...) __printf(2, 3); void dmem_cgroup_unregister_region(struct dmem_cgroup_region *region); int dmem_cgroup_try_charge(struct dmem_cgroup_region *region, u64 size, struct dmem_cgroup_pool_state **ret_pool, @@ -24,11 +56,17 @@ void dmem_cgroup_uncharge(struct dmem_cgroup_pool_state *pool, u64 size); bool dmem_cgroup_state_evict_valuable(struct dmem_cgroup_pool_state *limit_pool, struct dmem_cgroup_pool_state *test_pool, bool ignore_low, bool *ret_hit_low); +bool dmem_cgroup_below_min(struct dmem_cgroup_pool_state *root, + struct dmem_cgroup_pool_state *test); +bool dmem_cgroup_below_low(struct dmem_cgroup_pool_state *root, + struct dmem_cgroup_pool_state *test); +struct dmem_cgroup_pool_state *dmem_cgroup_get_common_ancestor(struct dmem_cgroup_pool_state *a, + struct dmem_cgroup_pool_state *b); void dmem_cgroup_pool_state_put(struct dmem_cgroup_pool_state *pool); #else -static inline __printf(2,3) struct dmem_cgroup_region * -dmem_cgroup_register_region(u64 size, const char *name_fmt, ...) +static inline __printf(2, 3) struct dmem_cgroup_region * +dmem_cgroup_register_region(const struct dmem_cgroup_init *init, const char *name_fmt, ...) { return NULL; } @@ -59,6 +97,25 @@ bool dmem_cgroup_state_evict_valuable(struct dmem_cgroup_pool_state *limit_pool, return true; } +static inline bool dmem_cgroup_below_min(struct dmem_cgroup_pool_state *root, + struct dmem_cgroup_pool_state *test) +{ + return false; +} + +static inline bool dmem_cgroup_below_low(struct dmem_cgroup_pool_state *root, + struct dmem_cgroup_pool_state *test) +{ + return false; +} + +static inline +struct dmem_cgroup_pool_state *dmem_cgroup_get_common_ancestor(struct dmem_cgroup_pool_state *a, + struct dmem_cgroup_pool_state *b) +{ + return NULL; +} + static inline void dmem_cgroup_pool_state_put(struct dmem_cgroup_pool_state *pool) { } diff --git a/include/linux/cgroup_rdma.h b/include/linux/cgroup_rdma.h index 80edae03c313..9a5c9ee728e7 100644 --- a/include/linux/cgroup_rdma.h +++ b/include/linux/cgroup_rdma.h @@ -24,12 +24,17 @@ struct rdma_cgroup { * that belongs to this cgroup. */ struct list_head rpools; + + /* Handles for rdma.events[.local] */ + struct cgroup_file events_file; + struct cgroup_file events_local_file; }; struct rdmacg_device { struct list_head dev_node; struct list_head rpools; char *name; + u32 index; }; /* diff --git a/include/linux/cleanup.h b/include/linux/cleanup.h index dbc4162921e9..b1b5698cbf1b 100644 --- a/include/linux/cleanup.h +++ b/include/linux/cleanup.h @@ -286,15 +286,18 @@ static __always_inline _type class_##_name##_constructor(_init_args) \ __no_context_analysis \ { _type t = _init; return t; } -#define EXTEND_CLASS(_name, ext, _init, _init_args...) \ -typedef lock_##_name##_t lock_##_name##ext##_t; \ +#define EXTEND_CLASS_COND(_name, ext, _cond, _init, _init_args...) \ +typedef lock_##_name##_t lock_##_name##ext##_t; \ typedef class_##_name##_t class_##_name##ext##_t; \ -static __always_inline void class_##_name##ext##_destructor(class_##_name##_t *p) \ -{ class_##_name##_destructor(p); } \ +static __always_inline void class_##_name##ext##_destructor(class_##_name##_t *_T) \ +{ if (_cond) return; class_##_name##_destructor(_T); } \ static __always_inline class_##_name##_t class_##_name##ext##_constructor(_init_args) \ __no_context_analysis \ { class_##_name##_t t = _init; return t; } +#define EXTEND_CLASS(_name, ext, _init, _init_args...) \ + EXTEND_CLASS_COND(_name, ext, 0, _init, _init_args) + #define CLASS(_name, var) \ class_##_name##_t var __cleanup(class_##_name##_destructor) = \ class_##_name##_constructor @@ -394,12 +397,13 @@ static __maybe_unused const bool class_##_name##_is_conditional = _is_cond __DEFINE_GUARD_LOCK_PTR(_name, _T) #define DEFINE_GUARD(_name, _type, _lock, _unlock) \ - DEFINE_CLASS(_name, _type, if (!__GUARD_IS_ERR(_T)) { _unlock; }, ({ _lock; _T; }), _type _T); \ + static __always_inline __nonnull_args(1) _type class_##_name##_constructor(_type _T); \ + DEFINE_CLASS(_name, _type, _unlock, ({ _lock; _T; }), _type _T); \ DEFINE_CLASS_IS_GUARD(_name) #define DEFINE_GUARD_COND_4(_name, _ext, _lock, _cond) \ __DEFINE_CLASS_IS_CONDITIONAL(_name##_ext, true); \ - EXTEND_CLASS(_name, _ext, \ + EXTEND_CLASS_COND(_name, _ext, __GUARD_IS_ERR(*_T), \ ({ void *_t = _T; int _RET = (_lock); if (_T && !(_cond)) _t = ERR_PTR(_RET); _t; }), \ class_##_name##_t _T) \ static __always_inline void * class_##_name##_ext##_lock_ptr(class_##_name##_t *_T) \ @@ -488,13 +492,14 @@ typedef struct { \ static __always_inline void class_##_name##_destructor(class_##_name##_t *_T) \ __no_context_analysis \ { \ - if (!__GUARD_IS_ERR(_T->lock)) { _unlock; } \ + _unlock; \ } \ \ __DEFINE_GUARD_LOCK_PTR(_name, &_T->lock) #define __DEFINE_LOCK_GUARD_1(_name, _type, ...) \ -static __always_inline class_##_name##_t class_##_name##_constructor(_type *l) \ +static __always_inline __nonnull_args(1) \ +class_##_name##_t class_##_name##_constructor(_type *l) \ __no_context_analysis \ { \ class_##_name##_t _t = { .lock = l }, *_T = &_t; \ @@ -565,7 +570,7 @@ __DEFINE_LOCK_GUARD_0(_name, _lock) #define DEFINE_LOCK_GUARD_1_COND_4(_name, _ext, _lock, _cond) \ __DEFINE_CLASS_IS_CONDITIONAL(_name##_ext, true); \ - EXTEND_CLASS(_name, _ext, \ + EXTEND_CLASS_COND(_name, _ext, __GUARD_IS_ERR(_T->lock), \ ({ class_##_name##_t _t = { .lock = l }, *_T = &_t;\ int _RET = (_lock); \ if (_T->lock && !(_cond)) _T->lock = ERR_PTR(_RET);\ diff --git a/include/linux/clk-provider.h b/include/linux/clk-provider.h index 630705a47129..dc8e161f9c2c 100644 --- a/include/linux/clk-provider.h +++ b/include/linux/clk-provider.h @@ -6,31 +6,44 @@ #ifndef __LINUX_CLK_PROVIDER_H #define __LINUX_CLK_PROVIDER_H +#include <dt-bindings/clock/clock.h> #include <linux/of.h> #include <linux/of_clk.h> -/* - * flags used across common struct clk. these flags should only affect the - * top-level framework. custom flags for dealing with hardware specifics - * belong in struct clk_foo +/** + * DOC: clk framework flags + * + * Flags used across common struct clk. These flags should only affect the + * top-level framework. Custom flags for dealing with hardware specifics + * belong in struct clk_foo. * - * Please update clk_flags[] in drivers/clk/clk.c when making changes here! + * * CLK_SET_RATE_GATE - must be gated across rate change + * * CLK_SET_PARENT_GATE - must be gated across re-parent + * * CLK_SET_RATE_PARENT - propagate rate change up one level + * * CLK_IGNORE_UNUSED - do not gate even if unused + * * CLK_GET_RATE_NOCACHE - do not use the cached clk rate + * * CLK_SET_RATE_NO_REPARENT - don't re-parent on rate change + * * CLK_GET_ACCURACY_NOCACHE - do not use the cached clk accuracy + * * CLK_RECALC_NEW_RATES - recalc rates after notifications + * * CLK_SET_RATE_UNGATE - clock needs to run to set rate + * * CLK_IS_CRITICAL - do not gate, ever + * * CLK_OPS_PARENT_ENABLE - parents need enable during gate/ungate, set rate and re-parent + * * CLK_DUTY_CYCLE_PARENT - duty cycle call may be forwarded to the parent clock */ -#define CLK_SET_RATE_GATE BIT(0) /* must be gated across rate change */ -#define CLK_SET_PARENT_GATE BIT(1) /* must be gated across re-parent */ -#define CLK_SET_RATE_PARENT BIT(2) /* propagate rate change up one level */ -#define CLK_IGNORE_UNUSED BIT(3) /* do not gate even if unused */ +/* Please update clk_flags[] in drivers/clk/clk.c when making changes here! */ +#define CLK_SET_RATE_GATE BIT(0) +#define CLK_SET_PARENT_GATE BIT(1) +#define CLK_SET_RATE_PARENT BIT(2) +#define CLK_IGNORE_UNUSED BIT(3) /* unused */ /* unused */ -#define CLK_GET_RATE_NOCACHE BIT(6) /* do not use the cached clk rate */ -#define CLK_SET_RATE_NO_REPARENT BIT(7) /* don't re-parent on rate change */ -#define CLK_GET_ACCURACY_NOCACHE BIT(8) /* do not use the cached clk accuracy */ -#define CLK_RECALC_NEW_RATES BIT(9) /* recalc rates after notifications */ -#define CLK_SET_RATE_UNGATE BIT(10) /* clock needs to run to set rate */ -#define CLK_IS_CRITICAL BIT(11) /* do not gate, ever */ -/* parents need enable during gate/ungate, set rate and re-parent */ +#define CLK_GET_RATE_NOCACHE BIT(6) +#define CLK_SET_RATE_NO_REPARENT BIT(7) +#define CLK_GET_ACCURACY_NOCACHE BIT(8) +#define CLK_RECALC_NEW_RATES BIT(9) +#define CLK_SET_RATE_UNGATE BIT(10) +#define CLK_IS_CRITICAL BIT(11) #define CLK_OPS_PARENT_ENABLE BIT(12) -/* duty cycle call may be forwarded to the parent clock */ #define CLK_DUTY_CYCLE_PARENT BIT(13) struct clk; @@ -84,6 +97,26 @@ struct clk_duty { unsigned int den; }; +enum clk_ssc_method { + CLK_SPREAD_NO = CLK_SSC_NO_SPREAD, + CLK_SPREAD_CENTER = CLK_SSC_CENTER_SPREAD, + CLK_SPREAD_UP = CLK_SSC_UP_SPREAD, + CLK_SPREAD_DOWN = CLK_SSC_DOWN_SPREAD, +}; + +/** + * struct clk_spread_spectrum - Structure encoding spread spectrum of a clock + * + * @modfreq_hz: Modulation frequency + * @spread_bp: Modulation percent in permyriad + * @method: Modulation method + */ +struct clk_spread_spectrum { + u32 modfreq_hz; + u32 spread_bp; + enum clk_ssc_method method; +}; + /** * struct clk_ops - Callback operations for hardware clocks; these are to * be provided by the clock implementation, and will be called by drivers @@ -136,10 +169,6 @@ struct clk_duty { * 0. Returns the calculated rate. Optional, but recommended - if * this op is not set then clock rate will be initialized to 0. * - * @round_rate: Given a target rate as input, returns the closest rate actually - * supported by the clock. The parent rate is an input/output - * parameter. - * * @determine_rate: Given a target rate as input, returns the closest rate * actually supported by the clock, and optionally the parent clock * that should be used to provide the clock rate. @@ -163,13 +192,13 @@ struct clk_duty { * * @set_rate: Change the rate of this clock. The requested rate is specified * by the second argument, which should typically be the return - * of .round_rate call. The third argument gives the parent rate - * which is likely helpful for most .set_rate implementation. + * of .determine_rate call. The third argument gives the parent + * rate which is likely helpful for most .set_rate implementation. * Returns 0 on success, -EERROR otherwise. * * @set_rate_and_parent: Change the rate and the parent of this clock. The * requested rate is specified by the second argument, which - * should typically be the return of .round_rate call. The + * should typically be the return of clk_round_rate() call. The * third argument gives the parent rate which is likely helpful * for most .set_rate_and_parent implementation. The fourth * argument gives the parent index. This callback is optional (and @@ -178,6 +207,12 @@ struct clk_duty { * separately via calls to .set_parent and .set_rate. * Returns 0 on success, -EERROR otherwise. * + * @set_spread_spectrum: Optional callback used to configure the spread + * spectrum modulation frequency, percentage, and method + * to reduce EMI by spreading the clock frequency over a + * wider range. + * Returns 0 on success, -EERROR otherwise. + * * @recalc_accuracy: Recalculate the accuracy of this clock. The clock accuracy * is expressed in ppb (parts per billion). The parent accuracy is * an input parameter. @@ -244,8 +279,6 @@ struct clk_ops { void (*restore_context)(struct clk_hw *hw); unsigned long (*recalc_rate)(struct clk_hw *hw, unsigned long parent_rate); - long (*round_rate)(struct clk_hw *hw, unsigned long rate, - unsigned long *parent_rate); int (*determine_rate)(struct clk_hw *hw, struct clk_rate_request *req); int (*set_parent)(struct clk_hw *hw, u8 index); @@ -255,6 +288,8 @@ struct clk_ops { int (*set_rate_and_parent)(struct clk_hw *hw, unsigned long rate, unsigned long parent_rate, u8 index); + int (*set_spread_spectrum)(struct clk_hw *hw, + const struct clk_spread_spectrum *ss_conf); unsigned long (*recalc_accuracy)(struct clk_hw *hw, unsigned long parent_accuracy); int (*get_phase)(struct clk_hw *hw); @@ -679,7 +714,7 @@ struct clk_div_table { * @lock: register lock * * Clock with an adjustable divider affecting its output frequency. Implements - * .recalc_rate, .set_rate and .round_rate + * .recalc_rate, .set_rate and .determine_rate * * @flags: * CLK_DIVIDER_ONE_BASED - by default the divisor is the value read from the @@ -739,14 +774,6 @@ extern const struct clk_ops clk_divider_ro_ops; unsigned long divider_recalc_rate(struct clk_hw *hw, unsigned long parent_rate, unsigned int val, const struct clk_div_table *table, unsigned long flags, unsigned long width); -long divider_round_rate_parent(struct clk_hw *hw, struct clk_hw *parent, - unsigned long rate, unsigned long *prate, - const struct clk_div_table *table, - u8 width, unsigned long flags); -long divider_ro_round_rate_parent(struct clk_hw *hw, struct clk_hw *parent, - unsigned long rate, unsigned long *prate, - const struct clk_div_table *table, u8 width, - unsigned long flags, unsigned int val); int divider_determine_rate(struct clk_hw *hw, struct clk_rate_request *req, const struct clk_div_table *table, u8 width, unsigned long flags); @@ -948,6 +975,26 @@ struct clk *clk_register_divider_table(struct device *dev, const char *name, (shift), (width), (clk_divider_flags), \ NULL, (lock)) /** + * devm_clk_hw_register_divider_parent_data - register a divider clock with the + * clock framework + * @dev: device registering this clock + * @name: name of this clock + * @parent_data: parent clk data + * @flags: framework-specific flags + * @reg: register address to adjust divider + * @shift: number of bits to shift the bitfield + * @width: width of the bitfield + * @clk_divider_flags: divider-specific flags for this clock + * @lock: shared register lock for this clock + */ +#define devm_clk_hw_register_divider_parent_data(dev, name, parent_data, \ + flags, reg, shift, width, \ + clk_divider_flags, lock) \ + __devm_clk_hw_register_divider((dev), NULL, (name), NULL, NULL, \ + (parent_data), (flags), (reg), (shift), \ + (width), (clk_divider_flags), NULL, \ + (lock)) +/** * devm_clk_hw_register_divider_table - register a table based divider clock * with the clock framework (devres variant) * @dev: device registering this clock @@ -1126,7 +1173,7 @@ void of_fixed_factor_clk_setup(struct device_node *node); * * Clock with a fixed multiplier and divider. The output frequency is the * parent clock rate divided by div and multiplied by mult. - * Implements .recalc_rate, .set_rate, .round_rate and .recalc_accuracy + * Implements .recalc_rate, .set_rate, .determine_rate and .recalc_accuracy * * Flags: * * CLK_FIXED_FACTOR_FIXED_ACCURACY - Use the value in @acc instead of the @@ -1146,13 +1193,16 @@ struct clk_fixed_factor { #define to_clk_fixed_factor(_hw) container_of(_hw, struct clk_fixed_factor, hw) extern const struct clk_ops clk_fixed_factor_ops; +struct clk_hw * +__clk_hw_register_fixed_factor(struct device *dev, struct device_node *np, + const char *name, const char *parent_name, + const struct clk_hw *parent_hw, const struct clk_parent_data *pdata, + unsigned long flags, unsigned int mult, unsigned int div, + unsigned long acc, unsigned int fixflags, bool devm); struct clk *clk_register_fixed_factor(struct device *dev, const char *name, const char *parent_name, unsigned long flags, unsigned int mult, unsigned int div); void clk_unregister_fixed_factor(struct clk *clk); -struct clk_hw *clk_hw_register_fixed_factor(struct device *dev, - const char *name, const char *parent_name, unsigned long flags, - unsigned int mult, unsigned int div); struct clk_hw *clk_hw_register_fixed_factor_fwname(struct device *dev, struct device_node *np, const char *name, const char *fw_name, unsigned long flags, unsigned int mult, unsigned int div); @@ -1164,9 +1214,6 @@ struct clk_hw *clk_hw_register_fixed_factor_index(struct device *dev, const char *name, unsigned int index, unsigned long flags, unsigned int mult, unsigned int div); void clk_hw_unregister_fixed_factor(struct clk_hw *hw); -struct clk_hw *devm_clk_hw_register_fixed_factor(struct device *dev, - const char *name, const char *parent_name, unsigned long flags, - unsigned int mult, unsigned int div); struct clk_hw *devm_clk_hw_register_fixed_factor_fwname(struct device *dev, struct device_node *np, const char *name, const char *fw_name, unsigned long flags, unsigned int mult, unsigned int div); @@ -1178,13 +1225,45 @@ struct clk_hw *devm_clk_hw_register_fixed_factor_index(struct device *dev, const char *name, unsigned int index, unsigned long flags, unsigned int mult, unsigned int div); -struct clk_hw *devm_clk_hw_register_fixed_factor_parent_hw(struct device *dev, - const char *name, const struct clk_hw *parent_hw, - unsigned long flags, unsigned int mult, unsigned int div); - -struct clk_hw *clk_hw_register_fixed_factor_parent_hw(struct device *dev, - const char *name, const struct clk_hw *parent_hw, - unsigned long flags, unsigned int mult, unsigned int div); +#define clk_hw_register_fixed_factor(dev, name, parent_name, \ + flags, mult, div) \ + __clk_hw_register_fixed_factor((dev), NULL, (name), (parent_name), \ + NULL, NULL, (flags), (mult), (div), \ + 0, 0, false) +#define clk_hw_register_fixed_factor_pdata(dev, np, name, pdata, \ + flags, mult, div, acc, fixflags) \ + __clk_hw_register_fixed_factor((dev), (np), (name), NULL, NULL, \ + (pdata), (flags), (mult), (div), \ + (acc), (fixflags), false) +#define devm_clk_hw_register_fixed_factor(dev, name, parent_name, flags, \ + mult, div) \ + __clk_hw_register_fixed_factor((dev), NULL, (name), (parent_name), \ + NULL, NULL, (flags), (mult), (div), 0, \ + 0, true) +/** + * devm_clk_hw_register_fixed_factor_parent_hw - Register a fixed factor clock with + * pointer to parent clock + * @dev: device that is registering this clock + * @name: name of this clock + * @parent_hw: pointer to parent clk + * @flags: fixed factor flags + * @mult: multiplier + * @div: divider + * + * Return: Pointer to fixed factor clk_hw structure that was registered or + * an error pointer. + */ +#define devm_clk_hw_register_fixed_factor_parent_hw(dev, name, parent_hw, \ + flags, mult, div) \ + __clk_hw_register_fixed_factor((dev), NULL, (name), NULL, \ + (parent_hw), NULL, (flags), (mult), \ + (div), 0, 0, true) + +#define clk_hw_register_fixed_factor_parent_hw(dev, name, parent_hw, flags, \ + mult, div) \ + __clk_hw_register_fixed_factor((dev), NULL, (name), NULL, \ + (parent_hw), NULL, (flags), (mult), \ + (div), 0, 0, false) /** * struct clk_fractional_divider - adjustable fractional divider clock * @@ -1254,7 +1333,7 @@ void clk_hw_unregister_fractional_divider(struct clk_hw *hw); * @lock: register lock * * Clock with an adjustable multiplier affecting its output frequency. - * Implements .recalc_rate, .set_rate and .round_rate + * Implements .recalc_rate, .set_rate and .determine_rate * * @flags: * CLK_MULTIPLIER_ZERO_BYPASS - By default, the multiplier is the value read @@ -1425,11 +1504,14 @@ int clk_mux_determine_rate_flags(struct clk_hw *hw, unsigned long flags); int clk_hw_determine_rate_no_reparent(struct clk_hw *hw, struct clk_rate_request *req); +int clk_determine_rate_noop(struct clk_hw *hw, struct clk_rate_request *req); void clk_hw_reparent(struct clk_hw *hw, struct clk_hw *new_parent); void clk_hw_get_rate_range(struct clk_hw *hw, unsigned long *min_rate, unsigned long *max_rate); void clk_hw_set_rate_range(struct clk_hw *hw, unsigned long min_rate, unsigned long max_rate); +int clk_hw_set_spread_spectrum(struct clk_hw *hw, + const struct clk_spread_spectrum *ss_conf); static inline void __clk_hw_set_clk(struct clk_hw *dst, struct clk_hw *src) { @@ -1437,26 +1519,6 @@ static inline void __clk_hw_set_clk(struct clk_hw *dst, struct clk_hw *src) dst->core = src->core; } -static inline long divider_round_rate(struct clk_hw *hw, unsigned long rate, - unsigned long *prate, - const struct clk_div_table *table, - u8 width, unsigned long flags) -{ - return divider_round_rate_parent(hw, clk_hw_get_parent(hw), - rate, prate, table, width, flags); -} - -static inline long divider_ro_round_rate(struct clk_hw *hw, unsigned long rate, - unsigned long *prate, - const struct clk_div_table *table, - u8 width, unsigned long flags, - unsigned int val) -{ - return divider_ro_round_rate_parent(hw, clk_hw_get_parent(hw), - rate, prate, table, width, flags, - val); -} - /* * FIXME clock api without lock protection */ diff --git a/include/linux/clk.h b/include/linux/clk.h index 998ba3f261da..db9cf184fd99 100644 --- a/include/linux/clk.h +++ b/include/linux/clk.h @@ -503,6 +503,22 @@ int __must_check devm_clk_bulk_get(struct device *dev, int num_clks, int __must_check devm_clk_bulk_get_optional(struct device *dev, int num_clks, struct clk_bulk_data *clks); /** + * devm_clk_bulk_get_enable - Get and enable bulk clocks (managed) + * @dev: device for clock "consumer" + * @num_clks: the number of clk_bulk_data + * @clks: pointer to the clk_bulk_data table of consumer + * + * Behaves the same as devm_clk_bulk_get() but also prepares and enables the + * clocks in one operation with management. The clks will automatically be + * disabled, unprepared and freed when the device is unbound. + * + * Return: 0 if all clocks specified in clk_bulk_data table are obtained and + * enabled successfully. Otherwise returns valid IS_ERR() condition containing + * errno. + */ +int __must_check devm_clk_bulk_get_enable(struct device *dev, int num_clks, + struct clk_bulk_data *clks); +/** * devm_clk_bulk_get_optional_enable - Get and enable optional bulk clocks (managed) * @dev: device for clock "consumer" * @num_clks: the number of clk_bulk_data @@ -951,8 +967,8 @@ int clk_set_parent(struct clk *clk, struct clk *parent); * clk_get_parent - get the parent clock source for this clock * @clk: clock source * - * Returns struct clk corresponding to parent clock source, or - * valid IS_ERR() condition containing errno. + * Returns struct clk corresponding to parent clock source, or NULL + * if clk is NULL. */ struct clk *clk_get_parent(struct clk *clk); @@ -1052,6 +1068,13 @@ static inline int __must_check devm_clk_bulk_get_optional(struct device *dev, return 0; } +static inline int __must_check devm_clk_bulk_get_enable(struct device *dev, + int num_clks, + struct clk_bulk_data *clks) +{ + return 0; +} + static inline int __must_check devm_clk_bulk_get_optional_enable(struct device *dev, int num_clks, struct clk_bulk_data *clks) diff --git a/include/linux/clk/qcom.h b/include/linux/clk/qcom.h new file mode 100644 index 000000000000..11ac2d42ff9e --- /dev/null +++ b/include/linux/clk/qcom.h @@ -0,0 +1,67 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2026, Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#ifndef __LINUX_CLK_QCOM_H +#define __LINUX_CLK_QCOM_H + +#include <linux/clk-provider.h> +#include <linux/errno.h> +#include <linux/kconfig.h> +#include <linux/regmap.h> +#include <linux/types.h> + +struct device; +struct platform_device; +struct regulator_bulk_data; + +/** + * struct qcom_clk_ref_desc - descriptor for a clkref_en gate clock + * @name: clock name exposed to the common clock framework + * @offset: clkref_en register offset from the block base + * @regulator_names: optional supply names enabled while preparing the clock + * @num_regulators: number of entries in @regulator_names + */ +struct qcom_clk_ref_desc { + const char *name; + u32 offset; + const char * const *regulator_names; + unsigned int num_regulators; +}; + +/** + * struct qcom_clk_ref - per-clock data for a clkref_en gate clock + * @hw: common clock framework hardware clock handle + * @regmap: register map backing the clkref_en register + * @desc: clock descriptor copied at registration time + * @regulators: optional bulk regulator handles for @desc.regulator_names + */ +struct qcom_clk_ref { + struct clk_hw hw; + struct regmap *regmap; + struct qcom_clk_ref_desc desc; + struct regulator_bulk_data *regulators; +}; + +#if IS_ENABLED(CONFIG_COMMON_CLK_QCOM) + +int qcom_clk_ref_probe(struct platform_device *pdev, + const struct regmap_config *config, + const struct qcom_clk_ref_desc * const *descs, + size_t num_clk_refs); + +#else + +static inline int +qcom_clk_ref_probe(struct platform_device *pdev, + const struct regmap_config *config, + const struct qcom_clk_ref_desc * const *descs, + size_t num_clk_refs) +{ + return -EOPNOTSUPP; +} + +#endif + +#endif diff --git a/include/linux/clk/renesas.h b/include/linux/clk/renesas.h index c360df9fa735..c9495558cd5c 100644 --- a/include/linux/clk/renesas.h +++ b/include/linux/clk/renesas.h @@ -53,6 +53,9 @@ static inline void rzg2l_cpg_dsi_div_set_divider(u8 divider, int target) { } * various parameters used to configure a PLL. These limits ensure * the PLL operates within valid and stable ranges. * + * @input_fref: Reference input frequency to the PLL (in Hz). If set + * to 0, a default value of 24MHz is used. + * * @fout: Output frequency range (in MHz) * @fout.min: Minimum allowed output frequency * @fout.max: Maximum allowed output frequency @@ -78,6 +81,8 @@ static inline void rzg2l_cpg_dsi_div_set_divider(u8 divider, int target) { } * @k.max: Maximum delta-sigma value */ struct rzv2h_pll_limits { + u32 input_fref; + struct { u32 min; u32 max; @@ -164,28 +169,51 @@ struct rzv2h_pll_div_pars { .k = { .min = -32768, .max = 32767 }, \ } \ -#ifdef CONFIG_CLK_RZV2H -bool rzv2h_get_pll_pars(const struct rzv2h_pll_limits *limits, - struct rzv2h_pll_pars *pars, u64 freq_millihz); +#define RZG3E_CPG_PLL_DSI0_LIMITS(name) \ + static const struct rzv2h_pll_limits (name) = { \ + .fout = { .min = 25 * MEGA, .max = 1218 * MEGA }, \ + .fvco = { .min = 1600 * MEGA, .max = 3200 * MEGA }, \ + .m = { .min = 64, .max = 533 }, \ + .p = { .min = 1, .max = 4 }, \ + .s = { .min = 0, .max = 6 }, \ + .k = { .min = -32768, .max = 32767 }, \ + } \ -bool rzv2h_get_pll_divs_pars(const struct rzv2h_pll_limits *limits, - struct rzv2h_pll_div_pars *pars, - const u8 *table, u8 table_size, u64 freq_millihz); +#define RZG3E_CPG_PLL_DSI1_LIMITS(name) \ + static const struct rzv2h_pll_limits (name) = { \ + .fout = { .min = 25 * MEGA, .max = 609 * MEGA }, \ + .fvco = { .min = 1600 * MEGA, .max = 3200 * MEGA }, \ + .m = { .min = 64, .max = 533 }, \ + .p = { .min = 1, .max = 4 }, \ + .s = { .min = 0, .max = 6 }, \ + .k = { .min = -32768, .max = 32767 }, \ + } \ + +#ifdef CONFIG_CLK_RZV2H_CPG_LIB +bool rzv2h_cpg_get_pll_pars(const struct rzv2h_pll_limits *limits, + struct rzv2h_pll_pars *pars, u64 freq_millihz); + +bool rzv2h_cpg_get_pll_divs_pars(const struct rzv2h_pll_limits *limits, + struct rzv2h_pll_div_pars *pars, + const u8 *table, u8 table_size, u64 freq_millihz); #else -static inline bool rzv2h_get_pll_pars(const struct rzv2h_pll_limits *limits, - struct rzv2h_pll_pars *pars, - u64 freq_millihz) +static inline bool rzv2h_cpg_get_pll_pars(const struct rzv2h_pll_limits *limits, + struct rzv2h_pll_pars *pars, + u64 freq_millihz) { return false; } -static inline bool rzv2h_get_pll_divs_pars(const struct rzv2h_pll_limits *limits, - struct rzv2h_pll_div_pars *pars, - const u8 *table, u8 table_size, - u64 freq_millihz) +static inline bool rzv2h_cpg_get_pll_divs_pars(const struct rzv2h_pll_limits *limits, + struct rzv2h_pll_div_pars *pars, + const u8 *table, u8 table_size, + u64 freq_millihz) { return false; } #endif +#define rzv2h_get_pll_pars rzv2h_cpg_get_pll_pars +#define rzv2h_get_pll_divs_pars rzv2h_cpg_get_pll_divs_pars + #endif diff --git a/include/linux/clockchips.h b/include/linux/clockchips.h index b0df28ddd394..6adb72761246 100644 --- a/include/linux/clockchips.h +++ b/include/linux/clockchips.h @@ -43,9 +43,9 @@ enum clock_event_state { /* * Clock event features */ -# define CLOCK_EVT_FEAT_PERIODIC 0x000001 -# define CLOCK_EVT_FEAT_ONESHOT 0x000002 -# define CLOCK_EVT_FEAT_KTIME 0x000004 +# define CLOCK_EVT_FEAT_PERIODIC 0x000001 +# define CLOCK_EVT_FEAT_ONESHOT 0x000002 +# define CLOCK_EVT_FEAT_CLOCKSOURCE_COUPLED 0x000004 /* * x86(64) specific (mis)features: @@ -73,6 +73,7 @@ enum clock_event_state { * level handler of the event source * @set_next_event: set next event function using a clocksource delta * @set_next_ktime: set next event function using a direct ktime value + * @set_next_coupled: set next event function for clocksource coupled mode * @next_event: local storage for the next event in oneshot mode * @max_delta_ns: maximum delta value in ns * @min_delta_ns: minimum delta value in ns @@ -80,6 +81,8 @@ enum clock_event_state { * @shift: nanoseconds to cycles divisor (power of two) * @state_use_accessors:current state of the device, assigned by the core code * @features: features + * @cs_id: Clocksource ID to denote the clocksource for coupled mode + * @next_event_forced: True if the last programming was a forced event * @retries: number of forced programming retries * @set_state_periodic: switch state to periodic * @set_state_oneshot: switch state to oneshot @@ -101,6 +104,7 @@ struct clock_event_device { void (*event_handler)(struct clock_event_device *); int (*set_next_event)(unsigned long evt, struct clock_event_device *); int (*set_next_ktime)(ktime_t expires, struct clock_event_device *); + void (*set_next_coupled)(u64 cycles, struct clock_event_device *); ktime_t next_event; u64 max_delta_ns; u64 min_delta_ns; @@ -108,6 +112,8 @@ struct clock_event_device { u32 shift; enum clock_event_state state_use_accessors; unsigned int features; + enum clocksource_ids cs_id; + unsigned int next_event_forced; unsigned long retries; int (*set_state_periodic)(struct clock_event_device *); diff --git a/include/linux/clocksource.h b/include/linux/clocksource.h index 65b7c41471c3..283d7297aa79 100644 --- a/include/linux/clocksource.h +++ b/include/linux/clocksource.h @@ -25,14 +25,28 @@ struct clocksource_base; struct clocksource; struct module; -#if defined(CONFIG_ARCH_CLOCKSOURCE_DATA) || \ - defined(CONFIG_GENERIC_GETTIMEOFDAY) +#if defined(CONFIG_GENERIC_GETTIMEOFDAY) #include <asm/clocksource.h> #endif #include <vdso/clocksource.h> /** + * struct clocksource_hw_snapshot - Snapshot for the underlying hardware counter of derived + * clocksources like kvmclock or Hyper-V scaled TSC + * @hw_cycles: The hardware counter value + * @hw_csid: Clocksource ID of the hardware counter + * + * Such clocksources must implement the read_snapshot() callback and fill in the + * hardware counter value, the clocksource ID of the hardware counter and derive + * the actual clocksource cycles from @hw_cycles to provide an atomic snapshot + */ +struct clocksource_hw_snapshot { + u64 hw_cycles; + enum clocksource_ids hw_csid; +}; + +/** * struct clocksource - hardware abstraction for a free running counter * Provides mostly state-free accessors to the underlying hardware. * This is the structure used for system time. @@ -44,8 +58,6 @@ struct module; * @shift: Cycle to nanosecond divisor (power of two) * @max_idle_ns: Maximum idle time permitted by the clocksource (nsecs) * @maxadj: Maximum adjustment value to mult (~11%) - * @uncertainty_margin: Maximum uncertainty in nanoseconds per half second. - * Zero says to use default WATCHDOG_THRESHOLD. * @archdata: Optional arch-specific data * @max_cycles: Maximum safe cycle value which won't overflow on * multiplication @@ -75,6 +87,14 @@ struct module; * @flags: Flags describing special properties * @base: Hardware abstraction for clock on which a clocksource * is based + * @read_snapshot: Extended @read() function for clocksources such as + * kvmclock or the Hyper-V scaled TSC where the actual + * clocksource value for timekeeping is calculated from an + * underlying hardware counter. Returns the timekeeping + * relevant cycle value and stores the raw value of the + * underlying counter from which it was calculated + * including the clocksource ID of that counter in the + * clocksource hardware snapshot. * @enable: Optional function to enable the clocksource * @disable: Optional function to disable the clocksource * @suspend: Optional suspend function for the clocksource @@ -105,10 +125,6 @@ struct clocksource { u32 shift; u64 max_idle_ns; u32 maxadj; - u32 uncertainty_margin; -#ifdef CONFIG_ARCH_CLOCKSOURCE_DATA - struct arch_clocksource_data archdata; -#endif u64 max_cycles; u64 max_raw_delta; const char *name; @@ -120,6 +136,7 @@ struct clocksource { unsigned long flags; struct clocksource_base *base; + u64 (*read_snapshot)(struct clocksource *cs, struct clocksource_hw_snapshot *chs); int (*enable)(struct clocksource *cs); void (*disable)(struct clocksource *cs); void (*suspend)(struct clocksource *cs); @@ -133,6 +150,7 @@ struct clocksource { struct list_head wd_list; u64 cs_last; u64 wd_last; + unsigned int wd_cpu; #endif struct module *owner; }; @@ -142,13 +160,19 @@ struct clocksource { */ #define CLOCK_SOURCE_IS_CONTINUOUS 0x01 #define CLOCK_SOURCE_MUST_VERIFY 0x02 +#define CLOCK_SOURCE_CALIBRATED 0x04 #define CLOCK_SOURCE_WATCHDOG 0x10 #define CLOCK_SOURCE_VALID_FOR_HRES 0x20 #define CLOCK_SOURCE_UNSTABLE 0x40 #define CLOCK_SOURCE_SUSPEND_NONSTOP 0x80 #define CLOCK_SOURCE_RESELECT 0x100 -#define CLOCK_SOURCE_VERIFY_PERCPU 0x200 +#define CLOCK_SOURCE_CAN_INLINE_READ 0x200 +#define CLOCK_SOURCE_HAS_COUPLED_CLOCK_EVENT 0x400 + +#define CLOCK_SOURCE_WDTEST 0x800 +#define CLOCK_SOURCE_WDTEST_PERCPU 0x1000 + /* simplify initialization of mask field */ #define CLOCKSOURCE_MASK(bits) GENMASK_ULL((bits) - 1, 0) @@ -236,8 +260,9 @@ clocks_calc_mult_shift(u32 *mult, u32 *shift, u32 from, u32 to, u32 minsec); */ extern int __clocksource_register_scale(struct clocksource *cs, u32 scale, u32 freq); -extern void -__clocksource_update_freq_scale(struct clocksource *cs, u32 scale, u32 freq); +extern int +__devm_clocksource_register_scale(struct device *dev, struct clocksource *cs, + u32 scale, u32 freq); /* * Don't call this unless you are a default clocksource @@ -258,14 +283,16 @@ static inline int clocksource_register_khz(struct clocksource *cs, u32 khz) return __clocksource_register_scale(cs, 1000, khz); } -static inline void __clocksource_update_freq_hz(struct clocksource *cs, u32 hz) +static inline int devm_clocksource_register_hz(struct device *dev, + struct clocksource *cs, u32 hz) { - __clocksource_update_freq_scale(cs, 1, hz); + return __devm_clocksource_register_scale(dev, cs, 1, hz); } -static inline void __clocksource_update_freq_khz(struct clocksource *cs, u32 khz) +static inline int devm_clocksource_register_khz(struct device *dev, + struct clocksource *cs, u32 khz) { - __clocksource_update_freq_scale(cs, 1000, khz); + return __devm_clocksource_register_scale(dev, cs, 1000, khz); } #ifdef CONFIG_ARCH_CLOCKSOURCE_INIT @@ -298,21 +325,6 @@ static inline void timer_probe(void) {} #define TIMER_ACPI_DECLARE(name, table_id, fn) \ ACPI_DECLARE_PROBE_ENTRY(timer, name, table_id, 0, NULL, 0, fn) -static inline unsigned int clocksource_get_max_watchdog_retry(void) -{ - /* - * When system is in the boot phase or under heavy workload, there - * can be random big latencies during the clocksource/watchdog - * read, so allow retries to filter the noise latency. As the - * latency's frequency and maximum value goes up with the number of - * CPUs, scale the number of retries with the number of online - * CPUs. - */ - return (ilog2(num_online_cpus()) / 2) + 1; -} - -void clocksource_verify_percpu(struct clocksource *cs); - /** * struct clocksource_base - hardware abstraction for clock on which a clocksource * is based diff --git a/include/linux/cma.h b/include/linux/cma.h index d0793eaaadaa..8555d38a97b1 100644 --- a/include/linux/cma.h +++ b/include/linux/cma.h @@ -61,14 +61,4 @@ extern int cma_for_each_area(int (*it)(struct cma *cma, void *data), void *data) extern bool cma_intersects(struct cma *cma, unsigned long start, unsigned long end); extern void cma_reserve_pages_on_error(struct cma *cma); - -#ifdef CONFIG_DMA_CMA -extern bool cma_skip_dt_default_reserved_mem(void); -#else -static inline bool cma_skip_dt_default_reserved_mem(void) -{ - return false; -} -#endif - #endif diff --git a/include/linux/cnt32_to_63.h b/include/linux/cnt32_to_63.h deleted file mode 100644 index 064428479f2d..000000000000 --- a/include/linux/cnt32_to_63.h +++ /dev/null @@ -1,104 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Extend a 32-bit counter to 63 bits - * - * Author: Nicolas Pitre - * Created: December 3, 2006 - * Copyright: MontaVista Software, Inc. - */ - -#ifndef __LINUX_CNT32_TO_63_H__ -#define __LINUX_CNT32_TO_63_H__ - -#include <linux/compiler.h> -#include <linux/types.h> -#include <asm/byteorder.h> - -/* this is used only to give gcc a clue about good code generation */ -union cnt32_to_63 { - struct { -#if defined(__LITTLE_ENDIAN) - u32 lo, hi; -#elif defined(__BIG_ENDIAN) - u32 hi, lo; -#endif - }; - u64 val; -}; - - -/** - * cnt32_to_63 - Expand a 32-bit counter to a 63-bit counter - * @cnt_lo: The low part of the counter - * - * Many hardware clock counters are only 32 bits wide and therefore have - * a relatively short period making wrap-arounds rather frequent. This - * is a problem when implementing sched_clock() for example, where a 64-bit - * non-wrapping monotonic value is expected to be returned. - * - * To overcome that limitation, let's extend a 32-bit counter to 63 bits - * in a completely lock free fashion. Bits 0 to 31 of the clock are provided - * by the hardware while bits 32 to 62 are stored in memory. The top bit in - * memory is used to synchronize with the hardware clock half-period. When - * the top bit of both counters (hardware and in memory) differ then the - * memory is updated with a new value, incrementing it when the hardware - * counter wraps around. - * - * Because a word store in memory is atomic then the incremented value will - * always be in synch with the top bit indicating to any potential concurrent - * reader if the value in memory is up to date or not with regards to the - * needed increment. And any race in updating the value in memory is harmless - * as the same value would simply be stored more than once. - * - * The restrictions for the algorithm to work properly are: - * - * 1) this code must be called at least once per each half period of the - * 32-bit counter; - * - * 2) this code must not be preempted for a duration longer than the - * 32-bit counter half period minus the longest period between two - * calls to this code; - * - * Those requirements ensure proper update to the state bit in memory. - * This is usually not a problem in practice, but if it is then a kernel - * timer should be scheduled to manage for this code to be executed often - * enough. - * - * And finally: - * - * 3) the cnt_lo argument must be seen as a globally incrementing value, - * meaning that it should be a direct reference to the counter data which - * can be evaluated according to a specific ordering within the macro, - * and not the result of a previous evaluation stored in a variable. - * - * For example, this is wrong: - * - * u32 partial = get_hw_count(); - * u64 full = cnt32_to_63(partial); - * return full; - * - * This is fine: - * - * u64 full = cnt32_to_63(get_hw_count()); - * return full; - * - * Note that the top bit (bit 63) in the returned value should be considered - * as garbage. It is not cleared here because callers are likely to use a - * multiplier on the returned value which can get rid of the top bit - * implicitly by making the multiplier even, therefore saving on a runtime - * clear-bit instruction. Otherwise caller must remember to clear the top - * bit explicitly. - */ -#define cnt32_to_63(cnt_lo) \ -({ \ - static u32 __m_cnt_hi; \ - union cnt32_to_63 __x; \ - __x.hi = __m_cnt_hi; \ - smp_rmb(); \ - __x.lo = (cnt_lo); \ - if (unlikely((s32)(__x.hi ^ __x.lo) < 0)) \ - __m_cnt_hi = __x.hi = (__x.hi ^ 0x80000000) + (__x.hi >> 31); \ - __x.val; \ -}) - -#endif diff --git a/include/linux/cnum.h b/include/linux/cnum.h new file mode 100644 index 000000000000..49b7d0c7645d --- /dev/null +++ b/include/linux/cnum.h @@ -0,0 +1,82 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ + +#ifndef _LINUX_CNUM_H +#define _LINUX_CNUM_H + +#include <linux/types.h> + +/* + * cnum32: a circular number. + * A unified representation for signed and unsigned ranges. + * + * Assume that a 32-bit range is a circle, with 0 being in the 12 o'clock + * position, numbers placed sequentially in clockwise order and U32_MAX + * in the 11 o'clock position. Signed values map onto the same circle: + * S32_MAX sits at 5 o'clock, S32_MIN sits at 6 o'clock (opposite 0), + * negative values occupy the left half and positive values the right half. + * + * @cnum32 represents an arc on this circle drawn clockwise. + * @base corresponds to the first value of the range. + * @size corresponds to the number of integers in the range excluding @base. + * (The @base is excluded to avoid integer overflow when representing the full + * 0..U32_MAX range, which corresponds to 2^32, which can't be stored in u32). + * + * For example: {U32_MAX, 1} corresponds to signed range [-1, 0], + * {S32_MAX, 1} corresponds to unsigned range [S32_MAX, S32_MIN]. + */ +struct cnum32 { + u32 base; + u32 size; +}; + +#define CNUM32_UNBOUNDED ((struct cnum32){ .base = 0, .size = U32_MAX }) +#define CNUM32_EMPTY ((struct cnum32){ .base = U32_MAX, .size = U32_MAX }) + +struct cnum32 cnum32_from_urange(u32 min, u32 max); +struct cnum32 cnum32_from_srange(s32 min, s32 max); +u32 cnum32_umin(struct cnum32 cnum); +u32 cnum32_umax(struct cnum32 cnum); +s32 cnum32_smin(struct cnum32 cnum); +s32 cnum32_smax(struct cnum32 cnum); +struct cnum32 cnum32_intersect(struct cnum32 a, struct cnum32 b); +void cnum32_intersect_with(struct cnum32 *dst, struct cnum32 src); +void cnum32_intersect_with_urange(struct cnum32 *dst, u32 min, u32 max); +void cnum32_intersect_with_srange(struct cnum32 *dst, s32 min, s32 max); +bool cnum32_contains(struct cnum32 cnum, u32 v); +bool cnum32_is_const(struct cnum32 cnum); +bool cnum32_is_empty(struct cnum32 cnum); +struct cnum32 cnum32_add(struct cnum32 a, struct cnum32 b); +struct cnum32 cnum32_negate(struct cnum32 a); +bool cnum32_is_subset(struct cnum32 outer, struct cnum32 inner); + +/* Same as cnum32 but for 64-bit ranges */ +struct cnum64 { + u64 base; + u64 size; +}; + +#define CNUM64_UNBOUNDED ((struct cnum64){ .base = 0, .size = U64_MAX }) +#define CNUM64_EMPTY ((struct cnum64){ .base = U64_MAX, .size = U64_MAX }) + +struct cnum64 cnum64_from_urange(u64 min, u64 max); +struct cnum64 cnum64_from_srange(s64 min, s64 max); +u64 cnum64_umin(struct cnum64 cnum); +u64 cnum64_umax(struct cnum64 cnum); +s64 cnum64_smin(struct cnum64 cnum); +s64 cnum64_smax(struct cnum64 cnum); +struct cnum64 cnum64_intersect(struct cnum64 a, struct cnum64 b); +void cnum64_intersect_with(struct cnum64 *dst, struct cnum64 src); +void cnum64_intersect_with_urange(struct cnum64 *dst, u64 min, u64 max); +void cnum64_intersect_with_srange(struct cnum64 *dst, s64 min, s64 max); +bool cnum64_contains(struct cnum64 cnum, u64 v); +bool cnum64_is_const(struct cnum64 cnum); +bool cnum64_is_empty(struct cnum64 cnum); +struct cnum64 cnum64_add(struct cnum64 a, struct cnum64 b); +struct cnum64 cnum64_negate(struct cnum64 a); +bool cnum64_is_subset(struct cnum64 outer, struct cnum64 inner); + +struct cnum32 cnum32_from_cnum64(struct cnum64 cnum); +struct cnum64 cnum64_cnum32_intersect(struct cnum64 a, struct cnum32 b); + +#endif /* _LINUX_CNUM_H */ diff --git a/include/linux/codetag.h b/include/linux/codetag.h index 8ea2a5f7c98a..a25a085c2df1 100644 --- a/include/linux/codetag.h +++ b/include/linux/codetag.h @@ -74,8 +74,11 @@ struct codetag_iterator { .flags = 0, \ } -void codetag_lock_module_list(struct codetag_type *cttype, bool lock); +void codetag_lock_module_list(struct codetag_type *cttype); bool codetag_trylock_module_list(struct codetag_type *cttype); +void codetag_unlock_module_list(struct codetag_type *cttype); +unsigned long codetag_get_content_id(struct codetag_type *cttype); +unsigned int codetag_get_count(struct codetag_type *cttype); struct codetag_iterator codetag_get_ct_iter(struct codetag_type *cttype); struct codetag *codetag_next_ct(struct codetag_iterator *iter); diff --git a/include/linux/comedi/comedi_isadma.h b/include/linux/comedi/comedi_isadma.h index 9d2b12db7e6e..7514ce222fa6 100644 --- a/include/linux/comedi/comedi_isadma.h +++ b/include/linux/comedi/comedi_isadma.h @@ -48,11 +48,11 @@ struct comedi_isadma_desc { */ struct comedi_isadma { struct device *dev; - struct comedi_isadma_desc *desc; int n_desc; int cur_dma; unsigned int chan; unsigned int chan2; + struct comedi_isadma_desc desc[] __counted_by(n_desc); }; #if IS_ENABLED(CONFIG_ISA_DMA_API) diff --git a/include/linux/comedi/comedidev.h b/include/linux/comedi/comedidev.h index 35fdc41845ce..577a08f37aee 100644 --- a/include/linux/comedi/comedidev.h +++ b/include/linux/comedi/comedidev.h @@ -1026,10 +1026,55 @@ int comedi_load_firmware(struct comedi_device *dev, struct device *hw_dev, unsigned long context), unsigned long context); -int __comedi_request_region(struct comedi_device *dev, - unsigned long start, unsigned long len); -int comedi_request_region(struct comedi_device *dev, - unsigned long start, unsigned long len); +int __comedi_check_request_region(struct comedi_device *dev, + unsigned long start, unsigned long len, + unsigned long minstart, unsigned long maxend, + unsigned long minalign); +int comedi_check_request_region(struct comedi_device *dev, + unsigned long start, unsigned long len, + unsigned long minstart, unsigned long maxend, + unsigned long minalign); + +/** + * __comedi_request_region() - Request an I/O region for a legacy driver + * @dev: COMEDI device. + * @start: Base address of the I/O region. + * @len: Length of the I/O region. + * + * Requests the specified I/O port region which must start at a non-zero + * address. + * + * Returns 0 on success, -EINVAL if @start is 0, or -EIO if the request + * fails. + */ +static inline int __comedi_request_region(struct comedi_device *dev, + unsigned long start, + unsigned long len) +{ + return __comedi_check_request_region(dev, start, len, 0, ~0ul, 1); +} + +/** + * comedi_request_region() - Request an I/O region for a legacy driver + * @dev: COMEDI device. + * @start: Base address of the I/O region. + * @len: Length of the I/O region. + * + * Requests the specified I/O port region which must start at a non-zero + * address. + * + * On success, @dev->iobase is set to the base address of the region and + * @dev->iolen is set to its length. + * + * Returns 0 on success, -EINVAL if @start is 0, or -EIO if the request + * fails. + */ +static inline int comedi_request_region(struct comedi_device *dev, + unsigned long start, unsigned long len) +{ + return comedi_check_request_region(dev, start, len, 0, ~0ul, 1); +} + void comedi_legacy_detach(struct comedi_device *dev); int comedi_auto_config(struct device *hardware_device, diff --git a/include/linux/compaction.h b/include/linux/compaction.h index 173d9c07a895..66a2f70e9e01 100644 --- a/include/linux/compaction.h +++ b/include/linux/compaction.h @@ -2,6 +2,8 @@ #ifndef _LINUX_COMPACTION_H #define _LINUX_COMPACTION_H +#include <linux/swap.h> + /* * Determines how hard direct compaction should try to succeed. * Lower value means higher priority, analogically to reclaim priority. @@ -56,6 +58,7 @@ enum compact_result { }; struct alloc_context; /* in mm/internal.h */ +struct capture_control; /* in mm/internal.h */ /* * Number of free order-0 pages that should be available above given watermark @@ -73,11 +76,9 @@ static inline unsigned long compact_gap(unsigned int order) * effectively limited by COMPACT_CLUSTER_MAX, as that's the maximum * that the migrate scanner can have isolated on migrate list, and free * scanner is only invoked when the number of isolated free pages is - * lower than that. But it's not worth to complicate the formula here - * as a bigger gap for higher orders than strictly necessary can also - * improve chances of compaction success. + * lower than that. */ - return 2UL << order; + return min(2UL << order, COMPACT_CLUSTER_MAX); } static inline int current_is_kcompactd(void) @@ -92,7 +93,7 @@ extern int fragmentation_index(struct zone *zone, unsigned int order); extern enum compact_result try_to_compact_pages(gfp_t gfp_mask, unsigned int order, unsigned int alloc_flags, const struct alloc_context *ac, enum compact_priority prio, - struct page **page); + struct capture_control *capc); extern void reset_isolation_suitable(pg_data_t *pgdat); extern bool compaction_suitable(struct zone *zone, int order, unsigned long watermark, int highest_zoneidx); @@ -101,7 +102,7 @@ extern void compaction_defer_reset(struct zone *zone, int order, bool alloc_success); bool compaction_zonelist_suitable(struct alloc_context *ac, int order, - int alloc_flags); + int alloc_flags, gfp_t gfp_mask); extern void __meminit kcompactd_run(int nid); extern void __meminit kcompactd_stop(int nid); diff --git a/include/linux/compat.h b/include/linux/compat.h index 56cebaff0c91..8da0a15c95f4 100644 --- a/include/linux/compat.h +++ b/include/linux/compat.h @@ -72,6 +72,10 @@ __diag_push(); \ __diag_ignore(GCC, 8, "-Wattribute-alias", \ "Type aliasing is used to sanitize syscall arguments");\ + __diag_ignore(clang, 23, "-Wunknown-warning-option", \ + "Avoid breaking versions without -Wattribute-alias"); \ + __diag_ignore(clang, 23, "-Wattribute-alias", \ + "Type aliasing is used to sanitize syscall arguments"); \ asmlinkage long compat_sys##name(__MAP(x,__SC_DECL,__VA_ARGS__)) \ __attribute__((alias(__stringify(__se_compat_sys##name)))); \ ALLOW_ERROR_INJECTION(compat_sys##name, ERRNO); \ diff --git a/include/linux/compiler-clang.h b/include/linux/compiler-clang.h index e1123dd28486..e606ed6c7539 100644 --- a/include/linux/compiler-clang.h +++ b/include/linux/compiler-clang.h @@ -5,15 +5,6 @@ /* Compiler specific definitions for Clang compiler */ -/* - * Clang prior to 17 is being silly and considers many __cleanup() variables - * as unused (because they are, their sole purpose is to go out of scope). - * - * https://github.com/llvm/llvm-project/commit/877210faa447f4cc7db87812f8ed80e398fedd61 - */ -#undef __cleanup -#define __cleanup(func) __maybe_unused __attribute__((__cleanup__(func))) - /* all clang versions usable with the kernel support KASAN ABI version 5 */ #define KASAN_ABI_VERSION 5 @@ -131,10 +122,16 @@ #define __diag_str(s) __diag_str1(s) #define __diag(s) _Pragma(__diag_str(clang diagnostic s)) -#define __diag_clang_13(s) __diag(s) +#if CONFIG_CLANG_VERSION >= 230000 +#define __diag_clang_23(s) __diag(s) +#else +#define __diag_clang_23(s) +#endif + +#define __diag_clang_all(s) __diag(s) #define __diag_ignore_all(option, comment) \ - __diag_clang(13, ignore, option) + __diag_clang(all, ignore, option) /* * clang has horrible behavior with "g" or "rm" constraints for asm diff --git a/include/linux/compiler-context-analysis.h b/include/linux/compiler-context-analysis.h index 00c074a2ccb0..8302ebc2ea8c 100644 --- a/include/linux/compiler-context-analysis.h +++ b/include/linux/compiler-context-analysis.h @@ -39,12 +39,14 @@ # define __assumes_shared_ctx_lock(...) __attribute__((assert_shared_capability(__VA_ARGS__))) /** - * __guarded_by - struct member and globals attribute, declares variable - * only accessible within active context + * __guarded_by() - struct member and globals attribute, declares variable + * only accessible within active context + * @...: context lock instance pointer(s) * * Declares that the struct member or global variable is only accessible within - * the context entered by the given context lock. Read operations on the data - * require shared access, while write operations require exclusive access. + * the context entered by the given context lock(s). Read operations on the data + * require shared access to at least one of the context locks, while write + * operations require exclusive access to all listed context locks. * * .. code-block:: c * @@ -52,17 +54,24 @@ * spinlock_t lock; * long counter __guarded_by(&lock); * }; + * + * struct some_state { + * spinlock_t lock1, lock2; + * long counter __guarded_by(&lock1, &lock2); + * }; */ # define __guarded_by(...) __attribute__((guarded_by(__VA_ARGS__))) /** - * __pt_guarded_by - struct member and globals attribute, declares pointed-to - * data only accessible within active context + * __pt_guarded_by() - struct member and globals attribute, declares pointed-to + * data only accessible within active context + * @...: context lock instance pointer(s) * * Declares that the data pointed to by the struct member pointer or global * pointer is only accessible within the context entered by the given context - * lock. Read operations on the data require shared access, while write - * operations require exclusive access. + * lock(s). Read operations on the data require shared access to at least one + * of the context locks, while write operations require exclusive access to all + * listed context locks. * * .. code-block:: c * @@ -70,6 +79,11 @@ * spinlock_t lock; * long *counter __pt_guarded_by(&lock); * }; + * + * struct some_state { + * spinlock_t lock1, lock2; + * long *counter __pt_guarded_by(&lock1, &lock2); + * }; */ # define __pt_guarded_by(...) __attribute__((pt_guarded_by(__VA_ARGS__))) @@ -320,6 +334,38 @@ static inline void _context_unsafe_alias(void **p) { } */ #define __releases(...) __releases_ctx_lock(__VA_ARGS__) +/* + * Clang's analysis does not care precisely about the value, only that it is + * either zero or non-zero. So the __cond_acquires() interface might be + * misleading if we say that @ret is the value returned if acquired. Instead, + * provide symbolic variants which we translate. + */ +#define __cond_acquires_impl_not_true(x, ...) __try_acquires##__VA_ARGS__##_ctx_lock(0, x) +#define __cond_acquires_impl_not_false(x, ...) __try_acquires##__VA_ARGS__##_ctx_lock(1, x) +#define __cond_acquires_impl_not_nonzero(x, ...) __try_acquires##__VA_ARGS__##_ctx_lock(0, x) +#define __cond_acquires_impl_not_0(x, ...) __try_acquires##__VA_ARGS__##_ctx_lock(1, x) +#define __cond_acquires_impl_not_nonnull(x, ...) __try_acquires##__VA_ARGS__##_ctx_lock(0, x) +#define __cond_acquires_impl_not_NULL(x, ...) __try_acquires##__VA_ARGS__##_ctx_lock(1, x) + +/** + * __cond_releases() - function attribute, function conditionally + * releases a context lock exclusively + * @ret: abstract value returned by function if context lock releases + * @x: context lock instance pointer + * + * Function attribute declaring that the function conditionally releases the + * given context lock instance @x exclusively. The associated context(s) must + * be active on entry. The function return value @ret denotes when the context + * lock is released. + * + * @ret may be one of: true, false, nonzero, 0, nonnull, NULL. + * + * NOTE: clang does not have a native attribute for this; instead implement + * it as an unconditional release and a conditional acquire for the + * inverted condition -- which is semantically equivalent. + */ +#define __cond_releases(ret, x) __releases(x) __cond_acquires_impl_not_##ret(x) + /** * __acquire() - function to acquire context lock exclusively * @x: context lock instance pointer diff --git a/include/linux/compiler.h b/include/linux/compiler.h index af16624b29fd..cb2f6050bdf7 100644 --- a/include/linux/compiler.h +++ b/include/linux/compiler.h @@ -149,10 +149,7 @@ void ftrace_likely_update(struct ftrace_likely_data *f, int val, #endif #ifndef RELOC_HIDE -# define RELOC_HIDE(ptr, off) \ - ({ unsigned long __ptr; \ - __ptr = (unsigned long) (ptr); \ - (typeof(ptr)) (__ptr + (off)); }) +# define RELOC_HIDE(ptr, off) ((typeof(ptr))((unsigned long)(ptr) + (off))) #endif #define absolute_pointer(val) RELOC_HIDE((void *)(val), 0) diff --git a/include/linux/compiler_attributes.h b/include/linux/compiler_attributes.h index c16d4199bf92..476c4c560d17 100644 --- a/include/linux/compiler_attributes.h +++ b/include/linux/compiler_attributes.h @@ -231,6 +231,15 @@ #define noinline __attribute__((__noinline__)) /* + * Note: deliberately not named '__nonnull', to avoid clashing with glibc's + * __nonnull() when kernel and userspace headers are combined. + * + * gcc: https://gcc.gnu.org/onlinedocs/gcc/Common-Attributes.html#index-nonnull + * clang: https://clang.llvm.org/docs/AttributeReference.html#nonnull + */ +#define __nonnull_args(x...) __attribute__((__nonnull__(x))) + +/* * Optional: only supported since gcc >= 8 * Optional: not supported by clang * @@ -397,6 +406,17 @@ #endif /* + * Optional: not supported by clang + * + * gcc: https://gcc.gnu.org/onlinedocs/gcc/Common-Attributes.html#index-noipa + */ +#if __has_attribute(noipa) +# define __noipa __attribute__((noipa)) +#else +# define __noipa +#endif + +/* * gcc: https://gcc.gnu.org/onlinedocs/gcc/Common-Function-Attributes.html#index-weak-function-attribute * gcc: https://gcc.gnu.org/onlinedocs/gcc/Common-Variable-Attributes.html#index-weak-variable-attribute */ diff --git a/include/linux/compiler_types.h b/include/linux/compiler_types.h index 890076d0974b..c5921f139007 100644 --- a/include/linux/compiler_types.h +++ b/include/linux/compiler_types.h @@ -432,18 +432,11 @@ struct ftrace_likely_data { #define at_least #endif -/* Do not trap wrapping arithmetic within an annotated function. */ -#ifdef CONFIG_UBSAN_INTEGER_WRAP -# define __signed_wrap __attribute__((no_sanitize("signed-integer-overflow"))) -#else -# define __signed_wrap -#endif - /* Section for code which can't be instrumented at all */ #define __noinstr_section(section) \ noinline notrace __attribute((__section__(section))) \ __no_kcsan __no_sanitize_address __no_profile __no_sanitize_coverage \ - __no_sanitize_memory __signed_wrap + __no_sanitize_memory #define noinstr __noinstr_section(".noinstr.text") @@ -641,6 +634,9 @@ struct ftrace_likely_data { #else #define __unqual_scalar_typeof(x) __typeof_unqual__(x) #endif + +#include <asm/percpu_types.h> + #endif /* !__ASSEMBLY__ */ /* @@ -718,6 +714,10 @@ struct ftrace_likely_data { #define __diag_GCC(version, severity, string) #endif +#ifndef __diag_clang +#define __diag_clang(version, severity, string) +#endif + #define __diag_push() __diag(push) #define __diag_pop() __diag(pop) diff --git a/include/linux/console.h b/include/linux/console.h index 5520e4477ad7..502d1abe3f50 100644 --- a/include/linux/console.h +++ b/include/linux/console.h @@ -173,7 +173,7 @@ static inline void con_debug_leave(void) { } * @CON_BRL: Indicates a braille device which is exempt from * receiving the printk spam for obvious reasons. * @CON_EXTENDED: The console supports the extended output format of - * /dev/kmesg which requires a larger output buffer. + * /dev/kmsg which requires a larger output buffer. * @CON_SUSPENDED: Indicates if a console is suspended. If true, the * printing callbacks must not be called. * @CON_NBCON: Console can operate outside of the legacy style console_lock @@ -718,8 +718,6 @@ extern bool console_suspend_enabled; extern void console_suspend_all(void); extern void console_resume_all(void); -int mda_console_init(void); - void vcs_make_sysfs(int index); void vcs_remove_sysfs(int index); diff --git a/include/linux/console_struct.h b/include/linux/console_struct.h index d5ca855116df..fe915afdece5 100644 --- a/include/linux/console_struct.h +++ b/include/linux/console_struct.h @@ -13,8 +13,8 @@ #ifndef _LINUX_CONSOLE_STRUCT_H #define _LINUX_CONSOLE_STRUCT_H -#include <linux/wait.h> #include <linux/vt.h> +#include <linux/wait.h> #include <linux/workqueue.h> struct uni_pagedict; @@ -58,6 +58,33 @@ struct vc_state { bool reverse; }; +/** + * struct vc_font - Describes a font + * @width: The width of a single glyph in bits + * @height: The height of a single glyph in scanlines + * @charcount: The number of glyphs in the font + * @data: The raw font data + * + * Font data is organized as an array of glyphs. Each glyph is a bitmap with + * set bits indicating the foreground color. Unset bits indicate background + * color. The fields @width and @height store a single glyph's number of + * horizontal bits and vertical scanlines. If width is not a multiple of 8, + * there are trailing bits to fill up the byte. These bits should not be drawn. + * + * The field @data points to the first glyph's first byte. The value @charcount + * gives the number of glyphs in the font. There are no empty scanlines between + * two adjacent glyphs. + */ +struct vc_font { + unsigned int width; + unsigned int height; + unsigned int charcount; + const unsigned char *data; +}; + +unsigned int vc_font_pitch(const struct vc_font *font); +unsigned int vc_font_size(const struct vc_font *font); + /* * Example: vc_data of a console that was scrolled 3 lines down. * @@ -120,9 +147,9 @@ struct vc_data { unsigned short vc_complement_mask; /* [#] Xor mask for mouse pointer */ unsigned short vc_s_complement_mask; /* Saved mouse pointer mask */ unsigned long vc_pos; /* Cursor address */ - /* fonts */ + /* fonts */ unsigned short vc_hi_font_mask; /* [#] Attribute set for upper 256 chars of font or 0 if not supported */ - struct console_font vc_font; /* Current VC font set */ + struct vc_font vc_font; /* Current VC font set */ unsigned short vc_video_erase_char; /* Background erase character */ /* VT terminal data */ unsigned int vc_state; /* Escape sequence parser state */ diff --git a/include/linux/consolemap.h b/include/linux/consolemap.h index 6180b803795c..539d488fdc03 100644 --- a/include/linux/consolemap.h +++ b/include/linux/consolemap.h @@ -28,8 +28,7 @@ int conv_uni_to_pc(struct vc_data *conp, long ucs); u32 conv_8bit_to_uni(unsigned char c); int conv_uni_to_8bit(u32 uni); void console_map_init(void); -bool ucs_is_double_width(uint32_t cp); -bool ucs_is_zero_width(uint32_t cp); +unsigned int ucs_get_width(uint32_t cp); u32 ucs_recompose(u32 base, u32 mark); u32 ucs_get_fallback(u32 cp); #else @@ -62,14 +61,9 @@ static inline int conv_uni_to_8bit(u32 uni) static inline void console_map_init(void) { } -static inline bool ucs_is_double_width(uint32_t cp) +static inline unsigned int ucs_get_width(uint32_t cp) { - return false; -} - -static inline bool ucs_is_zero_width(uint32_t cp) -{ - return false; + return 1; } static inline u32 ucs_recompose(u32 base, u32 mark) diff --git a/include/linux/container_of.h b/include/linux/container_of.h index 1f6ebf27d962..28db38e9ee3e 100644 --- a/include/linux/container_of.h +++ b/include/linux/container_of.h @@ -17,11 +17,10 @@ * Do not use container_of() in new code. */ #define container_of(ptr, type, member) ({ \ - void *__mptr = (void *)(ptr); \ - static_assert(__same_type(*(ptr), ((type *)0)->member) || \ + static_assert(__same_type(*(ptr), typeof_member(type, member)) || \ __same_type(*(ptr), void), \ "pointer type mismatch in container_of()"); \ - ((type *)(__mptr - offsetof(type, member))); }) + (type *)((void *)(ptr) - offsetof(type, member)); }) /** * container_of_const - cast a member of a structure out to the containing diff --git a/include/linux/coreboot.h b/include/linux/coreboot.h new file mode 100644 index 000000000000..5d40ca7a1d89 --- /dev/null +++ b/include/linux/coreboot.h @@ -0,0 +1,90 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * coreboot.h + * + * Coreboot device and driver interfaces. + * + * Copyright 2014 Gerd Hoffmann <kraxel@redhat.com> + * Copyright 2017 Google Inc. + * Copyright 2017 Samuel Holland <samuel@sholland.org> + */ + +#ifndef _LINUX_COREBOOT_H +#define _LINUX_COREBOOT_H + +#include <linux/compiler_attributes.h> +#include <linux/stddef.h> +#include <linux/types.h> + +typedef __aligned(4) u64 cb_u64; + +/* List of coreboot entry structures that is used */ + +#define CB_TAG_FRAMEBUFFER 0x12 +#define LB_TAG_CBMEM_ENTRY 0x31 + +/* Generic */ +struct coreboot_table_entry { + u32 tag; + u32 size; +}; + +/* Points to a CBMEM entry */ +struct lb_cbmem_ref { + u32 tag; + u32 size; + + cb_u64 cbmem_addr; +}; + +/* Corresponds to LB_TAG_CBMEM_ENTRY */ +struct lb_cbmem_entry { + u32 tag; + u32 size; + + cb_u64 address; + u32 entry_size; + u32 id; +}; + +#define LB_FRAMEBUFFER_ORIENTATION_NORMAL 0 +#define LB_FRAMEBUFFER_ORIENTATION_BOTTOM_UP 1 +#define LB_FRAMEBUFFER_ORIENTATION_LEFT_UP 2 +#define LB_FRAMEBUFFER_ORIENTATION_RIGHT_UP 3 + +/* Describes framebuffer setup by coreboot */ +struct lb_framebuffer { + u32 tag; + u32 size; + + cb_u64 physical_address; + u32 x_resolution; + u32 y_resolution; + u32 bytes_per_line; + u8 bits_per_pixel; + u8 red_mask_pos; + u8 red_mask_size; + u8 green_mask_pos; + u8 green_mask_size; + u8 blue_mask_pos; + u8 blue_mask_size; + u8 reserved_mask_pos; + u8 reserved_mask_size; + u8 orientation; +}; + +/* + * True if the coreboot-provided data is large enough to hold information + * on the linear framebuffer. False otherwise. + */ +#define LB_FRAMEBUFFER_HAS_LFB(__fb) \ + ((__fb)->size >= offsetofend(struct lb_framebuffer, reserved_mask_size)) + +/* + * True if the coreboot-provided data is large enough to hold information + * on the display orientation. False otherwise. + */ +#define LB_FRAMEBUFFER_HAS_ORIENTATION(__fb) \ + ((__fb)->size >= offsetofend(struct lb_framebuffer, orientation)) + +#endif /* _LINUX_COREBOOT_H */ diff --git a/include/linux/coredump.h b/include/linux/coredump.h index 68861da4cf7c..7b38ee2e7913 100644 --- a/include/linux/coredump.h +++ b/include/linux/coredump.h @@ -5,6 +5,7 @@ #include <linux/types.h> #include <linux/mm.h> #include <linux/fs.h> +#include <linux/sched/coredump.h> #include <asm/siginfo.h> #ifdef CONFIG_COREDUMP @@ -20,7 +21,10 @@ struct coredump_params { const kernel_siginfo_t *siginfo; struct file *file; unsigned long limit; + /* MMF_DUMP_FILTER_* bits, snapshot of mm->flags at dump start. */ unsigned long mm_flags; + /* Snapshot of dumpable at dump start. */ + enum task_dumpable dumpable; int cpu; loff_t written; loff_t pos; diff --git a/include/linux/coresight.h b/include/linux/coresight.h index 2b48be97fcd0..ddf18c970e34 100644 --- a/include/linux/coresight.h +++ b/include/linux/coresight.h @@ -141,6 +141,8 @@ struct csdev_access { .base = (_addr), \ }) +#define CORESIGHT_DESC_CPU_BOUND BIT(0) + /** * struct coresight_desc - description of a component required from drivers * @type: as defined by @coresight_dev_type. @@ -153,6 +155,8 @@ struct csdev_access { * in the component's sysfs sub-directory. * @name: name for the coresight device, also shown under sysfs. * @access: Describe access to the device + * @flags: The descritpion flags. + * @cpu: The CPU this component is affined to. */ struct coresight_desc { enum coresight_dev_type type; @@ -163,6 +167,8 @@ struct coresight_desc { const struct attribute_group **groups; const char *name; struct csdev_access access; + u32 flags; + int cpu; }; /** @@ -251,6 +257,7 @@ struct coresight_trace_id_map { * by @coresight_ops. * @access: Device i/o access abstraction for this device. * @dev: The device entity associated to this component. + * @path: Activated path pointer (only used for source device). * @mode: The device mode, i.e sysFS, Perf or disabled. This is actually * an 'enum cs_mode' but stored in an atomic type. Access is always * through atomic APIs, ensuring SMP-safe synchronisation between @@ -260,6 +267,7 @@ struct coresight_trace_id_map { * device's spinlock when the coresight_mutex held and mode == * CS_MODE_SYSFS. Otherwise it must be accessed from inside the * spinlock. + * @cpu: The CPU this component is affined to (-1 for not CPU bound). * @orphan: true if the component has connections that haven't been linked. * @sysfs_sink_activated: 'true' when a sink has been selected for use via sysfs * by writing a 1 to the 'enable_sink' file. A sink can be @@ -284,8 +292,10 @@ struct coresight_device { const struct coresight_ops *ops; struct csdev_access access; struct device dev; + struct coresight_path *path; atomic_t mode; int refcnt; + int cpu; bool orphan; /* sink specific fields */ bool sysfs_sink_activated; @@ -306,24 +316,19 @@ struct coresight_device { * coresight_dev_list - Mapping for devices to "name" index for device * names. * + * @node: Node on the global device index list. * @nr_idx: Number of entries already allocated. * @pfx: Prefix pattern for device name. * @fwnode_list: Array of fwnode_handles associated with each allocated * index, upto nr_idx entries. */ struct coresight_dev_list { + struct list_head node; int nr_idx; - const char *pfx; + char *pfx; struct fwnode_handle **fwnode_list; }; -#define DEFINE_CORESIGHT_DEVLIST(var, dev_pfx) \ -static struct coresight_dev_list (var) = { \ - .pfx = dev_pfx, \ - .nr_idx = 0, \ - .fwnode_list = NULL, \ -} - #define to_coresight_device(d) container_of(d, struct coresight_device, dev) /** @@ -339,9 +344,9 @@ struct coresight_path { }; enum cs_mode { - CS_MODE_DISABLED, - CS_MODE_SYSFS, - CS_MODE_PERF, + CS_MODE_DISABLED = 0, + CS_MODE_SYSFS = BIT(0), + CS_MODE_PERF = BIT(1), }; #define coresight_ops(csdev) csdev->ops @@ -392,15 +397,12 @@ struct coresight_ops_link { /** * struct coresight_ops_source - basic operations for a source * Operations available for sources. - * @cpu_id: returns the value of the CPU number this component - * is associated to. * @enable: enables tracing for a source. * @disable: disables tracing for a source. * @resume_perf: resumes tracing for a source in perf session. * @pause_perf: pauses tracing for a source in perf session. */ struct coresight_ops_source { - int (*cpu_id)(struct coresight_device *csdev); int (*enable)(struct coresight_device *csdev, struct perf_event *event, enum cs_mode mode, struct coresight_path *path); void (*disable)(struct coresight_device *csdev, @@ -438,6 +440,8 @@ struct coresight_ops_panic { struct coresight_ops { int (*trace_id)(struct coresight_device *csdev, enum cs_mode mode, struct coresight_device *sink); + int (*pm_save_disable)(struct coresight_device *csdev); + void (*pm_restore_enable)(struct coresight_device *csdev); const struct coresight_ops_sink *sink_ops; const struct coresight_ops_link *link_ops; const struct coresight_ops_source *source_ops; @@ -607,6 +611,12 @@ static inline bool coresight_is_percpu_source(struct coresight_device *csdev) (csdev->subtype.source_subtype == CORESIGHT_DEV_SUBTYPE_SOURCE_PROC); } +static inline bool coresight_is_software_source(struct coresight_device *csdev) +{ + return csdev && coresight_is_device_source(csdev) && + (csdev->subtype.source_subtype == CORESIGHT_DEV_SUBTYPE_SOURCE_SOFTWARE); +} + static inline bool coresight_is_percpu_sink(struct coresight_device *csdev) { return csdev && (csdev->type == CORESIGHT_DEV_TYPE_SINK) && @@ -663,8 +673,7 @@ void coresight_clear_self_claim_tag(struct csdev_access *csa); void coresight_clear_self_claim_tag_unlocked(struct csdev_access *csa); void coresight_disclaim_device(struct coresight_device *csdev); void coresight_disclaim_device_unlocked(struct coresight_device *csdev); -char *coresight_alloc_device_name(struct coresight_dev_list *devs, - struct device *dev); +char *coresight_alloc_device_name(const char *prefix, struct device *dev); bool coresight_loses_context_with_cpu(struct device *dev); @@ -697,8 +706,11 @@ coresight_find_output_type(struct coresight_platform_data *pdata, enum coresight_dev_type type, union coresight_dev_subtype subtype); -int coresight_init_driver(const char *drv, struct amba_driver *amba_drv, - struct platform_driver *pdev_drv, struct module *owner); +int coresight_init_driver_with_owner(const char *drv, struct amba_driver *amba_drv, + struct platform_driver *pdev_drv, struct module *owner, + const char *mod_name); +#define coresight_init_driver(drv, amba_drv, pdev_drv) \ + coresight_init_driver_with_owner(drv, amba_drv, pdev_drv, THIS_MODULE, KBUILD_MODNAME) void coresight_remove_driver(struct amba_driver *amba_drv, struct platform_driver *pdev_drv); diff --git a/include/linux/count_zeros.h b/include/linux/count_zeros.h index 5b8ff5ac660d..b72ba3faa036 100644 --- a/include/linux/count_zeros.h +++ b/include/linux/count_zeros.h @@ -18,7 +18,7 @@ * * If the MSB of @x is set, the result is 0. * If only the LSB of @x is set, then the result is BITS_PER_LONG-1. - * If @x is 0 then the result is COUNT_LEADING_ZEROS_0. + * If @x is 0 then the result is BITS_PER_LONG. */ static inline int count_leading_zeros(unsigned long x) { @@ -28,8 +28,6 @@ static inline int count_leading_zeros(unsigned long x) return BITS_PER_LONG - fls64(x); } -#define COUNT_LEADING_ZEROS_0 BITS_PER_LONG - /** * count_trailing_zeros - Count the number of zeros from the LSB forwards * @x: The value @@ -38,16 +36,11 @@ static inline int count_leading_zeros(unsigned long x) * * If the LSB of @x is set, the result is 0. * If only the MSB of @x is set, then the result is BITS_PER_LONG-1. - * If @x is 0 then the result is COUNT_TRAILING_ZEROS_0. + * If @x is 0 then the result is BITS_PER_LONG. */ static inline int count_trailing_zeros(unsigned long x) { -#define COUNT_TRAILING_ZEROS_0 (-1) - - if (sizeof(x) == 4) - return ffs(x); - else - return (x != 0) ? __ffs(x) : COUNT_TRAILING_ZEROS_0; + return x ? __ffs(x) : BITS_PER_LONG; } #endif /* _LINUX_BITOPS_COUNT_ZEROS_H_ */ diff --git a/include/linux/cpu.h b/include/linux/cpu.h index 8239cd95a005..9b6b0d87fdb0 100644 --- a/include/linux/cpu.h +++ b/include/linux/cpu.h @@ -229,8 +229,8 @@ static inline bool cpu_attack_vector_mitigated(enum cpu_attack_vectors v) #define smt_mitigations SMT_MITIGATIONS_OFF #endif -int arch_get_indir_br_lp_status(struct task_struct *t, unsigned long __user *status); -int arch_set_indir_br_lp_status(struct task_struct *t, unsigned long status); -int arch_lock_indir_br_lp_status(struct task_struct *t, unsigned long status); +int arch_prctl_get_branch_landing_pad_state(struct task_struct *t, unsigned long __user *state); +int arch_prctl_set_branch_landing_pad_state(struct task_struct *t, unsigned long state); +int arch_prctl_lock_branch_landing_pad_state(struct task_struct *t); #endif /* _LINUX_CPU_H_ */ diff --git a/include/linux/cpufreq.h b/include/linux/cpufreq.h index cc894fc38971..35ce665edfd8 100644 --- a/include/linux/cpufreq.h +++ b/include/linux/cpufreq.h @@ -79,8 +79,9 @@ struct cpufreq_policy { * called, but you're in IRQ context */ struct freq_constraints constraints; - struct freq_qos_request *min_freq_req; - struct freq_qos_request *max_freq_req; + struct freq_qos_request min_freq_req; + struct freq_qos_request max_freq_req; + struct freq_qos_request boost_freq_req; struct cpufreq_frequency_table *freq_table; enum cpufreq_table_sorting freq_table_sorted; @@ -145,6 +146,9 @@ struct cpufreq_policy { /* Per policy boost supported flag. */ bool boost_supported; + /* Pending policy->min/max update for the driver */ + bool update_limits; + /* Cached frequency lookup from cpufreq_driver_resolve_freq. */ unsigned int cached_target_freq; unsigned int cached_resolved_idx; @@ -232,7 +236,7 @@ static inline bool policy_is_inactive(struct cpufreq_policy *policy) static inline bool policy_is_shared(struct cpufreq_policy *policy) { - return cpumask_weight(policy->cpus) > 1; + return cpumask_nth(1, policy->cpus) < nr_cpumask_bits; } #ifdef CONFIG_CPU_FREQ @@ -372,9 +376,10 @@ struct cpufreq_driver { * conditions) scale invariance can be disabled, which causes the * schedutil governor to fall back to the latter. */ - void (*adjust_perf)(unsigned int cpu, + void (*adjust_perf)(struct cpufreq_policy *policy, unsigned long min_perf, unsigned long target_perf, + unsigned long max_perf, unsigned long capacity); /* @@ -433,7 +438,7 @@ struct cpufreq_driver { /* * Set by drivers that need to update internal upper and lower boundaries along * with the target frequency and so the core and governors should also invoke - * the diver if the target frequency does not change, but the policy min or max + * the driver if the target frequency does not change, but the policy min or max * may have changed. */ #define CPUFREQ_NEED_UPDATE_LIMITS BIT(0) @@ -617,9 +622,10 @@ struct cpufreq_governor { /* Pass a target to the cpufreq driver */ unsigned int cpufreq_driver_fast_switch(struct cpufreq_policy *policy, unsigned int target_freq); -void cpufreq_driver_adjust_perf(unsigned int cpu, +void cpufreq_driver_adjust_perf(struct cpufreq_policy *policy, unsigned long min_perf, unsigned long target_perf, + unsigned long max_perf, unsigned long capacity); bool cpufreq_driver_has_adjust_perf(void); int cpufreq_driver_target(struct cpufreq_policy *policy, diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h index 62cd7b35a29c..feb32949aeea 100644 --- a/include/linux/cpuhotplug.h +++ b/include/linux/cpuhotplug.h @@ -92,7 +92,6 @@ enum cpuhp_state { CPUHP_NET_DEV_DEAD, CPUHP_IOMMU_IOVA_DEAD, CPUHP_AP_ARM_CACHE_B15_RAC_DEAD, - CPUHP_PADATA_DEAD, CPUHP_AP_DTPM_CPU_DEAD, CPUHP_RANDOM_PREPARE, CPUHP_WORKQUEUE_PREP, @@ -181,8 +180,6 @@ enum cpuhp_state { CPUHP_AP_DUMMY_TIMER_STARTING, CPUHP_AP_ARM_XEN_STARTING, CPUHP_AP_ARM_XEN_RUNSTATE_STARTING, - CPUHP_AP_ARM_CORESIGHT_STARTING, - CPUHP_AP_ARM_CORESIGHT_CTI_STARTING, CPUHP_AP_ARM64_ISNDEP_STARTING, CPUHP_AP_SMPCFD_DYING, CPUHP_AP_HRTIMERS_DYING, @@ -201,6 +198,7 @@ enum cpuhp_state { CPUHP_AP_IRQ_AFFINITY_ONLINE, CPUHP_AP_BLK_MQ_ONLINE, CPUHP_AP_ARM_MVEBU_SYNC_CLOCKS, + CPUHP_AP_ARM_CORESIGHT_ONLINE, CPUHP_AP_X86_INTEL_EPB_ONLINE, CPUHP_AP_PERF_ONLINE, CPUHP_AP_PERF_X86_ONLINE, diff --git a/include/linux/cpuhplock.h b/include/linux/cpuhplock.h index 286b3ab92e15..42f6a095ba5b 100644 --- a/include/linux/cpuhplock.h +++ b/include/linux/cpuhplock.h @@ -12,9 +12,6 @@ struct device; -extern int lockdep_is_cpus_held(void); -extern int lockdep_is_cpus_write_held(void); - #ifdef CONFIG_HOTPLUG_CPU void cpus_write_lock(void); void cpus_write_unlock(void); @@ -22,6 +19,8 @@ void cpus_read_lock(void); void cpus_read_unlock(void); int cpus_read_trylock(void); void lockdep_assert_cpus_held(void); +int lockdep_is_cpus_held(void); +int lockdep_is_cpus_write_held(void); void cpu_hotplug_disable_offlining(void); void cpu_hotplug_disable(void); void cpu_hotplug_enable(void); @@ -38,6 +37,8 @@ static inline void cpus_read_lock(void) { } static inline void cpus_read_unlock(void) { } static inline int cpus_read_trylock(void) { return true; } static inline void lockdep_assert_cpus_held(void) { } +static inline int lockdep_is_cpus_held(void) { return 1; } +static inline int lockdep_is_cpus_write_held(void) { return 1; } static inline void cpu_hotplug_disable_offlining(void) { } static inline void cpu_hotplug_disable(void) { } static inline void cpu_hotplug_enable(void) { } diff --git a/include/linux/cpuidle.h b/include/linux/cpuidle.h index 4073690504a7..a2485348def3 100644 --- a/include/linux/cpuidle.h +++ b/include/linux/cpuidle.h @@ -188,6 +188,7 @@ extern void cpuidle_driver_state_disabled(struct cpuidle_driver *drv, int idx, extern void cpuidle_unregister_driver(struct cpuidle_driver *drv); extern int cpuidle_register_device(struct cpuidle_device *dev); extern void cpuidle_unregister_device(struct cpuidle_device *dev); +extern void cpuidle_unregister_device_no_lock(struct cpuidle_device *dev); extern int cpuidle_register(struct cpuidle_driver *drv, const struct cpumask *const coupled_cpus); extern void cpuidle_unregister(struct cpuidle_driver *drv); @@ -226,6 +227,7 @@ static inline void cpuidle_unregister_driver(struct cpuidle_driver *drv) { } static inline int cpuidle_register_device(struct cpuidle_device *dev) {return -ENODEV; } static inline void cpuidle_unregister_device(struct cpuidle_device *dev) { } +static inline void cpuidle_unregister_device_no_lock(struct cpuidle_device *dev) {} static inline int cpuidle_register(struct cpuidle_driver *drv, const struct cpumask *const coupled_cpus) {return -ENODEV; } diff --git a/include/linux/cpumask.h b/include/linux/cpumask.h index 80211900f373..4c8bb6953107 100644 --- a/include/linux/cpumask.h +++ b/include/linux/cpumask.h @@ -15,6 +15,7 @@ #include <linux/numa.h> #include <linux/threads.h> #include <linux/types.h> +#include <vdso/page.h> #include <asm/bug.h> @@ -1314,23 +1315,6 @@ static __always_inline bool cpu_dying(unsigned int cpu) #endif /* NR_CPUS > BITS_PER_LONG */ /** - * cpumap_print_to_pagebuf - copies the cpumask into the buffer either - * as comma-separated list of cpus or hex values of cpumask - * @list: indicates whether the cpumap must be list - * @mask: the cpumask to copy - * @buf: the buffer to copy into - * - * Return: the length of the (null-terminated) @buf string, zero if - * nothing is copied. - */ -static __always_inline ssize_t -cpumap_print_to_pagebuf(bool list, char *buf, const struct cpumask *mask) -{ - return bitmap_print_to_pagebuf(list, buf, cpumask_bits(mask), - nr_cpu_ids); -} - -/** * cpumap_print_bitmask_to_buf - copies the cpumask into the buffer as * hex values of cpumask * diff --git a/include/linux/cpuset.h b/include/linux/cpuset.h index 65d76a38974b..914983387e95 100644 --- a/include/linux/cpuset.h +++ b/include/linux/cpuset.h @@ -80,10 +80,11 @@ extern void lockdep_assert_cpuset_lock_held(void); extern void cpuset_cpus_allowed_locked(struct task_struct *p, struct cpumask *mask); extern void cpuset_cpus_allowed(struct task_struct *p, struct cpumask *mask); extern bool cpuset_cpus_allowed_fallback(struct task_struct *p); +extern int cpuset_num_cpus(struct cgroup *cgroup); extern nodemask_t cpuset_mems_allowed(struct task_struct *p); #define cpuset_current_mems_allowed (current->mems_allowed) void cpuset_init_current_mems_allowed(void); -int cpuset_nodemask_valid_mems_allowed(nodemask_t *nodemask); +int cpuset_nodemask_valid_mems_allowed(const nodemask_t *nodemask); extern bool cpuset_current_node_allowed(int node, gfp_t gfp_mask); @@ -216,6 +217,11 @@ static inline bool cpuset_cpus_allowed_fallback(struct task_struct *p) return false; } +static inline int cpuset_num_cpus(struct cgroup *cgroup) +{ + return num_online_cpus(); +} + static inline nodemask_t cpuset_mems_allowed(struct task_struct *p) { return node_possible_map; @@ -224,7 +230,7 @@ static inline nodemask_t cpuset_mems_allowed(struct task_struct *p) #define cpuset_current_mems_allowed (node_states[N_MEMORY]) static inline void cpuset_init_current_mems_allowed(void) {} -static inline int cpuset_nodemask_valid_mems_allowed(nodemask_t *nodemask) +static inline int cpuset_nodemask_valid_mems_allowed(const nodemask_t *nodemask) { return 1; } @@ -273,6 +279,7 @@ static inline void dl_rebuild_rd_accounting(void) static inline void rebuild_sched_domains(void) { + guard(cpus_read_lock)(); partition_sched_domains(1, NULL, NULL); } diff --git a/include/linux/crash_core.h b/include/linux/crash_core.h index d35726d6a415..bc087124cd78 100644 --- a/include/linux/crash_core.h +++ b/include/linux/crash_core.h @@ -34,13 +34,6 @@ static inline void arch_kexec_protect_crashkres(void) { } static inline void arch_kexec_unprotect_crashkres(void) { } #endif -#ifdef CONFIG_CRASH_DM_CRYPT -int crash_load_dm_crypt_keys(struct kimage *image); -ssize_t dm_crypt_keys_read(char *buf, size_t count, u64 *ppos); -#else -static inline int crash_load_dm_crypt_keys(struct kimage *image) {return 0; } -#endif - #ifndef arch_crash_handle_hotplug_event static inline void arch_crash_handle_hotplug_event(struct kimage *image, void *arg) { } #endif @@ -66,6 +59,9 @@ extern int crash_exclude_mem_range(struct crash_mem *mem, unsigned long long mend); extern int crash_prepare_elf64_headers(struct crash_mem *mem, int need_kernel_map, void **addr, unsigned long *sz); +extern int crash_prepare_headers(int need_kernel_map, void **addr, + unsigned long *sz, unsigned long *nr_mem_ranges); +extern int crash_exclude_core_ranges(struct crash_mem **cmem); struct kimage; struct kexec_segment; @@ -83,6 +79,12 @@ int kexec_should_crash(struct task_struct *p); int kexec_crash_loaded(void); void crash_save_cpu(struct pt_regs *regs, int cpu); extern int kimage_crash_copy_vmcoreinfo(struct kimage *image); +extern unsigned int arch_get_system_nr_ranges(void); +extern int arch_crash_populate_cmem(struct crash_mem *cmem); +extern int arch_crash_exclude_ranges(struct crash_mem *cmem); +extern int arch_crash_exclude_mem_range(struct crash_mem **mem, + unsigned long long mstart, + unsigned long long mend); #else /* !CONFIG_CRASH_DUMP*/ struct pt_regs; @@ -96,4 +98,11 @@ static inline void crash_save_cpu(struct pt_regs *regs, int cpu) {}; static inline int kimage_crash_copy_vmcoreinfo(struct kimage *image) { return 0; }; #endif /* CONFIG_CRASH_DUMP*/ +#ifdef CONFIG_CRASH_DM_CRYPT +int crash_load_dm_crypt_keys(struct kimage *image); +ssize_t dm_crypt_keys_read(char *buf, size_t count, u64 *ppos); +#else +static inline int crash_load_dm_crypt_keys(struct kimage *image) {return 0; } +#endif + #endif /* LINUX_CRASH_CORE_H */ diff --git a/include/linux/crash_reserve.h b/include/linux/crash_reserve.h index f0dc03d94ca2..30864d90d7f5 100644 --- a/include/linux/crash_reserve.h +++ b/include/linux/crash_reserve.h @@ -14,9 +14,11 @@ extern struct resource crashk_res; extern struct resource crashk_low_res; extern struct range crashk_cma_ranges[]; + +#define CRASHK_CMA_RANGES_MAX 4 #if defined(CONFIG_CMA) && defined(CONFIG_ARCH_HAS_GENERIC_CRASHKERNEL_RESERVATION) #define CRASHKERNEL_CMA -#define CRASHKERNEL_CMA_RANGES_MAX 4 +#define CRASHKERNEL_CMA_RANGES_MAX (CRASHK_CMA_RANGES_MAX) extern int crashk_cma_cnt; #else #define crashk_cma_cnt 0 diff --git a/include/linux/cred.h b/include/linux/cred.h index ed1609d78cd7..6ef1750c93e2 100644 --- a/include/linux/cred.h +++ b/include/linux/cred.h @@ -33,12 +33,14 @@ struct group_info { /** * get_group_info - Get a reference to a group info structure - * @group_info: The group info to reference + * @gi: The group info to reference * * This gets a reference to a set of supplementary groups. * * If the caller is accessing a task's credentials, they must hold the RCU read * lock when reading. + * + * Returns: @gi */ static inline struct group_info *get_group_info(struct group_info *gi) { @@ -209,6 +211,8 @@ DEFINE_CLASS(override_creds, * usage count. The purpose of this is to attempt to catch at compile time the * accidental alteration of a set of credentials that should be considered * immutable. + * + * Returns: @cred when the references are acquired, NULL otherwise. */ static inline const struct cred *get_cred_many(const struct cred *cred, int nr) { @@ -246,8 +250,8 @@ static inline const struct cred *get_cred_rcu(const struct cred *cred) } /** - * put_cred - Release a reference to a set of credentials - * @cred: The credentials to release + * put_cred_many - Release a reference to a set of credentials + * @_cred: The credentials to release * @nr: Number of references to release * * Release a reference to a set of credentials, deleting them when the last ref @@ -367,7 +371,6 @@ DEFINE_FREE(put_cred, struct cred *, if (!IS_ERR_OR_NULL(_T)) put_cred(_T)) }) #define task_uid(task) (task_cred_xxx((task), uid)) -#define task_euid(task) (task_cred_xxx((task), euid)) #define task_ucounts(task) (task_cred_xxx((task), ucounts)) #define current_cred_xxx(xxx) \ diff --git a/include/linux/crypto.h b/include/linux/crypto.h index a2137e19be7d..b7c97f1c47c9 100644 --- a/include/linux/crypto.h +++ b/include/linux/crypto.h @@ -409,7 +409,6 @@ int crypto_has_alg(const char *name, u32 type, u32 mask); */ struct crypto_tfm { - refcount_t refcnt; u32 crt_flags; diff --git a/include/linux/damon.h b/include/linux/damon.h index a4fea23da857..0c8b7ddef9ab 100644 --- a/include/linux/damon.h +++ b/include/linux/damon.h @@ -1,30 +1,25 @@ /* SPDX-License-Identifier: GPL-2.0 */ /* * DAMON api - * - * Author: SeongJae Park <sj@kernel.org> */ #ifndef _DAMON_H_ #define _DAMON_H_ +#include <linux/math64.h> #include <linux/memcontrol.h> #include <linux/mutex.h> +#include <linux/prandom.h> #include <linux/time64.h> #include <linux/types.h> -#include <linux/random.h> /* Minimal region size. Every damon_region is aligned by this. */ #define DAMON_MIN_REGION_SZ PAGE_SIZE +/* Maximum number of monitoring probes. */ +#define DAMON_MAX_PROBES (4) /* Max priority score for DAMON-based operation schemes */ #define DAMOS_MAX_SCORE (99) -/* Get a random number in [l, r) */ -static inline unsigned long damon_rand(unsigned long l, unsigned long r) -{ - return l + get_random_u32_below(r - l); -} - /** * struct damon_addr_range - Represents an address region of [@start, @end). * @start: Start address of the region (inclusive). @@ -50,24 +45,18 @@ struct damon_size_range { * @ar: The address range of the region. * @sampling_addr: Address of the sample for the next access check. * @nr_accesses: Access frequency of this region. - * @nr_accesses_bp: @nr_accesses in basis point (0.01%) that updated for - * each sampling interval. - * @list: List head for siblings. + * @probe_hits: Number of probe-positive region samples. * @age: Age of this region. * + * For any use case, @ar should be non-zero positive size. damon_set_regions() + * does the validation. + * * @nr_accesses is reset to zero for every &damon_attrs->aggr_interval and be * increased for every &damon_attrs->sample_interval if an access to the region * during the last sampling interval is found. The update of this field should * not be done with direct access but with the helper function, * damon_update_region_access_rate(). * - * @nr_accesses_bp is another representation of @nr_accesses in basis point - * (1 in 10,000) that updated for every &damon_attrs->sample_interval in a - * manner similar to moving sum. By the algorithm, this value becomes - * @nr_accesses * 10000 for every &struct damon_attrs->aggr_interval. This can - * be used when the aggregation interval is too huge and therefore cannot wait - * for it before getting the access monitoring results. - * * @age is initially zero, increased for each aggregation interval, and reset * to zero again if the access frequency is significantly changed. If two * regions are merged into a new region, both @nr_accesses and @age of the new @@ -77,20 +66,19 @@ struct damon_region { struct damon_addr_range ar; unsigned long sampling_addr; unsigned int nr_accesses; - unsigned int nr_accesses_bp; - struct list_head list; - + unsigned char probe_hits[DAMON_MAX_PROBES]; unsigned int age; -/* private: Internal value for age calculation. */ +/* private: internal use only. */ + /* List head for siblings. */ + struct list_head list; + /* for age calculation. */ unsigned int last_nr_accesses; + unsigned char last_probe_hits[DAMON_MAX_PROBES]; }; /** * struct damon_target - Represents a monitoring target. * @pid: The PID of the virtual address space to monitor. - * @nr_regions: Number of monitoring target regions of this target. - * @regions_list: Head of the monitoring target regions of this target. - * @list: List head for siblings. * @obsolete: Whether the commit destination target is obsolete. * * Each monitoring context could have multiple targets. For example, a context @@ -104,10 +92,14 @@ struct damon_region { */ struct damon_target { struct pid *pid; + bool obsolete; +/* private: */ + /* Number of monitoring target regions of this target. */ unsigned int nr_regions; + /* Head of the monitoring target regions of this target. */ struct list_head regions_list; + /* List head for siblings. */ struct list_head list; - bool obsolete; }; /** @@ -119,6 +111,7 @@ struct damon_target { * @DAMOS_PAGEOUT: Reclaim the region. * @DAMOS_HUGEPAGE: Call ``madvise()`` for the region with MADV_HUGEPAGE. * @DAMOS_NOHUGEPAGE: Call ``madvise()`` for the region with MADV_NOHUGEPAGE. + * @DAMOS_COLLAPSE: Call ``madvise()`` for the region with MADV_COLLAPSE. * @DAMOS_LRU_PRIO: Prioritize the region on its LRU lists. * @DAMOS_LRU_DEPRIO: Deprioritize the region on its LRU lists. * @DAMOS_MIGRATE_HOT: Migrate the regions prioritizing warmer regions. @@ -138,6 +131,7 @@ enum damos_action { DAMOS_PAGEOUT, DAMOS_HUGEPAGE, DAMOS_NOHUGEPAGE, + DAMOS_COLLAPSE, DAMOS_LRU_PRIO, DAMOS_LRU_DEPRIO, DAMOS_MIGRATE_HOT, @@ -157,6 +151,8 @@ enum damos_action { * @DAMOS_QUOTA_NODE_MEMCG_FREE_BP: MemFree ratio of a node for a cgroup. * @DAMOS_QUOTA_ACTIVE_MEM_BP: Active to total LRU memory ratio. * @DAMOS_QUOTA_INACTIVE_MEM_BP: Inactive to total LRU memory ratio. + * @DAMOS_QUOTA_NODE_ELIGIBLE_MEM_BP: Scheme-eligible memory ratio of a + * node in basis points (0-10000). * @NR_DAMOS_QUOTA_GOAL_METRICS: Number of DAMOS quota goal metrics. * * Metrics equal to larger than @NR_DAMOS_QUOTA_GOAL_METRICS are unsupported. @@ -170,6 +166,7 @@ enum damos_quota_goal_metric { DAMOS_QUOTA_NODE_MEMCG_FREE_BP, DAMOS_QUOTA_ACTIVE_MEM_BP, DAMOS_QUOTA_INACTIVE_MEM_BP, + DAMOS_QUOTA_NODE_ELIGIBLE_MEM_BP, NR_DAMOS_QUOTA_GOAL_METRICS, }; @@ -178,10 +175,8 @@ enum damos_quota_goal_metric { * @metric: Metric to be used for representing the goal. * @target_value: Target value of @metric to achieve with the tuning. * @current_value: Current value of @metric. - * @last_psi_total: Last measured total PSI * @nid: Node id. * @memcg_id: Memcg id. - * @list: List head for siblings. * * Data structure for getting the current score of the quota tuning goal. The * score is calculated by how close @current_value and @target_value are. Then @@ -204,22 +199,39 @@ struct damos_quota_goal { unsigned long current_value; /* metric-dependent fields */ union { - u64 last_psi_total; struct { int nid; u64 memcg_id; }; +/* private: */ + /* Last measured total PSI */ + u64 last_psi_total; + }; +/* private: */ + /* List head for siblings. */ struct list_head list; }; /** + * enum damos_quota_goal_tuner - Goal-based quota tuning logic. + * @DAMOS_QUOTA_GOAL_TUNER_CONSIST: Aim long term consistent quota. + * @DAMOS_QUOTA_GOAL_TUNER_TEMPORAL: Aim zero quota asap. + */ +enum damos_quota_goal_tuner { + DAMOS_QUOTA_GOAL_TUNER_CONSIST, + DAMOS_QUOTA_GOAL_TUNER_TEMPORAL, +}; + +/** * struct damos_quota - Controls the aggressiveness of the given scheme. * @reset_interval: Charge reset interval in milliseconds. * @ms: Maximum milliseconds that the scheme can use. * @sz: Maximum bytes of memory that the action can be applied. - * @goals: Head of quota tuning goals (&damos_quota_goal) list. + * @goal_tuner: Goal-based @esz tuning algorithm to use. * @esz: Effective size quota in bytes. + * @fail_charge_num: Failed regions charge rate numerator. + * @fail_charge_denom: Failed regions charge rate denominator. * * @weight_sz: Weight of the region's size for prioritization. * @weight_nr_accesses: Weight of the region's nr_accesses for prioritization. @@ -242,13 +254,17 @@ struct damos_quota_goal { * the scheme's action. DAMON then compares it against &sz and uses smaller * one as the effective quota. * - * If @goals is not empty, DAMON calculates yet another size quota based on the + * If goals is not empty, DAMON calculates yet another size quota based on the * goals using its internal feedback loop algorithm, for every @reset_interval. * Then, if the new size quota is smaller than the effective quota, it uses the * new size quota as the effective quota. * * The resulting effective size quota in bytes is set to @esz. * + * For DAMOS action applying failed amount of regions, charging those same to + * those that the action has successfully applied may be unfair. For the + * reason, 'the size * @fail_charge_num / @fail_charge_denom' is charged. + * * For selecting regions within the quota, DAMON prioritizes current scheme's * target memory regions using the &struct damon_operations->get_scheme_score. * You could customize the prioritization logic by setting &weight_sz, @@ -259,14 +275,20 @@ struct damos_quota { unsigned long reset_interval; unsigned long ms; unsigned long sz; - struct list_head goals; + + enum damos_quota_goal_tuner goal_tuner; unsigned long esz; + unsigned int fail_charge_num; + unsigned int fail_charge_denom; + unsigned int weight_sz; unsigned int weight_nr_accesses; unsigned int weight_age; /* private: */ + /* Head of quota tuning goals (&damos_quota_goal) list. */ + struct list_head goals; /* For throughput estimation */ unsigned long total_charged_sz; unsigned long total_charged_ns; @@ -376,16 +398,15 @@ struct damos_stat { * @DAMOS_FILTER_TYPE_TARGET: Data Access Monitoring target. * @NR_DAMOS_FILTER_TYPES: Number of filter types. * - * The anon pages type and memcg type filters are handled by underlying - * &struct damon_operations as a part of scheme action trying, and therefore - * accounted as 'tried'. In contrast, other types are handled by core layer - * before trying of the action and therefore not accounted as 'tried'. + * All types except &DAMOS_FILTER_TYPE_ADDR and &DAMOS_FILTER_TYPE_TARGET + * are handled by the underlying &struct damon_operations as a part of scheme + * action trying, and therefore accounted as 'tried'. In contrast, + * &DAMOS_FILTER_TYPE_ADDR and &DAMOS_FILTER_TYPE_TARGET filters are handled + * by the core layer before trying of the action, and therefore not accounted + * as 'tried'. * - * The support of the filters that handled by &struct damon_operations depend - * on the running &struct damon_operations. - * &enum DAMON_OPS_PADDR supports both anon pages type and memcg type filters, - * while &enum DAMON_OPS_VADDR and &enum DAMON_OPS_FVADDR don't support any of - * the two types. + * Support for the operations-handled filters depends on the running + * &struct damon_operations. */ enum damos_filter_type { DAMOS_FILTER_TYPE_ANON, @@ -410,7 +431,6 @@ enum damos_filter_type { * &damon_ctx->adaptive_targets if @type is * DAMOS_FILTER_TYPE_TARGET. * @sz_range: Size range if @type is DAMOS_FILTER_TYPE_HUGEPAGE_SIZE. - * @list: List head for siblings. * * Before applying the &damos->action to a memory region, DAMOS checks if each * byte of the region matches to this given condition and avoid applying the @@ -428,6 +448,8 @@ struct damos_filter { int target_idx; struct damon_size_range sz_range; }; +/* private: */ + /* List head for siblings. */ struct list_head list; }; @@ -499,12 +521,8 @@ struct damos_migrate_dests { * @wmarks: Watermarks for automated (in)activation of this scheme. * @migrate_dests: Destination nodes if @action is "migrate_{hot,cold}". * @target_nid: Destination node if @action is "migrate_{hot,cold}". - * @core_filters: Additional set of &struct damos_filter for &action. - * @ops_filters: ops layer handling &struct damos_filter objects list. - * @last_applied: Last @action applied ops-managing entity. * @stat: Statistics of this scheme. * @max_nr_snapshots: Upper limit of nr_snapshots stat. - * @list: List head for siblings. * * For each @apply_interval_us, DAMON finds regions which fit in the * &pattern and applies &action to those. To avoid consuming too much @@ -526,16 +544,7 @@ struct damos_migrate_dests { * * Before applying the &action to a memory region, &struct damon_operations * implementation could check pages of the region and skip &action to respect - * &core_filters - * - * The minimum entity that @action can be applied depends on the underlying - * &struct damon_operations. Since it may not be aligned with the core layer - * abstract, namely &struct damon_region, &struct damon_operations could apply - * @action to same entity multiple times. Large folios that underlying on - * multiple &struct damon region objects could be such examples. The &struct - * damon_operations can use @last_applied to avoid that. DAMOS core logic - * unsets @last_applied when each regions walking for applying the scheme is - * finished. + * &struct damos_filter. * * After applying the &action to each region, &stat is updated. * @@ -546,6 +555,16 @@ struct damos { struct damos_access_pattern pattern; enum damos_action action; unsigned long apply_interval_us; + struct damos_quota quota; + struct damos_watermarks wmarks; + union { + struct { + int target_nid; + struct damos_migrate_dests migrate_dests; + }; + }; + struct damos_stat stat; + unsigned long max_nr_snapshots; /* private: internal use only */ /* * number of sample intervals that should be passed before applying @@ -562,20 +581,25 @@ struct damos { /* whether to reject core/ops filters umatched regions */ bool core_filters_default_reject; bool ops_filters_default_reject; -/* public: */ - struct damos_quota quota; - struct damos_watermarks wmarks; - union { - struct { - int target_nid; - struct damos_migrate_dests migrate_dests; - }; - }; + /* Additional set of &struct damos_filter for &action. */ struct list_head core_filters; + /* ops layer handling &struct damos_filter objects list. */ struct list_head ops_filters; + /* + * Last @action applied ops-managing entity. + * + * The minimum entity that @action can be applied depends on the + * underlying &struct damon_operations. Since it may not be aligned + * with the core layer abstract, namely &struct damon_region, &struct + * damon_operations could apply @action to same entity multiple times. + * Large folios that underlying on multiple &struct damon region + * objects could be such examples. The &struct damon_operations can + * use @last_applied to avoid that. DAMOS core logic unsets + * @last_applied when each regions walking for applying the scheme is + * finished. + */ void *last_applied; - struct damos_stat stat; - unsigned long max_nr_snapshots; + /* List head for siblings. */ struct list_head list; }; @@ -603,6 +627,7 @@ enum damon_ops_id { * @update: Update operations-related data structures. * @prepare_access_checks: Prepare next access check of target regions. * @check_accesses: Check the accesses to target regions. + * @apply_probes: Apply probes for each region. * @get_scheme_score: Get the score of a region for a scheme. * @apply_scheme: Apply a DAMON-based operation scheme. * @target_valid: Determine if the target is valid. @@ -629,6 +654,11 @@ enum damon_ops_id { * last preparation and update the number of observed accesses of each region. * It should also return max number of observed accesses that made as a result * of its update. The value will be used for regions adjustment threshold. + * @apply_probes should apply the data attribute probes to each region and + * accordingly update the probe hits counter of the region. It should also + * set &damon_region->sampling_addr of each region if ``set_samples`` is true. + * It should also return maximum probe hits weighted sum of regions if + * ``return_max_wsum`` is true. * @get_scheme_score should return the priority score of a region for a scheme * as an integer in [0, &DAMOS_MAX_SCORE]. * @apply_scheme is called from @kdamond when a region for user provided @@ -646,9 +676,10 @@ struct damon_operations { void (*update)(struct damon_ctx *context); void (*prepare_access_checks)(struct damon_ctx *context); unsigned int (*check_accesses)(struct damon_ctx *context); + unsigned int (*apply_probes)(struct damon_ctx *context, + bool set_samples, bool return_max_wsum); int (*get_scheme_score)(struct damon_ctx *context, - struct damon_target *t, struct damon_region *r, - struct damos *scheme); + struct damon_region *r, struct damos *scheme); unsigned long (*apply_scheme)(struct damon_ctx *context, struct damon_target *t, struct damon_region *r, struct damos *scheme, unsigned long *sz_filter_passed); @@ -709,6 +740,51 @@ struct damon_intervals_goal { }; /** + * enum damon_filter_type - Type of &struct damon_filter + * + * @DAMON_FILTER_TYPE_ANON: Anonymous pages. + * @DAMON_FILTER_TYPE_MEMCG: Specific memcg's pages. + */ +enum damon_filter_type { + DAMON_FILTER_TYPE_ANON, + DAMON_FILTER_TYPE_MEMCG, +}; + +/** + * struct damon_filter - DAMON region filter for &struct damon_probe. + * + * @type: Type of the region. + * @matching: Whether this filter is for the type-matching ones. + * @allow: Whether the @type-@matching ones should pass this filter. + * @memcg_id: Memcg id of the question if @type is DAMON_FILTER_MEMCG. + */ +struct damon_filter { + enum damon_filter_type type; + bool matching; + bool allow; + union { + u64 memcg_id; + }; +/* private: */ + /* Siblings list. */ + struct list_head list; +}; + +/** + * struct damon_probe - Data region attribute probe. + * + * @weight: Relative priority of the attribute for this probe. + */ +struct damon_probe { + unsigned int weight; +/* private: */ + /* Filters for assessing if a given region is for this probe. */ + struct list_head filters; + /* Siblings list. */ + struct list_head list; +}; + +/** * struct damon_attrs - Monitoring attributes for accuracy/overhead control. * * @sample_interval: The time between access samplings. @@ -771,11 +847,9 @@ struct damon_attrs { * thread other than the kdamond should be made using safe DAMON APIs, * including damon_call() and damos_walk(). * - * @ops: Set of monitoring operations for given use cases. * @addr_unit: Scale factor for core to ops address conversion. * @min_region_sz: Minimum region size. - * @adaptive_targets: Head of monitoring targets (&damon_target) list. - * @schemes: Head of schemes (&damos) list. + * @pause: Pause kdamond main loop. */ struct damon_ctx { struct damon_attrs attrs; @@ -805,25 +879,59 @@ struct damon_ctx { /* lists of &struct damon_call_control */ struct list_head call_controls; + bool call_controls_obsolete; struct mutex call_controls_lock; struct damos_walk_control *walk_control; + bool walk_control_obsolete; struct mutex walk_control_lock; + /* + * indicate if this may be corrupted. Currentonly this is set only for + * damon_commit_ctx() failure. + */ + bool maybe_corrupted; + /* Working thread of the given DAMON context */ struct task_struct *kdamond; /* Protects @kdamond field access */ struct mutex kdamond_lock; /* public: */ - struct damon_operations ops; unsigned long addr_unit; unsigned long min_region_sz; + bool pause; +/* private: */ + /* Set of monitoring operations for given use cases. */ + struct damon_operations ops; + /* Head of monitoring targets (&damon_target) list. */ struct list_head adaptive_targets; + /* Head of probes (&damon_probe) list. */ + struct list_head probes; + /* Head of schemes (&damos) list. */ struct list_head schemes; + + /* @rnd_state: Per-ctx PRNG state for damon_rand(). */ + struct rnd_state rnd_state; }; +/* Get a random number in [@l, @r) using @ctx's lockless PRNG. */ +static inline unsigned long damon_rand(struct damon_ctx *ctx, + unsigned long l, unsigned long r) +{ + unsigned long span = r - l; + u64 rnd; + + if (span <= U32_MAX) { + rnd = prandom_u32_state(&ctx->rnd_state); + return l + (unsigned long)((rnd * span) >> 32); + } + rnd = ((u64)prandom_u32_state(&ctx->rnd_state) << 32) | + prandom_u32_state(&ctx->rnd_state); + return l + mul_u64_u64_shr(rnd, span, 64); +} + static inline struct damon_region *damon_next_region(struct damon_region *r) { return container_of(r->list.next, struct damon_region, list); @@ -849,15 +957,26 @@ static inline unsigned long damon_sz_region(struct damon_region *r) return r->ar.end - r->ar.start; } +#define damon_for_each_filter(f, p) \ + list_for_each_entry(f, &(p)->filters, list) + +#define damon_for_each_filter_safe(f, next, p) \ + list_for_each_entry_safe(f, next, &(p)->filters, list) + +#define damon_for_each_probe(p, ctx) \ + list_for_each_entry(p, &(ctx)->probes, list) + +#define damon_for_each_probe_safe(p, next, ctx) \ + list_for_each_entry_safe(p, next, &(ctx)->probes, list) #define damon_for_each_region(r, t) \ - list_for_each_entry(r, &t->regions_list, list) + list_for_each_entry(r, &(t)->regions_list, list) #define damon_for_each_region_from(r, t) \ - list_for_each_entry_from(r, &t->regions_list, list) + list_for_each_entry_from(r, &(t)->regions_list, list) #define damon_for_each_region_safe(r, next, t) \ - list_for_each_entry_safe(r, next, &t->regions_list, list) + list_for_each_entry_safe(r, next, &(t)->regions_list, list) #define damon_for_each_target(t, ctx) \ list_for_each_entry(t, &(ctx)->adaptive_targets, list) @@ -872,7 +991,7 @@ static inline unsigned long damon_sz_region(struct damon_region *r) list_for_each_entry_safe(s, next, &(ctx)->schemes, list) #define damos_for_each_quota_goal(goal, quota) \ - list_for_each_entry(goal, "a->goals, list) + list_for_each_entry(goal, &(quota)->goals, list) #define damos_for_each_quota_goal_safe(goal, next, quota) \ list_for_each_entry_safe(goal, next, &(quota)->goals, list) @@ -891,25 +1010,25 @@ static inline unsigned long damon_sz_region(struct damon_region *r) #ifdef CONFIG_DAMON -struct damon_region *damon_new_region(unsigned long start, unsigned long end); +struct damon_filter *damon_new_filter(enum damon_filter_type type, + bool matching, bool allow); +void damon_add_filter(struct damon_probe *probe, struct damon_filter *f); +void damon_destroy_filter(struct damon_filter *f); -/* - * Add a region between two other regions - */ -static inline void damon_insert_region(struct damon_region *r, - struct damon_region *prev, struct damon_region *next, - struct damon_target *t) -{ - __list_add(&r->list, &prev->list, &next->list); - t->nr_regions++; -} +struct damon_probe *damon_new_probe(void); +void damon_add_probe(struct damon_ctx *ctx, struct damon_probe *probe); + +struct damon_region *damon_new_region(unsigned long start, unsigned long end); +unsigned int damon_nr_accesses_mvsum(struct damon_region *r, + struct damon_ctx *ctx); +unsigned char damon_probe_hits_mvsum(int probe_idx, struct damon_region *r, + struct damon_ctx *ctx); +unsigned int damon_probe_hits_wsum(struct damon_region *r, bool last, + struct damon_ctx *ctx); -void damon_add_region(struct damon_region *r, struct damon_target *t); -void damon_destroy_region(struct damon_region *r, struct damon_target *t); int damon_set_regions(struct damon_target *t, struct damon_addr_range *ranges, unsigned int nr_ranges, unsigned long min_region_sz); -void damon_update_region_access_rate(struct damon_region *r, bool accessed, - struct damon_attrs *attrs); +void damon_update_region_access_rate(struct damon_region *r, bool accessed); struct damos_filter *damos_new_filter(enum damos_filter_type type, bool matching, bool allow); @@ -956,25 +1075,32 @@ static inline bool damon_target_has_pid(const struct damon_ctx *ctx) return ctx->ops.id == DAMON_OPS_VADDR || ctx->ops.id == DAMON_OPS_FVADDR; } -static inline unsigned int damon_max_nr_accesses(const struct damon_attrs *attrs) +/* Returns number of samples per aggregation interval */ +static inline unsigned int damon_nr_samples_per_aggr( + const struct damon_attrs *attrs) { - /* {aggr,sample}_interval are unsigned long, hence could overflow */ - return min(attrs->aggr_interval / attrs->sample_interval, + unsigned long sample_interval; + unsigned long nr_samples; + + sample_interval = attrs->sample_interval ? : 1; + nr_samples = min(attrs->aggr_interval / sample_interval, (unsigned long)UINT_MAX); + return nr_samples ? : 1; } bool damon_initialized(void); int damon_start(struct damon_ctx **ctxs, int nr_ctxs, bool exclusive); -int damon_stop(struct damon_ctx **ctxs, int nr_ctxs); +void damon_stop(struct damon_ctx **ctxs, int nr_ctxs); bool damon_is_running(struct damon_ctx *ctx); int damon_kdamond_pid(struct damon_ctx *ctx); int damon_call(struct damon_ctx *ctx, struct damon_call_control *control); int damos_walk(struct damon_ctx *ctx, struct damos_walk_control *control); -int damon_set_region_biggest_system_ram_default(struct damon_target *t, +int damon_set_region_system_rams_default(struct damon_target *t, unsigned long *start, unsigned long *end, + unsigned long addr_unit, unsigned long min_region_sz); #endif /* CONFIG_DAMON */ diff --git a/include/linux/dax.h b/include/linux/dax.h index bf103f317cac..fe6c3ded1b50 100644 --- a/include/linux/dax.h +++ b/include/linux/dax.h @@ -54,6 +54,7 @@ struct dax_device *alloc_dax(void *private, const struct dax_operations *ops); void *dax_holder(struct dax_device *dax_dev); void put_dax(struct dax_device *dax_dev); void kill_dax(struct dax_device *dax_dev); +struct dax_device *dax_dev_get(dev_t devt); void dax_write_cache(struct dax_device *dax_dev, bool wc); bool dax_write_cache_enabled(struct dax_device *dax_dev); bool dax_synchronous(struct dax_device *dax_dev); @@ -69,7 +70,7 @@ static inline bool daxdev_mapping_supported(const struct vm_area_desc *desc, const struct inode *inode, struct dax_device *dax_dev) { - if (!vma_desc_test_flags(desc, VMA_SYNC_BIT)) + if (!vma_desc_test(desc, VMA_SYNC_BIT)) return true; if (!IS_DAX(inode)) return false; @@ -115,7 +116,7 @@ static inline bool daxdev_mapping_supported(const struct vm_area_desc *desc, const struct inode *inode, struct dax_device *dax_dev) { - return !vma_desc_test_flags(desc, VMA_SYNC_BIT); + return !vma_desc_test(desc, VMA_SYNC_BIT); } static inline size_t dax_recovery_write(struct dax_device *dax_dev, pgoff_t pgoff, void *addr, size_t bytes, struct iov_iter *i) @@ -130,7 +131,6 @@ int dax_add_host(struct dax_device *dax_dev, struct gendisk *disk); void dax_remove_host(struct gendisk *disk); struct dax_device *fs_dax_get_by_bdev(struct block_device *bdev, u64 *start_off, void *holder, const struct dax_holder_operations *ops); -void fs_put_dax(struct dax_device *dax_dev, void *holder); #else static inline int dax_add_host(struct dax_device *dax_dev, struct gendisk *disk) { @@ -145,14 +145,15 @@ static inline struct dax_device *fs_dax_get_by_bdev(struct block_device *bdev, { return NULL; } -static inline void fs_put_dax(struct dax_device *dax_dev, void *holder) -{ -} #endif /* CONFIG_BLOCK && CONFIG_FS_DAX */ #if IS_ENABLED(CONFIG_FS_DAX) +void fs_put_dax(struct dax_device *dax_dev, void *holder); +int fs_dax_get(struct dax_device *dax_dev, void *holder, + const struct dax_holder_operations *hops); int dax_writeback_mapping_range(struct address_space *mapping, struct dax_device *dax_dev, struct writeback_control *wbc); +int dax_folio_reset_order(struct folio *folio); struct page *dax_layout_busy_page(struct address_space *mapping); struct page *dax_layout_busy_page_range(struct address_space *mapping, loff_t start, loff_t end); @@ -163,6 +164,15 @@ dax_entry_t dax_lock_mapping_entry(struct address_space *mapping, void dax_unlock_mapping_entry(struct address_space *mapping, unsigned long index, dax_entry_t cookie); #else +static inline void fs_put_dax(struct dax_device *dax_dev, void *holder) +{ +} + +static inline int fs_dax_get(struct dax_device *dax_dev, void *holder, + const struct dax_holder_operations *hops) +{ + return -EOPNOTSUPP; +} static inline struct page *dax_layout_busy_page(struct address_space *mapping) { return NULL; @@ -242,6 +252,7 @@ static inline void dax_break_layout_final(struct inode *inode) bool dax_alive(struct dax_device *dax_dev); void *dax_get_private(struct dax_device *dax_dev); +int dax_set_ops(struct dax_device *dax_dev, const struct dax_operations *ops); long dax_direct_access(struct dax_device *dax_dev, pgoff_t pgoff, long nr_pages, enum dax_access_mode mode, void **kaddr, unsigned long *pfn); size_t dax_copy_from_iter(struct dax_device *dax_dev, pgoff_t pgoff, void *addr, diff --git a/include/linux/dcache.h b/include/linux/dcache.h index 898c60d21c92..4b1ff99608e0 100644 --- a/include/linux/dcache.h +++ b/include/linux/dcache.h @@ -88,6 +88,7 @@ union shortname_store { #define d_lock d_lockref.lock #define d_iname d_shortname.string +struct completion_list; struct dentry { /* RCU lookup touched fields */ @@ -115,20 +116,28 @@ struct dentry { * possible! */ - union { - struct list_head d_lru; /* LRU list */ - wait_queue_head_t *d_wait; /* in-lookup ones only */ - }; + struct list_head d_lru; /* LRU list */ struct hlist_node d_sib; /* child of parent list */ struct hlist_head d_children; /* our children */ /* - * d_alias and d_rcu can share memory + * the following members can share memory - their uses are + * mutually exclusive. */ union { - struct hlist_node d_alias; /* inode alias list */ - struct hlist_bl_node d_in_lookup_hash; /* only for in-lookup ones */ + /* positives: inode alias list */ + struct hlist_node d_alias; + /* in-lookup ones (all negative, live): hash chain */ + struct hlist_bl_node d_in_lookup_hash; + /* killed ones: (already negative) used to schedule freeing */ struct rcu_head d_rcu; - } d_u; + /* + * live non-in-lookup negatives: used if shrink_dcache_tree() + * races with eviction by another thread and needs to wait for + * this dentry to get killed . Remains NULL for almost all + * negative dentries. + */ + struct completion_list *waiters; + }; }; /* @@ -198,6 +207,9 @@ enum dentry_flags { DCACHE_REFERENCED = BIT(6), /* Recently used, don't discard. */ DCACHE_DONTCACHE = BIT(7), /* Purge from memory on final dput() */ DCACHE_CANT_MOUNT = BIT(8), + DCACHE_LOOKUP_WAITERS = BIT(9), /* A thread is waiting for + * PAR_LOOKUP to clear + */ DCACHE_SHRINK_LIST = BIT(10), DCACHE_OP_WEAK_REVALIDATE = BIT(11), /* @@ -244,8 +256,7 @@ extern void d_delete(struct dentry *); /* allocate/de-allocate */ extern struct dentry * d_alloc(struct dentry *, const struct qstr *); extern struct dentry * d_alloc_anon(struct super_block *); -extern struct dentry * d_alloc_parallel(struct dentry *, const struct qstr *, - wait_queue_head_t *); +extern struct dentry * d_alloc_parallel(struct dentry *, const struct qstr *); extern struct dentry * d_splice_alias(struct inode *, struct dentry *); /* weird procfs mess; *NOT* exported */ extern struct dentry * d_splice_alias_ops(struct inode *, struct dentry *, @@ -264,11 +275,12 @@ extern void d_invalidate(struct dentry *); extern struct dentry * d_make_root(struct inode *); extern void d_mark_tmpfile(struct file *, struct inode *); +int d_mark_tmpfile_name(struct file *file, const struct qstr *name); extern void d_tmpfile(struct file *, struct inode *); extern struct dentry *d_find_alias(struct inode *); extern void d_prune_aliases(struct inode *); -extern void d_dispose_if_unused(struct dentry *, struct list_head *); +extern bool __move_to_shrink_list(struct dentry *, struct list_head *); extern void shrink_dentry_list(struct list_head *); extern struct dentry *d_find_alias_rcu(struct inode *); @@ -353,6 +365,24 @@ static inline struct dentry *dget(struct dentry *dentry) return dentry; } +/* dentry->d_inode->i_lock must be held by caller */ +static inline bool dget_alias_ilocked(struct dentry *dentry) +{ + if (likely(!(READ_ONCE(dentry->d_flags) & DCACHE_NORCU))) { + lockref_get(&dentry->d_lockref); + return true; + } + // NORCU dentries with zero refcount MUST NOT be grabbed + spin_lock(&dentry->d_lock); + if (dentry->d_lockref.count > 0) { + dget_dlock(dentry); + spin_unlock(&dentry->d_lock); + return true; + } + spin_unlock(&dentry->d_lock); + return false; +} + extern struct dentry *dget_parent(struct dentry *dentry); /** @@ -615,4 +645,8 @@ void set_default_d_op(struct super_block *, const struct dentry_operations *); struct dentry *d_make_persistent(struct dentry *, struct inode *); void d_make_discardable(struct dentry *dentry); +/* inode->i_lock must be held over that */ +#define for_each_alias(dentry, inode) \ + hlist_for_each_entry(dentry, &(inode)->i_dentry, d_alias) + #endif /* __LINUX_DCACHE_H */ diff --git a/include/linux/delay.h b/include/linux/delay.h index 46412c00033a..82409e55b6ae 100644 --- a/include/linux/delay.h +++ b/include/linux/delay.h @@ -17,6 +17,8 @@ extern unsigned long loops_per_jiffy; #include <asm/delay.h> +bool delay_read_timer(unsigned long *t); + /* * Using udelay() for intervals greater than a few milliseconds can * risk overflow for high loops_per_jiffy (high bogomips) machines. The @@ -110,7 +112,7 @@ static const unsigned int max_slack_shift = 2; * fsleep - flexible sleep which autoselects the best mechanism * @usecs: requested sleep duration in microseconds * - * flseep() selects the best mechanism that will provide maximum 25% slack + * fsleep() selects the best mechanism that will provide maximum 25% slack * to the requested sleep duration. Therefore it uses: * * * udelay() loop for sleep durations <= 10 microseconds to avoid hrtimer diff --git a/include/linux/designware_i2c.h b/include/linux/designware_i2c.h new file mode 100644 index 000000000000..53f37f18a722 --- /dev/null +++ b/include/linux/designware_i2c.h @@ -0,0 +1,107 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Synopsys DesignWare I2C register definitions + * + * Copyright (C) 2026, Intel Corporation + */ + +#ifndef __LINUX_DESIGNWARE_I2C_H +#define __LINUX_DESIGNWARE_I2C_H + +#include <linux/bits.h> + +/* + * Registers offset + */ +#define DW_IC_CON 0x00 +#define DW_IC_TAR 0x04 +#define DW_IC_SAR 0x08 +#define DW_IC_DATA_CMD 0x10 +#define DW_IC_SS_SCL_HCNT 0x14 +#define DW_IC_SS_SCL_LCNT 0x18 +#define DW_IC_FS_SCL_HCNT 0x1c +#define DW_IC_FS_SCL_LCNT 0x20 +#define DW_IC_HS_SCL_HCNT 0x24 +#define DW_IC_HS_SCL_LCNT 0x28 +#define DW_IC_INTR_STAT 0x2c +#define DW_IC_INTR_MASK 0x30 +#define DW_IC_RAW_INTR_STAT 0x34 +#define DW_IC_RX_TL 0x38 +#define DW_IC_TX_TL 0x3c +#define DW_IC_CLR_INTR 0x40 +#define DW_IC_CLR_RX_UNDER 0x44 +#define DW_IC_CLR_RX_OVER 0x48 +#define DW_IC_CLR_TX_OVER 0x4c +#define DW_IC_CLR_RD_REQ 0x50 +#define DW_IC_CLR_TX_ABRT 0x54 +#define DW_IC_CLR_RX_DONE 0x58 +#define DW_IC_CLR_ACTIVITY 0x5c +#define DW_IC_CLR_STOP_DET 0x60 +#define DW_IC_CLR_START_DET 0x64 +#define DW_IC_CLR_GEN_CALL 0x68 +#define DW_IC_ENABLE 0x6c +#define DW_IC_STATUS 0x70 +#define DW_IC_TXFLR 0x74 +#define DW_IC_RXFLR 0x78 +#define DW_IC_SDA_HOLD 0x7c +#define DW_IC_TX_ABRT_SOURCE 0x80 +#define DW_IC_ENABLE_STATUS 0x9c +#define DW_IC_CLR_RESTART_DET 0xa8 +#define DW_IC_SMBUS_INTR_STAT 0xc8 +#define DW_IC_SMBUS_INTR_MASK 0xcc +#define DW_IC_CLR_SMBUS_INTR 0xd4 +#define DW_IC_COMP_PARAM_1 0xf4 +#define DW_IC_COMP_VERSION 0xf8 +#define DW_IC_COMP_TYPE 0xfc + +/* DW_IC_CON bits */ +#define DW_IC_CON_MASTER BIT(0) +#define DW_IC_CON_SPEED_STD (1 << 1) +#define DW_IC_CON_SPEED_FAST (2 << 1) +#define DW_IC_CON_SPEED_HIGH (3 << 1) +#define DW_IC_CON_SPEED_MASK GENMASK(2, 1) +#define DW_IC_CON_10BITADDR_SLAVE BIT(3) +#define DW_IC_CON_10BITADDR_MASTER BIT(4) +#define DW_IC_CON_RESTART_EN BIT(5) +#define DW_IC_CON_SLAVE_DISABLE BIT(6) +#define DW_IC_CON_STOP_DET_IFADDRESSED BIT(7) +#define DW_IC_CON_TX_EMPTY_CTRL BIT(8) +#define DW_IC_CON_RX_FIFO_FULL_HLD_CTRL BIT(9) +#define DW_IC_CON_BUS_CLEAR_CTRL BIT(11) + +/* DW_IC_DATA_CMD bits */ +#define DW_IC_DATA_CMD_DAT GENMASK(7, 0) +#define DW_IC_DATA_CMD_FIRST_DATA_BYTE BIT(11) + +/* DW_IC_INTR_* bits */ +#define DW_IC_INTR_RX_UNDER BIT(0) +#define DW_IC_INTR_RX_OVER BIT(1) +#define DW_IC_INTR_RX_FULL BIT(2) +#define DW_IC_INTR_TX_OVER BIT(3) +#define DW_IC_INTR_TX_EMPTY BIT(4) +#define DW_IC_INTR_RD_REQ BIT(5) +#define DW_IC_INTR_TX_ABRT BIT(6) +#define DW_IC_INTR_RX_DONE BIT(7) +#define DW_IC_INTR_ACTIVITY BIT(8) +#define DW_IC_INTR_STOP_DET BIT(9) +#define DW_IC_INTR_START_DET BIT(10) +#define DW_IC_INTR_GEN_CALL BIT(11) +#define DW_IC_INTR_RESTART_DET BIT(12) +#define DW_IC_INTR_MST_ON_HOLD BIT(13) + +/* DW_IC_ENABLE bits */ +#define DW_IC_ENABLE_ENABLE BIT(0) +#define DW_IC_ENABLE_ABORT BIT(1) + +/* DW_IC_STATUS bits */ +#define DW_IC_STATUS_ACTIVITY BIT(0) +#define DW_IC_STATUS_TFE BIT(2) +#define DW_IC_STATUS_RFNE BIT(3) +#define DW_IC_STATUS_MASTER_ACTIVITY BIT(5) +#define DW_IC_STATUS_SLAVE_ACTIVITY BIT(6) +#define DW_IC_STATUS_MASTER_HOLD_TX_FIFO_EMPTY BIT(7) + +/* DW_IC_SMBUS_INTR_* bits */ +#define DW_IC_SMBUS_INTR_ALERT BIT(10) + +#endif /* __LINUX_DESIGNWARE_I2C_H */ diff --git a/include/linux/device-id/acpi.h b/include/linux/device-id/acpi.h new file mode 100644 index 000000000000..65800cefddca --- /dev/null +++ b/include/linux/device-id/acpi.h @@ -0,0 +1,32 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_ACPI_H +#define LINUX_DEVICE_ID_ACPI_H + +#ifdef __KERNEL__ +#include <linux/types.h> +typedef unsigned long kernel_ulong_t; +#endif + +#define ACPI_ID_LEN 16 + +struct acpi_device_id { + __u8 id[ACPI_ID_LEN]; + kernel_ulong_t driver_data; + __u32 cls; + __u32 cls_msk; +}; + +/** + * ACPI_DEVICE_CLASS - macro used to describe an ACPI device with + * the PCI-defined class-code information + * + * @_cls : the class, subclass, prog-if triple for this device + * @_msk : the class mask for this device + * + * This macro is used to create a struct acpi_device_id that matches a + * specific PCI class. The .id and .driver_data fields will be left + * initialized with the default value. + */ +#define ACPI_DEVICE_CLASS(_cls, _msk) .cls = (_cls), .cls_msk = (_msk), + +#endif /* ifndef LINUX_DEVICE_ID_ACPI_H */ diff --git a/include/linux/device-id/amba.h b/include/linux/device-id/amba.h new file mode 100644 index 000000000000..114d66a784ac --- /dev/null +++ b/include/linux/device-id/amba.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_AMBA_H +#define LINUX_DEVICE_ID_AMBA_H + +/** + * struct amba_id - identifies a device on an AMBA bus + * @id: The significant bits if the hardware device ID + * @mask: Bitmask specifying which bits of the id field are significant when + * matching. A driver binds to a device when ((hardware device ID) & mask) + * == id. + * @data: Private data used by the driver. + */ +struct amba_id { + unsigned int id; + unsigned int mask; + void *data; +}; + +#endif /* ifndef LINUX_DEVICE_ID_AMBA_H */ diff --git a/include/linux/device-id/ap.h b/include/linux/device-id/ap.h new file mode 100644 index 000000000000..e050abebbf3d --- /dev/null +++ b/include/linux/device-id/ap.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_AP_H +#define LINUX_DEVICE_ID_AP_H + +#ifdef __KERNEL__ +#include <linux/types.h> +#endif + +#define AP_DEVICE_ID_MATCH_CARD_TYPE 0x01 +#define AP_DEVICE_ID_MATCH_QUEUE_TYPE 0x02 + +/* s390 AP bus devices */ +struct ap_device_id { + __u16 match_flags; /* which fields to match against */ + __u8 dev_type; /* device type */ +}; + +#endif /* ifndef LINUX_DEVICE_ID_AP_H */ diff --git a/include/linux/device-id/apr.h b/include/linux/device-id/apr.h new file mode 100644 index 000000000000..f282608ea018 --- /dev/null +++ b/include/linux/device-id/apr.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_APR_H +#define LINUX_DEVICE_ID_APR_H + +#ifdef __KERNEL__ +#include <linux/types.h> +typedef unsigned long kernel_ulong_t; +#endif + +#define APR_NAME_SIZE 32 +#define APR_MODULE_PREFIX "apr:" + +struct apr_device_id { + char name[APR_NAME_SIZE]; + __u32 domain_id; + __u32 svc_id; + __u32 svc_version; + kernel_ulong_t driver_data; /* Data private to the driver */ +}; + +#endif /* ifndef LINUX_DEVICE_ID_APR_H */ diff --git a/include/linux/device-id/auxiliary.h b/include/linux/device-id/auxiliary.h new file mode 100644 index 000000000000..9d512dfb23dd --- /dev/null +++ b/include/linux/device-id/auxiliary.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_AUXILIARY_H +#define LINUX_DEVICE_ID_AUXILIARY_H + +#ifdef __KERNEL__ +typedef unsigned long kernel_ulong_t; +#endif + +#define AUXILIARY_NAME_SIZE 40 +#define AUXILIARY_MODULE_PREFIX "auxiliary:" + +struct auxiliary_device_id { + char name[AUXILIARY_NAME_SIZE]; + kernel_ulong_t driver_data; +}; + +#endif /* ifndef LINUX_DEVICE_ID_AUXILIARY_H */ diff --git a/include/linux/device-id/bcma.h b/include/linux/device-id/bcma.h new file mode 100644 index 000000000000..3e6b973dc4ae --- /dev/null +++ b/include/linux/device-id/bcma.h @@ -0,0 +1,25 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_BCMA_H +#define LINUX_DEVICE_ID_BCMA_H + +#ifdef __KERNEL__ +#include <linux/types.h> +#endif + +#define BCMA_CORE(_manuf, _id, _rev, _class) \ + { .manuf = _manuf, .id = _id, .rev = _rev, .class = _class, } + +#define BCMA_ANY_MANUF 0xFFFF +#define BCMA_ANY_ID 0xFFFF +#define BCMA_ANY_REV 0xFF +#define BCMA_ANY_CLASS 0xFF + +/* Broadcom's specific AMBA core, see drivers/bcma/ */ +struct bcma_device_id { + __u16 manuf; + __u16 id; + __u8 rev; + __u8 class; +} __attribute__((packed,aligned(2))); + +#endif /* ifndef LINUX_DEVICE_ID_BCMA_H */ diff --git a/include/linux/device-id/ccw.h b/include/linux/device-id/ccw.h new file mode 100644 index 000000000000..6b7086aa5ca8 --- /dev/null +++ b/include/linux/device-id/ccw.h @@ -0,0 +1,27 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_CCW_H +#define LINUX_DEVICE_ID_CCW_H + +#ifdef __KERNEL__ +#include <linux/types.h> +typedef unsigned long kernel_ulong_t; +#endif + +#define CCW_DEVICE_ID_MATCH_CU_TYPE 0x01 +#define CCW_DEVICE_ID_MATCH_CU_MODEL 0x02 +#define CCW_DEVICE_ID_MATCH_DEVICE_TYPE 0x04 +#define CCW_DEVICE_ID_MATCH_DEVICE_MODEL 0x08 + +/* s390 CCW devices */ +struct ccw_device_id { + __u16 match_flags; /* which fields to match against */ + + __u16 cu_type; /* control unit type */ + __u16 dev_type; /* device type */ + __u8 cu_model; /* control unit model */ + __u8 dev_model; /* device model */ + + kernel_ulong_t driver_info; +}; + +#endif /* ifndef LINUX_DEVICE_ID_CCW_H */ diff --git a/include/linux/device-id/cdx.h b/include/linux/device-id/cdx.h new file mode 100644 index 000000000000..c6cb2b5cab5a --- /dev/null +++ b/include/linux/device-id/cdx.h @@ -0,0 +1,40 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_CDX_H +#define LINUX_DEVICE_ID_CDX_H + +#ifdef __KERNEL__ +#include <linux/types.h> +#endif + +#define CDX_ANY_ID (0xFFFF) + +enum { + CDX_ID_F_VFIO_DRIVER_OVERRIDE = 1, +}; + +/** + * struct cdx_device_id - CDX device identifier + * @vendor: Vendor ID + * @device: Device ID + * @subvendor: Subsystem vendor ID (or CDX_ANY_ID) + * @subdevice: Subsystem device ID (or CDX_ANY_ID) + * @class: Device class + * Most drivers do not need to specify class/class_mask + * as vendor/device is normally sufficient. + * @class_mask: Limit which sub-fields of the class field are compared. + * @override_only: Match only when dev->driver_override is this driver. + * + * Type of entries in the "device Id" table for CDX devices supported by + * a CDX device driver. + */ +struct cdx_device_id { + __u16 vendor; + __u16 device; + __u16 subvendor; + __u16 subdevice; + __u32 class; + __u32 class_mask; + __u32 override_only; +}; + +#endif /* ifndef LINUX_DEVICE_ID_CDX_H */ diff --git a/include/linux/device-id/coreboot.h b/include/linux/device-id/coreboot.h new file mode 100644 index 000000000000..ff459879781e --- /dev/null +++ b/include/linux/device-id/coreboot.h @@ -0,0 +1,20 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_COREBOOT_H +#define LINUX_DEVICE_ID_COREBOOT_H + +#ifdef __KERNEL__ +#include <linux/types.h> +typedef unsigned long kernel_ulong_t; +#endif + +/** + * struct coreboot_device_id - Identifies a coreboot table entry + * @tag: tag ID + * @driver_data: driver specific data + */ +struct coreboot_device_id { + __u32 tag; + kernel_ulong_t driver_data; +}; + +#endif /* ifndef LINUX_DEVICE_ID_COREBOOT_H */ diff --git a/include/linux/device-id/css.h b/include/linux/device-id/css.h new file mode 100644 index 000000000000..67435bb22618 --- /dev/null +++ b/include/linux/device-id/css.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_CSS_H +#define LINUX_DEVICE_ID_CSS_H + +#ifdef __KERNEL__ +#include <linux/types.h> +typedef unsigned long kernel_ulong_t; +#endif + +/* s390 css bus devices (subchannels) */ +struct css_device_id { + __u8 match_flags; + __u8 type; /* subchannel type */ + kernel_ulong_t driver_data; +}; + +#endif /* ifndef LINUX_DEVICE_ID_CSS_H */ diff --git a/include/linux/device-id/dfl.h b/include/linux/device-id/dfl.h new file mode 100644 index 000000000000..bd0c9dbafeeb --- /dev/null +++ b/include/linux/device-id/dfl.h @@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_DFL_H +#define LINUX_DEVICE_ID_DFL_H + +#ifdef __KERNEL__ +#include <linux/types.h> +typedef unsigned long kernel_ulong_t; +#endif + +/* + * DFL (Device Feature List) + * + * DFL defines a linked list of feature headers within the device MMIO space to + * provide an extensible way of adding features. Software can walk through these + * predefined data structures to enumerate features. It is now used in the FPGA. + * See Documentation/fpga/dfl.rst for more information. + * + * The dfl bus type is introduced to match the individual feature devices (dfl + * devices) for specific dfl drivers. + */ + +/** + * struct dfl_device_id - dfl device identifier + * @type: DFL FIU type of the device. See enum dfl_id_type. + * @feature_id: feature identifier local to its DFL FIU type. + * @driver_data: driver specific data. + */ +struct dfl_device_id { + __u16 type; + __u16 feature_id; + kernel_ulong_t driver_data; +}; + +#endif /* ifndef LINUX_DEVICE_ID_DFL_H */ diff --git a/include/linux/device-id/dmi.h b/include/linux/device-id/dmi.h new file mode 100644 index 000000000000..fdc4adbad133 --- /dev/null +++ b/include/linux/device-id/dmi.h @@ -0,0 +1,58 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_DMI_H +#define LINUX_DEVICE_ID_DMI_H + +#define DMI_MATCH(a, b) { .slot = a, .substr = b } +#define DMI_EXACT_MATCH(a, b) { .slot = a, .substr = b, .exact_match = 1 } + +/* dmi */ +enum dmi_field { + DMI_NONE, + DMI_BIOS_VENDOR, + DMI_BIOS_VERSION, + DMI_BIOS_DATE, + DMI_BIOS_RELEASE, + DMI_EC_FIRMWARE_RELEASE, + DMI_SYS_VENDOR, + DMI_PRODUCT_NAME, + DMI_PRODUCT_VERSION, + DMI_PRODUCT_SERIAL, + DMI_PRODUCT_UUID, + DMI_PRODUCT_SKU, + DMI_PRODUCT_FAMILY, + DMI_BOARD_VENDOR, + DMI_BOARD_NAME, + DMI_BOARD_VERSION, + DMI_BOARD_SERIAL, + DMI_BOARD_ASSET_TAG, + DMI_CHASSIS_VENDOR, + DMI_CHASSIS_TYPE, + DMI_CHASSIS_VERSION, + DMI_CHASSIS_SERIAL, + DMI_CHASSIS_ASSET_TAG, + DMI_STRING_MAX, + DMI_OEM_STRING, /* special case - will not be in dmi_ident */ +}; + +struct dmi_strmatch { + unsigned char slot:7; + unsigned char exact_match:1; + char substr[79]; +}; + +struct dmi_system_id { + int (*callback)(const struct dmi_system_id *); + const char *ident; + struct dmi_strmatch matches[4]; + void *driver_data; +}; + +/* + * struct dmi_device_id appears during expansion of + * "MODULE_DEVICE_TABLE(dmi, x)". Compiler doesn't look inside it + * but this is enough for gcc 3.4.6 to error out: + * error: storage size of '__mod_dmi_device_table' isn't known + */ +#define dmi_device_id dmi_system_id + +#endif /* ifndef LINUX_DEVICE_ID_DMI_H */ diff --git a/include/linux/device-id/eisa.h b/include/linux/device-id/eisa.h new file mode 100644 index 000000000000..1eeae6247524 --- /dev/null +++ b/include/linux/device-id/eisa.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_EISA_H +#define LINUX_DEVICE_ID_EISA_H + +#ifdef __KERNEL__ +typedef unsigned long kernel_ulong_t; +#endif + +/* EISA */ + +#define EISA_SIG_LEN 8 + +/* The EISA signature, in ASCII form, null terminated */ +struct eisa_device_id { + char sig[EISA_SIG_LEN]; + kernel_ulong_t driver_data; +}; + +#define EISA_DEVICE_MODALIAS_FMT "eisa:s%s" + +#endif /* ifndef LINUX_DEVICE_ID_EISA_H */ diff --git a/include/linux/device-id/fsl_mc.h b/include/linux/device-id/fsl_mc.h new file mode 100644 index 000000000000..0cfa94923670 --- /dev/null +++ b/include/linux/device-id/fsl_mc.h @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_FSL_MC_H +#define LINUX_DEVICE_ID_FSL_MC_H + +#ifdef __KERNEL__ +#include <linux/types.h> +#endif + +/** + * struct fsl_mc_device_id - MC object device identifier + * @vendor: vendor ID + * @obj_type: MC object type + * + * Type of entries in the "device Id" table for MC object devices supported by + * a MC object device driver. The last entry of the table has vendor set to 0x0 + */ +struct fsl_mc_device_id { + __u16 vendor; + const char obj_type[16]; +}; + +#endif /* ifndef LINUX_DEVICE_ID_FSL_MC_H */ diff --git a/include/linux/device-id/hda.h b/include/linux/device-id/hda.h new file mode 100644 index 000000000000..42542580f955 --- /dev/null +++ b/include/linux/device-id/hda.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_HDA_H +#define LINUX_DEVICE_ID_HDA_H + +#ifdef __KERNEL__ +#include <linux/types.h> +#endif + +struct hda_device_id { + __u32 vendor_id; + __u32 rev_id; + __u8 api_version; + const char *name; + unsigned long driver_data; +}; + +#endif /* ifndef LINUX_DEVICE_ID_HDA_H */ diff --git a/include/linux/device-id/hid.h b/include/linux/device-id/hid.h new file mode 100644 index 000000000000..e865fc64bf94 --- /dev/null +++ b/include/linux/device-id/hid.h @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_HID_H +#define LINUX_DEVICE_ID_HID_H + +#ifdef __KERNEL__ +#include <linux/types.h> +typedef unsigned long kernel_ulong_t; +#endif + +#define HID_ANY_ID (~0) +#define HID_BUS_ANY 0xffff +#define HID_GROUP_ANY 0x0000 + +struct hid_device_id { + __u16 bus; + __u16 group; + __u32 vendor; + __u32 product; + kernel_ulong_t driver_data; +}; + +#endif /* ifndef LINUX_DEVICE_ID_HID_H */ diff --git a/include/linux/device-id/hv_vmbus.h b/include/linux/device-id/hv_vmbus.h new file mode 100644 index 000000000000..a7682c05619f --- /dev/null +++ b/include/linux/device-id/hv_vmbus.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_HV_VMBUS_H +#define LINUX_DEVICE_ID_HV_VMBUS_H + +#ifdef __KERNEL__ +#include <linux/uuid.h> +typedef unsigned long kernel_ulong_t; +#endif + +/* + * For Hyper-V devices we use the device guid as the id. + */ +struct hv_vmbus_device_id { + guid_t guid; + kernel_ulong_t driver_data; /* Data private to the driver */ +}; + +#endif /* ifndef LINUX_DEVICE_ID_HV_VMBUS_H */ diff --git a/include/linux/device-id/i2c.h b/include/linux/device-id/i2c.h new file mode 100644 index 000000000000..21f9b581e7a9 --- /dev/null +++ b/include/linux/device-id/i2c.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_I2C_H +#define LINUX_DEVICE_ID_I2C_H + +#ifdef __KERNEL__ +typedef unsigned long kernel_ulong_t; +#endif + +/* i2c */ + +#define I2C_NAME_SIZE 20 +#define I2C_MODULE_PREFIX "i2c:" + +struct i2c_device_id { + char name[I2C_NAME_SIZE]; + kernel_ulong_t driver_data; /* Data private to the driver */ +}; + +#endif /* ifndef LINUX_DEVICE_ID_I2C_H */ diff --git a/include/linux/device-id/i3c.h b/include/linux/device-id/i3c.h new file mode 100644 index 000000000000..5d8222c9f908 --- /dev/null +++ b/include/linux/device-id/i3c.h @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_I3C_H +#define LINUX_DEVICE_ID_I3C_H + +#ifdef __KERNEL__ +#include <linux/types.h> +#endif + +/* i3c */ + +#define I3C_MATCH_DCR 0x1 +#define I3C_MATCH_MANUF 0x2 +#define I3C_MATCH_PART 0x4 +#define I3C_MATCH_EXTRA_INFO 0x8 + +struct i3c_device_id { + __u8 match_flags; + __u8 dcr; + __u16 manuf_id; + __u16 part_id; + __u16 extra_info; + + const void *data; +}; + +#endif /* ifndef LINUX_DEVICE_ID_I3C_H */ diff --git a/include/linux/device-id/ieee1394.h b/include/linux/device-id/ieee1394.h new file mode 100644 index 000000000000..63023964724a --- /dev/null +++ b/include/linux/device-id/ieee1394.h @@ -0,0 +1,27 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_IEEE1394_H +#define LINUX_DEVICE_ID_IEEE1394_H + +#ifdef __KERNEL__ +#include <linux/types.h> +typedef unsigned long kernel_ulong_t; +#endif + +#define IEEE1394_MATCH_VENDOR_ID 0x0001 +#define IEEE1394_MATCH_MODEL_ID 0x0002 +#define IEEE1394_MATCH_SPECIFIER_ID 0x0004 +#define IEEE1394_MATCH_VERSION 0x0008 + +struct ieee1394_device_id { + __u32 match_flags; + __u32 vendor_id; + __u32 model_id; + __u32 specifier_id; + __u32 version; + union { + kernel_ulong_t driver_data; + const void *driver_data_ptr; + }; +}; + +#endif /* ifndef LINUX_DEVICE_ID_IEEE1394_H */ diff --git a/include/linux/device-id/input.h b/include/linux/device-id/input.h new file mode 100644 index 000000000000..66d7e78d32c0 --- /dev/null +++ b/include/linux/device-id/input.h @@ -0,0 +1,62 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_INPUT_H +#define LINUX_DEVICE_ID_INPUT_H + +#ifdef __KERNEL__ +#include <linux/types.h> +typedef unsigned long kernel_ulong_t; +#endif + +/* Input */ +#define INPUT_DEVICE_ID_EV_MAX 0x1f +#define INPUT_DEVICE_ID_KEY_MIN_INTERESTING 0x71 +#define INPUT_DEVICE_ID_KEY_MAX 0x2ff +#define INPUT_DEVICE_ID_REL_MAX 0x0f +#define INPUT_DEVICE_ID_ABS_MAX 0x3f +#define INPUT_DEVICE_ID_MSC_MAX 0x07 +#define INPUT_DEVICE_ID_LED_MAX 0x0f +#define INPUT_DEVICE_ID_SND_MAX 0x07 +#define INPUT_DEVICE_ID_FF_MAX 0x7f +#define INPUT_DEVICE_ID_SW_MAX 0x11 +#define INPUT_DEVICE_ID_PROP_MAX 0x1f + +#define INPUT_DEVICE_ID_MATCH_BUS 1 +#define INPUT_DEVICE_ID_MATCH_VENDOR 2 +#define INPUT_DEVICE_ID_MATCH_PRODUCT 4 +#define INPUT_DEVICE_ID_MATCH_VERSION 8 + +#define INPUT_DEVICE_ID_MATCH_EVBIT 0x0010 +#define INPUT_DEVICE_ID_MATCH_KEYBIT 0x0020 +#define INPUT_DEVICE_ID_MATCH_RELBIT 0x0040 +#define INPUT_DEVICE_ID_MATCH_ABSBIT 0x0080 +#define INPUT_DEVICE_ID_MATCH_MSCIT 0x0100 +#define INPUT_DEVICE_ID_MATCH_LEDBIT 0x0200 +#define INPUT_DEVICE_ID_MATCH_SNDBIT 0x0400 +#define INPUT_DEVICE_ID_MATCH_FFBIT 0x0800 +#define INPUT_DEVICE_ID_MATCH_SWBIT 0x1000 +#define INPUT_DEVICE_ID_MATCH_PROPBIT 0x2000 + +struct input_device_id { + + kernel_ulong_t flags; + + __u16 bustype; + __u16 vendor; + __u16 product; + __u16 version; + + kernel_ulong_t evbit[INPUT_DEVICE_ID_EV_MAX / BITS_PER_LONG + 1]; + kernel_ulong_t keybit[INPUT_DEVICE_ID_KEY_MAX / BITS_PER_LONG + 1]; + kernel_ulong_t relbit[INPUT_DEVICE_ID_REL_MAX / BITS_PER_LONG + 1]; + kernel_ulong_t absbit[INPUT_DEVICE_ID_ABS_MAX / BITS_PER_LONG + 1]; + kernel_ulong_t mscbit[INPUT_DEVICE_ID_MSC_MAX / BITS_PER_LONG + 1]; + kernel_ulong_t ledbit[INPUT_DEVICE_ID_LED_MAX / BITS_PER_LONG + 1]; + kernel_ulong_t sndbit[INPUT_DEVICE_ID_SND_MAX / BITS_PER_LONG + 1]; + kernel_ulong_t ffbit[INPUT_DEVICE_ID_FF_MAX / BITS_PER_LONG + 1]; + kernel_ulong_t swbit[INPUT_DEVICE_ID_SW_MAX / BITS_PER_LONG + 1]; + kernel_ulong_t propbit[INPUT_DEVICE_ID_PROP_MAX / BITS_PER_LONG + 1]; + + kernel_ulong_t driver_info; +}; + +#endif /* ifndef LINUX_DEVICE_ID_INPUT_H */ diff --git a/include/linux/device-id/ipack.h b/include/linux/device-id/ipack.h new file mode 100644 index 000000000000..7f9b425e1e30 --- /dev/null +++ b/include/linux/device-id/ipack.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_IPACK_H +#define LINUX_DEVICE_ID_IPACK_H + +#ifdef __KERNEL__ +#include <linux/types.h> +#endif + +#define IPACK_ANY_FORMAT 0xff +#define IPACK_ANY_ID (~0) +struct ipack_device_id { + __u8 format; /* Format version or IPACK_ANY_ID */ + __u32 vendor; /* Vendor ID or IPACK_ANY_ID */ + __u32 device; /* Device ID or IPACK_ANY_ID */ +}; + +#endif /* ifndef LINUX_DEVICE_ID_IPACK_H */ diff --git a/include/linux/device-id/isapnp.h b/include/linux/device-id/isapnp.h new file mode 100644 index 000000000000..ba659c32650a --- /dev/null +++ b/include/linux/device-id/isapnp.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_ISAPNP_H +#define LINUX_DEVICE_ID_ISAPNP_H + +#ifdef __KERNEL__ +typedef unsigned long kernel_ulong_t; +#endif + +#define ISAPNP_ANY_ID 0xffff +struct isapnp_device_id { + unsigned short card_vendor, card_device; + unsigned short vendor, function; + kernel_ulong_t driver_data; /* data private to the driver */ +}; + +#endif /* ifndef LINUX_DEVICE_ID_ISAPNP_H */ diff --git a/include/linux/device-id/ishtp.h b/include/linux/device-id/ishtp.h new file mode 100644 index 000000000000..c66f144b4cb6 --- /dev/null +++ b/include/linux/device-id/ishtp.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_ISHTP_H +#define LINUX_DEVICE_ID_ISHTP_H + +#ifdef __KERNEL__ +#include <linux/uuid.h> +typedef unsigned long kernel_ulong_t; +#endif + +/* ISHTP (Integrated Sensor Hub Transport Protocol) */ + +#define ISHTP_MODULE_PREFIX "ishtp:" + +/** + * struct ishtp_device_id - ISHTP device identifier + * @guid: GUID of the device. + * @driver_data: pointer to driver specific data + */ +struct ishtp_device_id { + guid_t guid; + kernel_ulong_t driver_data; +}; + +#endif /* ifndef LINUX_DEVICE_ID_ISHTP_H */ diff --git a/include/linux/device-id/mcb.h b/include/linux/device-id/mcb.h new file mode 100644 index 000000000000..a5daec32c21c --- /dev/null +++ b/include/linux/device-id/mcb.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_MCB_H +#define LINUX_DEVICE_ID_MCB_H + +#ifdef __KERNEL__ +#include <linux/types.h> +typedef unsigned long kernel_ulong_t; +#endif + +struct mcb_device_id { + __u16 device; + kernel_ulong_t driver_data; +}; + +#endif /* ifndef LINUX_DEVICE_ID_MCB_H */ diff --git a/include/linux/device-id/mdio.h b/include/linux/device-id/mdio.h new file mode 100644 index 000000000000..e6cda2d91ddb --- /dev/null +++ b/include/linux/device-id/mdio.h @@ -0,0 +1,35 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_MDIO_H +#define LINUX_DEVICE_ID_MDIO_H + +#ifdef __KERNEL__ +#include <linux/types.h> +#endif + +#define MDIO_MODULE_PREFIX "mdio:" + +#define MDIO_ID_FMT "%u%u%u%u%u%u%u%u%u%u%u%u%u%u%u%u%u%u%u%u%u%u%u%u%u%u%u%u%u%u%u%u" +#define MDIO_ID_ARGS(_id) \ + ((_id)>>31) & 1, ((_id)>>30) & 1, ((_id)>>29) & 1, ((_id)>>28) & 1, \ + ((_id)>>27) & 1, ((_id)>>26) & 1, ((_id)>>25) & 1, ((_id)>>24) & 1, \ + ((_id)>>23) & 1, ((_id)>>22) & 1, ((_id)>>21) & 1, ((_id)>>20) & 1, \ + ((_id)>>19) & 1, ((_id)>>18) & 1, ((_id)>>17) & 1, ((_id)>>16) & 1, \ + ((_id)>>15) & 1, ((_id)>>14) & 1, ((_id)>>13) & 1, ((_id)>>12) & 1, \ + ((_id)>>11) & 1, ((_id)>>10) & 1, ((_id)>>9) & 1, ((_id)>>8) & 1, \ + ((_id)>>7) & 1, ((_id)>>6) & 1, ((_id)>>5) & 1, ((_id)>>4) & 1, \ + ((_id)>>3) & 1, ((_id)>>2) & 1, ((_id)>>1) & 1, (_id) & 1 + +/** + * struct mdio_device_id - identifies PHY devices on an MDIO/MII bus + * @phy_id: The result of + * (mdio_read(&MII_PHYSID1) << 16 | mdio_read(&MII_PHYSID2)) & @phy_id_mask + * for this PHY type + * @phy_id_mask: Defines the significant bits of @phy_id. A value of 0 + * is used to terminate an array of struct mdio_device_id. + */ +struct mdio_device_id { + __u32 phy_id; + __u32 phy_id_mask; +}; + +#endif /* ifndef LINUX_DEVICE_ID_MDIO_H */ diff --git a/include/linux/device-id/mei_cl.h b/include/linux/device-id/mei_cl.h new file mode 100644 index 000000000000..aeeb95da4b29 --- /dev/null +++ b/include/linux/device-id/mei_cl.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_MEI_CL_H +#define LINUX_DEVICE_ID_MEI_CL_H + +#ifdef __KERNEL__ +#include <linux/types.h> +#include <linux/mei_uuid.h> +typedef unsigned long kernel_ulong_t; +#endif + +#define MEI_CL_MODULE_PREFIX "mei:" +#define MEI_CL_NAME_SIZE 32 +#define MEI_CL_VERSION_ANY 0xff + +/** + * struct mei_cl_device_id - MEI client device identifier + * @name: helper name + * @uuid: client uuid + * @version: client protocol version + * @driver_info: information used by the driver. + * + * identifies mei client device by uuid and name + */ +struct mei_cl_device_id { + char name[MEI_CL_NAME_SIZE]; + uuid_le uuid; + __u8 version; + kernel_ulong_t driver_info; +}; + +#endif /* ifndef LINUX_DEVICE_ID_MEI_CL_H */ diff --git a/include/linux/device-id/mhi.h b/include/linux/device-id/mhi.h new file mode 100644 index 000000000000..30a27a71a57f --- /dev/null +++ b/include/linux/device-id/mhi.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_MHI_H +#define LINUX_DEVICE_ID_MHI_H + +#ifdef __KERNEL__ +typedef unsigned long kernel_ulong_t; +#endif + +#define MHI_DEVICE_MODALIAS_FMT "mhi:%s" +#define MHI_NAME_SIZE 32 + +#define MHI_EP_DEVICE_MODALIAS_FMT "mhi_ep:%s" + +/** + * struct mhi_device_id - MHI device identification + * @chan: MHI channel name + * @driver_data: driver data; + */ +struct mhi_device_id { + const char chan[MHI_NAME_SIZE]; + kernel_ulong_t driver_data; +}; + +#endif /* ifndef LINUX_DEVICE_ID_MHI_H */ diff --git a/include/linux/device-id/mips_cdmm.h b/include/linux/device-id/mips_cdmm.h new file mode 100644 index 000000000000..e1623884103a --- /dev/null +++ b/include/linux/device-id/mips_cdmm.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_MIPS_CDMM_H +#define LINUX_DEVICE_ID_MIPS_CDMM_H + +#ifdef __KERNEL__ +#include <linux/types.h> +#endif + +/** + * struct mips_cdmm_device_id - identifies devices in MIPS CDMM bus + * @type: Device type identifier. + */ +struct mips_cdmm_device_id { + __u8 type; +}; + +#endif /* ifndef LINUX_DEVICE_ID_MIPS_CDMM_H */ diff --git a/include/linux/device-id/of.h b/include/linux/device-id/of.h new file mode 100644 index 000000000000..28ea360f8f9a --- /dev/null +++ b/include/linux/device-id/of.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_OF_H +#define LINUX_DEVICE_ID_OF_H + +/* + * Struct used for matching a device + */ +struct of_device_id { + char name[32]; + char type[32]; + char compatible[128]; + const void *data; +}; + +#endif /* ifndef LINUX_DEVICE_ID_OF_H */ diff --git a/include/linux/device-id/parisc.h b/include/linux/device-id/parisc.h new file mode 100644 index 000000000000..2974eb26a8be --- /dev/null +++ b/include/linux/device-id/parisc.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_PARISC_H +#define LINUX_DEVICE_ID_PARISC_H + +#ifdef __KERNEL__ +#include <linux/types.h> +#endif + +#define PA_HWTYPE_ANY_ID 0xff +#define PA_HVERSION_REV_ANY_ID 0xff +#define PA_HVERSION_ANY_ID 0xffff +#define PA_SVERSION_ANY_ID 0xffffffff + +struct parisc_device_id { + __u8 hw_type; /* 5 bits used */ + __u8 hversion_rev; /* 4 bits */ + __u16 hversion; /* 12 bits */ + __u32 sversion; /* 20 bits */ +}; + +#endif /* ifndef LINUX_DEVICE_ID_PARISC_H */ diff --git a/include/linux/device-id/pci.h b/include/linux/device-id/pci.h new file mode 100644 index 000000000000..4a635d531cd6 --- /dev/null +++ b/include/linux/device-id/pci.h @@ -0,0 +1,54 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_PCI_H +#define LINUX_DEVICE_ID_PCI_H + +#ifdef __KERNEL__ +#include <linux/types.h> +typedef unsigned long kernel_ulong_t; +#endif + +#define PCI_ANY_ID (~0) + +enum { + PCI_ID_F_VFIO_DRIVER_OVERRIDE = 1, +}; + +/** + * struct pci_device_id - PCI device ID structure + * @vendor: Vendor ID to match (or PCI_ANY_ID) + * @device: Device ID to match (or PCI_ANY_ID) + * @subvendor: Subsystem vendor ID to match (or PCI_ANY_ID) + * @subdevice: Subsystem device ID to match (or PCI_ANY_ID) + * @class: Device class, subclass, and "interface" to match. + * See Appendix D of the PCI Local Bus Spec or + * include/linux/pci_ids.h for a full list of classes. + * Most drivers do not need to specify class/class_mask + * as vendor/device is normally sufficient. + * @class_mask: Limit which sub-fields of the class field are compared. + * See drivers/scsi/sym53c8xx_2/ for example of usage. + * @driver_data: Data private to the driver. + * Most drivers don't need to use driver_data field. + * Best practice is to use driver_data as an index + * into a static list of equivalent device types, + * instead of using it as a pointer. + * @override_only: Match only when dev->driver_override is this driver. + */ +struct pci_device_id { + __u32 vendor, device; /* Vendor and device ID or PCI_ANY_ID*/ + __u32 subvendor, subdevice; /* Subsystem ID's or PCI_ANY_ID */ + __u32 class, class_mask; /* (class,subclass,prog-if) triplet */ + kernel_ulong_t driver_data; /* Data private to the driver */ + __u32 override_only; +}; + +/* pci_epf */ + +#define PCI_EPF_NAME_SIZE 20 +#define PCI_EPF_MODULE_PREFIX "pci_epf:" + +struct pci_epf_device_id { + char name[PCI_EPF_NAME_SIZE]; + kernel_ulong_t driver_data; +}; + +#endif /* ifndef LINUX_DEVICE_ID_PCI_H */ diff --git a/include/linux/device-id/pcmcia.h b/include/linux/device-id/pcmcia.h new file mode 100644 index 000000000000..c0d809be2338 --- /dev/null +++ b/include/linux/device-id/pcmcia.h @@ -0,0 +1,48 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_PCMCIA_H +#define LINUX_DEVICE_ID_PCMCIA_H + +#ifdef __KERNEL__ +#include <linux/types.h> +typedef unsigned long kernel_ulong_t; +#endif + +/* PCMCIA */ + +#define PCMCIA_DEV_ID_MATCH_MANF_ID 0x0001 +#define PCMCIA_DEV_ID_MATCH_CARD_ID 0x0002 +#define PCMCIA_DEV_ID_MATCH_FUNC_ID 0x0004 +#define PCMCIA_DEV_ID_MATCH_FUNCTION 0x0008 +#define PCMCIA_DEV_ID_MATCH_PROD_ID1 0x0010 +#define PCMCIA_DEV_ID_MATCH_PROD_ID2 0x0020 +#define PCMCIA_DEV_ID_MATCH_PROD_ID3 0x0040 +#define PCMCIA_DEV_ID_MATCH_PROD_ID4 0x0080 +#define PCMCIA_DEV_ID_MATCH_DEVICE_NO 0x0100 +#define PCMCIA_DEV_ID_MATCH_FAKE_CIS 0x0200 +#define PCMCIA_DEV_ID_MATCH_ANONYMOUS 0x0400 + +struct pcmcia_device_id { + __u16 match_flags; + + __u16 manf_id; + __u16 card_id; + + __u8 func_id; + + /* for real multi-function devices */ + __u8 function; + + /* for pseudo multi-function devices */ + __u8 device_no; + + __u32 prod_id_hash[4]; + + /* not matched against in kernelspace */ + const char * prod_id[4]; + + /* not matched against */ + kernel_ulong_t driver_info; + char * cisfile; +}; + +#endif /* ifndef LINUX_DEVICE_ID_PCMCIA_H */ diff --git a/include/linux/device-id/platform.h b/include/linux/device-id/platform.h new file mode 100644 index 000000000000..d6beeb4a2574 --- /dev/null +++ b/include/linux/device-id/platform.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_PLATFORM_H +#define LINUX_DEVICE_ID_PLATFORM_H + +#ifdef __KERNEL__ +typedef unsigned long kernel_ulong_t; +#endif + +#define PLATFORM_NAME_SIZE 24 +#define PLATFORM_MODULE_PREFIX "platform:" + +struct platform_device_id { + char name[PLATFORM_NAME_SIZE]; + kernel_ulong_t driver_data; +}; + +#endif /* ifndef LINUX_DEVICE_ID_PLATFORM_H */ diff --git a/include/linux/device-id/pnp.h b/include/linux/device-id/pnp.h new file mode 100644 index 000000000000..325f17216df8 --- /dev/null +++ b/include/linux/device-id/pnp.h @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_PNP_H +#define LINUX_DEVICE_ID_PNP_H + +#ifdef __KERNEL__ +#include <linux/types.h> +typedef unsigned long kernel_ulong_t; +#endif + +#define PNP_ID_LEN 8 +#define PNP_MAX_DEVICES 8 + +struct pnp_device_id { + __u8 id[PNP_ID_LEN]; + kernel_ulong_t driver_data; +}; + +struct pnp_card_device_id { + __u8 id[PNP_ID_LEN]; + kernel_ulong_t driver_data; + struct { + __u8 id[PNP_ID_LEN]; + } devs[PNP_MAX_DEVICES]; +}; + +#endif /* ifndef LINUX_DEVICE_ID_PNP_H */ diff --git a/include/linux/device-id/rio.h b/include/linux/device-id/rio.h new file mode 100644 index 000000000000..31addf69ad1f --- /dev/null +++ b/include/linux/device-id/rio.h @@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_RIO_H +#define LINUX_DEVICE_ID_RIO_H + +#ifdef __KERNEL__ +#include <linux/types.h> +#endif + +/* RapidIO */ + +#define RIO_ANY_ID 0xffff + +/** + * struct rio_device_id - RIO device identifier + * @did: RapidIO device ID + * @vid: RapidIO vendor ID + * @asm_did: RapidIO assembly device ID + * @asm_vid: RapidIO assembly vendor ID + * + * Identifies a RapidIO device based on both the device/vendor IDs and + * the assembly device/vendor IDs. + */ +struct rio_device_id { + __u16 did, vid; + __u16 asm_did, asm_vid; +}; + +#endif /* ifndef LINUX_DEVICE_ID_RIO_H */ diff --git a/include/linux/device-id/rpmsg.h b/include/linux/device-id/rpmsg.h new file mode 100644 index 000000000000..dd53e7b7dc4f --- /dev/null +++ b/include/linux/device-id/rpmsg.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_RPMSG_H +#define LINUX_DEVICE_ID_RPMSG_H + +#ifdef __KERNEL__ +typedef unsigned long kernel_ulong_t; +#endif + +/* rpmsg */ + +#define RPMSG_NAME_SIZE 32 +#define RPMSG_DEVICE_MODALIAS_FMT "rpmsg:%s" + +struct rpmsg_device_id { + char name[RPMSG_NAME_SIZE]; + kernel_ulong_t driver_data; +}; + +#endif /* ifndef LINUX_DEVICE_ID_RPMSG_H */ diff --git a/include/linux/device-id/sdio.h b/include/linux/device-id/sdio.h new file mode 100644 index 000000000000..609d60a10cd3 --- /dev/null +++ b/include/linux/device-id/sdio.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_SDIO_H +#define LINUX_DEVICE_ID_SDIO_H + +#ifdef __KERNEL__ +#include <linux/types.h> +typedef unsigned long kernel_ulong_t; +#endif + +/* SDIO */ + +#define SDIO_ANY_ID (~0) + +struct sdio_device_id { + __u8 class; /* Standard interface or SDIO_ANY_ID */ + __u16 vendor; /* Vendor or SDIO_ANY_ID */ + __u16 device; /* Device ID or SDIO_ANY_ID */ + kernel_ulong_t driver_data; /* Data private to the driver */ +}; + +#endif /* ifndef LINUX_DEVICE_ID_SDIO_H */ diff --git a/include/linux/device-id/sdw.h b/include/linux/device-id/sdw.h new file mode 100644 index 000000000000..dbed7fde1ace --- /dev/null +++ b/include/linux/device-id/sdw.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_SDW_H +#define LINUX_DEVICE_ID_SDW_H + +#ifdef __KERNEL__ +#include <linux/types.h> +typedef unsigned long kernel_ulong_t; +#endif + +struct sdw_device_id { + __u16 mfg_id; + __u16 part_id; + __u8 sdw_version; + __u8 class_id; + kernel_ulong_t driver_data; +}; + +#endif /* ifndef LINUX_DEVICE_ID_SDW_H */ diff --git a/include/linux/device-id/serio.h b/include/linux/device-id/serio.h new file mode 100644 index 000000000000..b4c02adf220c --- /dev/null +++ b/include/linux/device-id/serio.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_SERIO_H +#define LINUX_DEVICE_ID_SERIO_H + +#ifdef __KERNEL__ +#include <linux/types.h> +#endif + +#define SERIO_ANY 0xff + +struct serio_device_id { + __u8 type; + __u8 extra; + __u8 id; + __u8 proto; +}; + +#endif /* ifndef LINUX_DEVICE_ID_SERIO_H */ diff --git a/include/linux/device-id/slim.h b/include/linux/device-id/slim.h new file mode 100644 index 000000000000..54f8abe09e18 --- /dev/null +++ b/include/linux/device-id/slim.h @@ -0,0 +1,23 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_SLIM_H +#define LINUX_DEVICE_ID_SLIM_H + +#ifdef __KERNEL__ +#include <linux/types.h> +typedef unsigned long kernel_ulong_t; +#endif + +/* SLIMbus */ + +#define SLIMBUS_NAME_SIZE 32 +#define SLIMBUS_MODULE_PREFIX "slim:" + +struct slim_device_id { + __u16 manf_id, prod_code; + __u16 dev_index, instance; + + /* Data private to the driver */ + kernel_ulong_t driver_data; +}; + +#endif /* ifndef LINUX_DEVICE_ID_SLIM_H */ diff --git a/include/linux/device-id/spi.h b/include/linux/device-id/spi.h new file mode 100644 index 000000000000..7a77d0d6672b --- /dev/null +++ b/include/linux/device-id/spi.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_SPI_H +#define LINUX_DEVICE_ID_SPI_H + +#ifdef __KERNEL__ +typedef unsigned long kernel_ulong_t; +#endif + +/* spi */ + +#define SPI_NAME_SIZE 32 +#define SPI_MODULE_PREFIX "spi:" + +struct spi_device_id { + char name[SPI_NAME_SIZE]; + kernel_ulong_t driver_data; /* Data private to the driver */ +}; + +#endif /* ifndef LINUX_DEVICE_ID_SPI_H */ diff --git a/include/linux/device-id/spmi.h b/include/linux/device-id/spmi.h new file mode 100644 index 000000000000..a821c42cfc9a --- /dev/null +++ b/include/linux/device-id/spmi.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_SPMI_H +#define LINUX_DEVICE_ID_SPMI_H + +#ifdef __KERNEL__ +typedef unsigned long kernel_ulong_t; +#endif + +#define SPMI_NAME_SIZE 32 +#define SPMI_MODULE_PREFIX "spmi:" + +struct spmi_device_id { + char name[SPMI_NAME_SIZE]; + kernel_ulong_t driver_data; /* Data private to the driver */ +}; + +#endif /* ifndef LINUX_DEVICE_ID_SPMI_H */ diff --git a/include/linux/device-id/ssam.h b/include/linux/device-id/ssam.h new file mode 100644 index 000000000000..550cca115a55 --- /dev/null +++ b/include/linux/device-id/ssam.h @@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_SSAM_H +#define LINUX_DEVICE_ID_SSAM_H + +#ifdef __KERNEL__ +#include <linux/types.h> +typedef unsigned long kernel_ulong_t; +#endif + +/* Surface System Aggregator Module */ + +#define SSAM_MATCH_TARGET 0x1 +#define SSAM_MATCH_INSTANCE 0x2 +#define SSAM_MATCH_FUNCTION 0x4 + +struct ssam_device_id { + __u8 match_flags; + + __u8 domain; + __u8 category; + __u8 target; + __u8 instance; + __u8 function; + + kernel_ulong_t driver_data; +}; + +#endif /* ifndef LINUX_DEVICE_ID_SSAM_H */ diff --git a/include/linux/device-id/ssb.h b/include/linux/device-id/ssb.h new file mode 100644 index 000000000000..678d40828299 --- /dev/null +++ b/include/linux/device-id/ssb.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_SSB_H +#define LINUX_DEVICE_ID_SSB_H + +#ifdef __KERNEL__ +#include <linux/types.h> +#endif + +#define SSB_ANY_VENDOR 0xFFFF +#define SSB_ANY_ID 0xFFFF +#define SSB_ANY_REV 0xFF + +/* SSB core, see drivers/ssb/ */ +struct ssb_device_id { + __u16 vendor; + __u16 coreid; + __u8 revision; + __u8 __pad; +} __attribute__((packed, aligned(2))); + +#define SSB_DEVICE(_vendor, _coreid, _revision) \ + { .vendor = _vendor, .coreid = _coreid, .revision = _revision } + +#endif /* ifndef LINUX_DEVICE_ID_SSB_H */ diff --git a/include/linux/device-id/tb.h b/include/linux/device-id/tb.h new file mode 100644 index 000000000000..4c62edff8bb7 --- /dev/null +++ b/include/linux/device-id/tb.h @@ -0,0 +1,37 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_TB_H +#define LINUX_DEVICE_ID_TB_H + +#ifdef __KERNEL__ +#include <linux/types.h> +typedef unsigned long kernel_ulong_t; +#endif + +#define TBSVC_MATCH_PROTOCOL_KEY 0x0001 +#define TBSVC_MATCH_PROTOCOL_ID 0x0002 +#define TBSVC_MATCH_PROTOCOL_VERSION 0x0004 +#define TBSVC_MATCH_PROTOCOL_REVISION 0x0008 + +/** + * struct tb_service_id - Thunderbolt service identifiers + * @match_flags: Flags used to match the structure + * @protocol_key: Protocol key the service supports + * @protocol_id: Protocol id the service supports + * @protocol_version: Version of the protocol + * @protocol_revision: Revision of the protocol software + * @driver_data: Driver specific data + * + * Thunderbolt XDomain services are exposed as devices where each device + * carries the protocol information the service supports. Thunderbolt + * XDomain service drivers match against that information. + */ +struct tb_service_id { + __u32 match_flags; + char protocol_key[8 + 1]; + __u32 protocol_id; + __u32 protocol_version; + __u32 protocol_revision; + kernel_ulong_t driver_data; +}; + +#endif /* ifndef LINUX_DEVICE_ID_TB_H */ diff --git a/include/linux/device-id/tee_client.h b/include/linux/device-id/tee_client.h new file mode 100644 index 000000000000..ed81f4228185 --- /dev/null +++ b/include/linux/device-id/tee_client.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_TEE_CLIENT_H +#define LINUX_DEVICE_ID_TEE_CLIENT_H + +#ifdef __KERNEL__ +#include <linux/uuid.h> +#endif + +/** + * struct tee_client_device_id - tee based device identifier + * @uuid: For TEE based client devices we use the device uuid as + * the identifier. + */ +struct tee_client_device_id { + uuid_t uuid; +}; + +#endif /* ifndef LINUX_DEVICE_ID_TEE_CLIENT_H */ diff --git a/include/linux/device-id/typec.h b/include/linux/device-id/typec.h new file mode 100644 index 000000000000..dc234733408b --- /dev/null +++ b/include/linux/device-id/typec.h @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_TYPEC_H +#define LINUX_DEVICE_ID_TYPEC_H + +#ifdef __KERNEL__ +#include <linux/types.h> +typedef unsigned long kernel_ulong_t; +#endif + +/* USB Type-C Alternate Modes */ + +#define TYPEC_ANY_MODE 0x7 + +/** + * struct typec_device_id - USB Type-C alternate mode identifiers + * @svid: Standard or Vendor ID + * @mode: Mode index + * @driver_data: Driver specific data + */ +struct typec_device_id { + __u16 svid; + __u8 mode; + kernel_ulong_t driver_data; +}; + +#endif /* ifndef LINUX_DEVICE_ID_TYPEC_H */ diff --git a/include/linux/device-id/ulpi.h b/include/linux/device-id/ulpi.h new file mode 100644 index 000000000000..b5105b4cfacc --- /dev/null +++ b/include/linux/device-id/ulpi.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_ULPI_H +#define LINUX_DEVICE_ID_ULPI_H + +#ifdef __KERNEL__ +#include <linux/types.h> +typedef unsigned long kernel_ulong_t; +#endif + +struct ulpi_device_id { + __u16 vendor; + __u16 product; + kernel_ulong_t driver_data; +}; + +#endif /* ifndef LINUX_DEVICE_ID_ULPI_H */ diff --git a/include/linux/device-id/usb.h b/include/linux/device-id/usb.h new file mode 100644 index 000000000000..a7ce5f6e1106 --- /dev/null +++ b/include/linux/device-id/usb.h @@ -0,0 +1,111 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_USB_H +#define LINUX_DEVICE_ID_USB_H + +#ifdef __KERNEL__ +#include <linux/types.h> +typedef unsigned long kernel_ulong_t; +#endif + +/* + * Device table entry for "new style" table-driven USB drivers. + * User mode code can read these tables to choose which modules to load. + * Declare the table as a MODULE_DEVICE_TABLE. + * + * A probe() parameter will point to a matching entry from this table. + * Use the driver_info field for each match to hold information tied + * to that match: device quirks, etc. + * + * Terminate the driver's table with an all-zeroes entry. + * Use the flag values to control which fields are compared. + */ + +/** + * struct usb_device_id - identifies USB devices for probing and hotplugging + * @match_flags: Bit mask controlling which of the other fields are used to + * match against new devices. Any field except for driver_info may be + * used, although some only make sense in conjunction with other fields. + * This is usually set by a USB_DEVICE_*() macro, which sets all + * other fields in this structure except for driver_info. + * @idVendor: USB vendor ID for a device; numbers are assigned + * by the USB forum to its members. + * @idProduct: Vendor-assigned product ID. + * @bcdDevice_lo: Low end of range of vendor-assigned product version numbers. + * This is also used to identify individual product versions, for + * a range consisting of a single device. + * @bcdDevice_hi: High end of version number range. The range of product + * versions is inclusive. + * @bDeviceClass: Class of device; numbers are assigned + * by the USB forum. Products may choose to implement classes, + * or be vendor-specific. Device classes specify behavior of all + * the interfaces on a device. + * @bDeviceSubClass: Subclass of device; associated with bDeviceClass. + * @bDeviceProtocol: Protocol of device; associated with bDeviceClass. + * @bInterfaceClass: Class of interface; numbers are assigned + * by the USB forum. Products may choose to implement classes, + * or be vendor-specific. Interface classes specify behavior only + * of a given interface; other interfaces may support other classes. + * @bInterfaceSubClass: Subclass of interface; associated with bInterfaceClass. + * @bInterfaceProtocol: Protocol of interface; associated with bInterfaceClass. + * @bInterfaceNumber: Number of interface; composite devices may use + * fixed interface numbers to differentiate between vendor-specific + * interfaces. + * @driver_info: Holds information used by the driver. Usually it holds + * a pointer to a descriptor understood by the driver, or perhaps + * device flags. + * + * In most cases, drivers will create a table of device IDs by using + * USB_DEVICE(), or similar macros designed for that purpose. + * They will then export it to userspace using MODULE_DEVICE_TABLE(), + * and provide it to the USB core through their usb_driver structure. + * + * See the usb_match_id() function for information about how matches are + * performed. Briefly, you will normally use one of several macros to help + * construct these entries. Each entry you provide will either identify + * one or more specific products, or will identify a class of products + * which have agreed to behave the same. You should put the more specific + * matches towards the beginning of your table, so that driver_info can + * record quirks of specific products. + */ +struct usb_device_id { + /* which fields to match against? */ + __u16 match_flags; + + /* Used for product specific matches; range is inclusive */ + __u16 idVendor; + __u16 idProduct; + __u16 bcdDevice_lo; + __u16 bcdDevice_hi; + + /* Used for device class matches */ + __u8 bDeviceClass; + __u8 bDeviceSubClass; + __u8 bDeviceProtocol; + + /* Used for interface class matches */ + __u8 bInterfaceClass; + __u8 bInterfaceSubClass; + __u8 bInterfaceProtocol; + + /* Used for vendor-specific interface matches */ + __u8 bInterfaceNumber; + + /* not matched against */ + kernel_ulong_t driver_info + __attribute__((aligned(sizeof(kernel_ulong_t)))); +}; + +/* Some useful macros to use to create struct usb_device_id */ +#define USB_DEVICE_ID_MATCH_VENDOR 0x0001 +#define USB_DEVICE_ID_MATCH_PRODUCT 0x0002 +#define USB_DEVICE_ID_MATCH_DEV_LO 0x0004 +#define USB_DEVICE_ID_MATCH_DEV_HI 0x0008 +#define USB_DEVICE_ID_MATCH_DEV_CLASS 0x0010 +#define USB_DEVICE_ID_MATCH_DEV_SUBCLASS 0x0020 +#define USB_DEVICE_ID_MATCH_DEV_PROTOCOL 0x0040 +#define USB_DEVICE_ID_MATCH_INT_CLASS 0x0080 +#define USB_DEVICE_ID_MATCH_INT_SUBCLASS 0x0100 +#define USB_DEVICE_ID_MATCH_INT_PROTOCOL 0x0200 +#define USB_DEVICE_ID_MATCH_INT_NUMBER 0x0400 + +#endif /* ifndef LINUX_DEVICE_ID_USB_H */ diff --git a/include/linux/device-id/vchiq.h b/include/linux/device-id/vchiq.h new file mode 100644 index 000000000000..16b1b874c02d --- /dev/null +++ b/include/linux/device-id/vchiq.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_VCHIQ_H +#define LINUX_DEVICE_ID_VCHIQ_H + +struct vchiq_device_id { + char name[32]; +}; + +#endif /* ifndef LINUX_DEVICE_ID_VCHIQ_H */ diff --git a/include/linux/device-id/vio.h b/include/linux/device-id/vio.h new file mode 100644 index 000000000000..8ed7ea9cec07 --- /dev/null +++ b/include/linux/device-id/vio.h @@ -0,0 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_VIO_H +#define LINUX_DEVICE_ID_VIO_H + +/* VIO */ +struct vio_device_id { + char type[32]; + char compat[32]; +}; + +#endif /* ifndef LINUX_DEVICE_ID_VIO_H */ diff --git a/include/linux/device-id/virtio.h b/include/linux/device-id/virtio.h new file mode 100644 index 000000000000..9648a42d4f26 --- /dev/null +++ b/include/linux/device-id/virtio.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_VIRTIO_H +#define LINUX_DEVICE_ID_VIRTIO_H + +#ifdef __KERNEL__ +#include <linux/types.h> +#endif + +#define VIRTIO_DEV_ANY_ID 0xffffffff + +struct virtio_device_id { + __u32 device; + __u32 vendor; +}; + +#endif /* ifndef LINUX_DEVICE_ID_VIRTIO_H */ diff --git a/include/linux/device-id/wmi.h b/include/linux/device-id/wmi.h new file mode 100644 index 000000000000..9f55c83c5203 --- /dev/null +++ b/include/linux/device-id/wmi.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_WMI_H +#define LINUX_DEVICE_ID_WMI_H + +/* WMI */ + +#define WMI_MODULE_PREFIX "wmi:" + +/** + * struct wmi_device_id - WMI device identifier + * @guid_string: 36 char string of the form fa50ff2b-f2e8-45de-83fa-65417f2f49ba + * @context: pointer to driver specific data + */ +struct wmi_device_id { + const char guid_string[UUID_STRING_LEN+1]; + const void *context; +}; + +#endif /* ifndef LINUX_DEVICE_ID_WMI_H */ diff --git a/include/linux/device-id/x86_cpu.h b/include/linux/device-id/x86_cpu.h new file mode 100644 index 000000000000..f44e5253ccca --- /dev/null +++ b/include/linux/device-id/x86_cpu.h @@ -0,0 +1,44 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_X86_CPU_H +#define LINUX_DEVICE_ID_X86_CPU_H + +#ifdef __KERNEL__ +#include <linux/types.h> +typedef unsigned long kernel_ulong_t; +#endif + +/* Wild cards for x86_cpu_id::vendor, family, model and feature */ +#define X86_VENDOR_ANY 0xffff +#define X86_FAMILY_ANY 0 +#define X86_MODEL_ANY 0 +#define X86_STEPPING_ANY 0 +#define X86_STEP_MIN 0 +#define X86_STEP_MAX 0xf +#define X86_PLATFORM_ANY 0x0 +#define X86_FEATURE_ANY 0 /* Same as FPU, you can't test for that */ +#define X86_CPU_TYPE_ANY 0 + +/* + * Match x86 CPUs for CPU specific drivers. + * See documentation of "x86_match_cpu" for details. + */ + +/* + * MODULE_DEVICE_TABLE expects this struct to be called x86cpu_device_id. + * Although gcc seems to ignore this error, clang fails without this define. + */ +#define x86cpu_device_id x86_cpu_id +struct x86_cpu_id { + __u16 vendor; + __u16 family; + __u16 model; + __u16 steppings; + __u16 feature; /* bit index */ + /* Solely for kernel-internal use: DO NOT EXPORT to userspace! */ + __u16 flags; + __u8 platform_mask; + __u8 type; + kernel_ulong_t driver_data; +}; + +#endif /* ifndef LINUX_DEVICE_ID_X86_CPU_H */ diff --git a/include/linux/device-id/zorro.h b/include/linux/device-id/zorro.h new file mode 100644 index 000000000000..26827b9b90b6 --- /dev/null +++ b/include/linux/device-id/zorro.h @@ -0,0 +1,23 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_DEVICE_ID_ZORRO_H +#define LINUX_DEVICE_ID_ZORRO_H + +#ifdef __KERNEL__ +#include <linux/types.h> +typedef unsigned long kernel_ulong_t; +#endif + +#define ZORRO_WILDCARD (0xffffffff) /* not official */ + +#define ZORRO_DEVICE_MODALIAS_FMT "zorro:i%08X" + +struct zorro_device_id { + __u32 id; /* Device ID or ZORRO_WILDCARD */ + union { + /* Data private to the driver */ + kernel_ulong_t driver_data; + const void *driver_data_ptr; + }; +}; + +#endif /* ifndef LINUX_DEVICE_ID_ZORRO_H */ diff --git a/include/linux/device-mapper.h b/include/linux/device-mapper.h index 38f625af6ab4..cd4faaf5d427 100644 --- a/include/linux/device-mapper.h +++ b/include/linux/device-mapper.h @@ -755,4 +755,11 @@ static inline unsigned long to_bytes(sector_t n) return (n << SECTOR_SHIFT); } +static inline void dm_stack_bs_limits(struct queue_limits *limits, unsigned int bs) +{ + limits->logical_block_size = max(limits->logical_block_size, bs); + limits->physical_block_size = max(limits->physical_block_size, bs); + limits->io_min = max(limits->io_min, bs); +} + #endif /* _LINUX_DEVICE_MAPPER_H */ diff --git a/include/linux/device.h b/include/linux/device.h index e65d564f01cd..aee79fd6b32b 100644 --- a/include/linux/device.h +++ b/include/linux/device.h @@ -38,7 +38,6 @@ struct device_private; struct device_driver; struct driver_private; struct module; -struct class; struct subsys_private; struct device_node; struct fwnode_handle; @@ -100,14 +99,24 @@ struct device_type { * struct device_attribute - Interface for exporting device attributes. * @attr: sysfs attribute definition. * @show: Show handler. + * @show_const: Show handler (read-only). * @store: Store handler. + * @store_const: Store handler (read-only). */ struct device_attribute { struct attribute attr; - ssize_t (*show)(struct device *dev, struct device_attribute *attr, - char *buf); - ssize_t (*store)(struct device *dev, struct device_attribute *attr, - const char *buf, size_t count); + __SYSFS_FUNCTION_ALTERNATIVE( + ssize_t (*show)(struct device *dev, struct device_attribute *attr, + char *buf); + ssize_t (*show_const)(struct device *dev, const struct device_attribute *attr, + char *buf); + ); + __SYSFS_FUNCTION_ALTERNATIVE( + ssize_t (*store)(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count); + ssize_t (*store_const)(struct device *dev, const struct device_attribute *attr, + const char *buf, size_t count); + ); }; /** @@ -135,6 +144,77 @@ ssize_t device_store_bool(struct device *dev, struct device_attribute *attr, ssize_t device_show_string(struct device *dev, struct device_attribute *attr, char *buf); +typedef ssize_t __device_show_handler_const(struct device *dev, const struct device_attribute *attr, + char *buf); +typedef ssize_t __device_store_handler_const(struct device *dev, const struct device_attribute *attr, + const char *buf, size_t count); + +#ifdef CONFIG_CFI + +#define __DEVICE_ATTR_SHOW_STORE(_show, _store) \ + .show = _Generic(_show, \ + __device_show_handler_const * : NULL, \ + default : _show \ + ), \ + .show_const = _Generic(_show, \ + __device_show_handler_const * : _show, \ + default : NULL \ + ), \ + .store = _Generic(_store, \ + __device_store_handler_const * : NULL, \ + default : _store \ + ), \ + .store_const = _Generic(_store, \ + __device_store_handler_const * : _store, \ + default : NULL \ + ), + +#else + +#define __DEVICE_ATTR_SHOW_STORE(_show, _store) \ + .show = _Generic(_show, \ + __device_show_handler_const * : (void *)_show, \ + default : _show \ + ), \ + .store = _Generic(_store, \ + __device_store_handler_const * : (void *)_store, \ + default : _store \ + ), \ + +#endif + + +#define __DEVICE_ATTR(_name, _mode, _show, _store) { \ + .attr = {.name = __stringify(_name), \ + .mode = VERIFY_OCTAL_PERMISSIONS(_mode) }, \ + __DEVICE_ATTR_SHOW_STORE(_show, _store) \ +} + +#define __DEVICE_ATTR_RO_MODE(_name, _mode) \ + __DEVICE_ATTR(_name, _mode, _name##_show, NULL) + +#define __DEVICE_ATTR_RO(_name) \ + __DEVICE_ATTR_RO_MODE(_name, 0444) + +#define __DEVICE_ATTR_WO(_name) \ + __DEVICE_ATTR(_name, 0200, NULL, _name##_store) + +#define __DEVICE_ATTR_RW_MODE(_name, _mode) \ + __DEVICE_ATTR(_name, _mode, _name##_show, _name##_store) + +#define __DEVICE_ATTR_RW(_name) \ + __DEVICE_ATTR_RW_MODE(_name, 0644) + +#ifdef CONFIG_DEBUG_LOCK_ALLOC +#define __DEVICE_ATTR_IGNORE_LOCKDEP(_name, _mode, _show, _store) { \ + .attr = {.name = __stringify(_name), .mode = _mode, \ + .ignore_lockdep = true }, \ + __DEVICE_ATTR_SHOW_STORE(_show, _store) \ +} +#else +#define __DEVICE_ATTR_IGNORE_LOCKDEP __DEVICE_ATTR +#endif + /** * DEVICE_ATTR - Define a device attribute. * @_name: Attribute name. @@ -155,20 +235,7 @@ ssize_t device_show_string(struct device *dev, struct device_attribute *attr, * }; */ #define DEVICE_ATTR(_name, _mode, _show, _store) \ - struct device_attribute dev_attr_##_name = __ATTR(_name, _mode, _show, _store) - -/** - * DEVICE_ATTR_PREALLOC - Define a preallocated device attribute. - * @_name: Attribute name. - * @_mode: File mode. - * @_show: Show handler. Optional, but mandatory if attribute is readable. - * @_store: Store handler. Optional, but mandatory if attribute is writable. - * - * Like DEVICE_ATTR(), but ``SYSFS_PREALLOC`` is set on @_mode. - */ -#define DEVICE_ATTR_PREALLOC(_name, _mode, _show, _store) \ - struct device_attribute dev_attr_##_name = \ - __ATTR_PREALLOC(_name, _mode, _show, _store) + struct device_attribute dev_attr_##_name = __DEVICE_ATTR(_name, _mode, _show, _store) /** * DEVICE_ATTR_RW - Define a read-write device attribute. @@ -178,7 +245,7 @@ ssize_t device_show_string(struct device *dev, struct device_attribute *attr, * and @_store is <_name>_store. */ #define DEVICE_ATTR_RW(_name) \ - struct device_attribute dev_attr_##_name = __ATTR_RW(_name) + struct device_attribute dev_attr_##_name = __DEVICE_ATTR_RW(_name) /** * DEVICE_ATTR_ADMIN_RW - Define an admin-only read-write device attribute. @@ -187,7 +254,22 @@ ssize_t device_show_string(struct device *dev, struct device_attribute *attr, * Like DEVICE_ATTR_RW(), but @_mode is 0600. */ #define DEVICE_ATTR_ADMIN_RW(_name) \ - struct device_attribute dev_attr_##_name = __ATTR_RW_MODE(_name, 0600) + struct device_attribute dev_attr_##_name = __DEVICE_ATTR_RW_MODE(_name, 0600) + +/** + * DEVICE_ATTR_RW_NAMED - Define a read-write device attribute with a sysfs name + * that differs from the function name. + * @_name: Attribute function preface + * @_attrname: Attribute name as it wil be exposed in the sysfs. + * + * Like DEVICE_ATTR_RW(), but allows for reusing names under separate paths in + * the same driver. + */ +#define DEVICE_ATTR_RW_NAMED(_name, _attrname) \ + struct device_attribute dev_attr_##_name = { \ + .attr = { .name = _attrname, .mode = 0644 }, \ + __DEVICE_ATTR_SHOW_STORE(_name##_show, _name##_store) \ + } /** * DEVICE_ATTR_RO - Define a readable device attribute. @@ -196,7 +278,7 @@ ssize_t device_show_string(struct device *dev, struct device_attribute *attr, * Like DEVICE_ATTR(), but @_mode is 0444 and @_show is <_name>_show. */ #define DEVICE_ATTR_RO(_name) \ - struct device_attribute dev_attr_##_name = __ATTR_RO(_name) + struct device_attribute dev_attr_##_name = __DEVICE_ATTR_RO(_name) /** * DEVICE_ATTR_ADMIN_RO - Define an admin-only readable device attribute. @@ -205,7 +287,22 @@ ssize_t device_show_string(struct device *dev, struct device_attribute *attr, * Like DEVICE_ATTR_RO(), but @_mode is 0400. */ #define DEVICE_ATTR_ADMIN_RO(_name) \ - struct device_attribute dev_attr_##_name = __ATTR_RO_MODE(_name, 0400) + struct device_attribute dev_attr_##_name = __DEVICE_ATTR_RO_MODE(_name, 0400) + +/** + * DEVICE_ATTR_RO_NAMED - Define a read-only device attribute with a sysfs name + * that differs from the function name. + * @_name: Attribute function preface + * @_attrname: Attribute name as it wil be exposed in the sysfs. + * + * Like DEVICE_ATTR_RO(), but allows for reusing names under separate paths in + * the same driver. + */ +#define DEVICE_ATTR_RO_NAMED(_name, _attrname) \ + struct device_attribute dev_attr_##_name = { \ + .attr = { .name = _attrname, .mode = 0444 }, \ + __DEVICE_ATTR_SHOW_STORE(_name##_show, NULL) \ + } /** * DEVICE_ATTR_WO - Define an admin-only writable device attribute. @@ -214,7 +311,22 @@ ssize_t device_show_string(struct device *dev, struct device_attribute *attr, * Like DEVICE_ATTR(), but @_mode is 0200 and @_store is <_name>_store. */ #define DEVICE_ATTR_WO(_name) \ - struct device_attribute dev_attr_##_name = __ATTR_WO(_name) + struct device_attribute dev_attr_##_name = __DEVICE_ATTR_WO(_name) + +/** + * DEVICE_ATTR_WO_NAMED - Define a read-only device attribute with a sysfs name + * that differs from the function name. + * @_name: Attribute function preface + * @_attrname: Attribute name as it wil be exposed in the sysfs. + * + * Like DEVICE_ATTR_WO(), but allows for reusing names under separate paths in + * the same driver. + */ +#define DEVICE_ATTR_WO_NAMED(_name, _attrname) \ + struct device_attribute dev_attr_##_name = { \ + .attr = { .name = _attrname, .mode = 0200 }, \ + __DEVICE_ATTR_SHOW_STORE(NULL, _name##_store) \ + } /** * DEVICE_ULONG_ATTR - Define a device attribute backed by an unsigned long. @@ -227,7 +339,7 @@ ssize_t device_show_string(struct device *dev, struct device_attribute *attr, */ #define DEVICE_ULONG_ATTR(_name, _mode, _var) \ struct dev_ext_attribute dev_attr_##_name = \ - { __ATTR(_name, _mode, device_show_ulong, device_store_ulong), &(_var) } + { __DEVICE_ATTR(_name, _mode, device_show_ulong, device_store_ulong), &(_var) } /** * DEVICE_INT_ATTR - Define a device attribute backed by an int. @@ -239,7 +351,7 @@ ssize_t device_show_string(struct device *dev, struct device_attribute *attr, */ #define DEVICE_INT_ATTR(_name, _mode, _var) \ struct dev_ext_attribute dev_attr_##_name = \ - { __ATTR(_name, _mode, device_show_int, device_store_int), &(_var) } + { __DEVICE_ATTR(_name, _mode, device_show_int, device_store_int), &(_var) } /** * DEVICE_BOOL_ATTR - Define a device attribute backed by a bool. @@ -251,7 +363,7 @@ ssize_t device_show_string(struct device *dev, struct device_attribute *attr, */ #define DEVICE_BOOL_ATTR(_name, _mode, _var) \ struct dev_ext_attribute dev_attr_##_name = \ - { __ATTR(_name, _mode, device_show_bool, device_store_bool), &(_var) } + { __DEVICE_ATTR(_name, _mode, device_show_bool, device_store_bool), &(_var) } /** * DEVICE_STRING_ATTR_RO - Define a device attribute backed by a r/o string. @@ -264,11 +376,11 @@ ssize_t device_show_string(struct device *dev, struct device_attribute *attr, */ #define DEVICE_STRING_ATTR_RO(_name, _mode, _var) \ struct dev_ext_attribute dev_attr_##_name = \ - { __ATTR(_name, (_mode) & ~0222, device_show_string, NULL), (_var) } + { __DEVICE_ATTR(_name, (_mode) & ~0222, device_show_string, NULL), (_var) } #define DEVICE_ATTR_IGNORE_LOCKDEP(_name, _mode, _show, _store) \ struct device_attribute dev_attr_##_name = \ - __ATTR_IGNORE_LOCKDEP(_name, _mode, _show, _store) + __DEVICE_ATTR_IGNORE_LOCKDEP(_name, _mode, _show, _store) int device_create_file(struct device *device, const struct device_attribute *entry); @@ -459,6 +571,52 @@ struct device_physical_location { }; /** + * enum struct_device_flags - Flags in struct device + * + * Each flag should have a set of accessor functions created via + * __create_dev_flag_accessors() for each access. + * + * @DEV_FLAG_READY_TO_PROBE: If set then device_add() has finished enough + * initialization that probe could be called. + * @DEV_FLAG_CAN_MATCH: The device has matched with a driver at least once or it + * is in a bus (like AMBA) which can't check for matching drivers + * until other devices probe successfully. + * @DEV_FLAG_DMA_IOMMU: Device is using default IOMMU implementation for DMA and + * doesn't rely on dma_ops structure. + * @DEV_FLAG_DMA_SKIP_SYNC: DMA sync operations can be skipped for coherent + * buffers. + * @DEV_FLAG_DMA_OPS_BYPASS: If set then the dma_ops are bypassed for the + * streaming DMA operations (->map_* / ->unmap_* / ->sync_*), and + * optional (if the coherent mask is large enough) also for dma + * allocations. This flag is managed by the dma ops instance from + * ->dma_supported. + * @DEV_FLAG_STATE_SYNCED: The hardware state of this device has been synced to + * match the software state of this device by calling the + * driver/bus sync_state() callback. + * @DEV_FLAG_DMA_COHERENT: This particular device is dma coherent, even if the + * architecture supports non-coherent devices. + * @DEV_FLAG_OF_NODE_REUSED: Set if the device-tree node is shared with an + * ancestor device. + * @DEV_FLAG_OFFLINE_DISABLED: If set, the device is permanently online. + * @DEV_FLAG_OFFLINE: Set after successful invocation of bus type's .offline(). + * @DEV_FLAG_COUNT: Number of defined struct_device_flags. + */ +enum struct_device_flags { + DEV_FLAG_READY_TO_PROBE = 0, + DEV_FLAG_CAN_MATCH = 1, + DEV_FLAG_DMA_IOMMU = 2, + DEV_FLAG_DMA_SKIP_SYNC = 3, + DEV_FLAG_DMA_OPS_BYPASS = 4, + DEV_FLAG_STATE_SYNCED = 5, + DEV_FLAG_DMA_COHERENT = 6, + DEV_FLAG_OF_NODE_REUSED = 7, + DEV_FLAG_OFFLINE_DISABLED = 8, + DEV_FLAG_OFFLINE = 9, + + DEV_FLAG_COUNT +}; + +/** * struct device - The basic device structure * @parent: The device's "parent" device, the device to which it is attached. * In most cases, a parent device is some sort of bus or host @@ -532,27 +690,7 @@ struct device_physical_location { * @removable: Whether the device can be removed from the system. This * should be set by the subsystem / bus driver that discovered * the device. - * - * @offline_disabled: If set, the device is permanently online. - * @offline: Set after successful invocation of bus type's .offline(). - * @of_node_reused: Set if the device-tree node is shared with an ancestor - * device. - * @state_synced: The hardware state of this device has been synced to match - * the software state of this device by calling the driver/bus - * sync_state() callback. - * @can_match: The device has matched with a driver at least once or it is in - * a bus (like AMBA) which can't check for matching drivers until - * other devices probe successfully. - * @dma_coherent: this particular device is dma coherent, even if the - * architecture supports non-coherent devices. - * @dma_ops_bypass: If set to %true then the dma_ops are bypassed for the - * streaming DMA operations (->map_* / ->unmap_* / ->sync_*), - * and optionall (if the coherent mask is large enough) also - * for dma allocations. This flag is managed by the dma ops - * instance from ->dma_supported. - * @dma_skip_sync: DMA sync operations can be skipped for coherent buffers. - * @dma_iommu: Device is using default IOMMU implementation for DMA and - * doesn't rely on dma_ops structure. + * @flags: DEV_FLAG_XXX flags. Use atomic bitfield operations to modify. * * At the lowest level, every device in a Linux system is represented by an * instance of struct device. The device structure contains the information @@ -656,27 +794,44 @@ struct device { enum device_removable removable; - bool offline_disabled:1; - bool offline:1; - bool of_node_reused:1; - bool state_synced:1; - bool can_match:1; -#if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_DEVICE) || \ - defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU) || \ - defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL) - bool dma_coherent:1; -#endif -#ifdef CONFIG_DMA_OPS_BYPASS - bool dma_ops_bypass : 1; -#endif -#ifdef CONFIG_DMA_NEED_SYNC - bool dma_skip_sync:1; -#endif -#ifdef CONFIG_IOMMU_DMA - bool dma_iommu:1; -#endif + DECLARE_BITMAP(flags, DEV_FLAG_COUNT); }; +#define __create_dev_flag_accessors(accessor_name, flag_name) \ +static inline bool dev_##accessor_name(const struct device *dev) \ +{ \ + return test_bit(flag_name, dev->flags); \ +} \ +static inline void dev_set_##accessor_name(struct device *dev) \ +{ \ + set_bit(flag_name, dev->flags); \ +} \ +static inline void dev_clear_##accessor_name(struct device *dev) \ +{ \ + clear_bit(flag_name, dev->flags); \ +} \ +static inline void dev_assign_##accessor_name(struct device *dev, bool value) \ +{ \ + assign_bit(flag_name, dev->flags, value); \ +} \ +static inline bool dev_test_and_set_##accessor_name(struct device *dev) \ +{ \ + return test_and_set_bit(flag_name, dev->flags); \ +} + +__create_dev_flag_accessors(ready_to_probe, DEV_FLAG_READY_TO_PROBE); +__create_dev_flag_accessors(can_match, DEV_FLAG_CAN_MATCH); +__create_dev_flag_accessors(dma_iommu, DEV_FLAG_DMA_IOMMU); +__create_dev_flag_accessors(dma_skip_sync, DEV_FLAG_DMA_SKIP_SYNC); +__create_dev_flag_accessors(dma_ops_bypass, DEV_FLAG_DMA_OPS_BYPASS); +__create_dev_flag_accessors(state_synced, DEV_FLAG_STATE_SYNCED); +__create_dev_flag_accessors(dma_coherent, DEV_FLAG_DMA_COHERENT); +__create_dev_flag_accessors(of_node_reused, DEV_FLAG_OF_NODE_REUSED); +__create_dev_flag_accessors(offline_disabled, DEV_FLAG_OFFLINE_DISABLED); +__create_dev_flag_accessors(offline, DEV_FLAG_OFFLINE); + +#undef __create_dev_flag_accessors + /** * struct device_link - Device link representation. * @supplier: The device on the supplier end of the link. @@ -965,23 +1120,13 @@ static inline void device_unlock(struct device *dev) } DEFINE_GUARD(device, struct device *, device_lock(_T), device_unlock(_T)) +DEFINE_GUARD_COND(device, _intr, device_lock_interruptible(_T), _RET == 0) static inline void device_lock_assert(struct device *dev) { lockdep_assert_held(&dev->mutex); } -static inline bool dev_has_sync_state(struct device *dev) -{ - if (!dev) - return false; - if (dev->driver && dev->driver->sync_state) - return true; - if (dev->bus && dev->bus->sync_state) - return true; - return false; -} - static inline int dev_set_drv_sync_state(struct device *dev, void (*fn)(struct device *dev)) { @@ -1185,9 +1330,9 @@ device_create_with_groups(const struct class *cls, struct device *parent, dev_t void device_destroy(const struct class *cls, dev_t devt); int __must_check device_add_groups(struct device *dev, - const struct attribute_group **groups); + const struct attribute_group *const *groups); void device_remove_groups(struct device *dev, - const struct attribute_group **groups); + const struct attribute_group *const *groups); static inline int __must_check device_add_group(struct device *dev, const struct attribute_group *grp) diff --git a/include/linux/device/bus.h b/include/linux/device/bus.h index c1b463cd6464..a38f7229b8f4 100644 --- a/include/linux/device/bus.h +++ b/include/linux/device/bus.h @@ -83,9 +83,9 @@ struct fwnode_handle; struct bus_type { const char *name; const char *dev_name; - const struct attribute_group **bus_groups; - const struct attribute_group **dev_groups; - const struct attribute_group **drv_groups; + const struct attribute_group *const *bus_groups; + const struct attribute_group *const *dev_groups; + const struct attribute_group *const *drv_groups; int (*match)(struct device *dev, const struct device_driver *drv); int (*uevent)(const struct device *dev, struct kobj_uevent_env *env); diff --git a/include/linux/device/class.h b/include/linux/device/class.h index 65880e60c720..9db1d61ba743 100644 --- a/include/linux/device/class.h +++ b/include/linux/device/class.h @@ -34,7 +34,7 @@ struct fwnode_handle; * @class_release: Called to release this class. * @dev_release: Called to release the device. * @shutdown_pre: Called at shut-down time before driver shutdown. - * @ns_type: Callbacks so sysfs can detemine namespaces. + * @ns_type: Callbacks so sysfs can determine namespaces. * @namespace: Namespace of the device belongs to this class. * @get_ownership: Allows class to specify uid/gid of the sysfs directories * for the devices belonging to the class. Usually tied to @@ -50,8 +50,8 @@ struct fwnode_handle; struct class { const char *name; - const struct attribute_group **class_groups; - const struct attribute_group **dev_groups; + const struct attribute_group *const *class_groups; + const struct attribute_group *const *dev_groups; int (*dev_uevent)(const struct device *dev, struct kobj_uevent_env *env); char *(*devnode)(const struct device *dev, umode_t *mode); @@ -62,7 +62,7 @@ struct class { int (*shutdown_pre)(struct device *dev); const struct kobj_ns_type_operations *ns_type; - const void *(*namespace)(const struct device *dev); + const struct ns_common *(*namespace)(const struct device *dev); void (*get_ownership)(const struct device *dev, kuid_t *uid, kgid_t *gid); @@ -180,9 +180,9 @@ struct class_attribute { struct class_attribute class_attr_##_name = __ATTR_WO(_name) int __must_check class_create_file_ns(const struct class *class, const struct class_attribute *attr, - const void *ns); + const struct ns_common *ns); void class_remove_file_ns(const struct class *class, const struct class_attribute *attr, - const void *ns); + const struct ns_common *ns); static inline int __must_check class_create_file(const struct class *class, const struct class_attribute *attr) diff --git a/include/linux/device/driver.h b/include/linux/device/driver.h index bbc67ec513ed..768a1334c0a1 100644 --- a/include/linux/device/driver.h +++ b/include/linux/device/driver.h @@ -19,6 +19,8 @@ #include <linux/pm.h> #include <linux/device/bus.h> #include <linux/module.h> +#include <linux/device-id/acpi.h> +#include <linux/device-id/of.h> /** * enum probe_type - device driver probe type to try @@ -114,8 +116,8 @@ struct device_driver { void (*shutdown) (struct device *dev); int (*suspend) (struct device *dev, pm_message_t state); int (*resume) (struct device *dev); - const struct attribute_group **groups; - const struct attribute_group **dev_groups; + const struct attribute_group *const *groups; + const struct attribute_group *const *dev_groups; const struct dev_pm_ops *pm; void (*coredump) (struct device *dev); @@ -123,8 +125,8 @@ struct device_driver { struct driver_private *p; struct { /* - * Called after remove() and after all devres entries have been - * processed. This is a Rust only callback. + * Called after remove() but before devres entries are released. + * This is a Rust only callback. */ void (*post_unbind_rust)(struct device *dev); } p_cb; @@ -160,8 +162,6 @@ int __must_check driver_create_file(const struct device_driver *driver, void driver_remove_file(const struct device_driver *driver, const struct driver_attribute *attr); -int driver_set_override(struct device *dev, const char **override, - const char *s, size_t len); int __must_check driver_for_each_device(struct device_driver *drv, struct device *start, void *data, device_iter_t fn); struct device *driver_find_device(const struct device_driver *drv, diff --git a/include/linux/dfl.h b/include/linux/dfl.h index 1f02db0c1897..f28e70652080 100644 --- a/include/linux/dfl.h +++ b/include/linux/dfl.h @@ -9,7 +9,7 @@ #define __LINUX_DFL_H #include <linux/device.h> -#include <linux/mod_devicetable.h> +#include <linux/device-id/dfl.h> /** * enum dfl_id_type - define the DFL FIU types diff --git a/include/linux/dibs.h b/include/linux/dibs.h index c75607f8a5cf..d3e0777f25ae 100644 --- a/include/linux/dibs.h +++ b/include/linux/dibs.h @@ -439,7 +439,7 @@ static inline void *dibs_get_priv(struct dibs_dev *dev, /** * dibs_dev_alloc() - allocate and reference device structure * - * The following fields will be valid upon successful return: dev + * The following fields will be valid upon successful return: dev, lock * NOTE: Use put_device(dibs_get_dev(@dibs)) to give up your reference instead * of freeing @dibs @dev directly once you have successfully called this * function. diff --git a/include/linux/dio.h b/include/linux/dio.h index 464331c4c4a7..894e50b91792 100644 --- a/include/linux/dio.h +++ b/include/linux/dio.h @@ -112,9 +112,7 @@ struct dio_driver { #define DIOII_END 0x20000000 /* end of DIO-II space */ #define DIOII_DEVSIZE 0x00400000 /* size of a DIO-II device */ -/* Highest valid select code. If we add DIO-II support this should become - * 256 for everything except HP320, which only has DIO. - */ +/* highest valid select code */ #define DIO_SCMAX (hp300_model == HP_320 ? 32 : 256) #define DIOII_SCBASE 132 /* lowest DIO-II select code */ #define DIO_SCINHOLE(scode) (((scode) >= 32) && ((scode) < DIOII_SCBASE)) diff --git a/include/linux/dm-io.h b/include/linux/dm-io.h index 7b2968612b7e..674683894064 100644 --- a/include/linux/dm-io.h +++ b/include/linux/dm-io.h @@ -27,7 +27,7 @@ struct page_list { struct page *page; }; -typedef void (*io_notify_fn)(unsigned int long error, void *context); +typedef void (*io_notify_fn)(unsigned long int error, unsigned long int unsup, void *context); enum dm_io_mem_type { DM_IO_PAGE_LIST,/* Page list */ @@ -80,8 +80,8 @@ void dm_io_client_destroy(struct dm_io_client *client); * error occurred doing io to the corresponding region. */ int dm_io(struct dm_io_request *io_req, unsigned int num_regions, - struct dm_io_region *region, unsigned int long *sync_error_bits, - unsigned short ioprio); + struct dm_io_region *region, unsigned long int *sync_error_bits, + unsigned long int *sync_unsup_bits, unsigned short ioprio); #endif /* __KERNEL__ */ #endif /* _LINUX_DM_IO_H */ diff --git a/include/linux/dma-buf.h b/include/linux/dma-buf.h index 133b9e637b55..d15b2b31d3c9 100644 --- a/include/linux/dma-buf.h +++ b/include/linux/dma-buf.h @@ -322,13 +322,13 @@ struct dma_buf { * @vmapping_counter: * * Used internally to refcnt the vmaps returned by dma_buf_vmap(). - * Protected by @lock. + * Protected by @resv. */ unsigned vmapping_counter; /** * @vmap_ptr: - * The current vmap ptr if @vmapping_counter > 0. Protected by @lock. + * The current vmap ptr if @vmapping_counter > 0. Protected by @resv. */ struct iosys_map vmap_ptr; @@ -407,7 +407,7 @@ struct dma_buf { * through the device. * * - Dynamic importers should set fences for any access that they can't - * disable immediately from their &dma_buf_attach_ops.move_notify + * disable immediately from their &dma_buf_attach_ops.invalidate_mappings * callback. * * IMPORTANT: @@ -446,7 +446,7 @@ struct dma_buf_attach_ops { bool allow_peer2peer; /** - * @move_notify: [optional] notification that the DMA-buf is moving + * @invalidate_mappings: [optional] notification that the DMA-buf is moving * * If this callback is provided the framework can avoid pinning the * backing store while mappings exists. @@ -456,14 +456,10 @@ struct dma_buf_attach_ops { * called with this lock held as well. This makes sure that no mapping * is created concurrently with an ongoing move operation. * - * Mappings stay valid and are not directly affected by this callback. - * But the DMA-buf can now be in a different physical location, so all - * mappings should be destroyed and re-created as soon as possible. - * - * New mappings can be created after this callback returns, and will - * point to the new location of the DMA-buf. + * See the kdoc for dma_buf_invalidate_mappings() for details on the + * required behavior. */ - void (*move_notify)(struct dma_buf_attachment *attach); + void (*invalidate_mappings)(struct dma_buf_attachment *attach); }; /** @@ -571,6 +567,7 @@ void dma_buf_unpin(struct dma_buf_attachment *attach); struct dma_buf *dma_buf_export(const struct dma_buf_export_info *exp_info); int dma_buf_fd(struct dma_buf *dmabuf, int flags); +void dma_buf_fd_install(struct dma_buf *dmabuf, int fd); struct dma_buf *dma_buf_get(int fd); void dma_buf_put(struct dma_buf *dmabuf); @@ -578,7 +575,8 @@ struct sg_table *dma_buf_map_attachment(struct dma_buf_attachment *, enum dma_data_direction); void dma_buf_unmap_attachment(struct dma_buf_attachment *, struct sg_table *, enum dma_data_direction); -void dma_buf_move_notify(struct dma_buf *dma_buf); +void dma_buf_invalidate_mappings(struct dma_buf *dma_buf); +bool dma_buf_attach_revocable(struct dma_buf_attachment *attach); int dma_buf_begin_cpu_access(struct dma_buf *dma_buf, enum dma_data_direction dir); int dma_buf_end_cpu_access(struct dma_buf *dma_buf, diff --git a/include/linux/dma-buf/heaps/cma.h b/include/linux/dma-buf/heaps/cma.h deleted file mode 100644 index e751479e21e7..000000000000 --- a/include/linux/dma-buf/heaps/cma.h +++ /dev/null @@ -1,16 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef DMA_BUF_HEAP_CMA_H_ -#define DMA_BUF_HEAP_CMA_H_ - -struct cma; - -#ifdef CONFIG_DMABUF_HEAPS_CMA -int dma_heap_cma_register_heap(struct cma *cma); -#else -static inline int dma_heap_cma_register_heap(struct cma *cma) -{ - return 0; -} -#endif // CONFIG_DMABUF_HEAPS_CMA - -#endif // DMA_BUF_HEAP_CMA_H_ diff --git a/include/linux/dma-direct.h b/include/linux/dma-direct.h index c249912456f9..daa31a1adf7b 100644 --- a/include/linux/dma-direct.h +++ b/include/linux/dma-direct.h @@ -77,6 +77,10 @@ static inline dma_addr_t dma_range_map_max(const struct bus_dma_region *map) #ifndef phys_to_dma_unencrypted #define phys_to_dma_unencrypted phys_to_dma #endif + +#ifndef phys_to_dma_encrypted +#define phys_to_dma_encrypted phys_to_dma +#endif #else static inline dma_addr_t __phys_to_dma(struct device *dev, phys_addr_t paddr) { @@ -90,6 +94,12 @@ static inline dma_addr_t phys_to_dma_unencrypted(struct device *dev, { return dma_addr_unencrypted(__phys_to_dma(dev, paddr)); } + +static inline dma_addr_t phys_to_dma_encrypted(struct device *dev, + phys_addr_t paddr) +{ + return dma_addr_encrypted(__phys_to_dma(dev, paddr)); +} /* * If memory encryption is supported, phys_to_dma will set the memory encryption * bit in the DMA address, and dma_to_phys will clear it. @@ -125,12 +135,20 @@ static inline bool force_dma_unencrypted(struct device *dev) #endif /* CONFIG_ARCH_HAS_FORCE_DMA_UNENCRYPTED */ static inline bool dma_capable(struct device *dev, dma_addr_t addr, size_t size, - bool is_ram) + bool is_ram, unsigned long attrs) { dma_addr_t end = addr + size - 1; if (addr == DMA_MAPPING_ERROR) return false; + /* + * The DMA address was derived from encrypted RAM, but this device + * requires unencrypted DMA addresses. Treat it as not DMA-capable + * so the caller can fall back to a suitable SWIOTLB pool. + */ + if (!(attrs & DMA_ATTR_CC_SHARED) && force_dma_unencrypted(dev)) + return false; + if (is_ram && !IS_ENABLED(CONFIG_ARCH_DMA_ADDR_T_64BIT) && min(addr, end) < phys_to_dma(dev, PFN_PHYS(min_low_pfn))) return false; diff --git a/include/linux/dma-fence-array.h b/include/linux/dma-fence-array.h index 079b3dec0a16..0c49d7ccefb6 100644 --- a/include/linux/dma-fence-array.h +++ b/include/linux/dma-fence-array.h @@ -28,7 +28,6 @@ struct dma_fence_array_cb { /** * struct dma_fence_array - fence to represent an array of fences * @base: fence base class - * @lock: spinlock for fence handling * @num_fences: number of fences in the array * @num_pending: fences in the array still pending * @fences: array of the fences @@ -38,7 +37,6 @@ struct dma_fence_array_cb { struct dma_fence_array { struct dma_fence base; - spinlock_t lock; unsigned num_fences; atomic_t num_pending; struct dma_fence **fences; @@ -82,13 +80,11 @@ to_dma_fence_array(struct dma_fence *fence) struct dma_fence_array *dma_fence_array_alloc(int num_fences); void dma_fence_array_init(struct dma_fence_array *array, int num_fences, struct dma_fence **fences, - u64 context, unsigned seqno, - bool signal_on_any); + u64 context, unsigned seqno); struct dma_fence_array *dma_fence_array_create(int num_fences, struct dma_fence **fences, - u64 context, unsigned seqno, - bool signal_on_any); + u64 context, unsigned seqno); bool dma_fence_match_context(struct dma_fence *fence, u64 context); diff --git a/include/linux/dma-fence-chain.h b/include/linux/dma-fence-chain.h index 5cd3ba53b4a1..705c4394ac0d 100644 --- a/include/linux/dma-fence-chain.h +++ b/include/linux/dma-fence-chain.h @@ -20,7 +20,6 @@ * @prev: previous fence of the chain * @prev_seqno: original previous seqno before garbage collection * @fence: encapsulated fence - * @lock: spinlock for fence handling */ struct dma_fence_chain { struct dma_fence base; @@ -46,7 +45,6 @@ struct dma_fence_chain { */ struct irq_work work; }; - spinlock_t lock; }; @@ -82,9 +80,8 @@ dma_fence_chain_contained(struct dma_fence *fence) } /** - * dma_fence_chain_alloc - * - * Returns a new struct dma_fence_chain object or NULL on failure. + * dma_fence_chain_alloc - Returns a new &struct dma_fence_chain object or + * %NULL on failure. * * This specialized allocator has to be a macro for its allocations to be * accounted separately (to have a separate alloc_tag). The typecast is @@ -94,7 +91,8 @@ dma_fence_chain_contained(struct dma_fence *fence) kmalloc_obj(struct dma_fence_chain) /** - * dma_fence_chain_free + * dma_fence_chain_free - Frees an allocated but not used + * &struct dma_fence_chain object. * @chain: chain node to free * * Frees up an allocated but not used struct dma_fence_chain object. This diff --git a/include/linux/dma-fence-unwrap.h b/include/linux/dma-fence-unwrap.h index 62df222fe0f1..7bfacdf79de2 100644 --- a/include/linux/dma-fence-unwrap.h +++ b/include/linux/dma-fence-unwrap.h @@ -8,6 +8,8 @@ #ifndef __LINUX_DMA_FENCE_UNWRAP_H #define __LINUX_DMA_FENCE_UNWRAP_H +#include <linux/types.h> + struct dma_fence; /** @@ -48,11 +50,11 @@ struct dma_fence *dma_fence_unwrap_next(struct dma_fence_unwrap *cursor); for (fence = dma_fence_unwrap_first(head, cursor); fence; \ fence = dma_fence_unwrap_next(cursor)) -struct dma_fence *__dma_fence_unwrap_merge(unsigned int num_fences, +struct dma_fence *__dma_fence_unwrap_merge(size_t num_fences, struct dma_fence **fences, struct dma_fence_unwrap *cursors); -int dma_fence_dedup_array(struct dma_fence **array, int num_fences); +size_t dma_fence_dedup_array(struct dma_fence **array, size_t num_fences); /** * dma_fence_unwrap_merge - unwrap and merge fences diff --git a/include/linux/dma-fence.h b/include/linux/dma-fence.h index d4c92fd35092..ffa99b930843 100644 --- a/include/linux/dma-fence.h +++ b/include/linux/dma-fence.h @@ -34,7 +34,8 @@ struct seq_file; * @ops: dma_fence_ops associated with this fence * @rcu: used for releasing fence with kfree_rcu * @cb_list: list of all callbacks to call - * @lock: spin_lock_irqsave used for locking + * @extern_lock: external spin_lock_irqsave used for locking (deprecated) + * @inline_lock: alternative internal spin_lock_irqsave used for locking * @context: execution context this fence belongs to, returned by * dma_fence_context_alloc() * @seqno: the sequence number of this fence inside the execution context, @@ -48,6 +49,8 @@ struct seq_file; * atomic ops (bit_*), so taking the spinlock will not be needed most * of the time. * + * DMA_FENCE_FLAG_INITIALIZED_BIT - fence was initialized + * DMA_FENCE_FLAG_INLINE_LOCK_BIT - use inline spinlock instead of external one * DMA_FENCE_FLAG_SIGNALED_BIT - fence is already signaled * DMA_FENCE_FLAG_TIMESTAMP_BIT - timestamp recorded for fence signaling * DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT - enable_signaling might have been called @@ -65,8 +68,11 @@ struct seq_file; * been completed, or never called at all. */ struct dma_fence { - spinlock_t *lock; - const struct dma_fence_ops *ops; + union { + spinlock_t *extern_lock; + spinlock_t inline_lock; + }; + const struct dma_fence_ops __rcu *ops; /* * We clear the callback list on kref_put so that by the time we * release the fence it is unused. No one should be adding to the @@ -98,6 +104,8 @@ struct dma_fence { }; enum dma_fence_flag_bits { + DMA_FENCE_FLAG_INITIALIZED_BIT, + DMA_FENCE_FLAG_INLINE_LOCK_BIT, DMA_FENCE_FLAG_SEQNO64_BIT, DMA_FENCE_FLAG_SIGNALED_BIT, DMA_FENCE_FLAG_TIMESTAMP_BIT, @@ -133,6 +141,9 @@ struct dma_fence_ops { * compute the name at runtime, without having it to store permanently * for each fence, or build a cache of some sort. * + * The returned string is RCU protected and can be freed after the fence + * signaled and a RCU grace period passed. + * * This callback is mandatory. */ const char * (*get_driver_name)(struct dma_fence *fence); @@ -145,6 +156,9 @@ struct dma_fence_ops { * having it to store permanently for each fence, or build a cache of * some sort. * + * The returned string is RCU protected and can be freed after the fence + * signaled and a RCU grace period passed. + * * This callback is mandatory. */ const char * (*get_timeline_name)(struct dma_fence *fence); @@ -218,6 +232,10 @@ struct dma_fence_ops { * timed out. Can also return other error values on custom implementations, * which should be treated as if the fence is signaled. For example a hardware * lockup could be reported like that. + * + * Implementing this callback prevents the fence from detaching after + * signaling and so it is necessary for the module providing the + * dma_fence_ops to stay loaded as long as the dma_fence exists. */ signed long (*wait)(struct dma_fence *fence, bool intr, signed long timeout); @@ -229,6 +247,13 @@ struct dma_fence_ops { * Can be called from irq context. This callback is optional. If it is * NULL, then dma_fence_free() is instead called as the default * implementation. + * + * Implementing this callback prevents the fence from detaching after + * signaling and so it is necessary for the module providing the + * dma_fence_ops to stay loaded as long as the dma_fence exists. + * + * If the callback is implemented the memory backing the dma_fence + * object must be freed RCU safe. */ void (*release)(struct dma_fence *fence); @@ -264,6 +289,19 @@ void dma_fence_free(struct dma_fence *fence); void dma_fence_describe(struct dma_fence *fence, struct seq_file *seq); /** + * dma_fence_was_initialized - test if fence was initialized + * @fence: fence to test + * + * Return: True if fence was ever initialized, false otherwise. Works correctly + * only when memory backing the fence structure is zero initialized on + * allocation. + */ +static inline bool dma_fence_was_initialized(struct dma_fence *fence) +{ + return fence && test_bit(DMA_FENCE_FLAG_INITIALIZED_BIT, &fence->flags); +} + +/** * dma_fence_put - decreases refcount of the fence * @fence: fence to reduce refcount of */ @@ -351,6 +389,45 @@ dma_fence_get_rcu_safe(struct dma_fence __rcu **fencep) } while (1); } +/** + * dma_fence_spinlock - return pointer to the spinlock protecting the fence + * @fence: the fence to get the lock from + * + * Return either the pointer to the embedded or the external spin lock. + */ +static inline spinlock_t *dma_fence_spinlock(struct dma_fence *fence) +{ + return test_bit(DMA_FENCE_FLAG_INLINE_LOCK_BIT, &fence->flags) ? + &fence->inline_lock : fence->extern_lock; +} + +/** + * dma_fence_lock_irqsave - irqsave lock the fence + * @fence: the fence to lock + * @flags: where to store the CPU flags. + * + * Lock the fence, preventing it from changing to the signaled state. + */ +#define dma_fence_lock_irqsave(fence, flags) \ + spin_lock_irqsave(dma_fence_spinlock(fence), flags) + +/** + * dma_fence_unlock_irqrestore - unlock the fence and irqrestore + * @fence: the fence to unlock + * @flags: the CPU flags to restore + * + * Unlock the fence, allowing it to change its state to signaled again. + */ +#define dma_fence_unlock_irqrestore(fence, flags) \ + spin_unlock_irqrestore(dma_fence_spinlock(fence), flags) + +/** + * dma_fence_assert_held - lockdep assertion that fence is locked + * @fence: the fence which should be locked + */ +#define dma_fence_assert_held(fence) \ + lockdep_assert_held(dma_fence_spinlock(fence)); + #ifdef CONFIG_LOCKDEP bool dma_fence_begin_signalling(void); void dma_fence_end_signalling(bool cookie); @@ -377,7 +454,7 @@ int dma_fence_add_callback(struct dma_fence *fence, dma_fence_func_t func); bool dma_fence_remove_callback(struct dma_fence *fence, struct dma_fence_cb *cb); -void dma_fence_enable_sw_signaling(struct dma_fence *fence); +void dma_fence_enable_signaling(struct dma_fence *fence); /** * DOC: Safe external access to driver provided object members @@ -439,13 +516,19 @@ dma_fence_test_signaled_flag(struct dma_fence *fence) static inline bool dma_fence_is_signaled_locked(struct dma_fence *fence) { + const struct dma_fence_ops *ops; + if (dma_fence_test_signaled_flag(fence)) return true; - if (fence->ops->signaled && fence->ops->signaled(fence)) { + rcu_read_lock(); + ops = rcu_dereference(fence->ops); + if (ops && ops->signaled && ops->signaled(fence)) { + rcu_read_unlock(); dma_fence_signal_locked(fence); return true; } + rcu_read_unlock(); return false; } @@ -457,7 +540,7 @@ dma_fence_is_signaled_locked(struct dma_fence *fence) * Returns true if the fence was already signaled, false if not. Since this * function doesn't enable signaling, it is not guaranteed to ever return * true if dma_fence_add_callback(), dma_fence_wait() or - * dma_fence_enable_sw_signaling() haven't been called before. + * dma_fence_enable_signaling() haven't been called before. * * It's recommended for seqno fences to call dma_fence_signal when the * operation is complete, it makes it possible to prevent issues from @@ -469,13 +552,19 @@ dma_fence_is_signaled_locked(struct dma_fence *fence) static inline bool dma_fence_is_signaled(struct dma_fence *fence) { + const struct dma_fence_ops *ops; + if (dma_fence_test_signaled_flag(fence)) return true; - if (fence->ops->signaled && fence->ops->signaled(fence)) { + rcu_read_lock(); + ops = rcu_dereference(fence->ops); + if (ops && ops->signaled && ops->signaled(fence)) { + rcu_read_unlock(); dma_fence_signal(fence); return true; } + rcu_read_unlock(); return false; } @@ -680,7 +769,7 @@ extern const struct dma_fence_ops dma_fence_chain_ops; */ static inline bool dma_fence_is_array(struct dma_fence *fence) { - return fence->ops == &dma_fence_array_ops; + return rcu_access_pointer(fence->ops) == &dma_fence_array_ops; } /** @@ -691,7 +780,7 @@ static inline bool dma_fence_is_array(struct dma_fence *fence) */ static inline bool dma_fence_is_chain(struct dma_fence *fence) { - return fence->ops == &dma_fence_chain_ops; + return rcu_access_pointer(fence->ops) == &dma_fence_chain_ops; } /** diff --git a/include/linux/dma-map-ops.h b/include/linux/dma-map-ops.h index 60b63756df82..8fae2b7deb20 100644 --- a/include/linux/dma-map-ops.h +++ b/include/linux/dma-map-ops.h @@ -91,14 +91,8 @@ static inline void set_dma_ops(struct device *dev, #endif /* CONFIG_ARCH_HAS_DMA_OPS */ #ifdef CONFIG_DMA_CMA -extern struct cma *dma_contiguous_default_area; - -static inline struct cma *dev_get_cma_area(struct device *dev) -{ - if (dev && dev->cma_area) - return dev->cma_area; - return dma_contiguous_default_area; -} +struct cma *dev_get_cma_area(struct device *dev); +struct cma *dma_contiguous_get_area_by_idx(unsigned int idx); void dma_contiguous_reserve(phys_addr_t addr_limit); int __init dma_contiguous_reserve_area(phys_addr_t size, phys_addr_t base, @@ -117,6 +111,10 @@ static inline struct cma *dev_get_cma_area(struct device *dev) { return NULL; } +static inline struct cma *dma_contiguous_get_area_by_idx(unsigned int idx) +{ + return NULL; +} static inline void dma_contiguous_reserve(phys_addr_t limit) { } @@ -147,9 +145,6 @@ static inline void dma_free_contiguous(struct device *dev, struct page *page, { __free_pages(page, get_order(size)); } -static inline void dma_contiguous_early_fixup(phys_addr_t base, unsigned long size) -{ -} #endif /* CONFIG_DMA_CMA*/ #ifdef CONFIG_DMA_DECLARE_COHERENT @@ -217,9 +212,10 @@ void *dma_common_pages_remap(struct page **pages, size_t size, pgprot_t prot, void dma_common_free_remap(void *cpu_addr, size_t size); struct page *dma_alloc_from_pool(struct device *dev, size_t size, - void **cpu_addr, gfp_t flags, + void **cpu_addr, gfp_t flags, unsigned long attrs, bool (*phys_addr_ok)(struct device *, phys_addr_t, size_t)); bool dma_free_from_pool(struct device *dev, void *start, size_t size); +bool dma_free_from_pool_page(struct device *dev, struct page *page, size_t size); int dma_direct_set_offset(struct device *dev, phys_addr_t cpu_start, dma_addr_t dma_start, u64 size); @@ -230,7 +226,7 @@ int dma_direct_set_offset(struct device *dev, phys_addr_t cpu_start, extern bool dma_default_coherent; static inline bool dev_is_dma_coherent(struct device *dev) { - return dev->dma_coherent; + return dev_dma_coherent(dev); } #else #define dma_default_coherent true @@ -245,8 +241,8 @@ static inline void dma_reset_need_sync(struct device *dev) { #ifdef CONFIG_DMA_NEED_SYNC /* Reset it only once so that the function can be called on hotpath */ - if (unlikely(dev->dma_skip_sync)) - dev->dma_skip_sync = false; + if (unlikely(dev_dma_skip_sync(dev))) + dev_clear_dma_skip_sync(dev); #endif } @@ -361,6 +357,12 @@ static inline void arch_sync_dma_for_cpu(phys_addr_t paddr, size_t size, } #endif /* ARCH_HAS_SYNC_DMA_FOR_CPU */ +#ifndef CONFIG_ARCH_HAS_BATCHED_DMA_SYNC +static inline void arch_sync_dma_flush(void) +{ +} +#endif + #ifdef CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL void arch_sync_dma_for_cpu_all(void); #else diff --git a/include/linux/dma-mapping.h b/include/linux/dma-mapping.h index 29973baa0581..a3e880649fa4 100644 --- a/include/linux/dma-mapping.h +++ b/include/linux/dma-mapping.h @@ -9,7 +9,7 @@ #include <linux/bug.h> #include <linux/cache.h> -/** +/* * List of possible attributes associated with a DMA mapping. The semantics * of each attribute should be defined in Documentation/core-api/dma-attributes.rst. */ @@ -80,11 +80,36 @@ #define DMA_ATTR_MMIO (1UL << 10) /* - * DMA_ATTR_CPU_CACHE_CLEAN: Indicates the CPU will not dirty any cacheline - * overlapping this buffer while it is mapped for DMA. All mappings sharing - * a cacheline must have this attribute for this to be considered safe. + * DMA_ATTR_DEBUGGING_IGNORE_CACHELINES: Indicates the CPU cache line can be + * overlapped. All mappings sharing a cacheline must have this attribute for + * this to be considered safe. + */ +#define DMA_ATTR_DEBUGGING_IGNORE_CACHELINES (1UL << 11) + +/* + * DMA_ATTR_REQUIRE_COHERENT: Indicates that DMA coherency is required. + * All mappings that carry this attribute can't work with SWIOTLB and cache + * flushing. + */ +#define DMA_ATTR_REQUIRE_COHERENT (1UL << 12) +/* + * DMA_ATTR_CC_SHARED: Indicates the DMA mapping is shared (decrypted) for + * confidential computing guests. For normal system memory the caller must have + * called set_memory_decrypted(), and pgprot_decrypted must be used when + * creating CPU PTEs for the mapping. The same shared semantic may be passed + * to the vIOMMU when it sets up the IOPTE. For MMIO use together with + * DMA_ATTR_MMIO to indicate shared MMIO. Unless DMA_ATTR_MMIO is provided + * a struct page is required. + */ +#define DMA_ATTR_CC_SHARED (1UL << 13) + +/* + * __DMA_ATTR_ALLOC_CC_SHARED: Internal DMA-mapping attribute used by + * allocation paths that create shared (decrypted) backing pages for + * confidential computing guests. Drivers must not pass this attribute to + * dma_alloc_attrs(). */ -#define DMA_ATTR_CPU_CACHE_CLEAN (1UL << 11) +#define __DMA_ATTR_ALLOC_CC_SHARED (1UL << 14) /* * A dma_addr_t can hold any valid DMA or bus address for the platform. It can @@ -248,8 +273,8 @@ static inline void *dma_alloc_attrs(struct device *dev, size_t size, { return NULL; } -static void dma_free_attrs(struct device *dev, size_t size, void *cpu_addr, - dma_addr_t dma_handle, unsigned long attrs) +static inline void dma_free_attrs(struct device *dev, size_t size, + void *cpu_addr, dma_addr_t dma_handle, unsigned long attrs) { } static inline void *dmam_alloc_attrs(struct device *dev, size_t size, @@ -412,7 +437,7 @@ bool __dma_need_sync(struct device *dev, dma_addr_t dma_addr); static inline bool dma_dev_need_sync(const struct device *dev) { /* Always call DMA sync operations when debugging is enabled */ - return !dev->dma_skip_sync || IS_ENABLED(CONFIG_DMA_API_DEBUG); + return !dev_dma_skip_sync(dev) || IS_ENABLED(CONFIG_DMA_API_DEBUG); } static inline void dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr, diff --git a/include/linux/dma/edma.h b/include/linux/dma/edma.h index 270b5458aecf..3c8e2ef9dee0 100644 --- a/include/linux/dma/edma.h +++ b/include/linux/dma/edma.h @@ -14,6 +14,8 @@ #define EDMA_MAX_WR_CH 8 #define EDMA_MAX_RD_CH 8 +#define HDMA_MAX_WR_CH 64 +#define HDMA_MAX_RD_CH 64 struct dw_edma; @@ -55,9 +57,48 @@ enum dw_edma_map_format { /** * enum dw_edma_chip_flags - Flags specific to an eDMA chip * @DW_EDMA_CHIP_LOCAL: eDMA is used locally by an endpoint + * @DW_EDMA_CHIP_PARTIAL: Only channels described by this instance are + * owned by this driver. Controller-wide state + * must be preserved, and layouts with shared + * direction-wide registers must only be shared at + * direction granularity. Layouts with per-channel + * registers may be shared at channel granularity. */ enum dw_edma_chip_flags { DW_EDMA_CHIP_LOCAL = BIT(0), + DW_EDMA_CHIP_PARTIAL = BIT(1), +}; + +/** + * enum dw_edma_ch_irq_mode - per-channel interrupt routing control + * @DW_EDMA_CH_IRQ_LOCAL: local interrupt only (edma_int[]) + * @DW_EDMA_CH_IRQ_REMOTE: remote interrupt only (IMWr/MSI), without + * delivering local edma_int[]. + * + * DesignWare EP eDMA can signal interrupts locally through the edma_int[] + * bus, and remotely using posted memory writes (IMWr) that may be + * interpreted as MSI/MSI-X by the RC. + * + * For the v0 eDMA linked-list programming path, DMA_*_INT_MASK gates the local + * edma_int[] assertion, while there is no dedicated per-channel mask for IMWr + * generation. To request a remote-only interrupt, Synopsys recommends setting + * both LIE and RIE, and masking the local interrupt in DMA_*_INT_MASK. See the + * DesignWare endpoint databook 6.30a, Linked List Mode interrupt handling + * ("Software Programming of an Endpoint's LIE and RIE Bits for Linked List + * Transfers", Attention). + * + * A local (DW_EDMA_CHIP_LOCAL) instance never issues transfers on a + * remote-routed channel: REMOTE routing on such an instance denotes a channel + * handed over to and driven by the remote side, and the recipe above is + * applied by the driving instance. + * + * HDMA linked-list watermark interrupts have the same LWIE/RWIE guidance. HDMA + * non-linked-list mode has dedicated local and remote stop/abort interrupt + * enables. + */ +enum dw_edma_ch_irq_mode { + DW_EDMA_CH_IRQ_LOCAL = 0, + DW_EDMA_CH_IRQ_REMOTE, }; /** @@ -73,7 +114,10 @@ enum dw_edma_chip_flags { * @ll_region_rd: DMA descriptor link list memory for read channel * @dt_region_wr: DMA data memory for write channel * @dt_region_rd: DMA data memory for read channel + * @db_irq: Virtual IRQ dedicated to interrupt emulation + * @db_offset: Offset from DMA register base * @mf: DMA register map format + * @func_no: PCI endpoint function number used by DMA TLPs * @dw: struct dw_edma that is filled by dw_edma_probe() */ struct dw_edma_chip { @@ -87,16 +131,22 @@ struct dw_edma_chip { u16 ll_wr_cnt; u16 ll_rd_cnt; /* link list address */ - struct dw_edma_region ll_region_wr[EDMA_MAX_WR_CH]; - struct dw_edma_region ll_region_rd[EDMA_MAX_RD_CH]; + struct dw_edma_region ll_region_wr[HDMA_MAX_WR_CH]; + struct dw_edma_region ll_region_rd[HDMA_MAX_RD_CH]; /* data region */ - struct dw_edma_region dt_region_wr[EDMA_MAX_WR_CH]; - struct dw_edma_region dt_region_rd[EDMA_MAX_RD_CH]; + struct dw_edma_region dt_region_wr[HDMA_MAX_WR_CH]; + struct dw_edma_region dt_region_rd[HDMA_MAX_RD_CH]; + + /* interrupt emulation */ + int db_irq; + resource_size_t db_offset; enum dw_edma_map_format mf; + u8 func_no; struct dw_edma *dw; + bool cfg_non_ll; }; /* Export to the platform drivers */ diff --git a/include/linux/dma/qcom-gpi-dma.h b/include/linux/dma/qcom-gpi-dma.h index 6680dd1a43c6..332be28427e4 100644 --- a/include/linux/dma/qcom-gpi-dma.h +++ b/include/linux/dma/qcom-gpi-dma.h @@ -8,6 +8,9 @@ /** * enum spi_transfer_cmd - spi transfer commands + * @SPI_TX: SPI peripheral TX command + * @SPI_RX: SPI peripheral RX command + * @SPI_DUPLEX: SPI peripheral Duplex command */ enum spi_transfer_cmd { SPI_TX = 1, @@ -64,7 +67,7 @@ enum i2c_op { * @set_config: set peripheral config * @rx_len: receive length for buffer * @op: i2c cmd - * @muli-msg: is part of multi i2c r-w msgs + * @multi_msg: is part of multi i2c r-w msgs */ struct gpi_i2c_config { u8 set_config; diff --git a/include/linux/dma/qcom_bam_dma.h b/include/linux/dma/qcom_bam_dma.h index 68fc0e643b1b..d9d07a9ab313 100644 --- a/include/linux/dma/qcom_bam_dma.h +++ b/include/linux/dma/qcom_bam_dma.h @@ -13,9 +13,12 @@ * supported by BAM DMA Engine. * * @cmd_and_addr - upper 8 bits command and lower 24 bits register address. - * @data - for write command: content to be written into peripheral register. - * for read command: dest addr to write peripheral register value. - * @mask - register mask. + * @data - For write command: content to be written into peripheral register. + * For read command: lower 32 bits of destination address. + * @mask - For write command: register write mask. + * For read command on BAM v1.6.0+: upper 4 bits of destination address. + * For read command on BAM < v1.6.0: ignored by hardware. + * Setting to 0 ensures 32-bit addressing compatibility. * @reserved - for future usage. * */ @@ -42,6 +45,10 @@ enum bam_command_type { * @addr: target address * @cmd: BAM command * @data: actual data for write and dest addr for read in le32 + * + * For BAM v1.6.0+, the mask field behavior depends on command type: + * - Write commands: mask = write mask (typically 0xffffffff) + * - Read commands: mask = upper 4 bits of destination address (0 for 32-bit) */ static inline void bam_prep_ce_le32(struct bam_cmd_element *bam_ce, u32 addr, @@ -50,7 +57,11 @@ bam_prep_ce_le32(struct bam_cmd_element *bam_ce, u32 addr, bam_ce->cmd_and_addr = cpu_to_le32((addr & 0xffffff) | ((cmd & 0xff) << 24)); bam_ce->data = data; - bam_ce->mask = cpu_to_le32(0xffffffff); + if (cmd == BAM_READ_COMMAND) + bam_ce->mask = cpu_to_le32(0x0); /* 32-bit addressing */ + else + bam_ce->mask = cpu_to_le32(0xffffffff); /* Write mask */ + bam_ce->reserved = 0; } /* @@ -60,7 +71,7 @@ bam_prep_ce_le32(struct bam_cmd_element *bam_ce, u32 addr, * @bam_ce: BAM command element * @addr: target address * @cmd: BAM command - * @data: actual data for write and dest addr for read + * @data: actual data for write and destination address for read */ static inline void bam_prep_ce(struct bam_cmd_element *bam_ce, u32 addr, diff --git a/include/linux/dma/ti-cppi5.h b/include/linux/dma/ti-cppi5.h index c53c0f6e3b1a..3fe19b75ddf7 100644 --- a/include/linux/dma/ti-cppi5.h +++ b/include/linux/dma/ti-cppi5.h @@ -16,8 +16,8 @@ * struct cppi5_desc_hdr_t - Descriptor header, present in all types of * descriptors * @pkt_info0: Packet info word 0 (n/a in Buffer desc) - * @pkt_info0: Packet info word 1 (n/a in Buffer desc) - * @pkt_info0: Packet info word 2 (n/a in Buffer desc) + * @pkt_info1: Packet info word 1 (n/a in Buffer desc) + * @pkt_info2: Packet info word 2 (n/a in Buffer desc) * @src_dst_tag: Packet info word 3 (n/a in Buffer desc) */ struct cppi5_desc_hdr_t { @@ -35,7 +35,7 @@ struct cppi5_desc_hdr_t { * @buf_info1: word 8: Buffer valid data length * @org_buf_len: word 9: Original buffer length * @org_buf_ptr: word 10/11: Original buffer pointer - * @epib[0]: Extended Packet Info Data (optional, 4 words), and/or + * @epib: Extended Packet Info Data (optional, 4 words), and/or * Protocol Specific Data (optional, 0-128 bytes in * multiples of 4), and/or * Other Software Data (0-N bytes, optional) @@ -132,7 +132,7 @@ struct cppi5_desc_epib_t { /** * struct cppi5_monolithic_desc_t - Monolithic-mode packet descriptor * @hdr: Descriptor header - * @epib[0]: Extended Packet Info Data (optional, 4 words), and/or + * @epib: Extended Packet Info Data (optional, 4 words), and/or * Protocol Specific Data (optional, 0-128 bytes in * multiples of 4), and/or * Other Software Data (0-N bytes, optional) @@ -179,7 +179,7 @@ static inline void cppi5_desc_dump(void *desc, u32 size) * cppi5_desc_is_tdcm - check if the paddr indicates Teardown Complete Message * @paddr: Physical address of the packet popped from the ring * - * Returns true if the address indicates TDCM + * Returns: true if the address indicates TDCM */ static inline bool cppi5_desc_is_tdcm(dma_addr_t paddr) { @@ -190,7 +190,7 @@ static inline bool cppi5_desc_is_tdcm(dma_addr_t paddr) * cppi5_desc_get_type - get descriptor type * @desc_hdr: packet descriptor/TR header * - * Returns descriptor type: + * Returns: descriptor type: * CPPI5_INFO0_DESC_TYPE_VAL_HOST * CPPI5_INFO0_DESC_TYPE_VAL_MONO * CPPI5_INFO0_DESC_TYPE_VAL_TR @@ -205,7 +205,7 @@ static inline u32 cppi5_desc_get_type(struct cppi5_desc_hdr_t *desc_hdr) * cppi5_desc_get_errflags - get Error Flags from Desc * @desc_hdr: packet/TR descriptor header * - * Returns Error Flags from Packet/TR Descriptor + * Returns: Error Flags from Packet/TR Descriptor */ static inline u32 cppi5_desc_get_errflags(struct cppi5_desc_hdr_t *desc_hdr) { @@ -307,7 +307,7 @@ static inline void cppi5_desc_set_tags_ids(struct cppi5_desc_hdr_t *desc_hdr, * @psdata_size: PSDATA size * @sw_data_size: SWDATA size * - * Returns required Host Packet Descriptor size + * Returns: required Host Packet Descriptor size * 0 - if PSDATA > CPPI5_INFO0_HDESC_PSDATA_MAX_SIZE */ static inline u32 cppi5_hdesc_calc_size(bool epib, u32 psdata_size, @@ -381,6 +381,8 @@ cppi5_hdesc_update_psdata_size(struct cppi5_host_desc_t *desc, u32 psdata_size) /** * cppi5_hdesc_get_psdata_size - get PSdata size in bytes * @desc: Host packet descriptor + * + * Returns: PSdata size in bytes */ static inline u32 cppi5_hdesc_get_psdata_size(struct cppi5_host_desc_t *desc) { @@ -398,7 +400,7 @@ static inline u32 cppi5_hdesc_get_psdata_size(struct cppi5_host_desc_t *desc) * cppi5_hdesc_get_pktlen - get Packet Length from HDesc * @desc: Host packet descriptor * - * Returns Packet Length from Host Packet Descriptor + * Returns: Packet Length from Host Packet Descriptor */ static inline u32 cppi5_hdesc_get_pktlen(struct cppi5_host_desc_t *desc) { @@ -408,6 +410,7 @@ static inline u32 cppi5_hdesc_get_pktlen(struct cppi5_host_desc_t *desc) /** * cppi5_hdesc_set_pktlen - set Packet Length in HDesc * @desc: Host packet descriptor + * @pkt_len: Packet length to set */ static inline void cppi5_hdesc_set_pktlen(struct cppi5_host_desc_t *desc, u32 pkt_len) @@ -420,7 +423,7 @@ static inline void cppi5_hdesc_set_pktlen(struct cppi5_host_desc_t *desc, * cppi5_hdesc_get_psflags - get Protocol Specific Flags from HDesc * @desc: Host packet descriptor * - * Returns Protocol Specific Flags from Host Packet Descriptor + * Returns: Protocol Specific Flags from Host Packet Descriptor */ static inline u32 cppi5_hdesc_get_psflags(struct cppi5_host_desc_t *desc) { @@ -431,6 +434,7 @@ static inline u32 cppi5_hdesc_get_psflags(struct cppi5_host_desc_t *desc) /** * cppi5_hdesc_set_psflags - set Protocol Specific Flags in HDesc * @desc: Host packet descriptor + * @ps_flags: Protocol Specific flags to set */ static inline void cppi5_hdesc_set_psflags(struct cppi5_host_desc_t *desc, u32 ps_flags) @@ -442,8 +446,10 @@ static inline void cppi5_hdesc_set_psflags(struct cppi5_host_desc_t *desc, } /** - * cppi5_hdesc_get_errflags - get Packet Type from HDesc + * cppi5_hdesc_get_pkttype - get Packet Type from HDesc * @desc: Host packet descriptor + * + * Returns: Packet type */ static inline u32 cppi5_hdesc_get_pkttype(struct cppi5_host_desc_t *desc) { @@ -452,7 +458,7 @@ static inline u32 cppi5_hdesc_get_pkttype(struct cppi5_host_desc_t *desc) } /** - * cppi5_hdesc_get_errflags - set Packet Type in HDesc + * cppi5_hdesc_set_pkttype - set Packet Type in HDesc * @desc: Host packet descriptor * @pkt_type: Packet Type */ @@ -501,7 +507,7 @@ static inline void cppi5_hdesc_reset_to_original(struct cppi5_host_desc_t *desc) /** * cppi5_hdesc_link_hbdesc - link Host Buffer Descriptor to HDesc * @desc: Host Packet Descriptor - * @buf_desc: Host Buffer Descriptor physical address + * @hbuf_desc: Host Buffer Descriptor physical address * * add and link Host Buffer Descriptor to HDesc */ @@ -527,7 +533,7 @@ static inline void cppi5_hdesc_reset_hbdesc(struct cppi5_host_desc_t *desc) * cppi5_hdesc_epib_present - check if EPIB present * @desc_hdr: packet descriptor/TR header * - * Returns true if EPIB present in the packet + * Returns: true if EPIB present in the packet */ static inline bool cppi5_hdesc_epib_present(struct cppi5_desc_hdr_t *desc_hdr) { @@ -538,7 +544,7 @@ static inline bool cppi5_hdesc_epib_present(struct cppi5_desc_hdr_t *desc_hdr) * cppi5_hdesc_get_psdata - Get pointer on PSDATA * @desc: Host packet descriptor * - * Returns pointer on PSDATA in HDesc. + * Returns: pointer on PSDATA in HDesc. * NULL - if ps_data placed at the start of data buffer. */ static inline void *cppi5_hdesc_get_psdata(struct cppi5_host_desc_t *desc) @@ -568,7 +574,7 @@ static inline void *cppi5_hdesc_get_psdata(struct cppi5_host_desc_t *desc) * cppi5_hdesc_get_swdata - Get pointer on swdata * @desc: Host packet descriptor * - * Returns pointer on SWDATA in HDesc. + * Returns: pointer on SWDATA in HDesc. * NOTE. It's caller responsibility to be sure hdesc actually has swdata. */ static inline void *cppi5_hdesc_get_swdata(struct cppi5_host_desc_t *desc) @@ -648,6 +654,7 @@ enum cppi5_tr_types { CPPI5_TR_TYPE11, /* type12-14: Reserved */ CPPI5_TR_TYPE15 = 15, + /* private: */ CPPI5_TR_TYPE_MAX }; @@ -673,6 +680,7 @@ enum cppi5_tr_event_size { CPPI5_TR_EVENT_SIZE_ICNT1_DEC, CPPI5_TR_EVENT_SIZE_ICNT2_DEC, CPPI5_TR_EVENT_SIZE_ICNT3_DEC, + /* private: */ CPPI5_TR_EVENT_SIZE_MAX }; @@ -690,6 +698,7 @@ enum cppi5_tr_trigger { CPPI5_TR_TRIGGER_GLOBAL0, CPPI5_TR_TRIGGER_GLOBAL1, CPPI5_TR_TRIGGER_LOCAL_EVENT, + /* private: */ CPPI5_TR_TRIGGER_MAX }; @@ -711,6 +720,7 @@ enum cppi5_tr_trigger_type { CPPI5_TR_TRIGGER_TYPE_ICNT2_DEC, CPPI5_TR_TRIGGER_TYPE_ICNT3_DEC, CPPI5_TR_TRIGGER_TYPE_ALL, + /* private: */ CPPI5_TR_TRIGGER_TYPE_MAX }; @@ -815,7 +825,7 @@ struct cppi5_tr_type3_t { * destination * @dicnt1: Total loop iteration count for level 1 for destination * @dicnt2: Total loop iteration count for level 2 for destination - * @sicnt3: Total loop iteration count for level 3 (outermost) for + * @dicnt3: Total loop iteration count for level 3 (outermost) for * destination */ struct cppi5_tr_type15_t { @@ -887,6 +897,7 @@ enum cppi5_tr_resp_status_type { CPPI5_TR_RESPONSE_STATUS_UNSUPPORTED_ERR, CPPI5_TR_RESPONSE_STATUS_TRANSFER_EXCEPTION, CPPI5_TR_RESPONSE_STATUS__TEARDOWN_FLUSH, + /* private: */ CPPI5_TR_RESPONSE_STATUS_MAX }; @@ -903,6 +914,7 @@ enum cppi5_tr_resp_status_submission { CPPI5_TR_RESPONSE_STATUS_SUBMISSION_ICNT0, CPPI5_TR_RESPONSE_STATUS_SUBMISSION_FIFO_FULL, CPPI5_TR_RESPONSE_STATUS_SUBMISSION_OWN, + /* private: */ CPPI5_TR_RESPONSE_STATUS_SUBMISSION_MAX }; @@ -931,6 +943,7 @@ enum cppi5_tr_resp_status_unsupported { CPPI5_TR_RESPONSE_STATUS_UNSUPPORTED_DFMT, CPPI5_TR_RESPONSE_STATUS_UNSUPPORTED_SECTR, CPPI5_TR_RESPONSE_STATUS_UNSUPPORTED_AMODE_SPECIFIC, + /* private: */ CPPI5_TR_RESPONSE_STATUS_UNSUPPORTED_MAX }; @@ -939,7 +952,7 @@ enum cppi5_tr_resp_status_unsupported { * @tr_count: number of TR records * @tr_size: Nominal size of TR record (max) [16, 32, 64, 128] * - * Returns required TR Descriptor size + * Returns: required TR Descriptor size */ static inline size_t cppi5_trdesc_calc_size(u32 tr_count, u32 tr_size) { @@ -955,7 +968,7 @@ static inline size_t cppi5_trdesc_calc_size(u32 tr_count, u32 tr_size) /** * cppi5_trdesc_init - Init TR Descriptor - * @desc: TR Descriptor + * @desc_hdr: TR Descriptor * @tr_count: number of TR records * @tr_size: Nominal size of TR record (max) [16, 32, 64, 128] * @reload_idx: Absolute index to jump to on the 2nd and following passes @@ -1044,7 +1057,7 @@ static inline void cppi5_tr_set_trigger(cppi5_tr_flags_t *flags, } /** - * cppi5_tr_cflag_set - Update the Configuration specific flags + * cppi5_tr_csf_set - Update the Configuration specific flags * @flags: Pointer to the TR's flags * @csf: Configuration specific flags * diff --git a/include/linux/dmaengine.h b/include/linux/dmaengine.h index 99efe2b9b4ea..fe33a20abc61 100644 --- a/include/linux/dmaengine.h +++ b/include/linux/dmaengine.h @@ -322,6 +322,8 @@ struct dma_router { * @slave: ptr to the device using this channel * @cookie: last cookie value returned to client * @completed_cookie: last completed cookie for this channel + * @lock: protect between config and prepare transfer when driver have not + * implemented callback device_prep_config_sg(). * @chan_id: channel ID for sysfs * @dev: class device for sysfs * @name: backlink name for sysfs @@ -341,6 +343,12 @@ struct dma_chan { dma_cookie_t cookie; dma_cookie_t completed_cookie; + /* + * protect between config and prepare transfer because *_prep() may be + * called from complete callback, which is in GFP_NOSLEEP context. + */ + spinlock_t lock; + /* sysfs */ int chan_id; struct dma_chan_dev *dev; @@ -623,7 +631,7 @@ struct dma_async_tx_descriptor { void *callback_param; struct dmaengine_unmap_data *unmap; enum dma_desc_metadata_mode desc_metadata_mode; - struct dma_descriptor_metadata_ops *metadata_ops; + const struct dma_descriptor_metadata_ops *metadata_ops; #ifdef CONFIG_ASYNC_TX_ENABLE_CHANNEL_SWITCH struct dma_async_tx_descriptor *next; struct dma_async_tx_descriptor *parent; @@ -835,6 +843,7 @@ struct dma_filter { * where the address and size of each segment is located in one entry of * the dma_vec array. * @device_prep_slave_sg: prepares a slave dma operation + * @device_prep_config_sg: prepares a slave DMA operation with dma_slave_config * @device_prep_dma_cyclic: prepare a cyclic dma operation suitable for audio. * The function takes a buffer of size buf_len. The callback function will * be called after period_len bytes have been transferred. @@ -934,6 +943,10 @@ struct dma_device { struct dma_chan *chan, struct scatterlist *sgl, unsigned int sg_len, enum dma_transfer_direction direction, unsigned long flags, void *context); + struct dma_async_tx_descriptor *(*device_prep_config_sg)( + struct dma_chan *chan, struct scatterlist *sgl, + unsigned int sg_len, enum dma_transfer_direction direction, + unsigned long flags, struct dma_slave_config *config); struct dma_async_tx_descriptor *(*device_prep_dma_cyclic)( struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len, size_t period_len, enum dma_transfer_direction direction, @@ -974,29 +987,52 @@ static inline bool is_slave_direction(enum dma_transfer_direction direction) (direction == DMA_DEV_TO_DEV); } -static inline struct dma_async_tx_descriptor *dmaengine_prep_slave_single( - struct dma_chan *chan, dma_addr_t buf, size_t len, - enum dma_transfer_direction dir, unsigned long flags) +static inline struct dma_async_tx_descriptor * +dmaengine_prep_config_single(struct dma_chan *chan, dma_addr_t buf, size_t len, + enum dma_transfer_direction dir, + unsigned long flags, + struct dma_slave_config *config) { struct scatterlist sg; + + if (!chan || !chan->device) + return NULL; + sg_init_table(&sg, 1); sg_dma_address(&sg) = buf; sg_dma_len(&sg) = len; - if (!chan || !chan->device || !chan->device->device_prep_slave_sg) + if (chan->device->device_prep_config_sg) + return chan->device->device_prep_config_sg(chan, &sg, 1, dir, + flags, config); + + if (config) + if (dmaengine_slave_config(chan, config)) + return NULL; + + if (!chan->device->device_prep_slave_sg) return NULL; return chan->device->device_prep_slave_sg(chan, &sg, 1, dir, flags, NULL); } +static inline struct dma_async_tx_descriptor * +dmaengine_prep_slave_single(struct dma_chan *chan, dma_addr_t buf, size_t len, + enum dma_transfer_direction dir, + unsigned long flags) +{ + return dmaengine_prep_config_single(chan, buf, len, dir, flags, NULL); +} + /** * dmaengine_prep_peripheral_dma_vec() - Prepare a DMA scatter-gather descriptor * @chan: The channel to be used for this descriptor * @vecs: The array of DMA vectors that should be transferred * @nents: The number of DMA vectors in the array * @dir: Specifies the direction of the data transfer - * @flags: DMA engine flags + * @flags: DMA engine flags - DMA_PREP_REPEAT can be used to mark a cyclic + * DMA transfer */ static inline struct dma_async_tx_descriptor *dmaengine_prep_peripheral_dma_vec( struct dma_chan *chan, const struct dma_vec *vecs, size_t nents, @@ -1009,17 +1045,115 @@ static inline struct dma_async_tx_descriptor *dmaengine_prep_peripheral_dma_vec( dir, flags); } -static inline struct dma_async_tx_descriptor *dmaengine_prep_slave_sg( - struct dma_chan *chan, struct scatterlist *sgl, unsigned int sg_len, - enum dma_transfer_direction dir, unsigned long flags) +static inline struct dma_async_tx_descriptor * +dmaengine_prep_config_sg(struct dma_chan *chan, struct scatterlist *sgl, + unsigned int sg_len, enum dma_transfer_direction dir, + unsigned long flags, struct dma_slave_config *config) { - if (!chan || !chan->device || !chan->device->device_prep_slave_sg) + if (!chan || !chan->device) + return NULL; + + if (chan->device->device_prep_config_sg) + return chan->device->device_prep_config_sg(chan, sgl, sg_len, + dir, flags, config); + + if (config) + if (dmaengine_slave_config(chan, config)) + return NULL; + + if (!chan->device->device_prep_slave_sg) return NULL; return chan->device->device_prep_slave_sg(chan, sgl, sg_len, dir, flags, NULL); } +static inline struct dma_async_tx_descriptor * +dmaengine_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, + unsigned int sg_len, enum dma_transfer_direction dir, + unsigned long flags) +{ + return dmaengine_prep_config_sg(chan, sgl, sg_len, dir, flags, NULL); +} + +/** + * dmaengine_prep_config_sg_safe - prepare a scatter-gather DMA transfer + * with atomic slave configuration update + * @chan: DMA channel + * @sgl: scatterlist for the transfer + * @sg_len: number of entries in @sgl + * @dir: DMA transfer direction + * @flags: transfer preparation flags + * @config: DMA slave configuration for this transfer + * + * Prepare a DMA scatter-gather transfer together with a corresponding slave + * configuration update in a re-entrant and race-safe manner. + * + * DMA engine drivers may implement the optional + * device_prep_config_sg() callback to perform both the slave configuration + * and descriptor preparation atomically. In this case, the operation is + * fully handled by the DMA engine driver. + * + * If the DMA engine driver does not implement device_prep_config_sg(), falls + * back to calling dmaengine_slave_config() followed by dmaengine_prep_slave_sg(). + * The fallback path is protected by a per-channel spinlock to ensure that + * concurrent callers cannot interleave configuration and descriptor preparation + * on the same DMA channel. + * + * Return: Pointer to a prepared DMA async transaction descriptor on success, + * or %NULL if the transfer could not be prepared. + */ +static inline struct dma_async_tx_descriptor * +dmaengine_prep_config_sg_safe(struct dma_chan *chan, struct scatterlist *sgl, + unsigned int sg_len, + enum dma_transfer_direction dir, + unsigned long flags, + struct dma_slave_config *config) +{ + struct dma_async_tx_descriptor *tx; + unsigned long spinlock_flags; + + if (!chan || !chan->device) + return NULL; + + if (!chan->device->device_prep_config_sg) + spin_lock_irqsave(&chan->lock, spinlock_flags); + + tx = dmaengine_prep_config_sg(chan, sgl, sg_len, dir, flags, config); + + if (!chan->device->device_prep_config_sg) + spin_unlock_irqrestore(&chan->lock, spinlock_flags); + + return tx; +} + +/** + * dmaengine_prep_config_single_safe - prepare a single-buffer DMA transfer + * with atomic slave configuration update + * @chan: DMA channel + * @buf: DMA buffer address + * @len: length of the transfer in bytes + * @dir: DMA transfer direction + * @flags: transfer preparation flags + * @config: DMA slave configuration for this transfer + * + * Detail see dmaengine_prep_config_sg_safe(). + */ +static inline struct dma_async_tx_descriptor * +dmaengine_prep_config_single_safe(struct dma_chan *chan, dma_addr_t buf, + size_t len, enum dma_transfer_direction dir, + unsigned long flags, + struct dma_slave_config *config) +{ + struct scatterlist sg; + + sg_init_table(&sg, 1); + sg_dma_address(&sg) = buf; + sg_dma_len(&sg) = len; + + return dmaengine_prep_config_sg_safe(chan, &sg, 1, dir, flags, config); +} + #ifdef CONFIG_RAPIDIO_DMA_ENGINE struct rio_dma_ext; static inline struct dma_async_tx_descriptor *dmaengine_prep_rio_sg( diff --git a/include/linux/dmar.h b/include/linux/dmar.h index 692b2b445761..63e35df2cef4 100644 --- a/include/linux/dmar.h +++ b/include/linux/dmar.h @@ -24,6 +24,7 @@ struct acpi_dmar_header; #define DMAR_INTR_REMAP 0x1 #define DMAR_X2APIC_OPT_OUT 0x2 #define DMAR_PLATFORM_OPT_IN 0x4 +#define DMAR_REMAP_OPT_OUT 0x8 struct intel_iommu; diff --git a/include/linux/dmi.h b/include/linux/dmi.h index 927f8a8b7a1d..fbb62f7c3111 100644 --- a/include/linux/dmi.h +++ b/include/linux/dmi.h @@ -4,7 +4,7 @@ #include <linux/list.h> #include <linux/kobject.h> -#include <linux/mod_devicetable.h> +#include <linux/device-id/dmi.h> /* enum dmi_field is in mod_devicetable.h */ @@ -60,6 +60,7 @@ enum dmi_entry_type { DMI_ENTRY_OOB_REMOTE_ACCESS, DMI_ENTRY_BIS_ENTRY, DMI_ENTRY_SYSTEM_BOOT, + DMI_ENTRY_64_MEM_ERROR, DMI_ENTRY_MGMT_DEV, DMI_ENTRY_MGMT_DEV_COMPONENT, DMI_ENTRY_MGMT_DEV_THRES, @@ -69,6 +70,10 @@ enum dmi_entry_type { DMI_ENTRY_ADDITIONAL, DMI_ENTRY_ONBOARD_DEV_EXT, DMI_ENTRY_MGMT_CONTROLLER_HOST, + DMI_ENTRY_TPM_DEVICE, + DMI_ENTRY_PROCESSOR_ADDITIONAL, + DMI_ENTRY_FIRMWARE_INVENTORY, + DMI_ENTRY_STRING_PROPERTY, DMI_ENTRY_INACTIVE = 126, DMI_ENTRY_END_OF_TABLE = 127, }; @@ -86,6 +91,21 @@ struct dmi_device { void *device_data; /* Type specific data */ }; +#define DMI_A_INFO_ENT_MIN_SIZE 0x6 +struct dmi_a_info_entry { + u8 length; + u16 handle; + u8 offset; + u8 str_num; + u8 value[]; +} __packed; + +#define DMI_A_INFO_MIN_SIZE 0xB +struct dmi_a_info { + struct dmi_header header; + u8 count; +} __packed; + #ifdef CONFIG_DMI struct dmi_dev_onboard { @@ -115,6 +135,7 @@ extern void dmi_memdev_name(u16 handle, const char **bank, const char **device); extern u64 dmi_memdev_size(u16 handle); extern u8 dmi_memdev_type(u16 handle); extern u16 dmi_memdev_handle(int slot); +const char *dmi_string_nosave(const struct dmi_header *dm, u8 s); #else @@ -148,6 +169,8 @@ static inline u8 dmi_memdev_type(u16 handle) { return 0x0; } static inline u16 dmi_memdev_handle(int slot) { return 0xffff; } static inline const struct dmi_system_id * dmi_first_match(const struct dmi_system_id *list) { return NULL; } +static inline const char * + dmi_string_nosave(const struct dmi_header *dm, u8 s) { return ""; } #endif diff --git a/include/linux/dpll.h b/include/linux/dpll.h index 2ce295b46b8c..7a6f8796cda2 100644 --- a/include/linux/dpll.h +++ b/include/linux/dpll.h @@ -13,6 +13,7 @@ #include <linux/netdevice.h> #include <linux/notifier.h> #include <linux/rtnetlink.h> +#include <net/netlink.h> struct dpll_device; struct dpll_pin; @@ -52,9 +53,28 @@ struct dpll_device_ops { int (*phase_offset_avg_factor_get)(const struct dpll_device *dpll, void *dpll_priv, u32 *factor, struct netlink_ext_ack *extack); + int (*freq_monitor_set)(const struct dpll_device *dpll, void *dpll_priv, + enum dpll_feature_state state, + struct netlink_ext_ack *extack); + int (*freq_monitor_get)(const struct dpll_device *dpll, void *dpll_priv, + enum dpll_feature_state *state, + struct netlink_ext_ack *extack); +}; + +enum dpll_ffo_type { + DPLL_FFO_PORT_RXTX_RATE, + DPLL_FFO_PIN_DEVICE, + + __DPLL_FFO_TYPE_MAX, +}; + +struct dpll_ffo_param { + enum dpll_ffo_type type; + s64 ffo; }; struct dpll_pin_ops { + unsigned long supported_ffo; int (*frequency_set)(const struct dpll_pin *pin, void *pin_priv, const struct dpll_device *dpll, void *dpll_priv, const u64 frequency, @@ -79,6 +99,12 @@ struct dpll_pin_ops { const struct dpll_device *dpll, void *dpll_priv, enum dpll_pin_state *state, struct netlink_ext_ack *extack); + int (*operstate_on_dpll_get)(const struct dpll_pin *pin, + void *pin_priv, + const struct dpll_device *dpll, + void *dpll_priv, + enum dpll_pin_operstate *operstate, + struct netlink_ext_ack *extack); int (*state_on_pin_set)(const struct dpll_pin *pin, void *pin_priv, const struct dpll_pin *parent_pin, void *parent_pin_priv, @@ -109,7 +135,12 @@ struct dpll_pin_ops { struct netlink_ext_ack *extack); int (*ffo_get)(const struct dpll_pin *pin, void *pin_priv, const struct dpll_device *dpll, void *dpll_priv, - s64 *ffo, struct netlink_ext_ack *extack); + struct dpll_ffo_param *ffo, + struct netlink_ext_ack *extack); + int (*measured_freq_get)(const struct dpll_pin *pin, void *pin_priv, + const struct dpll_device *dpll, + void *dpll_priv, u64 *measured_freq, + struct netlink_ext_ack *extack); int (*esync_set)(const struct dpll_pin *pin, void *pin_priv, const struct dpll_device *dpll, void *dpll_priv, u64 freq, struct netlink_ext_ack *extack); @@ -202,13 +233,18 @@ struct dpll_pin_notifier_info { u64 clock_id; const struct fwnode_handle *fwnode; const struct dpll_pin_properties *prop; + u64 src_clock_id; }; #if IS_ENABLED(CONFIG_DPLL) void dpll_netdev_pin_set(struct net_device *dev, struct dpll_pin *dpll_pin); void dpll_netdev_pin_clear(struct net_device *dev); -size_t dpll_netdev_pin_handle_size(const struct net_device *dev); +static inline size_t dpll_netdev_pin_handle_size(void) +{ + return nla_total_size(4); /* DPLL_A_PIN_ID */ +} + int dpll_netdev_add_pin_handle(struct sk_buff *msg, const struct net_device *dev); @@ -219,7 +255,7 @@ static inline void dpll_netdev_pin_set(struct net_device *dev, struct dpll_pin *dpll_pin) { } static inline void dpll_netdev_pin_clear(struct net_device *dev) { } -static inline size_t dpll_netdev_pin_handle_size(const struct net_device *dev) +static inline size_t dpll_netdev_pin_handle_size(void) { return 0; } @@ -274,8 +310,10 @@ void dpll_pin_on_pin_unregister(struct dpll_pin *parent, struct dpll_pin *pin, int dpll_pin_ref_sync_pair_add(struct dpll_pin *pin, struct dpll_pin *ref_sync_pin); +int __dpll_device_change_ntf(struct dpll_device *dpll); int dpll_device_change_ntf(struct dpll_device *dpll); +int __dpll_pin_change_ntf(struct dpll_pin *pin); int dpll_pin_change_ntf(struct dpll_pin *pin); int register_dpll_notifier(struct notifier_block *nb); diff --git a/include/linux/drbd.h b/include/linux/drbd.h deleted file mode 100644 index 5468a2399d48..000000000000 --- a/include/linux/drbd.h +++ /dev/null @@ -1,392 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* - drbd.h - Kernel module for 2.6.x Kernels - - This file is part of DRBD by Philipp Reisner and Lars Ellenberg. - - Copyright (C) 2001-2008, LINBIT Information Technologies GmbH. - Copyright (C) 2001-2008, Philipp Reisner <philipp.reisner@linbit.com>. - Copyright (C) 2001-2008, Lars Ellenberg <lars.ellenberg@linbit.com>. - - -*/ -#ifndef DRBD_H -#define DRBD_H -#include <asm/types.h> - -#ifdef __KERNEL__ -#include <linux/types.h> -#include <asm/byteorder.h> -#else -#include <sys/types.h> -#include <sys/wait.h> -#include <limits.h> - -/* Although the Linux source code makes a difference between - generic endianness and the bitfields' endianness, there is no - architecture as of Linux-2.6.24-rc4 where the bitfields' endianness - does not match the generic endianness. */ - -#if __BYTE_ORDER == __LITTLE_ENDIAN -#define __LITTLE_ENDIAN_BITFIELD -#elif __BYTE_ORDER == __BIG_ENDIAN -#define __BIG_ENDIAN_BITFIELD -#else -# error "sorry, weird endianness on this box" -#endif - -#endif - -enum drbd_io_error_p { - EP_PASS_ON, /* FIXME should the better be named "Ignore"? */ - EP_CALL_HELPER, - EP_DETACH -}; - -enum drbd_fencing_p { - FP_NOT_AVAIL = -1, /* Not a policy */ - FP_DONT_CARE = 0, - FP_RESOURCE, - FP_STONITH -}; - -enum drbd_disconnect_p { - DP_RECONNECT, - DP_DROP_NET_CONF, - DP_FREEZE_IO -}; - -enum drbd_after_sb_p { - ASB_DISCONNECT, - ASB_DISCARD_YOUNGER_PRI, - ASB_DISCARD_OLDER_PRI, - ASB_DISCARD_ZERO_CHG, - ASB_DISCARD_LEAST_CHG, - ASB_DISCARD_LOCAL, - ASB_DISCARD_REMOTE, - ASB_CONSENSUS, - ASB_DISCARD_SECONDARY, - ASB_CALL_HELPER, - ASB_VIOLENTLY -}; - -enum drbd_on_no_data { - OND_IO_ERROR, - OND_SUSPEND_IO -}; - -enum drbd_on_congestion { - OC_BLOCK, - OC_PULL_AHEAD, - OC_DISCONNECT, -}; - -enum drbd_read_balancing { - RB_PREFER_LOCAL, - RB_PREFER_REMOTE, - RB_ROUND_ROBIN, - RB_LEAST_PENDING, - RB_CONGESTED_REMOTE, - RB_32K_STRIPING, - RB_64K_STRIPING, - RB_128K_STRIPING, - RB_256K_STRIPING, - RB_512K_STRIPING, - RB_1M_STRIPING, -}; - -/* KEEP the order, do not delete or insert. Only append. */ -enum drbd_ret_code { - ERR_CODE_BASE = 100, - NO_ERROR = 101, - ERR_LOCAL_ADDR = 102, - ERR_PEER_ADDR = 103, - ERR_OPEN_DISK = 104, - ERR_OPEN_MD_DISK = 105, - ERR_DISK_NOT_BDEV = 107, - ERR_MD_NOT_BDEV = 108, - ERR_DISK_TOO_SMALL = 111, - ERR_MD_DISK_TOO_SMALL = 112, - ERR_BDCLAIM_DISK = 114, - ERR_BDCLAIM_MD_DISK = 115, - ERR_MD_IDX_INVALID = 116, - ERR_IO_MD_DISK = 118, - ERR_MD_INVALID = 119, - ERR_AUTH_ALG = 120, - ERR_AUTH_ALG_ND = 121, - ERR_NOMEM = 122, - ERR_DISCARD_IMPOSSIBLE = 123, - ERR_DISK_CONFIGURED = 124, - ERR_NET_CONFIGURED = 125, - ERR_MANDATORY_TAG = 126, - ERR_MINOR_INVALID = 127, - ERR_INTR = 129, /* EINTR */ - ERR_RESIZE_RESYNC = 130, - ERR_NO_PRIMARY = 131, - ERR_RESYNC_AFTER = 132, - ERR_RESYNC_AFTER_CYCLE = 133, - ERR_PAUSE_IS_SET = 134, - ERR_PAUSE_IS_CLEAR = 135, - ERR_PACKET_NR = 137, - ERR_NO_DISK = 138, - ERR_NOT_PROTO_C = 139, - ERR_NOMEM_BITMAP = 140, - ERR_INTEGRITY_ALG = 141, /* DRBD 8.2 only */ - ERR_INTEGRITY_ALG_ND = 142, /* DRBD 8.2 only */ - ERR_CPU_MASK_PARSE = 143, /* DRBD 8.2 only */ - ERR_CSUMS_ALG = 144, /* DRBD 8.2 only */ - ERR_CSUMS_ALG_ND = 145, /* DRBD 8.2 only */ - ERR_VERIFY_ALG = 146, /* DRBD 8.2 only */ - ERR_VERIFY_ALG_ND = 147, /* DRBD 8.2 only */ - ERR_CSUMS_RESYNC_RUNNING= 148, /* DRBD 8.2 only */ - ERR_VERIFY_RUNNING = 149, /* DRBD 8.2 only */ - ERR_DATA_NOT_CURRENT = 150, - ERR_CONNECTED = 151, /* DRBD 8.3 only */ - ERR_PERM = 152, - ERR_NEED_APV_93 = 153, - ERR_STONITH_AND_PROT_A = 154, - ERR_CONG_NOT_PROTO_A = 155, - ERR_PIC_AFTER_DEP = 156, - ERR_PIC_PEER_DEP = 157, - ERR_RES_NOT_KNOWN = 158, - ERR_RES_IN_USE = 159, - ERR_MINOR_CONFIGURED = 160, - ERR_MINOR_OR_VOLUME_EXISTS = 161, - ERR_INVALID_REQUEST = 162, - ERR_NEED_APV_100 = 163, - ERR_NEED_ALLOW_TWO_PRI = 164, - ERR_MD_UNCLEAN = 165, - ERR_MD_LAYOUT_CONNECTED = 166, - ERR_MD_LAYOUT_TOO_BIG = 167, - ERR_MD_LAYOUT_TOO_SMALL = 168, - ERR_MD_LAYOUT_NO_FIT = 169, - ERR_IMPLICIT_SHRINK = 170, - /* insert new ones above this line */ - AFTER_LAST_ERR_CODE -}; - -#define DRBD_PROT_A 1 -#define DRBD_PROT_B 2 -#define DRBD_PROT_C 3 - -enum drbd_role { - R_UNKNOWN = 0, - R_PRIMARY = 1, /* role */ - R_SECONDARY = 2, /* role */ - R_MASK = 3, -}; - -/* The order of these constants is important. - * The lower ones (<C_WF_REPORT_PARAMS) indicate - * that there is no socket! - * >=C_WF_REPORT_PARAMS ==> There is a socket - */ -enum drbd_conns { - C_STANDALONE, - C_DISCONNECTING, /* Temporal state on the way to StandAlone. */ - C_UNCONNECTED, /* >= C_UNCONNECTED -> inc_net() succeeds */ - - /* These temporal states are all used on the way - * from >= C_CONNECTED to Unconnected. - * The 'disconnect reason' states - * I do not allow to change between them. */ - C_TIMEOUT, - C_BROKEN_PIPE, - C_NETWORK_FAILURE, - C_PROTOCOL_ERROR, - C_TEAR_DOWN, - - C_WF_CONNECTION, - C_WF_REPORT_PARAMS, /* we have a socket */ - C_CONNECTED, /* we have introduced each other */ - C_STARTING_SYNC_S, /* starting full sync by admin request. */ - C_STARTING_SYNC_T, /* starting full sync by admin request. */ - C_WF_BITMAP_S, - C_WF_BITMAP_T, - C_WF_SYNC_UUID, - - /* All SyncStates are tested with this comparison - * xx >= C_SYNC_SOURCE && xx <= C_PAUSED_SYNC_T */ - C_SYNC_SOURCE, - C_SYNC_TARGET, - C_VERIFY_S, - C_VERIFY_T, - C_PAUSED_SYNC_S, - C_PAUSED_SYNC_T, - - C_AHEAD, - C_BEHIND, - - C_MASK = 31 -}; - -enum drbd_disk_state { - D_DISKLESS, - D_ATTACHING, /* In the process of reading the meta-data */ - D_FAILED, /* Becomes D_DISKLESS as soon as we told it the peer */ - /* when >= D_FAILED it is legal to access mdev->ldev */ - D_NEGOTIATING, /* Late attaching state, we need to talk to the peer */ - D_INCONSISTENT, - D_OUTDATED, - D_UNKNOWN, /* Only used for the peer, never for myself */ - D_CONSISTENT, /* Might be D_OUTDATED, might be D_UP_TO_DATE ... */ - D_UP_TO_DATE, /* Only this disk state allows applications' IO ! */ - D_MASK = 15 -}; - -union drbd_state { -/* According to gcc's docs is the ... - * The order of allocation of bit-fields within a unit (C90 6.5.2.1, C99 6.7.2.1). - * Determined by ABI. - * pointed out by Maxim Uvarov q<muvarov@ru.mvista.com> - * even though we transmit as "cpu_to_be32(state)", - * the offsets of the bitfields still need to be swapped - * on different endianness. - */ - struct { -#if defined(__LITTLE_ENDIAN_BITFIELD) - unsigned role:2 ; /* 3/4 primary/secondary/unknown */ - unsigned peer:2 ; /* 3/4 primary/secondary/unknown */ - unsigned conn:5 ; /* 17/32 cstates */ - unsigned disk:4 ; /* 8/16 from D_DISKLESS to D_UP_TO_DATE */ - unsigned pdsk:4 ; /* 8/16 from D_DISKLESS to D_UP_TO_DATE */ - unsigned susp:1 ; /* 2/2 IO suspended no/yes (by user) */ - unsigned aftr_isp:1 ; /* isp .. imposed sync pause */ - unsigned peer_isp:1 ; - unsigned user_isp:1 ; - unsigned susp_nod:1 ; /* IO suspended because no data */ - unsigned susp_fen:1 ; /* IO suspended because fence peer handler runs*/ - unsigned _pad:9; /* 0 unused */ -#elif defined(__BIG_ENDIAN_BITFIELD) - unsigned _pad:9; - unsigned susp_fen:1 ; - unsigned susp_nod:1 ; - unsigned user_isp:1 ; - unsigned peer_isp:1 ; - unsigned aftr_isp:1 ; /* isp .. imposed sync pause */ - unsigned susp:1 ; /* 2/2 IO suspended no/yes */ - unsigned pdsk:4 ; /* 8/16 from D_DISKLESS to D_UP_TO_DATE */ - unsigned disk:4 ; /* 8/16 from D_DISKLESS to D_UP_TO_DATE */ - unsigned conn:5 ; /* 17/32 cstates */ - unsigned peer:2 ; /* 3/4 primary/secondary/unknown */ - unsigned role:2 ; /* 3/4 primary/secondary/unknown */ -#else -# error "this endianness is not supported" -#endif - }; - unsigned int i; -}; - -enum drbd_state_rv { - SS_CW_NO_NEED = 4, - SS_CW_SUCCESS = 3, - SS_NOTHING_TO_DO = 2, - SS_SUCCESS = 1, - SS_UNKNOWN_ERROR = 0, /* Used to sleep longer in _drbd_request_state */ - SS_TWO_PRIMARIES = -1, - SS_NO_UP_TO_DATE_DISK = -2, - SS_NO_LOCAL_DISK = -4, - SS_NO_REMOTE_DISK = -5, - SS_CONNECTED_OUTDATES = -6, - SS_PRIMARY_NOP = -7, - SS_RESYNC_RUNNING = -8, - SS_ALREADY_STANDALONE = -9, - SS_CW_FAILED_BY_PEER = -10, - SS_IS_DISKLESS = -11, - SS_DEVICE_IN_USE = -12, - SS_NO_NET_CONFIG = -13, - SS_NO_VERIFY_ALG = -14, /* drbd-8.2 only */ - SS_NEED_CONNECTION = -15, /* drbd-8.2 only */ - SS_LOWER_THAN_OUTDATED = -16, - SS_NOT_SUPPORTED = -17, /* drbd-8.2 only */ - SS_IN_TRANSIENT_STATE = -18, /* Retry after the next state change */ - SS_CONCURRENT_ST_CHG = -19, /* Concurrent cluster side state change! */ - SS_O_VOL_PEER_PRI = -20, - SS_OUTDATE_WO_CONN = -21, - SS_AFTER_LAST_ERROR = -22, /* Keep this at bottom */ -}; - -#define SHARED_SECRET_MAX 64 - -#define MDF_CONSISTENT (1 << 0) -#define MDF_PRIMARY_IND (1 << 1) -#define MDF_CONNECTED_IND (1 << 2) -#define MDF_FULL_SYNC (1 << 3) -#define MDF_WAS_UP_TO_DATE (1 << 4) -#define MDF_PEER_OUT_DATED (1 << 5) -#define MDF_CRASHED_PRIMARY (1 << 6) -#define MDF_AL_CLEAN (1 << 7) -#define MDF_AL_DISABLED (1 << 8) - -#define MAX_PEERS 32 - -enum drbd_uuid_index { - UI_CURRENT, - UI_BITMAP, - UI_HISTORY_START, - UI_HISTORY_END, - UI_SIZE, /* nl-packet: number of dirty bits */ - UI_FLAGS, /* nl-packet: flags */ - UI_EXTENDED_SIZE /* Everything. */ -}; - -#define HISTORY_UUIDS MAX_PEERS - -enum drbd_timeout_flag { - UT_DEFAULT = 0, - UT_DEGRADED = 1, - UT_PEER_OUTDATED = 2, -}; - -enum drbd_notification_type { - NOTIFY_EXISTS, - NOTIFY_CREATE, - NOTIFY_CHANGE, - NOTIFY_DESTROY, - NOTIFY_CALL, - NOTIFY_RESPONSE, - - NOTIFY_CONTINUES = 0x8000, - NOTIFY_FLAGS = NOTIFY_CONTINUES, -}; - -enum drbd_peer_state { - P_INCONSISTENT = 3, - P_OUTDATED = 4, - P_DOWN = 5, - P_PRIMARY = 6, - P_FENCING = 7, -}; - -#define UUID_JUST_CREATED ((__u64)4) - -enum write_ordering_e { - WO_NONE, - WO_DRAIN_IO, - WO_BDEV_FLUSH, - WO_BIO_BARRIER -}; - -/* magic numbers used in meta data and network packets */ -#define DRBD_MAGIC 0x83740267 -#define DRBD_MAGIC_BIG 0x835a -#define DRBD_MAGIC_100 0x8620ec20 - -#define DRBD_MD_MAGIC_07 (DRBD_MAGIC+3) -#define DRBD_MD_MAGIC_08 (DRBD_MAGIC+4) -#define DRBD_MD_MAGIC_84_UNCLEAN (DRBD_MAGIC+5) - - -/* how I came up with this magic? - * base64 decode "actlog==" ;) */ -#define DRBD_AL_MAGIC 0x69cb65a2 - -/* these are of type "int" */ -#define DRBD_MD_INDEX_INTERNAL -1 -#define DRBD_MD_INDEX_FLEX_EXT -2 -#define DRBD_MD_INDEX_FLEX_INT -3 - -#define DRBD_CPU_MASK_SIZE 32 - -#endif diff --git a/include/linux/drbd_config.h b/include/linux/drbd_config.h deleted file mode 100644 index d215365c6bb1..000000000000 --- a/include/linux/drbd_config.h +++ /dev/null @@ -1,16 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * drbd_config.h - * DRBD's compile time configuration. - */ - -#ifndef DRBD_CONFIG_H -#define DRBD_CONFIG_H - -extern const char *drbd_buildtag(void); - -#define REL_VERSION "8.4.11" -#define PRO_VERSION_MIN 86 -#define PRO_VERSION_MAX 101 - -#endif diff --git a/include/linux/drbd_genl.h b/include/linux/drbd_genl.h deleted file mode 100644 index 53f44b8cd75f..000000000000 --- a/include/linux/drbd_genl.h +++ /dev/null @@ -1,536 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * General overview: - * full generic netlink message: - * |nlmsghdr|genlmsghdr|<payload> - * - * payload: - * |optional fixed size family header|<sequence of netlink attributes> - * - * sequence of netlink attributes: - * I chose to have all "top level" attributes NLA_NESTED, - * corresponding to some real struct. - * So we have a sequence of |tla, len|<nested nla sequence> - * - * nested nla sequence: - * may be empty, or contain a sequence of netlink attributes - * representing the struct fields. - * - * The tag number of any field (regardless of containing struct) - * will be available as T_ ## field_name, - * so you cannot have the same field name in two differnt structs. - * - * The tag numbers themselves are per struct, though, - * so should always begin at 1 (not 0, that is the special "NLA_UNSPEC" type, - * which we won't use here). - * The tag numbers are used as index in the respective nla_policy array. - * - * GENL_struct(tag_name, tag_number, struct name, struct fields) - struct and policy - * genl_magic_struct.h - * generates the struct declaration, - * generates an entry in the tla enum, - * genl_magic_func.h - * generates an entry in the static tla policy - * with .type = NLA_NESTED - * generates the static <struct_name>_nl_policy definition, - * and static conversion functions - * - * genl_magic_func.h - * - * GENL_mc_group(group) - * genl_magic_struct.h - * does nothing - * genl_magic_func.h - * defines and registers the mcast group, - * and provides a send helper - * - * GENL_notification(op_name, op_num, mcast_group, tla list) - * These are notifications to userspace. - * - * genl_magic_struct.h - * generates an entry in the genl_ops enum, - * genl_magic_func.h - * does nothing - * - * mcast group: the name of the mcast group this notification should be - * expected on - * tla list: the list of expected top level attributes, - * for documentation and sanity checking. - * - * GENL_op(op_name, op_num, flags and handler, tla list) - "genl operations" - * These are requests from userspace. - * - * _op and _notification share the same "number space", - * op_nr will be assigned to "genlmsghdr->cmd" - * - * genl_magic_struct.h - * generates an entry in the genl_ops enum, - * genl_magic_func.h - * generates an entry in the static genl_ops array, - * and static register/unregister functions to - * genl_register_family(). - * - * flags and handler: - * GENL_op_init( .doit = x, .dumpit = y, .flags = something) - * GENL_doit(x) => .dumpit = NULL, .flags = GENL_ADMIN_PERM - * tla list: the list of expected top level attributes, - * for documentation and sanity checking. - */ - -/* - * STRUCTS - */ - -/* this is sent kernel -> userland on various error conditions, and contains - * informational textual info, which is supposedly human readable. - * The computer relevant return code is in the drbd_genlmsghdr. - */ -GENL_struct(DRBD_NLA_CFG_REPLY, 1, drbd_cfg_reply, - /* "arbitrary" size strings, nla_policy.len = 0 */ - __str_field(1, DRBD_GENLA_F_MANDATORY, info_text, 0) -) - -/* Configuration requests typically need a context to operate on. - * Possible keys are device minor (fits in the drbd_genlmsghdr), - * the replication link (aka connection) name, - * and/or the replication group (aka resource) name, - * and the volume id within the resource. */ -GENL_struct(DRBD_NLA_CFG_CONTEXT, 2, drbd_cfg_context, - __u32_field(1, DRBD_GENLA_F_MANDATORY, ctx_volume) - __str_field(2, DRBD_GENLA_F_MANDATORY, ctx_resource_name, 128) - __bin_field(3, DRBD_GENLA_F_MANDATORY, ctx_my_addr, 128) - __bin_field(4, DRBD_GENLA_F_MANDATORY, ctx_peer_addr, 128) -) - -GENL_struct(DRBD_NLA_DISK_CONF, 3, disk_conf, - __str_field(1, DRBD_F_REQUIRED | DRBD_F_INVARIANT, backing_dev, 128) - __str_field(2, DRBD_F_REQUIRED | DRBD_F_INVARIANT, meta_dev, 128) - __s32_field(3, DRBD_F_REQUIRED | DRBD_F_INVARIANT, meta_dev_idx) - - /* use the resize command to try and change the disk_size */ - __u64_field(4, DRBD_GENLA_F_MANDATORY | DRBD_F_INVARIANT, disk_size) - /* we could change the max_bio_bvecs, - * but it won't propagate through the stack */ - __u32_field(5, DRBD_GENLA_F_MANDATORY | DRBD_F_INVARIANT, max_bio_bvecs) - - __u32_field_def(6, DRBD_GENLA_F_MANDATORY, on_io_error, DRBD_ON_IO_ERROR_DEF) - __u32_field_def(7, DRBD_GENLA_F_MANDATORY, fencing, DRBD_FENCING_DEF) - - __u32_field_def(8, DRBD_GENLA_F_MANDATORY, resync_rate, DRBD_RESYNC_RATE_DEF) - __s32_field_def(9, DRBD_GENLA_F_MANDATORY, resync_after, DRBD_MINOR_NUMBER_DEF) - __u32_field_def(10, DRBD_GENLA_F_MANDATORY, al_extents, DRBD_AL_EXTENTS_DEF) - __u32_field_def(11, DRBD_GENLA_F_MANDATORY, c_plan_ahead, DRBD_C_PLAN_AHEAD_DEF) - __u32_field_def(12, DRBD_GENLA_F_MANDATORY, c_delay_target, DRBD_C_DELAY_TARGET_DEF) - __u32_field_def(13, DRBD_GENLA_F_MANDATORY, c_fill_target, DRBD_C_FILL_TARGET_DEF) - __u32_field_def(14, DRBD_GENLA_F_MANDATORY, c_max_rate, DRBD_C_MAX_RATE_DEF) - __u32_field_def(15, DRBD_GENLA_F_MANDATORY, c_min_rate, DRBD_C_MIN_RATE_DEF) - __u32_field_def(20, DRBD_GENLA_F_MANDATORY, disk_timeout, DRBD_DISK_TIMEOUT_DEF) - __u32_field_def(21, 0 /* OPTIONAL */, read_balancing, DRBD_READ_BALANCING_DEF) - __u32_field_def(25, 0 /* OPTIONAL */, rs_discard_granularity, DRBD_RS_DISCARD_GRANULARITY_DEF) - - __flg_field_def(16, DRBD_GENLA_F_MANDATORY, disk_barrier, DRBD_DISK_BARRIER_DEF) - __flg_field_def(17, DRBD_GENLA_F_MANDATORY, disk_flushes, DRBD_DISK_FLUSHES_DEF) - __flg_field_def(18, DRBD_GENLA_F_MANDATORY, disk_drain, DRBD_DISK_DRAIN_DEF) - __flg_field_def(19, DRBD_GENLA_F_MANDATORY, md_flushes, DRBD_MD_FLUSHES_DEF) - __flg_field_def(23, 0 /* OPTIONAL */, al_updates, DRBD_AL_UPDATES_DEF) - __flg_field_def(24, 0 /* OPTIONAL */, discard_zeroes_if_aligned, DRBD_DISCARD_ZEROES_IF_ALIGNED_DEF) - __flg_field_def(26, 0 /* OPTIONAL */, disable_write_same, DRBD_DISABLE_WRITE_SAME_DEF) -) - -GENL_struct(DRBD_NLA_RESOURCE_OPTS, 4, res_opts, - __str_field_def(1, DRBD_GENLA_F_MANDATORY, cpu_mask, DRBD_CPU_MASK_SIZE) - __u32_field_def(2, DRBD_GENLA_F_MANDATORY, on_no_data, DRBD_ON_NO_DATA_DEF) -) - -GENL_struct(DRBD_NLA_NET_CONF, 5, net_conf, - __str_field_def(1, DRBD_GENLA_F_MANDATORY | DRBD_F_SENSITIVE, - shared_secret, SHARED_SECRET_MAX) - __str_field_def(2, DRBD_GENLA_F_MANDATORY, cram_hmac_alg, SHARED_SECRET_MAX) - __str_field_def(3, DRBD_GENLA_F_MANDATORY, integrity_alg, SHARED_SECRET_MAX) - __str_field_def(4, DRBD_GENLA_F_MANDATORY, verify_alg, SHARED_SECRET_MAX) - __str_field_def(5, DRBD_GENLA_F_MANDATORY, csums_alg, SHARED_SECRET_MAX) - __u32_field_def(6, DRBD_GENLA_F_MANDATORY, wire_protocol, DRBD_PROTOCOL_DEF) - __u32_field_def(7, DRBD_GENLA_F_MANDATORY, connect_int, DRBD_CONNECT_INT_DEF) - __u32_field_def(8, DRBD_GENLA_F_MANDATORY, timeout, DRBD_TIMEOUT_DEF) - __u32_field_def(9, DRBD_GENLA_F_MANDATORY, ping_int, DRBD_PING_INT_DEF) - __u32_field_def(10, DRBD_GENLA_F_MANDATORY, ping_timeo, DRBD_PING_TIMEO_DEF) - __u32_field_def(11, DRBD_GENLA_F_MANDATORY, sndbuf_size, DRBD_SNDBUF_SIZE_DEF) - __u32_field_def(12, DRBD_GENLA_F_MANDATORY, rcvbuf_size, DRBD_RCVBUF_SIZE_DEF) - __u32_field_def(13, DRBD_GENLA_F_MANDATORY, ko_count, DRBD_KO_COUNT_DEF) - __u32_field_def(14, DRBD_GENLA_F_MANDATORY, max_buffers, DRBD_MAX_BUFFERS_DEF) - __u32_field_def(15, DRBD_GENLA_F_MANDATORY, max_epoch_size, DRBD_MAX_EPOCH_SIZE_DEF) - __u32_field_def(16, DRBD_GENLA_F_MANDATORY, unplug_watermark, DRBD_UNPLUG_WATERMARK_DEF) - __u32_field_def(17, DRBD_GENLA_F_MANDATORY, after_sb_0p, DRBD_AFTER_SB_0P_DEF) - __u32_field_def(18, DRBD_GENLA_F_MANDATORY, after_sb_1p, DRBD_AFTER_SB_1P_DEF) - __u32_field_def(19, DRBD_GENLA_F_MANDATORY, after_sb_2p, DRBD_AFTER_SB_2P_DEF) - __u32_field_def(20, DRBD_GENLA_F_MANDATORY, rr_conflict, DRBD_RR_CONFLICT_DEF) - __u32_field_def(21, DRBD_GENLA_F_MANDATORY, on_congestion, DRBD_ON_CONGESTION_DEF) - __u32_field_def(22, DRBD_GENLA_F_MANDATORY, cong_fill, DRBD_CONG_FILL_DEF) - __u32_field_def(23, DRBD_GENLA_F_MANDATORY, cong_extents, DRBD_CONG_EXTENTS_DEF) - __flg_field_def(24, DRBD_GENLA_F_MANDATORY, two_primaries, DRBD_ALLOW_TWO_PRIMARIES_DEF) - __flg_field(25, DRBD_GENLA_F_MANDATORY | DRBD_F_INVARIANT, discard_my_data) - __flg_field_def(26, DRBD_GENLA_F_MANDATORY, tcp_cork, DRBD_TCP_CORK_DEF) - __flg_field_def(27, DRBD_GENLA_F_MANDATORY, always_asbp, DRBD_ALWAYS_ASBP_DEF) - __flg_field(28, DRBD_GENLA_F_MANDATORY | DRBD_F_INVARIANT, tentative) - __flg_field_def(29, DRBD_GENLA_F_MANDATORY, use_rle, DRBD_USE_RLE_DEF) - /* 9: __u32_field_def(30, DRBD_GENLA_F_MANDATORY, fencing_policy, DRBD_FENCING_DEF) */ - /* 9: __str_field_def(31, DRBD_GENLA_F_MANDATORY, name, SHARED_SECRET_MAX) */ - /* 9: __u32_field(32, DRBD_F_REQUIRED | DRBD_F_INVARIANT, peer_node_id) */ - __flg_field_def(33, 0 /* OPTIONAL */, csums_after_crash_only, DRBD_CSUMS_AFTER_CRASH_ONLY_DEF) - __u32_field_def(34, 0 /* OPTIONAL */, sock_check_timeo, DRBD_SOCKET_CHECK_TIMEO_DEF) -) - -GENL_struct(DRBD_NLA_SET_ROLE_PARMS, 6, set_role_parms, - __flg_field(1, DRBD_GENLA_F_MANDATORY, assume_uptodate) -) - -GENL_struct(DRBD_NLA_RESIZE_PARMS, 7, resize_parms, - __u64_field(1, DRBD_GENLA_F_MANDATORY, resize_size) - __flg_field(2, DRBD_GENLA_F_MANDATORY, resize_force) - __flg_field(3, DRBD_GENLA_F_MANDATORY, no_resync) - __u32_field_def(4, 0 /* OPTIONAL */, al_stripes, DRBD_AL_STRIPES_DEF) - __u32_field_def(5, 0 /* OPTIONAL */, al_stripe_size, DRBD_AL_STRIPE_SIZE_DEF) -) - -GENL_struct(DRBD_NLA_STATE_INFO, 8, state_info, - /* the reason of the broadcast, - * if this is an event triggered broadcast. */ - __u32_field(1, DRBD_GENLA_F_MANDATORY, sib_reason) - __u32_field(2, DRBD_F_REQUIRED, current_state) - __u64_field(3, DRBD_GENLA_F_MANDATORY, capacity) - __u64_field(4, DRBD_GENLA_F_MANDATORY, ed_uuid) - - /* These are for broadcast from after state change work. - * prev_state and new_state are from the moment the state change took - * place, new_state is not neccessarily the same as current_state, - * there may have been more state changes since. Which will be - * broadcasted soon, in their respective after state change work. */ - __u32_field(5, DRBD_GENLA_F_MANDATORY, prev_state) - __u32_field(6, DRBD_GENLA_F_MANDATORY, new_state) - - /* if we have a local disk: */ - __bin_field(7, DRBD_GENLA_F_MANDATORY, uuids, (UI_SIZE*sizeof(__u64))) - __u32_field(8, DRBD_GENLA_F_MANDATORY, disk_flags) - __u64_field(9, DRBD_GENLA_F_MANDATORY, bits_total) - __u64_field(10, DRBD_GENLA_F_MANDATORY, bits_oos) - /* and in case resync or online verify is active */ - __u64_field(11, DRBD_GENLA_F_MANDATORY, bits_rs_total) - __u64_field(12, DRBD_GENLA_F_MANDATORY, bits_rs_failed) - - /* for pre and post notifications of helper execution */ - __str_field(13, DRBD_GENLA_F_MANDATORY, helper, 32) - __u32_field(14, DRBD_GENLA_F_MANDATORY, helper_exit_code) - - __u64_field(15, 0, send_cnt) - __u64_field(16, 0, recv_cnt) - __u64_field(17, 0, read_cnt) - __u64_field(18, 0, writ_cnt) - __u64_field(19, 0, al_writ_cnt) - __u64_field(20, 0, bm_writ_cnt) - __u32_field(21, 0, ap_bio_cnt) - __u32_field(22, 0, ap_pending_cnt) - __u32_field(23, 0, rs_pending_cnt) -) - -GENL_struct(DRBD_NLA_START_OV_PARMS, 9, start_ov_parms, - __u64_field(1, DRBD_GENLA_F_MANDATORY, ov_start_sector) - __u64_field(2, DRBD_GENLA_F_MANDATORY, ov_stop_sector) -) - -GENL_struct(DRBD_NLA_NEW_C_UUID_PARMS, 10, new_c_uuid_parms, - __flg_field(1, DRBD_GENLA_F_MANDATORY, clear_bm) -) - -GENL_struct(DRBD_NLA_TIMEOUT_PARMS, 11, timeout_parms, - __u32_field(1, DRBD_F_REQUIRED, timeout_type) -) - -GENL_struct(DRBD_NLA_DISCONNECT_PARMS, 12, disconnect_parms, - __flg_field(1, DRBD_GENLA_F_MANDATORY, force_disconnect) -) - -GENL_struct(DRBD_NLA_DETACH_PARMS, 13, detach_parms, - __flg_field(1, DRBD_GENLA_F_MANDATORY, force_detach) -) - -GENL_struct(DRBD_NLA_RESOURCE_INFO, 15, resource_info, - __u32_field(1, 0, res_role) - __flg_field(2, 0, res_susp) - __flg_field(3, 0, res_susp_nod) - __flg_field(4, 0, res_susp_fen) - /* __flg_field(5, 0, res_weak) */ -) - -GENL_struct(DRBD_NLA_DEVICE_INFO, 16, device_info, - __u32_field(1, 0, dev_disk_state) -) - -GENL_struct(DRBD_NLA_CONNECTION_INFO, 17, connection_info, - __u32_field(1, 0, conn_connection_state) - __u32_field(2, 0, conn_role) -) - -GENL_struct(DRBD_NLA_PEER_DEVICE_INFO, 18, peer_device_info, - __u32_field(1, 0, peer_repl_state) - __u32_field(2, 0, peer_disk_state) - __u32_field(3, 0, peer_resync_susp_user) - __u32_field(4, 0, peer_resync_susp_peer) - __u32_field(5, 0, peer_resync_susp_dependency) -) - -GENL_struct(DRBD_NLA_RESOURCE_STATISTICS, 19, resource_statistics, - __u32_field(1, 0, res_stat_write_ordering) -) - -GENL_struct(DRBD_NLA_DEVICE_STATISTICS, 20, device_statistics, - __u64_field(1, 0, dev_size) /* (sectors) */ - __u64_field(2, 0, dev_read) /* (sectors) */ - __u64_field(3, 0, dev_write) /* (sectors) */ - __u64_field(4, 0, dev_al_writes) /* activity log writes (count) */ - __u64_field(5, 0, dev_bm_writes) /* bitmap writes (count) */ - __u32_field(6, 0, dev_upper_pending) /* application requests in progress */ - __u32_field(7, 0, dev_lower_pending) /* backing device requests in progress */ - __flg_field(8, 0, dev_upper_blocked) - __flg_field(9, 0, dev_lower_blocked) - __flg_field(10, 0, dev_al_suspended) /* activity log suspended */ - __u64_field(11, 0, dev_exposed_data_uuid) - __u64_field(12, 0, dev_current_uuid) - __u32_field(13, 0, dev_disk_flags) - __bin_field(14, 0, history_uuids, HISTORY_UUIDS * sizeof(__u64)) -) - -GENL_struct(DRBD_NLA_CONNECTION_STATISTICS, 21, connection_statistics, - __flg_field(1, 0, conn_congested) -) - -GENL_struct(DRBD_NLA_PEER_DEVICE_STATISTICS, 22, peer_device_statistics, - __u64_field(1, 0, peer_dev_received) /* sectors */ - __u64_field(2, 0, peer_dev_sent) /* sectors */ - __u32_field(3, 0, peer_dev_pending) /* number of requests */ - __u32_field(4, 0, peer_dev_unacked) /* number of requests */ - __u64_field(5, 0, peer_dev_out_of_sync) /* sectors */ - __u64_field(6, 0, peer_dev_resync_failed) /* sectors */ - __u64_field(7, 0, peer_dev_bitmap_uuid) - __u32_field(9, 0, peer_dev_flags) -) - -GENL_struct(DRBD_NLA_NOTIFICATION_HEADER, 23, drbd_notification_header, - __u32_field(1, DRBD_GENLA_F_MANDATORY, nh_type) -) - -GENL_struct(DRBD_NLA_HELPER, 24, drbd_helper_info, - __str_field(1, DRBD_GENLA_F_MANDATORY, helper_name, 32) - __u32_field(2, DRBD_GENLA_F_MANDATORY, helper_status) -) - -/* - * Notifications and commands (genlmsghdr->cmd) - */ -GENL_mc_group(events) - - /* kernel -> userspace announcement of changes */ -GENL_notification( - DRBD_EVENT, 1, events, - GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_F_REQUIRED) - GENL_tla_expected(DRBD_NLA_STATE_INFO, DRBD_F_REQUIRED) - GENL_tla_expected(DRBD_NLA_NET_CONF, DRBD_GENLA_F_MANDATORY) - GENL_tla_expected(DRBD_NLA_DISK_CONF, DRBD_GENLA_F_MANDATORY) - GENL_tla_expected(DRBD_NLA_SYNCER_CONF, DRBD_GENLA_F_MANDATORY) -) - - /* query kernel for specific or all info */ -GENL_op( - DRBD_ADM_GET_STATUS, 2, - GENL_op_init( - .doit = drbd_adm_get_status, - .dumpit = drbd_adm_get_status_all, - /* anyone may ask for the status, - * it is broadcasted anyways */ - ), - /* To select the object .doit. - * Or a subset of objects in .dumpit. */ - GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_GENLA_F_MANDATORY) -) - - /* add DRBD minor devices as volumes to resources */ -GENL_op(DRBD_ADM_NEW_MINOR, 5, GENL_doit(drbd_adm_new_minor), - GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_F_REQUIRED)) -GENL_op(DRBD_ADM_DEL_MINOR, 6, GENL_doit(drbd_adm_del_minor), - GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_F_REQUIRED)) - - /* add or delete resources */ -GENL_op(DRBD_ADM_NEW_RESOURCE, 7, GENL_doit(drbd_adm_new_resource), - GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_F_REQUIRED)) -GENL_op(DRBD_ADM_DEL_RESOURCE, 8, GENL_doit(drbd_adm_del_resource), - GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_F_REQUIRED)) - -GENL_op(DRBD_ADM_RESOURCE_OPTS, 9, - GENL_doit(drbd_adm_resource_opts), - GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_F_REQUIRED) - GENL_tla_expected(DRBD_NLA_RESOURCE_OPTS, DRBD_GENLA_F_MANDATORY) -) - -GENL_op( - DRBD_ADM_CONNECT, 10, - GENL_doit(drbd_adm_connect), - GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_F_REQUIRED) - GENL_tla_expected(DRBD_NLA_NET_CONF, DRBD_F_REQUIRED) -) - -GENL_op( - DRBD_ADM_CHG_NET_OPTS, 29, - GENL_doit(drbd_adm_net_opts), - GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_F_REQUIRED) - GENL_tla_expected(DRBD_NLA_NET_CONF, DRBD_F_REQUIRED) -) - -GENL_op(DRBD_ADM_DISCONNECT, 11, GENL_doit(drbd_adm_disconnect), - GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_F_REQUIRED)) - -GENL_op(DRBD_ADM_ATTACH, 12, - GENL_doit(drbd_adm_attach), - GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_F_REQUIRED) - GENL_tla_expected(DRBD_NLA_DISK_CONF, DRBD_F_REQUIRED) -) - -GENL_op(DRBD_ADM_CHG_DISK_OPTS, 28, - GENL_doit(drbd_adm_disk_opts), - GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_F_REQUIRED) - GENL_tla_expected(DRBD_NLA_DISK_OPTS, DRBD_F_REQUIRED) -) - -GENL_op( - DRBD_ADM_RESIZE, 13, - GENL_doit(drbd_adm_resize), - GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_F_REQUIRED) - GENL_tla_expected(DRBD_NLA_RESIZE_PARMS, DRBD_GENLA_F_MANDATORY) -) - -GENL_op( - DRBD_ADM_PRIMARY, 14, - GENL_doit(drbd_adm_set_role), - GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_F_REQUIRED) - GENL_tla_expected(DRBD_NLA_SET_ROLE_PARMS, DRBD_F_REQUIRED) -) - -GENL_op( - DRBD_ADM_SECONDARY, 15, - GENL_doit(drbd_adm_set_role), - GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_F_REQUIRED) - GENL_tla_expected(DRBD_NLA_SET_ROLE_PARMS, DRBD_F_REQUIRED) -) - -GENL_op( - DRBD_ADM_NEW_C_UUID, 16, - GENL_doit(drbd_adm_new_c_uuid), - GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_F_REQUIRED) - GENL_tla_expected(DRBD_NLA_NEW_C_UUID_PARMS, DRBD_GENLA_F_MANDATORY) -) - -GENL_op( - DRBD_ADM_START_OV, 17, - GENL_doit(drbd_adm_start_ov), - GENL_tla_expected(DRBD_NLA_START_OV_PARMS, DRBD_GENLA_F_MANDATORY) -) - -GENL_op(DRBD_ADM_DETACH, 18, GENL_doit(drbd_adm_detach), - GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_F_REQUIRED) - GENL_tla_expected(DRBD_NLA_DETACH_PARMS, DRBD_GENLA_F_MANDATORY)) - -GENL_op(DRBD_ADM_INVALIDATE, 19, GENL_doit(drbd_adm_invalidate), - GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_F_REQUIRED)) -GENL_op(DRBD_ADM_INVAL_PEER, 20, GENL_doit(drbd_adm_invalidate_peer), - GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_F_REQUIRED)) -GENL_op(DRBD_ADM_PAUSE_SYNC, 21, GENL_doit(drbd_adm_pause_sync), - GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_F_REQUIRED)) -GENL_op(DRBD_ADM_RESUME_SYNC, 22, GENL_doit(drbd_adm_resume_sync), - GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_F_REQUIRED)) -GENL_op(DRBD_ADM_SUSPEND_IO, 23, GENL_doit(drbd_adm_suspend_io), - GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_F_REQUIRED)) -GENL_op(DRBD_ADM_RESUME_IO, 24, GENL_doit(drbd_adm_resume_io), - GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_F_REQUIRED)) -GENL_op(DRBD_ADM_OUTDATE, 25, GENL_doit(drbd_adm_outdate), - GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_F_REQUIRED)) -GENL_op(DRBD_ADM_GET_TIMEOUT_TYPE, 26, GENL_doit(drbd_adm_get_timeout_type), - GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_F_REQUIRED)) -GENL_op(DRBD_ADM_DOWN, 27, GENL_doit(drbd_adm_down), - GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_F_REQUIRED)) - -GENL_op(DRBD_ADM_GET_RESOURCES, 30, - GENL_op_init( - .dumpit = drbd_adm_dump_resources, - ), - GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_GENLA_F_MANDATORY) - GENL_tla_expected(DRBD_NLA_RESOURCE_INFO, DRBD_GENLA_F_MANDATORY) - GENL_tla_expected(DRBD_NLA_RESOURCE_STATISTICS, DRBD_GENLA_F_MANDATORY)) - -GENL_op(DRBD_ADM_GET_DEVICES, 31, - GENL_op_init( - .dumpit = drbd_adm_dump_devices, - .done = drbd_adm_dump_devices_done, - ), - GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_GENLA_F_MANDATORY) - GENL_tla_expected(DRBD_NLA_DEVICE_INFO, DRBD_GENLA_F_MANDATORY) - GENL_tla_expected(DRBD_NLA_DEVICE_STATISTICS, DRBD_GENLA_F_MANDATORY)) - -GENL_op(DRBD_ADM_GET_CONNECTIONS, 32, - GENL_op_init( - .dumpit = drbd_adm_dump_connections, - .done = drbd_adm_dump_connections_done, - ), - GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_GENLA_F_MANDATORY) - GENL_tla_expected(DRBD_NLA_CONNECTION_INFO, DRBD_GENLA_F_MANDATORY) - GENL_tla_expected(DRBD_NLA_CONNECTION_STATISTICS, DRBD_GENLA_F_MANDATORY)) - -GENL_op(DRBD_ADM_GET_PEER_DEVICES, 33, - GENL_op_init( - .dumpit = drbd_adm_dump_peer_devices, - .done = drbd_adm_dump_peer_devices_done, - ), - GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_GENLA_F_MANDATORY) - GENL_tla_expected(DRBD_NLA_PEER_DEVICE_INFO, DRBD_GENLA_F_MANDATORY) - GENL_tla_expected(DRBD_NLA_PEER_DEVICE_STATISTICS, DRBD_GENLA_F_MANDATORY)) - -GENL_notification( - DRBD_RESOURCE_STATE, 34, events, - GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_F_REQUIRED) - GENL_tla_expected(DRBD_NLA_NOTIFICATION_HEADER, DRBD_F_REQUIRED) - GENL_tla_expected(DRBD_NLA_RESOURCE_INFO, DRBD_F_REQUIRED) - GENL_tla_expected(DRBD_NLA_RESOURCE_STATISTICS, DRBD_F_REQUIRED)) - -GENL_notification( - DRBD_DEVICE_STATE, 35, events, - GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_F_REQUIRED) - GENL_tla_expected(DRBD_NLA_NOTIFICATION_HEADER, DRBD_F_REQUIRED) - GENL_tla_expected(DRBD_NLA_DEVICE_INFO, DRBD_F_REQUIRED) - GENL_tla_expected(DRBD_NLA_DEVICE_STATISTICS, DRBD_F_REQUIRED)) - -GENL_notification( - DRBD_CONNECTION_STATE, 36, events, - GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_F_REQUIRED) - GENL_tla_expected(DRBD_NLA_NOTIFICATION_HEADER, DRBD_F_REQUIRED) - GENL_tla_expected(DRBD_NLA_CONNECTION_INFO, DRBD_F_REQUIRED) - GENL_tla_expected(DRBD_NLA_CONNECTION_STATISTICS, DRBD_F_REQUIRED)) - -GENL_notification( - DRBD_PEER_DEVICE_STATE, 37, events, - GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_F_REQUIRED) - GENL_tla_expected(DRBD_NLA_NOTIFICATION_HEADER, DRBD_F_REQUIRED) - GENL_tla_expected(DRBD_NLA_PEER_DEVICE_INFO, DRBD_F_REQUIRED) - GENL_tla_expected(DRBD_NLA_PEER_DEVICE_STATISTICS, DRBD_F_REQUIRED)) - -GENL_op( - DRBD_ADM_GET_INITIAL_STATE, 38, - GENL_op_init( - .dumpit = drbd_adm_get_initial_state, - ), - GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_GENLA_F_MANDATORY)) - -GENL_notification( - DRBD_HELPER, 40, events, - GENL_tla_expected(DRBD_NLA_CFG_CONTEXT, DRBD_F_REQUIRED) - GENL_tla_expected(DRBD_NLA_HELPER, DRBD_F_REQUIRED)) - -GENL_notification( - DRBD_INITIAL_STATE_DONE, 41, events, - GENL_tla_expected(DRBD_NLA_NOTIFICATION_HEADER, DRBD_F_REQUIRED)) diff --git a/include/linux/drbd_genl_api.h b/include/linux/drbd_genl_api.h deleted file mode 100644 index 70682c058027..000000000000 --- a/include/linux/drbd_genl_api.h +++ /dev/null @@ -1,56 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef DRBD_GENL_STRUCT_H -#define DRBD_GENL_STRUCT_H - -/** - * struct drbd_genlmsghdr - DRBD specific header used in NETLINK_GENERIC requests - * @minor: - * For admin requests (user -> kernel): which minor device to operate on. - * For (unicast) replies or informational (broadcast) messages - * (kernel -> user): which minor device the information is about. - * If we do not operate on minors, but on connections or resources, - * the minor value shall be (~0), and the attribute DRBD_NLA_CFG_CONTEXT - * is used instead. - * @flags: possible operation modifiers (relevant only for user->kernel): - * DRBD_GENL_F_SET_DEFAULTS - * @volume: - * When creating a new minor (adding it to a resource), the resource needs - * to know which volume number within the resource this is supposed to be. - * The volume number corresponds to the same volume number on the remote side, - * whereas the minor number on the remote side may be different - * (union with flags). - * @ret_code: kernel->userland unicast cfg reply return code (union with flags); - */ -struct drbd_genlmsghdr { - __u32 minor; - union { - __u32 flags; - __s32 ret_code; - }; -}; - -/* To be used in drbd_genlmsghdr.flags */ -enum { - DRBD_GENL_F_SET_DEFAULTS = 1, -}; - -enum drbd_state_info_bcast_reason { - SIB_GET_STATUS_REPLY = 1, - SIB_STATE_CHANGE = 2, - SIB_HELPER_PRE = 3, - SIB_HELPER_POST = 4, - SIB_SYNC_PROGRESS = 5, -}; - -/* hack around predefined gcc/cpp "linux=1", - * we cannot possibly include <1/drbd_genl.h> */ -#undef linux - -#include <linux/drbd.h> -#define GENL_MAGIC_VERSION 1 -#define GENL_MAGIC_FAMILY drbd -#define GENL_MAGIC_FAMILY_HDRSZ sizeof(struct drbd_genlmsghdr) -#define GENL_MAGIC_INCLUDE_FILE <linux/drbd_genl.h> -#include <linux/genl_magic_struct.h> - -#endif diff --git a/include/linux/drbd_limits.h b/include/linux/drbd_limits.h deleted file mode 100644 index 5b042fb427e9..000000000000 --- a/include/linux/drbd_limits.h +++ /dev/null @@ -1,251 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - drbd_limits.h - This file is part of DRBD by Philipp Reisner and Lars Ellenberg. -*/ - -/* - * Our current limitations. - * Some of them are hard limits, - * some of them are arbitrary range limits, that make it easier to provide - * feedback about nonsense settings for certain configurable values. - */ - -#ifndef DRBD_LIMITS_H -#define DRBD_LIMITS_H 1 - -#define DEBUG_RANGE_CHECK 0 - -#define DRBD_MINOR_COUNT_MIN 1U -#define DRBD_MINOR_COUNT_MAX 255U -#define DRBD_MINOR_COUNT_DEF 32U -#define DRBD_MINOR_COUNT_SCALE '1' - -#define DRBD_VOLUME_MAX 65534U - -#define DRBD_DIALOG_REFRESH_MIN 0U -#define DRBD_DIALOG_REFRESH_MAX 600U -#define DRBD_DIALOG_REFRESH_SCALE '1' - -/* valid port number */ -#define DRBD_PORT_MIN 1U -#define DRBD_PORT_MAX 0xffffU -#define DRBD_PORT_SCALE '1' - -/* startup { */ - /* if you want more than 3.4 days, disable */ -#define DRBD_WFC_TIMEOUT_MIN 0U -#define DRBD_WFC_TIMEOUT_MAX 300000U -#define DRBD_WFC_TIMEOUT_DEF 0U -#define DRBD_WFC_TIMEOUT_SCALE '1' - -#define DRBD_DEGR_WFC_TIMEOUT_MIN 0U -#define DRBD_DEGR_WFC_TIMEOUT_MAX 300000U -#define DRBD_DEGR_WFC_TIMEOUT_DEF 0U -#define DRBD_DEGR_WFC_TIMEOUT_SCALE '1' - -#define DRBD_OUTDATED_WFC_TIMEOUT_MIN 0U -#define DRBD_OUTDATED_WFC_TIMEOUT_MAX 300000U -#define DRBD_OUTDATED_WFC_TIMEOUT_DEF 0U -#define DRBD_OUTDATED_WFC_TIMEOUT_SCALE '1' -/* }*/ - -/* net { */ - /* timeout, unit centi seconds - * more than one minute timeout is not useful */ -#define DRBD_TIMEOUT_MIN 1U -#define DRBD_TIMEOUT_MAX 600U -#define DRBD_TIMEOUT_DEF 60U /* 6 seconds */ -#define DRBD_TIMEOUT_SCALE '1' - - /* If backing disk takes longer than disk_timeout, mark the disk as failed */ -#define DRBD_DISK_TIMEOUT_MIN 0U /* 0 = disabled */ -#define DRBD_DISK_TIMEOUT_MAX 6000U /* 10 Minutes */ -#define DRBD_DISK_TIMEOUT_DEF 0U /* disabled */ -#define DRBD_DISK_TIMEOUT_SCALE '1' - - /* active connection retries when C_WF_CONNECTION */ -#define DRBD_CONNECT_INT_MIN 1U -#define DRBD_CONNECT_INT_MAX 120U -#define DRBD_CONNECT_INT_DEF 10U /* seconds */ -#define DRBD_CONNECT_INT_SCALE '1' - - /* keep-alive probes when idle */ -#define DRBD_PING_INT_MIN 1U -#define DRBD_PING_INT_MAX 120U -#define DRBD_PING_INT_DEF 10U -#define DRBD_PING_INT_SCALE '1' - - /* timeout for the ping packets.*/ -#define DRBD_PING_TIMEO_MIN 1U -#define DRBD_PING_TIMEO_MAX 300U -#define DRBD_PING_TIMEO_DEF 5U -#define DRBD_PING_TIMEO_SCALE '1' - - /* max number of write requests between write barriers */ -#define DRBD_MAX_EPOCH_SIZE_MIN 1U -#define DRBD_MAX_EPOCH_SIZE_MAX 20000U -#define DRBD_MAX_EPOCH_SIZE_DEF 2048U -#define DRBD_MAX_EPOCH_SIZE_SCALE '1' - - /* I don't think that a tcp send buffer of more than 10M is useful */ -#define DRBD_SNDBUF_SIZE_MIN 0U -#define DRBD_SNDBUF_SIZE_MAX (10U<<20) -#define DRBD_SNDBUF_SIZE_DEF 0U -#define DRBD_SNDBUF_SIZE_SCALE '1' - -#define DRBD_RCVBUF_SIZE_MIN 0U -#define DRBD_RCVBUF_SIZE_MAX (10U<<20) -#define DRBD_RCVBUF_SIZE_DEF 0U -#define DRBD_RCVBUF_SIZE_SCALE '1' - - /* @4k PageSize -> 128kB - 512MB */ -#define DRBD_MAX_BUFFERS_MIN 32U -#define DRBD_MAX_BUFFERS_MAX 131072U -#define DRBD_MAX_BUFFERS_DEF 2048U -#define DRBD_MAX_BUFFERS_SCALE '1' - - /* @4k PageSize -> 4kB - 512MB */ -#define DRBD_UNPLUG_WATERMARK_MIN 1U -#define DRBD_UNPLUG_WATERMARK_MAX 131072U -#define DRBD_UNPLUG_WATERMARK_DEF (DRBD_MAX_BUFFERS_DEF/16) -#define DRBD_UNPLUG_WATERMARK_SCALE '1' - - /* 0 is disabled. - * 200 should be more than enough even for very short timeouts */ -#define DRBD_KO_COUNT_MIN 0U -#define DRBD_KO_COUNT_MAX 200U -#define DRBD_KO_COUNT_DEF 7U -#define DRBD_KO_COUNT_SCALE '1' -/* } */ - -/* syncer { */ - /* FIXME allow rate to be zero? */ -#define DRBD_RESYNC_RATE_MIN 1U -/* channel bonding 10 GbE, or other hardware */ -#define DRBD_RESYNC_RATE_MAX (4 << 20) -#define DRBD_RESYNC_RATE_DEF 250U -#define DRBD_RESYNC_RATE_SCALE 'k' /* kilobytes */ - -#define DRBD_AL_EXTENTS_MIN 67U - /* we use u16 as "slot number", (u16)~0 is "FREE". - * If you use >= 292 kB on-disk ring buffer, - * this is the maximum you can use: */ -#define DRBD_AL_EXTENTS_MAX 0xfffeU -#define DRBD_AL_EXTENTS_DEF 1237U -#define DRBD_AL_EXTENTS_SCALE '1' - -#define DRBD_MINOR_NUMBER_MIN -1 -#define DRBD_MINOR_NUMBER_MAX ((1 << 20) - 1) -#define DRBD_MINOR_NUMBER_DEF -1 -#define DRBD_MINOR_NUMBER_SCALE '1' - -/* } */ - -/* drbdsetup XY resize -d Z - * you are free to reduce the device size to nothing, if you want to. - * the upper limit with 64bit kernel, enough ram and flexible meta data - * is 1 PiB, currently. */ -/* DRBD_MAX_SECTORS */ -#define DRBD_DISK_SIZE_MIN 0LLU -#define DRBD_DISK_SIZE_MAX (1LLU * (2LLU << 40)) -#define DRBD_DISK_SIZE_DEF 0LLU /* = disabled = no user size... */ -#define DRBD_DISK_SIZE_SCALE 's' /* sectors */ - -#define DRBD_ON_IO_ERROR_DEF EP_DETACH -#define DRBD_FENCING_DEF FP_DONT_CARE -#define DRBD_AFTER_SB_0P_DEF ASB_DISCONNECT -#define DRBD_AFTER_SB_1P_DEF ASB_DISCONNECT -#define DRBD_AFTER_SB_2P_DEF ASB_DISCONNECT -#define DRBD_RR_CONFLICT_DEF ASB_DISCONNECT -#define DRBD_ON_NO_DATA_DEF OND_IO_ERROR -#define DRBD_ON_CONGESTION_DEF OC_BLOCK -#define DRBD_READ_BALANCING_DEF RB_PREFER_LOCAL - -#define DRBD_MAX_BIO_BVECS_MIN 0U -#define DRBD_MAX_BIO_BVECS_MAX 128U -#define DRBD_MAX_BIO_BVECS_DEF 0U -#define DRBD_MAX_BIO_BVECS_SCALE '1' - -#define DRBD_C_PLAN_AHEAD_MIN 0U -#define DRBD_C_PLAN_AHEAD_MAX 300U -#define DRBD_C_PLAN_AHEAD_DEF 20U -#define DRBD_C_PLAN_AHEAD_SCALE '1' - -#define DRBD_C_DELAY_TARGET_MIN 1U -#define DRBD_C_DELAY_TARGET_MAX 100U -#define DRBD_C_DELAY_TARGET_DEF 10U -#define DRBD_C_DELAY_TARGET_SCALE '1' - -#define DRBD_C_FILL_TARGET_MIN 0U -#define DRBD_C_FILL_TARGET_MAX (1U<<20) /* 500MByte in sec */ -#define DRBD_C_FILL_TARGET_DEF 100U /* Try to place 50KiB in socket send buffer during resync */ -#define DRBD_C_FILL_TARGET_SCALE 's' /* sectors */ - -#define DRBD_C_MAX_RATE_MIN 250U -#define DRBD_C_MAX_RATE_MAX (4U << 20) -#define DRBD_C_MAX_RATE_DEF 102400U -#define DRBD_C_MAX_RATE_SCALE 'k' /* kilobytes */ - -#define DRBD_C_MIN_RATE_MIN 0U -#define DRBD_C_MIN_RATE_MAX (4U << 20) -#define DRBD_C_MIN_RATE_DEF 250U -#define DRBD_C_MIN_RATE_SCALE 'k' /* kilobytes */ - -#define DRBD_CONG_FILL_MIN 0U -#define DRBD_CONG_FILL_MAX (10U<<21) /* 10GByte in sectors */ -#define DRBD_CONG_FILL_DEF 0U -#define DRBD_CONG_FILL_SCALE 's' /* sectors */ - -#define DRBD_CONG_EXTENTS_MIN DRBD_AL_EXTENTS_MIN -#define DRBD_CONG_EXTENTS_MAX DRBD_AL_EXTENTS_MAX -#define DRBD_CONG_EXTENTS_DEF DRBD_AL_EXTENTS_DEF -#define DRBD_CONG_EXTENTS_SCALE DRBD_AL_EXTENTS_SCALE - -#define DRBD_PROTOCOL_DEF DRBD_PROT_C - -#define DRBD_DISK_BARRIER_DEF 0U -#define DRBD_DISK_FLUSHES_DEF 1U -#define DRBD_DISK_DRAIN_DEF 1U -#define DRBD_MD_FLUSHES_DEF 1U -#define DRBD_TCP_CORK_DEF 1U -#define DRBD_AL_UPDATES_DEF 1U - -/* We used to ignore the discard_zeroes_data setting. - * To not change established (and expected) behaviour, - * by default assume that, for discard_zeroes_data=0, - * we can make that an effective discard_zeroes_data=1, - * if we only explicitly zero-out unaligned partial chunks. */ -#define DRBD_DISCARD_ZEROES_IF_ALIGNED_DEF 1U - -/* Some backends pretend to support WRITE SAME, - * but fail such requests when they are actually submitted. - * This is to tell DRBD to not even try. */ -#define DRBD_DISABLE_WRITE_SAME_DEF 0U - -#define DRBD_ALLOW_TWO_PRIMARIES_DEF 0U -#define DRBD_ALWAYS_ASBP_DEF 0U -#define DRBD_USE_RLE_DEF 1U -#define DRBD_CSUMS_AFTER_CRASH_ONLY_DEF 0U - -#define DRBD_AL_STRIPES_MIN 1U -#define DRBD_AL_STRIPES_MAX 1024U -#define DRBD_AL_STRIPES_DEF 1U -#define DRBD_AL_STRIPES_SCALE '1' - -#define DRBD_AL_STRIPE_SIZE_MIN 4U -#define DRBD_AL_STRIPE_SIZE_MAX 16777216U -#define DRBD_AL_STRIPE_SIZE_DEF 32U -#define DRBD_AL_STRIPE_SIZE_SCALE 'k' /* kilobytes */ - -#define DRBD_SOCKET_CHECK_TIMEO_MIN 0U -#define DRBD_SOCKET_CHECK_TIMEO_MAX DRBD_PING_TIMEO_MAX -#define DRBD_SOCKET_CHECK_TIMEO_DEF 0U -#define DRBD_SOCKET_CHECK_TIMEO_SCALE '1' - -#define DRBD_RS_DISCARD_GRANULARITY_MIN 0U -#define DRBD_RS_DISCARD_GRANULARITY_MAX (1U<<20) /* 1MiByte */ -#define DRBD_RS_DISCARD_GRANULARITY_DEF 0U /* disabled by default */ -#define DRBD_RS_DISCARD_GRANULARITY_SCALE '1' /* bytes */ - -#endif diff --git a/include/linux/dsa/loop.h b/include/linux/dsa/loop.h deleted file mode 100644 index b8fef35591aa..000000000000 --- a/include/linux/dsa/loop.h +++ /dev/null @@ -1,42 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef DSA_LOOP_H -#define DSA_LOOP_H - -#include <linux/if_vlan.h> -#include <linux/types.h> -#include <linux/ethtool.h> -#include <net/dsa.h> - -struct dsa_loop_vlan { - u16 members; - u16 untagged; -}; - -struct dsa_loop_mib_entry { - char name[ETH_GSTRING_LEN]; - unsigned long val; -}; - -enum dsa_loop_mib_counters { - DSA_LOOP_PHY_READ_OK, - DSA_LOOP_PHY_READ_ERR, - DSA_LOOP_PHY_WRITE_OK, - DSA_LOOP_PHY_WRITE_ERR, - __DSA_LOOP_CNT_MAX, -}; - -struct dsa_loop_port { - struct dsa_loop_mib_entry mib[__DSA_LOOP_CNT_MAX]; - u16 pvid; - int mtu; -}; - -struct dsa_loop_priv { - struct mii_bus *bus; - unsigned int port_base; - struct dsa_loop_vlan vlans[VLAN_N_VID]; - struct net_device *netdev; - struct dsa_loop_port ports[DSA_MAX_PORTS]; -}; - -#endif /* DSA_LOOP_H */ diff --git a/include/linux/dsa/tag_netc.h b/include/linux/dsa/tag_netc.h new file mode 100644 index 000000000000..fe964722e5b0 --- /dev/null +++ b/include/linux/dsa/tag_netc.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: GPL-2.0 + * + * Copyright 2025-2026 NXP + */ + +#ifndef __NET_DSA_TAG_NETC_H +#define __NET_DSA_TAG_NETC_H + +#include <linux/skbuff.h> +#include <net/dsa.h> + +#define NETC_TAG_MAX_LEN 14 + +#endif diff --git a/include/linux/dtlk.h b/include/linux/dtlk.h deleted file mode 100644 index 27b95e70bde3..000000000000 --- a/include/linux/dtlk.h +++ /dev/null @@ -1,86 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#define DTLK_MINOR 0 -#define DTLK_IO_EXTENT 0x02 - - /* ioctl's use magic number of 0xa3 */ -#define DTLK_INTERROGATE 0xa390 /* get settings from the DoubleTalk */ -#define DTLK_STATUS 0xa391 /* get status from the DoubleTalk */ - - -#define DTLK_CLEAR 0x18 /* stops speech */ - -#define DTLK_MAX_RETRIES (loops_per_jiffy/(10000/HZ)) - - /* TTS Port Status Flags */ -#define TTS_READABLE 0x80 /* mask for bit which is nonzero if a - byte can be read from the TTS port */ -#define TTS_SPEAKING 0x40 /* mask for SYNC bit, which is nonzero - while DoubleTalk is producing - output with TTS, PCM or CVSD - synthesizers or tone generators - (that is, all but LPC) */ -#define TTS_SPEAKING2 0x20 /* mask for SYNC2 bit, - which falls to zero up to 0.4 sec - before speech stops */ -#define TTS_WRITABLE 0x10 /* mask for RDY bit, which when set to - 1, indicates the TTS port is ready - to accept a byte of data. The RDY - bit goes zero 2-3 usec after - writing, and goes 1 again 180-190 - usec later. */ -#define TTS_ALMOST_FULL 0x08 /* mask for AF bit: When set to 1, - indicates that less than 300 free - bytes are available in the TTS - input buffer. AF is always 0 in the - PCM, TGN and CVSD modes. */ -#define TTS_ALMOST_EMPTY 0x04 /* mask for AE bit: When set to 1, - indicates that less than 300 bytes - of data remain in DoubleTalk's - input (TTS or PCM) buffer. AE is - always 1 in the TGN and CVSD - modes. */ - - /* LPC speak commands */ -#define LPC_5220_NORMAL 0x60 /* 5220 format decoding table, normal rate */ -#define LPC_5220_FAST 0x64 /* 5220 format decoding table, fast rate */ -#define LPC_D6_NORMAL 0x20 /* D6 format decoding table, normal rate */ -#define LPC_D6_FAST 0x24 /* D6 format decoding table, fast rate */ - - /* LPC Port Status Flags (valid only after one of the LPC - speak commands) */ -#define LPC_SPEAKING 0x80 /* mask for TS bit: When set to 1, - indicates the LPC synthesizer is - producing speech.*/ -#define LPC_BUFFER_LOW 0x40 /* mask for BL bit: When set to 1, - indicates that the hardware LPC - data buffer has less than 30 bytes - remaining. (Total internal buffer - size = 4096 bytes.) */ -#define LPC_BUFFER_EMPTY 0x20 /* mask for BE bit: When set to 1, - indicates that the LPC data buffer - ran out of data (error condition if - TS is also 1). */ - - /* data returned by Interrogate command */ -struct dtlk_settings -{ - unsigned short serial_number; /* 0-7Fh:0-7Fh */ - unsigned char rom_version[24]; /* null terminated string */ - unsigned char mode; /* 0=Character; 1=Phoneme; 2=Text */ - unsigned char punc_level; /* nB; 0-7 */ - unsigned char formant_freq; /* nF; 0-9 */ - unsigned char pitch; /* nP; 0-99 */ - unsigned char speed; /* nS; 0-9 */ - unsigned char volume; /* nV; 0-9 */ - unsigned char tone; /* nX; 0-2 */ - unsigned char expression; /* nE; 0-9 */ - unsigned char ext_dict_loaded; /* 1=exception dictionary loaded */ - unsigned char ext_dict_status; /* 1=exception dictionary enabled */ - unsigned char free_ram; /* # pages (truncated) remaining for - text buffer */ - unsigned char articulation; /* nA; 0-9 */ - unsigned char reverb; /* nR; 0-9 */ - unsigned char eob; /* 7Fh value indicating end of - parameter block */ - unsigned char has_indexing; /* nonzero if indexing is implemented */ -}; diff --git a/include/linux/edac.h b/include/linux/edac.h index fa32f2aca22f..f7a8218f9cc0 100644 --- a/include/linux/edac.h +++ b/include/linux/edac.h @@ -184,6 +184,7 @@ static inline char *mc_event_error_type(const unsigned int err_type) * @MEM_DDR5: Unbuffered DDR5 RAM * @MEM_RDDR5: Registered DDR5 RAM * @MEM_LRDDR5: Load-Reduced DDR5 memory. + * @MEM_LPDDR5: Low-Power DDR5 memory. * @MEM_NVDIMM: Non-volatile RAM * @MEM_WIO2: Wide I/O 2. * @MEM_HBM2: High bandwidth Memory Gen 2. @@ -216,6 +217,7 @@ enum mem_type { MEM_DDR5, MEM_RDDR5, MEM_LRDDR5, + MEM_LPDDR5, MEM_NVDIMM, MEM_WIO2, MEM_HBM2, @@ -247,6 +249,7 @@ enum mem_type { #define MEM_FLAG_DDR5 BIT(MEM_DDR5) #define MEM_FLAG_RDDR5 BIT(MEM_RDDR5) #define MEM_FLAG_LRDDR5 BIT(MEM_LRDDR5) +#define MEM_FLAG_LPDDR5 BIT(MEM_LPDDR5) #define MEM_FLAG_NVDIMM BIT(MEM_NVDIMM) #define MEM_FLAG_WIO2 BIT(MEM_WIO2) #define MEM_FLAG_HBM2 BIT(MEM_HBM2) @@ -541,17 +544,6 @@ struct mem_ctl_info { struct csrow_info **csrows; unsigned int nr_csrows, num_cschannel; - /* - * Memory Controller hierarchy - * - * There are basically two types of memory controller: the ones that - * sees memory sticks ("dimms"), and the ones that sees memory ranks. - * All old memory controllers enumerate memories per rank, but most - * of the recent drivers enumerate memories per DIMM, instead. - * When the memory controller is per rank, csbased is true. - */ - unsigned int n_layers; - struct edac_mc_layer *layers; bool csbased; /* @@ -606,9 +598,18 @@ struct mem_ctl_info { int op_state; struct dentry *debugfs; - u8 fake_inject_layer[EDAC_MAX_LAYERS]; - bool fake_inject_ue; - u16 fake_inject_count; + + /* + * Memory Controller hierarchy + * + * There are basically two types of memory controller: the ones that + * sees memory sticks ("dimms"), and the ones that sees memory ranks. + * All old memory controllers enumerate memories per rank, but most + * of the recent drivers enumerate memories per DIMM, instead. + * When the memory controller is per rank, csbased is true. + */ + unsigned int n_layers; + struct edac_mc_layer layers[] __counted_by(n_layers); }; #define mci_for_each_dimm(mci, dimm) \ diff --git a/include/linux/efi.h b/include/linux/efi.h index 664898d09ff5..aa15ff88539b 100644 --- a/include/linux/efi.h +++ b/include/linux/efi.h @@ -61,7 +61,7 @@ typedef void *efi_handle_t; /* * The UEFI spec and EDK2 reference implementation both define EFI_GUID as - * struct { u32 a; u16; b; u16 c; u8 d[8]; }; and so the implied alignment + * struct { u32 a; u16 b; u16 c; u8 d[8]; }; and so the implied alignment * is 32 bits not 8 bits like our guid_t. In some cases (i.e., on 32-bit ARM), * this means that firmware services invoked by the kernel may assume that * efi_guid_t* arguments are 32-bit aligned, and use memory accessors that @@ -705,7 +705,7 @@ efi_guidcmp (efi_guid_t left, efi_guid_t right) } static inline char * -efi_guid_to_str(efi_guid_t *guid, char *out) +efi_guid_to_str(const efi_guid_t *guid, char *out) { sprintf(out, "%pUl", guid->b); return out; @@ -832,27 +832,6 @@ extern int __init parse_efi_signature_list( const void *data, size_t size, efi_element_handler_t (*get_handler_for_guid)(const efi_guid_t *)); -/** - * efi_range_is_wc - check the WC bit on an address range - * @start: starting kvirt address - * @len: length of range - * - * Consult the EFI memory map and make sure it's ok to set this range WC. - * Returns true or false. - */ -static inline int efi_range_is_wc(unsigned long start, unsigned long len) -{ - unsigned long i; - - for (i = 0; i < len; i += (1UL << EFI_PAGE_SHIFT)) { - unsigned long paddr = __pa(start + i); - if (!(efi_mem_attributes(paddr) & EFI_MEMORY_WC)) - return 0; - } - /* The range checked out */ - return 1; -} - /* * We play games with efi_enabled so that the compiler will, if * possible, remove EFI-related code altogether. @@ -1233,8 +1212,8 @@ efi_call_acpi_prm_handler(efi_status_t (__efiapi *handler_addr)(u64, void *), /* * efi_runtime_service() function identifiers. - * "NONE" is used by efi_recover_from_page_fault() to check if the page - * fault happened while executing an efi runtime service. + * "NONE" is used by efi_crash_gracefully_on_page_fault() to check if the + * page fault happened while executing an efi runtime service. */ enum efi_rts_ids { EFI_NONE, @@ -1277,6 +1256,8 @@ extern struct efi_runtime_work efi_rts_work; /* Workqueue to queue EFI Runtime Services */ extern struct workqueue_struct *efi_rts_wq; +void __noreturn efi_rts_park_worker(void); + struct linux_efi_memreserve { int size; // allocated size of the array atomic_t count; // number of entries used diff --git a/include/linux/efs_vh.h b/include/linux/efs_vh.h deleted file mode 100644 index 206c5270f7b8..000000000000 --- a/include/linux/efs_vh.h +++ /dev/null @@ -1,54 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * efs_vh.h - * - * Copyright (c) 1999 Al Smith - * - * Portions derived from IRIX header files (c) 1985 MIPS Computer Systems, Inc. - */ - -#ifndef __EFS_VH_H__ -#define __EFS_VH_H__ - -#define VHMAGIC 0xbe5a941 /* volume header magic number */ -#define NPARTAB 16 /* 16 unix partitions */ -#define NVDIR 15 /* max of 15 directory entries */ -#define BFNAMESIZE 16 /* max 16 chars in boot file name */ -#define VDNAMESIZE 8 - -struct volume_directory { - char vd_name[VDNAMESIZE]; /* name */ - __be32 vd_lbn; /* logical block number */ - __be32 vd_nbytes; /* file length in bytes */ -}; - -struct partition_table { /* one per logical partition */ - __be32 pt_nblks; /* # of logical blks in partition */ - __be32 pt_firstlbn; /* first lbn of partition */ - __be32 pt_type; /* use of partition */ -}; - -struct volume_header { - __be32 vh_magic; /* identifies volume header */ - __be16 vh_rootpt; /* root partition number */ - __be16 vh_swappt; /* swap partition number */ - char vh_bootfile[BFNAMESIZE]; /* name of file to boot */ - char pad[48]; /* device param space */ - struct volume_directory vh_vd[NVDIR]; /* other vol hdr contents */ - struct partition_table vh_pt[NPARTAB]; /* device partition layout */ - __be32 vh_csum; /* volume header checksum */ - __be32 vh_fill; /* fill out to 512 bytes */ -}; - -/* partition type sysv is used for EFS format CD-ROM partitions */ -#define SGI_SYSV 0x05 -#define SGI_EFS 0x07 -#define IS_EFS(x) (((x) == SGI_EFS) || ((x) == SGI_SYSV)) - -struct pt_types { - int pt_type; - char *pt_name; -}; - -#endif /* __EFS_VH_H__ */ - diff --git a/include/linux/eisa.h b/include/linux/eisa.h index cf55630b595b..52a97dc4c85a 100644 --- a/include/linux/eisa.h +++ b/include/linux/eisa.h @@ -4,7 +4,7 @@ #include <linux/ioport.h> #include <linux/device.h> -#include <linux/mod_devicetable.h> +#include <linux/device-id/eisa.h> #define EISA_MAX_SLOTS 8 diff --git a/include/linux/energy_model.h b/include/linux/energy_model.h index e7497f804644..c909a8ba22e8 100644 --- a/include/linux/energy_model.h +++ b/include/linux/energy_model.h @@ -248,7 +248,7 @@ static inline unsigned long em_cpu_energy(struct em_perf_domain *pd, struct em_perf_state *ps; int i; - WARN_ONCE(!rcu_read_lock_held(), "EM: rcu read lock needed\n"); + lockdep_assert(rcu_read_lock_any_held()); if (!sum_util) return 0; @@ -267,7 +267,7 @@ static inline unsigned long em_cpu_energy(struct em_perf_domain *pd, * Find the lowest performance state of the Energy Model above the * requested performance. */ - em_table = rcu_dereference(pd->em_table); + em_table = rcu_dereference_all(pd->em_table); i = em_pd_get_efficient_state(em_table->state, pd, max_util); ps = &em_table->state[i]; diff --git a/include/linux/entry-common.h b/include/linux/entry-common.h index f83ca0abf2cd..fa2854fed1f2 100644 --- a/include/linux/entry-common.h +++ b/include/linux/entry-common.h @@ -6,9 +6,11 @@ #include <linux/irq-entry-common.h> #include <linux/livepatch.h> #include <linux/ptrace.h> +#include <linux/randomize_kstack.h> #include <linux/resume_user_mode.h> #include <linux/seccomp.h> #include <linux/sched.h> +#include <linux/syscall_user_dispatch.h> #include <asm/entry-common.h> #include <asm/syscall.h> @@ -18,70 +20,51 @@ #endif /* - * SYSCALL_WORK flags handled in syscall_enter_from_user_mode() + * SYSCALL_WORK flags handled in syscall_enter_from_user_mode_work() */ -#ifndef ARCH_SYSCALL_WORK_ENTER -# define ARCH_SYSCALL_WORK_ENTER (0) -#endif - -/* - * SYSCALL_WORK flags handled in syscall_exit_to_user_mode() - */ -#ifndef ARCH_SYSCALL_WORK_EXIT -# define ARCH_SYSCALL_WORK_EXIT (0) -#endif - #define SYSCALL_WORK_ENTER (SYSCALL_WORK_SECCOMP | \ SYSCALL_WORK_SYSCALL_TRACEPOINT | \ SYSCALL_WORK_SYSCALL_TRACE | \ SYSCALL_WORK_SYSCALL_EMU | \ SYSCALL_WORK_SYSCALL_AUDIT | \ SYSCALL_WORK_SYSCALL_USER_DISPATCH | \ - SYSCALL_WORK_SYSCALL_RSEQ_SLICE | \ - ARCH_SYSCALL_WORK_ENTER) + SYSCALL_WORK_SYSCALL_RSEQ_SLICE) +/* + * SYSCALL_WORK flags handled in syscall_exit_to_user_mode() + */ #define SYSCALL_WORK_EXIT (SYSCALL_WORK_SYSCALL_TRACEPOINT | \ SYSCALL_WORK_SYSCALL_TRACE | \ SYSCALL_WORK_SYSCALL_AUDIT | \ SYSCALL_WORK_SYSCALL_USER_DISPATCH | \ - SYSCALL_WORK_SYSCALL_EXIT_TRAP | \ - ARCH_SYSCALL_WORK_EXIT) + SYSCALL_WORK_SYSCALL_EXIT_TRAP) /** - * arch_ptrace_report_syscall_entry - Architecture specific ptrace_report_syscall_entry() wrapper + * arch_ptrace_report_syscall_permit_entry - Architecture specific wrapper for + * ptrace_report_syscall_permit_entry() + * @regs: Pointer to the register state at syscall entry * - * Invoked from syscall_trace_enter() to wrap ptrace_report_syscall_entry(). + * Invoked from syscall_trace_enter() to wrap ptrace_report_syscall_permit_entry(). * - * This allows architecture specific ptrace_report_syscall_entry() + * This allows architecture specific ptrace_report_syscall_permit_entry() * implementations. If not defined by the architecture this falls back to - * to ptrace_report_syscall_entry(). + * to ptrace_report_syscall_permit_entry(). */ -static __always_inline int arch_ptrace_report_syscall_entry(struct pt_regs *regs); +static __always_inline bool arch_ptrace_report_syscall_permit_entry(struct pt_regs *regs); -#ifndef arch_ptrace_report_syscall_entry -static __always_inline int arch_ptrace_report_syscall_entry(struct pt_regs *regs) +#ifndef arch_ptrace_report_syscall_permit_entry +static __always_inline bool arch_ptrace_report_syscall_permit_entry(struct pt_regs *regs) { - return ptrace_report_syscall_entry(regs); + return ptrace_report_syscall_permit_entry(regs); } #endif -bool syscall_user_dispatch(struct pt_regs *regs); -long trace_syscall_enter(struct pt_regs *regs, long syscall); +void trace_syscall_enter(struct pt_regs *regs); void trace_syscall_exit(struct pt_regs *regs, long ret); +void syscall_enter_audit(struct pt_regs *regs); -static inline void syscall_enter_audit(struct pt_regs *regs, long syscall) +static __always_inline long syscall_trace_enter(struct pt_regs *regs, unsigned long work, + long syscall) { - if (unlikely(audit_context())) { - unsigned long args[6]; - - syscall_get_arguments(current, regs, args); - audit_syscall_entry(syscall, args[0], args[1], args[2], args[3]); - } -} - -static __always_inline long syscall_trace_enter(struct pt_regs *regs, unsigned long work) -{ - long syscall, ret = 0; - /* * Handle Syscall User Dispatch. This must comes first, since * the ABI here can be something that doesn't make sense for @@ -89,7 +72,7 @@ static __always_inline long syscall_trace_enter(struct pt_regs *regs, unsigned l */ if (work & SYSCALL_WORK_SYSCALL_USER_DISPATCH) { if (syscall_user_dispatch(regs)) - return -1L; + return false; } /* @@ -98,31 +81,38 @@ static __always_inline long syscall_trace_enter(struct pt_regs *regs, unsigned l * through hrtimer_interrupt(). */ if (work & SYSCALL_WORK_SYSCALL_RSEQ_SLICE) - rseq_syscall_enter_work(syscall_get_nr(current, regs)); + rseq_syscall_enter_work(syscall); /* Handle ptrace */ if (work & (SYSCALL_WORK_SYSCALL_TRACE | SYSCALL_WORK_SYSCALL_EMU)) { - ret = arch_ptrace_report_syscall_entry(regs); - if (ret || (work & SYSCALL_WORK_SYSCALL_EMU)) - return -1L; + if (!arch_ptrace_report_syscall_permit_entry(regs) || + (work & SYSCALL_WORK_SYSCALL_EMU)) + return false; + + /* ptrace might have changed work flags */ + work = READ_ONCE(current_thread_info()->syscall_work); } /* Do seccomp after ptrace, to catch any tracer changes. */ if (work & SYSCALL_WORK_SECCOMP) { - ret = __secure_computing(); - if (ret == -1L) - return ret; + if (!__seccomp_permit_syscall()) + return false; } - /* Either of the above might have changed the syscall number */ - syscall = syscall_get_nr(current, regs); - if (unlikely(work & SYSCALL_WORK_SYSCALL_TRACEPOINT)) - syscall = trace_syscall_enter(regs, syscall); + trace_syscall_enter(regs); - syscall_enter_audit(regs, syscall); + /* + * The config check works around broken compilers which fail to + * eliminate the dead code in case of CONFIG_AUDITSYSCALL=n as they + * insist on creating a always false runtime condition based on + * audit_context() which returns NULL in that case. The explicit + * IS_ENABLED() check makes that madness go away. + */ + if (IS_ENABLED(CONFIG_AUDITSYSCALL) && unlikely(audit_context())) + syscall_enter_audit(regs); - return ret ? : syscall; + return true; } /** @@ -131,36 +121,63 @@ static __always_inline long syscall_trace_enter(struct pt_regs *regs, unsigned l * @regs: Pointer to currents pt_regs * @syscall: The syscall number * - * Invoked from architecture specific syscall entry code with interrupts - * enabled after invoking enter_from_user_mode(), enabling interrupts and - * extra architecture specific work. + * Invoked from architecture specific syscall entry code with interrupts enabled + * after invoking enter_from_user_mode(), enabling interrupts and extra + * architecture specific work with the syscall return value preset to -ENOSYS. * - * Returns: The original or a modified syscall number + * Returns: True if the syscall should be invoked, False otherwise. * - * If the returned syscall number is -1 then the syscall should be - * skipped. In this case the caller may invoke syscall_set_error() or - * syscall_set_return_value() first. If neither of those are called and -1 - * is returned, then the syscall will fail with ENOSYS. + * If the return value is false, the caller must skip the syscall and leave the + * syscall return value unmodified as it might have been set by one of the entry + * work functions. * * It handles the following work items: * * 1) syscall_work flag dependent invocations of - * ptrace_report_syscall_entry(), __secure_computing(), trace_sys_enter() + * ptrace_report_syscall_permit_entry(), __seccomp_permit_syscall(), trace_sys_enter() * 2) Invocation of audit_syscall_entry() */ -static __always_inline long syscall_enter_from_user_mode_work(struct pt_regs *regs, long syscall) +static __always_inline bool syscall_enter_from_user_mode_work(struct pt_regs *regs, long *syscall) { unsigned long work = READ_ONCE(current_thread_info()->syscall_work); - if (work & SYSCALL_WORK_ENTER) - syscall = syscall_trace_enter(regs, work); + if (!(work & SYSCALL_WORK_ENTER)) + return true; + + if (unlikely(!syscall_trace_enter(regs, work, *syscall))) + return false; - return syscall; + /* Reread the syscall number as it might have been modified */ + *syscall = syscall_get_nr(current, regs); + + return true; } /** - * syscall_enter_from_user_mode - Establish state and check and handle work - * before invoking a syscall + * enter_from_user_mode_randomize_stack - Establish state and add stack randomization + * before invoking syscall_enter_from_user_mode_work() + * @regs: Pointer to currents pt_regs + * + * Invoked from architecture specific syscall entry code with interrupts + * disabled. The calling code has to be non-instrumentable. When the function + * returns all state is correct, interrupts are still disabled and the + * subsequent functions can be instrumented. + * + * Implemented as a macro so that the stack randomization is effective + * throughout the function in which it is invoked. An inline would only make it + * effective in the scope of the inline function. + */ +#define enter_from_user_mode_randomize_stack(regs) \ +do { \ + enter_from_user_mode(regs); \ + instrumentation_begin(); \ + add_random_kstack_offset_irqsoff(); \ + instrumentation_end(); \ +} while (0) + +/** + * syscall_enter_from_user_mode_randomize_stack - Establish state and check and handle work + * before invoking a syscall * @regs: Pointer to currents pt_regs * @syscall: The syscall number * @@ -169,31 +186,32 @@ static __always_inline long syscall_enter_from_user_mode_work(struct pt_regs *re * function returns all state is correct, interrupts are enabled and the * subsequent functions can be instrumented. * - * This is the combination of enter_from_user_mode() and + * This is the combination of enter_from_user_mode_randomize_stack() and * syscall_enter_from_user_mode_work() to be used when there is no * architecture specific work to be done between the two. * * Returns: The original or a modified syscall number. See * syscall_enter_from_user_mode_work() for further explanation. + * + * Implemented as a macro to make stack randomization effective in the calling + * scope. */ -static __always_inline long syscall_enter_from_user_mode(struct pt_regs *regs, long syscall) -{ - long ret; - - enter_from_user_mode(regs); - - instrumentation_begin(); - local_irq_enable(); - ret = syscall_enter_from_user_mode_work(regs, syscall); - instrumentation_end(); - - return ret; -} +#define syscall_enter_from_user_mode_randomize_stack(regs, syscall) \ +({ \ + enter_from_user_mode_randomize_stack(regs); \ + \ + instrumentation_begin(); \ + local_irq_enable(); \ + long _ret = syscall_enter_from_user_mode_work(regs, syscall); \ + instrumentation_end(); \ + \ + _ret; \ +}) /* - * If SYSCALL_EMU is set, then the only reason to report is when - * SINGLESTEP is set (i.e. PTRACE_SYSEMU_SINGLESTEP). This syscall - * instruction has been already reported in syscall_enter_from_user_mode(). + * If SYSCALL_EMU is set, then the only reason to report is when SINGLESTEP is + * set (i.e. PTRACE_SYSEMU_SINGLESTEP). This syscall instruction has been + * already reported in syscall_enter_from_user_mode_work(). */ static __always_inline bool report_single_step(unsigned long work) { @@ -205,6 +223,8 @@ static __always_inline bool report_single_step(unsigned long work) /** * arch_ptrace_report_syscall_exit - Architecture specific ptrace_report_syscall_exit() + * @regs: Pointer to the register state at syscall exit + * @step: Indicates a single-step exit rather than a normal syscall exit * * This allows architecture specific ptrace_report_syscall_exit() * implementations. If not defined by the architecture this falls back to @@ -239,10 +259,8 @@ static __always_inline void syscall_exit_work(struct pt_regs *regs, unsigned lon * of these syscalls is unknown. */ if (work & SYSCALL_WORK_SYSCALL_USER_DISPATCH) { - if (unlikely(current->syscall_dispatch.on_dispatch)) { - current->syscall_dispatch.on_dispatch = false; + if (syscall_user_dispatch_clear_on_dispatch()) return; - } } audit_syscall_exit(regs); @@ -321,7 +339,7 @@ static __always_inline void syscall_exit_to_user_mode(struct pt_regs *regs) { instrumentation_begin(); syscall_exit_to_user_mode_work(regs); - local_irq_disable_exit_to_user(); + local_irq_disable(); syscall_exit_to_user_mode_prepare(regs); instrumentation_end(); exit_to_user_mode(); diff --git a/include/linux/err.h b/include/linux/err.h index 8c37be0620ab..d3e38d5b3a98 100644 --- a/include/linux/err.h +++ b/include/linux/err.h @@ -36,7 +36,7 @@ * * Return: A pointer with @error encoded within its value. */ -static inline void * __must_check ERR_PTR(long error) +static __always_inline void * __must_check ERR_PTR(long error) { return (void *) error; } @@ -60,7 +60,7 @@ static inline void * __must_check ERR_PTR(long error) * @ptr: An error pointer. * Return: The error code within @ptr. */ -static inline long __must_check PTR_ERR(__force const void *ptr) +static __always_inline long __must_check PTR_ERR(__force const void *ptr) { return (long) ptr; } @@ -73,7 +73,7 @@ static inline long __must_check PTR_ERR(__force const void *ptr) * @ptr: The pointer to check. * Return: true if @ptr is an error pointer, false otherwise. */ -static inline bool __must_check IS_ERR(__force const void *ptr) +static __always_inline bool __must_check IS_ERR(__force const void *ptr) { return IS_ERR_VALUE((unsigned long)ptr); } @@ -87,7 +87,7 @@ static inline bool __must_check IS_ERR(__force const void *ptr) * * Like IS_ERR(), but also returns true for a null pointer. */ -static inline bool __must_check IS_ERR_OR_NULL(__force const void *ptr) +static __always_inline bool __must_check IS_ERR_OR_NULL(__force const void *ptr) { return unlikely(!ptr) || IS_ERR_VALUE((unsigned long)ptr); } @@ -99,7 +99,7 @@ static inline bool __must_check IS_ERR_OR_NULL(__force const void *ptr) * Explicitly cast an error-valued pointer to another pointer type in such a * way as to make it clear that's what's going on. */ -static inline void * __must_check ERR_CAST(__force const void *ptr) +static __always_inline void * __must_check ERR_CAST(__force const void *ptr) { /* cast away the const */ return (void *) ptr; @@ -122,7 +122,7 @@ static inline void * __must_check ERR_CAST(__force const void *ptr) * * Return: The error code within @ptr if it is an error pointer; 0 otherwise. */ -static inline int __must_check PTR_ERR_OR_ZERO(__force const void *ptr) +static __always_inline int __must_check PTR_ERR_OR_ZERO(__force const void *ptr) { if (IS_ERR(ptr)) return PTR_ERR(ptr); diff --git a/include/linux/ethtool.h b/include/linux/ethtool.h index 798abec67a1b..253600c0eccd 100644 --- a/include/linux/ethtool.h +++ b/include/linux/ethtool.h @@ -176,6 +176,8 @@ static inline u32 ethtool_rxfh_indir_default(u32 index, u32 n_rx_rings) * struct ethtool_rxfh_context - a custom RSS context configuration * @indir_size: Number of u32 entries in indirection table * @key_size: Size of hash key, in bytes + * @indir_user_size: number of user provided entries for the + * indirection table * @priv_size: Size of driver private data, in bytes * @hfunc: RSS hash function identifier. One of the %ETH_RSS_HASH_* * @input_xfrm: Defines how the input data is transformed. Valid values are one @@ -186,6 +188,7 @@ static inline u32 ethtool_rxfh_indir_default(u32 index, u32 n_rx_rings) struct ethtool_rxfh_context { u32 indir_size; u32 key_size; + u32 indir_user_size; u16 priv_size; u8 hfunc; u8 input_xfrm; @@ -214,6 +217,13 @@ static inline u8 *ethtool_rxfh_context_key(struct ethtool_rxfh_context *ctx) } void ethtool_rxfh_context_lost(struct net_device *dev, u32 context_id); +void ethtool_rxfh_indir_lost(struct net_device *dev); +bool ethtool_rxfh_indir_can_resize(struct net_device *dev, const u32 *tbl, + u32 old_size, u32 new_size); +void ethtool_rxfh_indir_resize(struct net_device *dev, u32 *tbl, + u32 old_size, u32 new_size); +int ethtool_rxfh_ctxs_can_resize(struct net_device *dev, u32 new_indir_size); +void ethtool_rxfh_ctxs_resize(struct net_device *dev, u32 new_indir_size); struct link_mode_info { int speed; @@ -315,6 +325,8 @@ struct ethtool_link_ksettings { extern int __ethtool_get_link_ksettings(struct net_device *dev, struct ethtool_link_ksettings *link_ksettings); +int netif_get_link_ksettings(struct net_device *dev, + struct ethtool_link_ksettings *link_ksettings); struct ethtool_keee { __ETHTOOL_DECLARE_LINK_MODE_MASK(supported); @@ -332,6 +344,8 @@ struct kernel_ethtool_coalesce { u32 tx_aggr_max_bytes; u32 tx_aggr_max_frames; u32 tx_aggr_time_usecs; + u32 rx_cqe_frames; + u32 rx_cqe_nsecs; }; /** @@ -380,7 +394,9 @@ bool ethtool_convert_link_mode_to_legacy_u32(u32 *legacy_u32, #define ETHTOOL_COALESCE_TX_AGGR_TIME_USECS BIT(26) #define ETHTOOL_COALESCE_RX_PROFILE BIT(27) #define ETHTOOL_COALESCE_TX_PROFILE BIT(28) -#define ETHTOOL_COALESCE_ALL_PARAMS GENMASK(28, 0) +#define ETHTOOL_COALESCE_RX_CQE_FRAMES BIT(29) +#define ETHTOOL_COALESCE_RX_CQE_NSECS BIT(30) +#define ETHTOOL_COALESCE_ALL_PARAMS GENMASK(30, 0) #define ETHTOOL_COALESCE_USECS \ (ETHTOOL_COALESCE_RX_USECS | ETHTOOL_COALESCE_TX_USECS) @@ -512,12 +528,14 @@ struct ethtool_eth_ctrl_stats { * * Equivalent to `30.3.4.3 aPAUSEMACCtrlFramesReceived` * from the standard. + * @tx_pause_storm_events: TX pause storm event count (see ethtool.yaml). */ struct ethtool_pause_stats { enum ethtool_mac_stats_src src; struct_group(stats, u64 tx_pause_frames; u64 rx_pause_frames; + u64 tx_pause_storm_events; ); }; @@ -544,6 +562,7 @@ struct ethtool_fec_hist { u64 per_lane[ETHTOOL_MAX_LANES]; } values[ETHTOOL_FEC_HIST_MAX]; const struct ethtool_fec_hist_range *ranges; + struct ethtool_fec_hist_range ranges_buf[ETHTOOL_FEC_HIST_MAX]; }; /** * struct ethtool_fec_stats - statistics for IEEE 802.3 FEC @@ -912,6 +931,20 @@ struct kernel_ethtool_ts_info { u32 rx_filters; }; +/* Bits for ethtool_ops::op_needs_rtnl + * LINKSETTINGS cover a number of commands, but in most cases we want to keep + * these bits separate, per GET and SET. GET is much easier to "unlock". + */ +#define ETHTOOL_OP_NEEDS_RTNL_LINKSETTINGS BIT(0) +#define ETHTOOL_OP_NEEDS_RTNL_SPFLAGS BIT(1) +#define ETHTOOL_OP_NEEDS_RTNL_SRINGPARAM BIT(2) +#define ETHTOOL_OP_NEEDS_RTNL_SCHANNELS BIT(3) +#define ETHTOOL_OP_NEEDS_RTNL_SCOALESCE BIT(4) +#define ETHTOOL_OP_NEEDS_RTNL_GPAUSEPARAM BIT(5) +#define ETHTOOL_OP_NEEDS_RTNL_SPAUSEPARAM BIT(6) +#define ETHTOOL_OP_NEEDS_RTNL_RSS BIT(7) +#define ETHTOOL_OP_NEEDS_RTNL_GLINK BIT(8) + /** * struct ethtool_ops - optional netdev operations * @supported_input_xfrm: supported types of input xfrm from %RXH_XFRM_*. @@ -938,6 +971,17 @@ struct kernel_ethtool_ts_info { * @supported_coalesce_params: supported types of interrupt coalescing. * @supported_ring_params: supported ring params. * @supported_hwtstamp_qualifiers: bitfield of supported hwtstamp qualifier. + * @op_needs_rtnl: mask of %ETHTOOL_OP_NEEDS_RTNL_* bits. + * For use with ops-locked drivers (ignored otherwise). Selects which + * ethtool callbacks driver needs to still be executed under rtnl_lock + * (in addition to the netdev instance lock). + * The following commonly used core APIs currently require rtnl_lock + * (this list may not be exhaustive): + * - phylink helpers (note that phydev is currently unsupported!) + * - netdev_update_features() + * - netif_set_real_num_tx_queues() + * - ethtool_op_get_link() (syncs link watch under rtnl_lock) + * * @get_drvinfo: Report driver/device information. Modern drivers no * longer have to implement this callback. Most fields are * correctly filled in by the core using system information, or @@ -1013,6 +1057,12 @@ struct kernel_ethtool_ts_info { * @get_sset_count: Get number of strings that @get_strings will write. * @get_rxnfc: Get RX flow classification rules. Returns a negative * error code or zero. + * Note that for %ETHTOOL_GRXCLSRLALL rule_cnt and size of the arrays + * is user-provided, and not guaranteed to match what driver would + * have reported via %ETHTOOL_GRXCLSRLCNT. Drivers must return -%EMSGSIZE + * when rule_cnt is too small. rule_locs is %NULL when rule_cnt is zero. + * On success drivers must set rule_cnt to the number of locations they + * filled in, the core copies out exactly that many. * @set_rxnfc: Set RX flow classification rules. Returns a negative * error code or zero. * @flash_device: Write a firmware image to device's flash memory. @@ -1136,8 +1186,14 @@ struct kernel_ethtool_ts_info { * @get_mm_stats: Query the 802.3 MAC Merge layer statistics. * * All operations are optional (i.e. the function pointer may be set - * to %NULL) and callers must take this into account. Callers must - * hold the RTNL lock. + * to %NULL) and callers must take this into account. + * + * For traditional drivers callers hold ``rtnl_lock`` across the call. + * For "ops locked" drivers (see Documentation/networking/netdevices.rst) + * callers instead hold the netdev instance lock (``netdev_lock_ops``); + * ``rtnl_lock`` is additionally held only for callbacks for which + * the driver opts in via the matching ``ETHTOOL_OP_NEEDS_RTNL_*`` bit + * in @op_needs_rtnl. * * See the structures used by these operations for further documentation. * Note that for all operations using a structure ending with a zero- @@ -1160,6 +1216,7 @@ struct ethtool_ops { u32 supported_coalesce_params; u32 supported_ring_params; u32 supported_hwtstamp_qualifiers; + u32 op_needs_rtnl; void (*get_drvinfo)(struct net_device *, struct ethtool_drvinfo *); int (*get_regs_len)(struct net_device *); void (*get_regs)(struct net_device *, struct ethtool_regs *, void *); @@ -1331,12 +1388,17 @@ int ethtool_virtdev_set_link_ksettings(struct net_device *dev, * @rss_ctx: XArray of custom RSS contexts * @rss_lock: Protects entries in @rss_ctx. May be taken from * within RTNL. + * @rss_indir_user_size: Number of user provided entries for the default + * (context 0) indirection table. + * @phys_id_busy: Loop blinking the device LED is running. * @wol_enabled: Wake-on-LAN is enabled * @module_fw_flash_in_progress: Module firmware flashing is in progress. */ struct ethtool_netdev_state { struct xarray rss_ctx; struct mutex rss_lock; + u32 rss_indir_user_size; + unsigned phys_id_busy:1; unsigned wol_enabled:1; unsigned module_fw_flash_in_progress:1; }; diff --git a/include/linux/ethtool_netlink.h b/include/linux/ethtool_netlink.h index 39254b2726c0..15881f34ef66 100644 --- a/include/linux/ethtool_netlink.h +++ b/include/linux/ethtool_netlink.h @@ -7,9 +7,6 @@ #include <linux/ethtool.h> #include <linux/netdevice.h> -#define __ETHTOOL_LINK_MODE_MASK_NWORDS \ - DIV_ROUND_UP(__ETHTOOL_LINK_MODE_MASK_NBITS, 32) - #define ETHTOOL_PAUSE_STAT_CNT (__ETHTOOL_A_PAUSE_STAT_CNT - \ ETHTOOL_A_PAUSE_STAT_TX_FRAMES) diff --git a/include/linux/eventpoll.h b/include/linux/eventpoll.h index ea9ca0e4172a..de1c738aa8ad 100644 --- a/include/linux/eventpoll.h +++ b/include/linux/eventpoll.h @@ -39,12 +39,16 @@ static inline void eventpoll_release(struct file *file) { /* - * Fast check to avoid the get/release of the semaphore. Since - * we're doing this outside the semaphore lock, it might return - * false negatives, but we don't care. It'll help in 99.99% of cases - * to avoid the semaphore lock. False positives simply cannot happen - * because the file in on the way to be removed and nobody ( but - * eventpoll ) has still a reference to this file. + * Fast check to skip the slow path in the common case where the + * file was never attached to an epoll. Safe without file->f_lock + * because every f_ep writer excludes a concurrent __fput() on + * @file: + * - ep_insert() requires the file alive (refcount > 0); + * - ep_remove() holds @file pinned via epi_fget() across the + * write; + * - eventpoll_release_file() runs from __fput() itself. + * We are in __fput() here, so none of those can race us: a NULL + * observation truly means no epoll path has work left on @file. */ if (likely(!READ_ONCE(file->f_ep))) return; @@ -57,8 +61,16 @@ static inline void eventpoll_release(struct file *file) eventpoll_release_file(file); } +struct epoll_key { + struct file *file; + int fd; +} __packed; + +int do_epoll_ctl_file(struct file *f, int op, struct epoll_key *tf, + struct epoll_event *epds, bool nonblock); int do_epoll_ctl(int epfd, int op, int fd, struct epoll_event *epds, bool nonblock); +bool is_file_epoll(struct file *f); /* Tells if the epoll_ctl(2) operation needs an event copy from userspace */ static inline int ep_op_has_event(int op) diff --git a/include/linux/evm.h b/include/linux/evm.h index ddece4a6b25d..913f4573b203 100644 --- a/include/linux/evm.h +++ b/include/linux/evm.h @@ -18,6 +18,8 @@ extern enum integrity_status evm_verifyxattr(struct dentry *dentry, const char *xattr_name, void *xattr_value, size_t xattr_value_len); +int evm_fix_hmac(struct dentry *dentry, const char *xattr_name, + const char *xattr_value, size_t xattr_value_len); int evm_inode_init_security(struct inode *inode, struct inode *dir, const struct qstr *qstr, struct xattr *xattrs, int *xattr_count); @@ -51,6 +53,12 @@ static inline enum integrity_status evm_verifyxattr(struct dentry *dentry, { return INTEGRITY_UNKNOWN; } + +static inline int evm_fix_hmac(struct dentry *dentry, const char *xattr_name, + const char *xattr_value, size_t xattr_value_len) +{ + return -EOPNOTSUPP; +} #endif static inline int evm_inode_init_security(struct inode *inode, struct inode *dir, diff --git a/include/linux/execmem.h b/include/linux/execmem.h index 7de229134e30..1bd34925d1aa 100644 --- a/include/linux/execmem.h +++ b/include/linux/execmem.h @@ -89,7 +89,7 @@ static inline int execmem_restore_rox(void *ptr, size_t size) { return 0; } * @end: address space end (inclusive) * @fallback_start: start of the secondary address space range for fallback * allocations on architectures that require it - * @fallback_end: start of the secondary address space (inclusive) + * @fallback_end: end of the secondary address space (inclusive) * @pgprot: permissions for memory in this address space * @alignment: alignment required for text allocations * @flags: options for memory allocations for this range diff --git a/include/linux/export-internal.h b/include/linux/export-internal.h index d445705ac13c..726054614752 100644 --- a/include/linux/export-internal.h +++ b/include/linux/export-internal.h @@ -37,14 +37,14 @@ * section flag requires it. Use '%progbits' instead of '@progbits' since the * former apparently works on all arches according to the binutils source. */ -#define __KSYMTAB(name, sym, sec, ns) \ +#define __KSYMTAB(name, sym, ns) \ asm(" .section \"__ksymtab_strings\",\"aMS\",%progbits,1" "\n" \ "__kstrtab_" #name ":" "\n" \ " .asciz \"" #name "\"" "\n" \ "__kstrtabns_" #name ":" "\n" \ " .asciz \"" ns "\"" "\n" \ " .previous" "\n" \ - " .section \"___ksymtab" sec "+" #name "\", \"a\"" "\n" \ + " .section \"___ksymtab+" #name "\", \"a\"" "\n" \ __KSYM_ALIGN "\n" \ "__ksymtab_" #name ":" "\n" \ __KSYM_REF(sym) "\n" \ @@ -59,14 +59,22 @@ #define KSYM_FUNC(name) name #endif -#define KSYMTAB_FUNC(name, sec, ns) __KSYMTAB(name, KSYM_FUNC(name), sec, ns) -#define KSYMTAB_DATA(name, sec, ns) __KSYMTAB(name, name, sec, ns) +#define KSYMTAB_FUNC(name, ns) __KSYMTAB(name, KSYM_FUNC(name), ns) +#define KSYMTAB_DATA(name, ns) __KSYMTAB(name, name, ns) -#define SYMBOL_CRC(sym, crc, sec) \ - asm(".section \"___kcrctab" sec "+" #sym "\",\"a\"" "\n" \ - ".balign 4" "\n" \ - "__crc_" #sym ":" "\n" \ - ".long " #crc "\n" \ - ".previous" "\n") +#define SYMBOL_CRC(sym, crc) \ + asm(" .section \"___kcrctab+" #sym "\",\"a\"" "\n" \ + " .balign 4" "\n" \ + "__crc_" #sym ":" "\n" \ + " .long " #crc "\n" \ + " .previous" "\n" \ + ) + +#define SYMBOL_FLAGS(sym, flags) \ + asm(" .section \"___kflagstab+" #sym "\",\"a\"" "\n" \ + "__flags_" #sym ":" "\n" \ + " .byte " #flags "\n" \ + " .previous" "\n" \ + ) #endif /* __LINUX_EXPORT_INTERNAL_H__ */ diff --git a/include/linux/exportfs.h b/include/linux/exportfs.h index 8bcdba28b406..c835bc64f4fa 100644 --- a/include/linux/exportfs.h +++ b/include/linux/exportfs.h @@ -6,9 +6,8 @@ #include <linux/path.h> struct dentry; -struct iattr; +struct exportfs_block_ops; struct inode; -struct iomap; struct super_block; struct vfsmount; @@ -260,19 +259,13 @@ struct handle_to_path_ctx { * @commit_metadata: * @commit_metadata should commit metadata changes to stable storage. * - * @get_uuid: - * Get a filesystem unique signature exposed to clients. - * - * @map_blocks: - * Map and, if necessary, allocate blocks for a layout. - * - * @commit_blocks: - * Commit blocks in a layout once the client is done with them. - * * @flags: * Allows the filesystem to communicate to nfsd that it may want to do things * differently when dealing with it. * + * @block_ops: + * Operations for layout grants to block on the underlying device. + * * Locking rules: * get_parent is called with child->d_inode->i_rwsem down * get_name is not (which is possibly inconsistent) @@ -290,12 +283,6 @@ struct export_operations { struct dentry * (*get_parent)(struct dentry *child); int (*commit_metadata)(struct inode *inode); - int (*get_uuid)(struct super_block *sb, u8 *buf, u32 *len, u64 *offset); - int (*map_blocks)(struct inode *inode, loff_t offset, - u64 len, struct iomap *iomap, - bool write, u32 *device_generation); - int (*commit_blocks)(struct inode *inode, struct iomap *iomaps, - int nr_iomaps, struct iattr *iattr); int (*permission)(struct handle_to_path_ctx *ctx, unsigned int oflags); struct file * (*open)(const struct path *path, unsigned int oflags); #define EXPORT_OP_NOWCC (0x1) /* don't collect v3 wcc data */ @@ -308,6 +295,10 @@ struct export_operations { #define EXPORT_OP_FLUSH_ON_CLOSE (0x20) /* fs flushes file data on close */ #define EXPORT_OP_NOLOCKS (0x40) /* no file locking support */ unsigned long flags; + +#ifdef CONFIG_EXPORTFS_BLOCK_OPS + const struct exportfs_block_ops *block_ops; +#endif }; /** diff --git a/include/linux/exportfs_block.h b/include/linux/exportfs_block.h new file mode 100644 index 000000000000..de519b7b599b --- /dev/null +++ b/include/linux/exportfs_block.h @@ -0,0 +1,88 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2014-2026 Christoph Hellwig. + * + * Support for exportfs-based layout grants for direct block device access. + */ +#ifndef LINUX_EXPORTFS_BLOCK_H +#define LINUX_EXPORTFS_BLOCK_H 1 + +#include <linux/blkdev.h> +#include <linux/exportfs.h> +#include <linux/fs.h> + +struct inode; +struct iomap; +struct super_block; + +/* + * There are the two types of block-style layout support: + * - In-band implies a device identified by a unique cookie inside the actual + * device address space checked by the ->get_uuid method as used by the pNFS + * block layout. This is a bit dangerous and deprecated. + * - Out of band implies identification by out of band unique identifiers + * specified by the storage protocol, which is much safer and used by the + * pNFS SCSI/NVMe layouts. + */ +typedef unsigned int __bitwise expfs_block_layouts_t; +#define EXPFS_BLOCK_FLAG(__bit) \ + ((__force expfs_block_layouts_t)(1u << __bit)) +#define EXPFS_BLOCK_IN_BAND_ID EXPFS_BLOCK_FLAG(0) +#define EXPFS_BLOCK_OUT_OF_BAND_ID EXPFS_BLOCK_FLAG(1) + +struct exportfs_block_ops { + /* + * Returns the EXPFS_BLOCK_* bitmap of supported layout types. + */ + expfs_block_layouts_t (*layouts_supported)(struct super_block *sb); + + /* + * Get the in-band device unique signature exposed to clients. + */ + int (*get_uuid)(struct super_block *sb, u8 *buf, u32 *len, u64 *offset); + + /* + * Map blocks for direct block access. + * If @write is %true, also allocate the blocks for the range if needed. + */ + int (*map_blocks)(struct inode *inode, loff_t offset, u64 len, + struct iomap *iomap, bool write, + u32 *device_generation); + + /* + * Commit blocks previously handed out by ->map_blocks and written to by + * the client. + */ + int (*commit_blocks)(struct inode *inode, struct iomap *iomaps, + int nr_iomaps, loff_t new_size); +}; + +static inline bool +exportfs_bdev_supports_out_of_band_id(struct block_device *bdev) +{ + return bdev->bd_disk->fops->pr_ops && + bdev->bd_disk->fops->get_unique_id; +} + +#ifdef CONFIG_EXPORTFS_BLOCK_OPS +static inline expfs_block_layouts_t +exportfs_layouts_supported(struct super_block *sb) +{ + const struct exportfs_block_ops *bops = sb->s_export_op->block_ops; + + if (!bops || + !bops->layouts_supported || + WARN_ON_ONCE(!bops->map_blocks) || + WARN_ON_ONCE(!bops->commit_blocks)) + return 0; + return bops->layouts_supported(sb); +} +#else +static inline expfs_block_layouts_t +exportfs_layouts_supported(struct super_block *sb) +{ + return 0; +} +#endif /* CONFIG_EXPORTFS_BLOCK_OPS */ + +#endif /* LINUX_EXPORTFS_BLOCK_H */ diff --git a/include/linux/f2fs_fs.h b/include/linux/f2fs_fs.h index dc41722fcc9d..bb2b6cd5d507 100644 --- a/include/linux/f2fs_fs.h +++ b/include/linux/f2fs_fs.h @@ -80,6 +80,9 @@ enum stop_cp_reason { STOP_CP_REASON_NO_SEGMENT, STOP_CP_REASON_CORRUPTED_FREE_BITMAP, STOP_CP_REASON_CORRUPTED_NID, + STOP_CP_REASON_READ_META, + STOP_CP_REASON_READ_NODE, + STOP_CP_REASON_READ_DATA, STOP_CP_REASON_MAX, }; @@ -104,6 +107,7 @@ enum f2fs_error { ERROR_CORRUPTED_XATTR, ERROR_INVALID_NODE_REFERENCE, ERROR_INCONSISTENT_NAT, + ERROR_INCONSISTENT_ORPHAN, ERROR_MAX, }; diff --git a/include/linux/fb.h b/include/linux/fb.h index 6d4a58084fd5..e9a26e82322a 100644 --- a/include/linux/fb.h +++ b/include/linux/fb.h @@ -217,14 +217,12 @@ struct fb_deferred_io { /* delay between mkwrite and deferred handler */ unsigned long delay; bool sort_pagereflist; /* sort pagelist by offset */ - int open_count; /* number of opened files; protected by fb_info lock */ - struct mutex lock; /* mutex that protects the pageref list */ - struct list_head pagereflist; /* list of pagerefs for touched pages */ - struct address_space *mapping; /* page cache object for fb device */ /* callback */ struct page *(*get_page)(struct fb_info *info, unsigned long offset); void (*deferred_io)(struct fb_info *info, struct list_head *pagelist); }; + +struct fb_deferred_io_state; #endif /* @@ -484,9 +482,8 @@ struct fb_info { #ifdef CONFIG_FB_DEFERRED_IO struct delayed_work deferred_work; - unsigned long npagerefs; - struct fb_deferred_io_pageref *pagerefs; struct fb_deferred_io *fbdefio; + struct fb_deferred_io_state *fbdefio_state; #endif const struct fb_ops *fbops; @@ -536,6 +533,8 @@ extern int fb_set_var(struct fb_info *info, struct fb_var_screeninfo *var); extern int fb_pan_display(struct fb_info *info, struct fb_var_screeninfo *var); extern int fb_blank(struct fb_info *info, int blank); +int fb_set_var_from_user(struct fb_info *info, struct fb_var_screeninfo *var); + /* * Helpers for framebuffers in I/O memory */ @@ -605,10 +604,11 @@ extern int register_framebuffer(struct fb_info *fb_info); extern void unregister_framebuffer(struct fb_info *fb_info); extern int devm_register_framebuffer(struct device *dev, struct fb_info *fb_info); extern char* fb_get_buffer_offset(struct fb_info *info, struct fb_pixmap *buf, u32 size); -extern void fb_pad_unaligned_buffer(u8 *dst, u32 d_pitch, u8 *src, u32 idx, - u32 height, u32 shift_high, u32 shift_low, u32 mod); -extern void fb_pad_aligned_buffer(u8 *dst, u32 d_pitch, u8 *src, u32 s_pitch, u32 height); +void fb_pad_unaligned_buffer(u8 *dst, u32 d_pitch, const u8 *src, u32 idx, u32 height, + u32 shift_high, u32 shift_low, u32 mod); +void fb_pad_aligned_buffer(u8 *dst, u32 d_pitch, const u8 *src, u32 s_pitch, u32 height); extern void fb_set_suspend(struct fb_info *info, int state); +extern void fb_switch_outputs(struct fb_info *info); extern int fb_get_color_depth(struct fb_var_screeninfo *var, struct fb_fix_screeninfo *fix); extern int fb_get_options(const char *name, char **option); @@ -633,8 +633,8 @@ static inline struct device *dev_of_fbinfo(const struct fb_info *info) #endif } -static inline void __fb_pad_aligned_buffer(u8 *dst, u32 d_pitch, - u8 *src, u32 s_pitch, u32 height) +static inline void __fb_pad_aligned_buffer(u8 *dst, u32 d_pitch, const u8 *src, u32 s_pitch, + u32 height) { u32 i, j; diff --git a/include/linux/fbcon.h b/include/linux/fbcon.h deleted file mode 100644 index f206370060e1..000000000000 --- a/include/linux/fbcon.h +++ /dev/null @@ -1,55 +0,0 @@ -#ifndef _LINUX_FBCON_H -#define _LINUX_FBCON_H - -#include <linux/compiler_types.h> - -struct fb_blit_caps; -struct fb_info; -struct fb_var_screeninfo; -struct fb_videomode; - -#ifdef CONFIG_FRAMEBUFFER_CONSOLE -void __init fb_console_init(void); -void __exit fb_console_exit(void); -int fbcon_fb_registered(struct fb_info *info); -void fbcon_fb_unregistered(struct fb_info *info); -void fbcon_fb_unbind(struct fb_info *info); -void fbcon_suspended(struct fb_info *info); -void fbcon_resumed(struct fb_info *info); -int fbcon_mode_deleted(struct fb_info *info, - struct fb_videomode *mode); -void fbcon_delete_modelist(struct list_head *head); -void fbcon_new_modelist(struct fb_info *info); -void fbcon_get_requirement(struct fb_info *info, - struct fb_blit_caps *caps); -void fbcon_fb_blanked(struct fb_info *info, int blank); -int fbcon_modechange_possible(struct fb_info *info, - struct fb_var_screeninfo *var); -void fbcon_update_vcs(struct fb_info *info, bool all); -void fbcon_remap_all(struct fb_info *info); -int fbcon_set_con2fb_map_ioctl(void __user *argp); -int fbcon_get_con2fb_map_ioctl(void __user *argp); -#else -static inline void fb_console_init(void) {} -static inline void fb_console_exit(void) {} -static inline int fbcon_fb_registered(struct fb_info *info) { return 0; } -static inline void fbcon_fb_unregistered(struct fb_info *info) {} -static inline void fbcon_fb_unbind(struct fb_info *info) {} -static inline void fbcon_suspended(struct fb_info *info) {} -static inline void fbcon_resumed(struct fb_info *info) {} -static inline int fbcon_mode_deleted(struct fb_info *info, - struct fb_videomode *mode) { return 0; } -static inline void fbcon_delete_modelist(struct list_head *head) {} -static inline void fbcon_new_modelist(struct fb_info *info) {} -static inline void fbcon_get_requirement(struct fb_info *info, - struct fb_blit_caps *caps) {} -static inline void fbcon_fb_blanked(struct fb_info *info, int blank) {} -static inline int fbcon_modechange_possible(struct fb_info *info, - struct fb_var_screeninfo *var) { return 0; } -static inline void fbcon_update_vcs(struct fb_info *info, bool all) {} -static inline void fbcon_remap_all(struct fb_info *info) {} -static inline int fbcon_set_con2fb_map_ioctl(void __user *argp) { return 0; } -static inline int fbcon_get_con2fb_map_ioctl(void __user *argp) { return 0; } -#endif - -#endif /* _LINUX_FBCON_H */ diff --git a/include/linux/fcntl.h b/include/linux/fcntl.h index a332e79b3207..6ad6b9e7a226 100644 --- a/include/linux/fcntl.h +++ b/include/linux/fcntl.h @@ -4,13 +4,31 @@ #include <linux/stat.h> #include <uapi/linux/fcntl.h> +#include <uapi/linux/openat2.h> /* List of all valid flags for the open/openat flags argument: */ #define VALID_OPEN_FLAGS \ (O_RDONLY | O_WRONLY | O_RDWR | O_CREAT | O_EXCL | O_NOCTTY | O_TRUNC | \ O_APPEND | O_NDELAY | O_NONBLOCK | __O_SYNC | O_DSYNC | \ FASYNC | O_DIRECT | O_LARGEFILE | O_DIRECTORY | O_NOFOLLOW | \ - O_NOATIME | O_CLOEXEC | O_PATH | __O_TMPFILE) + O_NOATIME | O_CLOEXEC | O_PATH | __O_TMPFILE | O_EMPTYPATH) + +/* List of all valid flags for openat2(2)'s how->flags argument. */ +#define VALID_OPENAT2_FLAGS (VALID_OPEN_FLAGS | OPENAT2_REGULAR) + +/* + * Kernel-internal carrier for OPENAT2_REGULAR. The UAPI bit lives in the + * upper 32 bits of open_how::flags so open()/openat() cannot encode it. + * build_open_flags() translates it to this internal flag, which then + * propagates through op->open_flag and f->f_flags exactly like __FMODE_EXEC. + * do_dentry_open() strips it so userspace cannot observe it via + * fcntl(F_GETFL). + * + * Bit 30 is not claimed by any O_* flag on any architecture and stays clear + * of the sign bit of the int op->open_flag. fcntl_init() enforces that it + * never aliases an open-flag bit. + */ +#define __O_REGULAR (1 << 30) /* List of all valid flags for the how->resolve argument: */ #define VALID_RESOLVE_FLAGS \ diff --git a/include/linux/fileattr.h b/include/linux/fileattr.h index 3780904a63a6..58044b598016 100644 --- a/include/linux/fileattr.h +++ b/include/linux/fileattr.h @@ -16,7 +16,8 @@ /* Read-only inode flags */ #define FS_XFLAG_RDONLY_MASK \ - (FS_XFLAG_PREALLOC | FS_XFLAG_HASATTR | FS_XFLAG_VERITY) + (FS_XFLAG_PREALLOC | FS_XFLAG_HASATTR | FS_XFLAG_VERITY | \ + FS_XFLAG_CASEFOLD | FS_XFLAG_CASENONPRESERVING) /* Flags to indicate valid value of fsx_ fields */ #define FS_XFLAG_VALUES_MASK \ diff --git a/include/linux/filelock.h b/include/linux/filelock.h index d2c9740e26a8..ec11cc6b4c58 100644 --- a/include/linux/filelock.h +++ b/include/linux/filelock.h @@ -4,19 +4,22 @@ #include <linux/fs.h> -#define FL_POSIX 1 -#define FL_FLOCK 2 -#define FL_DELEG 4 /* NFSv4 delegation */ -#define FL_ACCESS 8 /* not trying to lock, just looking */ -#define FL_EXISTS 16 /* when unlocking, test for existence */ -#define FL_LEASE 32 /* lease held on this file */ -#define FL_CLOSE 64 /* unlock on close */ -#define FL_SLEEP 128 /* A blocking lock */ -#define FL_DOWNGRADE_PENDING 256 /* Lease is being downgraded */ -#define FL_UNLOCK_PENDING 512 /* Lease is being broken */ -#define FL_OFDLCK 1024 /* lock is "owned" by struct file */ -#define FL_LAYOUT 2048 /* outstanding pNFS layout */ -#define FL_RECLAIM 4096 /* reclaiming from a reboot server */ +#define FL_POSIX BIT(0) /* POSIX lock */ +#define FL_FLOCK BIT(1) /* BSD lock */ +#define FL_DELEG BIT(2) /* NFSv4 delegation */ +#define FL_ACCESS BIT(3) /* not trying to lock, just looking */ +#define FL_EXISTS BIT(4) /* when unlocking, test for existence */ +#define FL_LEASE BIT(5) /* file lease */ +#define FL_CLOSE BIT(6) /* unlock on close */ +#define FL_SLEEP BIT(7) /* A blocking lock */ +#define FL_DOWNGRADE_PENDING BIT(8) /* Lease is being downgraded */ +#define FL_UNLOCK_PENDING BIT(9) /* Lease is being broken */ +#define FL_OFDLCK BIT(10) /* POSIX lock "owned" by struct file */ +#define FL_LAYOUT BIT(11) /* outstanding pNFS layout */ +#define FL_RECLAIM BIT(12) /* reclaiming from a reboot server */ +#define FL_IGN_DIR_CREATE BIT(13) /* ignore DIR_CREATE events */ +#define FL_IGN_DIR_DELETE BIT(14) /* ignore DIR_DELETE events */ +#define FL_IGN_DIR_RENAME BIT(15) /* ignore DIR_RENAME events */ #define FL_CLOSE_POSIX (FL_POSIX | FL_CLOSE) @@ -26,6 +29,15 @@ */ #define FILE_LOCK_DEFERRED 1 +#define LEASE_BREAK_LEASE BIT(0) // break leases and delegations +#define LEASE_BREAK_DELEG BIT(1) // break delegations only +#define LEASE_BREAK_LAYOUT BIT(2) // break layouts only +#define LEASE_BREAK_NONBLOCK BIT(3) // non-blocking break +#define LEASE_BREAK_OPEN_RDONLY BIT(4) // readonly open event +#define LEASE_BREAK_DIR_CREATE BIT(5) // dir deleg create event +#define LEASE_BREAK_DIR_DELETE BIT(6) // dir deleg delete event +#define LEASE_BREAK_DIR_RENAME BIT(7) // dir deleg rename event + struct file_lock; struct file_lease; @@ -50,6 +62,7 @@ struct lease_manager_operations { void (*lm_setup)(struct file_lease *, void **); bool (*lm_breaker_owns_lease)(struct file_lease *); int (*lm_open_conflict)(struct file *, int); + bool (*lm_breaker_timedout)(struct file_lease *fl); }; struct lock_manager { @@ -215,19 +228,13 @@ int locks_lock_inode_wait(struct inode *inode, struct file_lock *fl); void locks_init_lease(struct file_lease *); void locks_free_lease(struct file_lease *fl); struct file_lease *locks_alloc_lease(void); - -#define LEASE_BREAK_LEASE BIT(0) // break leases and delegations -#define LEASE_BREAK_DELEG BIT(1) // break delegations only -#define LEASE_BREAK_LAYOUT BIT(2) // break layouts only -#define LEASE_BREAK_NONBLOCK BIT(3) // non-blocking break -#define LEASE_BREAK_OPEN_RDONLY BIT(4) // readonly open event - int __break_lease(struct inode *inode, unsigned int flags); void lease_get_mtime(struct inode *, struct timespec64 *time); int generic_setlease(struct file *, int, struct file_lease **, void **priv); int kernel_setlease(struct file *, int, struct file_lease **, void **); int vfs_setlease(struct file *, int, struct file_lease **, void **); int lease_modify(struct file_lease *, int, struct list_head *); +u32 inode_lease_ignore_mask(struct inode *inode); struct notifier_block; int lease_register_notifier(struct notifier_block *); @@ -515,12 +522,26 @@ static inline bool is_delegated(struct delegated_inode *di) return di->di_inode; } -static inline int try_break_deleg(struct inode *inode, +/** + * try_break_deleg - do a non-blocking delegation break + * @inode: inode that should have its delegations broken + * @flags: extra LEASE_BREAK_* flags to pass to break_deleg() + * @di: returns pointer to delegated inode (may be NULL) + * + * Break delegations in a non-blocking fashion. If there are + * outstanding delegations and @di is set, then an extra reference + * will be taken on @inode and @di->di_inode will be populated so + * that it may be waited upon. + * + * Returns 0 if there is no need to wait or an error. If -EWOULDBLOCK + * is returned, then @di will be populated (if non-NULL). + */ +static inline int try_break_deleg(struct inode *inode, unsigned int flags, struct delegated_inode *di) { int ret; - ret = break_deleg(inode, LEASE_BREAK_NONBLOCK); + ret = break_deleg(inode, flags | LEASE_BREAK_NONBLOCK); if (ret == -EWOULDBLOCK && di) { di->di_inode = inode; ihold(inode); @@ -563,7 +584,7 @@ static inline bool is_delegated(struct delegated_inode *di) return false; } -static inline int break_lease(struct inode *inode, bool wait) +static inline int break_lease(struct inode *inode, unsigned int mode) { return 0; } @@ -573,7 +594,7 @@ static inline int break_deleg(struct inode *inode, unsigned int flags) return 0; } -static inline int try_break_deleg(struct inode *inode, +static inline int try_break_deleg(struct inode *inode, unsigned int flags, struct delegated_inode *delegated_inode) { return 0; diff --git a/include/linux/filter.h b/include/linux/filter.h index 44d7ae95ddbc..39decde7fc73 100644 --- a/include/linux/filter.h +++ b/include/linux/filter.h @@ -21,7 +21,7 @@ #include <linux/if_vlan.h> #include <linux/vmalloc.h> #include <linux/sockptr.h> -#include <crypto/sha1.h> +#include <linux/static_call.h> #include <linux/u64_stats_sync.h> #include <net/sch_generic.h> @@ -59,8 +59,9 @@ struct ctl_table_header; #define BPF_REG_H BPF_REG_9 /* hlen, callee-saved */ /* Kernel hidden auxiliary/helper register. */ -#define BPF_REG_AX MAX_BPF_REG -#define MAX_BPF_EXT_REG (MAX_BPF_REG + 1) +#define BPF_REG_PARAMS MAX_BPF_REG +#define BPF_REG_AX (MAX_BPF_REG + 1) +#define MAX_BPF_EXT_REG (MAX_BPF_REG + 2) #define MAX_BPF_JIT_REG MAX_BPF_EXT_REG /* unused opcode to mark special call to bpf_tail_call() helper */ @@ -382,6 +383,61 @@ static inline bool insn_is_cast_user(const struct bpf_insn *insn) /* Legacy alias */ #define BPF_STX_XADD(SIZE, DST, SRC, OFF) BPF_ATOMIC_OP(SIZE, BPF_ADD, DST, SRC, OFF) +/* + * Given a BPF_ATOMIC instruction @atomic_insn, return true if it is an + * atomic load or store, and false if it is a read-modify-write instruction. + */ +static inline bool +bpf_atomic_is_load_store(const struct bpf_insn *atomic_insn) +{ + switch (atomic_insn->imm) { + case BPF_LOAD_ACQ: + case BPF_STORE_REL: + return true; + default: + return false; + } +} + +/* + * A load-acquire is the only BPF_STX class instruction that reads into + * dst_reg from src_reg + off16, i.e. it has the operand roles of a BPF_LDX. + * Unlike bpf_atomic_is_load_store(), @insn is not assumed to be a BPF_ATOMIC + * instruction here, so that callers which walk all instruction classes can + * use this directly. + */ +static inline bool bpf_atomic_is_load_acq(const struct bpf_insn *insn) +{ + return BPF_CLASS(insn->code) == BPF_STX && + (BPF_MODE(insn->code) == BPF_ATOMIC || + BPF_MODE(insn->code) == BPF_PROBE_ATOMIC) && + insn->imm == BPF_LOAD_ACQ; +} + +/* + * Given an instruction @insn, return the number of the BPF register that a + * BPF_ATOMIC reads the value at its memory operand into, or -1 if there is + * no such register. That is the register a BPF_PROBE_ATOMIC has to clear when + * the access faults. Like bpf_atomic_is_load_acq(), @insn is not assumed to + * be a BPF_ATOMIC here. + */ +static inline int bpf_atomic_load_reg(const struct bpf_insn *insn) +{ + if (BPF_CLASS(insn->code) != BPF_STX || + (BPF_MODE(insn->code) != BPF_ATOMIC && + BPF_MODE(insn->code) != BPF_PROBE_ATOMIC)) + return -1; + + switch (insn->imm) { + case BPF_LOAD_ACQ: + return insn->dst_reg; + case BPF_CMPXCHG: + return BPF_REG_0; + default: + return (insn->imm & BPF_FETCH) ? insn->src_reg : -1; + } +} + /* Memory store, *(uint *) (dst_reg + off16) = imm32 */ #define BPF_ST_MEM(SIZE, DST, OFF, IMM) \ @@ -749,6 +805,27 @@ static inline u32 bpf_prog_run_pin_on_cpu(const struct bpf_prog *prog, return ret; } +static inline bool is_stack_arg_ldx(const struct bpf_insn *insn) +{ + return insn->code == (BPF_LDX | BPF_MEM | BPF_DW) && + insn->src_reg == BPF_REG_PARAMS && + insn->off > 0 && insn->off % 8 == 0; +} + +static inline bool is_stack_arg_st(const struct bpf_insn *insn) +{ + return insn->code == (BPF_ST | BPF_MEM | BPF_DW) && + insn->dst_reg == BPF_REG_PARAMS && + insn->off < 0 && insn->off % 8 == 0; +} + +static inline bool is_stack_arg_stx(const struct bpf_insn *insn) +{ + return insn->code == (BPF_STX | BPF_MEM | BPF_DW) && + insn->dst_reg == BPF_REG_PARAMS && + insn->off < 0 && insn->off % 8 == 0; +} + #define BPF_SKB_CB_LEN QDISC_CB_PRIV_LEN struct bpf_skb_data_end { @@ -1092,22 +1169,25 @@ bpf_jit_binary_lock_ro(struct bpf_binary_header *hdr) return set_memory_rox((unsigned long)hdr, hdr->size >> PAGE_SHIFT); } -int sk_filter_trim_cap(struct sock *sk, struct sk_buff *skb, unsigned int cap, - enum skb_drop_reason *reason); +enum skb_drop_reason +sk_filter_trim_cap(struct sock *sk, struct sk_buff *skb, unsigned int cap); static inline int sk_filter(struct sock *sk, struct sk_buff *skb) { - enum skb_drop_reason ignore_reason; + enum skb_drop_reason drop_reason; - return sk_filter_trim_cap(sk, skb, 1, &ignore_reason); + drop_reason = sk_filter_trim_cap(sk, skb, 1); + return drop_reason ? -EPERM : 0; } -static inline int sk_filter_reason(struct sock *sk, struct sk_buff *skb, - enum skb_drop_reason *reason) +static inline enum skb_drop_reason +sk_filter_reason(struct sock *sk, struct sk_buff *skb) { - return sk_filter_trim_cap(sk, skb, 1, reason); + return sk_filter_trim_cap(sk, skb, 1); } +struct bpf_prog *__bpf_prog_select_runtime(struct bpf_verifier_env *env, struct bpf_prog *fp, + int *err); struct bpf_prog *bpf_prog_select_runtime(struct bpf_prog *fp, int *err); void bpf_prog_free(struct bpf_prog *fp); @@ -1149,17 +1229,16 @@ bool sk_filter_charge(struct sock *sk, struct sk_filter *fp); void sk_filter_uncharge(struct sock *sk, struct sk_filter *fp); u64 __bpf_call_base(u64 r1, u64 r2, u64 r3, u64 r4, u64 r5); -#define __bpf_call_base_args \ - ((u64 (*)(u64, u64, u64, u64, u64, const struct bpf_insn *)) \ - (void *)__bpf_call_base) -struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog); +struct bpf_prog *bpf_int_jit_compile(struct bpf_verifier_env *env, struct bpf_prog *prog); void bpf_jit_compile(struct bpf_prog *prog); bool bpf_jit_needs_zext(void); bool bpf_jit_inlines_helper_call(s32 imm); bool bpf_jit_supports_subprog_tailcalls(void); bool bpf_jit_supports_percpu_insn(void); bool bpf_jit_supports_kfunc_call(void); +bool bpf_jit_supports_stack_args(void); +bool bpf_jit_supports_arena_args(void); bool bpf_jit_supports_far_kfunc_call(void); bool bpf_jit_supports_exceptions(void); bool bpf_jit_supports_ptr_xchg(void); @@ -1184,6 +1263,18 @@ static inline bool bpf_dump_raw_ok(const struct cred *cred) struct bpf_prog *bpf_patch_insn_single(struct bpf_prog *prog, u32 off, const struct bpf_insn *patch, u32 len); + +#ifdef CONFIG_BPF_SYSCALL +struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off, + const struct bpf_insn *patch, u32 len); +#else +static inline struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off, + const struct bpf_insn *patch, u32 len) +{ + return ERR_PTR(-ENOTSUPP); +} +#endif /* CONFIG_BPF_SYSCALL */ + int bpf_remove_insns(struct bpf_prog *prog, u32 off, u32 cnt); static inline bool xdp_return_frame_no_direct(void) @@ -1267,6 +1358,15 @@ extern long bpf_jit_limit_max; typedef void (*bpf_jit_fill_hole_t)(void *area, unsigned int size); +/* + * Flush the indirect branch predictors before reusing JIT memory, so that + * indirect jumps into a newly written program don't reuse predictions left + * behind by an old program that occupied the same space. + */ +void bpf_arch_pred_flush(void); +DECLARE_STATIC_CALL(bpf_arch_pred_flush, bpf_arch_pred_flush); +DECLARE_STATIC_KEY_FALSE(bpf_pred_flush_enabled); + void bpf_jit_fill_hole_with_zero(void *area, unsigned int size); struct bpf_binary_header * @@ -1281,7 +1381,7 @@ void bpf_jit_free(struct bpf_prog *fp); struct bpf_binary_header * bpf_jit_binary_pack_hdr(const struct bpf_prog *fp); -void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insns); +void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insns, bool was_classic); void bpf_prog_pack_free(void *ptr, u32 size); static inline bool bpf_prog_kallsyms_verify_off(const struct bpf_prog *fp) @@ -1295,7 +1395,8 @@ bpf_jit_binary_pack_alloc(unsigned int proglen, u8 **ro_image, unsigned int alignment, struct bpf_binary_header **rw_hdr, u8 **rw_image, - bpf_jit_fill_hole_t bpf_fill_ill_insns); + bpf_jit_fill_hole_t bpf_fill_ill_insns, + bool was_classic); int bpf_jit_binary_pack_finalize(struct bpf_binary_header *ro_header, struct bpf_binary_header *rw_header); void bpf_jit_binary_pack_free(struct bpf_binary_header *ro_header, @@ -1310,9 +1411,14 @@ int bpf_jit_get_func_addr(const struct bpf_prog *prog, const char *bpf_jit_get_prog_name(struct bpf_prog *prog); -struct bpf_prog *bpf_jit_blind_constants(struct bpf_prog *fp); +struct bpf_prog *bpf_jit_blind_constants(struct bpf_verifier_env *env, struct bpf_prog *prog); void bpf_jit_prog_release_other(struct bpf_prog *fp, struct bpf_prog *fp_other); +static inline bool bpf_prog_need_blind(const struct bpf_prog *prog) +{ + return prog->blinding_requested && !prog->blinded; +} + static inline void bpf_jit_dump(unsigned int flen, unsigned int proglen, u32 pass, void *image) { @@ -1451,6 +1557,20 @@ static inline void bpf_prog_kallsyms_del(struct bpf_prog *fp) { } +static inline bool bpf_prog_need_blind(const struct bpf_prog *prog) +{ + return false; +} + +static inline +struct bpf_prog *bpf_jit_blind_constants(struct bpf_verifier_env *env, struct bpf_prog *prog) +{ + return prog; +} + +static inline void bpf_jit_prog_release_other(struct bpf_prog *fp, struct bpf_prog *fp_other) +{ +} #endif /* CONFIG_BPF_JIT */ void bpf_prog_kallsyms_del_all(struct bpf_prog *fp); diff --git a/include/linux/find.h b/include/linux/find.h index 9d720ad92bc1..a129f8205fb6 100644 --- a/include/linux/find.h +++ b/include/linux/find.h @@ -22,8 +22,6 @@ extern unsigned long _find_first_bit(const unsigned long *addr, unsigned long si unsigned long __find_nth_bit(const unsigned long *addr, unsigned long size, unsigned long n); unsigned long __find_nth_and_bit(const unsigned long *addr1, const unsigned long *addr2, unsigned long size, unsigned long n); -unsigned long __find_nth_andnot_bit(const unsigned long *addr1, const unsigned long *addr2, - unsigned long size, unsigned long n); unsigned long __find_nth_and_andnot_bit(const unsigned long *addr1, const unsigned long *addr2, const unsigned long *addr3, unsigned long size, unsigned long n); @@ -251,7 +249,7 @@ unsigned long find_nth_bit(const unsigned long *addr, unsigned long size, unsign * @n: The number of set bit, which position is needed, counting from 0 * * Returns the bit number of the N'th set bit. - * If no such, returns @size. + * If no such, returns >= @size. */ static __always_inline unsigned long find_nth_and_bit(const unsigned long *addr1, const unsigned long *addr2, @@ -279,7 +277,7 @@ unsigned long find_nth_and_bit(const unsigned long *addr1, const unsigned long * * @n: The number of set bit, which position is needed, counting from 0 * * Returns the bit number of the N'th set bit. - * If no such, returns @size. + * If no such, returns >= @size. */ static __always_inline unsigned long find_nth_and_andnot_bit(const unsigned long *addr1, @@ -657,8 +655,8 @@ unsigned long find_next_bit_le(const void *addr, unsigned /** * for_each_clear_bitrange_from - iterate over all unset bit ranges [b; e) - * @b: bit offset of start of current bitrange (first set bit); must be initialized - * @e: bit offset of end of current bitrange (first unset bit) + * @b: bit offset of start of current bitrange (first unset bit); must be initialized + * @e: bit offset of end of current bitrange (first set bit) * @addr: bitmap address to base the search on * @size: bitmap size in number of bits */ diff --git a/include/linux/firewire.h b/include/linux/firewire.h index 986d712e4d94..fd35a6570cd8 100644 --- a/include/linux/firewire.h +++ b/include/linux/firewire.h @@ -14,6 +14,7 @@ #include <linux/timer.h> #include <linux/types.h> #include <linux/workqueue.h> +#include <linux/device-id/ieee1394.h> #include <linux/atomic.h> #include <asm/byteorder.h> @@ -271,8 +272,6 @@ static inline void fw_unit_put(struct fw_unit *unit) #define fw_parent_device(unit) fw_device(unit->device.parent) -struct ieee1394_device_id; - struct fw_driver { struct device_driver driver; int (*probe)(struct fw_unit *unit, const struct ieee1394_device_id *id); diff --git a/include/linux/firmware.h b/include/linux/firmware.h index aae1b85ffc10..0fa3b027f02f 100644 --- a/include/linux/firmware.h +++ b/include/linux/firmware.h @@ -110,6 +110,9 @@ int request_firmware_nowait( struct module *module, bool uevent, const char *name, struct device *device, gfp_t gfp, void *context, void (*cont)(const struct firmware *fw, void *context)); +void request_firmware_nowait_cancel(struct device *device, void *context, + void (*cont)(const struct firmware *fw, + void *context)); int request_firmware_direct(const struct firmware **fw, const char *name, struct device *device); int request_firmware_into_buf(const struct firmware **firmware_p, @@ -157,6 +160,13 @@ static inline int request_firmware_nowait( return -EINVAL; } +static inline void request_firmware_nowait_cancel(struct device *device, + void *context, + void (*cont)(const struct firmware *fw, + void *context)) +{ +} + static inline void release_firmware(const struct firmware *fw) { } diff --git a/include/linux/firmware/cirrus/cs_dsp.h b/include/linux/firmware/cirrus/cs_dsp.h index 0ec1cdc5585d..4e3baa557068 100644 --- a/include/linux/firmware/cirrus/cs_dsp.h +++ b/include/linux/firmware/cirrus/cs_dsp.h @@ -179,6 +179,7 @@ struct cs_dsp { bool booted; bool running; + bool hibernating; struct list_head ctl_list; @@ -354,4 +355,6 @@ int cs_dsp_chunk_write(struct cs_dsp_chunk *ch, int nbits, u32 val); int cs_dsp_chunk_flush(struct cs_dsp_chunk *ch); int cs_dsp_chunk_read(struct cs_dsp_chunk *ch, int nbits); +void cs_dsp_hibernate(struct cs_dsp *dsp, bool hibernating); + #endif diff --git a/include/linux/firmware/intel/stratix10-smc.h b/include/linux/firmware/intel/stratix10-smc.h index 935dba3633b5..366309260121 100644 --- a/include/linux/firmware/intel/stratix10-smc.h +++ b/include/linux/firmware/intel/stratix10-smc.h @@ -67,6 +67,9 @@ * INTEL_SIP_SMC_STATUS_REJECTED: * Secure monitor software reject the service client's request. * + * INTEL_SIP_SMC_STATUS_NO_RESPONSE: + * Secure monitor software has no response for the request yet. + * * INTEL_SIP_SMC_STATUS_ERROR: * There is error during the process of service request. * @@ -77,6 +80,7 @@ #define INTEL_SIP_SMC_STATUS_OK 0x0 #define INTEL_SIP_SMC_STATUS_BUSY 0x1 #define INTEL_SIP_SMC_STATUS_REJECTED 0x2 +#define INTEL_SIP_SMC_STATUS_NO_RESPONSE 0x3 #define INTEL_SIP_SMC_STATUS_ERROR 0x4 #define INTEL_SIP_SMC_RSU_ERROR 0x7 @@ -430,6 +434,29 @@ INTEL_SIP_SMC_FAST_CALL_VAL(INTEL_SIP_SMC_FUNCID_FPGA_CONFIG_COMPLETED_WRITE) INTEL_SIP_SMC_FAST_CALL_VAL(INTEL_SIP_SMC_FUNCID_RSU_DCMF_STATUS) /** + * Request INTEL_SIP_SMC_RSU_GET_DEVICE_INFO + * + * Sync call used by service driver at EL1 to query QSPI device info from FW + * + * Call register usage: + * a0 INTEL_SIP_SMC_RSU_GET_DEVICE_INFO + * a1-7 not used + * + * Return status + * a0 INTEL_SIP_SMC_STATUS_OK + * a1 erasesize0 | size0 + * a2 erasesize1 | size1 + * a3 erasesize2 | size2 + * a4 erasesize3 | size3 + * Or + * + * a0 INTEL_SIP_SMC_RSU_ERROR + */ +#define INTEL_SIP_SMC_FUNCID_RSU_GET_DEVICE_INFO 22 +#define INTEL_SIP_SMC_RSU_GET_DEVICE_INFO \ + INTEL_SIP_SMC_FAST_CALL_VAL(INTEL_SIP_SMC_FUNCID_RSU_GET_DEVICE_INFO) + +/** * Request INTEL_SIP_SMC_SERVICE_COMPLETED * Sync call to check if the secure world have completed service request * or not. @@ -493,7 +520,7 @@ INTEL_SIP_SMC_FAST_CALL_VAL(INTEL_SIP_SMC_FUNCID_FPGA_CONFIG_COMPLETED_WRITE) * a3 not used */ #define INTEL_SIP_SMC_FUNCID_MBOX_SEND_CMD 60 - #define INTEL_SIP_SMC_MBOX_SEND_CMD \ +#define INTEL_SIP_SMC_MBOX_SEND_CMD \ INTEL_SIP_SMC_FAST_CALL_VAL(INTEL_SIP_SMC_FUNCID_MBOX_SEND_CMD) /** @@ -515,6 +542,25 @@ INTEL_SIP_SMC_FAST_CALL_VAL(INTEL_SIP_SMC_FUNCID_FPGA_CONFIG_COMPLETED_WRITE) INTEL_SIP_SMC_FAST_CALL_VAL(INTEL_SIP_SMC_SVC_FUNCID_VERSION) /** + * Request INTEL_SIP_SMC_ATF_BUILD_VER + * + * Sync call used to query the ATF Build Version + * + * Call register usage: + * a0 INTEL_SIP_SMC_ATF_BUILD_VER + * a1-a7 not used + * + * Return status: + * a0 INTEL_SIP_SMC_STATUS_OK + * a1 Major + * a2 Minor + * a3 Patch + */ +#define INTEL_SIP_SMC_ATF_BUILD_VERSION 155 +#define INTEL_SIP_SMC_ATF_BUILD_VER \ + INTEL_SIP_SMC_FAST_CALL_VAL(INTEL_SIP_SMC_ATF_BUILD_VERSION) + +/** * SMC call protocol for FPGA Crypto Service (FCS) * FUNCID starts from 90 */ @@ -587,7 +633,7 @@ INTEL_SIP_SMC_FAST_CALL_VAL(INTEL_SIP_SMC_FUNCID_FPGA_CONFIG_COMPLETED_WRITE) /** * Request INTEL_SIP_SMC_FUNCID_FCS_SEND_CERTIFICATE - * Sync call to send a signed certificate + * Async call to send a signed certificate * * Call register usage: * a0 INTEL_SIP_SMC_FCS_SEND_CERTIFICATE @@ -596,7 +642,7 @@ INTEL_SIP_SMC_FAST_CALL_VAL(INTEL_SIP_SMC_FUNCID_FPGA_CONFIG_COMPLETED_WRITE) * a3-a7 not used * * Return status: - * a0 INTEL_SIP_SMC_STATUS_OK or INTEL_SIP_SMC_FCS_REJECTED + * a0 INTEL_SIP_SMC_STATUS_OK or INTEL_SIP_SMC_REJECTED * a1-a3 not used */ #define INTEL_SIP_SMC_FUNCID_FCS_SEND_CERTIFICATE 93 @@ -605,25 +651,23 @@ INTEL_SIP_SMC_FAST_CALL_VAL(INTEL_SIP_SMC_FUNCID_FPGA_CONFIG_COMPLETED_WRITE) /** * Request INTEL_SIP_SMC_FCS_GET_PROVISION_DATA - * Sync call to dump all the fuses and key hashes + * Async call to dump all the fuses and key hashes * * Call register usage: * a0 INTEL_SIP_SMC_FCS_GET_PROVISION_DATA - * a1 the physical address for firmware to write structure of fuse and - * key hashes - * a2-a7 not used + * a1-a7 not used * * Return status: - * a0 INTEL_SIP_SMC_STATUS_OK, INTEL_SIP_SMC_FCS_ERROR or - * INTEL_SIP_SMC_FCS_REJECTED - * a1 mailbox error + * a0 INTEL_SIP_SMC_STATUS_OK, INTEL_SIP_SMC_STATUS_ERROR or + * INTEL_SIP_SMC_STATUS_REJECTED + * a1 mailbox error if a0 is INTEL_SIP_SMC_STATUS_ERROR * a2 physical address for the structure of fuse and key hashes * a3 the size of structure * */ #define INTEL_SIP_SMC_FUNCID_FCS_GET_PROVISION_DATA 94 #define INTEL_SIP_SMC_FCS_GET_PROVISION_DATA \ - INTEL_SIP_SMC_FAST_CALL_VAL(INTEL_SIP_SMC_FUNCID_FCS_GET_PROVISION_DATA) + INTEL_SIP_SMC_STD_CALL_VAL(INTEL_SIP_SMC_FUNCID_FCS_GET_PROVISION_DATA) /** * Request INTEL_SIP_SMC_HWMON_READTEMP @@ -681,6 +725,44 @@ INTEL_SIP_SMC_FAST_CALL_VAL(INTEL_SIP_SMC_FUNCID_FPGA_CONFIG_COMPLETED_WRITE) INTEL_SIP_SMC_ASYNC_VAL(INTEL_SIP_SMC_ASYNC_FUNC_ID_POLL) /** + * Request INTEL_SIP_SMC_ASYNC_HWMON_READTEMP + * Async call to request temperature + * + * Call register usage: + * a0 INTEL_SIP_SMC_ASYNC_HWMON_READTEMP + * a1 transaction job id + * a2 Temperature Channel + * a3-a17 not used + * + * Return status + * a0 INTEL_SIP_SMC_STATUS_OK, INTEL_SIP_SMC_STATUS_REJECTED + * or INTEL_SIP_SMC_STATUS_BUSY + * a1-a17 not used + */ +#define INTEL_SIP_SMC_ASYNC_FUNC_ID_HWMON_READTEMP 0xE8 +#define INTEL_SIP_SMC_ASYNC_HWMON_READTEMP \ + INTEL_SIP_SMC_ASYNC_VAL(INTEL_SIP_SMC_ASYNC_FUNC_ID_HWMON_READTEMP) + +/** + * Request INTEL_SIP_SMC_ASYNC_HWMON_READVOLT + * Async call to request voltage + * + * Call register usage: + * a0 INTEL_SIP_SMC_ASYNC_HWMON_READVOLT + * a1 transaction job id + * a2 Voltage Channel + * a3-a17 not used + * + * Return status + * a0 INTEL_SIP_SMC_STATUS_OK, INTEL_SIP_SMC_STATUS_REJECTED + * or INTEL_SIP_SMC_STATUS_BUSY + * a1-a17 not used + */ +#define INTEL_SIP_SMC_ASYNC_FUNC_ID_HWMON_READVOLT 0xE9 +#define INTEL_SIP_SMC_ASYNC_HWMON_READVOLT \ + INTEL_SIP_SMC_ASYNC_VAL(INTEL_SIP_SMC_ASYNC_FUNC_ID_HWMON_READVOLT) + +/** * Request INTEL_SIP_SMC_ASYNC_RSU_GET_SPT * Async call to get RSU SPT from SDM. * Call register usage: diff --git a/include/linux/firmware/intel/stratix10-svc-client.h b/include/linux/firmware/intel/stratix10-svc-client.h index 91013161e9db..af13dacdf5ac 100644 --- a/include/linux/firmware/intel/stratix10-svc-client.h +++ b/include/linux/firmware/intel/stratix10-svc-client.h @@ -128,6 +128,10 @@ struct stratix10_svc_chan; * @COMMAND_RSU_DCMF_STATUS: query firmware for the DCMF status * return status is SVC_STATUS_OK or SVC_STATUS_ERROR * + * @COMMAND_RSU_GET_DEVICE_INFO: query firmware for QSPI device info; + * return status is SVC_STATUS_OK, SVC_STATUS_ERROR, or SVC_STATUS_NO_SUPPORT + * (unsupported command / firmware compatibility path in the service layer). + * * @COMMAND_RSU_GET_SPT_TABLE: query firmware for SPT table * return status is SVC_STATUS_OK or SVC_STATUS_ERROR * @@ -154,6 +158,10 @@ struct stratix10_svc_chan; * * @COMMAND_HWMON_READVOLT: query the voltage from the hardware monitor, * return status is SVC_STATUS_OK or SVC_STATUS_ERROR + * + * @COMMAND_SMC_ATF_BUILD_VER: Non-mailbox SMC ATF Build Version, + * return status is SVC_STATUS_OK + * */ enum stratix10_svc_command_code { /* for FPGA */ @@ -170,6 +178,7 @@ enum stratix10_svc_command_code { COMMAND_RSU_MAX_RETRY, COMMAND_RSU_DCMF_VERSION, COMMAND_RSU_DCMF_STATUS, + COMMAND_RSU_GET_DEVICE_INFO, COMMAND_FIRMWARE_VERSION, COMMAND_RSU_GET_SPT_TABLE, /* for FCS */ @@ -187,7 +196,8 @@ enum stratix10_svc_command_code { COMMAND_SMC_SVC_VERSION = 200, /* for HWMON */ COMMAND_HWMON_READTEMP, - COMMAND_HWMON_READVOLT + COMMAND_HWMON_READVOLT, + COMMAND_SMC_ATF_BUILD_VER }; /** @@ -219,7 +229,12 @@ struct stratix10_svc_command_config_type { /** * struct stratix10_svc_cb_data - callback data structure from service layer * @status: the status of sent command - * @kaddr1: address of 1st completed data block + * @kaddr1: address of 1st completed data block, or command-specific payload. + * For COMMAND_RSU_GET_DEVICE_INFO on SVC_STATUS_OK or SVC_STATUS_ERROR, + * points to struct arm_smccc_1_2_regs filled by the SMC/HVC return + * registers (a0 status, a1-a4 packed device words per + * INTEL_SIP_SMC_RSU_GET_DEVICE_INFO). On SVC_STATUS_NO_SUPPORT (older + * firmware that does not handle this command), kaddr1 is NULL. * @kaddr2: address of 2nd completed data block * @kaddr3: address of 3rd completed data block */ diff --git a/include/linux/firmware/meson/meson_sm.h b/include/linux/firmware/meson/meson_sm.h index 8eaf8922ab02..3ebc2bd9a976 100644 --- a/include/linux/firmware/meson/meson_sm.h +++ b/include/linux/firmware/meson/meson_sm.h @@ -12,6 +12,7 @@ enum { SM_EFUSE_WRITE, SM_EFUSE_USER_MAX, SM_GET_CHIP_ID, + SM_THERMAL_CALIB_READ, SM_A1_PWRC_SET, SM_A1_PWRC_GET, }; @@ -27,5 +28,7 @@ int meson_sm_call_read(struct meson_sm_firmware *fw, void *buffer, unsigned int bsize, unsigned int cmd_index, u32 arg0, u32 arg1, u32 arg2, u32 arg3, u32 arg4); struct meson_sm_firmware *meson_sm_get(struct device_node *firmware_node); +int meson_sm_get_thermal_calib(struct meson_sm_firmware *fw, u32 *trim_info, + u32 tsensor_id); #endif /* _MESON_SM_FW_H_ */ diff --git a/include/linux/firmware/qcom/qcom_pas.h b/include/linux/firmware/qcom/qcom_pas.h new file mode 100644 index 000000000000..fb2ec3be6a16 --- /dev/null +++ b/include/linux/firmware/qcom/qcom_pas.h @@ -0,0 +1,55 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2010-2015, 2018-2019 The Linux Foundation. All rights reserved. + * Copyright (C) 2015 Linaro Ltd. + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + */ + +#ifndef __QCOM_PAS_H +#define __QCOM_PAS_H + +#include <linux/device.h> +#include <linux/err.h> +#include <linux/io.h> +#include <linux/types.h> + +struct qcom_pas_context { + struct device *dev; + u32 pas_id; + phys_addr_t mem_phys; + size_t mem_size; + void *ptr; + dma_addr_t phys; + ssize_t size; + bool use_tzmem; +}; + +static inline void __iomem *qcom_pas_ctx_map(struct qcom_pas_context *ctx) +{ + void __iomem *ptr = ioremap_wc(ctx->mem_phys, ctx->mem_size); + + if (!ptr) + dev_err(ctx->dev, "unable to map memory region: %pa+%zx\n", + &ctx->mem_phys, ctx->mem_size); + return ptr; +} + +bool qcom_pas_is_available(void); +struct qcom_pas_context *devm_qcom_pas_context_alloc(struct device *dev, + u32 pas_id, + phys_addr_t mem_phys, + size_t mem_size); +int qcom_pas_init_image(u32 pas_id, const void *metadata, size_t size, + struct qcom_pas_context *ctx); +struct resource_table *qcom_pas_get_rsc_table(struct qcom_pas_context *ctx, + void *input_rt, size_t input_rt_size, + size_t *output_rt_size); +int qcom_pas_mem_setup(u32 pas_id, phys_addr_t addr, phys_addr_t size); +int qcom_pas_auth_and_reset(u32 pas_id); +int qcom_pas_prepare_and_auth_reset(struct qcom_pas_context *ctx); +int qcom_pas_set_remote_state(u32 state, u32 pas_id); +int qcom_pas_shutdown(u32 pas_id); +bool qcom_pas_supported(u32 pas_id); +void qcom_pas_metadata_release(struct qcom_pas_context *ctx); + +#endif /* __QCOM_PAS_H */ diff --git a/include/linux/firmware/samsung/exynos-acpm-protocol.h b/include/linux/firmware/samsung/exynos-acpm-protocol.h index 2091da965a5a..c6b35c0ff300 100644 --- a/include/linux/firmware/samsung/exynos-acpm-protocol.h +++ b/include/linux/firmware/samsung/exynos-acpm-protocol.h @@ -14,57 +14,61 @@ struct acpm_handle; struct device_node; struct acpm_dvfs_ops { - int (*set_rate)(const struct acpm_handle *handle, - unsigned int acpm_chan_id, unsigned int clk_id, - unsigned long rate); - unsigned long (*get_rate)(const struct acpm_handle *handle, + int (*set_rate)(struct acpm_handle *handle, unsigned int acpm_chan_id, + unsigned int clk_id, unsigned long rate); + unsigned long (*get_rate)(struct acpm_handle *handle, unsigned int acpm_chan_id, unsigned int clk_id); }; struct acpm_pmic_ops { - int (*read_reg)(const struct acpm_handle *handle, - unsigned int acpm_chan_id, u8 type, u8 reg, u8 chan, - u8 *buf); - int (*bulk_read)(const struct acpm_handle *handle, - unsigned int acpm_chan_id, u8 type, u8 reg, u8 chan, - u8 count, u8 *buf); - int (*write_reg)(const struct acpm_handle *handle, - unsigned int acpm_chan_id, u8 type, u8 reg, u8 chan, - u8 value); - int (*bulk_write)(const struct acpm_handle *handle, - unsigned int acpm_chan_id, u8 type, u8 reg, u8 chan, - u8 count, const u8 *buf); - int (*update_reg)(const struct acpm_handle *handle, - unsigned int acpm_chan_id, u8 type, u8 reg, u8 chan, - u8 value, u8 mask); + int (*read_reg)(struct acpm_handle *handle, unsigned int acpm_chan_id, + u8 type, u8 reg, u8 chan, u8 *buf); + int (*bulk_read)(struct acpm_handle *handle, unsigned int acpm_chan_id, + u8 type, u8 reg, u8 chan, u8 count, u8 *buf); + int (*write_reg)(struct acpm_handle *handle, unsigned int acpm_chan_id, + u8 type, u8 reg, u8 chan, u8 value); + int (*bulk_write)(struct acpm_handle *handle, unsigned int acpm_chan_id, + u8 type, u8 reg, u8 chan, u8 count, const u8 *buf); + int (*update_reg)(struct acpm_handle *handle, unsigned int acpm_chan_id, + u8 type, u8 reg, u8 chan, u8 value, u8 mask); +}; + +struct acpm_tmu_ops { + int (*init)(struct acpm_handle *handle, unsigned int acpm_chan_id); + int (*read_temp)(struct acpm_handle *handle, unsigned int acpm_chan_id, + u8 tz, int *temp); + int (*set_threshold)(struct acpm_handle *handle, + unsigned int acpm_chan_id, u8 tz, + const u8 temperature[8], size_t tlen); + int (*set_interrupt_enable)(struct acpm_handle *handle, + unsigned int acpm_chan_id, u8 tz, u8 inten); + int (*tz_control)(struct acpm_handle *handle, unsigned int acpm_chan_id, + u8 tz, bool enable); + int (*clear_tz_irq)(struct acpm_handle *handle, + unsigned int acpm_chan_id, u8 tz); + int (*suspend)(struct acpm_handle *handle, unsigned int acpm_chan_id); + int (*resume)(struct acpm_handle *handle, unsigned int acpm_chan_id); }; struct acpm_ops { - struct acpm_dvfs_ops dvfs_ops; - struct acpm_pmic_ops pmic_ops; + struct acpm_dvfs_ops dvfs; + struct acpm_pmic_ops pmic; + struct acpm_tmu_ops tmu; }; /** * struct acpm_handle - Reference to an initialized protocol instance - * @ops: + * @ops: pointer to the constant ACPM protocol operations. */ struct acpm_handle { - struct acpm_ops ops; + const struct acpm_ops *ops; }; struct device; -#if IS_ENABLED(CONFIG_EXYNOS_ACPM_PROTOCOL) -const struct acpm_handle *devm_acpm_get_by_node(struct device *dev, - struct device_node *np); -#else - -static inline const struct acpm_handle *devm_acpm_get_by_node(struct device *dev, - struct device_node *np) -{ - return NULL; -} -#endif +struct acpm_handle *devm_acpm_get_by_node(struct device *dev, + struct device_node *np); +struct acpm_handle *devm_acpm_get_by_phandle(struct device *dev); #endif /* __EXYNOS_ACPM_PROTOCOL_H */ diff --git a/include/linux/firmware/thead/thead,th1520-aon.h b/include/linux/firmware/thead/thead,th1520-aon.h index dae132b66873..d81f5f6f5b90 100644 --- a/include/linux/firmware/thead/thead,th1520-aon.h +++ b/include/linux/firmware/thead/thead,th1520-aon.h @@ -97,80 +97,6 @@ struct th1520_aon_rpc_ack_common { #define RPC_GET_SVC_FLAG_ACK_TYPE(MESG) (((MESG)->svc & 0x40) >> 6) #define RPC_SET_SVC_FLAG_ACK_TYPE(MESG, ACK) ((MESG)->svc |= (ACK) << 6) -#define RPC_SET_BE64(MESG, OFFSET, SET_DATA) \ - do { \ - u8 *data = (u8 *)(MESG); \ - u64 _offset = (OFFSET); \ - u64 _set_data = (SET_DATA); \ - data[_offset + 7] = _set_data & 0xFF; \ - data[_offset + 6] = (_set_data & 0xFF00) >> 8; \ - data[_offset + 5] = (_set_data & 0xFF0000) >> 16; \ - data[_offset + 4] = (_set_data & 0xFF000000) >> 24; \ - data[_offset + 3] = (_set_data & 0xFF00000000) >> 32; \ - data[_offset + 2] = (_set_data & 0xFF0000000000) >> 40; \ - data[_offset + 1] = (_set_data & 0xFF000000000000) >> 48; \ - data[_offset + 0] = (_set_data & 0xFF00000000000000) >> 56; \ - } while (0) - -#define RPC_SET_BE32(MESG, OFFSET, SET_DATA) \ - do { \ - u8 *data = (u8 *)(MESG); \ - u64 _offset = (OFFSET); \ - u64 _set_data = (SET_DATA); \ - data[_offset + 3] = (_set_data) & 0xFF; \ - data[_offset + 2] = (_set_data & 0xFF00) >> 8; \ - data[_offset + 1] = (_set_data & 0xFF0000) >> 16; \ - data[_offset + 0] = (_set_data & 0xFF000000) >> 24; \ - } while (0) - -#define RPC_SET_BE16(MESG, OFFSET, SET_DATA) \ - do { \ - u8 *data = (u8 *)(MESG); \ - u64 _offset = (OFFSET); \ - u64 _set_data = (SET_DATA); \ - data[_offset + 1] = (_set_data) & 0xFF; \ - data[_offset + 0] = (_set_data & 0xFF00) >> 8; \ - } while (0) - -#define RPC_SET_U8(MESG, OFFSET, SET_DATA) \ - do { \ - u8 *data = (u8 *)(MESG); \ - data[OFFSET] = (SET_DATA) & 0xFF; \ - } while (0) - -#define RPC_GET_BE64(MESG, OFFSET, PTR) \ - do { \ - u8 *data = (u8 *)(MESG); \ - u64 _offset = (OFFSET); \ - *(u32 *)(PTR) = \ - (data[_offset + 7] | data[_offset + 6] << 8 | \ - data[_offset + 5] << 16 | data[_offset + 4] << 24 | \ - data[_offset + 3] << 32 | data[_offset + 2] << 40 | \ - data[_offset + 1] << 48 | data[_offset + 0] << 56); \ - } while (0) - -#define RPC_GET_BE32(MESG, OFFSET, PTR) \ - do { \ - u8 *data = (u8 *)(MESG); \ - u64 _offset = (OFFSET); \ - *(u32 *)(PTR) = \ - (data[_offset + 3] | data[_offset + 2] << 8 | \ - data[_offset + 1] << 16 | data[_offset + 0] << 24); \ - } while (0) - -#define RPC_GET_BE16(MESG, OFFSET, PTR) \ - do { \ - u8 *data = (u8 *)(MESG); \ - u64 _offset = (OFFSET); \ - *(u16 *)(PTR) = (data[_offset + 1] | data[_offset + 0] << 8); \ - } while (0) - -#define RPC_GET_U8(MESG, OFFSET, PTR) \ - do { \ - u8 *data = (u8 *)(MESG); \ - *(u8 *)(PTR) = (data[OFFSET]); \ - } while (0) - /* * Defines for SC PM Power Mode */ diff --git a/include/linux/firmware/xlnx-zynqmp.h b/include/linux/firmware/xlnx-zynqmp.h index d70dcd462b44..f4b3df614c77 100644 --- a/include/linux/firmware/xlnx-zynqmp.h +++ b/include/linux/firmware/xlnx-zynqmp.h @@ -3,7 +3,7 @@ * Xilinx Zynq MPSoC Firmware layer * * Copyright (C) 2014-2021 Xilinx - * Copyright (C) 2022 - 2025 Advanced Micro Devices, Inc. + * Copyright (C) 2022 - 2026 Advanced Micro Devices, Inc. * * Michal Simek <michal.simek@amd.com> * Davorin Mista <davorin.mista@aggios.com> @@ -50,6 +50,7 @@ /* PM API versions */ #define PM_API_VERSION_1 1 #define PM_API_VERSION_2 2 +#define PM_API_VERSION_3 3 #define PM_PINCTRL_PARAM_SET_VERSION 2 @@ -66,6 +67,7 @@ #define FIRMWARE_VERSION_MASK 0xFFFFU /* ATF only commands */ +#define TF_A_CLEAR_PM_STATE 0xa05 #define TF_A_PM_REGISTER_SGI 0xa04 #define PM_GET_TRUSTZONE_VERSION 0xa03 #define PM_SET_SUSPEND_MODE 0xa02 @@ -144,6 +146,12 @@ #define XPM_EVENT_ERROR_MASK_NOC_NCR BIT(13) #define XPM_EVENT_ERROR_MASK_NOC_CR BIT(12) +/* Node ID for all peripheral devices */ +#define PM_DEV_ALL_PERIPH 0x18224FFFU + +/* Node ID for all notifier callbacks */ +#define PM_ALL_NOTIFIERS 0xFFFFFFFFU + enum pm_module_id { PM_MODULE_ID = 0x0, XPM_MODULE_ID = 0x2, @@ -543,6 +551,18 @@ enum pm_gem_config_type { }; /** + * enum pm_node_status - Device node status provided by xilpm fw + * @PM_NODE_UNUSED: Device is not used + * @PM_NODE_RUNNING: Device is power-on and out of reset + * @PM_NODE_HALT: Device is power-on but in the reset state + */ +enum pm_node_status { + PM_NODE_UNUSED = 0, + PM_NODE_RUNNING = 1, + PM_NODE_HALT = 12, +}; + +/** * struct zynqmp_pm_query_data - PM query data * @qid: query ID * @arg1: Argument 1 of query data @@ -630,6 +650,10 @@ int zynqmp_pm_set_rpu_mode(u32 node_id, enum rpu_oper_mode rpu_mode); int zynqmp_pm_set_tcm_config(u32 node_id, enum rpu_tcm_comb tcm_mode); int zynqmp_pm_get_node_status(const u32 node, u32 *const status, u32 *const requirements, u32 *const usage); +int zynqmp_pm_get_rpu_node_status(const u32 node, u32 *const status, + u32 *const requirements, u32 *const usage); +int zynqmp_pm_start_rpu(const u32 node, const u64 bootaddr); +int zynqmp_pm_stop_rpu(const u32 node); int zynqmp_pm_set_sd_config(u32 node, enum pm_sd_config_type config, u32 value); int zynqmp_pm_set_gem_config(u32 node, enum pm_gem_config_type config, u32 value); @@ -939,6 +963,23 @@ static inline int zynqmp_pm_get_node_status(const u32 node, u32 *const status, return -ENODEV; } +static inline int zynqmp_pm_get_rpu_node_status(const u32 node, u32 *const status, + u32 *const requirements, + u32 *const usage) +{ + return -ENODEV; +} + +static inline int zynqmp_pm_start_rpu(const u32 node, const u64 bootaddr) +{ + return -ENODEV; +} + +static inline int zynqmp_pm_stop_rpu(const u32 node) +{ + return -ENODEV; +} + static inline int zynqmp_pm_set_sd_config(u32 node, enum pm_sd_config_type config, u32 value) diff --git a/include/linux/fixp-arith.h b/include/linux/fixp-arith.h index e485fb0c1201..4d4f4a7d4749 100644 --- a/include/linux/fixp-arith.h +++ b/include/linux/fixp-arith.h @@ -40,7 +40,7 @@ static const s32 sin_table[] = { }; /** - * __fixp_sin32() returns the sin of an angle in degrees + * __fixp_sin32() - returns the sin of an angle in degrees * * @degrees: angle, in degrees, from 0 to 360. * @@ -64,7 +64,7 @@ static inline s32 __fixp_sin32(int degrees) } /** - * fixp_sin32() returns the sin of an angle in degrees + * fixp_sin32() - returns the sin of an angle in degrees * * @degrees: angle, in degrees. The angle can be positive or negative * diff --git a/include/linux/pagevec.h b/include/linux/folio_batch.h index 63be5a451627..b45946adc50b 100644 --- a/include/linux/pagevec.h +++ b/include/linux/folio_batch.h @@ -1,18 +1,18 @@ /* SPDX-License-Identifier: GPL-2.0 */ /* - * include/linux/pagevec.h + * include/linux/folio_batch.h * * In many places it is efficient to batch an operation up against multiple * folios. A folio_batch is a container which is used for that. */ -#ifndef _LINUX_PAGEVEC_H -#define _LINUX_PAGEVEC_H +#ifndef _LINUX_FOLIO_BATCH_H +#define _LINUX_FOLIO_BATCH_H #include <linux/types.h> /* 31 pointers + header align the folio_batch structure to a power of two */ -#define PAGEVEC_SIZE 31 +#define FOLIO_BATCH_SIZE 31 struct folio; @@ -29,7 +29,7 @@ struct folio_batch { unsigned char nr; unsigned char i; bool percpu_pvec_drained; - struct folio *folios[PAGEVEC_SIZE]; + struct folio *folios[FOLIO_BATCH_SIZE]; }; /** @@ -58,7 +58,7 @@ static inline unsigned int folio_batch_count(const struct folio_batch *fbatch) static inline unsigned int folio_batch_space(const struct folio_batch *fbatch) { - return PAGEVEC_SIZE - fbatch->nr; + return FOLIO_BATCH_SIZE - fbatch->nr; } /** @@ -93,7 +93,7 @@ static inline struct folio *folio_batch_next(struct folio_batch *fbatch) return fbatch->folios[fbatch->i++]; } -void __folio_batch_release(struct folio_batch *pvec); +void __folio_batch_release(struct folio_batch *fbatch); static inline void folio_batch_release(struct folio_batch *fbatch) { @@ -102,4 +102,4 @@ static inline void folio_batch_release(struct folio_batch *fbatch) } void folio_batch_remove_exceptionals(struct folio_batch *fbatch); -#endif /* _LINUX_PAGEVEC_H */ +#endif /* _LINUX_FOLIO_BATCH_H */ diff --git a/include/linux/folio_queue.h b/include/linux/folio_queue.h index adab609c972e..f6d5f1f127c9 100644 --- a/include/linux/folio_queue.h +++ b/include/linux/folio_queue.h @@ -14,7 +14,7 @@ #ifndef _LINUX_FOLIO_QUEUE_H #define _LINUX_FOLIO_QUEUE_H -#include <linux/pagevec.h> +#include <linux/folio_batch.h> #include <linux/mm.h> /* @@ -29,12 +29,12 @@ */ struct folio_queue { struct folio_batch vec; /* Folios in the queue segment */ - u8 orders[PAGEVEC_SIZE]; /* Order of each folio */ + u8 orders[FOLIO_BATCH_SIZE]; /* Order of each folio */ struct folio_queue *next; /* Next queue segment or NULL */ struct folio_queue *prev; /* Previous queue segment of NULL */ unsigned long marks; /* 1-bit mark per folio */ unsigned long marks2; /* Second 1-bit mark per folio */ -#if PAGEVEC_SIZE > BITS_PER_LONG +#if FOLIO_BATCH_SIZE > BITS_PER_LONG #error marks is not big enough #endif unsigned int rreq_id; @@ -70,7 +70,7 @@ static inline void folioq_init(struct folio_queue *folioq, unsigned int rreq_id) */ static inline unsigned int folioq_nr_slots(const struct folio_queue *folioq) { - return PAGEVEC_SIZE; + return FOLIO_BATCH_SIZE; } /** diff --git a/include/linux/font.h b/include/linux/font.h index fd8625cd76b2..5e1cf9830084 100644 --- a/include/linux/font.h +++ b/include/linux/font.h @@ -11,47 +11,128 @@ #ifndef _VIDEO_FONT_H #define _VIDEO_FONT_H +#include <linux/math.h> #include <linux/types.h> +struct console_font; + +/* + * Glyphs + */ + +/** + * font_glyph_pitch - Calculates the number of bytes per scanline + * @width: The glyph width in bits per scanline + * + * A glyph's pitch is the number of bytes in a single scanline, rounded + * up to the next full byte. The parameter @width receives the number + * of visible bits per scanline. For example, if width is 14 bytes per + * scanline, the pitch is 2 bytes per scanline. If width is 8 bits per + * scanline, the pitch is 1 byte per scanline. + * + * Returns: + * The number of bytes in a single scanline of the glyph + */ +static inline unsigned int font_glyph_pitch(unsigned int width) +{ + return DIV_ROUND_UP(width, 8); +} + +/** + * font_glyph_size - Calculates the number of bytes per glyph + * @width: The glyph width in bits per scanline + * @vpitch: The number of scanlines in the glyph + * + * The number of bytes in a glyph depends on the pitch and the number + * of scanlines. font_glyph_size automatically calculates the pitch + * from the given width. The parameter @vpitch gives the number of + * scanlines, which is usually the glyph's height in scanlines. Fonts + * coming from user space can sometimes have a different vertical pitch + * with empty scanlines between two adjacent glyphs. + * + * Returns: the number of bytes per glyph + */ +static inline unsigned int font_glyph_size(unsigned int width, unsigned int vpitch) +{ + return font_glyph_pitch(width) * vpitch; +} + +/* + * font_data_t and helpers + */ + +/** + * typedef font_data_t - Raw font data + * + * Values of type font_data_t store a pointer to raw font data. The format + * is monochrome. Each bit sets a pixel of a stored glyph. Font data does + * not store geometry information for the individual glyphs. Users of the + * font have to store glyph size, pitch and character count separately. + * + * Font data in font_data_t is not equivalent to raw u8. Each pointer stores + * an additional hidden header before the font data. The layout is + * + * +------+-----------------------------+ + * | -16 | CRC32 Checksum (optional) | + * | -12 | <Unused> | + * | -8 | Number of data bytes | + * | -4 | Reference count | + * +------+-----------------------------+ + * | 0 | Data buffer | + * | ... | | + * +------+-----------------------------+ + * + * Use helpers to access font_data_t. Use font_data_buf() to get the stored data. + */ +typedef const unsigned char font_data_t; + +/** + * font_data_buf() - Returns the font data as raw bytes + * @fd: The font data + * + * Returns: + * The raw font data. The provided buffer is read-only. + */ +static inline const unsigned char *font_data_buf(font_data_t *fd) +{ + return (const unsigned char *)fd; +} + +font_data_t *font_data_import(const struct console_font *font, unsigned int vpitch, + u32 (*calc_csum)(u32, const void *, size_t)); +void font_data_get(font_data_t *fd); +bool font_data_put(font_data_t *fd); +unsigned int font_data_size(font_data_t *fd); +const unsigned char *font_data_glyph_buf(font_data_t *fd, + unsigned int width, unsigned int vpitch, + unsigned int c); +bool font_data_is_equal(font_data_t *lhs, font_data_t *rhs); +int font_data_export(font_data_t *fd, struct console_font *font, unsigned int vpitch); + +/* font_rotate.c */ +void font_glyph_rotate_90(const unsigned char *glyph, unsigned int width, unsigned int height, + unsigned char *out); +void font_glyph_rotate_180(const unsigned char *glyph, unsigned int width, unsigned int height, + unsigned char *out); +void font_glyph_rotate_270(const unsigned char *glyph, unsigned int width, unsigned int height, + unsigned char *out); +unsigned char *font_data_rotate(font_data_t *fd, unsigned int width, unsigned int height, + unsigned int charcount, unsigned int steps, + unsigned char *buf, size_t *bufsize); + +/* + * Font description + */ + struct font_desc { int idx; const char *name; unsigned int width, height; unsigned int charcount; - const void *data; + font_data_t *data; int pref; }; -#define VGA8x8_IDX 0 -#define VGA8x16_IDX 1 -#define PEARL8x8_IDX 2 -#define VGA6x11_IDX 3 -#define FONT7x14_IDX 4 -#define FONT10x18_IDX 5 -#define SUN8x16_IDX 6 -#define SUN12x22_IDX 7 -#define ACORN8x8_IDX 8 -#define MINI4x6_IDX 9 -#define FONT6x10_IDX 10 -#define TER16x32_IDX 11 -#define FONT6x8_IDX 12 -#define TER10x18_IDX 13 - -extern const struct font_desc font_vga_8x8, - font_vga_8x16, - font_pearl_8x8, - font_vga_6x11, - font_7x14, - font_10x18, - font_sun_8x16, - font_sun_12x22, - font_acorn_8x8, - font_mini_4x6, - font_6x10, - font_ter_16x32, - font_6x8, - font_ter_10x18; - /* Find a font with a specific name */ extern const struct font_desc *find_font(const char *name); @@ -65,17 +146,23 @@ extern const struct font_desc *get_default_font(int xres, int yres, /* Max. length for the name of a predefined font */ #define MAX_FONT_NAME 32 -/* Extra word getters */ -#define REFCOUNT(fd) (((int *)(fd))[-1]) -#define FNTSIZE(fd) (((int *)(fd))[-2]) -#define FNTCHARCNT(fd) (((int *)(fd))[-3]) -#define FNTSUM(fd) (((int *)(fd))[-4]) - -#define FONT_EXTRA_WORDS 4 +/* + * Built-in fonts + */ -struct font_data { - unsigned int extra[FONT_EXTRA_WORDS]; - const unsigned char data[]; -} __packed; +extern const struct font_desc font_10x18; +extern const struct font_desc font_6x10; +extern const struct font_desc font_6x8; +extern const struct font_desc font_7x14; +extern const struct font_desc font_acorn_8x8; +extern const struct font_desc font_mini_4x6; +extern const struct font_desc font_pearl_8x8; +extern const struct font_desc font_sun_12x22; +extern const struct font_desc font_sun_8x16; +extern const struct font_desc font_ter_10x18; +extern const struct font_desc font_ter_16x32; +extern const struct font_desc font_vga_6x11; +extern const struct font_desc font_vga_8x16; +extern const struct font_desc font_vga_8x8; #endif /* _VIDEO_FONT_H */ diff --git a/include/linux/fortify-string.h b/include/linux/fortify-string.h index 171982e53c9a..cf841dc71fef 100644 --- a/include/linux/fortify-string.h +++ b/include/linux/fortify-string.h @@ -26,7 +26,6 @@ #define FORTIFY_WRITE 1 #define EACH_FORTIFY_FUNC(macro) \ - macro(strncpy), \ macro(strnlen), \ macro(strlen), \ macro(strscpy), \ @@ -95,8 +94,6 @@ extern char *__underlying_strcat(char *p, const char *q) __RENAME(strcat); extern char *__underlying_strcpy(char *p, const char *q) __RENAME(strcpy); extern __kernel_size_t __underlying_strlen(const char *p) __RENAME(strlen); extern char *__underlying_strncat(char *p, const char *q, __kernel_size_t count) __RENAME(strncat); -extern char *__underlying_strncpy(char *p, const char *q, __kernel_size_t size) __RENAME(strncpy); - #else #if defined(__SANITIZE_MEMORY__) @@ -120,7 +117,6 @@ extern char *__underlying_strncpy(char *p, const char *q, __kernel_size_t size) #define __underlying_strcpy __builtin_strcpy #define __underlying_strlen __builtin_strlen #define __underlying_strncat __builtin_strncat -#define __underlying_strncpy __builtin_strncpy #endif @@ -159,50 +155,6 @@ extern char *__underlying_strncpy(char *p, const char *q, __kernel_size_t size) (bounds) < (length) \ ) -/** - * strncpy - Copy a string to memory with non-guaranteed NUL padding - * - * @p: pointer to destination of copy - * @q: pointer to NUL-terminated source string to copy - * @size: bytes to write at @p - * - * If strlen(@q) >= @size, the copy of @q will stop after @size bytes, - * and @p will NOT be NUL-terminated - * - * If strlen(@q) < @size, following the copy of @q, trailing NUL bytes - * will be written to @p until @size total bytes have been written. - * - * Do not use this function. While FORTIFY_SOURCE tries to avoid - * over-reads of @q, it cannot defend against writing unterminated - * results to @p. Using strncpy() remains ambiguous and fragile. - * Instead, please choose an alternative, so that the expectation - * of @p's contents is unambiguous: - * - * +--------------------+--------------------+------------+ - * | **p** needs to be: | padded to **size** | not padded | - * +====================+====================+============+ - * | NUL-terminated | strscpy_pad() | strscpy() | - * +--------------------+--------------------+------------+ - * | not NUL-terminated | strtomem_pad() | strtomem() | - * +--------------------+--------------------+------------+ - * - * Note strscpy*()'s differing return values for detecting truncation, - * and strtomem*()'s expectation that the destination is marked with - * __nonstring when it is a character array. - * - */ -__FORTIFY_INLINE __diagnose_as(__builtin_strncpy, 1, 2, 3) -char *strncpy(char * const POS p, const char *q, __kernel_size_t size) -{ - const size_t p_size = __member_size(p); - - if (__compiletime_lessthan(p_size, size)) - __write_overflow(); - if (p_size < size) - fortify_panic(FORTIFY_FUNC_strncpy, FORTIFY_WRITE, p_size, size, p); - return __underlying_strncpy(p, q, size); -} - extern __kernel_size_t __real_strnlen(const char *, __kernel_size_t) __RENAME(strnlen); /** * strnlen - Return bounded count of characters in a NUL-terminated string @@ -809,7 +761,6 @@ char *strcpy(char * const POS p, const char * const POS q) #undef __underlying_strcpy #undef __underlying_strlen #undef __underlying_strncat -#undef __underlying_strncpy #undef POS #undef POS0 diff --git a/include/linux/fprobe.h b/include/linux/fprobe.h index 0a3bcd1718f3..be1b38c981d4 100644 --- a/include/linux/fprobe.h +++ b/include/linux/fprobe.h @@ -94,6 +94,7 @@ int register_fprobe(struct fprobe *fp, const char *filter, const char *notfilter int register_fprobe_ips(struct fprobe *fp, unsigned long *addrs, int num); int register_fprobe_syms(struct fprobe *fp, const char **syms, int num); int unregister_fprobe(struct fprobe *fp); +int unregister_fprobe_async(struct fprobe *fp); bool fprobe_is_registered(struct fprobe *fp); int fprobe_count_ips_from_filter(const char *filter, const char *notfilter); #else @@ -113,6 +114,10 @@ static inline int unregister_fprobe(struct fprobe *fp) { return -EOPNOTSUPP; } +static inline int unregister_fprobe_async(struct fprobe *fp) +{ + return -EOPNOTSUPP; +} static inline bool fprobe_is_registered(struct fprobe *fp) { return false; diff --git a/include/linux/fs.h b/include/linux/fs.h index 8b3dd145b25e..f9d1e05e8ae6 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -55,8 +55,6 @@ struct bdi_writeback; struct bio; struct io_comp_batch; struct fiemap_extent_info; -struct hd_geometry; -struct iovec; struct kiocb; struct kobject; struct pipe_inode_info; @@ -440,11 +438,17 @@ struct address_space_operations { int (*swap_activate)(struct swap_info_struct *sis, struct file *file, sector_t *span); void (*swap_deactivate)(struct file *file); - int (*swap_rw)(struct kiocb *iocb, struct iov_iter *iter); }; extern const struct address_space_operations empty_aops; +/* Structure for tracking metadata buffer heads associated with the mapping */ +struct mapping_metadata_bhs { + struct address_space *mapping; /* Mapping bhs are associated with */ + spinlock_t lock; /* Lock protecting bh list */ + struct list_head list; /* The list of bhs (b_assoc_buffers) */ +}; + /** * struct address_space - Contents of a cacheable, mappable object. * @host: Owner, either the inode or the block_device. @@ -455,7 +459,6 @@ extern const struct address_space_operations empty_aops; * memory mappings. * @gfp_mask: Memory allocation flags to use for allocating pages. * @i_mmap_writable: Number of VM_SHARED, VM_MAYWRITE mappings. - * @nr_thps: Number of THPs in the pagecache (non-shmem only). * @i_mmap: Tree of private and shared mappings. * @i_mmap_rwsem: Protects @i_mmap and @i_mmap_writable. * @nrpages: Number of page entries, protected by the i_pages lock. @@ -464,8 +467,6 @@ extern const struct address_space_operations empty_aops; * @flags: Error bits and flags (AS_*). * @wb_err: The most recent error which has occurred. * @i_private_lock: For use by the owner of the address_space. - * @i_private_list: For use by the owner of the address_space. - * @i_private_data: For use by the owner of the address_space. */ struct address_space { struct inode *host; @@ -473,10 +474,6 @@ struct address_space { struct rw_semaphore invalidate_lock; gfp_t gfp_mask; atomic_t i_mmap_writable; -#ifdef CONFIG_READ_ONLY_THP_FOR_FS - /* number of thp, only for non-shmem files */ - atomic_t nr_thps; -#endif struct rb_root_cached i_mmap; unsigned long nrpages; pgoff_t writeback_index; @@ -484,9 +481,7 @@ struct address_space { unsigned long flags; errseq_t wb_err; spinlock_t i_private_lock; - struct list_head i_private_list; struct rw_semaphore i_mmap_rwsem; - void * i_private_data; } __attribute__((aligned(sizeof(long)))) __randomize_layout; /* * On most architectures that alignment is already the case; but @@ -744,7 +739,8 @@ enum inode_state_flags_enum { I_CREATING = (1U << 15), I_DONTCACHE = (1U << 16), I_SYNC_QUEUED = (1U << 17), - I_PINNING_NETFS_WB = (1U << 18) + I_PINNING_NETFS_WB = (1U << 18), + I_METADATA_WRITEBACK = (1U << 19), }; #define I_DIRTY_INODE (I_DIRTY_SYNC | I_DIRTY_DATASYNC) @@ -783,7 +779,7 @@ struct inode { #endif /* Stat data, not accessed from path walking */ - unsigned long i_ino; + u64 i_ino; /* * Filesystems may only read i_nlink directly. They shall use the * following functions for modification: @@ -1602,12 +1598,12 @@ struct timespec64 inode_set_ctime_deleg(struct inode *inode, static inline time64_t inode_get_atime_sec(const struct inode *inode) { - return inode->i_atime_sec; + return READ_ONCE(inode->i_atime_sec); } static inline long inode_get_atime_nsec(const struct inode *inode) { - return inode->i_atime_nsec; + return READ_ONCE(inode->i_atime_nsec); } static inline struct timespec64 inode_get_atime(const struct inode *inode) @@ -1621,8 +1617,8 @@ static inline struct timespec64 inode_get_atime(const struct inode *inode) static inline struct timespec64 inode_set_atime_to_ts(struct inode *inode, struct timespec64 ts) { - inode->i_atime_sec = ts.tv_sec; - inode->i_atime_nsec = ts.tv_nsec; + WRITE_ONCE(inode->i_atime_sec, ts.tv_sec); + WRITE_ONCE(inode->i_atime_nsec, ts.tv_nsec); return ts; } @@ -1637,12 +1633,12 @@ static inline struct timespec64 inode_set_atime(struct inode *inode, static inline time64_t inode_get_mtime_sec(const struct inode *inode) { - return inode->i_mtime_sec; + return READ_ONCE(inode->i_mtime_sec); } static inline long inode_get_mtime_nsec(const struct inode *inode) { - return inode->i_mtime_nsec; + return READ_ONCE(inode->i_mtime_nsec); } static inline struct timespec64 inode_get_mtime(const struct inode *inode) @@ -1655,8 +1651,8 @@ static inline struct timespec64 inode_get_mtime(const struct inode *inode) static inline struct timespec64 inode_set_mtime_to_ts(struct inode *inode, struct timespec64 ts) { - inode->i_mtime_sec = ts.tv_sec; - inode->i_mtime_nsec = ts.tv_nsec; + WRITE_ONCE(inode->i_mtime_sec, ts.tv_sec); + WRITE_ONCE(inode->i_mtime_nsec, ts.tv_nsec); return ts; } @@ -1681,12 +1677,12 @@ static inline struct timespec64 inode_set_mtime(struct inode *inode, static inline time64_t inode_get_ctime_sec(const struct inode *inode) { - return inode->i_ctime_sec; + return READ_ONCE(inode->i_ctime_sec); } static inline long inode_get_ctime_nsec(const struct inode *inode) { - return inode->i_ctime_nsec & ~I_CTIME_QUERIED; + return READ_ONCE(inode->i_ctime_nsec) & ~I_CTIME_QUERIED; } static inline struct timespec64 inode_get_ctime(const struct inode *inode) @@ -1917,12 +1913,9 @@ struct dir_context { */ #define COPY_FILE_SPLICE (1 << 0) -struct iov_iter; struct io_uring_cmd; struct offset_ctx; -typedef unsigned int __bitwise fop_flags_t; - struct file_operations { struct module *owner; fop_flags_t fop_flags; @@ -2007,7 +2000,7 @@ struct inode_operations { int (*readlink) (struct dentry *, char __user *,int); int (*create) (struct mnt_idmap *, struct inode *,struct dentry *, - umode_t, bool); + umode_t); int (*link) (struct dentry *,struct inode *,struct dentry *); int (*unlink) (struct inode *,struct dentry *); int (*symlink) (struct mnt_idmap *, struct inode *,struct dentry *, @@ -2058,8 +2051,9 @@ static inline bool can_mmap_file(struct file *file) return true; } -int __compat_vma_mmap(const struct file_operations *f_op, - struct file *file, struct vm_area_struct *vma); +void compat_set_desc_from_vma(struct vm_area_desc *desc, const struct file *file, + const struct vm_area_struct *vma); +int __compat_vma_mmap(struct vm_area_desc *desc, struct vm_area_struct *vma); int compat_vma_mmap(struct file *file, struct vm_area_struct *vma); static inline int vfs_mmap(struct file *file, struct vm_area_struct *vma) @@ -2217,8 +2211,28 @@ static inline void mark_inode_dirty_sync(struct inode *inode) __mark_inode_dirty(inode, I_DIRTY_SYNC); } +static inline void set_inode_metadata_writeback(struct inode *inode) +{ + spin_lock(&inode->i_lock); + inode_state_set(inode, I_METADATA_WRITEBACK); + spin_unlock(&inode->i_lock); +} + +/* + * returns the refcount on the inode. it can change arbitrarily. + */ +static inline int icount_read_once(const struct inode *inode) +{ + return atomic_read(&inode->i_count); +} + +/* + * returns the refcount on the inode. The lock guarantees no 0->1 or 1->0 transitions + * of the count are going to take place, otherwise it changes arbitrarily. + */ static inline int icount_read(const struct inode *inode) { + lockdep_assert_held(&inode->i_lock); return atomic_read(&inode->i_count); } @@ -2280,12 +2294,14 @@ struct file_system_type { #define FS_MGTIME 64 /* FS uses multigrain timestamps */ #define FS_LBS 128 /* FS supports LBS */ #define FS_POWER_FREEZE 256 /* Always freeze on suspend/hibernate */ +#define FS_USERNS_MOUNT_RESTRICTED 512 /* Restrict mount in userns if not already visible */ +#define FS_USERNS_DELEGATABLE 1024 /* Can be mounted inside userns from outside */ #define FS_RENAME_DOES_D_MOVE 32768 /* FS will handle d_move() during rename() internally. */ int (*init_fs_context)(struct fs_context *); const struct fs_parameter_spec *parameters; void (*kill_sb) (struct super_block *); struct module *owner; - struct file_system_type * next; + struct hlist_node list; struct hlist_head fs_supers; struct lock_class_key s_lock_key; @@ -2326,10 +2342,6 @@ void free_anon_bdev(dev_t); struct super_block *sget_fc(struct fs_context *fc, int (*test)(struct super_block *, struct fs_context *), int (*set)(struct super_block *, struct fs_context *)); -struct super_block *sget(struct file_system_type *type, - int (*test)(struct super_block *,void *), - int (*set)(struct super_block *,void *), - int flags, void *data); struct super_block *sget_dev(struct fs_context *fc, dev_t dev); /* Alas, no aliases. Too much hassle with bringing module.h everywhere */ @@ -2406,6 +2418,21 @@ static inline void super_set_sysfs_name_generic(struct super_block *sb, const ch extern void ihold(struct inode * inode); extern void iput(struct inode *); void iput_not_last(struct inode *); + +/** + * iput_if_not_last - drop an inode reference only if it is not the last one + * @inode: inode to put + * + * Returns true if the reference was dropped, false if this was the last + * reference and the caller must arrange for final iput() in a safe context. + */ +static inline bool __must_check iput_if_not_last(struct inode *inode) +{ + VFS_BUG_ON_INODE(inode_state_read_once(inode) & (I_FREEING | I_CLEAR), inode); + VFS_BUG_ON_INODE(icount_read_once(inode) < 1, inode); + return atomic_add_unless(&inode->i_count, -1, 1); +} + int inode_update_time(struct inode *inode, enum fs_update_time type, unsigned int flags); int generic_update_time(struct inode *inode, enum fs_update_time type, @@ -2437,6 +2464,11 @@ static inline struct mnt_idmap *file_mnt_idmap(const struct file *file) return mnt_idmap(file->f_path.mnt); } +static inline bool file_owner_or_capable(const struct file *file) +{ + return inode_owner_or_capable(file_mnt_idmap(file), file_inode(file)); +} + /** * is_idmapped_mnt - check whether a mount is mapped * @mnt: the mount to check @@ -2475,6 +2507,19 @@ struct file *dentry_create(struct path *path, int flags, umode_t mode, const struct cred *cred); const struct path *backing_file_user_path(const struct file *f); +#ifdef CONFIG_SECURITY +void *backing_file_security(const struct file *f); +void backing_file_set_security(struct file *f, void *security); +#else +static inline void *backing_file_security(const struct file *f) +{ + return NULL; +} +static inline void backing_file_set_security(struct file *f, void *security) +{ +} +#endif /* CONFIG_SECURITY */ + /* * When mmapping a file on a stackable filesystem (e.g., overlayfs), the file * stored in ->vm_file is a backing file whose f_inode is on the underlying @@ -2610,6 +2655,7 @@ extern int __must_check file_write_and_wait_range(struct file *file, loff_t start, loff_t end); int filemap_flush_range(struct address_space *mapping, loff_t start, loff_t end); +void filemap_dontcache_kick_writeback(struct address_space *mapping); static inline int file_write_and_wait(struct file *file) { @@ -2643,10 +2689,7 @@ static inline ssize_t generic_write_sync(struct kiocb *iocb, ssize_t count) if (ret) return ret; } else if (iocb->ki_flags & IOCB_DONTCACHE) { - struct address_space *mapping = iocb->ki_filp->f_mapping; - - filemap_flush_range(mapping, iocb->ki_pos - count, - iocb->ki_pos - 1); + filemap_dontcache_kick_writeback(iocb->ki_filp->f_mapping); } return count; @@ -2912,65 +2955,63 @@ static inline bool name_contains_dotdot(const char *name) #include <linux/err.h> /* needed for stackable file system support */ -extern loff_t default_llseek(struct file *file, loff_t offset, int whence); +loff_t default_llseek(struct file *file, loff_t offset, int whence); -extern loff_t vfs_llseek(struct file *file, loff_t offset, int whence); +loff_t vfs_llseek(struct file *file, loff_t offset, int whence); -extern int inode_init_always_gfp(struct super_block *, struct inode *, gfp_t); +int inode_init_always_gfp(struct super_block *sb, struct inode *inode, gfp_t gfp); static inline int inode_init_always(struct super_block *sb, struct inode *inode) { return inode_init_always_gfp(sb, inode, GFP_NOFS); } -extern void inode_init_once(struct inode *); -extern void address_space_init_once(struct address_space *mapping); -extern struct inode * igrab(struct inode *); -extern ino_t iunique(struct super_block *, ino_t); -extern int inode_needs_sync(struct inode *inode); -extern int inode_just_drop(struct inode *inode); +void inode_init_once(struct inode *inode); +void address_space_init_once(struct address_space *mapping); +struct inode *igrab(struct inode *inode); +ino_t iunique(struct super_block *sb, ino_t max_reserved); +int inode_needs_sync(struct inode *inode); +int inode_just_drop(struct inode *inode); static inline int inode_generic_drop(struct inode *inode) { return !inode->i_nlink || inode_unhashed(inode); } -extern void d_mark_dontcache(struct inode *inode); +void d_mark_dontcache(struct inode *inode); -extern struct inode *ilookup5_nowait(struct super_block *sb, - unsigned long hashval, int (*test)(struct inode *, void *), - void *data, bool *isnew); -extern struct inode *ilookup5(struct super_block *sb, unsigned long hashval, - int (*test)(struct inode *, void *), void *data); -extern struct inode *ilookup(struct super_block *sb, unsigned long ino); +struct inode *ilookup5_nowait(struct super_block *sb, u64 hashval, + int (*test)(struct inode *, void *), void *data, + bool *isnew); +struct inode *ilookup5(struct super_block *sb, u64 hashval, + int (*test)(struct inode *, void *), void *data); +struct inode *ilookup(struct super_block *sb, u64 ino); -extern struct inode *inode_insert5(struct inode *inode, unsigned long hashval, - int (*test)(struct inode *, void *), - int (*set)(struct inode *, void *), - void *data); -struct inode *iget5_locked(struct super_block *, unsigned long, +struct inode *inode_insert5(struct inode *inode, u64 hashval, + int (*test)(struct inode *, void *), + int (*set)(struct inode *, void *), void *data); +struct inode *iget5_locked(struct super_block *sb, u64 hashval, int (*test)(struct inode *, void *), - int (*set)(struct inode *, void *), void *); -struct inode *iget5_locked_rcu(struct super_block *, unsigned long, + int (*set)(struct inode *, void *), void *data); +struct inode *iget5_locked_rcu(struct super_block *sb, u64 hashval, int (*test)(struct inode *, void *), - int (*set)(struct inode *, void *), void *); -extern struct inode * iget_locked(struct super_block *, unsigned long); -extern struct inode *find_inode_nowait(struct super_block *, - unsigned long, - int (*match)(struct inode *, - unsigned long, void *), - void *data); -extern struct inode *find_inode_rcu(struct super_block *, unsigned long, - int (*)(struct inode *, void *), void *); -extern struct inode *find_inode_by_ino_rcu(struct super_block *, unsigned long); -extern int insert_inode_locked4(struct inode *, unsigned long, int (*test)(struct inode *, void *), void *); -extern int insert_inode_locked(struct inode *); + int (*set)(struct inode *, void *), void *data); +struct inode *iget_locked(struct super_block *sb, u64 ino); +struct inode *find_inode_nowait(struct super_block *sb, u64 hashval, + int (*match)(struct inode *, u64, void *), + void *data); +struct inode *find_inode_rcu(struct super_block *sb, u64 hashval, + int (*test)(struct inode *, void *), void *data); +struct inode *find_inode_by_ino_rcu(struct super_block *sb, u64 ino); +int insert_inode_locked4(struct inode *inode, u64 hashval, + int (*test)(struct inode *, void *), void *data); +int insert_inode_locked(struct inode *inode); #ifdef CONFIG_DEBUG_LOCK_ALLOC -extern void lockdep_annotate_inode_mutex_key(struct inode *inode); +void lockdep_annotate_inode_mutex_key(struct inode *inode); #else static inline void lockdep_annotate_inode_mutex_key(struct inode *inode) { }; #endif -extern void unlock_new_inode(struct inode *); -extern void discard_new_inode(struct inode *); -extern unsigned int get_next_ino(void); -extern void evict_inodes(struct super_block *sb); +void unlock_new_inode(struct inode *inode); +void discard_new_inode(struct inode *inode); +unsigned int get_next_ino(void); +void evict_inodes(struct super_block *sb); void dump_mapping(const struct address_space *); /* @@ -3015,21 +3056,21 @@ int setattr_should_drop_sgid(struct mnt_idmap *idmap, */ #define alloc_inode_sb(_sb, _cache, _gfp) kmem_cache_alloc_lru(_cache, &_sb->s_inode_lru, _gfp) -extern void __insert_inode_hash(struct inode *, unsigned long hashval); +void __insert_inode_hash(struct inode *inode, u64 hashval); static inline void insert_inode_hash(struct inode *inode) { __insert_inode_hash(inode, inode->i_ino); } -extern void __remove_inode_hash(struct inode *); +void __remove_inode_hash(struct inode *inode); static inline void remove_inode_hash(struct inode *inode) { if (!inode_unhashed(inode) && !hlist_fake(&inode->i_hash)) __remove_inode_hash(inode); } -extern void inode_sb_list_add(struct inode *inode); -extern void inode_lru_list_add(struct inode *inode); +void inode_sb_list_add(struct inode *inode); +void inode_lru_list_add(struct inode *inode); int generic_file_mmap(struct file *, struct vm_area_struct *); int generic_file_mmap_prepare(struct vm_area_desc *desc); @@ -3295,8 +3336,8 @@ void simple_offset_destroy(struct offset_ctx *octx); extern const struct file_operations simple_offset_dir_operations; -extern int __generic_file_fsync(struct file *, loff_t, loff_t, int); -extern int generic_file_fsync(struct file *, loff_t, loff_t, int); +extern int simple_fsync_noflush(struct file *, loff_t, loff_t, int); +extern int simple_fsync(struct file *, loff_t, loff_t, int); extern int generic_check_addressable(unsigned, u64); diff --git a/include/linux/fs/super.h b/include/linux/fs/super.h index f21ffbb6dea5..733d439f01ed 100644 --- a/include/linux/fs/super.h +++ b/include/linux/fs/super.h @@ -235,4 +235,14 @@ int freeze_super(struct super_block *super, enum freeze_holder who, int thaw_super(struct super_block *super, enum freeze_holder who, const void *freeze_owner); +int sb_init_dio_done_wq(struct super_block *sb); + +struct file; +struct file *fs_bdev_file_open_by_dev(dev_t dev, blk_mode_t mode, void *holder, + struct super_block *sb); +struct file *fs_bdev_file_open_by_path(const char *path, blk_mode_t mode, + void *holder, struct super_block *sb); +void fs_bdev_unregister(struct file *bdev_file, struct super_block *sb); +void fs_bdev_file_release(struct file *bdev_file, struct super_block *sb); + #endif /* _LINUX_FS_SUPER_H */ diff --git a/include/linux/fs/super_types.h b/include/linux/fs/super_types.h index fa7638b81246..ecd96aeb1cee 100644 --- a/include/linux/fs/super_types.h +++ b/include/linux/fs/super_types.h @@ -30,6 +30,7 @@ struct mount; struct mtd_info; struct quotactl_ops; struct shrinker; +struct super_dev; struct unicode_map; struct user_namespace; struct workqueue_struct; @@ -86,6 +87,8 @@ struct super_operations { void (*free_inode)(struct inode *inode); void (*dirty_inode)(struct inode *inode, int flags); int (*write_inode)(struct inode *inode, struct writeback_control *wbc); + int (*sync_inode_metadata)(struct inode *inode, + struct writeback_control *wbc); int (*drop_inode)(struct inode *inode); void (*evict_inode)(struct inode *inode); void (*put_super)(struct super_block *sb); @@ -132,6 +135,7 @@ struct super_operations { struct super_block { struct list_head s_list; /* Keep this first */ dev_t s_dev; /* search index; _not_ kdev_t */ + struct super_dev *s_super_dev; /* sget_fc()'s device table claim */ unsigned char s_blocksize_bits; unsigned long s_blocksize; loff_t s_maxbytes; /* Max file size */ @@ -145,7 +149,7 @@ struct super_block { unsigned long s_magic; struct dentry *s_root; struct rw_semaphore s_umount; - int s_count; + refcount_t s_passive; atomic_t s_active; #ifdef CONFIG_SECURITY void *s_security; @@ -162,7 +166,8 @@ struct super_block { struct unicode_map *s_encoding; __u16 s_encoding_flags; #endif - struct hlist_bl_head s_roots; /* alternate root dentries for NFS */ + struct hlist_head s_roots; /* alternate root dentries for NFS */ + spinlock_t s_roots_lock; struct mount *s_mounts; /* list of mounts; _not_ for fs use */ struct block_device *s_bdev; /* can go away once we use an accessor for @s_bdev_file */ struct file *s_bdev_file; @@ -274,6 +279,14 @@ struct super_block { /* number of fserrors that are being sent to fsnotify/filesystems */ refcount_t s_pending_errors; + +#ifdef CONFIG_CGROUP_WRITEBACK + /* + * Number of in-flight inode wb switches for this sb. Drained by + * cgroup_writeback_umount() before tear-down. + */ + atomic_t s_isw_nr_in_flight; +#endif } __randomize_layout; /* @@ -291,7 +304,7 @@ struct super_block { #define SB_NODIRATIME BIT(11) /* Do not update directory access times */ #define SB_SILENT BIT(15) #define SB_POSIXACL BIT(16) /* Supports POSIX ACLs */ -#define SB_INLINECRYPT BIT(17) /* Use blk-crypto for encrypted files */ +#define SB_INLINECRYPT BIT(17) /* Use inline crypto hardware if available */ #define SB_KERNMOUNT BIT(22) /* this is a kern_mount call */ #define SB_I_VERSION BIT(23) /* Update inode I_version field */ #define SB_LAZYTIME BIT(25) /* Update the on-disk [acm]times lazily */ @@ -326,7 +339,7 @@ struct super_block { #define SB_I_STABLE_WRITES 0x00000008 /* don't modify blks until WB is done */ /* sb->s_iflags to limit user namespace mounts */ -#define SB_I_USERNS_VISIBLE 0x00000010 /* fstype already mounted */ +#define SB_I_RESTRICTED_VARIANT 0x00000010 #define SB_I_IMA_UNVERIFIABLE_SIGNATURE 0x00000020 #define SB_I_UNTRUSTED_MOUNTER 0x00000040 #define SB_I_EVM_HMAC_UNSUPPORTED 0x00000080 @@ -338,5 +351,6 @@ struct super_block { #define SB_I_NOUMASK 0x00001000 /* VFS does not apply umask */ #define SB_I_NOIDMAP 0x00002000 /* No idmapped mounts on this superblock */ #define SB_I_ALLOW_HSM 0x00004000 /* Allow HSM events on this superblock */ +#define SB_I_NO_DATA_INTEGRITY 0x00008000 /* fs cannot guarantee data persistence on sync */ #endif /* _LINUX_FS_SUPER_TYPES_H */ diff --git a/include/linux/fs_parser.h b/include/linux/fs_parser.h index 5e8a3b546033..98b83708f92b 100644 --- a/include/linux/fs_parser.h +++ b/include/linux/fs_parser.h @@ -27,8 +27,8 @@ typedef int fs_param_type(struct p_log *, * The type of parameter expected. */ fs_param_type fs_param_is_bool, fs_param_is_u32, fs_param_is_s32, fs_param_is_u64, - fs_param_is_enum, fs_param_is_string, fs_param_is_blob, fs_param_is_blockdev, - fs_param_is_path, fs_param_is_fd, fs_param_is_uid, fs_param_is_gid, + fs_param_is_enum, fs_param_is_string, fs_param_is_blockdev, + fs_param_is_fd, fs_param_is_uid, fs_param_is_gid, fs_param_is_file_or_string; /* @@ -84,8 +84,6 @@ extern int fs_lookup_param(struct fs_context *fc, extern int lookup_constant(const struct constant_table tbl[], const char *name, int not_found); -extern const struct constant_table bool_names[]; - #ifdef CONFIG_VALIDATE_FS_PARSER extern bool fs_validate_description(const char *name, const struct fs_parameter_spec *desc); @@ -127,9 +125,7 @@ static inline bool fs_validate_description(const char *name, #define fsparam_enum(NAME, OPT, array) __fsparam(fs_param_is_enum, NAME, OPT, 0, array) #define fsparam_string(NAME, OPT) \ __fsparam(fs_param_is_string, NAME, OPT, 0, NULL) -#define fsparam_blob(NAME, OPT) __fsparam(fs_param_is_blob, NAME, OPT, 0, NULL) #define fsparam_bdev(NAME, OPT) __fsparam(fs_param_is_blockdev, NAME, OPT, 0, NULL) -#define fsparam_path(NAME, OPT) __fsparam(fs_param_is_path, NAME, OPT, 0, NULL) #define fsparam_fd(NAME, OPT) __fsparam(fs_param_is_fd, NAME, OPT, 0, NULL) #define fsparam_file_or_string(NAME, OPT) \ __fsparam(fs_param_is_file_or_string, NAME, OPT, 0, NULL) diff --git a/include/linux/fs_struct.h b/include/linux/fs_struct.h index 0070764b790a..97eef8d3863d 100644 --- a/include/linux/fs_struct.h +++ b/include/linux/fs_struct.h @@ -6,6 +6,7 @@ #include <linux/path.h> #include <linux/spinlock.h> #include <linux/seqlock.h> +#include <linux/vfsdebug.h> struct fs_struct { int users; @@ -16,6 +17,7 @@ struct fs_struct { } __randomize_layout; extern struct kmem_cache *fs_cachep; +extern struct fs_struct *userspace_init_fs; extern void exit_fs(struct task_struct *); extern void set_fs_root(struct fs_struct *, const struct path *); @@ -40,6 +42,8 @@ static inline void get_fs_pwd(struct fs_struct *fs, struct path *pwd) read_sequnlock_excl(&fs->seq); } +struct fs_struct *switch_fs_struct(struct fs_struct *new_fs); + extern bool current_chrooted(void); static inline int current_umask(void) @@ -47,4 +51,34 @@ static inline int current_umask(void) return current->fs->umask; } +/* + * Temporarily use userspace_init_fs for path resolution in kthreads. + * Callers should use scoped_with_init_fs() which automatically + * restores the original fs_struct at scope exit. + */ +static inline struct fs_struct *__override_init_fs(void) +{ + struct fs_struct *old_fs; + + old_fs = current->fs; + WRITE_ONCE(current->fs, userspace_init_fs); + return old_fs; +} + +static inline void __revert_init_fs(struct fs_struct *old_fs) +{ + VFS_WARN_ON_ONCE(current->fs != userspace_init_fs); + WRITE_ONCE(current->fs, old_fs); +} + +DEFINE_CLASS(__override_init_fs, + struct fs_struct *, + __revert_init_fs(_T), + __override_init_fs(), void) + +#define scoped_with_init_fs() \ + scoped_class(__override_init_fs, __UNIQUE_ID(label)) + +void __init init_userspace_fs(void); + #endif /* _LINUX_FS_STRUCT_H */ diff --git a/include/linux/fscrypt.h b/include/linux/fscrypt.h index 516aba5b858b..28f2108e29f3 100644 --- a/include/linux/fscrypt.h +++ b/include/linux/fscrypt.h @@ -57,6 +57,9 @@ struct fscrypt_name { /* Maximum value for the third parameter of fscrypt_operations.set_context(). */ #define FSCRYPT_SET_CONTEXT_MAX_SIZE 40 +/* Maximum supported number of block devices per filesystem */ +#define FSCRYPT_MAX_DEVICES 8 + #ifdef CONFIG_FS_ENCRYPTION /* Crypto operations for filesystems */ @@ -69,14 +72,15 @@ struct fscrypt_operations { ptrdiff_t inode_info_offs; /* - * If set, then fs/crypto/ will allocate a global bounce page pool the - * first time an encryption key is set up for a file. The bounce page - * pool is required by the following functions: - * - * - fscrypt_encrypt_pagecache_blocks() - * - fscrypt_zeroout_range() for files not using inline crypto - * - * If the filesystem doesn't use those, it doesn't need to set this. + * Set to 1 if the filesystem is block-based. This causes fs/crypto/ to + * set up the key for regular files as a blk_crypto_key. The filesystem + * then uses fscrypt_set_bio_crypt_ctx() and similar functions. + */ + unsigned int is_block_based : 1; + + /* + * Set to 1 if the filesystem uses fscrypt_encrypt_pagecache_blocks(). + * This enables the allocation of the bounce page pool it requires. */ unsigned int needs_bounce_pages : 1; @@ -181,21 +185,20 @@ struct fscrypt_operations { bool (*has_stable_inodes)(struct super_block *sb); /* - * Return an array of pointers to the block devices to which the - * filesystem may write encrypted file contents, NULL if the filesystem - * only has a single such block device, or an ERR_PTR() on error. + * Retrieve the list of block devices to which the filesystem may write + * encrypted file contents. * - * On successful non-NULL return, *num_devs is set to the number of - * devices in the returned array. The caller must free the returned - * array using kfree(). + * This writes the block_device pointers to @devs and returns the count + * (between 1 and FSCRYPT_MAX_DEVICES inclusively). * * If the filesystem can use multiple block devices (other than block * devices that aren't used for encrypted file contents, such as * external journal devices), and wants to support inline encryption, * then it must implement this function. Otherwise it's not needed. */ - struct block_device **(*get_devices)(struct super_block *sb, - unsigned int *num_devs); + unsigned int (*get_devices)( + struct super_block *sb, + struct block_device *devs[FSCRYPT_MAX_DEVICES]); }; int fscrypt_d_revalidate(struct inode *dir, const struct qstr *name, @@ -342,7 +345,6 @@ static inline void fscrypt_prepare_dentry(struct dentry *dentry, } /* crypto.c */ -void fscrypt_enqueue_decrypt_work(struct work_struct *); struct page *fscrypt_encrypt_pagecache_blocks(struct folio *folio, size_t len, size_t offs, gfp_t gfp_flags); @@ -350,8 +352,6 @@ int fscrypt_encrypt_block_inplace(const struct inode *inode, struct page *page, unsigned int len, unsigned int offs, u64 lblk_num); -int fscrypt_decrypt_pagecache_blocks(struct folio *folio, size_t len, - size_t offs); int fscrypt_decrypt_block_inplace(const struct inode *inode, struct page *page, unsigned int len, unsigned int offs, u64 lblk_num); @@ -448,11 +448,6 @@ bool fscrypt_match_name(const struct fscrypt_name *fname, const u8 *de_name, u32 de_name_len); u64 fscrypt_fname_siphash(const struct inode *dir, const struct qstr *name); -/* bio.c */ -bool fscrypt_decrypt_bio(struct bio *bio); -int fscrypt_zeroout_range(const struct inode *inode, pgoff_t lblk, - sector_t pblk, unsigned int len); - /* hooks.c */ int fscrypt_file_open(struct inode *inode, struct file *filp); int __fscrypt_prepare_link(struct inode *inode, struct inode *dir, @@ -509,9 +504,6 @@ static inline void fscrypt_prepare_dentry(struct dentry *dentry, } /* crypto.c */ -static inline void fscrypt_enqueue_decrypt_work(struct work_struct *work) -{ -} static inline struct page *fscrypt_encrypt_pagecache_blocks(struct folio *folio, size_t len, size_t offs, gfp_t gfp_flags) @@ -527,12 +519,6 @@ static inline int fscrypt_encrypt_block_inplace(const struct inode *inode, return -EOPNOTSUPP; } -static inline int fscrypt_decrypt_pagecache_blocks(struct folio *folio, - size_t len, size_t offs) -{ - return -EOPNOTSUPP; -} - static inline int fscrypt_decrypt_block_inplace(const struct inode *inode, struct page *page, unsigned int len, @@ -749,18 +735,6 @@ static inline int fscrypt_d_revalidate(struct inode *dir, const struct qstr *nam return 1; } -/* bio.c */ -static inline bool fscrypt_decrypt_bio(struct bio *bio) -{ - return true; -} - -static inline int fscrypt_zeroout_range(const struct inode *inode, pgoff_t lblk, - sector_t pblk, unsigned int len) -{ - return -EOPNOTSUPP; -} - /* hooks.c */ static inline int fscrypt_file_open(struct inode *inode, struct file *filp) @@ -860,99 +834,44 @@ static inline void fscrypt_set_ops(struct super_block *sb, #endif /* !CONFIG_FS_ENCRYPTION */ -/* inline_crypt.c */ +/* block.c */ #ifdef CONFIG_FS_ENCRYPTION_INLINE_CRYPT -bool __fscrypt_inode_uses_inline_crypto(const struct inode *inode); - -void fscrypt_set_bio_crypt_ctx(struct bio *bio, - const struct inode *inode, u64 first_lblk, - gfp_t gfp_mask); - -void fscrypt_set_bio_crypt_ctx_bh(struct bio *bio, - const struct buffer_head *first_bh, - gfp_t gfp_mask); +void fscrypt_set_bio_crypt_ctx(struct bio *bio, const struct inode *inode, + loff_t pos, gfp_t gfp_mask); bool fscrypt_mergeable_bio(struct bio *bio, const struct inode *inode, - u64 next_lblk); - -bool fscrypt_mergeable_bio_bh(struct bio *bio, - const struct buffer_head *next_bh); - -bool fscrypt_dio_supported(struct inode *inode); + loff_t pos); u64 fscrypt_limit_io_blocks(const struct inode *inode, u64 lblk, u64 nr_blocks); +int fscrypt_zeroout_range(const struct inode *inode, loff_t pos, + sector_t sector, u64 len); #else /* CONFIG_FS_ENCRYPTION_INLINE_CRYPT */ -static inline bool __fscrypt_inode_uses_inline_crypto(const struct inode *inode) -{ - return false; -} - static inline void fscrypt_set_bio_crypt_ctx(struct bio *bio, const struct inode *inode, - u64 first_lblk, gfp_t gfp_mask) { } - -static inline void fscrypt_set_bio_crypt_ctx_bh( - struct bio *bio, - const struct buffer_head *first_bh, - gfp_t gfp_mask) { } + loff_t pos, gfp_t gfp_mask) { } static inline bool fscrypt_mergeable_bio(struct bio *bio, const struct inode *inode, - u64 next_lblk) -{ - return true; -} - -static inline bool fscrypt_mergeable_bio_bh(struct bio *bio, - const struct buffer_head *next_bh) + loff_t pos) { return true; } -static inline bool fscrypt_dio_supported(struct inode *inode) -{ - return !fscrypt_needs_contents_encryption(inode); -} - static inline u64 fscrypt_limit_io_blocks(const struct inode *inode, u64 lblk, u64 nr_blocks) { return nr_blocks; } -#endif /* !CONFIG_FS_ENCRYPTION_INLINE_CRYPT */ -/** - * fscrypt_inode_uses_inline_crypto() - test whether an inode uses inline - * encryption - * @inode: an inode. If encrypted, its key must be set up. - * - * Return: true if the inode requires file contents encryption and if the - * encryption should be done in the block layer via blk-crypto rather - * than in the filesystem layer. - */ -static inline bool fscrypt_inode_uses_inline_crypto(const struct inode *inode) +static inline int fscrypt_zeroout_range(const struct inode *inode, loff_t pos, + sector_t sector, u64 len) { - return fscrypt_needs_contents_encryption(inode) && - __fscrypt_inode_uses_inline_crypto(inode); -} - -/** - * fscrypt_inode_uses_fs_layer_crypto() - test whether an inode uses fs-layer - * encryption - * @inode: an inode. If encrypted, its key must be set up. - * - * Return: true if the inode requires file contents encryption and if the - * encryption should be done in the filesystem layer rather than in the - * block layer via blk-crypto. - */ -static inline bool fscrypt_inode_uses_fs_layer_crypto(const struct inode *inode) -{ - return fscrypt_needs_contents_encryption(inode) && - !__fscrypt_inode_uses_inline_crypto(inode); + return -EOPNOTSUPP; } +#endif /* !CONFIG_FS_ENCRYPTION_INLINE_CRYPT */ /** * fscrypt_has_encryption_key() - check whether an inode has had its key set up @@ -1140,15 +1059,4 @@ static inline int fscrypt_encrypt_symlink(struct inode *inode, return 0; } -/* If *pagep is a bounce page, free it and set *pagep to the pagecache page */ -static inline void fscrypt_finalize_bounce_page(struct page **pagep) -{ - struct page *page = *pagep; - - if (fscrypt_is_bounce_page(page)) { - *pagep = fscrypt_pagecache_page(page); - fscrypt_free_bounce_page(page); - } -} - #endif /* _LINUX_FSCRYPT_H */ diff --git a/include/linux/fsl/guts.h b/include/linux/fsl/guts.h index fdb55ca47a4f..8d5ffb985fee 100644 --- a/include/linux/fsl/guts.h +++ b/include/linux/fsl/guts.h @@ -13,6 +13,7 @@ #include <linux/types.h> #include <linux/io.h> +#include <soc/fsl/phy-fsl-lynx.h> /* * Global Utility Registers. @@ -91,9 +92,15 @@ struct ccsr_guts { u32 iovselsr; /* 0x.00c0 - I/O voltage select status register Called 'elbcvselcr' on 86xx SOCs */ u8 res0c4[0x100 - 0xc4]; - u32 rcwsr[16]; /* 0x.0100 - Reset Control Word Status registers - There are 16 registers */ - u8 res140[0x224 - 0x140]; + /* 0x.0100 - read-only Reset Configuration Word Status registers in + * CCSR, or write-only Reset Configuration Word Control registers in + * DCSR. In both cases there are 32 registers. + */ + union { + u32 rcwsr[32]; + u32 rcwcr[32]; + }; + u8 res180[0x224 - 0x180]; u32 iodelay1; /* 0x.0224 - IO delay control register 1 */ u32 iodelay2; /* 0x.0228 - IO delay control register 2 */ u8 res22c[0x604 - 0x22c]; @@ -131,6 +138,11 @@ struct ccsr_guts { u32 srds2cr1; /* 0x.0f44 - SerDes2 Control Register 0 */ } __attribute__ ((packed)); +int fsl_guts_lane_validate(int serdes_idx, int lane, + enum lynx_lane_mode lane_mode); +int fsl_guts_lane_set_mode(int serdes_idx, int lane, + enum lynx_lane_mode lane_mode); + /* Alternate function signal multiplex control */ #define MPC85xx_PMUXCR_QE(x) (0x8000 >> (x)) diff --git a/include/linux/fsl/mc.h b/include/linux/fsl/mc.h index 897d6211c163..c25f0f7e6dd4 100644 --- a/include/linux/fsl/mc.h +++ b/include/linux/fsl/mc.h @@ -11,7 +11,7 @@ #define _FSL_MC_H_ #include <linux/device.h> -#include <linux/mod_devicetable.h> +#include <linux/device-id/fsl_mc.h> #include <linux/interrupt.h> #include <uapi/linux/fsl_mc.h> @@ -178,9 +178,6 @@ struct fsl_mc_obj_desc { * @regions: pointer to array of MMIO region entries * @irqs: pointer to array of pointers to interrupts allocated to this device * @resource: generic resource associated with this MC object device, if any. - * @driver_override: driver name to force a match; do not set directly, - * because core frees it; use driver_set_override() to - * set or clear it. * * Generic device object for MC object devices that are "attached" to a * MC bus. @@ -214,7 +211,6 @@ struct fsl_mc_device { struct fsl_mc_device_irq **irqs; struct fsl_mc_resource *resource; struct device_link *consumer_link; - const char *driver_override; }; #define to_fsl_mc_device(_dev) \ @@ -361,9 +357,11 @@ int mc_send_command(struct fsl_mc_io *mc_io, struct fsl_mc_command *cmd); #ifdef CONFIG_FSL_MC_BUS #define dev_is_fsl_mc(_dev) ((_dev)->bus == &fsl_mc_bus_type) +u32 fsl_mc_get_msi_id(struct device *dev); #else /* If fsl-mc bus is not present device cannot belong to fsl-mc bus */ #define dev_is_fsl_mc(_dev) (0) +#define fsl_mc_get_msi_id(_dev) (0) #endif /* Macro to check if a device is a container device */ @@ -423,10 +421,6 @@ int __must_check fsl_mc_object_allocate(struct fsl_mc_device *mc_dev, void fsl_mc_object_free(struct fsl_mc_device *mc_adev); -struct irq_domain *fsl_mc_msi_create_irq_domain(struct fwnode_handle *fwnode, - struct msi_domain_info *info, - struct irq_domain *parent); - int __must_check fsl_mc_allocate_irqs(struct fsl_mc_device *mc_dev); void fsl_mc_free_irqs(struct fsl_mc_device *mc_dev); diff --git a/include/linux/fsl/netc_global.h b/include/linux/fsl/netc_global.h index fdecca8c90f0..5b8ff528d369 100644 --- a/include/linux/fsl/netc_global.h +++ b/include/linux/fsl/netc_global.h @@ -5,6 +5,7 @@ #define __NETC_GLOBAL_H #include <linux/io.h> +#include <linux/io-64-nonatomic-lo-hi.h> static inline u32 netc_read(void __iomem *reg) { @@ -16,4 +17,9 @@ static inline void netc_write(void __iomem *reg, u32 val) iowrite32(val, reg); } +static inline u64 netc_read64(void __iomem *reg) +{ + return ioread64(reg); +} + #endif diff --git a/include/linux/fsl/ntmp.h b/include/linux/fsl/ntmp.h index 916dc4fe7de3..764ef2892608 100644 --- a/include/linux/fsl/ntmp.h +++ b/include/linux/fsl/ntmp.h @@ -1,11 +1,15 @@ /* SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) */ -/* Copyright 2025 NXP */ +/* Copyright 2025-2026 NXP */ #ifndef __NETC_NTMP_H #define __NETC_NTMP_H +#include <linux/bitmap.h> #include <linux/bitops.h> #include <linux/if_ether.h> +#define NTMP_NULL_ENTRY_ID 0xffffffffU +#define IPFT_MAX_PLD_LEN 24 + struct maft_keye_data { u8 mac_addr[ETH_ALEN]; __le16 resv; @@ -29,6 +33,18 @@ struct netc_cbdr_regs { struct netc_tbl_vers { u8 maft_ver; u8 rsst_ver; + u8 fdbt_ver; + u8 vft_ver; + u8 bpt_ver; + u8 ipft_ver; + u8 ett_ver; + u8 ect_ver; +}; + +struct netc_swcbd { + void *buf; + dma_addr_t dma; + size_t size; }; struct netc_cbdr { @@ -44,9 +60,10 @@ struct netc_cbdr { void *addr_base_align; dma_addr_t dma_base; dma_addr_t dma_base_align; + struct netc_swcbd *swcbd; /* Serialize the order of command BD ring */ - spinlock_t ring_lock; + struct mutex ring_lock; }; struct ntmp_user { @@ -54,6 +71,14 @@ struct ntmp_user { struct device *dev; struct netc_cbdr *ring; struct netc_tbl_vers tbl; + + /* NTMP table bitmaps for resource management */ + u32 ett_bitmap_size; + u32 ect_bitmap_size; + u16 maft_num_entries; + unsigned long *ett_gid_bitmap; /* only valid for switch */ + unsigned long *ect_gid_bitmap; /* only valid for switch */ + unsigned long *maft_eid_bitmap; /* only valid for ENETC */ }; struct maft_entry_data { @@ -61,10 +86,205 @@ struct maft_entry_data { struct maft_cfge_data cfge; }; +struct ipft_pld_byte { + u8 data; + u8 mask; +}; + +struct ipft_keye_data { + __le16 precedence; + __le16 resv0[3]; + __le16 frm_attr_flags; +#define IPFT_FAF_OVLAN BIT(2) +#define IPFT_FAF_IVLAN BIT(3) +#define IPFT_FAF_IP_HDR BIT(7) +#define IPFT_FAF_IP_VER6 BIT(8) +#define IPFT_FAF_L4_CODE GENMASK(11, 10) +#define IPFT_FAF_TCP_HDR 1 +#define IPFT_FAF_UDP_HDR 2 +#define IPFT_FAF_SCTP_HDR 3 +#define IPFT_FAF_WOL_MAGIC BIT(12) + __le16 frm_attr_flags_mask; + __le16 dscp; +#define IPFT_DSCP GENMASK(5, 0) +#define IPFT_DSCP_MASK GENMASK(11, 6) +#define IPFT_DSCP_MASK_ALL 0x3f + __le16 src_port; /* This field is reserved for ENETC */ +#define IPFT_SRC_PORT GENMASK(4, 0) +#define IPFT_SRC_PORT_MASK GENMASK(9, 5) +#define IPFT_SRC_PORT_MASK_ALL 0x1f + __be16 outer_vlan_tci; + __be16 outer_vlan_tci_mask; + u8 dmac[ETH_ALEN]; + u8 dmac_mask[ETH_ALEN]; + u8 smac[ETH_ALEN]; + u8 smac_mask[ETH_ALEN]; + __be16 inner_vlan_tci; + __be16 inner_vlan_tci_mask; + __be16 ethertype; + __be16 ethertype_mask; + u8 ip_protocol; + u8 ip_protocol_mask; + __le16 resv1[7]; + __be32 ip_src[4]; + __le32 resv2[2]; + __be32 ip_src_mask[4]; + __be16 l4_src_port; + __be16 l4_src_port_mask; + __le32 resv3; + __be32 ip_dst[4]; + __le32 resv4[2]; + __be32 ip_dst_mask[4]; + __be16 l4_dst_port; + __be16 l4_dst_port_mask; + __le32 resv5; + struct ipft_pld_byte byte[IPFT_MAX_PLD_LEN]; +}; + +struct ipft_cfge_data { + __le32 cfg; +#define IPFT_IPV GENMASK(3, 0) +#define IPFT_OIPV BIT(4) +#define IPFT_DR GENMASK(6, 5) +#define IPFT_ODR BIT(7) +#define IPFT_FLTFA GENMASK(10, 8) +#define IPFT_FLTFA_DISCARD 0 +#define IPFT_FLTFA_PERMIT 1 +/* Redirect is only for switch */ +#define IPFT_FLTFA_REDIRECT 2 +#define IPFT_IMIRE BIT(11) +#define IPFT_WOLTE BIT(12) +#define IPFT_FLTA GENMASK(14, 13) +#define IPFT_FLTA_RP 1 +#define IPFT_FLTA_IS 2 +#define IPFT_FLTA_SI_BITMAP 3 +#define IPFT_RPR GENMASK(16, 15) +#define IPFT_CTD BIT(17) +#define IPFT_HR GENMASK(21, 18) +#define IPFT_TIMECAPE BIT(22) +#define IPFT_RRT BIT(23) +#define IPFT_BL2F BIT(24) +#define IPFT_EVMEID GENMASK(31, 28) + __le32 flta_tgt; +}; + +struct ipft_entry_data { + u32 entry_id; /* hardware assigns entry ID */ + struct ipft_keye_data keye; + struct ipft_cfge_data cfge; +}; + +struct fdbt_keye_data { + u8 mac_addr[ETH_ALEN]; /* big-endian */ + __le16 resv0; + __le16 fid; +#define FDBT_FID GENMASK(11, 0) + __le16 resv1; +}; + +struct fdbt_cfge_data { + __le32 port_bitmap; +#define FDBT_PORT_BITMAP GENMASK(23, 0) + __le32 cfg; +#define FDBT_OETEID GENMASK(1, 0) +#define FDBT_EPORT GENMASK(6, 2) +#define FDBT_IMIRE BIT(7) +#define FDBT_CTD GENMASK(10, 9) +#define FDBT_DYNAMIC BIT(11) +#define FDBT_TIMECAPE BIT(12) + __le32 et_eid; +}; + +struct fdbt_entry_data { + u32 entry_id; + struct fdbt_keye_data keye; + struct fdbt_cfge_data cfge; + u8 acte; +#define FDBT_ACT_CNT GENMASK(6, 0) +#define FDBT_ACT_FLAG BIT(7) +}; + +struct vft_cfge_data { + __le32 bitmap_stg; +#define VFT_PORT_MEMBERSHIP GENMASK(23, 0) +#define VFT_STG_ID_MASK GENMASK(27, 24) +#define VFT_STG_ID(g) FIELD_PREP(VFT_STG_ID_MASK, (g)) + __le16 fid; +#define VFT_FID GENMASK(11, 0) + __le16 cfg; +#define VFT_MLO GENMASK(2, 0) +#define VFT_MFO GENMASK(4, 3) +#define VFT_IPMFE BIT(6) +#define VFT_IPMFLE BIT(7) +#define VFT_PGA BIT(8) +#define VFT_SFDA BIT(10) +#define VFT_OSFDA BIT(11) +#define VFT_FDBAFSS BIT(12) + __le32 eta_port_bitmap; +#define VFT_ETA_PORT_BITMAP GENMASK(23, 0) + __le32 et_eid; +}; + +struct ett_cfge_data { + __le16 efm_cfg; +#define ETT_EFM_MODE GENMASK(1, 0) +#define ETT_ESQA GENMASK(5, 4) +#define ETT_ECA GENMASK(8, 6) +#define ETT_ECA_INC 1 +#define ETT_EFM_LEN_CHANGE GENMASK(15, 9) +#define ETT_FRM_LEN_DEL_VLAN 0x7c +#define ETT_FRM_LEN_DEL_RTAG 0x7a +#define ETT_FRM_LEN_DEL_VLAN_RTAG 0x76 + __le16 efm_data_len; +#define ETT_EFM_DATA_LEN GENMASK(10, 0) + __le32 efm_eid; + __le32 ec_eid; + __le32 esqa_tgt_eid; +}; + +struct bpt_bpse_data { + __le32 amount_used; + __le32 amount_used_hwm; + u8 bpd_fc_state; +#define BPT_FC_STATE BIT(0) +#define BPT_BPD BIT(1) +} __packed; + +struct bpt_cfge_data { + u8 fccfg_sbpen; +#define BPT_FC_CFG GENMASK(2, 1) +#define BPT_FC_CFG_EN_BPFC 1 + u8 pfc_vector; + __le16 max_thresh; + __le16 fc_on_thresh; + __le16 fc_off_thresh; + __le16 sbp_thresh; + __le16 resv; + __le32 sbp_eid; + __le32 fc_ports; +}; + +union ntmp_fmt_eid { + __le32 index; +#define FMTEID_INDEX GENMASK(12, 0) + __le32 vuda_sqta; +#define FMTEID_VUDA GENMASK(1, 0) +#define FMTEID_VUDA_DEL_OTAG 2 +#define FMTEID_SQTA GENMASK(4, 2) +#define FMTEID_SQTA_DEL 2 +#define FMTEID_VUDA_SQTA BIT(13) + __le32 vara_vid; +#define FMTEID_VID GENMASK(11, 0) +#define FMTEID_VARA GENMASK(13, 12) +#define FMTEID_VARA_VID BIT(14) +}; + #if IS_ENABLED(CONFIG_NXP_NETC_LIB) int ntmp_init_cbdr(struct netc_cbdr *cbdr, struct device *dev, const struct netc_cbdr_regs *regs); void ntmp_free_cbdr(struct netc_cbdr *cbdr); +u32 ntmp_lookup_free_eid(unsigned long *bitmap, u32 size); +void ntmp_clear_eid_bitmap(unsigned long *bitmap, u32 entry_id); /* NTMP APIs */ int ntmp_maft_add_entry(struct ntmp_user *user, u32 entry_id, @@ -76,6 +296,34 @@ int ntmp_rsst_update_entry(struct ntmp_user *user, const u32 *table, int count); int ntmp_rsst_query_entry(struct ntmp_user *user, u32 *table, int count); +int ntmp_ipft_add_entry(struct ntmp_user *user, + struct ipft_entry_data *entry); +int ntmp_ipft_delete_entry(struct ntmp_user *user, u32 entry_id); +int ntmp_fdbt_add_entry(struct ntmp_user *user, u32 *entry_id, + const struct fdbt_keye_data *keye, + const struct fdbt_cfge_data *cfge); +int ntmp_fdbt_update_entry(struct ntmp_user *user, u32 entry_id, + const struct fdbt_cfge_data *cfge); +int ntmp_fdbt_delete_entry(struct ntmp_user *user, u32 entry_id); +int ntmp_fdbt_search_port_entry(struct ntmp_user *user, int port, + u32 *resume_entry_id, + struct fdbt_entry_data *entry); +int ntmp_fdbt_update_activity_element(struct ntmp_user *user); +int ntmp_fdbt_delete_ageing_entries(struct ntmp_user *user, u8 act_cnt); +int ntmp_fdbt_delete_port_dynamic_entries(struct ntmp_user *user, int port); +int ntmp_vft_add_entry(struct ntmp_user *user, u16 vid, + const struct vft_cfge_data *cfge); +int ntmp_vft_update_entry(struct ntmp_user *user, u16 vid, + const struct vft_cfge_data *cfge); +int ntmp_vft_delete_entry(struct ntmp_user *user, u16 vid); +int ntmp_ett_add_entry(struct ntmp_user *user, u32 entry_id, + const struct ett_cfge_data *cfge); +int ntmp_ett_update_entry(struct ntmp_user *user, u32 entry_id, + const struct ett_cfge_data *cfge); +int ntmp_ett_delete_entry(struct ntmp_user *user, u32 entry_id); +int ntmp_ect_update_entry(struct ntmp_user *user, u32 entry_id); +int ntmp_bpt_update_entry(struct ntmp_user *user, u32 entry_id, + const struct bpt_cfge_data *cfge); #else static inline int ntmp_init_cbdr(struct netc_cbdr *cbdr, struct device *dev, const struct netc_cbdr_regs *regs) diff --git a/include/linux/fsnotify.h b/include/linux/fsnotify.h index 079c18bcdbde..bda798bc67bc 100644 --- a/include/linux/fsnotify.h +++ b/include/linux/fsnotify.h @@ -257,6 +257,10 @@ static inline void fsnotify_move(struct inode *old_dir, struct inode *new_dir, __u32 new_dir_mask = FS_MOVED_TO; __u32 rename_mask = FS_RENAME; const struct qstr *new_name = &moved->d_name; + struct fsnotify_rename_data rd = { + .moved = moved, + .target = target, + }; if (isdir) { old_dir_mask |= FS_ISDIR; @@ -265,12 +269,12 @@ static inline void fsnotify_move(struct inode *old_dir, struct inode *new_dir, } /* Event with information about both old and new parent+name */ - fsnotify_name(rename_mask, moved, FSNOTIFY_EVENT_DENTRY, + fsnotify_name(rename_mask, &rd, FSNOTIFY_EVENT_RENAME, old_dir, old_name, 0); fsnotify_name(old_dir_mask, source, FSNOTIFY_EVENT_INODE, old_dir, old_name, fs_cookie); - fsnotify_name(new_dir_mask, source, FSNOTIFY_EVENT_INODE, + fsnotify_name(new_dir_mask, &rd, FSNOTIFY_EVENT_RENAME, new_dir, new_name, fs_cookie); if (target) diff --git a/include/linux/fsnotify_backend.h b/include/linux/fsnotify_backend.h index 95985400d3d8..618eed4d6d72 100644 --- a/include/linux/fsnotify_backend.h +++ b/include/linux/fsnotify_backend.h @@ -311,6 +311,7 @@ enum fsnotify_data_type { FSNOTIFY_EVENT_DENTRY, FSNOTIFY_EVENT_MNT, FSNOTIFY_EVENT_ERROR, + FSNOTIFY_EVENT_RENAME, }; struct fs_error_report { @@ -335,6 +336,11 @@ struct fsnotify_mnt { u64 mnt_id; }; +struct fsnotify_rename_data { + struct dentry *moved; /* the dentry that was renamed */ + struct inode *target; /* inode overwritten by rename, or NULL */ +}; + static inline struct inode *fsnotify_data_inode(const void *data, int data_type) { switch (data_type) { @@ -348,6 +354,8 @@ static inline struct inode *fsnotify_data_inode(const void *data, int data_type) return d_inode(file_range_path(data)->dentry); case FSNOTIFY_EVENT_ERROR: return ((struct fs_error_report *)data)->inode; + case FSNOTIFY_EVENT_RENAME: + return d_inode(((const struct fsnotify_rename_data *)data)->moved); default: return NULL; } @@ -363,6 +371,8 @@ static inline struct dentry *fsnotify_data_dentry(const void *data, int data_typ return ((const struct path *)data)->dentry; case FSNOTIFY_EVENT_FILE_RANGE: return file_range_path(data)->dentry; + case FSNOTIFY_EVENT_RENAME: + return ((struct fsnotify_rename_data *)data)->moved; default: return NULL; } @@ -395,6 +405,8 @@ static inline struct super_block *fsnotify_data_sb(const void *data, return file_range_path(data)->dentry->d_sb; case FSNOTIFY_EVENT_ERROR: return ((struct fs_error_report *) data)->sb; + case FSNOTIFY_EVENT_RENAME: + return ((const struct fsnotify_rename_data *)data)->moved->d_sb; default: return NULL; } @@ -430,6 +442,14 @@ static inline struct fs_error_report *fsnotify_data_error_report( } } +static inline struct inode *fsnotify_data_rename_target(const void *data, + int data_type) +{ + if (data_type == FSNOTIFY_EVENT_RENAME) + return ((const struct fsnotify_rename_data *)data)->target; + return NULL; +} + static inline const struct file_range *fsnotify_data_file_range( const void *data, int data_type) @@ -915,8 +935,10 @@ extern void fsnotify_clear_marks_by_group(struct fsnotify_group *group, unsigned int obj_type); extern void fsnotify_get_mark(struct fsnotify_mark *mark); extern void fsnotify_put_mark(struct fsnotify_mark *mark); +struct fsnotify_mark *fsnotify_next_mark(struct fsnotify_mark *mark); extern void fsnotify_finish_user_wait(struct fsnotify_iter_info *iter_info); extern bool fsnotify_prepare_user_wait(struct fsnotify_iter_info *iter_info); +extern void fsnotify_modify_mark_mask(struct fsnotify_mark *mark, u32 set, u32 clear); static inline void fsnotify_init_event(struct fsnotify_event *event) { diff --git a/include/linux/fsverity.h b/include/linux/fsverity.h index a8f9aa75b792..6c467ded9751 100644 --- a/include/linux/fsverity.h +++ b/include/linux/fsverity.h @@ -201,6 +201,8 @@ bool fsverity_verify_blocks(struct fsverity_info *vi, struct folio *folio, size_t len, size_t offset); void fsverity_verify_bio(struct fsverity_info *vi, struct bio *bio); void fsverity_enqueue_verify_work(struct work_struct *work); +void fsverity_fill_zerohash(struct folio *folio, size_t offset, size_t len, + struct fsverity_info *vi); #else /* !CONFIG_FS_VERITY */ @@ -281,6 +283,12 @@ static inline void fsverity_enqueue_verify_work(struct work_struct *work) WARN_ON_ONCE(1); } +static inline void fsverity_fill_zerohash(struct folio *folio, size_t offset, + size_t len, struct fsverity_info *vi) +{ + WARN_ON_ONCE(1); +} + #endif /* !CONFIG_FS_VERITY */ static inline bool fsverity_verify_folio(struct fsverity_info *vi, diff --git a/include/linux/ftrace.h b/include/linux/ftrace.h index c242fe49af4c..bd76a16a63af 100644 --- a/include/linux/ftrace.h +++ b/include/linux/ftrace.h @@ -31,7 +31,7 @@ #define ARCH_SUPPORTS_FTRACE_OPS 0 #endif -#ifdef CONFIG_TRACING +#ifdef CONFIG_TRACER_SNAPSHOT extern void ftrace_boot_snapshot(void); #else static inline void ftrace_boot_snapshot(void) { } @@ -415,6 +415,8 @@ struct ftrace_hash *alloc_ftrace_hash(int size_bits); void free_ftrace_hash(struct ftrace_hash *hash); struct ftrace_func_entry *add_ftrace_hash_entry_direct(struct ftrace_hash *hash, unsigned long ip, unsigned long direct); +void add_ftrace_hash_entry(struct ftrace_hash *hash, struct ftrace_func_entry *entry); +void ftrace_hash_remove(struct ftrace_hash *hash); /* The hash used to know what functions callbacks trace */ struct ftrace_ops_hash { @@ -551,6 +553,8 @@ int update_ftrace_direct_mod(struct ftrace_ops *ops, struct ftrace_hash *hash, b void ftrace_stub_direct_tramp(void); +unsigned long ftrace_hash_count(struct ftrace_hash *hash); + #else struct ftrace_ops; static inline unsigned long ftrace_find_rec_direct(unsigned long ip) @@ -590,6 +594,11 @@ static inline int update_ftrace_direct_mod(struct ftrace_ops *ops, struct ftrace return -ENODEV; } +static inline unsigned long ftrace_hash_count(struct ftrace_hash *hash) +{ + return 0; +} + /* * This must be implemented by the architecture. * It is the way the ftrace direct_ops helper, when called @@ -857,8 +866,9 @@ unsigned long ftrace_get_addr_new(struct dyn_ftrace *rec); unsigned long ftrace_get_addr_curr(struct dyn_ftrace *rec); extern ftrace_func_t ftrace_trace_function; +struct trace_array; -int ftrace_regex_open(struct ftrace_ops *ops, int flag, +int ftrace_regex_open(struct trace_array *tr, struct ftrace_ops *ops, int flag, struct inode *inode, struct file *file); ssize_t ftrace_filter_write(struct file *file, const char __user *ubuf, size_t cnt, loff_t *ppos); @@ -1068,7 +1078,7 @@ static inline unsigned long ftrace_location(unsigned long ip) * have them defined when ftrace is not enabled, but these * functions may still be called. Use a macro instead of inline. */ -#define ftrace_regex_open(ops, flag, inod, file) ({ -ENODEV; }) +#define ftrace_regex_open(tr, ops, flag, inode, file) ({ -ENODEV; }) #define ftrace_set_early_filter(ops, buf, enable) do { } while (0) #define ftrace_set_filter_ip(ops, ip, remove, reset) ({ -ENODEV; }) #define ftrace_set_filter_ips(ops, ips, cnt, remove, reset) ({ -ENODEV; }) diff --git a/include/linux/futex.h b/include/linux/futex.h index 9e9750f04980..18ed18d5cbc1 100644 --- a/include/linux/futex.h +++ b/include/linux/futex.h @@ -64,19 +64,15 @@ enum { static inline void futex_init_task(struct task_struct *tsk) { - tsk->robust_list = NULL; -#ifdef CONFIG_COMPAT - tsk->compat_robust_list = NULL; -#endif - INIT_LIST_HEAD(&tsk->pi_state_list); - tsk->pi_state_cache = NULL; - tsk->futex_state = FUTEX_STATE_OK; - mutex_init(&tsk->futex_exit_mutex); + memset(&tsk->futex, 0, sizeof(tsk->futex)); + INIT_LIST_HEAD(&tsk->futex.pi_state_list); + tsk->futex.state = FUTEX_STATE_OK; + mutex_init(&tsk->futex.exit_mutex); } void futex_exit_recursive(struct task_struct *tsk); -void futex_exit_release(struct task_struct *tsk); -void futex_exec_release(struct task_struct *tsk); +void futex_exit_exec_release(struct task_struct *tsk); +void futex_exec_done(struct task_struct *tsk); long do_futex(u32 __user *uaddr, int op, u32 val, ktime_t *timeout, u32 __user *uaddr2, u32 val2, u32 val3); @@ -85,22 +81,18 @@ int futex_hash_prctl(unsigned long arg2, unsigned long arg3, unsigned long arg4) #ifdef CONFIG_FUTEX_PRIVATE_HASH int futex_hash_allocate_default(void); void futex_hash_free(struct mm_struct *mm); -int futex_mm_init(struct mm_struct *mm); - -#else /* !CONFIG_FUTEX_PRIVATE_HASH */ +#else /* CONFIG_FUTEX_PRIVATE_HASH */ static inline int futex_hash_allocate_default(void) { return 0; } static inline int futex_hash_free(struct mm_struct *mm) { return 0; } -static inline int futex_mm_init(struct mm_struct *mm) { return 0; } -#endif /* CONFIG_FUTEX_PRIVATE_HASH */ +#endif /* !CONFIG_FUTEX_PRIVATE_HASH */ -#else /* !CONFIG_FUTEX */ +#else /* CONFIG_FUTEX */ static inline void futex_init_task(struct task_struct *tsk) { } static inline void futex_exit_recursive(struct task_struct *tsk) { } -static inline void futex_exit_release(struct task_struct *tsk) { } -static inline void futex_exec_release(struct task_struct *tsk) { } -static inline long do_futex(u32 __user *uaddr, int op, u32 val, - ktime_t *timeout, u32 __user *uaddr2, - u32 val2, u32 val3) +static inline void futex_exit_exec_release(struct task_struct *tsk) { } +static inline void futex_exec_done(struct task_struct *tsk) { } +static inline long do_futex(u32 __user *uaddr, int op, u32 val, ktime_t *timeout, + u32 __user *uaddr2, u32 val2, u32 val3) { return -EINVAL; } @@ -108,13 +100,63 @@ static inline int futex_hash_prctl(unsigned long arg2, unsigned long arg3, unsig { return -EINVAL; } -static inline int futex_hash_allocate_default(void) +static inline int futex_hash_allocate_default(void) { return 0; } +static inline int futex_hash_free(struct mm_struct *mm) { return 0; } +#endif /* !CONFIG_FUTEX */ + +#ifdef CONFIG_FUTEX_ROBUST_UNLOCK +#include <asm/futex_robust.h> + +void futex_reset_cs_ranges(struct futex_mm_data *fd); +void __futex_fixup_robust_unlock(struct pt_regs *regs, struct futex_unlock_cs_range *csr); + +static inline bool futex_within_robust_unlock(struct pt_regs *regs, + struct futex_unlock_cs_range *csr) { - return 0; + unsigned long ip = instruction_pointer(regs); + + return ip >= csr->start_ip && ip < csr->start_ip + csr->len; } -static inline int futex_hash_free(struct mm_struct *mm) { return 0; } -static inline int futex_mm_init(struct mm_struct *mm) { return 0; } -#endif +static inline void futex_fixup_robust_unlock(struct pt_regs *regs) +{ + struct futex_unlock_cs_range *csr; + + /* + * Avoid dereferencing current->mm if not returning from interrupt. + * current->rseq.event is going to be used subsequently, so bringing the + * cache line in is not a big deal. + */ + if (!current->rseq.event.user_irq) + return; + + csr = current->mm->futex.unlock.cs_ranges; + + /* The loop is optimized out for !COMPAT */ + for (int r = 0; r < FUTEX_ROBUST_MAX_CS_RANGES; r++, csr++) { + if (unlikely(futex_within_robust_unlock(regs, csr))) { + __futex_fixup_robust_unlock(regs, csr); + return; + } + } +} + +static inline void futex_set_vdso_cs_range(struct futex_mm_data *fd, unsigned int idx, + unsigned long start, unsigned long end, bool sz32) +{ + fd->unlock.cs_ranges[idx].start_ip = start; + fd->unlock.cs_ranges[idx].len = end - start; + fd->unlock.cs_ranges[idx].pop_size32 = sz32; +} +#else /* CONFIG_FUTEX_ROBUST_UNLOCK */ +static inline void futex_fixup_robust_unlock(struct pt_regs *regs) { } +#endif /* !CONFIG_FUTEX_ROBUST_UNLOCK */ + +#if defined(CONFIG_FUTEX_PRIVATE_HASH) || defined(CONFIG_FUTEX_ROBUST_UNLOCK) +void futex_mm_init(struct mm_struct *mm); +#else +static inline void futex_mm_init(struct mm_struct *mm) { } #endif + +#endif /* _LINUX_FUTEX_H */ diff --git a/include/linux/futex_types.h b/include/linux/futex_types.h new file mode 100644 index 000000000000..d320c0571f0c --- /dev/null +++ b/include/linux/futex_types.h @@ -0,0 +1,98 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LINUX_FUTEX_TYPES_H +#define _LINUX_FUTEX_TYPES_H + +#ifdef CONFIG_FUTEX +#include <linux/compiler_types.h> +#include <linux/mutex_types.h> +#include <linux/types.h> + +struct compat_robust_list_head; +struct futex_pi_state; +struct robust_list_head; + +/** + * struct futex_sched_data - Futex related per task data + * @robust_list: User space registered robust list pointer + * @compat_robust_list: User space registered robust list pointer for compat tasks + * @pi_state_list: List head for Priority Inheritance (PI) state management + * @pi_state_cache: Pointer to cache one PI state object per task + * @exit_mutex: Mutex for serializing exit + * @state: Futex handling state to handle exit races correctly + */ +struct futex_sched_data { + struct robust_list_head __user *robust_list; +#ifdef CONFIG_COMPAT + struct compat_robust_list_head __user *compat_robust_list; +#endif + struct list_head pi_state_list; + struct futex_pi_state *pi_state_cache; + struct mutex exit_mutex; + unsigned int state; +}; + +#ifdef CONFIG_FUTEX_PRIVATE_HASH +/** + * struct futex_mm_phash - Futex private hash related per MM data + * @lock: Mutex to protect the private hash operations + * @hash: RCU managed pointer to the private hash + * @hash_new: Pointer to a newly allocated private hash + * @batches: Batch state for RCU synchronization + * @rcu: RCU head for call_rcu() + * @atomic: Aggregate value for @hash_ref + * @ref: Per CPU reference counter for a private hash + */ +struct futex_mm_phash { + struct mutex lock; + struct futex_private_hash __rcu *hash; + struct futex_private_hash *hash_new; + unsigned long batches; + struct rcu_head rcu; + atomic_long_t atomic; + unsigned int __percpu *ref; +}; +#else /* CONFIG_FUTEX_ROBUST_UNLOCK */ +struct futex_mm_phash { }; +#endif /* !CONFIG_FUTEX_ROBUST_UNLOCK */ + +#ifdef CONFIG_FUTEX_ROBUST_UNLOCK +/** + * struct futex_unlock_cs_range - Range for the VDSO unlock critical section + * @start_ip: The start IP of the robust futex unlock critical section (inclusive) + * @len: The length of the robust futex unlock critical section + * @pop_size32: Pending OP pointer size indicator. 0 == 64-bit, 1 == 32-bit + */ +struct futex_unlock_cs_range { + unsigned long start_ip; + unsigned int len; + unsigned int pop_size32; +}; + +#define FUTEX_ROBUST_MAX_CS_RANGES (1 + IS_ENABLED(CONFIG_COMPAT)) + +/** + * struct futex_unlock_cs_ranges - Futex unlock VSDO critical sections + * @cs_ranges: Array of critical section ranges + */ +struct futex_unlock_cs_ranges { + struct futex_unlock_cs_range cs_ranges[FUTEX_ROBUST_MAX_CS_RANGES]; +}; +#else /* CONFIG_FUTEX_ROBUST_UNLOCK */ +struct futex_unlock_cs_ranges { }; +#endif /* !CONFIG_FUTEX_ROBUST_UNLOCK */ + +/** + * struct futex_mm_data - Futex related per MM data + * @phash: Futex private hash related data + * @unlock: Futex unlock VDSO critical sections + */ +struct futex_mm_data { + struct futex_mm_phash phash; + struct futex_unlock_cs_ranges unlock; +}; +#else /* CONFIG_FUTEX */ +struct futex_sched_data { }; +struct futex_mm_data { }; +#endif /* !CONFIG_FUTEX */ + +#endif /* _LINUX_FUTEX_TYPES_H */ diff --git a/include/linux/fwnode.h b/include/linux/fwnode.h index 097be89487bf..a9dcaf7e7076 100644 --- a/include/linux/fwnode.h +++ b/include/linux/fwnode.h @@ -15,6 +15,7 @@ #define _LINUX_FWNODE_H_ #include <linux/bits.h> +#include <linux/bitops.h> #include <linux/err.h> #include <linux/list.h> #include <linux/types.h> @@ -42,12 +43,12 @@ struct device; * suppliers. Only enforce ordering with suppliers that have * drivers. */ -#define FWNODE_FLAG_LINKS_ADDED BIT(0) -#define FWNODE_FLAG_NOT_DEVICE BIT(1) -#define FWNODE_FLAG_INITIALIZED BIT(2) -#define FWNODE_FLAG_NEEDS_CHILD_BOUND_ON_ADD BIT(3) -#define FWNODE_FLAG_BEST_EFFORT BIT(4) -#define FWNODE_FLAG_VISITED BIT(5) +#define FWNODE_FLAG_LINKS_ADDED 0 +#define FWNODE_FLAG_NOT_DEVICE 1 +#define FWNODE_FLAG_INITIALIZED 2 +#define FWNODE_FLAG_NEEDS_CHILD_BOUND_ON_ADD 3 +#define FWNODE_FLAG_BEST_EFFORT 4 +#define FWNODE_FLAG_VISITED 5 struct fwnode_handle { struct fwnode_handle *secondary; @@ -57,7 +58,7 @@ struct fwnode_handle { struct device *dev; struct list_head suppliers; struct list_head consumers; - u8 flags; + unsigned long flags; }; /* @@ -116,6 +117,8 @@ struct fwnode_reference_args { * @put: Put a reference to an fwnode. * @device_is_available: Return true if the device is available. * @device_get_match_data: Return the device driver match data. + * @device_dma_supported: Return true if DMA is supported. + * @device_get_dma_attr: Return the device DMA attribute. * @property_present: Return true if a property is present. * @property_read_bool: Return a boolean property value. * @property_read_int_array: Read an array of integer properties. Return zero on @@ -133,6 +136,8 @@ struct fwnode_reference_args { * endpoint node. * @graph_get_port_parent: Return the parent node of a port node. * @graph_parse_endpoint: Parse endpoint for port and endpoint id. + * @iomap: Map the I/O memory of a given index for a fwnode. + * @irq_get: Get the IRQ of a given index for a fwnode. * @add_links: Create fwnode links to all the suppliers of the fwnode. Return * zero on success, a negative error code otherwise. */ @@ -207,9 +212,36 @@ struct fwnode_operations { static inline void fwnode_init(struct fwnode_handle *fwnode, const struct fwnode_operations *ops) { + fwnode->secondary = NULL; fwnode->ops = ops; + fwnode->dev = NULL; INIT_LIST_HEAD(&fwnode->consumers); INIT_LIST_HEAD(&fwnode->suppliers); + fwnode->flags = 0; +} + +static inline void fwnode_set_flag(struct fwnode_handle *fwnode, + unsigned int bit) +{ + set_bit(bit, &fwnode->flags); +} + +static inline void fwnode_clear_flag(struct fwnode_handle *fwnode, + unsigned int bit) +{ + clear_bit(bit, &fwnode->flags); +} + +static inline void fwnode_assign_flag(struct fwnode_handle *fwnode, + unsigned int bit, bool value) +{ + assign_bit(bit, &fwnode->flags, value); +} + +static inline bool fwnode_test_flag(struct fwnode_handle *fwnode, + unsigned int bit) +{ + return test_bit(bit, &fwnode->flags); } static inline void fwnode_dev_initialized(struct fwnode_handle *fwnode, @@ -218,16 +250,14 @@ static inline void fwnode_dev_initialized(struct fwnode_handle *fwnode, if (IS_ERR_OR_NULL(fwnode)) return; - if (initialized) - fwnode->flags |= FWNODE_FLAG_INITIALIZED; - else - fwnode->flags &= ~FWNODE_FLAG_INITIALIZED; + fwnode_assign_flag(fwnode, FWNODE_FLAG_INITIALIZED, initialized); } int fwnode_link_add(struct fwnode_handle *con, struct fwnode_handle *sup, u8 flags); void fwnode_links_purge(struct fwnode_handle *fwnode); void fw_devlink_purge_absent_suppliers(struct fwnode_handle *fwnode); +void fw_devlink_refresh_fwnode(struct fwnode_handle *fwnode); bool fw_devlink_is_strict(void); #endif diff --git a/include/linux/generic_pt/common.h b/include/linux/generic_pt/common.h index 6a9a1acb5aad..07ef1c8341a4 100644 --- a/include/linux/generic_pt/common.h +++ b/include/linux/generic_pt/common.h @@ -134,6 +134,11 @@ enum pt_features { * significant amount of page table. */ PT_FEAT_FLUSH_RANGE_NO_GAPS, + /** + * @PT_FEAT_DETAILED_GATHER: Fill in the struct iommu_iotlb_gather pt + * sub structure with information about which levels were changed. + */ + PT_FEAT_DETAILED_GATHER, /* private: */ PT_FEAT_FMT_START, }; @@ -175,6 +180,26 @@ enum { PT_FEAT_VTDSS_FORCE_WRITEABLE, }; +struct pt_riscv_32 { + struct pt_common common; +}; + +struct pt_riscv_64 { + struct pt_common common; +}; + +enum { + /* + * Support the 64k contiguous page size following the Svnapot extension. + */ + PT_FEAT_RISCV_SVNAPOT_64K = PT_FEAT_FMT_START, + /* + * Support Svpbmt extension: encode page-based memory type (PBMT) in PTEs. + */ + PT_FEAT_RISCV_SVPBMT, + +}; + struct pt_x86_64 { struct pt_common common; }; diff --git a/include/linux/generic_pt/iommu.h b/include/linux/generic_pt/iommu.h index 9eefbb74efd0..dd0edd02a48a 100644 --- a/include/linux/generic_pt/iommu.h +++ b/include/linux/generic_pt/iommu.h @@ -66,6 +66,13 @@ struct pt_iommu { struct device *iommu_device; }; +static inline struct pt_iommu *iommupt_from_domain(struct iommu_domain *domain) +{ + if (!IS_ENABLED(CONFIG_IOMMU_PT) || !domain->is_iommupt) + return NULL; + return container_of(domain, struct pt_iommu, domain); +} + /** * struct pt_iommu_info - Details about the IOMMU page table * @@ -81,6 +88,56 @@ struct pt_iommu_info { struct pt_iommu_ops { /** + * @map_range: Install translation for an IOVA range + * @iommu_table: Table to manipulate + * @iova: IO virtual address to start + * @paddr: Physical/Output address to start + * @len: Length of the range starting from @iova + * @prot: A bitmap of IOMMU_READ/WRITE/CACHE/NOEXEC/MMIO + * @gfp: GFP flags for any memory allocations + * + * The range starting at IOVA will have paddr installed into it. The + * rage is automatically segmented into optimally sized table entries, + * and can have any valid alignment. + * + * On error the caller will probably want to invoke unmap on the range + * from iova up to the amount indicated by @mapped to return the table + * back to an unchanged state. + * + * Context: The caller must hold a write range lock that includes + * the whole range. + * + * Returns: -ERRNO on failure, 0 on success. The number of bytes of VA + * that were mapped are added to @mapped, @mapped is not zerod first. + */ + int (*map_range)(struct pt_iommu *iommu_table, dma_addr_t iova, + phys_addr_t paddr, dma_addr_t len, unsigned int prot, + gfp_t gfp, size_t *mapped); + + /** + * @unmap_range: Make a range of IOVA empty/not present + * @iommu_table: Table to manipulate + * @iova: IO virtual address to start + * @len: Length of the range starting from @iova + * @iotlb_gather: Gather struct that must be flushed on return + * + * unmap_range() will remove a translation created by map_range(). It + * cannot subdivide a mapping created by map_range(), so it should be + * called with IOVA ranges that match those passed to map_pages. The + * IOVA range can aggregate contiguous map_range() calls so long as no + * individual range is split. + * + * Context: The caller must hold a write range lock that includes + * the whole range. + * + * Returns: Number of bytes of VA unmapped. iova + res will be the + * point unmapping stopped. + */ + size_t (*unmap_range)(struct pt_iommu *iommu_table, dma_addr_t iova, + dma_addr_t len, + struct iommu_iotlb_gather *iotlb_gather); + + /** * @set_dirty: Make the iova write dirty * @iommu_table: Table to manipulate * @iova: IO virtual address to start @@ -194,14 +251,6 @@ struct pt_iommu_cfg { #define IOMMU_PROTOTYPES(fmt) \ phys_addr_t pt_iommu_##fmt##_iova_to_phys(struct iommu_domain *domain, \ dma_addr_t iova); \ - int pt_iommu_##fmt##_map_pages(struct iommu_domain *domain, \ - unsigned long iova, phys_addr_t paddr, \ - size_t pgsize, size_t pgcount, \ - int prot, gfp_t gfp, size_t *mapped); \ - size_t pt_iommu_##fmt##_unmap_pages( \ - struct iommu_domain *domain, unsigned long iova, \ - size_t pgsize, size_t pgcount, \ - struct iommu_iotlb_gather *iotlb_gather); \ int pt_iommu_##fmt##_read_and_clear_dirty( \ struct iommu_domain *domain, unsigned long iova, size_t size, \ unsigned long flags, struct iommu_dirty_bitmap *dirty); \ @@ -222,9 +271,7 @@ struct pt_iommu_cfg { * iommu_pt */ #define IOMMU_PT_DOMAIN_OPS(fmt) \ - .iova_to_phys = &pt_iommu_##fmt##_iova_to_phys, \ - .map_pages = &pt_iommu_##fmt##_map_pages, \ - .unmap_pages = &pt_iommu_##fmt##_unmap_pages + .iova_to_phys = &pt_iommu_##fmt##_iova_to_phys #define IOMMU_PT_DIRTY_OPS(fmt) \ .read_and_clear_dirty = &pt_iommu_##fmt##_read_and_clear_dirty @@ -275,6 +322,17 @@ struct pt_iommu_vtdss_hw_info { IOMMU_FORMAT(vtdss, vtdss_pt); +struct pt_iommu_riscv_64_cfg { + struct pt_iommu_cfg common; +}; + +struct pt_iommu_riscv_64_hw_info { + u64 ppn; + u8 fsc_iosatp_mode; +}; + +IOMMU_FORMAT(riscv_64, riscv_64pt); + struct pt_iommu_x86_64_cfg { struct pt_iommu_cfg common; /* 4 is a 57 bit 5 level table */ diff --git a/include/linux/genl_magic_func.h b/include/linux/genl_magic_func.h deleted file mode 100644 index d4da060b7532..000000000000 --- a/include/linux/genl_magic_func.h +++ /dev/null @@ -1,408 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef GENL_MAGIC_FUNC_H -#define GENL_MAGIC_FUNC_H - -#include <linux/args.h> -#include <linux/build_bug.h> -#include <linux/genl_magic_struct.h> - -/* - * Magic: declare tla policy {{{1 - * Magic: declare nested policies - * {{{2 - */ -#undef GENL_mc_group -#define GENL_mc_group(group) - -#undef GENL_notification -#define GENL_notification(op_name, op_num, mcast_group, tla_list) - -#undef GENL_op -#define GENL_op(op_name, op_num, handler, tla_list) - -#undef GENL_struct -#define GENL_struct(tag_name, tag_number, s_name, s_fields) \ - [tag_name] = { .type = NLA_NESTED }, - -static struct nla_policy CONCATENATE(GENL_MAGIC_FAMILY, _tla_nl_policy)[] = { -#include GENL_MAGIC_INCLUDE_FILE -}; - -#undef GENL_struct -#define GENL_struct(tag_name, tag_number, s_name, s_fields) \ -static struct nla_policy s_name ## _nl_policy[] __read_mostly = \ -{ s_fields }; - -#undef __field -#define __field(attr_nr, attr_flag, name, nla_type, _type, __get, \ - __put, __is_signed) \ - [attr_nr] = { .type = nla_type }, - -#undef __array -#define __array(attr_nr, attr_flag, name, nla_type, _type, maxlen, \ - __get, __put, __is_signed) \ - [attr_nr] = { .type = nla_type, \ - .len = maxlen - (nla_type == NLA_NUL_STRING) }, - -#include GENL_MAGIC_INCLUDE_FILE - -#ifndef __KERNEL__ -#ifndef pr_info -#define pr_info(args...) fprintf(stderr, args); -#endif -#endif - -#ifdef GENL_MAGIC_DEBUG -static void dprint_field(const char *dir, int nla_type, - const char *name, void *valp) -{ - __u64 val = valp ? *(__u32 *)valp : 1; - switch (nla_type) { - case NLA_U8: val = (__u8)val; - case NLA_U16: val = (__u16)val; - case NLA_U32: val = (__u32)val; - pr_info("%s attr %s: %d 0x%08x\n", dir, - name, (int)val, (unsigned)val); - break; - case NLA_U64: - val = *(__u64*)valp; - pr_info("%s attr %s: %lld 0x%08llx\n", dir, - name, (long long)val, (unsigned long long)val); - break; - case NLA_FLAG: - if (val) - pr_info("%s attr %s: set\n", dir, name); - break; - } -} - -static void dprint_array(const char *dir, int nla_type, - const char *name, const char *val, unsigned len) -{ - switch (nla_type) { - case NLA_NUL_STRING: - if (len && val[len-1] == '\0') - len--; - pr_info("%s attr %s: [len:%u] '%s'\n", dir, name, len, val); - break; - default: - /* we can always show 4 byte, - * thats what nlattr are aligned to. */ - pr_info("%s attr %s: [len:%u] %02x%02x%02x%02x ...\n", - dir, name, len, val[0], val[1], val[2], val[3]); - } -} - -#define DPRINT_TLA(a, op, b) pr_info("%s %s %s\n", a, op, b); - -/* Name is a member field name of the struct s. - * If s is NULL (only parsing, no copy requested in *_from_attrs()), - * nla is supposed to point to the attribute containing the information - * corresponding to that struct member. */ -#define DPRINT_FIELD(dir, nla_type, name, s, nla) \ - do { \ - if (s) \ - dprint_field(dir, nla_type, #name, &s->name); \ - else if (nla) \ - dprint_field(dir, nla_type, #name, \ - (nla_type == NLA_FLAG) ? NULL \ - : nla_data(nla)); \ - } while (0) - -#define DPRINT_ARRAY(dir, nla_type, name, s, nla) \ - do { \ - if (s) \ - dprint_array(dir, nla_type, #name, \ - s->name, s->name ## _len); \ - else if (nla) \ - dprint_array(dir, nla_type, #name, \ - nla_data(nla), nla_len(nla)); \ - } while (0) -#else -#define DPRINT_TLA(a, op, b) do {} while (0) -#define DPRINT_FIELD(dir, nla_type, name, s, nla) do {} while (0) -#define DPRINT_ARRAY(dir, nla_type, name, s, nla) do {} while (0) -#endif - -/* - * Magic: provide conversion functions {{{1 - * populate struct from attribute table: - * {{{2 - */ - -/* processing of generic netlink messages is serialized. - * use one static buffer for parsing of nested attributes */ -static struct nlattr *nested_attr_tb[128]; - -#undef GENL_struct -#define GENL_struct(tag_name, tag_number, s_name, s_fields) \ -/* *_from_attrs functions are static, but potentially unused */ \ -static int __ ## s_name ## _from_attrs(struct s_name *s, \ - struct genl_info *info, bool exclude_invariants) \ -{ \ - const int maxtype = ARRAY_SIZE(s_name ## _nl_policy)-1; \ - struct nlattr *tla = info->attrs[tag_number]; \ - struct nlattr **ntb = nested_attr_tb; \ - struct nlattr *nla; \ - int err; \ - BUILD_BUG_ON(ARRAY_SIZE(s_name ## _nl_policy) > ARRAY_SIZE(nested_attr_tb)); \ - if (!tla) \ - return -ENOMSG; \ - DPRINT_TLA(#s_name, "<=-", #tag_name); \ - err = drbd_nla_parse_nested(ntb, maxtype, tla, s_name ## _nl_policy); \ - if (err) \ - return err; \ - \ - s_fields \ - return 0; \ -} __attribute__((unused)) \ -static int s_name ## _from_attrs(struct s_name *s, \ - struct genl_info *info) \ -{ \ - return __ ## s_name ## _from_attrs(s, info, false); \ -} __attribute__((unused)) \ -static int s_name ## _from_attrs_for_change(struct s_name *s, \ - struct genl_info *info) \ -{ \ - return __ ## s_name ## _from_attrs(s, info, true); \ -} __attribute__((unused)) \ - -#define __assign(attr_nr, attr_flag, name, nla_type, type, assignment...) \ - nla = ntb[attr_nr]; \ - if (nla) { \ - if (exclude_invariants && !!((attr_flag) & DRBD_F_INVARIANT)) { \ - pr_info("<< must not change invariant attr: %s\n", #name); \ - return -EEXIST; \ - } \ - assignment; \ - } else if (exclude_invariants && !!((attr_flag) & DRBD_F_INVARIANT)) { \ - /* attribute missing from payload, */ \ - /* which was expected */ \ - } else if ((attr_flag) & DRBD_F_REQUIRED) { \ - pr_info("<< missing attr: %s\n", #name); \ - return -ENOMSG; \ - } - -#undef __field -#define __field(attr_nr, attr_flag, name, nla_type, type, __get, __put, \ - __is_signed) \ - __assign(attr_nr, attr_flag, name, nla_type, type, \ - if (s) \ - s->name = __get(nla); \ - DPRINT_FIELD("<<", nla_type, name, s, nla)) - -/* validate_nla() already checked nla_len <= maxlen appropriately. */ -#undef __array -#define __array(attr_nr, attr_flag, name, nla_type, type, maxlen, \ - __get, __put, __is_signed) \ - __assign(attr_nr, attr_flag, name, nla_type, type, \ - if (s) \ - s->name ## _len = \ - __get(s->name, nla, maxlen); \ - DPRINT_ARRAY("<<", nla_type, name, s, nla)) - -#include GENL_MAGIC_INCLUDE_FILE - -#undef GENL_struct -#define GENL_struct(tag_name, tag_number, s_name, s_fields) - -/* - * Magic: define op number to op name mapping {{{1 - * {{{2 - */ -static const char *CONCATENATE(GENL_MAGIC_FAMILY, _genl_cmd_to_str)(__u8 cmd) -{ - switch (cmd) { -#undef GENL_op -#define GENL_op(op_name, op_num, handler, tla_list) \ - case op_num: return #op_name; -#include GENL_MAGIC_INCLUDE_FILE - default: - return "unknown"; - } -} - -#ifdef __KERNEL__ -#include <linux/stringify.h> -/* - * Magic: define genl_ops {{{1 - * {{{2 - */ - -#undef GENL_op -#define GENL_op(op_name, op_num, handler, tla_list) \ -{ \ - handler \ - .cmd = op_name, \ -}, - -#define ZZZ_genl_ops CONCATENATE(GENL_MAGIC_FAMILY, _genl_ops) -static struct genl_ops ZZZ_genl_ops[] __read_mostly = { -#include GENL_MAGIC_INCLUDE_FILE -}; - -#undef GENL_op -#define GENL_op(op_name, op_num, handler, tla_list) - -/* - * Define the genl_family, multicast groups, {{{1 - * and provide register/unregister functions. - * {{{2 - */ -#define ZZZ_genl_family CONCATENATE(GENL_MAGIC_FAMILY, _genl_family) -static struct genl_family ZZZ_genl_family; -/* - * Magic: define multicast groups - * Magic: define multicast group registration helper - */ -#define ZZZ_genl_mcgrps CONCATENATE(GENL_MAGIC_FAMILY, _genl_mcgrps) -static const struct genl_multicast_group ZZZ_genl_mcgrps[] = { -#undef GENL_mc_group -#define GENL_mc_group(group) { .name = #group, }, -#include GENL_MAGIC_INCLUDE_FILE -}; - -enum CONCATENATE(GENL_MAGIC_FAMILY, group_ids) { -#undef GENL_mc_group -#define GENL_mc_group(group) CONCATENATE(GENL_MAGIC_FAMILY, _group_ ## group), -#include GENL_MAGIC_INCLUDE_FILE -}; - -#undef GENL_mc_group -#define GENL_mc_group(group) \ -static int CONCATENATE(GENL_MAGIC_FAMILY, _genl_multicast_ ## group)( \ - struct sk_buff *skb, gfp_t flags) \ -{ \ - unsigned int group_id = \ - CONCATENATE(GENL_MAGIC_FAMILY, _group_ ## group); \ - return genlmsg_multicast(&ZZZ_genl_family, skb, 0, \ - group_id, flags); \ -} - -#include GENL_MAGIC_INCLUDE_FILE - -#undef GENL_mc_group -#define GENL_mc_group(group) - -static struct genl_family ZZZ_genl_family __ro_after_init = { - .name = __stringify(GENL_MAGIC_FAMILY), - .version = GENL_MAGIC_VERSION, -#ifdef GENL_MAGIC_FAMILY_HDRSZ - .hdrsize = NLA_ALIGN(GENL_MAGIC_FAMILY_HDRSZ), -#endif - .maxattr = ARRAY_SIZE(CONCATENATE(GENL_MAGIC_FAMILY, _tla_nl_policy))-1, - .policy = CONCATENATE(GENL_MAGIC_FAMILY, _tla_nl_policy), - .ops = ZZZ_genl_ops, - .n_ops = ARRAY_SIZE(ZZZ_genl_ops), - .mcgrps = ZZZ_genl_mcgrps, - .resv_start_op = 42, /* drbd is currently the only user */ - .n_mcgrps = ARRAY_SIZE(ZZZ_genl_mcgrps), - .module = THIS_MODULE, -}; - -int CONCATENATE(GENL_MAGIC_FAMILY, _genl_register)(void) -{ - return genl_register_family(&ZZZ_genl_family); -} - -void CONCATENATE(GENL_MAGIC_FAMILY, _genl_unregister)(void) -{ - genl_unregister_family(&ZZZ_genl_family); -} - -/* - * Magic: provide conversion functions {{{1 - * populate skb from struct. - * {{{2 - */ - -#undef GENL_op -#define GENL_op(op_name, op_num, handler, tla_list) - -#undef GENL_struct -#define GENL_struct(tag_name, tag_number, s_name, s_fields) \ -static int s_name ## _to_skb(struct sk_buff *skb, struct s_name *s, \ - const bool exclude_sensitive) \ -{ \ - struct nlattr *tla = nla_nest_start(skb, tag_number); \ - if (!tla) \ - goto nla_put_failure; \ - DPRINT_TLA(#s_name, "-=>", #tag_name); \ - s_fields \ - nla_nest_end(skb, tla); \ - return 0; \ - \ -nla_put_failure: \ - if (tla) \ - nla_nest_cancel(skb, tla); \ - return -EMSGSIZE; \ -} \ -static inline int s_name ## _to_priv_skb(struct sk_buff *skb, \ - struct s_name *s) \ -{ \ - return s_name ## _to_skb(skb, s, 0); \ -} \ -static inline int s_name ## _to_unpriv_skb(struct sk_buff *skb, \ - struct s_name *s) \ -{ \ - return s_name ## _to_skb(skb, s, 1); \ -} - - -#undef __field -#define __field(attr_nr, attr_flag, name, nla_type, type, __get, __put, \ - __is_signed) \ - if (!exclude_sensitive || !((attr_flag) & DRBD_F_SENSITIVE)) { \ - DPRINT_FIELD(">>", nla_type, name, s, NULL); \ - if (__put(skb, attr_nr, s->name)) \ - goto nla_put_failure; \ - } - -#undef __array -#define __array(attr_nr, attr_flag, name, nla_type, type, maxlen, \ - __get, __put, __is_signed) \ - if (!exclude_sensitive || !((attr_flag) & DRBD_F_SENSITIVE)) { \ - DPRINT_ARRAY(">>",nla_type, name, s, NULL); \ - if (__put(skb, attr_nr, min_t(int, maxlen, \ - s->name ## _len + (nla_type == NLA_NUL_STRING)),\ - s->name)) \ - goto nla_put_failure; \ - } - -#include GENL_MAGIC_INCLUDE_FILE - - -/* Functions for initializing structs to default values. */ - -#undef __field -#define __field(attr_nr, attr_flag, name, nla_type, type, __get, __put, \ - __is_signed) -#undef __array -#define __array(attr_nr, attr_flag, name, nla_type, type, maxlen, \ - __get, __put, __is_signed) -#undef __u32_field_def -#define __u32_field_def(attr_nr, attr_flag, name, default) \ - x->name = default; -#undef __s32_field_def -#define __s32_field_def(attr_nr, attr_flag, name, default) \ - x->name = default; -#undef __flg_field_def -#define __flg_field_def(attr_nr, attr_flag, name, default) \ - x->name = default; -#undef __str_field_def -#define __str_field_def(attr_nr, attr_flag, name, maxlen) \ - memset(x->name, 0, sizeof(x->name)); \ - x->name ## _len = 0; -#undef GENL_struct -#define GENL_struct(tag_name, tag_number, s_name, s_fields) \ -static void set_ ## s_name ## _defaults(struct s_name *x) __attribute__((unused)); \ -static void set_ ## s_name ## _defaults(struct s_name *x) { \ -s_fields \ -} - -#include GENL_MAGIC_INCLUDE_FILE - -#endif /* __KERNEL__ */ - -/* }}}1 */ -#endif /* GENL_MAGIC_FUNC_H */ diff --git a/include/linux/genl_magic_struct.h b/include/linux/genl_magic_struct.h deleted file mode 100644 index 621b87a87d74..000000000000 --- a/include/linux/genl_magic_struct.h +++ /dev/null @@ -1,283 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef GENL_MAGIC_STRUCT_H -#define GENL_MAGIC_STRUCT_H - -#ifndef GENL_MAGIC_FAMILY -# error "you need to define GENL_MAGIC_FAMILY before inclusion" -#endif - -#ifndef GENL_MAGIC_VERSION -# error "you need to define GENL_MAGIC_VERSION before inclusion" -#endif - -#ifndef GENL_MAGIC_INCLUDE_FILE -# error "you need to define GENL_MAGIC_INCLUDE_FILE before inclusion" -#endif - -#include <linux/args.h> -#include <linux/types.h> -#include <net/genetlink.h> - -extern int CONCATENATE(GENL_MAGIC_FAMILY, _genl_register)(void); -extern void CONCATENATE(GENL_MAGIC_FAMILY, _genl_unregister)(void); - -/* - * Extension of genl attribute validation policies {{{2 - */ - -/* - * @DRBD_GENLA_F_MANDATORY: By default, netlink ignores attributes it does not - * know about. This flag can be set in nlattr->nla_type to indicate that this - * attribute must not be ignored. - * - * We check and remove this flag in drbd_nla_check_mandatory() before - * validating the attribute types and lengths via nla_parse_nested(). - */ -#define DRBD_GENLA_F_MANDATORY (1 << 14) - -/* - * Flags specific to drbd and not visible at the netlink layer, used in - * <struct>_from_attrs and <struct>_to_skb: - * - * @DRBD_F_REQUIRED: Attribute is required; a request without this attribute is - * invalid. - * - * @DRBD_F_SENSITIVE: Attribute includes sensitive information and must not be - * included in unpriviledged get requests or broadcasts. - * - * @DRBD_F_INVARIANT: Attribute is set when an object is initially created, but - * cannot subsequently be changed. - */ -#define DRBD_F_REQUIRED (1 << 0) -#define DRBD_F_SENSITIVE (1 << 1) -#define DRBD_F_INVARIANT (1 << 2) - -#define __nla_type(x) ((__u16)((x) & NLA_TYPE_MASK & ~DRBD_GENLA_F_MANDATORY)) - -/* }}}1 - * MAGIC - * multi-include macro expansion magic starts here - */ - -/* MAGIC helpers {{{2 */ - -static inline int nla_put_u64_0pad(struct sk_buff *skb, int attrtype, u64 value) -{ - return nla_put_64bit(skb, attrtype, sizeof(u64), &value, 0); -} - -/* possible field types */ -#define __flg_field(attr_nr, attr_flag, name) \ - __field(attr_nr, attr_flag, name, NLA_U8, char, \ - nla_get_u8, nla_put_u8, false) -#define __u8_field(attr_nr, attr_flag, name) \ - __field(attr_nr, attr_flag, name, NLA_U8, unsigned char, \ - nla_get_u8, nla_put_u8, false) -#define __u16_field(attr_nr, attr_flag, name) \ - __field(attr_nr, attr_flag, name, NLA_U16, __u16, \ - nla_get_u16, nla_put_u16, false) -#define __u32_field(attr_nr, attr_flag, name) \ - __field(attr_nr, attr_flag, name, NLA_U32, __u32, \ - nla_get_u32, nla_put_u32, false) -#define __s32_field(attr_nr, attr_flag, name) \ - __field(attr_nr, attr_flag, name, NLA_U32, __s32, \ - nla_get_u32, nla_put_u32, true) -#define __u64_field(attr_nr, attr_flag, name) \ - __field(attr_nr, attr_flag, name, NLA_U64, __u64, \ - nla_get_u64, nla_put_u64_0pad, false) -#define __str_field(attr_nr, attr_flag, name, maxlen) \ - __array(attr_nr, attr_flag, name, NLA_NUL_STRING, char, maxlen, \ - nla_strscpy, nla_put, false) -#define __bin_field(attr_nr, attr_flag, name, maxlen) \ - __array(attr_nr, attr_flag, name, NLA_BINARY, char, maxlen, \ - nla_memcpy, nla_put, false) - -/* fields with default values */ -#define __flg_field_def(attr_nr, attr_flag, name, default) \ - __flg_field(attr_nr, attr_flag, name) -#define __u32_field_def(attr_nr, attr_flag, name, default) \ - __u32_field(attr_nr, attr_flag, name) -#define __s32_field_def(attr_nr, attr_flag, name, default) \ - __s32_field(attr_nr, attr_flag, name) -#define __str_field_def(attr_nr, attr_flag, name, maxlen) \ - __str_field(attr_nr, attr_flag, name, maxlen) - -#define GENL_op_init(args...) args -#define GENL_doit(handler) \ - .doit = handler, \ - .flags = GENL_ADMIN_PERM, -#define GENL_dumpit(handler) \ - .dumpit = handler, \ - .flags = GENL_ADMIN_PERM, - -/* }}}1 - * Magic: define the enum symbols for genl_ops - * Magic: define the enum symbols for top level attributes - * Magic: define the enum symbols for nested attributes - * {{{2 - */ - -#undef GENL_struct -#define GENL_struct(tag_name, tag_number, s_name, s_fields) - -#undef GENL_mc_group -#define GENL_mc_group(group) - -#undef GENL_notification -#define GENL_notification(op_name, op_num, mcast_group, tla_list) \ - op_name = op_num, - -#undef GENL_op -#define GENL_op(op_name, op_num, handler, tla_list) \ - op_name = op_num, - -enum { -#include GENL_MAGIC_INCLUDE_FILE -}; - -#undef GENL_notification -#define GENL_notification(op_name, op_num, mcast_group, tla_list) - -#undef GENL_op -#define GENL_op(op_name, op_num, handler, attr_list) - -#undef GENL_struct -#define GENL_struct(tag_name, tag_number, s_name, s_fields) \ - tag_name = tag_number, - -enum { -#include GENL_MAGIC_INCLUDE_FILE -}; - -#undef GENL_struct -#define GENL_struct(tag_name, tag_number, s_name, s_fields) \ -enum { \ - s_fields \ -}; - -#undef __field -#define __field(attr_nr, attr_flag, name, nla_type, type, \ - __get, __put, __is_signed) \ - T_ ## name = (__u16)(attr_nr | ((attr_flag) & DRBD_GENLA_F_MANDATORY)), - -#undef __array -#define __array(attr_nr, attr_flag, name, nla_type, type, \ - maxlen, __get, __put, __is_signed) \ - T_ ## name = (__u16)(attr_nr | ((attr_flag) & DRBD_GENLA_F_MANDATORY)), - -#include GENL_MAGIC_INCLUDE_FILE - -/* }}}1 - * Magic: compile time assert unique numbers for operations - * Magic: -"- unique numbers for top level attributes - * Magic: -"- unique numbers for nested attributes - * {{{2 - */ - -#undef GENL_struct -#define GENL_struct(tag_name, tag_number, s_name, s_fields) - -#undef GENL_op -#define GENL_op(op_name, op_num, handler, attr_list) \ - case op_name: - -#undef GENL_notification -#define GENL_notification(op_name, op_num, mcast_group, tla_list) \ - case op_name: - -static inline void ct_assert_unique_operations(void) -{ - switch (0) { -#include GENL_MAGIC_INCLUDE_FILE - case 0: - ; - } -} - -#undef GENL_op -#define GENL_op(op_name, op_num, handler, attr_list) - -#undef GENL_notification -#define GENL_notification(op_name, op_num, mcast_group, tla_list) - -#undef GENL_struct -#define GENL_struct(tag_name, tag_number, s_name, s_fields) \ - case tag_number: - -static inline void ct_assert_unique_top_level_attributes(void) -{ - switch (0) { -#include GENL_MAGIC_INCLUDE_FILE - case 0: - ; - } -} - -#undef GENL_struct -#define GENL_struct(tag_name, tag_number, s_name, s_fields) \ -static inline void ct_assert_unique_ ## s_name ## _attributes(void) \ -{ \ - switch (0) { \ - s_fields \ - case 0: \ - ; \ - } \ -} - -#undef __field -#define __field(attr_nr, attr_flag, name, nla_type, type, __get, __put, \ - __is_signed) \ - case attr_nr: - -#undef __array -#define __array(attr_nr, attr_flag, name, nla_type, type, maxlen, \ - __get, __put, __is_signed) \ - case attr_nr: - -#include GENL_MAGIC_INCLUDE_FILE - -/* }}}1 - * Magic: declare structs - * struct <name> { - * fields - * }; - * {{{2 - */ - -#undef GENL_struct -#define GENL_struct(tag_name, tag_number, s_name, s_fields) \ -struct s_name { s_fields }; - -#undef __field -#define __field(attr_nr, attr_flag, name, nla_type, type, __get, __put, \ - __is_signed) \ - type name; - -#undef __array -#define __array(attr_nr, attr_flag, name, nla_type, type, maxlen, \ - __get, __put, __is_signed) \ - type name[maxlen]; \ - __u32 name ## _len; - -#include GENL_MAGIC_INCLUDE_FILE - -#undef GENL_struct -#define GENL_struct(tag_name, tag_number, s_name, s_fields) \ -enum { \ - s_fields \ -}; - -#undef __field -#define __field(attr_nr, attr_flag, name, nla_type, type, __get, __put, \ - is_signed) \ - F_ ## name ## _IS_SIGNED = is_signed, - -#undef __array -#define __array(attr_nr, attr_flag, name, nla_type, type, maxlen, \ - __get, __put, is_signed) \ - F_ ## name ## _IS_SIGNED = is_signed, - -#include GENL_MAGIC_INCLUDE_FILE - -/* }}}1 */ -#endif /* GENL_MAGIC_STRUCT_H */ diff --git a/include/linux/gfp.h b/include/linux/gfp.h index 51ef13ed756e..872bc53f32ec 100644 --- a/include/linux/gfp.h +++ b/include/linux/gfp.h @@ -17,28 +17,6 @@ struct mempolicy; #define __default_gfp(a,b,...) b #define default_gfp(...) __default_gfp(,##__VA_ARGS__,GFP_KERNEL) -/* Convert GFP flags to their corresponding migrate type */ -#define GFP_MOVABLE_MASK (__GFP_RECLAIMABLE|__GFP_MOVABLE) -#define GFP_MOVABLE_SHIFT 3 - -static inline int gfp_migratetype(const gfp_t gfp_flags) -{ - VM_WARN_ON((gfp_flags & GFP_MOVABLE_MASK) == GFP_MOVABLE_MASK); - BUILD_BUG_ON((1UL << GFP_MOVABLE_SHIFT) != ___GFP_MOVABLE); - BUILD_BUG_ON((___GFP_MOVABLE >> GFP_MOVABLE_SHIFT) != MIGRATE_MOVABLE); - BUILD_BUG_ON((___GFP_RECLAIMABLE >> GFP_MOVABLE_SHIFT) != MIGRATE_RECLAIMABLE); - BUILD_BUG_ON(((___GFP_MOVABLE | ___GFP_RECLAIMABLE) >> - GFP_MOVABLE_SHIFT) != MIGRATE_HIGHATOMIC); - - if (unlikely(page_group_by_mobility_disabled)) - return MIGRATE_UNMOVABLE; - - /* Group based on mobility */ - return (__force unsigned long)(gfp_flags & GFP_MOVABLE_MASK) >> GFP_MOVABLE_SHIFT; -} -#undef GFP_MOVABLE_MASK -#undef GFP_MOVABLE_SHIFT - static inline bool gfpflags_allow_blocking(const gfp_t gfp_flags) { return !!(gfp_flags & __GFP_DIRECT_RECLAIM); @@ -226,10 +204,6 @@ static inline void arch_free_page(struct page *page, int order) { } static inline void arch_alloc_page(struct page *page, int order) { } #endif -struct page *__alloc_pages_noprof(gfp_t gfp, unsigned int order, int preferred_nid, - nodemask_t *nodemask); -#define __alloc_pages(...) alloc_hooks(__alloc_pages_noprof(__VA_ARGS__)) - struct folio *__folio_alloc_noprof(gfp_t gfp, unsigned int order, int preferred_nid, nodemask_t *nodemask); #define __folio_alloc(...) alloc_hooks(__folio_alloc_noprof(__VA_ARGS__)) @@ -239,6 +213,8 @@ unsigned long alloc_pages_bulk_noprof(gfp_t gfp, int preferred_nid, struct page **page_array); #define __alloc_pages_bulk(...) alloc_hooks(alloc_pages_bulk_noprof(__VA_ARGS__)) +void free_pages_bulk(struct page **page_array, unsigned long nr_pages); + unsigned long alloc_pages_bulk_mempolicy_noprof(gfp_t gfp, unsigned long nr_pages, struct page **page_array); @@ -276,25 +252,9 @@ static inline void warn_if_node_offline(int this_node, gfp_t gfp_mask) dump_stack(); } -/* - * Allocate pages, preferring the node given as nid. The node must be valid and - * online. For more general interface, see alloc_pages_node(). - */ -static inline struct page * -__alloc_pages_node_noprof(int nid, gfp_t gfp_mask, unsigned int order) -{ - VM_BUG_ON(nid < 0 || nid >= MAX_NUMNODES); - warn_if_node_offline(nid, gfp_mask); - - return __alloc_pages_noprof(gfp_mask, order, nid, NULL); -} - -#define __alloc_pages_node(...) alloc_hooks(__alloc_pages_node_noprof(__VA_ARGS__)) - static inline struct folio *__folio_alloc_node_noprof(gfp_t gfp, unsigned int order, int nid) { - VM_BUG_ON(nid < 0 || nid >= MAX_NUMNODES); warn_if_node_offline(nid, gfp); return __folio_alloc_noprof(gfp, order, nid, NULL); @@ -307,14 +267,7 @@ struct folio *__folio_alloc_node_noprof(gfp_t gfp, unsigned int order, int nid) * prefer the current CPU's closest node. Otherwise node must be valid and * online. */ -static inline struct page *alloc_pages_node_noprof(int nid, gfp_t gfp_mask, - unsigned int order) -{ - if (nid == NUMA_NO_NODE) - nid = numa_mem_id(); - - return __alloc_pages_node_noprof(nid, gfp_mask, order); -} +struct page *alloc_pages_node_noprof(int nid, gfp_t gfp_mask, unsigned int order); #define alloc_pages_node(...) alloc_hooks(alloc_pages_node_noprof(__VA_ARGS__)) @@ -393,10 +346,6 @@ extern void free_pages(unsigned long addr, unsigned int order); #define __free_page(page) __free_pages((page), 0) #define free_page(addr) free_pages((addr), 0) -void page_alloc_init_cpuhp(void); -bool decay_pcp_high(struct zone *zone, struct per_cpu_pages *pcp); -void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp); -void drain_all_pages(struct zone *zone); void drain_local_pages(struct zone *zone); void page_alloc_init_late(void); @@ -467,6 +416,8 @@ void free_contig_frozen_range(unsigned long pfn, unsigned long nr_pages); void free_contig_range(unsigned long pfn, unsigned long nr_pages); #endif +void __free_contig_range(unsigned long pfn, unsigned long nr_pages); + DEFINE_FREE(free_page, void *, free_page((unsigned long)_T)) #endif /* __LINUX_GFP_H */ diff --git a/include/linux/gfp_types.h b/include/linux/gfp_types.h index 6c75df30a281..190191411009 100644 --- a/include/linux/gfp_types.h +++ b/include/linux/gfp_types.h @@ -55,7 +55,6 @@ enum { #ifdef CONFIG_LOCKDEP ___GFP_NOLOCKDEP_BIT, #endif - ___GFP_NO_OBJ_EXT_BIT, ___GFP_LAST_BIT }; @@ -96,7 +95,6 @@ enum { #else #define ___GFP_NOLOCKDEP 0 #endif -#define ___GFP_NO_OBJ_EXT BIT(___GFP_NO_OBJ_EXT_BIT) /* * Physical address zone modifiers (see linux/mmzone.h - low four bits) @@ -136,18 +134,14 @@ enum { * %__GFP_THISNODE forces the allocation to be satisfied from the requested * node with no fallbacks or placement policy enforcements. * - * %__GFP_ACCOUNT causes the allocation to be accounted to kmemcg. - * - * %__GFP_NO_OBJ_EXT causes slab allocation to have no object extension. - * mark_obj_codetag_empty() should be called upon freeing for objects allocated - * with this flag to indicate that their NULL tags are expected and normal. + * %__GFP_ACCOUNT causes the allocation to be accounted to the active + * cgroup context. */ #define __GFP_RECLAIMABLE ((__force gfp_t)___GFP_RECLAIMABLE) #define __GFP_WRITE ((__force gfp_t)___GFP_WRITE) #define __GFP_HARDWALL ((__force gfp_t)___GFP_HARDWALL) #define __GFP_THISNODE ((__force gfp_t)___GFP_THISNODE) #define __GFP_ACCOUNT ((__force gfp_t)___GFP_ACCOUNT) -#define __GFP_NO_OBJ_EXT ((__force gfp_t)___GFP_NO_OBJ_EXT) /** * DOC: Watermark modifiers @@ -273,17 +267,17 @@ enum { * * %__GFP_ZERO returns a zeroed page on success. * - * %__GFP_ZEROTAGS zeroes memory tags at allocation time if the memory itself - * is being zeroed (either via __GFP_ZERO or via init_on_alloc, provided that - * __GFP_SKIP_ZERO is not set). This flag is intended for optimization: setting - * memory tags at the same time as zeroing memory has minimal additional - * performance impact. + * %__GFP_ZEROTAGS zeroes memory tags at allocation time. Setting memory tags at + * the same time as zeroing memory (e.g., with __GFP_ZERO) has minimal + * additional performance impact. However, __GFP_ZEROTAGS also zeroes the tags + * even if memory is not getting zeroed at allocation time (e.g., + * with init_on_free). * * %__GFP_SKIP_KASAN makes KASAN skip unpoisoning on page allocation. * Used for userspace and vmalloc pages; the latter are unpoisoned by - * kasan_unpoison_vmalloc instead. For userspace pages, results in - * poisoning being skipped as well, see should_skip_kasan_poison for - * details. Only effective in HW_TAGS mode. + * kasan_unpoison_vmalloc instead. If passed to vmalloc, kasan_unpoison_vmalloc + * is skipped too. For userspace pages, results in poisoning being skipped as + * well, see should_skip_kasan_poison for details. Only effective in HW_TAGS mode. */ #define __GFP_NOWARN ((__force gfp_t)___GFP_NOWARN) #define __GFP_COMP ((__force gfp_t)___GFP_COMP) @@ -320,7 +314,7 @@ enum { * %ZONE_NORMAL or a lower zone for direct access but can direct reclaim. * * %GFP_KERNEL_ACCOUNT is the same as GFP_KERNEL, except the allocation is - * accounted to kmemcg. + * accounted to the active cgroup context. * * %GFP_NOWAIT is for kernel allocations that should not stall for direct * reclaim, start physical IO or use any filesystem callback. It is very diff --git a/include/linux/glob.h b/include/linux/glob.h index 861327b33e41..91595e750936 100644 --- a/include/linux/glob.h +++ b/include/linux/glob.h @@ -6,5 +6,6 @@ #include <linux/compiler.h> /* For __pure */ bool __pure glob_match(char const *pat, char const *str); +bool __pure glob_match_len(char const *pat, char const *str, size_t len); #endif /* _LINUX_GLOB_H */ diff --git a/include/linux/goldfish.h b/include/linux/goldfish.h index bcc17f95b906..98a4719c6776 100644 --- a/include/linux/goldfish.h +++ b/include/linux/goldfish.h @@ -2,8 +2,6 @@ #ifndef __LINUX_GOLDFISH_H #define __LINUX_GOLDFISH_H -#include <linux/kernel.h> -#include <linux/types.h> #include <linux/io.h> /* Helpers for Goldfish virtual platform */ @@ -15,26 +13,4 @@ #define gf_iowrite32 iowrite32 #endif -static inline void gf_write_ptr(const void *ptr, void __iomem *portl, - void __iomem *porth) -{ - const unsigned long addr = (unsigned long)ptr; - - gf_iowrite32(lower_32_bits(addr), portl); -#ifdef CONFIG_64BIT - gf_iowrite32(upper_32_bits(addr), porth); -#endif -} - -static inline void gf_write_dma_addr(const dma_addr_t addr, - void __iomem *portl, - void __iomem *porth) -{ - gf_iowrite32(lower_32_bits(addr), portl); -#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT - gf_iowrite32(upper_32_bits(addr), porth); -#endif -} - - #endif /* __LINUX_GOLDFISH_H */ diff --git a/include/linux/gpio.h b/include/linux/gpio.h index 8f85ddb26429..b0d4942a65de 100644 --- a/include/linux/gpio.h +++ b/include/linux/gpio.h @@ -2,13 +2,11 @@ /* * NOTE: This header *must not* be included. * - * This is the LEGACY GPIO bulk include file, including legacy APIs. It is - * used for GPIO drivers still referencing the global GPIO numberspace, - * and should not be included in new code. - * * If you're implementing a GPIO driver, only include <linux/gpio/driver.h> * If you're implementing a GPIO consumer, only include <linux/gpio/consumer.h> + * If you're using the legacy interfaces, include <linux/gpio/legacy.h> */ + #ifndef __LINUX_GPIO_H #define __LINUX_GPIO_H @@ -18,159 +16,7 @@ #endif #ifdef CONFIG_GPIOLIB_LEGACY - -struct device; - -/* make these flag values available regardless of GPIO kconfig options */ -#define GPIOF_IN ((1 << 0)) -#define GPIOF_OUT_INIT_LOW ((0 << 0) | (0 << 1)) -#define GPIOF_OUT_INIT_HIGH ((0 << 0) | (1 << 1)) - -#ifdef CONFIG_GPIOLIB -/* - * "valid" GPIO numbers are nonnegative and may be passed to - * setup routines like gpio_request(). Only some valid numbers - * can successfully be requested and used. - * - * Invalid GPIO numbers are useful for indicating no-such-GPIO in - * platform data and other tables. - */ -static inline bool gpio_is_valid(int number) -{ - /* only non-negative numbers are valid */ - return number >= 0; -} - -/* - * Platforms may implement their GPIO interface with library code, - * at a small performance cost for non-inlined operations and some - * extra memory (for code and for per-GPIO table entries). - */ - -/* Always use the library code for GPIO management calls, - * or when sleeping may be involved. - */ -int gpio_request(unsigned gpio, const char *label); -void gpio_free(unsigned gpio); - -static inline int gpio_direction_input(unsigned gpio) -{ - return gpiod_direction_input(gpio_to_desc(gpio)); -} -static inline int gpio_direction_output(unsigned gpio, int value) -{ - return gpiod_direction_output_raw(gpio_to_desc(gpio), value); -} - -static inline int gpio_get_value_cansleep(unsigned gpio) -{ - return gpiod_get_raw_value_cansleep(gpio_to_desc(gpio)); -} -static inline void gpio_set_value_cansleep(unsigned gpio, int value) -{ - gpiod_set_raw_value_cansleep(gpio_to_desc(gpio), value); -} - -static inline int gpio_get_value(unsigned gpio) -{ - return gpiod_get_raw_value(gpio_to_desc(gpio)); -} -static inline void gpio_set_value(unsigned gpio, int value) -{ - gpiod_set_raw_value(gpio_to_desc(gpio), value); -} - -static inline int gpio_to_irq(unsigned gpio) -{ - return gpiod_to_irq(gpio_to_desc(gpio)); -} - -int gpio_request_one(unsigned gpio, unsigned long flags, const char *label); - -int devm_gpio_request_one(struct device *dev, unsigned gpio, - unsigned long flags, const char *label); - -#else /* ! CONFIG_GPIOLIB */ - -#include <linux/kernel.h> - -#include <asm/bug.h> -#include <asm/errno.h> - -static inline bool gpio_is_valid(int number) -{ - return false; -} - -static inline int gpio_request(unsigned gpio, const char *label) -{ - return -ENOSYS; -} - -static inline int gpio_request_one(unsigned gpio, - unsigned long flags, const char *label) -{ - return -ENOSYS; -} - -static inline void gpio_free(unsigned gpio) -{ - might_sleep(); - - /* GPIO can never have been requested */ - WARN_ON(1); -} - -static inline int gpio_direction_input(unsigned gpio) -{ - return -ENOSYS; -} - -static inline int gpio_direction_output(unsigned gpio, int value) -{ - return -ENOSYS; -} - -static inline int gpio_get_value(unsigned gpio) -{ - /* GPIO can never have been requested or set as {in,out}put */ - WARN_ON(1); - return 0; -} - -static inline void gpio_set_value(unsigned gpio, int value) -{ - /* GPIO can never have been requested or set as output */ - WARN_ON(1); -} - -static inline int gpio_get_value_cansleep(unsigned gpio) -{ - /* GPIO can never have been requested or set as {in,out}put */ - WARN_ON(1); - return 0; -} - -static inline void gpio_set_value_cansleep(unsigned gpio, int value) -{ - /* GPIO can never have been requested or set as output */ - WARN_ON(1); -} - -static inline int gpio_to_irq(unsigned gpio) -{ - /* GPIO can never have been requested or set as input */ - WARN_ON(1); - return -EINVAL; -} - -static inline int devm_gpio_request_one(struct device *dev, unsigned gpio, - unsigned long flags, const char *label) -{ - WARN_ON(1); - return -EINVAL; -} - -#endif /* ! CONFIG_GPIOLIB */ +#include <linux/gpio/legacy.h> #endif /* CONFIG_GPIOLIB_LEGACY */ + #endif /* __LINUX_GPIO_H */ diff --git a/include/linux/gpio/consumer.h b/include/linux/gpio/consumer.h index 0d8408582918..fceeefd5f893 100644 --- a/include/linux/gpio/consumer.h +++ b/include/linux/gpio/consumer.h @@ -6,6 +6,8 @@ #include <linux/err.h> #include <linux/types.h> +#include "defs.h" + struct acpi_device; struct device; struct fwnode_handle; @@ -109,6 +111,7 @@ void devm_gpiod_unhinge(struct device *dev, struct gpio_desc *desc); void devm_gpiod_put_array(struct device *dev, struct gpio_descs *descs); int gpiod_get_direction(struct gpio_desc *desc); +bool gpiod_is_single_ended(struct gpio_desc *desc); int gpiod_direction_input(struct gpio_desc *desc); int gpiod_direction_output(struct gpio_desc *desc, int value); int gpiod_direction_output_raw(struct gpio_desc *desc, int value); @@ -335,6 +338,10 @@ static inline int gpiod_get_direction(const struct gpio_desc *desc) WARN_ON(desc); return -ENOSYS; } +static inline bool gpiod_is_single_ended(struct gpio_desc *desc) +{ + return false; +} static inline int gpiod_direction_input(struct gpio_desc *desc) { /* GPIO can never have been requested */ @@ -597,6 +604,15 @@ static inline int gpiod_disable_hw_timestamp_ns(struct gpio_desc *desc, #endif /* CONFIG_GPIOLIB && CONFIG_HTE */ static inline +struct gpio_desc *fwnode_gpiod_get(struct fwnode_handle *fwnode, + const char *con_id, + enum gpiod_flags flags, + const char *label) +{ + return fwnode_gpiod_get_index(fwnode, con_id, 0, flags, label); +} + +static inline struct gpio_desc *devm_fwnode_gpiod_get(struct device *dev, struct fwnode_handle *fwnode, const char *con_id, diff --git a/include/linux/gpio/defs.h b/include/linux/gpio/defs.h new file mode 100644 index 000000000000..b69fd7c041b2 --- /dev/null +++ b/include/linux/gpio/defs.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef __LINUX_GPIO_DEFS_H +#define __LINUX_GPIO_DEFS_H + +#define GPIO_LINE_DIRECTION_IN 1 +#define GPIO_LINE_DIRECTION_OUT 0 + +#endif /* __LINUX_GPIO_DEFS_H */ diff --git a/include/linux/gpio/driver.h b/include/linux/gpio/driver.h index fabe2baf7b50..17511434ed07 100644 --- a/include/linux/gpio/driver.h +++ b/include/linux/gpio/driver.h @@ -20,6 +20,8 @@ #include <asm/msi.h> #endif +#include "defs.h" + struct device; struct irq_chip; struct irq_data; @@ -42,9 +44,6 @@ union gpio_irq_fwspec { #endif }; -#define GPIO_LINE_DIRECTION_IN 1 -#define GPIO_LINE_DIRECTION_OUT 0 - /** * struct gpio_irq_chip - GPIO interrupt controller */ @@ -344,11 +343,17 @@ struct gpio_irq_chip { * @direction_output: configures signal "offset" as output, returns 0 on * success or a negative error number. This can be omitted on input-only * or output-only gpio chips. - * @get: returns value for signal "offset", 0=low, 1=high, or negative error + * @get: returns value for signal "offset", 0=low, 1=high, or negative error. + * the low and high values are defined as physical low on the line + * in/out to the connector such as a physical pad, pin or rail. The GPIO + * library has internal logic to handle lines that are active low, such + * as indicated by overstrike or #name in a schematic, and the driver + * should not try to second-guess the logic value of a line. * @get_multiple: reads values for multiple signals defined by "mask" and * stores them in "bits", returns 0 on success or negative error * @set: assigns output value for signal "offset", returns 0 on success or - * negative error value + * negative error value. The output value follows the same semantic + * rules as for @get. * @set_multiple: assigns output values for multiple signals defined by * "mask", returns 0 on success or negative error value * @set_config: optional hook for all kinds of settings. Uses the same diff --git a/include/linux/gpio/generic.h b/include/linux/gpio/generic.h index ff566dc9c3cb..de43c06c83ef 100644 --- a/include/linux/gpio/generic.h +++ b/include/linux/gpio/generic.h @@ -3,9 +3,15 @@ #ifndef __LINUX_GPIO_GENERIC_H #define __LINUX_GPIO_GENERIC_H +#include <linux/bits.h> +#include <linux/bug.h> #include <linux/cleanup.h> -#include <linux/gpio/driver.h> +#include <linux/container_of.h> +#include <linux/errno.h> #include <linux/spinlock.h> +#include <linux/types.h> + +#include <linux/gpio/driver.h> struct device; diff --git a/include/linux/gpio/gpio-nomadik.h b/include/linux/gpio/gpio-nomadik.h index 592a774a53cd..44c47645649c 100644 --- a/include/linux/gpio/gpio-nomadik.h +++ b/include/linux/gpio/gpio-nomadik.h @@ -32,9 +32,6 @@ struct fwnode_handle; #define NMK_GPIO_RWIMSC 0x50 #define NMK_GPIO_FWIMSC 0x54 #define NMK_GPIO_WKS 0x58 -/* These appear in DB8540 and later ASICs */ -#define NMK_GPIO_EDGELEVEL 0x5C -#define NMK_GPIO_LEVEL 0x60 /* Pull up/down values */ enum nmk_gpio_pull { @@ -114,8 +111,7 @@ struct nmk_gpio_chip { } /** - * enum prcm_gpiocr_reg_index - * Used to reference an PRCM GPIOCR register address. + * enum prcm_gpiocr_reg_index - Used to reference a PRCM GPIOCR register address. */ enum prcm_gpiocr_reg_index { PRCM_IDX_GPIOCR1, @@ -123,8 +119,7 @@ enum prcm_gpiocr_reg_index { PRCM_IDX_GPIOCR3 }; /** - * enum prcm_gpiocr_altcx_index - * Used to reference an Other alternate-C function. + * enum prcm_gpiocr_altcx_index - Used to reference an Other alternate-C function. */ enum prcm_gpiocr_altcx_index { PRCM_IDX_GPIOCR_ALTC1, @@ -135,7 +130,7 @@ enum prcm_gpiocr_altcx_index { }; /** - * struct prcm_gpio_altcx - Other alternate-C function + * struct prcm_gpiocr_altcx - Other alternate-C function * @used: other alternate-C function availability * @reg_index: PRCM GPIOCR register index used to control the function * @control_bit: PRCM GPIOCR bit used to control the function @@ -147,7 +142,7 @@ struct prcm_gpiocr_altcx { } __packed; /** - * struct prcm_gpio_altcx_pin_desc - Other alternate-C pin + * struct prcm_gpiocr_altcx_pin_desc - Other alternate-C pin * @pin: The pin number * @altcx: array of other alternate-C[1-4] functions */ @@ -193,7 +188,7 @@ struct nmk_pingroup { * numbering. * @npins: The number of entries in @pins. * @functions: The functions supported on this SoC. - * @nfunction: The number of entries in @functions. + * @nfunctions: The number of entries in @functions. * @groups: An array describing all pin groups the pin SoC supports. * @ngroups: The number of entries in @groups. * @altcx_pins: The pins that support Other alternate-C function on this SoC @@ -238,19 +233,6 @@ nmk_pinctrl_db8500_init(const struct nmk_pinctrl_soc_data **soc) #endif -#ifdef CONFIG_PINCTRL_DB8540 - -void nmk_pinctrl_db8540_init(const struct nmk_pinctrl_soc_data **soc); - -#else - -static inline void -nmk_pinctrl_db8540_init(const struct nmk_pinctrl_soc_data **soc) -{ -} - -#endif - struct platform_device; #ifdef CONFIG_DEBUG_FS diff --git a/include/linux/gpio/legacy.h b/include/linux/gpio/legacy.h new file mode 100644 index 000000000000..557ef635935e --- /dev/null +++ b/include/linux/gpio/legacy.h @@ -0,0 +1,173 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * This is the LEGACY GPIO include file, used only for legacy APIs. + * + * No new code should use this, but instead use the linux/gpio/consumer.h + * interfaces directly. + */ + +#ifndef __LINUX_GPIO_LEGACY_H +#define __LINUX_GPIO_LEGACY_H + +#include <linux/types.h> + +#ifdef CONFIG_GPIOLIB_LEGACY + +struct device; + +/* make these flag values available regardless of GPIO kconfig options */ +#define GPIOF_IN ((1 << 0)) +#define GPIOF_OUT_INIT_LOW ((0 << 0) | (0 << 1)) +#define GPIOF_OUT_INIT_HIGH ((0 << 0) | (1 << 1)) + +#ifdef CONFIG_GPIOLIB + +#include <linux/gpio/consumer.h> + +/* + * "valid" GPIO numbers are nonnegative and may be passed to + * setup routines like gpio_request(). Only some valid numbers + * can successfully be requested and used. + * + * Invalid GPIO numbers are useful for indicating no-such-GPIO in + * platform data and other tables. + */ +static inline bool gpio_is_valid(int number) +{ + /* only non-negative numbers are valid */ + return number >= 0; +} + +/* + * Platforms may implement their GPIO interface with library code, + * at a small performance cost for non-inlined operations and some + * extra memory (for code and for per-GPIO table entries). + */ + +/* Always use the library code for GPIO management calls, + * or when sleeping may be involved. + */ +int gpio_request(unsigned gpio, const char *label); +void gpio_free(unsigned gpio); + +static inline int gpio_direction_input(unsigned gpio) +{ + return gpiod_direction_input(gpio_to_desc(gpio)); +} +static inline int gpio_direction_output(unsigned gpio, int value) +{ + return gpiod_direction_output_raw(gpio_to_desc(gpio), value); +} + +static inline int gpio_get_value_cansleep(unsigned gpio) +{ + return gpiod_get_raw_value_cansleep(gpio_to_desc(gpio)); +} +static inline void gpio_set_value_cansleep(unsigned gpio, int value) +{ + gpiod_set_raw_value_cansleep(gpio_to_desc(gpio), value); +} + +static inline int gpio_get_value(unsigned gpio) +{ + return gpiod_get_raw_value(gpio_to_desc(gpio)); +} +static inline void gpio_set_value(unsigned gpio, int value) +{ + gpiod_set_raw_value(gpio_to_desc(gpio), value); +} + +static inline int gpio_to_irq(unsigned gpio) +{ + return gpiod_to_irq(gpio_to_desc(gpio)); +} + +int gpio_request_one(unsigned gpio, unsigned long flags, const char *label); + +int devm_gpio_request_one(struct device *dev, unsigned gpio, + unsigned long flags, const char *label); + +#else /* ! CONFIG_GPIOLIB */ + +#include <linux/kernel.h> + +#include <asm/bug.h> +#include <asm/errno.h> + +static inline bool gpio_is_valid(int number) +{ + return false; +} + +static inline int gpio_request(unsigned gpio, const char *label) +{ + return -ENOSYS; +} + +static inline int gpio_request_one(unsigned gpio, + unsigned long flags, const char *label) +{ + return -ENOSYS; +} + +static inline void gpio_free(unsigned gpio) +{ + might_sleep(); + + /* GPIO can never have been requested */ + WARN_ON(1); +} + +static inline int gpio_direction_input(unsigned gpio) +{ + return -ENOSYS; +} + +static inline int gpio_direction_output(unsigned gpio, int value) +{ + return -ENOSYS; +} + +static inline int gpio_get_value(unsigned gpio) +{ + /* GPIO can never have been requested or set as {in,out}put */ + WARN_ON(1); + return 0; +} + +static inline void gpio_set_value(unsigned gpio, int value) +{ + /* GPIO can never have been requested or set as output */ + WARN_ON(1); +} + +static inline int gpio_get_value_cansleep(unsigned gpio) +{ + /* GPIO can never have been requested or set as {in,out}put */ + WARN_ON(1); + return 0; +} + +static inline void gpio_set_value_cansleep(unsigned gpio, int value) +{ + /* GPIO can never have been requested or set as output */ + WARN_ON(1); +} + +static inline int gpio_to_irq(unsigned gpio) +{ + /* GPIO can never have been requested or set as input */ + WARN_ON(1); + return -EINVAL; +} + +static inline int devm_gpio_request_one(struct device *dev, unsigned gpio, + unsigned long flags, const char *label) +{ + WARN_ON(1); + return -EINVAL; +} + +#endif /* ! CONFIG_GPIOLIB */ +#endif /* CONFIG_GPIOLIB_LEGACY */ +#endif /* __LINUX_GPIO_LEGAGY_H */ diff --git a/include/linux/gpio/machine.h b/include/linux/gpio/machine.h index 44e5f162973e..1a141056716f 100644 --- a/include/linux/gpio/machine.h +++ b/include/linux/gpio/machine.h @@ -46,23 +46,6 @@ struct gpiod_lookup_table { struct gpiod_lookup table[]; }; -/** - * struct gpiod_hog - GPIO line hog table - * @chip_label: name of the chip the GPIO belongs to - * @chip_hwnum: hardware number (i.e. relative to the chip) of the GPIO - * @line_name: consumer name for the hogged line - * @lflags: bitmask of gpio_lookup_flags GPIO_* values - * @dflags: GPIO flags used to specify the direction and value - */ -struct gpiod_hog { - struct list_head list; - const char *chip_label; - u16 chip_hwnum; - const char *line_name; - unsigned long lflags; - int dflags; -}; - /* * Helper for lookup tables with just one single lookup for a device. */ @@ -71,7 +54,7 @@ static struct gpiod_lookup_table _name = { \ .dev_id = _dev_id, \ .table = { \ GPIO_LOOKUP(_key, _chip_hwnum, _con_id, _flags), \ - {}, \ + { } \ }, \ } @@ -95,24 +78,10 @@ static struct gpiod_lookup_table _name = { \ .flags = _flags, \ } -/* - * Simple definition of a single GPIO hog in an array. - */ -#define GPIO_HOG(_chip_label, _chip_hwnum, _line_name, _lflags, _dflags) \ -(struct gpiod_hog) { \ - .chip_label = _chip_label, \ - .chip_hwnum = _chip_hwnum, \ - .line_name = _line_name, \ - .lflags = _lflags, \ - .dflags = _dflags, \ -} - #ifdef CONFIG_GPIOLIB void gpiod_add_lookup_table(struct gpiod_lookup_table *table); void gpiod_add_lookup_tables(struct gpiod_lookup_table **tables, size_t n); void gpiod_remove_lookup_table(struct gpiod_lookup_table *table); -void gpiod_add_hogs(struct gpiod_hog *hogs); -void gpiod_remove_hogs(struct gpiod_hog *hogs); #else /* ! CONFIG_GPIOLIB */ static inline void gpiod_add_lookup_table(struct gpiod_lookup_table *table) {} @@ -120,8 +89,6 @@ static inline void gpiod_add_lookup_tables(struct gpiod_lookup_table **tables, size_t n) {} static inline void gpiod_remove_lookup_table(struct gpiod_lookup_table *table) {} -static inline void gpiod_add_hogs(struct gpiod_hog *hogs) {} -static inline void gpiod_remove_hogs(struct gpiod_hog *hogs) {} #endif /* CONFIG_GPIOLIB */ #endif /* __LINUX_GPIO_MACHINE_H */ diff --git a/include/linux/gpio/regmap.h b/include/linux/gpio/regmap.h index 12d154732ca9..6f52c140e50d 100644 --- a/include/linux/gpio/regmap.h +++ b/include/linux/gpio/regmap.h @@ -3,6 +3,8 @@ #ifndef _LINUX_GPIO_REGMAP_H #define _LINUX_GPIO_REGMAP_H +#include <linux/types.h> + struct device; struct fwnode_handle; struct gpio_regmap; @@ -14,6 +16,32 @@ struct regmap; #define GPIO_REGMAP_ADDR(addr) ((addr) ? : GPIO_REGMAP_ADDR_ZERO) /** + * enum gpio_regmap_operation - Operation type for gpio_regmap callbacks + * + * Traditionally, the operation type was inferred from the base register. + * However, that approach does not always work — for example, when all control + * bits of a single GPIO reside in the same register. This enum allows the + * callbacks (reg_mask_xlate and value_xlate) to explicitly distinguish between + * operation types. The user is free to choose which method to use. + * + * Read operation: + * @GPIO_REGMAP_GET_OP: Indicates a read operation to get the current GPIO value. + * + * Write operation: + * @GPIO_REGMAP_SET_OP: Indicates a write operation to set the GPIO output value. + * + * Direction operations: + * @GPIO_REGMAP_GET_DIR_OP: Indicates a read operation to get the GPIO direction. + * @GPIO_REGMAP_SET_DIR_OP: Indicates a write operation to set the GPIO direction. + */ +enum gpio_regmap_operation { + GPIO_REGMAP_GET_OP, + GPIO_REGMAP_SET_OP, + GPIO_REGMAP_GET_DIR_OP, + GPIO_REGMAP_SET_DIR_OP, +}; + +/** * struct gpio_regmap_config - Description of a generic regmap gpio_chip. * @parent: The parent device * @regmap: The regmap used to access the registers @@ -34,24 +62,38 @@ struct regmap; * @ngpio_per_reg: (Optional) Number of GPIOs per register * @irq_domain: (Optional) IRQ domain if the controller is * interrupt-capable - * @reg_mask_xlate: (Optional) Translates base address and GPIO - * offset to a register/bitmask pair. If not - * given the default gpio_regmap_simple_xlate() - * is used. + * @fixed_direction_mask: + * (Optional) Bitmap representing the GPIO lines that + * make use of the @fixed_direction_output list to + * enforce direction of the GPIO. If this is NULL + * and @fixed_direction_output is defined, ALL GPIOs + * are assumed to be fixed direction (out or in). * @fixed_direction_output: * (Optional) Bitmap representing the fixed direction of * the GPIO lines. Useful when there are GPIO lines with a * fixed direction mixed together in the same register. - * @drvdata: (Optional) Pointer to driver specific data which is - * not used by gpio-remap but is provided "as is" to the - * driver callback(s). - * @init_valid_mask: (Optional) Routine to initialize @valid_mask, to be used - * if not all GPIOs are valid. * @regmap_irq_chip: (Optional) Pointer on an regmap_irq_chip structure. If * set, a regmap-irq device will be created and the IRQ * domain will be set accordingly. * @regmap_irq_line: (Optional) The IRQ the device uses to signal interrupts. * @regmap_irq_flags: (Optional) The IRQF_ flags to use for the interrupt. + * @reg_mask_xlate: (Optional) Translates base address and GPIO + * offset to a register/bitmask pair. If not + * given the default gpio_regmap_simple_xlate() + * is used. + * @init_valid_mask: (Optional) Routine to initialize @valid_mask, to be used + * if not all GPIOs are valid. + * @value_xlate: (Optional) Routine to translate the register value and + * mask before writing. This allows driver-specific logic + * to append additional bits (like write-enable masks) + * dynamically based on the current operation + * (GPIO_REGMAP_SET_OP and GPIO_REGMAP_SET_DIR_OP). + * @set_config: (Optional) Callback for setting GPIO configuration such + * as debounce, drive strength, or other hardware specific + * settings. + * @drvdata: (Optional) Pointer to driver specific data which is + * not used by gpio-remap but is provided "as is" to the + * driver callback(s). * * The ->reg_mask_xlate translates a given base address and GPIO offset to * register and mask pair. The base address is one of the given register @@ -89,6 +131,7 @@ struct gpio_regmap_config { int reg_stride; int ngpio_per_reg; struct irq_domain *irq_domain; + unsigned long *fixed_direction_mask; unsigned long *fixed_direction_output; #ifdef CONFIG_REGMAP_IRQ @@ -97,14 +140,21 @@ struct gpio_regmap_config { unsigned long regmap_irq_flags; #endif - int (*reg_mask_xlate)(struct gpio_regmap *gpio, unsigned int base, - unsigned int offset, unsigned int *reg, - unsigned int *mask); + int (*reg_mask_xlate)(struct gpio_regmap *gpio, enum gpio_regmap_operation, + unsigned int base, unsigned int offset, + unsigned int *reg, unsigned int *mask); int (*init_valid_mask)(struct gpio_chip *gc, unsigned long *valid_mask, unsigned int ngpios); + int (*value_xlate)(struct gpio_regmap *gpio, enum gpio_regmap_operation, + unsigned int base, unsigned int offset, unsigned int reg, + unsigned int *mask, unsigned int *val); + + int (*set_config)(struct gpio_regmap *gpio, struct gpio_chip *chip, + unsigned int offset, unsigned long config); + void *drvdata; }; @@ -114,4 +164,10 @@ struct gpio_regmap *devm_gpio_regmap_register(struct device *dev, const struct gpio_regmap_config *config); void *gpio_regmap_get_drvdata(struct gpio_regmap *gpio); +int gpio_regmap_reqres_irq(struct gpio_regmap *gpio, unsigned int offset); +void gpio_regmap_relres_irq(struct gpio_regmap *gpio, unsigned int offset); + +void gpio_regmap_enable_irq(struct gpio_regmap *gpio, irq_hw_number_t hwirq); +void gpio_regmap_disable_irq(struct gpio_regmap *gpio, irq_hw_number_t hwirq); + #endif /* _LINUX_GPIO_REGMAP_H */ diff --git a/include/linux/gpio_keys.h b/include/linux/gpio_keys.h index 80fa930b04c6..e8d6dc290efb 100644 --- a/include/linux/gpio_keys.h +++ b/include/linux/gpio_keys.h @@ -25,7 +25,9 @@ struct device; */ struct gpio_keys_button { unsigned int code; +#ifdef CONFIG_GPIOLIB_LEGACY int gpio; +#endif int active_low; const char *desc; unsigned int type; diff --git a/include/linux/gpu_buddy.h b/include/linux/gpu_buddy.h new file mode 100644 index 000000000000..2c36124bb696 --- /dev/null +++ b/include/linux/gpu_buddy.h @@ -0,0 +1,299 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2021 Intel Corporation + */ + +#ifndef __GPU_BUDDY_H__ +#define __GPU_BUDDY_H__ + +#include <linux/bitops.h> +#include <linux/list.h> +#include <linux/slab.h> +#include <linux/sched.h> +#include <linux/rbtree.h> +#include <linux/rbtree_augmented.h> + +/** + * GPU_BUDDY_RANGE_ALLOCATION - Allocate within a specific address range + * + * When set, allocation is restricted to the range [start, end) specified + * in gpu_buddy_alloc_blocks(). Without this flag, start/end are ignored + * and allocation can use any free space. + */ +#define GPU_BUDDY_RANGE_ALLOCATION BIT(0) + +/** + * GPU_BUDDY_TOPDOWN_ALLOCATION - Allocate from top of address space + * + * Allocate starting from high addresses and working down. Useful for + * separating different allocation types (e.g., kernel vs userspace) + * to reduce fragmentation. + */ +#define GPU_BUDDY_TOPDOWN_ALLOCATION BIT(1) + +/** + * GPU_BUDDY_CONTIGUOUS_ALLOCATION - Require physically contiguous blocks + * + * The allocation must be satisfied with a single contiguous block. + * If the requested size cannot be allocated contiguously, the + * allocation fails with -ENOSPC. + */ +#define GPU_BUDDY_CONTIGUOUS_ALLOCATION BIT(2) + +/** + * GPU_BUDDY_CLEAR_ALLOCATION - Prefer pre-cleared (zeroed) memory + * + * Attempt to allocate from the clear tree first. If insufficient clear + * memory is available, falls back to dirty memory. Useful when the + * caller needs zeroed memory and wants to avoid GPU clear operations. + */ +#define GPU_BUDDY_CLEAR_ALLOCATION BIT(3) + +/** + * GPU_BUDDY_CLEARED - Mark returned blocks as cleared + * + * Used with gpu_buddy_free_list() to indicate that the memory being + * freed has been cleared (zeroed). The blocks will be placed in the + * clear tree for future GPU_BUDDY_CLEAR_ALLOCATION requests. + */ +#define GPU_BUDDY_CLEARED BIT(4) + +/** + * GPU_BUDDY_TRIM_DISABLE - Disable automatic block trimming + * + * By default, if an allocation is smaller than the allocated block, + * excess memory is trimmed and returned to the free pool. This flag + * disables trimming, keeping the full power-of-two block size. + */ +#define GPU_BUDDY_TRIM_DISABLE BIT(5) + +enum gpu_buddy_free_tree { + GPU_BUDDY_CLEAR_TREE = 0, + GPU_BUDDY_DIRTY_TREE, + GPU_BUDDY_MAX_FREE_TREES, +}; + +#define for_each_free_tree(tree) \ + for ((tree) = 0; (tree) < GPU_BUDDY_MAX_FREE_TREES; (tree)++) + +/** + * struct gpu_buddy_block - Block within a buddy allocator + * + * Each block in the buddy allocator is represented by this structure. + * Blocks are organized in a binary tree where each parent block can be + * split into two children (left and right buddies). The allocator manages + * blocks at various orders (power-of-2 sizes) from chunk_size up to the + * largest contiguous region. + * + * @private: Private data owned by the allocator user (e.g., driver-specific data) + * @link: List node for user ownership while block is allocated + */ +struct gpu_buddy_block { +/* private: */ + /* + * Header bit layout: + * - Bits 63:12: block offset within the address space + * - Bits 11:10: state (ALLOCATED, FREE, or SPLIT) + * - Bit 9: clear bit (1 if memory is zeroed) + * - Bits 8:6: reserved + * - Bits 5:0: order (log2 of size relative to chunk_size) + */ +#define GPU_BUDDY_HEADER_OFFSET GENMASK_ULL(63, 12) +#define GPU_BUDDY_HEADER_STATE GENMASK_ULL(11, 10) +#define GPU_BUDDY_ALLOCATED (1 << 10) +#define GPU_BUDDY_FREE (2 << 10) +#define GPU_BUDDY_SPLIT (3 << 10) +#define GPU_BUDDY_HEADER_CLEAR GENMASK_ULL(9, 9) +/* Free to be used, if needed in the future */ +#define GPU_BUDDY_HEADER_UNUSED GENMASK_ULL(8, 6) +#define GPU_BUDDY_HEADER_ORDER GENMASK_ULL(5, 0) + u64 header; + + struct gpu_buddy_block *left; + struct gpu_buddy_block *right; + struct gpu_buddy_block *parent; +/* public: */ + void *private; /* owned by creator */ + + /* + * While the block is allocated by the user through gpu_buddy_alloc*, + * the user has ownership of the link, for example to maintain within + * a list, if so desired. As soon as the block is freed with + * gpu_buddy_free* ownership is given back to the mm. + */ + union { +/* private: */ + struct rb_node rb; +/* public: */ + struct list_head link; + }; +/* private: */ + struct list_head tmp_link; + unsigned int subtree_max_alignment; +}; + +/* Order-zero must be at least SZ_4K */ +#define GPU_BUDDY_MAX_ORDER (63 - 12) + +/** + * struct gpu_buddy - GPU binary buddy allocator + * + * The buddy allocator provides efficient power-of-two memory allocation + * with fast allocation and free operations. It is commonly used for GPU + * memory management where allocations can be split into power-of-two + * block sizes. + * + * Locking should be handled by the user; a simple mutex around + * gpu_buddy_alloc_blocks() and gpu_buddy_free_block()/gpu_buddy_free_list() + * should suffice. + * + * @n_roots: Number of root blocks in the roots array. + * @max_order: Maximum block order (log2 of largest block size / chunk_size). + * @chunk_size: Minimum allocation granularity in bytes. Must be at least SZ_4K. + * @size: Total size of the address space managed by this allocator in bytes. + * @avail: Total free space currently available for allocation in bytes. + * @clear_avail: Free space available in the clear tree (zeroed memory) in bytes. + * This is a subset of @avail. + * @lock_dep_map: Annotates gpu_buddy API with a driver provided lock. + */ +struct gpu_buddy { +/* private: */ + /* + * Array of red-black trees for free block management. + * Indexed as free_trees[clear/dirty][order] where: + * - Index 0 (GPU_BUDDY_CLEAR_TREE): blocks with zeroed content + * - Index 1 (GPU_BUDDY_DIRTY_TREE): blocks with unknown content + * Each tree holds free blocks of the corresponding order. + */ + struct rb_root **free_trees; + /* + * Array of root blocks representing the top-level blocks of the + * binary tree(s). Multiple roots exist when the total size is not + * a power of two, with each root being the largest power-of-two + * that fits in the remaining space. + */ + struct gpu_buddy_block **roots; + /* + * Per-order free block scoreboard: free_scoreboard[order] holds the + * number of blocks of that order currently in the free state. + * Incremented in mark_free(), decremented wherever rbtree_remove() is + * called on a free block. + */ + u64 *free_scoreboard; + /* + * Per-order used block scoreboard: used_scoreboard[order] holds the + * number of blocks of that order currently in the allocated state. + * Incremented in mark_allocated(), decremented in mark_free() (guarded + * by gpu_buddy_block_is_allocated()) and in __gpu_buddy_free() when an + * allocated block is consumed directly during buddy coalescing. + */ + u64 *used_scoreboard; +/* public: */ + unsigned int n_roots; + unsigned int max_order; + u64 chunk_size; + u64 size; + u64 avail; + u64 clear_avail; +#ifdef CONFIG_LOCKDEP + struct lockdep_map *lock_dep_map; +#endif +}; + +#ifdef CONFIG_LOCKDEP +/** + * gpu_buddy_driver_set_lock() - Set the lock protecting accesses to GPU BUDDY + * @mm: Pointer to GPU buddy structure. + * @lock: the lock used to protect the gpu buddy. The locking primitive + * must contain a dep_map field. + * + * Call this to annotate gpu_buddy APIs which access/modify gpu_buddy manager + */ +#define gpu_buddy_driver_set_lock(mm, lock) \ + do { \ + struct gpu_buddy *__mm = (mm); \ + if (!WARN(__mm->lock_dep_map, "GPU BUDDY MM lock should be set only once.")) \ + __mm->lock_dep_map = &(lock)->dep_map; \ + } while (0) +#else +#define gpu_buddy_driver_set_lock(mm, lock) do { (void)(mm); (void)(lock); } while (0) +#endif + +#ifdef CONFIG_LOCKDEP +/** + * gpu_buddy_driver_lock_held() - Assert GPU BUDDY manager lock is held + * @mm: Pointer to the GPU BUDDY structure. + * + * Ensure driver lock is held. + */ +static inline void gpu_buddy_driver_lock_held(struct gpu_buddy *mm) +{ + if (mm->lock_dep_map) + lockdep_assert(lock_is_held_type(mm->lock_dep_map, 0)); +} +#else +static inline void gpu_buddy_driver_lock_held(struct gpu_buddy *mm) +{ +} +#endif + +static inline u64 +gpu_buddy_block_offset(const struct gpu_buddy_block *block) +{ + return block->header & GPU_BUDDY_HEADER_OFFSET; +} + +static inline unsigned int +gpu_buddy_block_order(struct gpu_buddy_block *block) +{ + return block->header & GPU_BUDDY_HEADER_ORDER; +} + +static inline bool +gpu_buddy_block_is_free(struct gpu_buddy_block *block) +{ + return (block->header & GPU_BUDDY_HEADER_STATE) == GPU_BUDDY_FREE; +} + +static inline bool +gpu_buddy_block_is_clear(struct gpu_buddy_block *block) +{ + return block->header & GPU_BUDDY_HEADER_CLEAR; +} + +static inline u64 +gpu_buddy_block_size(struct gpu_buddy *mm, + struct gpu_buddy_block *block) +{ + return mm->chunk_size << gpu_buddy_block_order(block); +} + +int gpu_buddy_init(struct gpu_buddy *mm, u64 size, u64 chunk_size); + +void gpu_buddy_fini(struct gpu_buddy *mm); + +int gpu_buddy_alloc_blocks(struct gpu_buddy *mm, + u64 start, u64 end, u64 size, + u64 min_page_size, + struct list_head *blocks, + unsigned long flags); + +int gpu_buddy_block_trim(struct gpu_buddy *mm, + u64 *start, + u64 new_size, + struct list_head *blocks); + +void gpu_buddy_reset_clear(struct gpu_buddy *mm, bool is_clear); + +void gpu_buddy_free_block(struct gpu_buddy *mm, struct gpu_buddy_block *block); + +struct gpu_buddy_block *gpu_buddy_allocated_addr_to_block(struct gpu_buddy *mm, u64 addr); + +void gpu_buddy_free_list(struct gpu_buddy *mm, + struct list_head *objects, + unsigned int flags); + +void gpu_buddy_print(struct gpu_buddy *mm); +void gpu_buddy_block_print(struct gpu_buddy *mm, + struct gpu_buddy_block *block); +#endif diff --git a/include/linux/greybus/greybus_id.h b/include/linux/greybus/greybus_id.h index f4c8440093e4..72f330a35569 100644 --- a/include/linux/greybus/greybus_id.h +++ b/include/linux/greybus/greybus_id.h @@ -1,14 +1,12 @@ /* SPDX-License-Identifier: GPL-2.0 */ /* FIXME - * move this to include/linux/mod_devicetable.h when merging + * move this to include/linux/device-id/greybus.h when merging */ #ifndef __LINUX_GREYBUS_ID_H #define __LINUX_GREYBUS_ID_H #include <linux/types.h> -#include <linux/mod_devicetable.h> - struct greybus_bundle_id { __u16 match_flags; diff --git a/include/linux/hardirq.h b/include/linux/hardirq.h index d57cab4d4c06..73b48dd9f135 100644 --- a/include/linux/hardirq.h +++ b/include/linux/hardirq.h @@ -92,6 +92,37 @@ void irq_exit_rcu(void); #define arch_nmi_exit() do { } while (0) #endif +#ifdef CONFIG_HAS_SEPARATE_PREEMPT_RESCHED_BITS +static __always_inline void __preempt_count_nmi_enter(void) +{ + __preempt_count_add(NMI_OFFSET + HARDIRQ_OFFSET); +} + +static __always_inline void __preempt_count_nmi_exit(void) +{ + __preempt_count_sub(NMI_OFFSET + HARDIRQ_OFFSET); +} +#else +DECLARE_PER_CPU(unsigned int, nmi_nesting); + +#define __preempt_count_nmi_enter() \ + do { \ + __preempt_count_add(HARDIRQ_OFFSET); \ + /* Maximum NMI nesting is 15. */ \ + BUG_ON(__this_cpu_read(nmi_nesting) >= 15); \ + __this_cpu_inc(nmi_nesting); \ + preempt_count_set(preempt_count() | NMI_MASK); \ + } while (0) + +#define __preempt_count_nmi_exit() \ + do { \ + __preempt_count_sub(HARDIRQ_OFFSET); \ + if (!__this_cpu_dec_return(nmi_nesting)) \ + preempt_count_set(preempt_count() & ~NMI_MASK); \ + } while (0) + +#endif + /* * NMI vs Tracing * -------------- @@ -102,21 +133,20 @@ void irq_exit_rcu(void); */ /* - * nmi_enter() can nest up to 15 times; see NMI_BITS. + * nmi_enter() can nest - nesting is tracked in a per-CPU counter. */ #define __nmi_enter() \ do { \ lockdep_off(); \ arch_nmi_enter(); \ - BUG_ON(in_nmi() == NMI_MASK); \ - __preempt_count_add(NMI_OFFSET + HARDIRQ_OFFSET); \ + __preempt_count_nmi_enter(); \ } while (0) #define nmi_enter() \ do { \ __nmi_enter(); \ lockdep_hardirq_enter(); \ - ct_nmi_enter(); \ + ct_nmi_enter(); \ instrumentation_begin(); \ ftrace_nmi_enter(); \ instrumentation_end(); \ @@ -125,7 +155,7 @@ void irq_exit_rcu(void); #define __nmi_exit() \ do { \ BUG_ON(!in_nmi()); \ - __preempt_count_sub(NMI_OFFSET + HARDIRQ_OFFSET); \ + __preempt_count_nmi_exit(); \ arch_nmi_exit(); \ lockdep_on(); \ } while (0) diff --git a/include/linux/hdlcdrv.h b/include/linux/hdlcdrv.h deleted file mode 100644 index 5d70c3f98f5b..000000000000 --- a/include/linux/hdlcdrv.h +++ /dev/null @@ -1,276 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * hdlcdrv.h -- HDLC packet radio network driver. - * The Linux soundcard driver for 1200 baud and 9600 baud packet radio - * (C) 1996-1998 by Thomas Sailer, HB9JNX/AE4WA - */ -#ifndef _HDLCDRV_H -#define _HDLCDRV_H - - -#include <linux/netdevice.h> -#include <linux/if.h> -#include <linux/spinlock.h> -#include <uapi/linux/hdlcdrv.h> - -#define HDLCDRV_MAGIC 0x5ac6e778 -#define HDLCDRV_HDLCBUFFER 32 /* should be a power of 2 for speed reasons */ -#define HDLCDRV_BITBUFFER 256 /* should be a power of 2 for speed reasons */ -#undef HDLCDRV_LOOPBACK /* define for HDLC debugging purposes */ -#define HDLCDRV_DEBUG - -/* maximum packet length, excluding CRC */ -#define HDLCDRV_MAXFLEN 400 - - -struct hdlcdrv_hdlcbuffer { - spinlock_t lock; - unsigned rd, wr; - unsigned short buf[HDLCDRV_HDLCBUFFER]; -}; - -#ifdef HDLCDRV_DEBUG -struct hdlcdrv_bitbuffer { - unsigned int rd; - unsigned int wr; - unsigned int shreg; - unsigned char buffer[HDLCDRV_BITBUFFER]; -}; - -static inline void hdlcdrv_add_bitbuffer(struct hdlcdrv_bitbuffer *buf, - unsigned int bit) -{ - unsigned char new; - - new = buf->shreg & 1; - buf->shreg >>= 1; - buf->shreg |= (!!bit) << 7; - if (new) { - buf->buffer[buf->wr] = buf->shreg; - buf->wr = (buf->wr+1) % sizeof(buf->buffer); - buf->shreg = 0x80; - } -} - -static inline void hdlcdrv_add_bitbuffer_word(struct hdlcdrv_bitbuffer *buf, - unsigned int bits) -{ - buf->buffer[buf->wr] = bits & 0xff; - buf->wr = (buf->wr+1) % sizeof(buf->buffer); - buf->buffer[buf->wr] = (bits >> 8) & 0xff; - buf->wr = (buf->wr+1) % sizeof(buf->buffer); - -} -#endif /* HDLCDRV_DEBUG */ - -/* -------------------------------------------------------------------- */ -/* - * Information that need to be kept for each driver. - */ - -struct hdlcdrv_ops { - /* - * first some informations needed by the hdlcdrv routines - */ - const char *drvname; - const char *drvinfo; - /* - * the routines called by the hdlcdrv routines - */ - int (*open)(struct net_device *); - int (*close)(struct net_device *); - int (*ioctl)(struct net_device *, void __user *, - struct hdlcdrv_ioctl *, int); -}; - -struct hdlcdrv_state { - int magic; - int opened; - - const struct hdlcdrv_ops *ops; - - struct { - int bitrate; - } par; - - struct hdlcdrv_pttoutput { - int dma2; - int seriobase; - int pariobase; - int midiiobase; - unsigned int flags; - } ptt_out; - - struct hdlcdrv_channel_params ch_params; - - struct hdlcdrv_hdlcrx { - struct hdlcdrv_hdlcbuffer hbuf; - unsigned long in_hdlc_rx; - /* 0 = sync hunt, != 0 receiving */ - int rx_state; - unsigned int bitstream; - unsigned int bitbuf; - int numbits; - unsigned char dcd; - - int len; - unsigned char *bp; - unsigned char buffer[HDLCDRV_MAXFLEN+2]; - } hdlcrx; - - struct hdlcdrv_hdlctx { - struct hdlcdrv_hdlcbuffer hbuf; - unsigned long in_hdlc_tx; - /* - * 0 = send flags - * 1 = send txtail (flags) - * 2 = send packet - */ - int tx_state; - int numflags; - unsigned int bitstream; - unsigned char ptt; - int calibrate; - int slotcnt; - - unsigned int bitbuf; - int numbits; - - int len; - unsigned char *bp; - unsigned char buffer[HDLCDRV_MAXFLEN+2]; - } hdlctx; - -#ifdef HDLCDRV_DEBUG - struct hdlcdrv_bitbuffer bitbuf_channel; - struct hdlcdrv_bitbuffer bitbuf_hdlc; -#endif /* HDLCDRV_DEBUG */ - - int ptt_keyed; - - /* queued skb for transmission */ - struct sk_buff *skb; -}; - - -/* -------------------------------------------------------------------- */ - -static inline int hdlcdrv_hbuf_full(struct hdlcdrv_hdlcbuffer *hb) -{ - unsigned long flags; - int ret; - - spin_lock_irqsave(&hb->lock, flags); - ret = !((HDLCDRV_HDLCBUFFER - 1 + hb->rd - hb->wr) % HDLCDRV_HDLCBUFFER); - spin_unlock_irqrestore(&hb->lock, flags); - return ret; -} - -/* -------------------------------------------------------------------- */ - -static inline int hdlcdrv_hbuf_empty(struct hdlcdrv_hdlcbuffer *hb) -{ - unsigned long flags; - int ret; - - spin_lock_irqsave(&hb->lock, flags); - ret = (hb->rd == hb->wr); - spin_unlock_irqrestore(&hb->lock, flags); - return ret; -} - -/* -------------------------------------------------------------------- */ - -static inline unsigned short hdlcdrv_hbuf_get(struct hdlcdrv_hdlcbuffer *hb) -{ - unsigned long flags; - unsigned short val; - unsigned newr; - - spin_lock_irqsave(&hb->lock, flags); - if (hb->rd == hb->wr) - val = 0; - else { - newr = (hb->rd+1) % HDLCDRV_HDLCBUFFER; - val = hb->buf[hb->rd]; - hb->rd = newr; - } - spin_unlock_irqrestore(&hb->lock, flags); - return val; -} - -/* -------------------------------------------------------------------- */ - -static inline void hdlcdrv_hbuf_put(struct hdlcdrv_hdlcbuffer *hb, - unsigned short val) -{ - unsigned newp; - unsigned long flags; - - spin_lock_irqsave(&hb->lock, flags); - newp = (hb->wr+1) % HDLCDRV_HDLCBUFFER; - if (newp != hb->rd) { - hb->buf[hb->wr] = val & 0xffff; - hb->wr = newp; - } - spin_unlock_irqrestore(&hb->lock, flags); -} - -/* -------------------------------------------------------------------- */ - -static inline void hdlcdrv_putbits(struct hdlcdrv_state *s, unsigned int bits) -{ - hdlcdrv_hbuf_put(&s->hdlcrx.hbuf, bits); -} - -static inline unsigned int hdlcdrv_getbits(struct hdlcdrv_state *s) -{ - unsigned int ret; - - if (hdlcdrv_hbuf_empty(&s->hdlctx.hbuf)) { - if (s->hdlctx.calibrate > 0) - s->hdlctx.calibrate--; - else - s->hdlctx.ptt = 0; - ret = 0; - } else - ret = hdlcdrv_hbuf_get(&s->hdlctx.hbuf); -#ifdef HDLCDRV_LOOPBACK - hdlcdrv_hbuf_put(&s->hdlcrx.hbuf, ret); -#endif /* HDLCDRV_LOOPBACK */ - return ret; -} - -static inline void hdlcdrv_channelbit(struct hdlcdrv_state *s, unsigned int bit) -{ -#ifdef HDLCDRV_DEBUG - hdlcdrv_add_bitbuffer(&s->bitbuf_channel, bit); -#endif /* HDLCDRV_DEBUG */ -} - -static inline void hdlcdrv_setdcd(struct hdlcdrv_state *s, int dcd) -{ - s->hdlcrx.dcd = !!dcd; -} - -static inline int hdlcdrv_ptt(struct hdlcdrv_state *s) -{ - return s->hdlctx.ptt || (s->hdlctx.calibrate > 0); -} - -/* -------------------------------------------------------------------- */ - -void hdlcdrv_receiver(struct net_device *, struct hdlcdrv_state *); -void hdlcdrv_transmitter(struct net_device *, struct hdlcdrv_state *); -void hdlcdrv_arbitrate(struct net_device *, struct hdlcdrv_state *); -struct net_device *hdlcdrv_register(const struct hdlcdrv_ops *ops, - unsigned int privsize, const char *ifname, - unsigned int baseaddr, unsigned int irq, - unsigned int dma); -void hdlcdrv_unregister(struct net_device *dev); - -/* -------------------------------------------------------------------- */ - - - -#endif /* _HDLCDRV_H */ diff --git a/include/linux/hdmi.h b/include/linux/hdmi.h index 96bda41d9148..8dab78e1f61b 100644 --- a/include/linux/hdmi.h +++ b/include/linux/hdmi.h @@ -50,6 +50,12 @@ enum hdmi_infoframe_type { HDMI_INFOFRAME_TYPE_DRM = 0x87, }; +/* HDMI spec maximum TMDS character rates, in Hz */ +#define HDMI_TMDS_CHAR_RATE_MIN_HZ 25000000 +#define HDMI_1_0_TMDS_CHAR_RATE_MAX_HZ 165000000 +#define HDMI_1_3_TMDS_CHAR_RATE_MAX_HZ 340000000 +#define HDMI_2_0_TMDS_CHAR_RATE_MAX_HZ 600000000 + #define HDMI_IEEE_OUI 0x000c03 #define HDMI_FORUM_IEEE_OUI 0xc45dd8 #define HDMI_INFOFRAME_HEADER_SIZE 4 diff --git a/include/linux/hfs_common.h b/include/linux/hfs_common.h index dadb5e0aa8a3..d6a615e74b26 100644 --- a/include/linux/hfs_common.h +++ b/include/linux/hfs_common.h @@ -166,6 +166,11 @@ struct hfsplus_attr_unistr { hfsplus_unichr unicode[HFSPLUS_ATTR_MAX_STRLEN]; } __packed; +enum { + HFS_REGULAR_NAME, + HFS_XATTR_NAME, +}; + struct hfs_extent { __be16 block; __be16 count; @@ -505,12 +510,24 @@ struct hfs_btree_header_rec { b-tree but not in extents b-tree (hfsplus). */ +/* HFS BTree misc info */ +#define HFS_TREE_HEAD 0 +#define HFS_BTREE_HDR_MAP_REC_INDEX 2 /* Map (bitmap) record in Header node */ +#define HFS_BTREE_MAP_NODE_REC_INDEX 0 /* Map record in Map Node */ + /* HFS+ BTree misc info */ -#define HFSPLUS_TREE_HEAD 0 +#define HFSPLUS_TREE_HEAD HFS_TREE_HEAD #define HFSPLUS_NODE_MXSZ 32768 +#define HFSPLUS_NODE_MINSZ 512 #define HFSPLUS_ATTR_TREE_NODE_SIZE 8192 #define HFSPLUS_BTREE_HDR_NODE_RECS_COUNT 3 +/* Map (bitmap) record in Header node */ +#define HFSPLUS_BTREE_HDR_MAP_REC_INDEX HFS_BTREE_HDR_MAP_REC_INDEX +/* Map record in Map Node */ +#define HFSPLUS_BTREE_MAP_NODE_REC_INDEX HFS_BTREE_MAP_NODE_REC_INDEX #define HFSPLUS_BTREE_HDR_USER_BYTES 128 +#define HFSPLUS_BTREE_MAP_NODE_RECS_COUNT 2 +#define HFSPLUS_BTREE_MAP_NODE_RESERVED_BYTES 2 /* btree key type */ #define HFSPLUS_KEY_CASEFOLDING 0xCF /* case-insensitive */ diff --git a/include/linux/hid-sensor-hub.h b/include/linux/hid-sensor-hub.h index e71056553108..ab5cc8db3fbb 100644 --- a/include/linux/hid-sensor-hub.h +++ b/include/linux/hid-sensor-hub.h @@ -43,6 +43,8 @@ struct hid_sensor_hub_attribute_info { * @attr_usage_id: Usage Id of a field, e.g. X-axis for a gyro. * @raw_size: Response size for a read request. * @raw_data: Place holder for received response. + * @index: Current write index into raw_data for multi-byte reads. + * @max_raw_size: Total buffer size for multi-byte reads; 0 for single-value reads. */ struct sensor_hub_pending { bool status; @@ -51,6 +53,8 @@ struct sensor_hub_pending { u32 attr_usage_id; int raw_size; u8 *raw_data; + u32 index; + u32 max_raw_size; }; /** @@ -184,6 +188,27 @@ int sensor_hub_input_attr_get_raw_value(struct hid_sensor_hub_device *hsdev, ); /** + * sensor_hub_input_attr_read_values() - Synchronous multi-byte read request + * @hsdev: Hub device instance. + * @usage_id: Attribute usage id of parent physical device as per spec + * @attr_usage_id: Attribute usage id as per spec + * @report_id: Report id to look for + * @flag: Synchronous or asynchronous read + * @buffer_size: Size of the buffer in bytes + * @buffer: Buffer to store the read data + * + * Issues a synchronous or asynchronous read request for an input attribute, + * accumulating data into the provided buffer until it is full. + * Return: 0 on success, -ETIMEDOUT if the device did not respond, or a + * negative error code. + */ +int sensor_hub_input_attr_read_values(struct hid_sensor_hub_device *hsdev, + u32 usage_id, u32 attr_usage_id, + u32 report_id, + enum sensor_hub_read_flags flag, + u32 buffer_size, u8 *buffer); + +/** * sensor_hub_set_feature() - Feature set request * @hsdev: Hub device instance. * @report_id: Report id to look for diff --git a/include/linux/hid.h b/include/linux/hid.h index 31324609af4d..8d17b741638c 100644 --- a/include/linux/hid.h +++ b/include/linux/hid.h @@ -18,7 +18,7 @@ #include <linux/types.h> #include <linux/slab.h> #include <linux/list.h> -#include <linux/mod_devicetable.h> /* hid_device_id */ +#include <linux/device-id/hid.h> #include <linux/timer.h> #include <linux/workqueue.h> #include <linux/input.h> @@ -634,6 +634,40 @@ enum hid_battery_status { HID_BATTERY_REPORTED, /* Device sent unsolicited battery strength report */ }; +/** + * struct hid_battery - represents a single battery power supply + * @dev: pointer to the parent hid_device + * @ps: the power supply instance + * @min: minimum battery value from HID descriptor + * @max: maximum battery value from HID descriptor + * @report_type: HID report type (input/feature) + * @report_id: HID report ID for this battery + * @report_offset: bit offset of the capacity field within its report + * @charge_status: current charging status + * @status: battery reporting status + * @capacity: current battery capacity (0-100) + * @avoid_query: if true, avoid querying battery (e.g., for stylus) + * @present: if true, battery is present (may be dynamic) + * @ratelimit_time: rate limiting for battery reports + * @list: list node for linking into hid_device's battery list + */ +struct hid_battery { + struct hid_device *dev; + struct power_supply *ps; + __s32 min; + __s32 max; + __s32 report_type; + __s32 report_id; + __s32 report_offset; + __s32 charge_status; + enum hid_battery_status status; + __s32 capacity; + bool avoid_query; + bool present; + ktime_t ratelimit_time; + struct list_head list; +}; + struct hid_driver; struct hid_ll_driver; @@ -670,20 +704,10 @@ struct hid_device { #ifdef CONFIG_HID_BATTERY_STRENGTH /* * Power supply information for HID devices which report - * battery strength. power_supply was successfully registered if - * battery is non-NULL. + * battery strength. Each battery is tracked separately in the + * batteries list. */ - struct power_supply *battery; - __s32 battery_capacity; - __s32 battery_min; - __s32 battery_max; - __s32 battery_report_type; - __s32 battery_report_id; - __s32 battery_charge_status; - enum hid_battery_status battery_status; - bool battery_avoid_query; - bool battery_present; - ktime_t battery_ratelimit_time; + struct list_head batteries; #endif unsigned long status; /* see STAT flags above */ @@ -699,6 +723,7 @@ struct hid_device { char name[128]; /* Device name */ char phys[64]; /* Device physical location */ char uniq[64]; /* Device unique identifier (serial #) */ + u64 firmware_version; /* Firmware version */ void *driver_data; @@ -744,6 +769,15 @@ static inline void hid_set_drvdata(struct hid_device *hdev, void *data) dev_set_drvdata(&hdev->dev, data); } +#ifdef CONFIG_HID_BATTERY_STRENGTH +static inline struct hid_battery *hid_get_battery(struct hid_device *hdev) +{ + if (list_empty(&hdev->batteries)) + return NULL; + return list_first_entry(&hdev->batteries, struct hid_battery, list); +} +#endif + #define HID_GLOBAL_STACK_SIZE 4 #define HID_COLLECTION_STACK_SIZE 4 @@ -989,7 +1023,7 @@ extern void hid_unregister_driver(struct hid_driver *); extern void hidinput_hid_event(struct hid_device *, struct hid_field *, struct hid_usage *, __s32); extern void hidinput_report_event(struct hid_device *hid, struct hid_report *report); -extern int hidinput_connect(struct hid_device *hid, unsigned int force); +extern int hidinput_connect(struct hid_device *hid, unsigned int connect_mask); extern void hidinput_disconnect(struct hid_device *); void hidinput_reset_resume(struct hid_device *hid); @@ -998,6 +1032,8 @@ struct hid_field *hid_find_field(struct hid_device *hdev, unsigned int report_ty int hid_set_field(struct hid_field *, unsigned, __s32); int hid_input_report(struct hid_device *hid, enum hid_report_type type, u8 *data, u32 size, int interrupt); +int hid_safe_input_report(struct hid_device *hid, enum hid_report_type type, u8 *data, + size_t bufsize, u32 size, int interrupt); struct hid_field *hidinput_get_led_field(struct hid_device *hid); unsigned int hidinput_count_leds(struct hid_device *hid); __s32 hidinput_calc_abs_res(const struct hid_field *field, __u16 code); @@ -1266,8 +1302,8 @@ static inline u32 hid_report_len(struct hid_report *report) return DIV_ROUND_UP(report->size, 8) + (report->id > 0); } -int hid_report_raw_event(struct hid_device *hid, enum hid_report_type type, u8 *data, u32 size, - int interrupt); +int hid_report_raw_event(struct hid_device *hid, enum hid_report_type type, u8 *data, + size_t bufsize, u32 size, int interrupt); /* HID quirks API */ unsigned long hid_lookup_quirk(const struct hid_device *hdev); @@ -1282,8 +1318,6 @@ void hid_quirks_exit(__u16 bus); dev_notice(&(hid)->dev, fmt, ##__VA_ARGS__) #define hid_warn(hid, fmt, ...) \ dev_warn(&(hid)->dev, fmt, ##__VA_ARGS__) -#define hid_warn_ratelimited(hid, fmt, ...) \ - dev_warn_ratelimited(&(hid)->dev, fmt, ##__VA_ARGS__) #define hid_info(hid, fmt, ...) \ dev_info(&(hid)->dev, fmt, ##__VA_ARGS__) #define hid_dbg(hid, fmt, ...) \ diff --git a/include/linux/hid_bpf.h b/include/linux/hid_bpf.h index a2e47dbcf82c..19fffa4574a4 100644 --- a/include/linux/hid_bpf.h +++ b/include/linux/hid_bpf.h @@ -72,8 +72,8 @@ struct hid_ops { int (*hid_hw_output_report)(struct hid_device *hdev, __u8 *buf, size_t len, u64 source, bool from_bpf); int (*hid_input_report)(struct hid_device *hid, enum hid_report_type type, - u8 *data, u32 size, int interrupt, u64 source, bool from_bpf, - bool lock_already_taken); + u8 *data, size_t bufsize, u32 size, int interrupt, u64 source, + bool from_bpf, bool lock_already_taken); struct module *owner; const struct bus_type *bus_type; }; @@ -200,7 +200,8 @@ struct hid_bpf { #ifdef CONFIG_HID_BPF u8 *dispatch_hid_bpf_device_event(struct hid_device *hid, enum hid_report_type type, u8 *data, - u32 *size, int interrupt, u64 source, bool from_bpf); + size_t *buf_size, u32 *size, int interrupt, u64 source, + bool from_bpf); int dispatch_hid_bpf_raw_requests(struct hid_device *hdev, unsigned char reportnum, __u8 *buf, u32 size, enum hid_report_type rtype, @@ -215,8 +216,11 @@ int hid_bpf_device_init(struct hid_device *hid); const u8 *call_hid_bpf_rdesc_fixup(struct hid_device *hdev, const u8 *rdesc, unsigned int *size); #else /* CONFIG_HID_BPF */ static inline u8 *dispatch_hid_bpf_device_event(struct hid_device *hid, enum hid_report_type type, - u8 *data, u32 *size, int interrupt, - u64 source, bool from_bpf) { return data; } + u8 *data, size_t *buf_size, u32 *size, + int interrupt, u64 source, bool from_bpf) +{ + return data; +} static inline int dispatch_hid_bpf_raw_requests(struct hid_device *hdev, unsigned char reportnum, u8 *buf, u32 size, enum hid_report_type rtype, diff --git a/include/linux/highmem-internal.h b/include/linux/highmem-internal.h index 0574c21ca45d..bb71e7dba4f7 100644 --- a/include/linux/highmem-internal.h +++ b/include/linux/highmem-internal.h @@ -262,7 +262,7 @@ static inline bool is_kmap_addr(const void *x) * @__addr: Virtual address to be unmapped * * Unmaps an address previously mapped by kmap_atomic() and re-enables - * pagefaults. Depending on PREEMP_RT configuration, re-enables also + * pagefaults. Depending on PREEMPT_RT configuration, re-enables also * migration and preemption. Users should not count on these side effects. * * Mappings should be unmapped in the reverse order that they were mapped. diff --git a/include/linux/highmem.h b/include/linux/highmem.h index af03db851a1d..d7aac9de1c8a 100644 --- a/include/linux/highmem.h +++ b/include/linux/highmem.h @@ -347,10 +347,11 @@ static inline void clear_highpage_kasan_tagged(struct page *page) #ifndef __HAVE_ARCH_TAG_CLEAR_HIGHPAGES -/* Return false to let people know we did not initialize the pages */ -static inline bool tag_clear_highpages(struct page *page, int numpages) +/* Returns true if the caller has to initialize the pages */ +static inline bool tag_clear_highpages(struct page *page, int numpages, + bool clear_pages) { - return false; + return clear_pages; } #endif diff --git a/include/linux/hisi_acc_qm.h b/include/linux/hisi_acc_qm.h index 51a6dc2b97e9..f7570a409905 100644 --- a/include/linux/hisi_acc_qm.h +++ b/include/linux/hisi_acc_qm.h @@ -102,6 +102,12 @@ #define QM_MIG_REGION_SEL 0x100198 #define QM_MIG_REGION_EN BIT(0) +#define QM_MAX_CHANNEL_NUM 8 +#define QM_CHANNEL_USAGE_OFFSET 0x1100 +#define QM_MAX_DEV_USAGE 100 +#define QM_DEV_USAGE_RATE 100 +#define QM_CHANNEL_ADDR_INTRVL 0x4 + /* uacce mode of the driver */ #define UACCE_MODE_NOUACCE 0 /* don't use uacce */ #define UACCE_MODE_SVA 1 /* use uacce sva mode */ @@ -109,10 +115,22 @@ #define QM_ECC_MBIT BIT(2) +/** + * enum qm_stop_reason - Queue manager stop reasons + * @QM_NORMAL: Graceful stop. Used for device unbind, driver removal, + * or runtime power management (runtime_suspend). + * @QM_SOFT_RESET: Error recovery reset. Triggered by unrecoverable hardware + * errors (e.g., PCIe AER, timeout) to recover device state. + * @QM_DOWN: Function Level Reset. Used when the device needs to + * be reset at the function level without resetting the link. + * @QM_SHUTDOWN: System shutdown. Used during system poweroff, reboot, or + * kexec to ensure hardware is in a safe state. + */ enum qm_stop_reason { QM_NORMAL, QM_SOFT_RESET, QM_DOWN, + QM_SHUTDOWN, }; enum qm_state { @@ -152,7 +170,6 @@ enum qm_vf_state { enum qm_misc_ctl_bits { QM_DRIVER_REMOVING = 0x0, - QM_RST_SCHED, QM_RESETTING, QM_MODULE_PARAM, }; @@ -243,6 +260,7 @@ enum acc_err_result { ACC_ERR_NONE, ACC_ERR_NEED_RESET, ACC_ERR_RECOVERED, + ACC_ERR_NEED_FUNC_RESET, }; struct hisi_qm_err_mask { @@ -359,6 +377,11 @@ struct qm_rsv_buf { struct qm_dma qcdma; }; +struct qm_channel { + int channel_num; + const char *channel_name[QM_MAX_CHANNEL_NUM]; +}; + struct hisi_qm { enum qm_hw_ver ver; enum qm_fun_type fun_type; @@ -433,6 +456,7 @@ struct hisi_qm { struct qm_err_isolate isolate_data; struct hisi_qm_cap_tables cap_tables; + struct qm_channel channel_data; }; struct hisi_qp_status { @@ -540,14 +564,13 @@ static inline void hisi_qm_del_list(struct hisi_qm *qm, struct hisi_qm_list *qm_ mutex_unlock(&qm_list->lock); } +int hisi_qm_register_uacce(struct hisi_qm *qm); int hisi_qm_q_num_set(const char *val, const struct kernel_param *kp, unsigned int device); int hisi_qm_init(struct hisi_qm *qm); void hisi_qm_uninit(struct hisi_qm *qm); int hisi_qm_start(struct hisi_qm *qm); int hisi_qm_stop(struct hisi_qm *qm, enum qm_stop_reason r); -int hisi_qm_start_qp(struct hisi_qp *qp, unsigned long arg); -void hisi_qm_stop_qp(struct hisi_qp *qp); int hisi_qp_send(struct hisi_qp *qp, const void *msg); void hisi_qm_debug_init(struct hisi_qm *qm); void hisi_qm_debug_regs_clear(struct hisi_qm *qm); diff --git a/include/linux/hmm.h b/include/linux/hmm.h index db75ffc949a7..6f04e3932f5b 100644 --- a/include/linux/hmm.h +++ b/include/linux/hmm.h @@ -123,6 +123,8 @@ struct hmm_range { * Please see Documentation/mm/hmm.rst for how to use the range API. */ int hmm_range_fault(struct hmm_range *range); +int hmm_range_fault_unlocked_timeout(struct hmm_range *range, + unsigned long timeout); /* * HMM_RANGE_DEFAULT_TIMEOUT - default timeout (ms) when waiting for a range diff --git a/include/linux/host1x.h b/include/linux/host1x.h index 5e7a63143a4a..a7a675783136 100644 --- a/include/linux/host1x.h +++ b/include/linux/host1x.h @@ -143,6 +143,12 @@ static inline struct host1x_bo_mapping *to_host1x_bo_mapping(struct kref *ref) return container_of(ref, struct host1x_bo_mapping, ref); } +/** + * struct host1x_bo_ops - operations implemented by a host1x_bo provider + * + * @pin: create a DMA mapping. Implementation must not touch the bo's refcount. + * @unpin: destroy a DMA mapping. Implementation must not touch the bo's refcount. + */ struct host1x_bo_ops { struct host1x_bo *(*get)(struct host1x_bo *bo); void (*put)(struct host1x_bo *bo); @@ -181,6 +187,7 @@ struct host1x_bo_mapping *host1x_bo_pin(struct device *dev, struct host1x_bo *bo enum dma_data_direction dir, struct host1x_bo_cache *cache); void host1x_bo_unpin(struct host1x_bo_mapping *map); +void host1x_bo_clear_cached_mappings(struct host1x_bo *bo); static inline void *host1x_bo_mmap(struct host1x_bo *bo) { @@ -453,16 +460,6 @@ void host1x_client_unregister(struct host1x_client *client); int host1x_client_suspend(struct host1x_client *client); int host1x_client_resume(struct host1x_client *client); -struct tegra_mipi_device; - -struct tegra_mipi_device *tegra_mipi_request(struct device *device, - struct device_node *np); -void tegra_mipi_free(struct tegra_mipi_device *device); -int tegra_mipi_enable(struct tegra_mipi_device *device); -int tegra_mipi_disable(struct tegra_mipi_device *device); -int tegra_mipi_start_calibration(struct tegra_mipi_device *device); -int tegra_mipi_finish_calibration(struct tegra_mipi_device *device); - /* host1x memory contexts */ struct host1x_memory_context { diff --git a/include/linux/hrtimer.h b/include/linux/hrtimer.h index 74adbd4e7003..29072d89e5cb 100644 --- a/include/linux/hrtimer.h +++ b/include/linux/hrtimer.h @@ -12,7 +12,7 @@ #ifndef _LINUX_HRTIMER_H #define _LINUX_HRTIMER_H -#include <linux/hrtimer_defs.h> +#include <linux/hrtimer_rearm.h> #include <linux/hrtimer_types.h> #include <linux/init.h> #include <linux/list.h> @@ -31,6 +31,13 @@ * soft irq context * HRTIMER_MODE_HARD - Timer callback function will be executed in * hard irq context even on PREEMPT_RT. + * HRTIMER_MODE_LAZY_REARM - Avoid reprogramming if the timer was the + * first expiring timer and is moved into the + * future. Special mode for the HRTICK timer to + * avoid extensive reprogramming of the hardware, + * which is expensive in virtual machines. Risks + * a pointless expiry, but that's better than + * reprogramming on every context switch, */ enum hrtimer_mode { HRTIMER_MODE_ABS = 0x00, @@ -38,6 +45,7 @@ enum hrtimer_mode { HRTIMER_MODE_PINNED = 0x02, HRTIMER_MODE_SOFT = 0x04, HRTIMER_MODE_HARD = 0x08, + HRTIMER_MODE_LAZY_REARM = 0x10, HRTIMER_MODE_ABS_PINNED = HRTIMER_MODE_ABS | HRTIMER_MODE_PINNED, HRTIMER_MODE_REL_PINNED = HRTIMER_MODE_REL | HRTIMER_MODE_PINNED, @@ -55,33 +63,6 @@ enum hrtimer_mode { HRTIMER_MODE_REL_PINNED_HARD = HRTIMER_MODE_REL_PINNED | HRTIMER_MODE_HARD, }; -/* - * Values to track state of the timer - * - * Possible states: - * - * 0x00 inactive - * 0x01 enqueued into rbtree - * - * The callback state is not part of the timer->state because clearing it would - * mean touching the timer after the callback, this makes it impossible to free - * the timer from the callback function. - * - * Therefore we track the callback state in: - * - * timer->base->cpu_base->running == timer - * - * On SMP it is possible to have a "callback function running and enqueued" - * status. It happens for example when a posix timer expired and the callback - * queued a signal. Between dropping the lock which protects the posix timer - * and reacquiring the base lock of the hrtimer, another CPU can deliver the - * signal and rearm the timer. - * - * All state transitions are protected by cpu_base->lock. - */ -#define HRTIMER_STATE_INACTIVE 0x00 -#define HRTIMER_STATE_ENQUEUED 0x01 - /** * struct hrtimer_sleeper - simple sleeper structure * @timer: embedded timer structure @@ -134,11 +115,6 @@ static inline ktime_t hrtimer_get_softexpires(const struct hrtimer *timer) return timer->_softexpires; } -static inline s64 hrtimer_get_expires_ns(const struct hrtimer *timer) -{ - return ktime_to_ns(timer->node.expires); -} - ktime_t hrtimer_cb_get_time(const struct hrtimer *timer); static inline ktime_t hrtimer_expires_remaining(const struct hrtimer *timer) @@ -146,24 +122,23 @@ static inline ktime_t hrtimer_expires_remaining(const struct hrtimer *timer) return ktime_sub(timer->node.expires, hrtimer_cb_get_time(timer)); } -static inline int hrtimer_is_hres_active(struct hrtimer *timer) -{ - return IS_ENABLED(CONFIG_HIGH_RES_TIMERS) ? - timer->base->cpu_base->hres_active : 0; -} - #ifdef CONFIG_HIGH_RES_TIMERS +extern unsigned int hrtimer_resolution; struct clock_event_device; extern void hrtimer_interrupt(struct clock_event_device *dev); -extern unsigned int hrtimer_resolution; +extern struct static_key_false hrtimer_highres_enabled_key; -#else +static inline bool hrtimer_highres_enabled(void) +{ + return static_branch_likely(&hrtimer_highres_enabled_key); +} +#else /* CONFIG_HIGH_RES_TIMERS */ #define hrtimer_resolution (unsigned int)LOW_RES_NSEC - -#endif +static inline bool hrtimer_highres_enabled(void) { return false; } +#endif /* !CONFIG_HIGH_RES_TIMERS */ static inline ktime_t __hrtimer_expires_remaining_adjusted(const struct hrtimer *timer, ktime_t now) @@ -230,6 +205,9 @@ static inline void destroy_hrtimer_on_stack(struct hrtimer *timer) { } extern void hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim, u64 range_ns, const enum hrtimer_mode mode); +extern bool hrtimer_start_range_ns_user(struct hrtimer *timer, ktime_t tim, + u64 range_ns, const enum hrtimer_mode mode); + /** * hrtimer_start - (re)start an hrtimer * @timer: the timer to be added @@ -247,17 +225,28 @@ static inline void hrtimer_start(struct hrtimer *timer, ktime_t tim, extern int hrtimer_cancel(struct hrtimer *timer); extern int hrtimer_try_to_cancel(struct hrtimer *timer); -static inline void hrtimer_start_expires(struct hrtimer *timer, - enum hrtimer_mode mode) +static inline void hrtimer_start_expires(struct hrtimer *timer, enum hrtimer_mode mode) { - u64 delta; ktime_t soft, hard; + u64 delta; + soft = hrtimer_get_softexpires(timer); hard = hrtimer_get_expires(timer); delta = ktime_to_ns(ktime_sub(hard, soft)); hrtimer_start_range_ns(timer, soft, delta, mode); } +static inline bool hrtimer_start_expires_user(struct hrtimer *timer, enum hrtimer_mode mode) +{ + ktime_t soft, hard; + u64 delta; + + soft = hrtimer_get_softexpires(timer); + hard = hrtimer_get_expires(timer); + delta = ktime_to_ns(ktime_sub(hard, soft)); + return hrtimer_start_range_ns_user(timer, soft, delta, mode); +} + void hrtimer_sleeper_start_expires(struct hrtimer_sleeper *sl, enum hrtimer_mode mode); @@ -278,8 +267,8 @@ static inline ktime_t hrtimer_get_remaining(const struct hrtimer *timer) return __hrtimer_get_remaining(timer, false); } -extern u64 hrtimer_get_next_event(void); -extern u64 hrtimer_next_event_without(const struct hrtimer *exclude); +extern ktime_t hrtimer_get_next_event(void); +extern ktime_t hrtimer_next_event_without(const struct hrtimer *exclude); extern bool hrtimer_active(const struct hrtimer *timer); @@ -293,41 +282,12 @@ extern bool hrtimer_active(const struct hrtimer *timer); */ static inline bool hrtimer_is_queued(struct hrtimer *timer) { - /* The READ_ONCE pairs with the update functions of timer->state */ - return !!(READ_ONCE(timer->state) & HRTIMER_STATE_ENQUEUED); -} - -/* - * Helper function to check, whether the timer is running the callback - * function - */ -static inline int hrtimer_callback_running(struct hrtimer *timer) -{ - return timer->base->running == timer; + /* The READ_ONCE pairs with the update functions of timer->is_queued */ + return READ_ONCE(timer->is_queued); } -/** - * hrtimer_update_function - Update the timer's callback function - * @timer: Timer to update - * @function: New callback function - * - * Only safe to call if the timer is not enqueued. Can be called in the callback function if the - * timer is not enqueued at the same time (see the comments above HRTIMER_STATE_ENQUEUED). - */ -static inline void hrtimer_update_function(struct hrtimer *timer, - enum hrtimer_restart (*function)(struct hrtimer *)) -{ -#ifdef CONFIG_PROVE_LOCKING - guard(raw_spinlock_irqsave)(&timer->base->cpu_base->lock); - - if (WARN_ON_ONCE(hrtimer_is_queued(timer))) - return; - - if (WARN_ON_ONCE(!function)) - return; -#endif - ACCESS_PRIVATE(timer, function) = function; -} +void hrtimer_update_function(struct hrtimer *timer, + enum hrtimer_restart (*function)(struct hrtimer *)); /* Forward a hrtimer so it expires after now: */ extern u64 diff --git a/include/linux/hrtimer_bases.h b/include/linux/hrtimer_bases.h new file mode 100644 index 000000000000..d4c83ec5c0f8 --- /dev/null +++ b/include/linux/hrtimer_bases.h @@ -0,0 +1,123 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LINUX_HRTIMER_BASES_H +#define _LINUX_HRTIMER_BASES_H + +#include <linux/hrtimer.h> +#include <linux/ktime.h> +#include <linux/timerqueue.h> +#include <linux/seqlock.h> + +#ifdef CONFIG_64BIT +# define __hrtimer_clock_base_align ____cacheline_aligned +#else +# define __hrtimer_clock_base_align +#endif + +/** + * struct hrtimer_clock_base - the timer base for a specific clock + * @cpu_base: per cpu clock base + * @index: clock type index for per_cpu support when moving a + * timer to a base on another cpu. + * @clockid: clock id for per_cpu support + * @seq: seqcount around __run_hrtimer + * @expires_next: Absolute time of the next event in this clock base + * @running: pointer to the currently running hrtimer + * @active: red black tree root node for the active timers + * @offset: offset of this clock to the monotonic base + */ +struct hrtimer_clock_base { + struct hrtimer_cpu_base *cpu_base; + const unsigned int index; + const clockid_t clockid; + seqcount_raw_spinlock_t seq; + ktime_t expires_next; + struct hrtimer *running; + struct timerqueue_linked_head active; + ktime_t offset; +} __hrtimer_clock_base_align; + +enum hrtimer_base_type { + HRTIMER_BASE_MONOTONIC, + HRTIMER_BASE_REALTIME, + HRTIMER_BASE_BOOTTIME, + HRTIMER_BASE_TAI, + HRTIMER_BASE_MONOTONIC_SOFT, + HRTIMER_BASE_REALTIME_SOFT, + HRTIMER_BASE_BOOTTIME_SOFT, + HRTIMER_BASE_TAI_SOFT, + HRTIMER_MAX_CLOCK_BASES +}; + +/** + * struct hrtimer_cpu_base - the per cpu clock bases + * @lock: lock protecting the base and associated clock bases and timers + * @cpu: cpu number + * @active_bases: Bitfield to mark bases with active timers + * @clock_was_set_seq: Sequence counter of clock was set events + * @hres_active: State of high resolution mode + * @deferred_rearm: A deferred rearm is pending + * @deferred_needs_update: The deferred rearm must re-evaluate the first timer + * @hang_detected: The last hrtimer interrupt detected a hang + * @softirq_activated: displays, if the softirq is raised - update of softirq + * related settings is not required then. + * @nr_events: Total number of hrtimer interrupt events + * @nr_retries: Total number of hrtimer interrupt retries + * @nr_hangs: Total number of hrtimer interrupt hangs + * @max_hang_time: Maximum time spent in hrtimer_interrupt + * @softirq_expiry_lock: Lock which is taken while softirq based hrtimer are expired + * @online: CPU is online from an hrtimers point of view + * @timer_waiters: A hrtimer_cancel() waiters for the timer callback to finish. + * @expires_next: Absolute time of the next event, is required for remote + * hrtimer enqueue; it is the total first expiry time (hard + * and soft hrtimer are taken into account) + * @next_timer: Pointer to the first expiring timer + * @softirq_expires_next: Time to check, if soft queues needs also to be expired + * @softirq_next_timer: Pointer to the first expiring softirq based timer + * @deferred_expires_next: Cached expires next value for deferred rearm + * @clock_base: Array of clock bases for this cpu + * + * Note: next_timer is just an optimization for __remove_hrtimer(). + * Do not dereference the pointer because it is not reliable on + * cross cpu removals. + */ +struct hrtimer_cpu_base { + raw_spinlock_t lock; + unsigned int cpu; + unsigned int active_bases; + u32 clock_was_set_seq; + bool hres_active; + bool deferred_rearm; + bool deferred_needs_update; + bool hang_detected; + bool softirq_activated; + bool online; +#ifdef CONFIG_HIGH_RES_TIMERS + unsigned int nr_events; + unsigned short nr_retries; + unsigned short nr_hangs; + unsigned int max_hang_time; +#endif +#ifdef CONFIG_PREEMPT_RT + spinlock_t softirq_expiry_lock; + atomic_t timer_waiters; +#endif + ktime_t expires_next; + struct hrtimer *next_timer; + ktime_t softirq_expires_next; + struct hrtimer *softirq_next_timer; + ktime_t deferred_expires_next; + struct hrtimer_clock_base clock_base[HRTIMER_MAX_CLOCK_BASES]; + call_single_data_t csd; +} ____cacheline_aligned; + + +/* + * Helper function to check, whether the timer is running the callback + * function + */ +static inline int hrtimer_callback_running(struct hrtimer *timer) +{ + return timer->base->running == timer; +} + +#endif diff --git a/include/linux/hrtimer_defs.h b/include/linux/hrtimer_defs.h deleted file mode 100644 index 02b010df6570..000000000000 --- a/include/linux/hrtimer_defs.h +++ /dev/null @@ -1,110 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _LINUX_HRTIMER_DEFS_H -#define _LINUX_HRTIMER_DEFS_H - -#include <linux/ktime.h> -#include <linux/timerqueue.h> -#include <linux/seqlock.h> - -#ifdef CONFIG_64BIT -# define __hrtimer_clock_base_align ____cacheline_aligned -#else -# define __hrtimer_clock_base_align -#endif - -/** - * struct hrtimer_clock_base - the timer base for a specific clock - * @cpu_base: per cpu clock base - * @index: clock type index for per_cpu support when moving a - * timer to a base on another cpu. - * @clockid: clock id for per_cpu support - * @seq: seqcount around __run_hrtimer - * @running: pointer to the currently running hrtimer - * @active: red black tree root node for the active timers - * @offset: offset of this clock to the monotonic base - */ -struct hrtimer_clock_base { - struct hrtimer_cpu_base *cpu_base; - unsigned int index; - clockid_t clockid; - seqcount_raw_spinlock_t seq; - struct hrtimer *running; - struct timerqueue_head active; - ktime_t offset; -} __hrtimer_clock_base_align; - -enum hrtimer_base_type { - HRTIMER_BASE_MONOTONIC, - HRTIMER_BASE_REALTIME, - HRTIMER_BASE_BOOTTIME, - HRTIMER_BASE_TAI, - HRTIMER_BASE_MONOTONIC_SOFT, - HRTIMER_BASE_REALTIME_SOFT, - HRTIMER_BASE_BOOTTIME_SOFT, - HRTIMER_BASE_TAI_SOFT, - HRTIMER_MAX_CLOCK_BASES, -}; - -/** - * struct hrtimer_cpu_base - the per cpu clock bases - * @lock: lock protecting the base and associated clock bases - * and timers - * @cpu: cpu number - * @active_bases: Bitfield to mark bases with active timers - * @clock_was_set_seq: Sequence counter of clock was set events - * @hres_active: State of high resolution mode - * @in_hrtirq: hrtimer_interrupt() is currently executing - * @hang_detected: The last hrtimer interrupt detected a hang - * @softirq_activated: displays, if the softirq is raised - update of softirq - * related settings is not required then. - * @nr_events: Total number of hrtimer interrupt events - * @nr_retries: Total number of hrtimer interrupt retries - * @nr_hangs: Total number of hrtimer interrupt hangs - * @max_hang_time: Maximum time spent in hrtimer_interrupt - * @softirq_expiry_lock: Lock which is taken while softirq based hrtimer are - * expired - * @online: CPU is online from an hrtimers point of view - * @timer_waiters: A hrtimer_cancel() invocation waits for the timer - * callback to finish. - * @expires_next: absolute time of the next event, is required for remote - * hrtimer enqueue; it is the total first expiry time (hard - * and soft hrtimer are taken into account) - * @next_timer: Pointer to the first expiring timer - * @softirq_expires_next: Time to check, if soft queues needs also to be expired - * @softirq_next_timer: Pointer to the first expiring softirq based timer - * @clock_base: array of clock bases for this cpu - * - * Note: next_timer is just an optimization for __remove_hrtimer(). - * Do not dereference the pointer because it is not reliable on - * cross cpu removals. - */ -struct hrtimer_cpu_base { - raw_spinlock_t lock; - unsigned int cpu; - unsigned int active_bases; - unsigned int clock_was_set_seq; - unsigned int hres_active : 1, - in_hrtirq : 1, - hang_detected : 1, - softirq_activated : 1, - online : 1; -#ifdef CONFIG_HIGH_RES_TIMERS - unsigned int nr_events; - unsigned short nr_retries; - unsigned short nr_hangs; - unsigned int max_hang_time; -#endif -#ifdef CONFIG_PREEMPT_RT - spinlock_t softirq_expiry_lock; - atomic_t timer_waiters; -#endif - ktime_t expires_next; - struct hrtimer *next_timer; - ktime_t softirq_expires_next; - struct hrtimer *softirq_next_timer; - struct hrtimer_clock_base clock_base[HRTIMER_MAX_CLOCK_BASES]; - call_single_data_t csd; -} ____cacheline_aligned; - - -#endif diff --git a/include/linux/hrtimer_rearm.h b/include/linux/hrtimer_rearm.h new file mode 100644 index 000000000000..17a81826bd9a --- /dev/null +++ b/include/linux/hrtimer_rearm.h @@ -0,0 +1,88 @@ +// SPDX-License-Identifier: GPL-2.0 +#ifndef _LINUX_HRTIMER_REARM_H +#define _LINUX_HRTIMER_REARM_H + +#include <linux/types.h> + +#ifdef CONFIG_HRTIMER_REARM_DEFERRED +#include <linux/irqflags.h> +#include <linux/lockdep.h> +#include <linux/preempt.h> +#include <linux/thread_info.h> + +void __hrtimer_rearm_deferred(void); + +/* + * This is purely CPU local, so check the TIF bit first to avoid the overhead of + * the atomic test_and_clear_bit() operation for the common case where the bit + * is not set. + */ +static __always_inline bool hrtimer_test_and_clear_rearm_deferred_tif(unsigned long tif_work) +{ + lockdep_assert_irqs_disabled(); + + if (unlikely(tif_work & _TIF_HRTIMER_REARM)) { + clear_thread_flag(TIF_HRTIMER_REARM); + return true; + } + return false; +} + +#define TIF_REARM_MASK (_TIF_NEED_RESCHED | _TIF_NEED_RESCHED_LAZY | _TIF_HRTIMER_REARM) + +/* Invoked from the exit to user before invoking exit_to_user_mode_loop() */ +static __always_inline bool +hrtimer_rearm_deferred_user_irq(unsigned long *tif_work, const unsigned long tif_mask) +{ + /* Help the compiler to optimize the function out for syscall returns */ + if (!(tif_mask & _TIF_HRTIMER_REARM)) + return false; + /* + * Rearm the timer if none of the resched flags is set before going into + * the loop which re-enables interrupts. + */ + if (unlikely((*tif_work & TIF_REARM_MASK) == _TIF_HRTIMER_REARM)) { + clear_thread_flag(TIF_HRTIMER_REARM); + __hrtimer_rearm_deferred(); + /* Don't go into the loop if HRTIMER_REARM was the only flag */ + *tif_work &= ~TIF_HRTIMER_REARM; + return !*tif_work; + } + return false; +} + +/* Invoked from the time slice extension decision function */ +static __always_inline void hrtimer_rearm_deferred_tif(unsigned long tif_work) +{ + if (hrtimer_test_and_clear_rearm_deferred_tif(tif_work)) + __hrtimer_rearm_deferred(); +} + +/* + * This is to be called on all irqentry_exit() paths that will enable + * interrupts. + */ +static __always_inline void hrtimer_rearm_deferred(void) +{ + hrtimer_rearm_deferred_tif(read_thread_flags()); +} + +/* + * Invoked from the scheduler on entry to __schedule() so it can defer + * rearming after the load balancing callbacks which might change hrtick. + */ +static __always_inline bool hrtimer_test_and_clear_rearm_deferred(void) +{ + return hrtimer_test_and_clear_rearm_deferred_tif(read_thread_flags()); +} + +#else /* CONFIG_HRTIMER_REARM_DEFERRED */ +static __always_inline void __hrtimer_rearm_deferred(void) { } +static __always_inline void hrtimer_rearm_deferred(void) { } +static __always_inline void hrtimer_rearm_deferred_tif(unsigned long tif_work) { } +static __always_inline bool +hrtimer_rearm_deferred_user_irq(unsigned long *tif_work, const unsigned long tif_mask) { return false; } +static __always_inline bool hrtimer_test_and_clear_rearm_deferred(void) { return false; } +#endif /* !CONFIG_HRTIMER_REARM_DEFERRED */ + +#endif diff --git a/include/linux/hrtimer_types.h b/include/linux/hrtimer_types.h index 8fbbb6bdf7a1..b5dacc8271a4 100644 --- a/include/linux/hrtimer_types.h +++ b/include/linux/hrtimer_types.h @@ -17,7 +17,7 @@ enum hrtimer_restart { /** * struct hrtimer - the basic hrtimer structure - * @node: timerqueue node, which also manages node.expires, + * @node: Linked timerqueue node, which also manages node.expires, * the absolute expiry time in the hrtimers internal * representation. The time is related to the clock on * which the timer is based. Is setup by adding @@ -28,23 +28,26 @@ enum hrtimer_restart { * was armed. * @function: timer expiry callback function * @base: pointer to the timer base (per cpu and per clock) - * @state: state information (See bit values above) + * @is_queued: Indicates whether a timer is enqueued or not * @is_rel: Set if the timer was armed relative * @is_soft: Set if hrtimer will be expired in soft interrupt context. * @is_hard: Set if hrtimer will be expired in hard interrupt context * even on RT. + * @is_lazy: Set if the timer is frequently rearmed to avoid updates + * of the clock event device * * The hrtimer structure must be initialized by hrtimer_setup() */ struct hrtimer { - struct timerqueue_node node; + struct timerqueue_linked_node node; + struct hrtimer_clock_base *base; + bool is_queued; + bool is_rel; + bool is_soft; + bool is_hard; + bool is_lazy; ktime_t _softexpires; enum hrtimer_restart (*__private function)(struct hrtimer *); - struct hrtimer_clock_base *base; - u8 state; - u8 is_rel; - u8 is_soft; - u8 is_hard; }; #endif /* _LINUX_HRTIMER_TYPES_H */ diff --git a/include/linux/hsi/hsi.h b/include/linux/hsi/hsi.h index 6ca92bff02c6..ea6bef9b6012 100644 --- a/include/linux/hsi/hsi.h +++ b/include/linux/hsi/hsi.h @@ -271,7 +271,7 @@ struct hsi_controller { struct module *owner; unsigned int id; unsigned int num_ports; - struct hsi_port **port; + struct hsi_port *port[] __counted_by(num_ports); }; #define to_hsi_controller(dev) container_of(dev, struct hsi_controller, device) diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h index a4d9f964dfde..c745f7ad2298 100644 --- a/include/linux/huge_mm.h +++ b/include/linux/huge_mm.h @@ -27,8 +27,8 @@ static inline void huge_pud_set_accessed(struct vm_fault *vmf, pud_t orig_pud) vm_fault_t do_huge_pmd_wp_page(struct vm_fault *vmf); bool madvise_free_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, pmd_t *pmd, unsigned long addr, unsigned long next); -int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, pmd_t *pmd, - unsigned long addr); +bool zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, pmd_t *pmd, + unsigned long addr); int zap_huge_pud(struct mmu_gather *tlb, struct vm_area_struct *vma, pud_t *pud, unsigned long addr); bool move_huge_pmd(struct vm_area_struct *vma, unsigned long old_addr, @@ -83,7 +83,7 @@ extern struct kobj_attribute thpsize_shmem_enabled_attr; * file is never split and the MAX_PAGECACHE_ORDER limit does not apply to * it. Same to PFNMAPs where there's neither page* nor pagecache. */ -#define THP_ORDERS_ALL_SPECIAL \ +#define THP_ORDERS_ALL_SPECIAL_DAX \ (BIT(PMD_ORDER) | BIT(PUD_ORDER)) #define THP_ORDERS_ALL_FILE_DEFAULT \ ((BIT(MAX_PAGECACHE_ORDER + 1) - 1) & ~BIT(0)) @@ -92,7 +92,7 @@ extern struct kobj_attribute thpsize_shmem_enabled_attr; * Mask of all large folio orders supported for THP. */ #define THP_ORDERS_ALL \ - (THP_ORDERS_ALL_ANON | THP_ORDERS_ALL_SPECIAL | THP_ORDERS_ALL_FILE_DEFAULT) + (THP_ORDERS_ALL_ANON | THP_ORDERS_ALL_SPECIAL_DAX | THP_ORDERS_ALL_FILE_DEFAULT) enum tva_type { TVA_SMAPS, /* Exposing "THPeligible:" in smaps. */ @@ -128,6 +128,8 @@ enum mthp_stat_item { MTHP_STAT_ANON_FAULT_ALLOC, MTHP_STAT_ANON_FAULT_FALLBACK, MTHP_STAT_ANON_FAULT_FALLBACK_CHARGE, + MTHP_STAT_COLLAPSE_ALLOC, + MTHP_STAT_COLLAPSE_ALLOC_FAILED, MTHP_STAT_ZSWPOUT, MTHP_STAT_SWPIN, MTHP_STAT_SWPIN_FALLBACK, @@ -142,6 +144,9 @@ enum mthp_stat_item { MTHP_STAT_SPLIT_DEFERRED, MTHP_STAT_NR_ANON, MTHP_STAT_NR_ANON_PARTIALLY_MAPPED, + MTHP_STAT_COLLAPSE_EXCEED_SWAP, + MTHP_STAT_COLLAPSE_EXCEED_NONE, + MTHP_STAT_COLLAPSE_EXCEED_SHARED, __MTHP_STAT_COUNT }; @@ -225,6 +230,7 @@ static inline bool thp_vma_suitable_order(struct vm_area_struct *vma, /* Don't have to check pgoff for anonymous vma */ if (!vma_is_anonymous(vma)) { + /* vma_start_pgoff() in mm.h so not available. */ if (!IS_ALIGNED((vma->vm_start >> PAGE_SHIFT) - vma->vm_pgoff, hpage_size >> PAGE_SHIFT)) return false; @@ -238,6 +244,31 @@ static inline bool thp_vma_suitable_order(struct vm_area_struct *vma, } /* + * Make sure huge_gfp is always more limited than limit_gfp. + * Some shmem users want THP allocation to be done less aggressively + * and only in certain zone. + */ +static inline gfp_t thp_shmem_limit_gfp_mask(gfp_t huge_gfp, gfp_t limit_gfp) +{ + gfp_t allowflags = __GFP_IO | __GFP_FS | __GFP_RECLAIM; + gfp_t denyflags = __GFP_NOWARN | __GFP_NORETRY; + gfp_t zoneflags = limit_gfp & GFP_ZONEMASK; + gfp_t result = huge_gfp & ~(allowflags | GFP_ZONEMASK); + + /* Allow allocations only from the originally specified zones. */ + result |= zoneflags; + + /* + * Minimize the result gfp by taking the union with the deny flags, + * and the intersection of the allow flags. + */ + result |= (limit_gfp & denyflags); + result |= (huge_gfp & limit_gfp) & allowflags; + + return result; +} + +/* * Filter the bitfield of input orders to the ones suitable for use in the vma. * See thp_vma_suitable_order(). * All orders that pass the checks are returned as a bitfield. @@ -360,9 +391,9 @@ static inline bool thp_disabled_by_hw(void) unsigned long thp_get_unmapped_area(struct file *filp, unsigned long addr, unsigned long len, unsigned long pgoff, unsigned long flags); -unsigned long thp_get_unmapped_area_vmflags(struct file *filp, unsigned long addr, +unsigned long thp_get_unmapped_area_vmaflags(struct file *filp, unsigned long addr, unsigned long len, unsigned long pgoff, unsigned long flags, - vm_flags_t vm_flags); + vma_flags_t vma_flags); enum split_type { SPLIT_TYPE_UNIFORM, @@ -389,35 +420,14 @@ static inline int split_huge_page_to_order(struct page *page, unsigned int new_o return split_huge_page_to_list_to_order(page, NULL, new_order); } -/** - * try_folio_split_to_order() - try to split a @folio at @page to @new_order - * using non uniform split. - * @folio: folio to be split - * @page: split to @new_order at the given page - * @new_order: the target split order - * - * Try to split a @folio at @page using non uniform split to @new_order, if - * non uniform split is not supported, fall back to uniform split. After-split - * folios are put back to LRU list. Use min_order_for_split() to get the lower - * bound of @new_order. - * - * Return: 0 - split is successful, otherwise split failed. - */ -static inline int try_folio_split_to_order(struct folio *folio, - struct page *page, unsigned int new_order) -{ - if (folio_check_splittable(folio, new_order, SPLIT_TYPE_NON_UNIFORM)) - return split_huge_page_to_order(&folio->page, new_order); - return folio_split(folio, new_order, page, NULL); -} static inline int split_huge_page(struct page *page) { return split_huge_page_to_list_to_order(page, NULL, 0); } + +int folio_memcg_alloc_deferred(struct folio *folio); + void deferred_split_folio(struct folio *folio, bool partially_mapped); -#ifdef CONFIG_MEMCG -void reparent_deferred_split_queue(struct mem_cgroup *memcg); -#endif void __split_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd, unsigned long address, bool freeze); @@ -462,6 +472,24 @@ void split_huge_pmd_address(struct vm_area_struct *vma, unsigned long address, void __split_huge_pud(struct vm_area_struct *vma, pud_t *pud, unsigned long address); +/** + * pud_is_huge() - Is this PUD either a huge PUD entry or a software leaf entry? + * @pud: The PUD to check. + * + * This is similar to pmd_is_huge(), but it checks at the PUD level. + * + * Returns: true if this PUD is huge, false otherwise. + */ +static inline bool pud_is_huge(pud_t pud) +{ + if (pud_present(pud)) + return pud_trans_huge(pud); + else if (!pud_none(pud)) + return true; + + return false; +} + #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD int change_huge_pud(struct mmu_gather *tlb, struct vm_area_struct *vma, pud_t *pudp, unsigned long addr, pgprot_t newprot, @@ -520,6 +548,8 @@ static inline bool folio_test_pmd_mappable(struct folio *folio) vm_fault_t do_huge_pmd_numa_page(struct vm_fault *vmf); +vm_fault_t do_huge_pmd_uffd_rwp(struct vm_fault *vmf); + vm_fault_t do_huge_pmd_device_private(struct vm_fault *vmf); extern struct folio *huge_zero_folio; @@ -558,7 +588,7 @@ static inline struct folio *get_persistent_huge_zero_folio(void) static inline bool thp_migration_supported(void) { - return IS_ENABLED(CONFIG_ARCH_ENABLE_THP_MIGRATION); + return IS_ENABLED(CONFIG_ARCH_HAS_PMD_SOFTLEAVES); } void split_huge_pmd_locked(struct vm_area_struct *vma, unsigned long address, @@ -581,6 +611,11 @@ static inline bool thp_vma_suitable_order(struct vm_area_struct *vma, return false; } +static inline gfp_t thp_shmem_limit_gfp_mask(gfp_t huge_gfp, gfp_t limit_gfp) +{ + return huge_gfp; +} + static inline unsigned long thp_vma_suitable_orders(struct vm_area_struct *vma, unsigned long addr, unsigned long orders) { @@ -600,18 +635,13 @@ static inline unsigned long thp_vma_allowable_orders(struct vm_area_struct *vma, #define thp_get_unmapped_area NULL static inline unsigned long -thp_get_unmapped_area_vmflags(struct file *filp, unsigned long addr, - unsigned long len, unsigned long pgoff, - unsigned long flags, vm_flags_t vm_flags) +thp_get_unmapped_area_vmaflags(struct file *filp, unsigned long addr, + unsigned long len, unsigned long pgoff, + unsigned long flags, vma_flags_t vma_flags) { return 0; } -static inline bool -can_split_folio(struct folio *folio, int caller_pins, int *pextra_pins) -{ - return false; -} static inline int split_huge_page_to_list_to_order(struct page *page, struct list_head *list, unsigned int new_order) @@ -642,15 +672,22 @@ static inline int split_folio_to_list(struct folio *folio, struct list_head *lis return -EINVAL; } -static inline int try_folio_split_to_order(struct folio *folio, - struct page *page, unsigned int new_order) +static inline int folio_split(struct folio *folio, unsigned int new_order, + struct page *page, struct list_head *list) { VM_WARN_ON_ONCE_FOLIO(1, folio); return -EINVAL; } -static inline void deferred_split_folio(struct folio *folio, bool partially_mapped) {} -static inline void reparent_deferred_split_queue(struct mem_cgroup *memcg) {} +static inline int folio_memcg_alloc_deferred(struct folio *folio) +{ + return 0; +} + +static inline void deferred_split_folio(struct folio *folio, bool partially_mapped) +{ +} + #define split_huge_pmd(__vma, __pmd, __address) \ do { } while (0) @@ -702,6 +739,11 @@ static inline spinlock_t *pud_trans_huge_lock(pud_t *pud, return NULL; } +static inline vm_fault_t do_huge_pmd_uffd_rwp(struct vm_fault *vmf) +{ + return 0; +} + static inline vm_fault_t do_huge_pmd_numa_page(struct vm_fault *vmf) { return 0; @@ -769,17 +811,21 @@ static inline bool pmd_is_huge(pmd_t pmd) { return false; } + +static inline bool pud_is_huge(pud_t pud) +{ + return false; +} #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ -static inline int split_folio_to_list_to_order(struct folio *folio, - struct list_head *list, int new_order) +static inline bool is_pmd_order(unsigned int order) { - return split_huge_page_to_list_to_order(&folio->page, list, new_order); + return order == HPAGE_PMD_ORDER; } static inline int split_folio_to_order(struct folio *folio, int new_order) { - return split_folio_to_list_to_order(folio, NULL, new_order); + return split_huge_page_to_list_to_order(&folio->page, NULL, new_order); } /** diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index 65910437be1c..16c4c4caa126 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -2,6 +2,7 @@ #ifndef _LINUX_HUGETLB_H #define _LINUX_HUGETLB_H +#include <linux/mempolicy.h> #include <linux/mm.h> #include <linux/mm_types.h> #include <linux/mmdebug.h> @@ -148,15 +149,14 @@ int hugetlb_mfill_atomic_pte(pte_t *dst_pte, struct folio **foliop); #endif /* CONFIG_USERFAULTFD */ long hugetlb_reserve_pages(struct inode *inode, long from, long to, - struct vm_area_desc *desc, vma_flags_t vma_flags); + struct vm_area_struct *vma, vma_flags_t vma_flags); long hugetlb_unreserve_pages(struct inode *inode, long start, long end, long freed); bool folio_isolate_hugetlb(struct folio *folio, struct list_head *list); int get_hwpoison_hugetlb_folio(struct folio *folio, bool *hugetlb, bool unpoison); -int get_huge_page_for_hwpoison(unsigned long pfn, int flags, - bool *migratable_cleared); void folio_putback_hugetlb(struct folio *folio); -void move_hugetlb_state(struct folio *old_folio, struct folio *new_folio, int reason); +void move_hugetlb_state(struct folio *old_folio, struct folio *new_folio, + enum migrate_reason reason); void hugetlb_fix_reserve_counts(struct inode *inode); extern struct mutex *hugetlb_fault_mutex_table; u32 hugetlb_fault_mutex_hash(struct address_space *mapping, pgoff_t idx); @@ -173,6 +173,7 @@ extern int movable_gigantic_pages __read_mostly; extern int sysctl_hugetlb_shm_group __read_mostly; extern struct list_head huge_boot_pages[MAX_NUMNODES]; +void hugetlb_bootmem_struct_page_init(void); void hugetlb_bootmem_alloc(void); extern nodemask_t hugetlb_bootmem_nodes; void hugetlb_bootmem_set_nodes(void); @@ -276,7 +277,6 @@ long hugetlb_change_protection(struct vm_area_struct *vma, void hugetlb_unshare_all_pmds(struct vm_area_struct *vma); void fixup_hugetlb_reservations(struct vm_area_struct *vma); void hugetlb_split(struct vm_area_struct *vma, unsigned long addr); -int hugetlb_vma_lock_alloc(struct vm_area_struct *vma); unsigned int arch_hugetlb_cma_order(void); @@ -422,18 +422,12 @@ static inline int get_hwpoison_hugetlb_folio(struct folio *folio, bool *hugetlb, return 0; } -static inline int get_huge_page_for_hwpoison(unsigned long pfn, int flags, - bool *migratable_cleared) -{ - return 0; -} - static inline void folio_putback_hugetlb(struct folio *folio) { } static inline void move_hugetlb_state(struct folio *old_folio, - struct folio *new_folio, int reason) + struct folio *new_folio, enum migrate_reason reason) { } @@ -469,11 +463,6 @@ static inline void fixup_hugetlb_reservations(struct vm_area_struct *vma) static inline void hugetlb_split(struct vm_area_struct *vma, unsigned long addr) {} -static inline int hugetlb_vma_lock_alloc(struct vm_area_struct *vma) -{ - return 0; -} - #endif /* !CONFIG_HUGETLB_PAGE */ #ifndef pgd_write @@ -518,6 +507,7 @@ static inline struct hugetlbfs_sb_info *HUGETLBFS_SB(struct super_block *sb) struct hugetlbfs_inode_info { struct inode vfs_inode; + struct resv_map *resv_map; unsigned int seals; }; @@ -686,24 +676,30 @@ struct hstate { char name[HSTATE_NAME_LEN]; }; -struct cma; - -struct huge_bootmem_page { - struct list_head list; - struct hstate *hstate; - unsigned long flags; - struct cma *cma; -}; - #define HUGE_BOOTMEM_HVO 0x0001 #define HUGE_BOOTMEM_ZONES_VALID 0x0002 #define HUGE_BOOTMEM_CMA 0x0004 -bool hugetlb_bootmem_page_zones_valid(int nid, struct huge_bootmem_page *m); - int isolate_or_dissolve_huge_folio(struct folio *folio, struct list_head *list); int replace_free_hugepage_folios(unsigned long start_pfn, unsigned long end_pfn); void wait_for_freed_hugetlb_folios(void); + +struct mempolicy_interpreted { + int nid; + nodemask_t *nodemask; + enum mempolicy_mode mode; +}; + +enum hugetlb_alloc_flag { + HUGETLB_ALLOC_CHARGE_CGROUP_RSVD_BIT = 0, + HUGETLB_ALLOC_USE_GLOBAL_RESERVATIONS_BIT, +}; + +#define HUGETLB_ALLOC_CHARG_CGROUP_RSVD BIT(HUGETLB_ALLOC_CHARGE_CGROUP_RSVD_BIT) +#define HUGETLB_ALLOC_USE_GLOBAL_RESERVATIONS BIT(HUGETLB_ALLOC_USE_GLOBAL_RESERVATIONS_BIT) + +struct folio *hugetlb_alloc_folio(struct hstate *h, + struct mempolicy_interpreted *mpoli, u8 alloc_flags); struct folio *alloc_hugetlb_folio(struct vm_area_struct *vma, unsigned long addr, bool cow_from_owner); struct folio *alloc_hugetlb_folio_nodemask(struct hstate *h, int preferred_nid, @@ -718,8 +714,8 @@ void restore_reserve_on_error(struct hstate *h, struct vm_area_struct *vma, unsigned long address, struct folio *folio); /* arch callback */ -int __init __alloc_bootmem_huge_page(struct hstate *h, int nid); -int __init alloc_bootmem_huge_page(struct hstate *h, int nid); +void *__init __alloc_bootmem_huge_page(struct hstate *h, int nid); +void *__init arch_alloc_bootmem_huge_page(struct hstate *h, int nid); bool __init hugetlb_node_alloc_supported(void); void __init hugetlb_add_hstate(unsigned order); @@ -777,10 +773,6 @@ static inline unsigned long huge_page_size(const struct hstate *h) return (unsigned long)PAGE_SIZE << h->order; } -extern unsigned long vma_kernel_pagesize(struct vm_area_struct *vma); - -extern unsigned long vma_mmu_pagesize(struct vm_area_struct *vma); - static inline unsigned long huge_page_mask(struct hstate *h) { return h->mask; @@ -796,6 +788,22 @@ static inline unsigned huge_page_shift(struct hstate *h) return h->order + PAGE_SHIFT; } +/** + * hugetlb_linear_page_index() - linear_page_index() but in hugetlb + * page size granularity. + * @vma: the hugetlb VMA + * @address: the virtual address within the VMA + * + * Return: the page offset within the mapping in huge page units. + */ +static inline pgoff_t hugetlb_linear_page_index(struct vm_area_struct *vma, + unsigned long address) +{ + struct hstate *h = hstate_vma(vma); + + return linear_page_index(vma, address) >> huge_page_order(h); +} + static inline bool order_is_gigantic(unsigned int order) { return order > MAX_PAGE_ORDER; @@ -956,7 +964,7 @@ static inline gfp_t htlb_modify_alloc_mask(struct hstate *h, gfp_t gfp_mask) return modified_mask; } -static inline bool htlb_allow_alloc_fallback(int reason) +static inline bool htlb_allow_alloc_fallback(enum migrate_reason reason) { bool allowed_fallback = false; @@ -1137,9 +1145,9 @@ alloc_hugetlb_folio_nodemask(struct hstate *h, int preferred_nid, return NULL; } -static inline int __alloc_bootmem_huge_page(struct hstate *h) +static inline void *__alloc_bootmem_huge_page(struct hstate *h, int nid) { - return 0; + return NULL; } static inline struct hstate *hstate_file(struct file *f) @@ -1177,16 +1185,6 @@ static inline unsigned long huge_page_mask(struct hstate *h) return PAGE_MASK; } -static inline unsigned long vma_kernel_pagesize(struct vm_area_struct *vma) -{ - return PAGE_SIZE; -} - -static inline unsigned long vma_mmu_pagesize(struct vm_area_struct *vma) -{ - return PAGE_SIZE; -} - static inline unsigned int huge_page_order(struct hstate *h) { return 0; @@ -1248,7 +1246,7 @@ static inline gfp_t htlb_modify_alloc_mask(struct hstate *h, gfp_t gfp_mask) return 0; } -static inline bool htlb_allow_alloc_fallback(int reason) +static inline bool htlb_allow_alloc_fallback(enum migrate_reason reason) { return false; } @@ -1271,6 +1269,9 @@ static inline void hugetlb_count_sub(long l, struct mm_struct *mm) { } +pte_t huge_ptep_get(struct mm_struct *mm, unsigned long addr, pte_t *ptep); +unsigned long huge_pte_dirty(pte_t pte); + static inline pte_t huge_ptep_clear_flush(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep) { @@ -1303,6 +1304,10 @@ static inline bool hugetlbfs_pagecache_present( static inline void hugetlb_bootmem_alloc(void) { } + +static inline void hugetlb_bootmem_struct_page_init(void) +{ +} #endif /* CONFIG_HUGETLB_PAGE */ static inline spinlock_t *huge_pte_lock(struct hstate *h, diff --git a/include/linux/hugetlb_cgroup.h b/include/linux/hugetlb_cgroup.h index e5d64b8b59c2..16d72c8c71f6 100644 --- a/include/linux/hugetlb_cgroup.h +++ b/include/linux/hugetlb_cgroup.h @@ -267,5 +267,5 @@ static inline void hugetlb_cgroup_migrate(struct folio *old_folio, { } -#endif /* CONFIG_MEM_RES_CTLR_HUGETLB */ +#endif /* CONFIG_CGROUP_HUGETLB */ #endif diff --git a/include/linux/hugetlb_inline.h b/include/linux/hugetlb_inline.h index 593f5d4e108b..5c29cd3223a1 100644 --- a/include/linux/hugetlb_inline.h +++ b/include/linux/hugetlb_inline.h @@ -6,11 +6,6 @@ #ifdef CONFIG_HUGETLB_PAGE -static inline bool is_vm_hugetlb_flags(vm_flags_t vm_flags) -{ - return !!(vm_flags & VM_HUGETLB); -} - static inline bool is_vma_hugetlb_flags(const vma_flags_t *flags) { return vma_flags_test(flags, VMA_HUGETLB_BIT); @@ -18,11 +13,6 @@ static inline bool is_vma_hugetlb_flags(const vma_flags_t *flags) #else -static inline bool is_vm_hugetlb_flags(vm_flags_t vm_flags) -{ - return false; -} - static inline bool is_vma_hugetlb_flags(const vma_flags_t *flags) { return false; @@ -30,9 +20,9 @@ static inline bool is_vma_hugetlb_flags(const vma_flags_t *flags) #endif -static inline bool is_vm_hugetlb_page(struct vm_area_struct *vma) +static inline bool is_vm_hugetlb_page(const struct vm_area_struct *vma) { - return is_vm_hugetlb_flags(vma->vm_flags); + return is_vma_hugetlb_flags(&vma->flags); } #endif diff --git a/include/linux/hw_random.h b/include/linux/hw_random.h index b77bc55a4cf3..1d3c1927986e 100644 --- a/include/linux/hw_random.h +++ b/include/linux/hw_random.h @@ -46,7 +46,7 @@ struct hwrng { unsigned long priv; unsigned short quality; - /* internal. */ + /* private: internal. */ struct list_head list; struct kref ref; struct work_struct cleanup_work; diff --git a/include/linux/hwmon-sysfs.h b/include/linux/hwmon-sysfs.h index d896713359cd..ee5b33185b2d 100644 --- a/include/linux/hwmon-sysfs.h +++ b/include/linux/hwmon-sysfs.h @@ -15,10 +15,10 @@ struct sensor_device_attribute{ int index; }; #define to_sensor_dev_attr(_dev_attr) \ - container_of(_dev_attr, struct sensor_device_attribute, dev_attr) + container_of_const(_dev_attr, struct sensor_device_attribute, dev_attr) -#define SENSOR_ATTR(_name, _mode, _show, _store, _index) \ - { .dev_attr = __ATTR(_name, _mode, _show, _store), \ +#define SENSOR_ATTR(_name, _mode, _show, _store, _index) \ + { .dev_attr = __DEVICE_ATTR(_name, _mode, _show, _store), \ .index = _index } #define SENSOR_ATTR_RO(_name, _func, _index) \ @@ -49,11 +49,11 @@ struct sensor_device_attribute_2 { u8 nr; }; #define to_sensor_dev_attr_2(_dev_attr) \ - container_of(_dev_attr, struct sensor_device_attribute_2, dev_attr) + container_of_const(_dev_attr, struct sensor_device_attribute_2, dev_attr) -#define SENSOR_ATTR_2(_name, _mode, _show, _store, _nr, _index) \ - { .dev_attr = __ATTR(_name, _mode, _show, _store), \ - .index = _index, \ +#define SENSOR_ATTR_2(_name, _mode, _show, _store, _nr, _index) \ + { .dev_attr = __DEVICE_ATTR(_name, _mode, _show, _store), \ + .index = _index, \ .nr = _nr } #define SENSOR_ATTR_2_RO(_name, _func, _nr, _index) \ diff --git a/include/linux/hwmon.h b/include/linux/hwmon.h index 301a83afbd66..dd713e193d0c 100644 --- a/include/linux/hwmon.h +++ b/include/linux/hwmon.h @@ -13,6 +13,7 @@ #define _HWMON_H_ #include <linux/bitops.h> +#include <linux/cleanup.h> struct device; struct attribute_group; @@ -39,6 +40,7 @@ enum hwmon_chip_attributes { hwmon_chip_power_reset_history, hwmon_chip_register_tz, hwmon_chip_update_interval, + hwmon_chip_update_interval_us, hwmon_chip_alarms, hwmon_chip_samples, hwmon_chip_curr_samples, @@ -55,6 +57,7 @@ enum hwmon_chip_attributes { #define HWMON_C_POWER_RESET_HISTORY BIT(hwmon_chip_power_reset_history) #define HWMON_C_REGISTER_TZ BIT(hwmon_chip_register_tz) #define HWMON_C_UPDATE_INTERVAL BIT(hwmon_chip_update_interval) +#define HWMON_C_UPDATE_INTERVAL_US BIT(hwmon_chip_update_interval_us) #define HWMON_C_ALARMS BIT(hwmon_chip_alarms) #define HWMON_C_SAMPLES BIT(hwmon_chip_samples) #define HWMON_C_CURR_SAMPLES BIT(hwmon_chip_curr_samples) @@ -495,6 +498,8 @@ char *devm_hwmon_sanitize_name(struct device *dev, const char *name); void hwmon_lock(struct device *dev); void hwmon_unlock(struct device *dev); +DEFINE_GUARD(hwmon_lock, struct device *, hwmon_lock(_T), hwmon_unlock(_T)) + /** * hwmon_is_bad_char - Is the char invalid in a hwmon name * @ch: the char to be considered diff --git a/include/linux/hwspinlock.h b/include/linux/hwspinlock.h index f35b42e8c5de..74b91244fe0e 100644 --- a/include/linux/hwspinlock.h +++ b/include/linux/hwspinlock.h @@ -25,34 +25,6 @@ struct hwspinlock; struct hwspinlock_device; struct hwspinlock_ops; -/** - * struct hwspinlock_pdata - platform data for hwspinlock drivers - * @base_id: base id for this hwspinlock device - * - * hwspinlock devices provide system-wide hardware locks that are used - * by remote processors that have no other way to achieve synchronization. - * - * To achieve that, each physical lock must have a system-wide id number - * that is agreed upon, otherwise remote processors can't possibly assume - * they're using the same hardware lock. - * - * Usually boards have a single hwspinlock device, which provides several - * hwspinlocks, and in this case, they can be trivially numbered 0 to - * (num-of-locks - 1). - * - * In case boards have several hwspinlocks devices, a different base id - * should be used for each hwspinlock device (they can't all use 0 as - * a starting id!). - * - * This platform data structure should be used to provide the base id - * for each device (which is trivially 0 when only a single hwspinlock - * device exists). It can be shared between different platforms, hence - * its location. - */ -struct hwspinlock_pdata { - int base_id; -}; - #ifdef CONFIG_HWSPINLOCK int hwspin_lock_register(struct hwspinlock_device *bank, struct device *dev, diff --git a/include/linux/hyperv.h b/include/linux/hyperv.h index dfc516c1c719..9e109d91aa14 100644 --- a/include/linux/hyperv.h +++ b/include/linux/hyperv.h @@ -21,7 +21,7 @@ #include <linux/timer.h> #include <linux/completion.h> #include <linux/device.h> -#include <linux/mod_devicetable.h> +#include <linux/device-id/hv_vmbus.h> #include <linux/interrupt.h> #include <linux/reciprocal_div.h> #include <hyperv/hvhdk.h> @@ -70,7 +70,8 @@ */ enum hv_gpadl_type { HV_GPADL_BUFFER, - HV_GPADL_RING + HV_GPADL_RING, + HV_GPADL_BUFFER_DECRYPTED }; /* Single-page buffer */ @@ -260,9 +261,8 @@ static inline u32 hv_get_avail_to_write_percent( * 5 . 2 (Windows Server 2019, RS5) * 5 . 3 (Windows Server 2022) * - * The WS2008 and WIN7 versions are listed here for - * completeness but are no longer supported in the - * Linux kernel. + * The WS2008, WIN7, WIN8, and WIN8_1 versions are listed here for + * completeness but are no longer supported in the Linux kernel. */ #define VMBUS_MAKE_VERSION(MAJ, MIN) ((((u32)MAJ) << 16) | (MIN)) @@ -1015,8 +1015,8 @@ struct vmbus_channel { /* The max size of a packet on this channel */ u32 max_pkt_size; - /* function to mmap ring buffer memory to the channel's sysfs ring attribute */ - int (*mmap_ring_buffer)(struct vmbus_channel *channel, struct vm_area_struct *vma); + /* function to mmap ring buffer memory to the channel's sysfs ring attribute */ + int (*mmap_prepare_ring_buffer)(struct vmbus_channel *channel, struct vm_area_desc *desc); /* boolean to control visibility of sysfs for ring buffer */ bool ring_sysfs_visible; @@ -1205,9 +1205,21 @@ extern int vmbus_establish_gpadl(struct vmbus_channel *channel, u32 size, struct vmbus_gpadl *gpadl); +extern int vmbus_establish_gpadl_caller_decrypted(struct vmbus_channel *channel, + void *kbuffer, + u32 size, + struct vmbus_gpadl *gpadl); + extern int vmbus_teardown_gpadl(struct vmbus_channel *channel, struct vmbus_gpadl *gpadl); +extern void *vmbus_alloc_buffer(struct vmbus_channel *channel, + u32 size, + struct page ***chunks_out, + u32 *chunk_cnt_out); + +extern void vmbus_free_buffer(void *addr, struct page **chunks, u32 chunk_cnt); + void vmbus_reset_channel_cb(struct vmbus_channel *channel); extern int vmbus_recvpacket(struct vmbus_channel *channel, @@ -1272,11 +1284,6 @@ struct hv_device { u16 device_id; struct device device; - /* - * Driver name to force a match. Do not set directly, because core - * frees it. Use driver_set_override() to set or clear it. - */ - const char *driver_override; struct vmbus_channel *channel; struct kset *channels_kset; @@ -1304,6 +1311,12 @@ static inline void *hv_get_drvdata(struct hv_device *dev) struct device *hv_get_vmbus_root_device(void); +#if IS_ENABLED(CONFIG_HYPERV_VMBUS) +bool hv_vmbus_exists(void); +#else +static inline bool hv_vmbus_exists(void) { return false; } +#endif + struct hv_ring_buffer_debug_info { u32 current_interrupt_mask; u32 current_read_index; @@ -1334,6 +1347,9 @@ int vmbus_allocate_mmio(struct resource **new, struct hv_device *device_obj, bool fb_overlap_ok); void vmbus_free_mmio(resource_size_t start, resource_size_t size); +void vmbus_initiate_unload(bool crash); +void vmbus_set_skip_unload(bool skip); + /* * GUID definitions of various offer types - services offered to the guest. */ diff --git a/include/linux/i2c-algo-bit.h b/include/linux/i2c-algo-bit.h index 7fd5575a368f..c4c43aec1e67 100644 --- a/include/linux/i2c-algo-bit.h +++ b/include/linux/i2c-algo-bit.h @@ -34,6 +34,7 @@ struct i2c_algo_bit_data { maximum 50 us for SMBus */ int timeout; /* in jiffies */ bool can_do_atomic; /* callbacks don't sleep, we can be atomic */ + bool skip_bit_test; /* override bit_test module parameter */ }; int i2c_bit_add_bus(struct i2c_adapter *); diff --git a/include/linux/i2c-atr.h b/include/linux/i2c-atr.h index 2bb54dc87c8e..b52a7b9ec536 100644 --- a/include/linux/i2c-atr.h +++ b/include/linux/i2c-atr.h @@ -71,7 +71,7 @@ struct i2c_atr_adap_desc { struct device *parent; struct fwnode_handle *bus_handle; size_t num_aliases; - u16 *aliases; + u16 *aliases __counted_by_ptr(num_aliases); }; /** diff --git a/include/linux/i2c.h b/include/linux/i2c.h index 20fd41b51d5c..14ab4d3055af 100644 --- a/include/linux/i2c.h +++ b/include/linux/i2c.h @@ -12,7 +12,7 @@ #include <linux/acpi.h> /* for acpi_handle */ #include <linux/bits.h> -#include <linux/mod_devicetable.h> +#include <linux/device-id/i2c.h> #include <linux/device.h> /* for struct device */ #include <linux/sched.h> /* for completion */ #include <linux/mutex.h> diff --git a/include/linux/i3c/ccc.h b/include/linux/i3c/ccc.h index ad59a4ae60d1..c6947dcb0f57 100644 --- a/include/linux/i3c/ccc.h +++ b/include/linux/i3c/ccc.h @@ -12,6 +12,8 @@ #include <linux/i3c/device.h> /* I3C CCC (Common Command Codes) related definitions */ +#define I3C_CCC_RETRIES 1 + #define I3C_CCC_DIRECT BIT(7) #define I3C_CCC_ID(id, broadcast) \ @@ -32,6 +34,7 @@ #define I3C_CCC_DEFSLVS I3C_CCC_ID(0x8, true) #define I3C_CCC_ENTTM I3C_CCC_ID(0xb, true) #define I3C_CCC_ENTHDR(x) I3C_CCC_ID(0x20 + (x), true) +#define I3C_CCC_SETAASA I3C_CCC_ID(0x29, true) /* Unicast-only commands */ #define I3C_CCC_SETDASA I3C_CCC_ID(0x7, false) @@ -343,11 +346,15 @@ struct i3c_ccc_getxtime { /** * struct i3c_ccc_cmd_payload - CCC payload * - * @len: payload length + * @len: requested payload length + * @actual_len: number of bytes received on a GET CCC (filled by the driver) + * @optional_bytes: GET CCCs may return up to this many fewer bytes than @len * @data: payload data. This buffer must be DMA-able */ struct i3c_ccc_cmd_payload { u16 len; + u16 actual_len; + u16 optional_bytes; void *data; }; @@ -372,12 +379,15 @@ struct i3c_ccc_cmd_dest { * @ndests: number of destinations. Should always be one for broadcast commands * @dests: array of destinations and associated payload for this CCC. Most of * the time, only one destination is provided + * @retries: number of times to retry a failed Direct GET CCC (see + * &I3C_CCC_RETRIES) * @err: I3C error code */ struct i3c_ccc_cmd { u8 rnw; u8 id; unsigned int ndests; + unsigned int retries; struct i3c_ccc_cmd_dest *dests; enum i3c_error_code err; }; diff --git a/include/linux/i3c/device.h b/include/linux/i3c/device.h index 971d53349b6f..0f065b883ee0 100644 --- a/include/linux/i3c/device.h +++ b/include/linux/i3c/device.h @@ -12,7 +12,7 @@ #include <linux/device.h> #include <linux/i2c.h> #include <linux/kconfig.h> -#include <linux/mod_devicetable.h> +#include <linux/device-id/i3c.h> #include <linux/module.h> /** diff --git a/include/linux/i3c/master.h b/include/linux/i3c/master.h index 592b646f6134..f7ceec2b4477 100644 --- a/include/linux/i3c/master.h +++ b/include/linux/i3c/master.h @@ -174,10 +174,19 @@ struct i3c_device_ibi_info { * assigned a dynamic address by the master. Will be used during * bus initialization to assign it a specific dynamic address * before starting DAA (Dynamic Address Assignment) + * @static_addr_method: Bitmap describing which methods of Dynamic Address + * Assignment from a Static Address are supported by this I3C Target. + * A value of 1 in a bit position indicates that the I3C target + * supports that method, and a value of 0 indicates that the I3C + * target does not support that method. + * Bit 0: SETDASA + * Bit 1: SETAASA + * All other bits are reserved. * @pid: I3C Provisioned ID exposed by the device. This is a unique identifier * that may be used to attach boardinfo to i3c_dev_desc when the device * does not have a static address - * @of_node: optional DT node in case the device has been described in the DT + * @fwnode: Firmware node (DT or ACPI) in case the device has been + * described in firmware * * This structure is used to attach board-level information to an I3C device. * Not all I3C devices connected on the bus will have a boardinfo. It's only @@ -188,8 +197,9 @@ struct i3c_dev_boardinfo { struct list_head node; u8 init_dyn_addr; u8 static_addr; + u8 static_addr_method; u64 pid; - struct device_node *of_node; + struct fwnode_handle *fwnode; }; /** @@ -228,6 +238,8 @@ struct i3c_dev_desc { * every time the I3C device is rediscovered with a different dynamic * address assigned * @bus: I3C bus this device is attached to + * @node: unregistered device list node, only for use by + * i3c_master_register_new_i3c_devs(), it is not protected by a lock * * I3C device object exposed to I3C device drivers. The takes care of linking * this object to the relevant &struct_i3c_dev_desc one. @@ -238,6 +250,7 @@ struct i3c_device { struct device dev; struct i3c_dev_desc *desc; struct i3c_bus *bus; + struct list_head node; }; /* @@ -259,6 +272,7 @@ struct i3c_device { #define I3C_BUS_THIGH_MIXED_MAX_NS 41 #define I3C_BUS_TIDLE_MIN_NS 200000 #define I3C_BUS_TLOW_OD_MIN_NS 200 +#define I3C_BUS_THIGH_INIT_OD_MIN_NS 200 /** * enum i3c_bus_mode - I3C bus mode @@ -494,7 +508,7 @@ struct i3c_master_controller_ops { int (*disable_hotjoin)(struct i3c_master_controller *master); int (*set_speed)(struct i3c_master_controller *master, enum i3c_open_drain_speed speed); int (*set_dev_nack_retry)(struct i3c_master_controller *master, - unsigned long dev_nack_retry_cnt); + unsigned int dev_nack_retry_cnt); }; /** @@ -511,15 +525,29 @@ struct i3c_master_controller_ops { * @hotjoin: true if the master support hotjoin * @rpm_allowed: true if Runtime PM allowed * @rpm_ibi_allowed: true if IBI and Hot-Join allowed while runtime suspended + * @ibi_wakeup: IBI can wakeup the system + * @shutting_down: set to true when master begins shutdown or unregister * @boardinfo.i3c: list of I3C boardinfo objects * @boardinfo.i2c: list of I2C boardinfo objects * @boardinfo: board-level information attached to devices connected on the bus * @bus: I3C bus exposed by this master - * @wq: workqueue which can be used by master + * @addr_method: Bitmap describing which methods of Address Assignment required + * to be run for discovering all the devices on the bus. + * Bit 0: SETDASA + * Bit 1: SETAASA + * All other bits are reserved. + * @wq: freezable workqueue which can be used by master * drivers if they need to postpone operations that need to take place * in a thread context. Typical examples are Hot Join processing which * requires taking the bus lock in maintenance, which in turn, can only * be done from a sleep-able context + * @hj_work: work item used to run DAA after a Hot-Join event is detected. + * Queued to @wq by i3c_master_queue_hotjoin() + * @reg_work: work item used to register newly discovered I3C devices with + * the driver model. Queued to @wq by i3c_master_do_daa_ext() so + * that device registration is deferred out of the DAA caller's + * context (notably the resume path), and is skipped if the + * controller is shutting down * @dev_nack_retry_count: retry count when slave device nack * * A &struct i3c_master_controller has to be registered to the I3C subsystem @@ -537,12 +565,17 @@ struct i3c_master_controller { unsigned int hotjoin: 1; unsigned int rpm_allowed: 1; unsigned int rpm_ibi_allowed: 1; + unsigned int ibi_wakeup: 1; + bool shutting_down; struct { struct list_head i3c; struct list_head i2c; } boardinfo; struct i3c_bus bus; + u8 addr_method; struct workqueue_struct *wq; + struct work_struct hj_work; + struct work_struct reg_work; unsigned int dev_nack_retry_count; }; @@ -596,14 +629,15 @@ int i3c_master_disec_locked(struct i3c_master_controller *master, u8 addr, u8 evts); int i3c_master_enec_locked(struct i3c_master_controller *master, u8 addr, u8 evts); +int i3c_master_enec_disec_locked(struct i3c_master_controller *master, u8 addr, + bool enable, u8 evts, bool suppress_m2); int i3c_master_entdaa_locked(struct i3c_master_controller *master); int i3c_master_defslvs_locked(struct i3c_master_controller *master); int i3c_master_get_free_addr(struct i3c_master_controller *master, u8 start_addr); -int i3c_master_add_i3c_dev_locked(struct i3c_master_controller *master, - u8 addr); +void i3c_master_add_i3c_dev_locked(struct i3c_master_controller *master, u8 addr); int i3c_master_do_daa(struct i3c_master_controller *master); int i3c_master_do_daa_ext(struct i3c_master_controller *master, bool rstdaa); struct i3c_dma *i3c_master_dma_map_single(struct device *dev, void *ptr, @@ -613,6 +647,8 @@ void i3c_master_dma_unmap_single(struct i3c_dma *dma_xfer); DEFINE_FREE(i3c_master_dma_unmap_single, void *, if (_T) i3c_master_dma_unmap_single(_T)) +int i3c_master_reattach_i3c_dev_locked(struct i3c_dev_desc *dev, + u8 old_dyn_addr); int i3c_master_set_info(struct i3c_master_controller *master, const struct i3c_device_info *info); @@ -623,6 +659,7 @@ int i3c_master_register(struct i3c_master_controller *master, void i3c_master_unregister(struct i3c_master_controller *master); int i3c_master_enable_hotjoin(struct i3c_master_controller *master); int i3c_master_disable_hotjoin(struct i3c_master_controller *master); +void i3c_master_queue_hotjoin(struct i3c_master_controller *master); /** * i3c_dev_get_master_data() - get master private data attached to an I3C @@ -729,6 +766,7 @@ void i3c_generic_ibi_recycle_slot(struct i3c_generic_ibi_pool *pool, struct i3c_ibi_slot *slot); void i3c_master_queue_ibi(struct i3c_dev_desc *dev, struct i3c_ibi_slot *slot); +bool i3c_master_has_wakeup_enabled_devs(struct i3c_master_controller *master); struct i3c_ibi_slot *i3c_master_get_free_ibi_slot(struct i3c_dev_desc *dev); diff --git a/include/linux/icmpv6.h b/include/linux/icmpv6.h index e3b3b0fa2a8f..2bd9f2157e6c 100644 --- a/include/linux/icmpv6.h +++ b/include/linux/icmpv6.h @@ -15,38 +15,13 @@ static inline struct icmp6hdr *icmp6_hdr(const struct sk_buff *skb) #if IS_ENABLED(CONFIG_IPV6) -typedef void ip6_icmp_send_t(struct sk_buff *skb, u8 type, u8 code, __u32 info, - const struct in6_addr *force_saddr, - const struct inet6_skb_parm *parm); void icmp6_send(struct sk_buff *skb, u8 type, u8 code, __u32 info, const struct in6_addr *force_saddr, const struct inet6_skb_parm *parm); -#if IS_BUILTIN(CONFIG_IPV6) -static inline void __icmpv6_send(struct sk_buff *skb, u8 type, u8 code, __u32 info, - const struct inet6_skb_parm *parm) -{ - icmp6_send(skb, type, code, info, NULL, parm); -} -static inline int inet6_register_icmp_sender(ip6_icmp_send_t *fn) -{ - BUILD_BUG_ON(fn != icmp6_send); - return 0; -} -static inline int inet6_unregister_icmp_sender(ip6_icmp_send_t *fn) -{ - BUILD_BUG_ON(fn != icmp6_send); - return 0; -} -#else -extern void __icmpv6_send(struct sk_buff *skb, u8 type, u8 code, __u32 info, - const struct inet6_skb_parm *parm); -extern int inet6_register_icmp_sender(ip6_icmp_send_t *fn); -extern int inet6_unregister_icmp_sender(ip6_icmp_send_t *fn); -#endif static inline void icmpv6_send(struct sk_buff *skb, u8 type, u8 code, __u32 info) { - __icmpv6_send(skb, type, code, info, IP6CB(skb)); + icmp6_send(skb, type, code, info, NULL, IP6CB(skb)); } int ip6_err_gen_icmpv6_unreach(struct sk_buff *skb, int nhs, int type, @@ -58,7 +33,7 @@ void icmpv6_ndo_send(struct sk_buff *skb_in, u8 type, u8 code, __u32 info); static inline void icmpv6_ndo_send(struct sk_buff *skb_in, u8 type, u8 code, __u32 info) { struct inet6_skb_parm parm = { 0 }; - __icmpv6_send(skb_in, type, code, info, &parm); + icmp6_send(skb_in, type, code, info, NULL, &parm); } #endif diff --git a/include/linux/ieee80211-eht.h b/include/linux/ieee80211-eht.h index f8e9f5d36d2a..b62297a978e7 100644 --- a/include/linux/ieee80211-eht.h +++ b/include/linux/ieee80211-eht.h @@ -9,7 +9,7 @@ * Copyright (c) 2006, Michael Wu <flamingice@sourmilk.net> * Copyright (c) 2013 - 2014 Intel Mobile Communications GmbH * Copyright (c) 2016 - 2017 Intel Deutschland GmbH - * Copyright (c) 2018 - 2025 Intel Corporation + * Copyright (c) 2018 - 2026 Intel Corporation */ #ifndef LINUX_IEEE80211_EHT_H @@ -251,8 +251,8 @@ struct ieee80211_eht_operation_info { #define IEEE80211_EHT_PHY_CAP5_SUPP_EXTRA_EHT_LTF 0x40 #define IEEE80211_EHT_PHY_CAP6_MAX_NUM_SUPP_EHT_LTF_MASK 0x07 -#define IEEE80211_EHT_PHY_CAP6_MCS15_SUPP_80MHZ 0x08 -#define IEEE80211_EHT_PHY_CAP6_MCS15_SUPP_160MHZ 0x30 +#define IEEE80211_EHT_PHY_CAP6_MCS15_SUPP_80MHZ 0x10 +#define IEEE80211_EHT_PHY_CAP6_MCS15_SUPP_160MHZ 0x20 #define IEEE80211_EHT_PHY_CAP6_MCS15_SUPP_320MHZ 0x40 #define IEEE80211_EHT_PHY_CAP6_MCS15_SUPP_MASK 0x78 #define IEEE80211_EHT_PHY_CAP6_EHT_DUP_6GHZ_SUPP 0x80 @@ -394,13 +394,23 @@ ieee80211_eht_oper_size_ok(const u8 *data, u8 len) } /* must validate ieee80211_eht_oper_size_ok() first */ +static inline const struct ieee80211_eht_operation_info * +ieee80211_eht_oper_info(const struct ieee80211_eht_operation *eht_oper) +{ + if (!(eht_oper->params & IEEE80211_EHT_OPER_INFO_PRESENT)) + return NULL; + + return (const void *)eht_oper->optional; +} + +/* must validate ieee80211_eht_oper_size_ok() first */ static inline u16 ieee80211_eht_oper_dis_subchan_bitmap(const struct ieee80211_eht_operation *eht_oper) { - const struct ieee80211_eht_operation_info *info = - (const void *)eht_oper->optional; + const struct ieee80211_eht_operation_info *info; - if (!(eht_oper->params & IEEE80211_EHT_OPER_INFO_PRESENT)) + info = ieee80211_eht_oper_info(eht_oper); + if (!info) return 0; if (!(eht_oper->params & IEEE80211_EHT_OPER_DISABLED_SUBCHANNEL_BITMAP_PRESENT)) @@ -471,6 +481,7 @@ struct ieee80211_multi_link_elem { #define IEEE80211_MLC_BASIC_PRES_MLD_CAPA_OP 0x0100 #define IEEE80211_MLC_BASIC_PRES_MLD_ID 0x0200 #define IEEE80211_MLC_BASIC_PRES_EXT_MLD_CAPA_OP 0x0400 +#define IEEE80211_MLC_BASIC_PRES_ENH_CRIT_UPD 0x0800 #define IEEE80211_MED_SYNC_DELAY_DURATION 0x00ff #define IEEE80211_MED_SYNC_DELAY_SYNC_OFDM_ED_THRESH 0x0f00 @@ -485,26 +496,28 @@ struct ieee80211_multi_link_elem { #define IEEE80211_MED_SYNC_DELAY_DEFAULT 0x10ac #define IEEE80211_EML_CAP_EMLSR_SUPP 0x0001 -#define IEEE80211_EML_CAP_EMLSR_PADDING_DELAY 0x000e -#define IEEE80211_EML_CAP_EMLSR_PADDING_DELAY_0US 0 -#define IEEE80211_EML_CAP_EMLSR_PADDING_DELAY_32US 1 -#define IEEE80211_EML_CAP_EMLSR_PADDING_DELAY_64US 2 -#define IEEE80211_EML_CAP_EMLSR_PADDING_DELAY_128US 3 -#define IEEE80211_EML_CAP_EMLSR_PADDING_DELAY_256US 4 -#define IEEE80211_EML_CAP_EMLSR_TRANSITION_DELAY 0x0070 +#define IEEE80211_EML_CAP_EML_PADDING_DELAY 0x000e +/* Described Tables 9-417i & 9-417k in 802.11be-2024, which have the same values */ +#define IEEE80211_EML_CAP_EML_PADDING_DELAY_0US 0 +#define IEEE80211_EML_CAP_EML_PADDING_DELAY_32US 1 +#define IEEE80211_EML_CAP_EML_PADDING_DELAY_64US 2 +#define IEEE80211_EML_CAP_EML_PADDING_DELAY_128US 3 +#define IEEE80211_EML_CAP_EML_PADDING_DELAY_256US 4 +#define IEEE80211_EML_CAP_EML_TRANSITION_DELAY 0x0070 +/* Described in Table 9-417j in 802.11be-2024 */ #define IEEE80211_EML_CAP_EMLSR_TRANSITION_DELAY_0US 0 #define IEEE80211_EML_CAP_EMLSR_TRANSITION_DELAY_16US 1 #define IEEE80211_EML_CAP_EMLSR_TRANSITION_DELAY_32US 2 #define IEEE80211_EML_CAP_EMLSR_TRANSITION_DELAY_64US 3 #define IEEE80211_EML_CAP_EMLSR_TRANSITION_DELAY_128US 4 #define IEEE80211_EML_CAP_EMLSR_TRANSITION_DELAY_256US 5 +/* Described in Table 9-417l in 802.11be-2024 */ +#define IEEE80211_EML_CAP_EMLMR_TRANSITION_DELAY_0US 0 +#define IEEE80211_EML_CAP_EMLMR_TRANSITION_DELAY_32US 1 +#define IEEE80211_EML_CAP_EMLMR_TRANSITION_DELAY_64US 2 +#define IEEE80211_EML_CAP_EMLMR_TRANSITION_DELAY_128US 3 +#define IEEE80211_EML_CAP_EMLMR_TRANSITION_DELAY_256US 4 #define IEEE80211_EML_CAP_EMLMR_SUPPORT 0x0080 -#define IEEE80211_EML_CAP_EMLMR_DELAY 0x0700 -#define IEEE80211_EML_CAP_EMLMR_DELAY_0US 0 -#define IEEE80211_EML_CAP_EMLMR_DELAY_32US 1 -#define IEEE80211_EML_CAP_EMLMR_DELAY_64US 2 -#define IEEE80211_EML_CAP_EMLMR_DELAY_128US 3 -#define IEEE80211_EML_CAP_EMLMR_DELAY_256US 4 #define IEEE80211_EML_CAP_TRANSITION_TIMEOUT 0x7800 #define IEEE80211_EML_CAP_TRANSITION_TIMEOUT_0 0 #define IEEE80211_EML_CAP_TRANSITION_TIMEOUT_128US 1 @@ -517,7 +530,6 @@ struct ieee80211_multi_link_elem { #define IEEE80211_EML_CAP_TRANSITION_TIMEOUT_16TU 8 #define IEEE80211_EML_CAP_TRANSITION_TIMEOUT_32TU 9 #define IEEE80211_EML_CAP_TRANSITION_TIMEOUT_64TU 10 -#define IEEE80211_EML_CAP_TRANSITION_TIMEOUT_128TU 11 #define IEEE80211_MLD_CAP_OP_MAX_SIMUL_LINKS 0x000f #define IEEE80211_MLD_CAP_OP_SRS_SUPPORT 0x0010 @@ -749,11 +761,13 @@ static inline u16 ieee80211_mle_get_mld_capa_op(const u8 *data) } /* Defined in Figure 9-1074t in P802.11be_D7.0 */ -#define IEEE80211_EHT_ML_EXT_MLD_CAPA_OP_PARAM_UPDATE 0x0001 -#define IEEE80211_EHT_ML_EXT_MLD_CAPA_OP_RECO_MAX_LINKS_MASK 0x001e -#define IEEE80211_EHT_ML_EXT_MLD_CAPA_NSTR_UPDATE 0x0020 -#define IEEE80211_EHT_ML_EXT_MLD_CAPA_EMLSR_ENA_ON_ONE_LINK 0x0040 -#define IEEE80211_EHT_ML_EXT_MLD_CAPA_BTM_MLD_RECO_MULTI_AP 0x0080 +#define IEEE80211_EHT_ML_EXT_MLD_CAPA_OP_PARAM_UPDATE 0x0001 +#define IEEE80211_EHT_ML_EXT_MLD_CAPA_OP_RECO_MAX_LINKS_MASK 0x001e +#define IEEE80211_EHT_ML_EXT_MLD_CAPA_NSTR_UPDATE 0x0020 +#define IEEE80211_EHT_ML_EXT_MLD_CAPA_EMLSR_ENA_ON_ONE_LINK 0x0040 +#define IEEE80211_EHT_ML_EXT_MLD_CAPA_BTM_MLD_RECO_MULTI_AP 0x0080 +/* defined by UHR Draft P802.11bn_D1.3 Figure 9-1147 */ +#define IEEE80211_UHR_ML_EXT_MLD_CAPA_ML_PM 0x0100 /** * ieee80211_mle_get_ext_mld_capa_op - returns the extended MLD capabilities @@ -797,6 +811,49 @@ static inline u16 ieee80211_mle_get_ext_mld_capa_op(const u8 *data) } /** + * ieee80211_mle_get_enh_crit_upd_info - returns the enhanced critical + * updates information + * @data: pointer to the multi-link element + * Return: the enhanced critical updates information field, or %NULL + * + * The element is assumed to be of the correct type (BASIC) and big enough, + * this must be checked using ieee80211_mle_type_ok(). + */ +static inline const struct ieee80211_enh_crit_upd * +ieee80211_mle_get_enh_crit_upd_info(const u8 *data) +{ + const struct ieee80211_multi_link_elem *mle = (const void *)data; + u16 control = le16_to_cpu(mle->control); + const u8 *common = mle->variable; + + /* + * common points now at the beginning of + * ieee80211_mle_basic_common_info + */ + common += sizeof(struct ieee80211_mle_basic_common_info); + + if (!(control & IEEE80211_MLC_BASIC_PRES_ENH_CRIT_UPD)) + return NULL; + + if (control & IEEE80211_MLC_BASIC_PRES_LINK_ID) + common += 1; + if (control & IEEE80211_MLC_BASIC_PRES_BSS_PARAM_CH_CNT) + common += 1; + if (control & IEEE80211_MLC_BASIC_PRES_MED_SYNC_DELAY) + common += 2; + if (control & IEEE80211_MLC_BASIC_PRES_EML_CAPA) + common += 2; + if (control & IEEE80211_MLC_BASIC_PRES_MLD_CAPA_OP) + common += 2; + if (control & IEEE80211_MLC_BASIC_PRES_MLD_ID) + common += 1; + if (control & IEEE80211_MLC_BASIC_PRES_EXT_MLD_CAPA_OP) + common += 2; + + return (const void *)common; +} + +/** * ieee80211_mle_get_mld_id - returns the MLD ID * @data: pointer to the multi-link element * Return: The MLD ID in the given multi-link element, or 0 if not present @@ -844,7 +901,7 @@ static inline bool ieee80211_mle_size_ok(const u8 *data, size_t len) const struct ieee80211_multi_link_elem *mle = (const void *)data; u8 fixed = sizeof(*mle); u8 common = 0; - bool check_common_len = false; + u8 common_len; u16 control; if (!data || len < fixed) @@ -855,7 +912,6 @@ static inline bool ieee80211_mle_size_ok(const u8 *data, size_t len) switch (u16_get_bits(control, IEEE80211_ML_CONTROL_TYPE)) { case IEEE80211_ML_CONTROL_TYPE_BASIC: common += sizeof(struct ieee80211_mle_basic_common_info); - check_common_len = true; if (control & IEEE80211_MLC_BASIC_PRES_LINK_ID) common += 1; if (control & IEEE80211_MLC_BASIC_PRES_BSS_PARAM_CH_CNT) @@ -870,14 +926,16 @@ static inline bool ieee80211_mle_size_ok(const u8 *data, size_t len) common += 1; if (control & IEEE80211_MLC_BASIC_PRES_EXT_MLD_CAPA_OP) common += 2; + if (control & IEEE80211_MLC_BASIC_PRES_ENH_CRIT_UPD) + common += 1; break; case IEEE80211_ML_CONTROL_TYPE_PREQ: common += sizeof(struct ieee80211_mle_preq_common_info); if (control & IEEE80211_MLC_PREQ_PRES_MLD_ID) common += 1; - check_common_len = true; break; case IEEE80211_ML_CONTROL_TYPE_RECONF: + common += 1; if (control & IEEE80211_MLC_RECONF_PRES_MLD_MAC_ADDR) common += ETH_ALEN; if (control & IEEE80211_MLC_RECONF_PRES_EML_CAPA) @@ -889,7 +947,6 @@ static inline bool ieee80211_mle_size_ok(const u8 *data, size_t len) break; case IEEE80211_ML_CONTROL_TYPE_TDLS: common += sizeof(struct ieee80211_mle_tdls_common_info); - check_common_len = true; break; case IEEE80211_ML_CONTROL_TYPE_PRIO_ACCESS: common = ETH_ALEN + 1; @@ -902,11 +959,9 @@ static inline bool ieee80211_mle_size_ok(const u8 *data, size_t len) if (len < fixed + common) return false; - if (!check_common_len) - return true; + common_len = mle->variable[0]; - /* if present, common length is the first octet there */ - return mle->variable[0] >= common; + return common_len >= common && common_len <= len - fixed; } /** @@ -946,6 +1001,8 @@ enum ieee80211_mle_subelems { #define IEEE80211_MLE_STA_CONTROL_NSTR_LINK_PAIR_PRESENT 0x0200 #define IEEE80211_MLE_STA_CONTROL_NSTR_BITMAP_SIZE 0x0400 #define IEEE80211_MLE_STA_CONTROL_BSS_PARAM_CHANGE_CNT_PRESENT 0x0800 +#define IEEE80211_MLE_STA_CONTROL_ENH_CRIT_UPD_PRESENT 0x1000 +#define IEEE80211_MLE_STA_CONTROL_AP_CONDUCTED_TX_PWR_PRESENT 0x2000 struct ieee80211_mle_per_sta_profile { __le16 control; @@ -991,6 +1048,12 @@ static inline bool ieee80211_mle_basic_sta_prof_size_ok(const u8 *data, if (control & IEEE80211_MLE_STA_CONTROL_BSS_PARAM_CHANGE_CNT_PRESENT) info_len += 1; + if (control & IEEE80211_MLE_STA_CONTROL_ENH_CRIT_UPD_PRESENT) + info_len += 1; + + if (control & IEEE80211_MLE_STA_CONTROL_AP_CONDUCTED_TX_PWR_PRESENT) + info_len += 1; + return prof->sta_info_len >= info_len && fixed + prof->sta_info_len - 1 <= len; } @@ -1031,17 +1094,59 @@ ieee80211_mle_basic_sta_prof_bss_param_ch_cnt(const struct ieee80211_mle_per_sta return *pos; } +/** + * ieee80211_mle_basic_sta_prof_enh_crit_upd - get per-STA profile enhanced + * critical updates field + * @prof: the per-STA profile, having been checked with + * ieee80211_mle_basic_sta_prof_size_ok() for the correct length + * + * Return: The enhanced critical updates field if present, %NULL otherwise. + */ +static inline const struct ieee80211_enh_crit_upd * +ieee80211_mle_basic_sta_prof_enh_crit_upd(const struct ieee80211_mle_per_sta_profile *prof) +{ + u16 control = le16_to_cpu(prof->control); + const u8 *pos = prof->variable; + + if (!(control & IEEE80211_MLE_STA_CONTROL_ENH_CRIT_UPD_PRESENT)) + return NULL; + + if (control & IEEE80211_MLE_STA_CONTROL_STA_MAC_ADDR_PRESENT) + pos += 6; + if (control & IEEE80211_MLE_STA_CONTROL_BEACON_INT_PRESENT) + pos += 2; + if (control & IEEE80211_MLE_STA_CONTROL_TSF_OFFS_PRESENT) + pos += 8; + if (control & IEEE80211_MLE_STA_CONTROL_DTIM_INFO_PRESENT) + pos += 2; + if (control & IEEE80211_MLE_STA_CONTROL_COMPLETE_PROFILE && + control & IEEE80211_MLE_STA_CONTROL_NSTR_LINK_PAIR_PRESENT) { + if (control & IEEE80211_MLE_STA_CONTROL_NSTR_BITMAP_SIZE) + pos += 2; + else + pos += 1; + } + if (control & IEEE80211_MLE_STA_CONTROL_BSS_PARAM_CHANGE_CNT_PRESENT) + pos += 1; + + return (const void *)pos; +} + #define IEEE80211_MLE_STA_RECONF_CONTROL_LINK_ID 0x000f #define IEEE80211_MLE_STA_RECONF_CONTROL_COMPLETE_PROFILE 0x0010 #define IEEE80211_MLE_STA_RECONF_CONTROL_STA_MAC_ADDR_PRESENT 0x0020 #define IEEE80211_MLE_STA_RECONF_CONTROL_AP_REM_TIMER_PRESENT 0x0040 -#define IEEE80211_MLE_STA_RECONF_CONTROL_OPERATION_TYPE 0x0780 -#define IEEE80211_MLE_STA_RECONF_CONTROL_OPERATION_TYPE_AP_REM 0 -#define IEEE80211_MLE_STA_RECONF_CONTROL_OPERATION_TYPE_OP_PARAM_UPDATE 1 -#define IEEE80211_MLE_STA_RECONF_CONTROL_OPERATION_TYPE_ADD_LINK 2 -#define IEEE80211_MLE_STA_RECONF_CONTROL_OPERATION_TYPE_DEL_LINK 3 -#define IEEE80211_MLE_STA_RECONF_CONTROL_OPERATION_TYPE_NSTR_STATUS 4 -#define IEEE80211_MLE_STA_RECONF_CONTROL_OPERATION_PARAMS_PRESENT 0x0800 +#define IEEE80211_MLE_STA_RECONF_CONTROL_OPERATION_TYPE 0x0780 +#define IEEE80211_MLE_STA_RECONF_CONTROL_OPERATION_TYPE_AP_REM 0 +#define IEEE80211_MLE_STA_RECONF_CONTROL_OPERATION_TYPE_OP_PARAM_UPDATE 1 +#define IEEE80211_MLE_STA_RECONF_CONTROL_OPERATION_TYPE_ADD_LINK 2 +#define IEEE80211_MLE_STA_RECONF_CONTROL_OPERATION_TYPE_DEL_LINK 3 +#define IEEE80211_MLE_STA_RECONF_CONTROL_OPERATION_TYPE_NSTR_STATUS 4 +#define IEEE80211_MLE_STA_RECONF_CONTROL_OPERATION_TYPE_UHR_OMP_UPD 5 +#define IEEE80211_MLE_STA_RECONF_CONTROL_OPERATION_PARAMS_PRESENT 0x0800 +#define IEEE80211_MLE_STA_RECONF_CONTROL_OPERATION_NSTR_BMAP_SIZE 0x1000 +#define IEEE80211_MLE_STA_RECONF_CONTROL_OPERATION_NSTR_IND_BMAP_PRES 0x2000 +#define IEEE80211_MLE_STA_RECONF_CONTROL_OPERATION_DSO_INFO_PRESENT 0x4000 /** * ieee80211_mle_reconf_sta_prof_size_ok - validate reconfiguration multi-link @@ -1121,14 +1226,20 @@ static inline bool ieee80211_tid_to_link_map_size_ok(const u8 *data, size_t len) static inline u32 ieee80211_emlsr_pad_delay_in_us(u16 eml_cap) { + u32 emlsr_supp = + u16_get_bits(eml_cap, IEEE80211_EML_CAP_EMLSR_SUPP); + + if (!emlsr_supp) + return 0; + /* IEEE Std 802.11be-2024 Table 9-417i—Encoding of the EMLSR * Padding Delay subfield. */ u32 pad_delay = u16_get_bits(eml_cap, - IEEE80211_EML_CAP_EMLSR_PADDING_DELAY); + IEEE80211_EML_CAP_EML_PADDING_DELAY); if (!pad_delay || - pad_delay > IEEE80211_EML_CAP_EMLSR_PADDING_DELAY_256US) + pad_delay > IEEE80211_EML_CAP_EML_PADDING_DELAY_256US) return 0; return 32 * (1 << (pad_delay - 1)); @@ -1145,12 +1256,18 @@ static inline u32 ieee80211_emlsr_pad_delay_in_us(u16 eml_cap) static inline u32 ieee80211_emlsr_trans_delay_in_us(u16 eml_cap) { + u32 emlsr_supp = + u16_get_bits(eml_cap, IEEE80211_EML_CAP_EMLSR_SUPP); + + if (!emlsr_supp) + return 0; + /* IEEE Std 802.11be-2024 Table 9-417j—Encoding of the EMLSR * Transition Delay subfield. */ u32 trans_delay = u16_get_bits(eml_cap, - IEEE80211_EML_CAP_EMLSR_TRANSITION_DELAY); + IEEE80211_EML_CAP_EML_TRANSITION_DELAY); /* invalid values also just use 0 */ if (!trans_delay || @@ -1161,11 +1278,73 @@ static inline u32 ieee80211_emlsr_trans_delay_in_us(u16 eml_cap) } /** - * ieee80211_eml_trans_timeout_in_us - Fetch the EMLSR Transition + * ieee80211_emlmr_pad_delay_in_us - Fetch the EMLMR Padding delay + * in microseconds + * @eml_cap: EML capabilities field value from common info field of + * the Multi-link element + * Return: the EMLMR Padding delay (in microseconds) encoded in the + * EML Capabilities field + */ + +static inline u32 ieee80211_emlmr_pad_delay_in_us(u16 eml_cap) +{ + u32 emlmr_supp = + u16_get_bits(eml_cap, IEEE80211_EML_CAP_EMLMR_SUPPORT); + + if (!emlmr_supp) + return 0; + + /* IEEE Std 802.11be-2024 Table 9-417k—Encoding of the EMLMR + * Padding Delay subfield. + */ + u32 pad_delay = u16_get_bits(eml_cap, + IEEE80211_EML_CAP_EML_PADDING_DELAY); + + if (!pad_delay || + pad_delay > IEEE80211_EML_CAP_EML_PADDING_DELAY_256US) + return 0; + + return 32 * (1 << (pad_delay - 1)); +} + +/** + * ieee80211_emlmr_trans_delay_in_us - Fetch the EMLMR Transition + * delay in microseconds + * @eml_cap: EML capabilities field value from common info field of + * the Multi-link element + * Return: the EMLMR Transition delay (in microseconds) encoded in the + * EML Capabilities field + */ + +static inline u32 ieee80211_emlmr_trans_delay_in_us(u16 eml_cap) +{ + u32 emlmr_supp = + u16_get_bits(eml_cap, IEEE80211_EML_CAP_EMLMR_SUPPORT); + + if (!emlmr_supp) + return 0; + + /* IEEE Std 802.11be-2024 Table 9-417l—Encoding of the EMLMR + * Transition Delay subfield. + */ + u32 trans_delay = + u16_get_bits(eml_cap, + IEEE80211_EML_CAP_EML_TRANSITION_DELAY); + + /* invalid values also just use 0 */ + if (!trans_delay || + trans_delay > IEEE80211_EML_CAP_EMLMR_TRANSITION_DELAY_256US) + return 0; + + return 32 * (1 << (trans_delay - 1)); +} + +/** + * ieee80211_eml_trans_timeout_in_us - Fetch the EML Transition * timeout value in microseconds * @eml_cap: EML capabilities field value from common info field of * the Multi-link element - * Return: the EMLSR Transition timeout (in microseconds) encoded in + * Return: the EML Transition timeout (in microseconds) encoded in * the EML Capabilities field */ @@ -1178,7 +1357,7 @@ static inline u32 ieee80211_eml_trans_timeout_in_us(u16 eml_cap) IEEE80211_EML_CAP_TRANSITION_TIMEOUT); /* invalid values also just use 0 */ - if (!timeout || timeout > IEEE80211_EML_CAP_TRANSITION_TIMEOUT_128TU) + if (!timeout || timeout > IEEE80211_EML_CAP_TRANSITION_TIMEOUT_64TU) return 0; return 128 * (1 << (timeout - 1)); diff --git a/include/linux/ieee80211-ht.h b/include/linux/ieee80211-ht.h index 21bbf470540f..7612b72f9c7c 100644 --- a/include/linux/ieee80211-ht.h +++ b/include/linux/ieee80211-ht.h @@ -281,6 +281,9 @@ enum ieee80211_back_actioncode { WLAN_ACTION_ADDBA_REQ = 0, WLAN_ACTION_ADDBA_RESP = 1, WLAN_ACTION_DELBA = 2, + WLAN_ACTION_NDP_ADDBA_REQ = 128, + WLAN_ACTION_NDP_ADDBA_RESP = 129, + WLAN_ACTION_NDP_DELBA = 130, }; /* BACK (block-ack) parties */ diff --git a/include/linux/ieee80211-mesh.h b/include/linux/ieee80211-mesh.h index 4b829bcb38b6..9e548b9173df 100644 --- a/include/linux/ieee80211-mesh.h +++ b/include/linux/ieee80211-mesh.h @@ -28,12 +28,73 @@ struct ieee80211s_hdr { u8 eaddr2[ETH_ALEN]; } __packed __aligned(2); +struct ieee80211_mesh_hwmp_preq_target { + u8 flags; + u8 addr[ETH_ALEN]; + __le32 sn; +} __packed; + +struct ieee80211_mesh_hwmp_preq_top { + u8 flags; + u8 hopcount; + u8 ttl; + __le32 preq_id; + u8 orig_addr[ETH_ALEN]; + __le32 orig_sn; + + /* optional AE, lifetime, metric, target */ + u8 variable[]; +} __packed; + +struct ieee80211_mesh_hwmp_preq_bottom { + __le32 lifetime; + __le32 metric; + u8 target_count; + struct ieee80211_mesh_hwmp_preq_target targets[]; +} __packed; + +struct ieee80211_mesh_hwmp_prep_top { + u8 flags; + u8 hopcount; + u8 ttl; + u8 target_addr[ETH_ALEN]; + __le32 target_sn; + + /* optional Target External Address */ + u8 variable[]; +} __packed; + +struct ieee80211_mesh_hwmp_prep_bottom { + __le32 lifetime; + __le32 metric; + u8 orig_addr[ETH_ALEN]; + __le32 orig_sn; +} __packed; + +struct ieee80211_mesh_hwmp_perr_dst { + u8 flags; + u8 addr[ETH_ALEN]; + __le32 sn; + /* optional Destination External Address */ + u8 variable[]; +} __packed; + +struct ieee80211_mesh_hwmp_perr { + u8 ttl; + u8 number_of_dst; + /* Destinations */ + u8 variable[]; +} __packed; + /* Mesh flags */ #define MESH_FLAGS_AE_A4 0x1 #define MESH_FLAGS_AE_A5_A6 0x2 #define MESH_FLAGS_AE 0x3 #define MESH_FLAGS_PS_DEEP 0x4 +/* HWMP IE processing macros */ +#define AE_F (1<<6) + /** * enum ieee80211_preq_flags - mesh PREQ element flags * @@ -227,4 +288,155 @@ enum ieee80211_root_mode_identifier { IEEE80211_PROACTIVE_RANN = 4, }; +static inline bool ieee80211_mesh_preq_prep_ae_enabled(const u8 *ie) +{ + return ie[0] & AE_F; +} + +static inline struct ieee80211_mesh_hwmp_preq_bottom * +ieee80211_mesh_hwmp_preq_get_bottom(const u8 *ie) +{ + struct ieee80211_mesh_hwmp_preq_top *top = (void *)ie; + + return (void *)&top->variable[ + ieee80211_mesh_preq_prep_ae_enabled(ie) ? ETH_ALEN : 0]; +} + +static inline struct ieee80211_mesh_hwmp_prep_bottom * +ieee80211_mesh_hwmp_prep_get_bottom(const u8 *ie) +{ + struct ieee80211_mesh_hwmp_prep_top *top = (void *)ie; + + return (void *)&top->variable[ + ieee80211_mesh_preq_prep_ae_enabled(ie) ? ETH_ALEN : 0]; +} + +static inline struct ieee80211_mesh_hwmp_perr_dst * +ieee80211_mesh_hwmp_perr_get_dst(const u8 *ie, u8 dst_idx) +{ + struct ieee80211_mesh_hwmp_perr *perr_ie = (void *)ie; + struct ieee80211_mesh_hwmp_perr_dst *dst; + u8 *pos = perr_ie->variable; + int i; + + for (i = 0; i < dst_idx + 1; i++) { + dst = (void *)pos; + pos += sizeof(struct ieee80211_mesh_hwmp_perr_dst) + + ((dst->flags & AE_F) ? ETH_ALEN : 0) + /* Destination External Address */ + + 2 /* Reason Code */; + } + + return dst; +} + +static inline u8 * +ieee80211_mesh_hwmp_perr_get_addr(const u8 *ie, u8 dst_idx) +{ + struct ieee80211_mesh_hwmp_perr_dst *dst = + ieee80211_mesh_hwmp_perr_get_dst(ie, dst_idx); + + return dst->addr; +} + +static inline u32 +ieee80211_mesh_hwmp_perr_get_sn(const u8 *ie, u8 dst_idx) +{ + struct ieee80211_mesh_hwmp_perr_dst *dst = + ieee80211_mesh_hwmp_perr_get_dst(ie, dst_idx); + + return le32_to_cpu(dst->sn); +} + +static inline u16 +ieee80211_mesh_hwmp_perr_get_rcode(const u8 *ie, u8 dst_idx) +{ + struct ieee80211_mesh_hwmp_perr_dst *dst = + ieee80211_mesh_hwmp_perr_get_dst(ie, dst_idx); + + return get_unaligned_le16(&dst->variable[ + (dst->flags & AE_F) ? ETH_ALEN : 0]); +} + +/* IEEE Std 802.11-2016 9.4.2.113 PREQ element */ +static inline bool ieee80211_mesh_preq_size_ok(const u8 *pos, u8 elen) +{ + struct ieee80211_mesh_hwmp_preq_bottom *preq_elem_bottom; + u8 target_count; + int needed; + + /* Check if the element contains flags */ + needed = sizeof(struct ieee80211_mesh_hwmp_preq_top); + if (elen < needed) + return false; + + /* Check if the element contains target_count */ + needed += (ieee80211_mesh_preq_prep_ae_enabled(pos) ? ETH_ALEN : 0) + /* Originator External Address */ + + sizeof(struct ieee80211_mesh_hwmp_preq_bottom); + if (elen < needed) + return false; + + preq_elem_bottom = ieee80211_mesh_hwmp_preq_get_bottom(pos); + target_count = preq_elem_bottom->target_count; + /* IEEE Std 802.11-2016 Table 14-10 to 14-16 */ + if (target_count < 1) + return false; + + needed += target_count * sizeof(struct ieee80211_mesh_hwmp_preq_target); + return elen == needed; +} + +/* IEEE Std 802.11-2016 9.4.2.114 PREP element */ +static inline bool ieee80211_mesh_prep_size_ok(const u8 *pos, u8 elen) +{ + u8 needed; + + /* Check if the element contains flags */ + needed = sizeof(struct ieee80211_mesh_hwmp_prep_top); + if (elen < needed) + return false; + + needed += (ieee80211_mesh_preq_prep_ae_enabled(pos) ? ETH_ALEN : 0) + /* Target External Address */ + + sizeof(struct ieee80211_mesh_hwmp_prep_bottom); + return elen == needed; +} + +/* IEEE Std 802.11-2016 9.4.2.115 PERR element */ +static inline bool ieee80211_mesh_perr_size_ok(const u8 *pos, u8 elen) +{ + struct ieee80211_mesh_hwmp_perr *perr_elem = (void *)pos; + const u8 *start = pos; + u8 number_of_dst; + int needed; + int i; + + needed = sizeof(struct ieee80211_mesh_hwmp_perr); + + /* Check if the element contains number of dst */ + if (elen < needed) + return false; + + pos += sizeof(struct ieee80211_mesh_hwmp_perr); + number_of_dst = perr_elem->number_of_dst; + + for (i = 0; i < number_of_dst; i++) { + struct ieee80211_mesh_hwmp_perr_dst *dst = (void *)pos; + u8 dst_len = sizeof(struct ieee80211_mesh_hwmp_perr_dst); + + /* Check if the element contains flags */ + if (elen < pos - start + dst_len) + return false; + + dst_len += ((dst->flags & AE_F) ? ETH_ALEN : 0) + /* Destination External Address */ + + 2 /* Reason Code */; + needed += dst_len; + pos += dst_len; + } + + return elen == needed; +} + #endif /* LINUX_IEEE80211_MESH_H */ diff --git a/include/linux/ieee80211-nan.h b/include/linux/ieee80211-nan.h index d07959bf8a90..455033955e54 100644 --- a/include/linux/ieee80211-nan.h +++ b/include/linux/ieee80211-nan.h @@ -9,7 +9,7 @@ * Copyright (c) 2006, Michael Wu <flamingice@sourmilk.net> * Copyright (c) 2013 - 2014 Intel Mobile Communications GmbH * Copyright (c) 2016 - 2017 Intel Deutschland GmbH - * Copyright (c) 2018 - 2025 Intel Corporation + * Copyright (c) 2018 - 2026 Intel Corporation */ #ifndef LINUX_IEEE80211_NAN_H @@ -23,6 +23,11 @@ #define NAN_OP_MODE_160MHZ 0x04 #define NAN_OP_MODE_PNDL_SUPPRTED 0x08 +#define NAN_DEV_CAPA_NUM_TX_ANT_POS 0 +#define NAN_DEV_CAPA_NUM_TX_ANT_MASK 0x0f +#define NAN_DEV_CAPA_NUM_RX_ANT_POS 4 +#define NAN_DEV_CAPA_NUM_RX_ANT_MASK 0xf0 + /* NAN Device capabilities, as defined in Wi-Fi Aware (TM) specification * Table 79 */ @@ -32,4 +37,41 @@ #define NAN_DEV_CAPA_NDPE_SUPPORTED 0x08 #define NAN_DEV_CAPA_S3_SUPPORTED 0x10 +/* NAN attributes, as defined in Wi-Fi Aware (TM) specification 4.0 Table 42 */ +#define NAN_ATTR_MASTER_INDICATION 0x00 +#define NAN_ATTR_CLUSTER_INFO 0x01 + +struct ieee80211_nan_attr { + u8 attr; + __le16 length; + u8 data[]; +} __packed; + +struct ieee80211_nan_master_indication { + u8 master_pref; + u8 random_factor; +} __packed; + +struct ieee80211_nan_anchor_master_info { + union { + __le64 master_rank; + struct { + u8 master_addr[ETH_ALEN]; + u8 random_factor; + u8 master_pref; + } __packed; + } __packed; + u8 hop_count; + __le32 ambtt; +} __packed; + +#define for_each_nan_attr(_attr, _data, _datalen) \ + for (_attr = (const struct ieee80211_nan_attr *)(_data); \ + (const u8 *)(_data) + (_datalen) - (const u8 *)_attr >= \ + (int)sizeof(*_attr) && \ + (const u8 *)(_data) + (_datalen) - (const u8 *)_attr >= \ + (int)sizeof(*_attr) + le16_to_cpu(_attr->length); \ + _attr = (const struct ieee80211_nan_attr *) \ + (_attr->data + le16_to_cpu(_attr->length))) + #endif /* LINUX_IEEE80211_NAN_H */ diff --git a/include/linux/ieee80211-s1g.h b/include/linux/ieee80211-s1g.h index 22dde4cbc1b0..3f9626ad3d97 100644 --- a/include/linux/ieee80211-s1g.h +++ b/include/linux/ieee80211-s1g.h @@ -556,7 +556,7 @@ static inline bool ieee80211_s1g_check_tim(const struct ieee80211_tim_ie *tim, */ err = ieee80211_s1g_find_target_block(&enc_blk, &target_aid, tim->virtual_map, - (const u8 *)tim + tim_len + 2); + (const u8 *)tim + tim_len); if (err) return false; diff --git a/include/linux/ieee80211-uhr.h b/include/linux/ieee80211-uhr.h index 132acced7d79..665d4b3a5b41 100644 --- a/include/linux/ieee80211-uhr.h +++ b/include/linux/ieee80211-uhr.h @@ -8,12 +8,20 @@ #define LINUX_IEEE80211_UHR_H #include <linux/types.h> +#include <linux/bitfield.h> #include <linux/if_ether.h> +#include "ieee80211-eht.h" #define IEEE80211_UHR_OPER_PARAMS_DPS_ENA 0x0001 #define IEEE80211_UHR_OPER_PARAMS_NPCA_ENA 0x0002 -#define IEEE80211_UHR_OPER_PARAMS_DBE_ENA 0x0004 -#define IEEE80211_UHR_OPER_PARAMS_PEDCA_ENA 0x0008 +#define IEEE80211_UHR_OPER_PARAMS_PEDCA_ENA 0x0004 +#define IEEE80211_UHR_OPER_PARAMS_DBE_ENA 0x0008 +#define IEEE80211_UHR_OPER_PARAMS_DBE_BW 0x0070 +#define IEEE80211_UHR_OPER_PARAMS_DUO_PRES 0x0080 +#define IEEE80211_UHR_OPER_PARAMS_DPS_PRES 0x0100 +#define IEEE80211_UHR_OPER_PARAMS_NPCA_PRES 0x0200 +#define IEEE80211_UHR_OPER_PARAMS_PEDCA_PRES 0x0400 +#define IEEE80211_UHR_OPER_PARAMS_DBE_PRES 0x0800 struct ieee80211_uhr_operation { __le16 params; @@ -29,13 +37,240 @@ struct ieee80211_uhr_operation { #define IEEE80211_UHR_NPCA_PARAMS_MOPLEN 0x00400000 #define IEEE80211_UHR_NPCA_PARAMS_DIS_SUBCH_BMAP_PRES 0x00800000 +/** + * struct ieee80211_uhr_npca_info - npca operation information + * + * This structure is the "NPCA Operation Parameters field format" of "UHR + * Operation Element" fields as described in P802.11bn_D1.3 + * subclause 9.4.2.353. See Figure 9-aa4. + * + * Refer to IEEE80211_UHR_NPCA* + * @params: + * NPCA Primary Channel - NPCA primary channel + * NPCA_Min Duration Threshold - Minimum duration of inter-BSS activity + * NPCA Switching Delay - + * Time needed by an NPCA AP to switch from the + * BSS primary channel to the NPCA primary channel + * in the unit of 4 µs. + * NPCA Switching Back Delay - + * Time to switch from the NPCA primary channel + * to the BSS primary channel in the unit of 4 µs. + * NPCA Initial QSRC - + * Initialize the EDCAF QSRC[AC] variables + * when an NPCA STA in the BSS + * switches to NPCA operation. + * NPCA MOPLEN - + * Indicates which conditions can be used to + * initiate an NPCA operation, + * 1 -> both PHYLEN NPCA operation and MOPLEN + * NPCA operation are + * permitted in the BSS + * 0 -> only PHYLEN NPCA operation is allowed in the BSS. + * NPCA Disabled Subchannel Bitmap Present - + * Indicates whether the NPCA Disabled Subchannel + * Bitmap field is present. A 1 in this field indicates that + * the NPCA Disabled Subchannel Bitmap field is present + * @dis_subch_bmap: + * A bit in the bitmap that lies within the BSS bandwidth is set + * to 1 to indicate that the corresponding 20 MHz subchannel is + * punctured and is set to 0 to indicate that the corresponding + * 20 MHz subchannel is not punctured. A bit in the bitmap that + * falls outside of the BSS bandwidth is reserved. This field is + * present when the value of the NPCA Disabled Subchannel Bitmap + * Field Present field is equal to 1, and not present, otherwise + */ struct ieee80211_uhr_npca_info { __le32 params; __le16 dis_subch_bmap[]; } __packed; -static inline bool ieee80211_uhr_oper_size_ok(const u8 *data, u8 len, - bool beacon) +#define IEEE80211_UHR_DPS_PADDING_DELAY 0x0000003F +#define IEEE80211_UHR_DPS_TRANSITION_DELAY 0x00003F00 +#define IEEE80211_UHR_DPS_ICF_REQUIRED 0x00010000 +#define IEEE80211_UHR_DPS_PARAMETERIZED_FLAG 0x00020000 +#define IEEE80211_UHR_DPS_LC_MODE_BW 0x001C0000 +#define IEEE80211_UHR_DPS_LC_MODE_NSS 0x01E00000 +#define IEEE80211_UHR_DPS_LC_MODE_MCS 0x1E000000 +#define IEEE80211_UHR_DPS_MOBILE_AP_DPS_STATIC_HCM 0x20000000 + +/** + * struct ieee80211_uhr_dps_info - DPS operation information + * + * This structure is the "DPS Operation Parameter field" of "UHR + * Operation Element" fields as described in P802.11bn_D1.3 + * subclause 9.4.1.87. See Figure 9-207u. + * + * Refer to IEEE80211_UHR_DPS* + * @params: + * DPS Padding Delay - + * Indicates the minimum MAC padding + * duration that is required by a DPS STA + * in an ICF to cause the STA to transition + * from the lower capability mode to the + * higher capability mode. The DPS Padding + * Delay field is in units of 4 µs. + * DPS Transition Delay - + * Indicates the amount of time required by a + * DPS STA to transition from the higher + * capability mode to the lower capability + * mode. The DPS Transition Delay field is in + * units of 4 µs. + * ICF Required - + * Indicates when the DPS assisting STA needs + * to transmit an ICF frame to the peer DPS STA + * before performing the frame exchanges with + * the peer DPS STA in a TXOP. + * 1 -> indicates that the transmission of the + * ICF frame to the peer DPS STA prior to + * any frame exchange is needed. + * 0 -> ICF transmission before the frame + * exchanges with the peer DPS STA is only + * needed if the frame exchange is performed + * in the HC mode. + * Parameterized Flag - + * 0 -> indicates that only 20 MHz, 1 SS, + * non-HT PPDU format with the data + * rate of 6, 12, and 24 Mb/s as the + * default mode are supported by the + * DPS STA in the LC mode + * 1 -> indicates that a bandwidth up to the + * bandwidth indicated in the LC Mode + * Bandwidth field, a number of spatial + * streams up to the NSS indicated in + * the LC Mode Nss field, and an MCS up + * to the MCS indicated in the LC Mode + * MCS fields are supported by the DPS + * STA in the LC mode as the + * parameterized mode. + * LC Mode Bandwidth - + * Indicates the maximum bandwidth supported + * by the STA in the LC mode. + * LC Mode NSS - + * Indicates the maximum number of the spatial + * streams supported by the STA in the LC mode. + * LC Mode MCS - + * Indicates the highest MCS supported by the STA + * in the LC mode. + * Mobile AP DPS Static HCM - + * 1 -> indicates that it will remain in the DPS high + * capability mode until the next TBTT on that + * link. + * 0 -> otherwise. + */ +struct ieee80211_uhr_dps_info { + __le32 params; +} __packed; + +#define IEEE80211_UHR_DBE_OPER_BANDWIDTH 0x07 +#define IEEE80211_UHR_DBE_OPER_DIS_SUBCHANNEL_BITMAP_PRES 0x08 + +/** + * enum ieee80211_uhr_dbe_oper_bw - DBE Operational Bandwidth + * + * Encoding for the DBE Operational Bandwidth field in the UHR Operation + * element (DBE Operation Parameters). + * + * @IEEE80211_UHR_DBE_OPER_BW_40: 40 MHz operational DBE bandwidth + * @IEEE80211_UHR_DBE_OPER_BW_80: 80 MHz operational DBE bandwidth + * @IEEE80211_UHR_DBE_OPER_BW_160: 160 MHz operational DBE bandwidth + * @IEEE80211_UHR_DBE_OPER_BW_320_1: 320-1 MHz operational DBE bandwidth + * @IEEE80211_UHR_DBE_OPER_BW_320_2: 320-2 MHz operational DBE bandwidth + */ +enum ieee80211_uhr_dbe_oper_bw { + IEEE80211_UHR_DBE_OPER_BW_40 = 1, + IEEE80211_UHR_DBE_OPER_BW_80 = 2, + IEEE80211_UHR_DBE_OPER_BW_160 = 3, + IEEE80211_UHR_DBE_OPER_BW_320_1 = 4, + IEEE80211_UHR_DBE_OPER_BW_320_2 = 5, +}; + +/** + * ieee80211_uhr_dbe_bw_mhz - get bandwidth in MHz from UHR DBE bandwidth + * @bw: UHR DBE bandwidth + * + * Return: the bandwidth in MHz, or -1 for invalid values + */ +static inline int ieee80211_uhr_dbe_bw_mhz(enum ieee80211_uhr_dbe_oper_bw bw) +{ + switch (bw) { + case IEEE80211_UHR_DBE_OPER_BW_40: + return 40; + case IEEE80211_UHR_DBE_OPER_BW_80: + return 80; + case IEEE80211_UHR_DBE_OPER_BW_160: + return 160; + case IEEE80211_UHR_DBE_OPER_BW_320_1: + case IEEE80211_UHR_DBE_OPER_BW_320_2: + return 320; + default: + return -1; + } +} + +/** + * struct ieee80211_uhr_dbe_info - DBE operation information + * + * This structure is the "DBE Operation Parameters field" of + * "UHR Operation Element" fields as described in P802.11bn_D1.3 + * subclause 9.4.2.353. See Figure 9-aa6. + * + * Refer to IEEE80211_UHR_DBE_OPER* + * @params: + * B0-B2 - DBE Operational Bandwidth field, see + * "enum ieee80211_uhr_dbe_oper_bw" for values. + * Value 0 is reserved. + * Value 1 indicates 40 MHz operational DBE bandwidth. + * Value 2 indicates 80 MHz operational DBE bandwidth. + * Value 3 indicates 160 MHz operational DBE bandwidth. + * Value 4 indicates 320-1 MHz operational DBE bandwidth. + * Value 5 indicates 320-2 MHz operational DBE bandwidth. + * Values 6 to 7 are reserved. + * B3 - DBE Disabled Subchannel Bitmap Present. + * @dis_subch_bmap: DBE Disabled Subchannel Bitmap field is set to indicate + * disabled 20 MHz subchannels within the DBE Bandwidth. + */ +struct ieee80211_uhr_dbe_info { + u8 params; + __le16 dis_subch_bmap[]; +} __packed; + +#define IEEE80211_UHR_P_EDCA_ECWMIN 0x0F +#define IEEE80211_UHR_P_EDCA_ECWMAX 0xF0 +#define IEEE80211_UHR_P_EDCA_AIFSN 0x000F +#define IEEE80211_UHR_P_EDCA_CW_DS 0x0030 +#define IEEE80211_UHR_P_EDCA_PSRC_THRESHOLD 0x01C0 +#define IEEE80211_UHR_P_EDCA_QSRC_THRESHOLD 0x0600 + +/** + * struct ieee80211_uhr_p_edca_info - P-EDCA operation information + * + * This structure is the "P-EDCA Operation Parameters field" of + * "UHR Operation Element" fields as described in P802.11bn_D1.3 + * subclause 9.4.2.353. See Figure 9-aa5. + * + * Refer to IEEE80211_UHR_P_EDCA* + * @p_edca_ec: P-EDCA ECWmin and ECWmax. + * These fields indicate the CWmin and CWmax values used by a + * P-EDCA STA during P-EDCA contention. + * @params: AIFSN, CW DS, PSRC threshold, and QSRC threshold. + * - The AIFSN field indicates the AIFSN value used by a P-EDCA STA + * during P-EDCA contention. + * - The CW DS field indicates the value used for randomization of the + * transmission slot of the DS-CTS frame. The value 3 is reserved. + * The value 0 indicates that randomization is not enabled. + * - The P-EDCA PSRC threshold field indicates the maximum number of + * allowed consecutive DS-CTS transmissions. The value 0 and values + * greater than 4 are reserved. + * - The P-EDCA QSRC threshold field indicates the value of the + * QSRC[AC_VO] counter required to start P-EDCA contention. The + * value 0 is reserved. + */ +struct ieee80211_uhr_p_edca_info { + u8 p_edca_ec; + __le16 params; +} __packed; + +static inline bool ieee80211_uhr_oper_size_ok(const u8 *data, u8 len) { const struct ieee80211_uhr_operation *oper = (const void *)data; u8 needed = sizeof(*oper); @@ -43,41 +278,73 @@ static inline bool ieee80211_uhr_oper_size_ok(const u8 *data, u8 len, if (len < needed) return false; - /* nothing else present in beacons */ - if (beacon) - return true; - - /* FIXME: DPS, DBE, P-EDCA (consider order, also relative to NPCA) */ + /* DPS Operation Parameters (fixed 4 bytes) */ + if (oper->params & cpu_to_le16(IEEE80211_UHR_OPER_PARAMS_DPS_PRES)) { + needed += sizeof(struct ieee80211_uhr_dps_info); + if (len < needed) + return false; + } - if (oper->params & cpu_to_le16(IEEE80211_UHR_OPER_PARAMS_NPCA_ENA)) { + /* NPCA Operation Parameters (fixed 4 bytes + optional 2 bytes) */ + if (oper->params & cpu_to_le16(IEEE80211_UHR_OPER_PARAMS_NPCA_PRES)) { const struct ieee80211_uhr_npca_info *npca = - (const void *)oper->variable; + (const void *)(data + needed); needed += sizeof(*npca); - if (len < needed) return false; - if (npca->params & cpu_to_le32(IEEE80211_UHR_NPCA_PARAMS_DIS_SUBCH_BMAP_PRES)) + if (npca->params & + cpu_to_le32(IEEE80211_UHR_NPCA_PARAMS_DIS_SUBCH_BMAP_PRES)) { needed += sizeof(npca->dis_subch_bmap[0]); + if (len < needed) + return false; + } + } + + /* P-EDCA Operation Parameters (fixed 3 bytes) */ + if (oper->params & cpu_to_le16(IEEE80211_UHR_OPER_PARAMS_PEDCA_PRES)) { + needed += sizeof(struct ieee80211_uhr_p_edca_info); + if (len < needed) + return false; + } + + /* DBE Operation Parameters (fixed 1 byte + optional 2 bytes) */ + if (oper->params & cpu_to_le16(IEEE80211_UHR_OPER_PARAMS_DBE_PRES)) { + const struct ieee80211_uhr_dbe_info *dbe = + (const void *)(data + needed); + + needed += sizeof(*dbe); + if (len < needed) + return false; + + if (dbe->params & + IEEE80211_UHR_DBE_OPER_DIS_SUBCHANNEL_BITMAP_PRES) { + needed += sizeof(dbe->dis_subch_bmap[0]); + if (len < needed) + return false; + } } return len >= needed; } -/* - * Note: cannot call this on the element coming from a beacon, - * must ensure ieee80211_uhr_oper_size_ok(..., false) first - */ +/* Note: must ensure ieee80211_uhr_oper_size_ok(...) first */ static inline const struct ieee80211_uhr_npca_info * ieee80211_uhr_npca_info(const struct ieee80211_uhr_operation *oper) { + const u8 *pos = oper->variable; + + if (!(oper->params & cpu_to_le16(IEEE80211_UHR_OPER_PARAMS_NPCA_PRES))) + return NULL; + if (!(oper->params & cpu_to_le16(IEEE80211_UHR_OPER_PARAMS_NPCA_ENA))) return NULL; - /* FIXME: DPS */ + if (oper->params & cpu_to_le16(IEEE80211_UHR_OPER_PARAMS_DPS_PRES)) + pos += sizeof(struct ieee80211_uhr_dps_info); - return (const void *)oper->variable; + return (const void *)pos; } static inline const __le16 * @@ -93,6 +360,35 @@ ieee80211_uhr_npca_dis_subch_bitmap(const struct ieee80211_uhr_operation *oper) return npca->dis_subch_bmap; } +/* Note: must ensure ieee80211_uhr_oper_size_ok(...) first */ +static inline const struct ieee80211_uhr_dbe_info * +ieee80211_uhr_oper_dbe_info(const struct ieee80211_uhr_operation *oper) +{ + const u8 *pos = oper->variable; + + if (!(oper->params & cpu_to_le16(IEEE80211_UHR_OPER_PARAMS_DBE_PRES))) + return NULL; + + if (!(oper->params & cpu_to_le16(IEEE80211_UHR_OPER_PARAMS_DBE_ENA))) + return NULL; + + if (oper->params & cpu_to_le16(IEEE80211_UHR_OPER_PARAMS_DPS_PRES)) + pos += sizeof(struct ieee80211_uhr_dps_info); + + if (oper->params & cpu_to_le16(IEEE80211_UHR_OPER_PARAMS_NPCA_PRES)) { + const struct ieee80211_uhr_npca_info *npca = (const void *)pos; + + pos += sizeof(*npca); + if (npca->params & cpu_to_le32(IEEE80211_UHR_NPCA_PARAMS_DIS_SUBCH_BMAP_PRES)) + pos += sizeof(npca->dis_subch_bmap[0]); + } + + if (oper->params & cpu_to_le16(IEEE80211_UHR_OPER_PARAMS_PEDCA_PRES)) + pos += sizeof(struct ieee80211_uhr_p_edca_info); + + return (const void *)pos; +} + #define IEEE80211_UHR_MAC_CAP0_DPS_SUPP 0x01 #define IEEE80211_UHR_MAC_CAP0_DPS_ASSIST_SUPP 0x02 #define IEEE80211_UHR_MAC_CAP0_DPS_AP_STATIC_HCM_SUPP 0x04 @@ -131,34 +427,81 @@ ieee80211_uhr_npca_dis_subch_bitmap(const struct ieee80211_uhr_operation *oper) #define IEEE80211_UHR_MAC_CAP_DBE_EHT_MCS_MAP_160_PRES 0x08 #define IEEE80211_UHR_MAC_CAP_DBE_EHT_MCS_MAP_320_PRES 0x10 -struct ieee80211_uhr_cap_mac { - u8 mac_cap[5]; +struct ieee80211_uhr_cap_dbe { + u8 cap; + /* present 0, 1 or 2 times depending on _PRES bits */ + struct ieee80211_eht_mcs_nss_supp_bw eht_mcs_map[]; } __packed; -struct ieee80211_uhr_cap { - struct ieee80211_uhr_cap_mac mac; - /* DBE, PHY capabilities */ - u8 variable[]; +/** + * enum ieee80211_uhr_dbe_max_supported_bw - DBE Maximum Supported Bandwidth + * + * As per spec P802.11bn_D1.3 "Table 9-bb5—Encoding of the DBE Maximum + * Supported Bandwidth field". + * + * @IEEE80211_UHR_DBE_MAX_BW_40: Indicates 40 MHz DBE max supported bw + * @IEEE80211_UHR_DBE_MAX_BW_80: Indicates 80 MHz DBE max supported bw + * @IEEE80211_UHR_DBE_MAX_BW_160: Indicates 160 MHz DBE max supported bw + * @IEEE80211_UHR_DBE_MAX_BW_320: Indicates 320 MHz DBE max supported bw + */ +enum ieee80211_uhr_dbe_max_supported_bw { + IEEE80211_UHR_DBE_MAX_BW_40 = 1, + IEEE80211_UHR_DBE_MAX_BW_80 = 2, + IEEE80211_UHR_DBE_MAX_BW_160 = 3, + IEEE80211_UHR_DBE_MAX_BW_320 = 4, +}; + +struct ieee80211_uhr_cap_mac { + u8 mac_cap[6]; } __packed; -#define IEEE80211_UHR_PHY_CAP_MAX_NSS_RX_SND_NDP_LE80 0x01 -#define IEEE80211_UHR_PHY_CAP_MAX_NSS_RX_DL_MU_LE80 0x02 -#define IEEE80211_UHR_PHY_CAP_MAX_NSS_RX_SND_NDP_160 0x04 -#define IEEE80211_UHR_PHY_CAP_MAX_NSS_RX_DL_MU_160 0x08 -#define IEEE80211_UHR_PHY_CAP_MAX_NSS_RX_SND_NDP_320 0x10 -#define IEEE80211_UHR_PHY_CAP_MAX_NSS_RX_DL_MU_320 0x20 -#define IEEE80211_UHR_PHY_CAP_ELR_RX 0x40 -#define IEEE80211_UHR_PHY_CAP_ELR_TX 0x80 +#define IEEE80211_UHR_PHY_CAP_MAX_NSS_RX_SND_NDP_LE80 0x00000001 +#define IEEE80211_UHR_PHY_CAP_MAX_NSS_RX_DL_MU_LE80 0x00000002 +#define IEEE80211_UHR_PHY_CAP_MAX_NSS_RX_SND_NDP_160 0x00000004 +#define IEEE80211_UHR_PHY_CAP_MAX_NSS_RX_DL_MU_160 0x00000008 +#define IEEE80211_UHR_PHY_CAP_MAX_NSS_RX_SND_NDP_320 0x00000010 +#define IEEE80211_UHR_PHY_CAP_MAX_NSS_RX_DL_MU_320 0x00000020 +#define IEEE80211_UHR_PHY_CAP_ELR_TX 0x00000040 +#define IEEE80211_UHR_PHY_CAP_ELR_RX 0x00000080 +#define IEEE80211_UHR_PHY_CAP_PART_BW_DL_MUMIMO 0x00000100 +#define IEEE80211_UHR_PHY_CAP_PART_BW_UL_MUMIMO 0x00000200 +#define IEEE80211_UHR_PHY_CAP_MCS15 0x00000400 +#define IEEE80211_UHR_PHY_CAP_2XLDPC_TX 0x00000800 +#define IEEE80211_UHR_PHY_CAP_2XLDPC_RX 0x00001000 +#define IEEE80211_UHR_PHY_CAP_UEQM_TX_MAX_NSS 0x00006000 +#define IEEE80211_UHR_PHY_CAP_UEQM_RX_MAX_NSS 0x00018000 +#define IEEE80211_UHR_PHY_CAP_CO_BF_JOINT_SOUNDING 0x00040000 +#define IEEE80211_UHR_PHY_CAP_IM_TX 0x00080000 +#define IEEE80211_UHR_PHY_CAP_IM_RX 0x00100000 +#define IEEE80211_UHR_PHY_CAP_CO_SR_MODE_1 0x00200000 +#define IEEE80211_UHR_PHY_CAP_CO_SR_MODE_2 0x00400000 +#define IEEE80211_UHR_PHY_CAP_DRU_DBW_20_IN_PBW_20 0x00800000 +#define IEEE80211_UHR_PHY_CAP_DRU_DBW_40_IN_PBW_40 0x01000000 +#define IEEE80211_UHR_PHY_CAP_DRU_DBW_80_IN_PBW_80 0x02000000 +#define IEEE80211_UHR_PHY_CAP_DRU_DBW_80_IN_PBW_160 0x04000000 +#define IEEE80211_UHR_PHY_CAP_DRU_DBW_80_IN_PBW_320 0x08000000 +#define IEEE80211_UHR_PHY_CAP_DRU_DBW_20_IN_PBW_GE80 0x10000000 +#define IEEE80211_UHR_PHY_CAP_DRU_DBW_40_IN_PBW_GE80 0x20000000 +#define IEEE80211_UHR_PHY_CAP_DRU_DBW_60_IN_PBW_GE80 0x40000000 +#define IEEE80211_UHR_PHY_CAP_DRU_RRU_HYBRID_MODE 0x80000000 struct ieee80211_uhr_cap_phy { - u8 cap; + __le32 cap; + u8 reserved; +} __packed; + +struct ieee80211_uhr_cap { + struct ieee80211_uhr_cap_mac mac; + struct ieee80211_uhr_cap_phy phy; + /* optional DBE capabilities */ + u8 variable[]; } __packed; static inline bool ieee80211_uhr_capa_size_ok(const u8 *data, u8 len, bool from_ap) { const struct ieee80211_uhr_cap *cap = (const void *)data; - size_t needed = sizeof(*cap) + sizeof(struct ieee80211_uhr_cap_phy); + size_t needed = sizeof(*cap); if (len < needed) return false; @@ -168,42 +511,60 @@ static inline bool ieee80211_uhr_capa_size_ok(const u8 *data, u8 len, * in the UHR MAC Capabilities Information field. */ if (from_ap && cap->mac.mac_cap[1] & IEEE80211_UHR_MAC_CAP1_DBE_SUPP) { - u8 dbe; + const struct ieee80211_uhr_cap_dbe *dbe; - needed += 1; + needed += sizeof(struct ieee80211_uhr_cap_dbe); if (len < needed) return false; - dbe = cap->variable[0]; + dbe = (const void *)cap->variable; - if (dbe & IEEE80211_UHR_MAC_CAP_DBE_EHT_MCS_MAP_160_PRES) - needed += 3; + if (dbe->cap & IEEE80211_UHR_MAC_CAP_DBE_EHT_MCS_MAP_160_PRES) + needed += sizeof(dbe->eht_mcs_map[0]); - if (dbe & IEEE80211_UHR_MAC_CAP_DBE_EHT_MCS_MAP_320_PRES) - needed += 3; + if (dbe->cap & IEEE80211_UHR_MAC_CAP_DBE_EHT_MCS_MAP_320_PRES) + needed += sizeof(dbe->eht_mcs_map[0]); } return len >= needed; } -static inline const struct ieee80211_uhr_cap_phy * -ieee80211_uhr_phy_cap(const struct ieee80211_uhr_cap *cap, bool from_ap) +#define IEEE80211_UHR_OM_PU_TO_128TU 11 + +/** + * ieee80211_uhr_capa_get_om_pu_to_us - get OM parameter update timeout in usec + * @cap: the UHR capability element, size must be validated + * + * Return: the OM parameter update timeout in usec, or -1 if it's not valid + */ +static inline int +ieee80211_uhr_capa_get_om_pu_to_us(const struct ieee80211_uhr_cap *cap) { - u8 offs = 0; + u8 timeout; - if (from_ap && cap->mac.mac_cap[1] & IEEE80211_UHR_MAC_CAP1_DBE_SUPP) { - u8 dbe = cap->variable[0]; + timeout = u8_get_bits(cap->mac.mac_cap[3], + IEEE80211_UHR_MAC_CAP3_UHR_OM_PU_TO_HIGH); + timeout <<= 2; + timeout |= u8_get_bits(cap->mac.mac_cap[2], + IEEE80211_UHR_MAC_CAP2_UHR_OM_PU_TO_LOW); - offs += 1; + if (timeout > IEEE80211_UHR_OM_PU_TO_128TU) + return -1; - if (dbe & IEEE80211_UHR_MAC_CAP_DBE_EHT_MCS_MAP_160_PRES) - offs += 3; + if (!timeout) + return 0; - if (dbe & IEEE80211_UHR_MAC_CAP_DBE_EHT_MCS_MAP_320_PRES) - offs += 3; - } + return 128 << (timeout - 1); +} + +/* only valid from AP, must check ieee80211_uhr_capa_size_ok(..., true) */ +static inline const struct ieee80211_uhr_cap_dbe * +ieee80211_uhr_dbe_cap(const struct ieee80211_uhr_cap *cap) +{ + if (!(cap->mac.mac_cap[1] & IEEE80211_UHR_MAC_CAP1_DBE_SUPP)) + return NULL; - return (const void *)&cap->variable[offs]; + return (const void *)cap->variable; } #define IEEE80211_SMD_INFO_CAPA_DL_DATA_FWD 0x01 @@ -217,4 +578,70 @@ struct ieee80211_smd_info { __le16 timeout; } __packed; +enum ieee80211_protected_uhr_action { + IEEE80211_PROTECTED_UHR_ACTION_LINK_RECONFIG_REQUEST = 0, + IEEE80211_PROTECTED_UHR_ACTION_LINK_RECONFIG_RESPONSE = 1, + IEEE80211_PROTECTED_UHR_ACTION_LINK_RECONFIG_NOTIFY = 2, +}; + +enum ieee80211_uhr_link_reconfig_request_type { + IEEE80211_UHR_LINK_RECONFIG_REQUEST_ST_PREP = 0, + IEEE80211_UHR_LINK_RECONFIG_REQUEST_ST_EXEC = 1, + IEEE80211_UHR_LINK_RECONFIG_REQUEST_OMP_REQUEST = 3, +}; + +enum ieee80211_uhr_link_reconfig_response_type { + IEEE80211_UHR_LINK_RECONFIG_RESPONSE_ST_PREP = 0, + IEEE80211_UHR_LINK_RECONFIG_RESPONSE_ST_EXEC = 1, +}; + +enum ieee80211_uhr_link_reconfig_notify_type { + IEEE80211_UHR_LINK_RECONFIG_NOTIFY_DL_DRAINED = 2, + IEEE80211_UHR_LINK_RECONFIG_NOTIFY_OMP_RESPONSE = 3, +}; + +enum ieee80211_uhr_mode_change_control { + IEEE80211_UHR_MODE_CHANGE_CONTROL_MODE_ID = 0x003f, + IEEE80211_UHR_MODE_CHANGE_CONTROL_MODE_ENABLE = 0x0040, + IEEE80211_UHR_MODE_CHANGE_CONTROL_MODE_UPDATE = 0x0080, + IEEE80211_UHR_MODE_CHANGE_CONTROL_MODE_LENGTH = 0x0f00, + IEEE80211_UHR_MODE_CHANGE_CONTROL_MODE_SPECIFIC = 0xf000, +}; + +enum ieee80211_uhr_mode_change_mode_id { + IEEE80211_UHR_MODE_CHANGE_MODE_ID_DPS = 0, + IEEE80211_UHR_MODE_CHANGE_MODE_ID_NPCA = 1, + IEEE80211_UHR_MODE_CHANGE_MODE_ID_DUO = 2, + IEEE80211_UHR_MODE_CHANGE_MODE_ID_DSO = 3, + IEEE80211_UHR_MODE_CHANGE_MODE_ID_P_EDCA = 4, + IEEE80211_UHR_MODE_CHANGE_MODE_ID_ELR_RX = 5, + IEEE80211_UHR_MODE_CHANGE_MODE_ID_AOM = 6, + IEEE80211_UHR_MODE_CHANGE_MODE_ID_LLI = 7, + IEEE80211_UHR_MODE_CHANGE_MODE_ID_CO_BF = 8, + IEEE80211_UHR_MODE_CHANGE_MODE_ID_CO_SR = 9, + IEEE80211_UHR_MODE_CHANGE_MODE_ID_EMLSR = 10, + IEEE80211_UHR_MODE_CHANGE_MODE_ID_DBE = 11, +}; + +struct ieee80211_uhr_mode_change_tuple { + __le16 control; + u8 variable[]; +} __packed; + +static inline int +ieee80211_uhr_mode_change_tuple_size(const struct ieee80211_uhr_mode_change_tuple *tuple) +{ + return sizeof(*tuple) + + le16_get_bits(tuple->control, + IEEE80211_UHR_MODE_CHANGE_CONTROL_MODE_LENGTH); +} + +#define for_each_uhr_mode_change_tuple(data, len, tuple) \ + for (tuple = (const void *)(data); \ + (len) - ((const u8 *)tuple - (data)) >= sizeof(*tuple) && \ + (len) - ((const u8 *)tuple - (data)) >= \ + ieee80211_uhr_mode_change_tuple_size(tuple); \ + tuple = (const void *)((const u8 *)tuple + \ + ieee80211_uhr_mode_change_tuple_size(tuple))) + #endif /* LINUX_IEEE80211_UHR_H */ diff --git a/include/linux/ieee80211.h b/include/linux/ieee80211.h index 0aa2fb8f88de..26e674038865 100644 --- a/include/linux/ieee80211.h +++ b/include/linux/ieee80211.h @@ -557,6 +557,17 @@ static inline bool ieee80211_is_reassoc_resp(__le16 fc) } /** + * ieee80211_is_assoc - check if (Re)association request/response frame + * @fc: frame control bytes in little-endian byteorder + * Return: whether or not the frame is an (re)association request or response + */ +static inline bool ieee80211_is_assoc(__le16 fc) +{ + return ieee80211_is_assoc_req(fc) || ieee80211_is_reassoc_req(fc) || + ieee80211_is_assoc_resp(fc) || ieee80211_is_reassoc_resp(fc); +} + +/** * ieee80211_is_probe_req - check if IEEE80211_FTYPE_MGMT && IEEE80211_STYPE_PROBE_REQ * @fc: frame control bytes in little-endian byteorder * Return: whether or not the frame is a probe request @@ -1046,31 +1057,28 @@ struct ieee80211_mgmt { } __packed probe_resp; struct { u8 category; + u8 action_code; union { struct { - u8 action_code; u8 dialog_token; u8 status_code; u8 variable[]; } __packed wme_action; struct{ - u8 action_code; + u8 no_fixed_fields[0]; u8 variable[]; } __packed chan_switch; struct{ - u8 action_code; struct ieee80211_ext_chansw_ie data; u8 variable[]; } __packed ext_chan_switch; struct{ - u8 action_code; u8 dialog_token; u8 element_id; u8 length; struct ieee80211_msrment_ie msr_elem; } __packed measurement; struct{ - u8 action_code; u8 dialog_token; __le16 capab; __le16 timeout; @@ -1079,7 +1087,6 @@ struct ieee80211_mgmt { u8 variable[]; } __packed addba_req; struct{ - u8 action_code; u8 dialog_token; __le16 status; __le16 capab; @@ -1088,54 +1095,45 @@ struct ieee80211_mgmt { u8 variable[]; } __packed addba_resp; struct{ - u8 action_code; __le16 params; __le16 reason_code; } __packed delba; struct { - u8 action_code; + u8 no_fixed_fields[0]; u8 variable[]; } __packed self_prot; struct{ - u8 action_code; + u8 no_fixed_fields[0]; u8 variable[]; } __packed mesh_action; struct { - u8 action; u8 trans_id[WLAN_SA_QUERY_TR_ID_LEN]; } __packed sa_query; struct { - u8 action; u8 smps_control; } __packed ht_smps; struct { - u8 action_code; u8 chanwidth; } __packed ht_notify_cw; struct { - u8 action_code; u8 dialog_token; __le16 capability; u8 variable[]; } __packed tdls_discover_resp; struct { - u8 action_code; u8 operating_mode; } __packed vht_opmode_notif; struct { - u8 action_code; u8 membership[WLAN_MEMBERSHIP_LEN]; u8 position[WLAN_USER_POSITION_LEN]; } __packed vht_group_notif; struct { - u8 action_code; u8 dialog_token; u8 tpc_elem_id; u8 tpc_elem_length; struct ieee80211_tpc_report_ie tpc; } __packed tpc_report; struct { - u8 action_code; u8 dialog_token; u8 follow_up; u8 tod[6]; @@ -1145,11 +1143,10 @@ struct ieee80211_mgmt { u8 variable[]; } __packed ftm; struct { - u8 action_code; + u8 no_fixed_fields[0]; u8 variable[]; } __packed s1g; struct { - u8 action_code; u8 dialog_token; u8 follow_up; u32 tod; @@ -1158,41 +1155,53 @@ struct ieee80211_mgmt { u8 max_toa_error; } __packed wnm_timing_msr; struct { - u8 action_code; u8 dialog_token; u8 variable[]; } __packed ttlm_req; struct { - u8 action_code; u8 dialog_token; __le16 status_code; u8 variable[]; } __packed ttlm_res; struct { - u8 action_code; + u8 no_fixed_fields[0]; + /* no variable fields either */ } __packed ttlm_tear_down; struct { - u8 action_code; u8 dialog_token; u8 variable[]; } __packed ml_reconf_req; struct { - u8 action_code; u8 dialog_token; u8 count; u8 variable[]; } __packed ml_reconf_resp; struct { - u8 action_code; + u8 no_fixed_fields[0]; u8 variable[]; } __packed epcs; struct { - u8 action_code; u8 dialog_token; u8 control; u8 variable[]; } __packed eml_omn; - } u; + struct { + u8 dialog_token; + u8 type; + u8 variable[]; + } __packed uhr_link_reconf_req; + struct { + u8 dialog_token; + u8 type; + u8 count; + u8 variable[]; + } __packed uhr_link_reconf_resp; + struct { + u8 dialog_token; + u8 type; + u8 variable[]; + } __packed uhr_link_reconf_notif; + }; } __packed action; DECLARE_FLEX_ARRAY(u8, body); /* Generic frame body */ } u; @@ -1210,9 +1219,15 @@ struct ieee80211_mgmt { #define BSS_MEMBERSHIP_SELECTOR_MIN BSS_MEMBERSHIP_SELECTOR_UHR_PHY -/* mgmt header + 1 byte category code */ -#define IEEE80211_MIN_ACTION_SIZE offsetof(struct ieee80211_mgmt, u.action.u) +#define IEEE80211_MIN_ACTION_SIZE(type) offsetofend(struct ieee80211_mgmt, u.action.type) +/* Link Reconfiguration Status Duple field */ +struct ieee80211_ml_reconf_status { + u8 info; + __le16 status; +} __packed; + +#define IEEE80211_ML_RECONF_LINK_ID_MASK 0xf /* Management MIC information element (IEEE 802.11w) for CMAC */ struct ieee80211_mmie { @@ -1358,6 +1373,7 @@ struct ieee80211_tdls_data { #define WLAN_AUTH_FILS_SK 4 #define WLAN_AUTH_FILS_SK_PFS 5 #define WLAN_AUTH_FILS_PK 6 +#define WLAN_AUTH_IEEE8021X 8 #define WLAN_AUTH_EPPKE 9 #define WLAN_AUTH_LEAP 128 @@ -1500,6 +1516,8 @@ enum ieee80211_statuscode { WLAN_STATUS_REJECT_DSE_BAND = 96, WLAN_STATUS_DENIED_WITH_SUGGESTED_BAND_AND_CHANNEL = 99, WLAN_STATUS_DENIED_DUE_TO_SPECTRUM_MANAGEMENT = 103, + /* 802.11ah */ + WLAN_STATUS_REJECTED_NDP_BLOCK_ACK_SUGGESTED = 109, /* 802.11ai */ WLAN_STATUS_FILS_AUTHENTICATION_FAILURE = 112, WLAN_STATUS_UNKNOWN_AUTHENTICATION_SERVER = 113, @@ -1507,6 +1525,7 @@ enum ieee80211_statuscode { WLAN_STATUS_SAE_PK = 127, WLAN_STATUS_DENIED_TID_TO_LINK_MAPPING = 133, WLAN_STATUS_PREF_TID_TO_LINK_MAPPING_SUGGESTED = 134, + WLAN_STATUS_8021X_AUTH_SUCCESS = 153, }; @@ -1845,6 +1864,7 @@ enum ieee80211_category { WLAN_CATEGORY_VHT = 21, WLAN_CATEGORY_S1G = 22, WLAN_CATEGORY_PROTECTED_EHT = 37, + WLAN_CATEGORY_PROTECTED_UHR = 43, WLAN_CATEGORY_VENDOR_SPECIFIC_PROTECTED = 126, WLAN_CATEGORY_VENDOR_SPECIFIC = 127, }; @@ -1929,6 +1949,11 @@ enum ieee80211_radio_measurement_actioncode { #define PMK_MAX_LEN 64 #define SAE_PASSWORD_MAX_LEN 128 +#define MICHAEL_MIC_LEN 8 + +void michael_mic(const u8 *key, struct ieee80211_hdr *hdr, + const u8 *data, size_t data_len, u8 *mic); + /* Public action codes (IEEE Std 802.11-2016, 9.6.8.1, Table 9-307) */ enum ieee80211_pub_actioncode { WLAN_PUB_ACTION_20_40_BSS_COEX = 0, @@ -2239,6 +2264,7 @@ struct ieee80211_multiple_bssid_configuration { #define WLAN_AKM_SUITE_WFA_DPP SUITE(WLAN_OUI_WFA, 2) #define WLAN_MAX_KEY_LEN 32 +#define WLAN_MAX_SECURE_LTF_KEYSEED_LEN 48 #define WLAN_PMK_NAME_LEN 16 #define WLAN_PMKID_LEN 16 @@ -2248,6 +2274,7 @@ struct ieee80211_multiple_bssid_configuration { #define WLAN_OUI_WFA 0x506f9a #define WLAN_OUI_TYPE_WFA_P2P 9 +#define WLAN_OUI_TYPE_WFA_NAN 0x13 #define WLAN_OUI_TYPE_WFA_DPP 0x1A #define WLAN_OUI_MICROSOFT 0x0050f2 #define WLAN_OUI_TYPE_MICROSOFT_WPA 1 @@ -2389,7 +2416,7 @@ static inline bool ieee80211_is_bufferable_mmpdu(struct sk_buff *skb) if (!ieee80211_is_action(fc)) return false; - if (skb->len < offsetofend(typeof(*mgmt), u.action.u.ftm.action_code)) + if (skb->len < IEEE80211_MIN_ACTION_SIZE(action_code)) return true; /* action frame - additionally check for non-bufferable FTM */ @@ -2398,8 +2425,8 @@ static inline bool ieee80211_is_bufferable_mmpdu(struct sk_buff *skb) mgmt->u.action.category != WLAN_CATEGORY_PROTECTED_DUAL_OF_ACTION) return true; - if (mgmt->u.action.u.ftm.action_code == WLAN_PUB_ACTION_FTM_REQUEST || - mgmt->u.action.u.ftm.action_code == WLAN_PUB_ACTION_FTM_RESPONSE) + if (mgmt->u.action.action_code == WLAN_PUB_ACTION_FTM_REQUEST || + mgmt->u.action.action_code == WLAN_PUB_ACTION_FTM_RESPONSE) return false; return true; @@ -2449,7 +2476,7 @@ static inline bool _ieee80211_is_robust_mgmt_frame(struct ieee80211_hdr *hdr) */ static inline bool ieee80211_is_robust_mgmt_frame(struct sk_buff *skb) { - if (skb->len < IEEE80211_MIN_ACTION_SIZE) + if (skb->len < IEEE80211_MIN_ACTION_SIZE(category)) return false; return _ieee80211_is_robust_mgmt_frame((void *)skb->data); } @@ -2465,7 +2492,7 @@ static inline bool ieee80211_is_public_action(struct ieee80211_hdr *hdr, { struct ieee80211_mgmt *mgmt = (void *)hdr; - if (len < IEEE80211_MIN_ACTION_SIZE) + if (len < IEEE80211_MIN_ACTION_SIZE(category)) return false; if (!ieee80211_is_action(hdr->frame_control)) return false; @@ -2483,13 +2510,14 @@ static inline bool ieee80211_is_public_action(struct ieee80211_hdr *hdr, static inline bool ieee80211_is_protected_dual_of_public_action(struct sk_buff *skb) { + struct ieee80211_mgmt *mgmt = (void *)skb->data; u8 action; if (!ieee80211_is_public_action((void *)skb->data, skb->len) || - skb->len < IEEE80211_MIN_ACTION_SIZE + 1) + skb->len < IEEE80211_MIN_ACTION_SIZE(action_code)) return false; - action = *(u8 *)(skb->data + IEEE80211_MIN_ACTION_SIZE); + action = mgmt->u.action.action_code; return action != WLAN_PUB_ACTION_20_40_BSS_COEX && action != WLAN_PUB_ACTION_DSE_REG_LOC_ANN && @@ -2528,7 +2556,7 @@ static inline bool _ieee80211_is_group_privacy_action(struct ieee80211_hdr *hdr) */ static inline bool ieee80211_is_group_privacy_action(struct sk_buff *skb) { - if (skb->len < IEEE80211_MIN_ACTION_SIZE) + if (skb->len < IEEE80211_MIN_ACTION_SIZE(category)) return false; return _ieee80211_is_group_privacy_action((void *)skb->data); } @@ -2599,6 +2627,7 @@ static inline int ieee80211_get_tdls_action(struct sk_buff *skb) /* convert frequencies */ #define MHZ_TO_KHZ(freq) ((freq) * 1000) #define KHZ_TO_MHZ(freq) ((freq) / 1000) +#define KHZ_TO_HZ(x) ((x) * 1000) #define PR_KHZ(f) KHZ_TO_MHZ(f), f % 1000 #define KHZ_F "%d.%03d" @@ -2624,8 +2653,7 @@ static inline bool ieee80211_action_contains_tpc(struct sk_buff *skb) if (!ieee80211_is_action(mgmt->frame_control)) return false; - if (skb->len < IEEE80211_MIN_ACTION_SIZE + - sizeof(mgmt->u.action.u.tpc_report)) + if (skb->len < IEEE80211_MIN_ACTION_SIZE(tpc_report)) return false; /* @@ -2644,12 +2672,11 @@ static inline bool ieee80211_action_contains_tpc(struct sk_buff *skb) return false; /* both spectrum mgmt and link measurement have same action code */ - if (mgmt->u.action.u.tpc_report.action_code != - WLAN_ACTION_SPCT_TPC_RPRT) + if (mgmt->u.action.action_code != WLAN_ACTION_SPCT_TPC_RPRT) return false; - if (mgmt->u.action.u.tpc_report.tpc_elem_id != WLAN_EID_TPC_REPORT || - mgmt->u.action.u.tpc_report.tpc_elem_length != + if (mgmt->u.action.tpc_report.tpc_elem_id != WLAN_EID_TPC_REPORT || + mgmt->u.action.tpc_report.tpc_elem_length != sizeof(struct ieee80211_tpc_report_ie)) return false; @@ -2665,16 +2692,15 @@ static inline bool ieee80211_is_timing_measurement(struct sk_buff *skb) { struct ieee80211_mgmt *mgmt = (void *)skb->data; - if (skb->len < IEEE80211_MIN_ACTION_SIZE) + if (skb->len < IEEE80211_MIN_ACTION_SIZE(wnm_timing_msr)) return false; if (!ieee80211_is_action(mgmt->frame_control)) return false; if (mgmt->u.action.category == WLAN_CATEGORY_WNM_UNPROTECTED && - mgmt->u.action.u.wnm_timing_msr.action_code == - WLAN_UNPROTECTED_WNM_ACTION_TIMING_MEASUREMENT_RESPONSE && - skb->len >= offsetofend(typeof(*mgmt), u.action.u.wnm_timing_msr)) + mgmt->u.action.action_code == + WLAN_UNPROTECTED_WNM_ACTION_TIMING_MEASUREMENT_RESPONSE) return true; return false; @@ -2689,15 +2715,13 @@ static inline bool ieee80211_is_ftm(struct sk_buff *skb) { struct ieee80211_mgmt *mgmt = (void *)skb->data; - if (!ieee80211_is_public_action((void *)mgmt, skb->len)) + if (skb->len < IEEE80211_MIN_ACTION_SIZE(ftm)) return false; - if (mgmt->u.action.u.ftm.action_code == - WLAN_PUB_ACTION_FTM_RESPONSE && - skb->len >= offsetofend(typeof(*mgmt), u.action.u.ftm)) - return true; + if (!ieee80211_is_public_action((void *)mgmt, skb->len)) + return false; - return false; + return mgmt->u.action.action_code == WLAN_PUB_ACTION_FTM_RESPONSE; } struct element { diff --git a/include/linux/if_bridge.h b/include/linux/if_bridge.h index c5fe3b2a53e8..75673b8bffcb 100644 --- a/include/linux/if_bridge.h +++ b/include/linux/if_bridge.h @@ -36,31 +36,60 @@ struct br_ip_list { struct br_ip addr; }; -#define BR_HAIRPIN_MODE BIT(0) -#define BR_BPDU_GUARD BIT(1) -#define BR_ROOT_BLOCK BIT(2) -#define BR_MULTICAST_FAST_LEAVE BIT(3) -#define BR_ADMIN_COST BIT(4) -#define BR_LEARNING BIT(5) -#define BR_FLOOD BIT(6) +enum bridge_flags_bit { + BR_HAIRPIN_MODE_BIT, + BR_BPDU_GUARD_BIT, + BR_ROOT_BLOCK_BIT, + BR_MULTICAST_FAST_LEAVE_BIT, + BR_ADMIN_COST_BIT, + BR_LEARNING_BIT, + BR_FLOOD_BIT, + BR_PROMISC_BIT, + BR_PROXYARP_BIT, + BR_LEARNING_SYNC_BIT, + BR_PROXYARP_WIFI_BIT, + BR_MCAST_FLOOD_BIT, + BR_MULTICAST_TO_UNICAST_BIT, + BR_VLAN_TUNNEL_BIT, + BR_BCAST_FLOOD_BIT, + BR_NEIGH_SUPPRESS_BIT, + BR_ISOLATED_BIT, + BR_MRP_AWARE_BIT, + BR_MRP_LOST_CONT_BIT, + BR_MRP_LOST_IN_CONT_BIT, + BR_TX_FWD_OFFLOAD_BIT, + BR_PORT_LOCKED_BIT, + BR_PORT_MAB_BIT, + BR_NEIGH_VLAN_SUPPRESS_BIT, + BR_NEIGH_FORWARD_GRAT_BIT, +}; + +#define BR_HAIRPIN_MODE BIT(BR_HAIRPIN_MODE_BIT) +#define BR_BPDU_GUARD BIT(BR_BPDU_GUARD_BIT) +#define BR_ROOT_BLOCK BIT(BR_ROOT_BLOCK_BIT) +#define BR_MULTICAST_FAST_LEAVE BIT(BR_MULTICAST_FAST_LEAVE_BIT) +#define BR_ADMIN_COST BIT(BR_ADMIN_COST_BIT) +#define BR_LEARNING BIT(BR_LEARNING_BIT) +#define BR_FLOOD BIT(BR_FLOOD_BIT) #define BR_AUTO_MASK (BR_FLOOD | BR_LEARNING) -#define BR_PROMISC BIT(7) -#define BR_PROXYARP BIT(8) -#define BR_LEARNING_SYNC BIT(9) -#define BR_PROXYARP_WIFI BIT(10) -#define BR_MCAST_FLOOD BIT(11) -#define BR_MULTICAST_TO_UNICAST BIT(12) -#define BR_VLAN_TUNNEL BIT(13) -#define BR_BCAST_FLOOD BIT(14) -#define BR_NEIGH_SUPPRESS BIT(15) -#define BR_ISOLATED BIT(16) -#define BR_MRP_AWARE BIT(17) -#define BR_MRP_LOST_CONT BIT(18) -#define BR_MRP_LOST_IN_CONT BIT(19) -#define BR_TX_FWD_OFFLOAD BIT(20) -#define BR_PORT_LOCKED BIT(21) -#define BR_PORT_MAB BIT(22) -#define BR_NEIGH_VLAN_SUPPRESS BIT(23) +#define BR_PROMISC BIT(BR_PROMISC_BIT) +#define BR_PROXYARP BIT(BR_PROXYARP_BIT) +#define BR_LEARNING_SYNC BIT(BR_LEARNING_SYNC_BIT) +#define BR_PROXYARP_WIFI BIT(BR_PROXYARP_WIFI_BIT) +#define BR_MCAST_FLOOD BIT(BR_MCAST_FLOOD_BIT) +#define BR_MULTICAST_TO_UNICAST BIT(BR_MULTICAST_TO_UNICAST_BIT) +#define BR_VLAN_TUNNEL BIT(BR_VLAN_TUNNEL_BIT) +#define BR_BCAST_FLOOD BIT(BR_BCAST_FLOOD_BIT) +#define BR_NEIGH_SUPPRESS BIT(BR_NEIGH_SUPPRESS_BIT) +#define BR_ISOLATED BIT(BR_ISOLATED_BIT) +#define BR_MRP_AWARE BIT(BR_MRP_AWARE_BIT) +#define BR_MRP_LOST_CONT BIT(BR_MRP_LOST_CONT_BIT) +#define BR_MRP_LOST_IN_CONT BIT(BR_MRP_LOST_IN_CONT_BIT) +#define BR_TX_FWD_OFFLOAD BIT(BR_TX_FWD_OFFLOAD_BIT) +#define BR_PORT_LOCKED BIT(BR_PORT_LOCKED_BIT) +#define BR_PORT_MAB BIT(BR_PORT_MAB_BIT) +#define BR_NEIGH_VLAN_SUPPRESS BIT(BR_NEIGH_VLAN_SUPPRESS_BIT) +#define BR_NEIGH_FORWARD_GRAT BIT(BR_NEIGH_FORWARD_GRAT_BIT) #define BR_DEFAULT_AGEING_TIME (300 * HZ) diff --git a/include/linux/if_pppox.h b/include/linux/if_pppox.h index db45d6f1c4f4..594d6dc3f4c9 100644 --- a/include/linux/if_pppox.h +++ b/include/linux/if_pppox.h @@ -25,8 +25,6 @@ struct pppoe_opt { struct net_device *dev; /* device associated with socket*/ int ifindex; /* ifindex of device associated with socket */ struct pppoe_addr pa; /* what this socket is bound to*/ - struct sockaddr_pppox relay; /* what socket data will be - relayed to (PPPoE relaying) */ struct work_struct padt_work;/* Work item for handling PADT */ }; @@ -53,16 +51,10 @@ struct pppox_sock { #define pppoe_dev proto.pppoe.dev #define pppoe_ifindex proto.pppoe.ifindex #define pppoe_pa proto.pppoe.pa -#define pppoe_relay proto.pppoe.relay static inline struct pppox_sock *pppox_sk(struct sock *sk) { - return (struct pppox_sock *)sk; -} - -static inline struct sock *sk_pppox(struct pppox_sock *po) -{ - return (struct sock *)po; + return container_of(sk, struct pppox_sock, sk); } struct module; @@ -80,14 +72,11 @@ extern void pppox_unbind_sock(struct sock *sk);/* delete ppp-channel binding */ extern int pppox_ioctl(struct socket *sock, unsigned int cmd, unsigned long arg); extern int pppox_compat_ioctl(struct socket *sock, unsigned int cmd, unsigned long arg); -#define PPPOEIOCSFWD32 _IOW(0xB1 ,0, compat_size_t) - /* PPPoX socket states */ enum { PPPOX_NONE = 0, /* initial state */ PPPOX_CONNECTED = 1, /* connection established ==TCP_ESTABLISHED */ PPPOX_BOUND = 2, /* bound to ppp device */ - PPPOX_RELAY = 4, /* forwarding is enabled */ PPPOX_DEAD = 16 /* dead, useless, please clean me up!*/ }; diff --git a/include/linux/if_team.h b/include/linux/if_team.h index ce97d891cf72..3d21e06fda67 100644 --- a/include/linux/if_team.h +++ b/include/linux/if_team.h @@ -27,10 +27,11 @@ struct team; struct team_port { struct net_device *dev; - struct hlist_node hlist; /* node in enabled ports hash list */ + struct hlist_node tx_hlist; /* node in tx-enabled ports hash list */ struct list_head list; /* node in ordinary list */ struct team *team; - int index; /* index of enabled port. If disabled, it's set to -1 */ + int tx_index; /* index of tx enabled port. If disabled, -1 */ + bool rx_enabled; bool linkup; /* either state.linkup or user.linkup */ @@ -75,14 +76,24 @@ static inline struct team_port *team_port_get_rcu(const struct net_device *dev) return rcu_dereference(dev->rx_handler_data); } +static inline bool team_port_rx_enabled(struct team_port *port) +{ + return READ_ONCE(port->rx_enabled); +} + +static inline bool team_port_tx_enabled(struct team_port *port) +{ + return READ_ONCE(port->tx_index) != -1; +} + static inline bool team_port_enabled(struct team_port *port) { - return port->index != -1; + return team_port_rx_enabled(port) && team_port_tx_enabled(port); } static inline bool team_port_txable(struct team_port *port) { - return port->linkup && team_port_enabled(port); + return port->linkup && team_port_tx_enabled(port); } static inline bool team_port_dev_txable(const struct net_device *port_dev) @@ -121,8 +132,7 @@ struct team_mode_ops { int (*port_enter)(struct team *team, struct team_port *port); void (*port_leave)(struct team *team, struct team_port *port); void (*port_change_dev_addr)(struct team *team, struct team_port *port); - void (*port_enabled)(struct team *team, struct team_port *port); - void (*port_disabled)(struct team *team, struct team_port *port); + void (*port_tx_disabled)(struct team *team, struct team_port *port); }; extern int team_modeop_port_enter(struct team *team, struct team_port *port); @@ -186,16 +196,16 @@ struct team_mode { #define TEAM_MODE_PRIV_SIZE (sizeof(long) * TEAM_MODE_PRIV_LONGS) struct team { - struct net_device *dev; /* associated netdevice */ struct team_pcpu_stats __percpu *pcpu_stats; const struct header_ops *header_ops_cache; /* - * List of enabled ports and their count + * List of tx-enabled ports and counts of rx and tx-enabled ports. */ - int en_port_count; - struct hlist_head en_port_hlist[TEAM_PORT_HASHENTRIES]; + int tx_en_port_count; + int rx_en_port_count; + struct hlist_head tx_en_port_hlist[TEAM_PORT_HASHENTRIES]; struct list_head port_list; /* list of all ports */ @@ -232,48 +242,50 @@ static inline int team_dev_queue_xmit(struct team *team, struct team_port *port, skb_set_queue_mapping(skb, qdisc_skb_cb(skb)->slave_dev_queue_mapping); skb->dev = port->dev; - if (unlikely(netpoll_tx_running(team->dev))) { + if (unlikely(netpoll_tx_running(netdev_from_priv(team)))) { team_netpoll_send_skb(port, skb); return 0; } return dev_queue_xmit(skb); } -static inline struct hlist_head *team_port_index_hash(struct team *team, - int port_index) +static inline struct hlist_head *team_tx_port_index_hash(struct team *team, + int tx_port_index) { - return &team->en_port_hlist[port_index & (TEAM_PORT_HASHENTRIES - 1)]; + unsigned int list_entry = tx_port_index & (TEAM_PORT_HASHENTRIES - 1); + + return &team->tx_en_port_hlist[list_entry]; } -static inline struct team_port *team_get_port_by_index(struct team *team, - int port_index) +static inline struct team_port *team_get_port_by_tx_index(struct team *team, + int tx_port_index) { + struct hlist_head *head = team_tx_port_index_hash(team, tx_port_index); struct team_port *port; - struct hlist_head *head = team_port_index_hash(team, port_index); - hlist_for_each_entry(port, head, hlist) - if (port->index == port_index) + hlist_for_each_entry(port, head, tx_hlist) + if (port->tx_index == tx_port_index) return port; return NULL; } static inline int team_num_to_port_index(struct team *team, unsigned int num) { - int en_port_count = READ_ONCE(team->en_port_count); + int tx_en_port_count = READ_ONCE(team->tx_en_port_count); - if (unlikely(!en_port_count)) + if (unlikely(!tx_en_port_count)) return 0; - return num % en_port_count; + return num % tx_en_port_count; } -static inline struct team_port *team_get_port_by_index_rcu(struct team *team, - int port_index) +static inline struct team_port *team_get_port_by_tx_index_rcu(struct team *team, + int tx_port_index) { + struct hlist_head *head = team_tx_port_index_hash(team, tx_port_index); struct team_port *port; - struct hlist_head *head = team_port_index_hash(team, port_index); - hlist_for_each_entry_rcu(port, head, hlist) - if (port->index == port_index) + hlist_for_each_entry_rcu(port, head, tx_hlist) + if (READ_ONCE(port->tx_index) == tx_port_index) return port; return NULL; } diff --git a/include/linux/if_tun.h b/include/linux/if_tun.h index 80166eb62f41..eeb9ed3c5a23 100644 --- a/include/linux/if_tun.h +++ b/include/linux/if_tun.h @@ -22,6 +22,8 @@ struct tun_msg_ctl { #if defined(CONFIG_TUN) || defined(CONFIG_TUN_MODULE) struct socket *tun_get_socket(struct file *); struct ptr_ring *tun_get_tx_ring(struct file *file); +/* Callers must hold the consumer_lock of the ring of file */ +void tun_wake_queue(struct file *file, int consumed); static inline bool tun_is_xdp_frame(void *ptr) { @@ -55,6 +57,8 @@ static inline struct ptr_ring *tun_get_tx_ring(struct file *f) return ERR_PTR(-EINVAL); } +static inline void tun_wake_queue(struct file *f, int consumed) {} + static inline bool tun_is_xdp_frame(void *ptr) { return false; diff --git a/include/linux/if_vlan.h b/include/linux/if_vlan.h index e6272f9c5e42..20cc16ea4e5a 100644 --- a/include/linux/if_vlan.h +++ b/include/linux/if_vlan.h @@ -147,11 +147,13 @@ extern __be16 vlan_dev_vlan_proto(const struct net_device *dev); * @priority: skb priority * @vlan_qos: vlan priority: (skb->priority << 13) & 0xE000 * @next: pointer to next struct + * @rcu: used for deferred freeing of mapping nodes */ struct vlan_priority_tci_mapping { u32 priority; u16 vlan_qos; - struct vlan_priority_tci_mapping *next; + struct vlan_priority_tci_mapping __rcu *next; + struct rcu_head rcu; }; struct proc_dir_entry; @@ -177,7 +179,7 @@ struct vlan_dev_priv { unsigned int nr_ingress_mappings; u32 ingress_priority_map[8]; unsigned int nr_egress_mappings; - struct vlan_priority_tci_mapping *egress_priority_map[16]; + struct vlan_priority_tci_mapping __rcu *egress_priority_map[16]; __be16 vlan_proto; u16 vlan_id; @@ -209,19 +211,24 @@ static inline u16 vlan_dev_get_egress_qos_mask(struct net_device *dev, u32 skprio) { struct vlan_priority_tci_mapping *mp; + u16 vlan_qos = 0; - smp_rmb(); /* coupled with smp_wmb() in vlan_dev_set_egress_priority() */ + rcu_read_lock(); - mp = vlan_dev_priv(dev)->egress_priority_map[(skprio & 0xF)]; + mp = rcu_dereference(vlan_dev_priv(dev)->egress_priority_map[skprio & 0xF]); while (mp) { if (mp->priority == skprio) { - return mp->vlan_qos; /* This should already be shifted - * to mask correctly with the - * VLAN's TCI */ + vlan_qos = READ_ONCE(mp->vlan_qos); + break; } - mp = mp->next; + mp = rcu_dereference(mp->next); } - return 0; + rcu_read_unlock(); + + /* This should already be shifted to mask correctly with + * the VLAN's TCI. + */ + return vlan_qos; } extern bool vlan_do_receive(struct sk_buff **skb); diff --git a/include/linux/igmp.h b/include/linux/igmp.h index 073b30a9b850..a0cf0398519f 100644 --- a/include/linux/igmp.h +++ b/include/linux/igmp.h @@ -57,20 +57,21 @@ struct ip_mc_socklist { }; struct ip_sf_list { - struct ip_sf_list *sf_next; + struct ip_sf_list __rcu *sf_next; unsigned long sf_count[2]; /* include/exclude counts */ __be32 sf_inaddr; unsigned char sf_gsresp; /* include in g & s response? */ unsigned char sf_oldin; /* change state */ unsigned char sf_crcount; /* retrans. left to send */ + struct rcu_head rcu; }; struct ip_mc_list { struct in_device *interface; __be32 multiaddr; unsigned int sfmode; - struct ip_sf_list *sources; - struct ip_sf_list *tomb; + struct ip_sf_list __rcu *sources; + struct ip_sf_list __rcu *tomb; unsigned long sfcount[2]; union { struct ip_mc_list *next; @@ -92,15 +93,166 @@ struct ip_mc_list { struct rcu_head rcu; }; +/* RFC3376, relevant sections: + * - 4.1.1. Maximum Response Code + * - 4.1.7. QQIC (Querier's Query Interval Code) + * + * For both MRC and QQIC, values >= 128 use the same floating-point + * encoding as follows: + * + * 0 1 2 3 4 5 6 7 + * +-+-+-+-+-+-+-+-+ + * |1| exp | mant | + * +-+-+-+-+-+-+-+-+ + */ +#define IGMPV3_FP_EXP(value) (((value) >> 4) & 0x07) +#define IGMPV3_FP_MAN(value) ((value) & 0x0f) + +/* IGMPv3 floating-point exponential field min threshold */ +#define IGMPV3_EXP_MIN_THRESHOLD 128 +/* IGMPv3 FP max threshold (mant = 0xF, exp = 7) -> 31744 */ +#define IGMPV3_EXP_MAX_THRESHOLD 31744 + +/* V3 exponential field encoding */ + +/* IGMPv3 MRC/QQIC 8-bit exponential field encode + * + * RFC3376, 4.1.1 & 4.1.7. defines only the decoding formula: + * MRT/QQI = (mant | 0x10) << (exp + 3) + * + * but does NOT define the encoding procedure. To derive exponent: + * + * For any value of mantissa and exponent, the decoding formula + * indicates that the "hidden bit" (0x10) is shifted 4 bits left + * to sit above the 4-bit mantissa. The RFC again shifts this + * entire block left by (exp + 3) to reconstruct the value. + * So, 'hidden bit' is the MSB which is shifted by (4 + exp + 3). + * + * Total left shift of the 'hidden bit' = 4 + (exp + 3) = exp + 7. + * This is the MSB at the 0-based bit position: (exp + 7). + * Since fls() is 1-based, fls(value) - 1 = exp + 7. + * + * Therefore: + * exp = fls(value) - 8 + * mant = (value >> (exp + 3)) & 0x0F + * + * Final encoding formula: + * 0x80 | (exp << 4) | mant + * + * Example (value = 3200): + * 0 1 + * 0 1 2 3 4 5 6 7 0 1 2 3 4 5 6 7 + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * |0 0 0 0 1 1 0 0 1 0 0 0 0 0 0 0| (value = 3200) + * | ^-^-mant^ ^..(exp+3)..^| exp = 4, mant = 9 + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * + * Encoded: + * 0x80 | (4 << 4) | 9 = 0xC9 + */ +static inline u8 igmpv3_exp_field_encode(unsigned long value) +{ + u8 mc_exp, mc_man; + + /* MRC/QQIC < 128 is literal */ + if (value < IGMPV3_EXP_MIN_THRESHOLD) + return value; + + /* Saturate at max representable (mant = 0xF, exp = 7) -> 31744 */ + if (value >= IGMPV3_EXP_MAX_THRESHOLD) + return 0xFF; + + mc_exp = fls(value) - 8; + mc_man = (value >> (mc_exp + 3)) & 0x0F; + + return 0x80 | (mc_exp << 4) | mc_man; +} + +/* Calculate Maximum Response Code from Max Resp Time + * + * RFC3376, relevant sections: + * - 4.1.1. Maximum Response Code + * - 8.3. Query Response Interval + * + * MRC represents the encoded form of Max Resp Time (MRT); once + * decoded, the resulting value is in units of 0.1 seconds (100 ms). + */ +static inline u8 igmpv3_mrc(unsigned long mrt) +{ + return igmpv3_exp_field_encode(mrt); +} + +/* Calculate Querier's Query Interval Code from Querier's Query Interval + * + * RFC3376, relevant sections: + * - 4.1.7. QQIC (Querier's Query Interval Code) + * - 8.2. Query Interval + * - 8.12. Older Version Querier Present Timeout + * (the [Query Interval] in the last Query received) + * + * QQIC represents the encoded form of Querier's Query Interval (QQI); + * once decoded, the resulting value is in units of seconds. + */ +static inline u8 igmpv3_qqic(unsigned long qi) +{ + return igmpv3_exp_field_encode(qi); +} + /* V3 exponential field decoding */ -#define IGMPV3_MASK(value, nb) ((nb)>=32 ? (value) : ((1<<(nb))-1) & (value)) -#define IGMPV3_EXP(thresh, nbmant, nbexp, value) \ - ((value) < (thresh) ? (value) : \ - ((IGMPV3_MASK(value, nbmant) | (1<<(nbmant))) << \ - (IGMPV3_MASK((value) >> (nbmant), nbexp) + (nbexp)))) - -#define IGMPV3_QQIC(value) IGMPV3_EXP(0x80, 4, 3, value) -#define IGMPV3_MRC(value) IGMPV3_EXP(0x80, 4, 3, value) + +/* IGMPv3 MRC/QQIC 8-bit exponential field decode + * + * RFC3376, 4.1.1 & 4.1.7. defines the decoding formula: + * 0 1 2 3 4 5 6 7 + * +-+-+-+-+-+-+-+-+ + * |1| exp | mant | + * +-+-+-+-+-+-+-+-+ + * Max Resp Time = (mant | 0x10) << (exp + 3) + * QQI = (mant | 0x10) << (exp + 3) + */ +static inline unsigned long igmpv3_exp_field_decode(const u8 code) +{ + if (code < IGMPV3_EXP_MIN_THRESHOLD) { + return code; + } else { + unsigned long mc_man, mc_exp; + + mc_exp = IGMPV3_FP_EXP(code); + mc_man = IGMPV3_FP_MAN(code); + + return (mc_man | 0x10) << (mc_exp + 3); + } +} + +/* Calculate Max Resp Time from Maximum Response Code + * + * RFC3376, relevant sections: + * - 4.1.1. Maximum Response Code + * - 8.3. Query Response Interval + * + * After decode, MRC represents the Maximum Response Time (MRT) in + * units of 0.1 seconds (100 ms). + */ +static inline unsigned long igmpv3_mrt(const struct igmpv3_query *ih3) +{ + return igmpv3_exp_field_decode(ih3->code); +} + +/* Calculate Querier's Query Interval from Querier's Query Interval Code + * + * RFC3376, relevant sections: + * - 4.1.7. QQIC (Querier's Query Interval Code) + * - 8.2. Query Interval + * - 8.12. Older Version Querier Present Timeout + * (the [Query Interval] in the last Query received) + * + * After decode, QQIC represents the Querier's Query Interval in units + * of seconds. + */ +static inline unsigned long igmpv3_qqi(const struct igmpv3_query *ih3) +{ + return igmpv3_exp_field_decode(ih3->qqic); +} static inline int ip_mc_may_pull(struct sk_buff *skb, unsigned int len) { diff --git a/include/linux/iio/adc/qcom-adc5-gen3-common.h b/include/linux/iio/adc/qcom-adc5-gen3-common.h new file mode 100644 index 000000000000..39cbfcbdb101 --- /dev/null +++ b/include/linux/iio/adc/qcom-adc5-gen3-common.h @@ -0,0 +1,209 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. + * + * Code used in the main and auxiliary Qualcomm PMIC voltage ADCs + * of type ADC5 Gen3. + */ + +#ifndef QCOM_ADC5_GEN3_COMMON_H +#define QCOM_ADC5_GEN3_COMMON_H + +#include <linux/auxiliary_bus.h> +#include <linux/bitfield.h> +#include <linux/bits.h> +#include <linux/device.h> +#include <linux/iio/adc/qcom-vadc-common.h> +#include <linux/regmap.h> +#include <linux/types.h> + +#define ADC5_GEN3_HS 0x45 +#define ADC5_GEN3_HS_BUSY BIT(7) +#define ADC5_GEN3_HS_READY BIT(0) + +#define ADC5_GEN3_STATUS1 0x46 +#define ADC5_GEN3_STATUS1_CONV_FAULT BIT(7) +#define ADC5_GEN3_STATUS1_THR_CROSS BIT(6) +#define ADC5_GEN3_STATUS1_EOC BIT(0) + +#define ADC5_GEN3_TM_EN_STS 0x47 +#define ADC5_GEN3_TM_HIGH_STS 0x48 +#define ADC5_GEN3_TM_LOW_STS 0x49 + +#define ADC5_GEN3_EOC_STS 0x4a +#define ADC5_GEN3_EOC_CHAN_0 BIT(0) + +#define ADC5_GEN3_EOC_CLR 0x4b +#define ADC5_GEN3_TM_HIGH_STS_CLR 0x4c +#define ADC5_GEN3_TM_LOW_STS_CLR 0x4d +#define ADC5_GEN3_CONV_ERR_CLR 0x4e +#define ADC5_GEN3_CONV_ERR_CLR_REQ BIT(0) + +#define ADC5_GEN3_SID 0x4f +#define ADC5_GEN3_SID_MASK GENMASK(3, 0) + +#define ADC5_GEN3_PERPH_CH 0x50 +#define ADC5_GEN3_CHAN_CONV_REQ BIT(7) + +#define ADC5_GEN3_TIMER_SEL 0x51 +#define ADC5_GEN3_TIME_IMMEDIATE 0x1 + +#define ADC5_GEN3_DIG_PARAM 0x52 +#define ADC5_GEN3_DIG_PARAM_CAL_SEL_MASK GENMASK(5, 4) +#define ADC5_GEN3_DIG_PARAM_DEC_RATIO_SEL_MASK GENMASK(3, 2) + +#define ADC5_GEN3_FAST_AVG 0x53 +#define ADC5_GEN3_FAST_AVG_CTL_EN BIT(7) +#define ADC5_GEN3_FAST_AVG_CTL_SAMPLES_MASK GENMASK(2, 0) + +#define ADC5_GEN3_ADC_CH_SEL_CTL 0x54 +#define ADC5_GEN3_DELAY_CTL 0x55 +#define ADC5_GEN3_HW_SETTLE_DELAY_MASK GENMASK(3, 0) + +#define ADC5_GEN3_CH_EN 0x56 +#define ADC5_GEN3_HIGH_THR_INT_EN BIT(1) +#define ADC5_GEN3_LOW_THR_INT_EN BIT(0) + +#define ADC5_GEN3_LOW_THR0 0x57 +#define ADC5_GEN3_LOW_THR1 0x58 +#define ADC5_GEN3_HIGH_THR0 0x59 +#define ADC5_GEN3_HIGH_THR1 0x5a + +#define ADC5_GEN3_CH_DATA0(channel) (0x5c + (channel) * 2) +#define ADC5_GEN3_CH_DATA1(channel) (0x5d + (channel) * 2) + +#define ADC5_GEN3_CONV_REQ 0xe5 +#define ADC5_GEN3_CONV_REQ_REQ BIT(0) + +#define ADC5_GEN3_VIRTUAL_SID_MASK GENMASK(15, 8) +#define ADC5_GEN3_CHANNEL_MASK GENMASK(7, 0) +#define ADC5_GEN3_V_CHAN(x) \ + (FIELD_PREP(ADC5_GEN3_VIRTUAL_SID_MASK, (x).sid) | (x).channel) + +/* ADC channels for PMIC5 Gen3 */ +#define ADC5_GEN3_REF_GND 0x00 +#define ADC5_GEN3_1P25VREF 0x01 +#define ADC5_GEN3_DIE_TEMP 0x03 +#define ADC5_GEN3_USB_SNS_V_16 0x11 +#define ADC5_GEN3_VIN_DIV16_MUX 0x12 +#define ADC5_GEN3_VPH_PWR 0x8e +#define ADC5_GEN3_VBAT_SNS_QBG 0x8f +/* 100k pull-up channels */ +#define ADC5_GEN3_AMUX1_THM_100K_PU 0x44 +#define ADC5_GEN3_AMUX2_THM_100K_PU 0x45 +#define ADC5_GEN3_AMUX3_THM_100K_PU 0x46 +#define ADC5_GEN3_AMUX4_THM_100K_PU 0x47 +#define ADC5_GEN3_AMUX5_THM_100K_PU 0x48 +#define ADC5_GEN3_AMUX6_THM_100K_PU 0x49 +#define ADC5_GEN3_AMUX1_GPIO_100K_PU 0x4a +#define ADC5_GEN3_AMUX2_GPIO_100K_PU 0x4b +#define ADC5_GEN3_AMUX3_GPIO_100K_PU 0x4c +#define ADC5_GEN3_AMUX4_GPIO_100K_PU 0x4d + +#define ADC5_MAX_CHANNEL 0xc0 + +enum adc5_cal_method { + ADC5_NO_CAL = 0, + ADC5_RATIOMETRIC_CAL, + ADC5_ABSOLUTE_CAL, +}; + +enum adc5_time_select { + MEAS_INT_DISABLE = 0, + MEAS_INT_IMMEDIATE, + MEAS_INT_50MS, + MEAS_INT_100MS, + MEAS_INT_1S, + MEAS_INT_NONE, +}; + +/** + * struct adc5_sdam_data - data per SDAM allocated for adc usage + * @base_addr: base address for the ADC SDAM peripheral. + * @irq_name: ADC IRQ name. + * @irq: ADC IRQ number. + */ +struct adc5_sdam_data { + u16 base_addr; + const char *irq_name; + int irq; +}; + +/** + * struct adc5_device_data - Top-level ADC device data + * @regmap: ADC peripheral register map field. + * @base: array of SDAM data. + * @num_sdams: number of ADC SDAM peripherals. + */ +struct adc5_device_data { + struct regmap *regmap; + struct adc5_sdam_data *base; + int num_sdams; +}; + +/** + * struct adc5_channel_common_prop - ADC channel properties (common to ADC and TM). + * @channel: channel number, refer to the channel list. + * @cal_method: calibration method. + * @decimation: sampling rate supported for the channel. + * @sid: ID of PMIC owning the channel. + * @label: Channel name used in device tree. + * @prescale: channel scaling performed on the input signal. + * @hw_settle_time_us: the time between AMUX being configured and the + * start of conversion in uS. + * @avg_samples: ability to provide single result from the ADC + * that is an average of multiple measurements. + * @scale_fn_type: Represents the scaling function to convert voltage + * physical units desired by the client for the channel. + */ +struct adc5_channel_common_prop { + unsigned int channel; + enum adc5_cal_method cal_method; + unsigned int decimation; + unsigned int sid; + const char *label; + unsigned int prescale; + unsigned int hw_settle_time_us; + unsigned int avg_samples; + enum vadc_scale_fn_type scale_fn_type; +}; + +/** + * struct tm5_aux_dev_wrapper - wrapper structure around TM auxiliary device + * @aux_dev: TM auxiliary device structure. + * @dev_data: Top-level ADC device data. + * @tm_props: Array of common ADC channel properties for TM channels. + * @n_tm_channels: number of TM channels. + */ +struct tm5_aux_dev_wrapper { + struct auxiliary_device aux_dev; + struct adc5_device_data *dev_data; + struct adc5_channel_common_prop *tm_props; + unsigned int n_tm_channels; +}; + +int adc5_gen3_read(struct adc5_device_data *adc, unsigned int sdam_index, + u16 offset, u8 *data, int len); + +int adc5_gen3_write(struct adc5_device_data *adc, unsigned int sdam_index, + u16 offset, u8 *data, int len); + +int adc5_gen3_poll_wait_hs(struct adc5_device_data *adc, + unsigned int sdam_index); + +void adc5_gen3_update_dig_param(struct adc5_channel_common_prop *prop, + u8 *data); + +int adc5_gen3_status_clear(struct adc5_device_data *adc, + int sdam_index, u16 offset, u8 *val, int len); + +void adc5_gen3_mutex_lock(struct device *dev); +void adc5_gen3_mutex_unlock(struct device *dev); +int adc5_gen3_get_scaled_reading(struct device *dev, + struct adc5_channel_common_prop *common_props, + int *val); +int adc5_gen3_therm_code_to_temp(struct device *dev, + struct adc5_channel_common_prop *common_props, + u16 code, int *val); + +#endif /* QCOM_ADC5_GEN3_COMMON_H */ diff --git a/include/linux/iio/backend.h b/include/linux/iio/backend.h index 7f815f3fed6a..3f95ed1fdf9e 100644 --- a/include/linux/iio/backend.h +++ b/include/linux/iio/backend.h @@ -85,6 +85,27 @@ enum iio_backend_filter_type { }; /** + * enum iio_backend_capabilities - Backend capabilities + * Backend capabilities can be used by frontends to check if a given + * functionality is supported by the backend. This is useful for frontend + * devices which are expected to work with alternative backend + * implementations. Capabilities are loosely coupled with operations, + * meaning that a capability requires certain operations to be implemented + * by the backend. A capability might be mapped to a single operation or + * multiple operations. + * + * @IIO_BACKEND_CAP_CALIBRATION: Backend supports digital interface + * calibration. Calibration procedure is device specific. + * @IIO_BACKEND_CAP_BUFFER: Support for IIO buffer interface. + * @IIO_BACKEND_CAP_ENABLE: Backend can be explicitly enabled/disabled. + */ +enum iio_backend_capabilities { + IIO_BACKEND_CAP_CALIBRATION = BIT(0), + IIO_BACKEND_CAP_BUFFER = BIT(1), + IIO_BACKEND_CAP_ENABLE = BIT(2), +}; + +/** * struct iio_backend_ops - operations structure for an iio_backend * @enable: Enable backend. * @disable: Disable backend. @@ -179,10 +200,12 @@ struct iio_backend_ops { * struct iio_backend_info - info structure for an iio_backend * @name: Backend name. * @ops: Backend operations. + * @caps: Backend capabilities. (bitmask of enum iio_backend_capabilities). */ struct iio_backend_info { const char *name; const struct iio_backend_ops *ops; + u32 caps; }; int iio_backend_chan_enable(struct iio_backend *back, unsigned int chan); @@ -235,8 +258,10 @@ int iio_backend_read_raw(struct iio_backend *back, long mask); int iio_backend_extend_chan_spec(struct iio_backend *back, struct iio_chan_spec *chan); +bool iio_backend_has_caps(struct iio_backend *back, u32 caps); void *iio_backend_get_priv(const struct iio_backend *conv); struct iio_backend *devm_iio_backend_get(struct device *dev, const char *name); +struct iio_backend *devm_iio_backend_get_by_index(struct device *dev, unsigned int index); struct iio_backend *devm_iio_backend_fwnode_get(struct device *dev, const char *name, struct fwnode_handle *fwnode); diff --git a/include/linux/iio/buffer.h b/include/linux/iio/buffer.h index d37f82678f71..745c98ef4e04 100644 --- a/include/linux/iio/buffer.h +++ b/include/linux/iio/buffer.h @@ -34,8 +34,16 @@ static inline int iio_push_to_buffers_with_timestamp(struct iio_dev *indio_dev, void *data, int64_t timestamp) { if (ACCESS_PRIVATE(indio_dev, scan_timestamp)) { - size_t ts_offset = indio_dev->scan_bytes / sizeof(int64_t) - 1; - ((int64_t *)data)[ts_offset] = timestamp; + size_t ts_offset = ACCESS_PRIVATE(indio_dev, scan_timestamp_offset); + + /* + * The size of indio_dev->scan_bytes is always aligned to the + * largest scan element's alignment (see iio_compute_scan_bytes()). + * So there may be padding after the timestamp. ts_offset contains + * the offset in bytes that was already computed for correctly + * aligning the timestamp. + */ + *(int64_t *)(data + ts_offset) = timestamp; } return iio_push_to_buffers(indio_dev, data); @@ -71,7 +79,7 @@ static inline int iio_push_to_buffers_with_ts(struct iio_dev *indio_dev, int iio_push_to_buffers_with_ts_unaligned(struct iio_dev *indio_dev, const void *data, size_t data_sz, - int64_t timestamp); + s64 timestamp); bool iio_validate_scan_mask_onehot(struct iio_dev *indio_dev, const unsigned long *mask); diff --git a/include/linux/iio/common/inv_sensors_timestamp.h b/include/linux/iio/common/inv_sensors_timestamp.h index 8d506f1e9df2..4f08204ede3b 100644 --- a/include/linux/iio/common/inv_sensors_timestamp.h +++ b/include/linux/iio/common/inv_sensors_timestamp.h @@ -13,9 +13,9 @@ * @init_period: chip initial period at reset in ns */ struct inv_sensors_timestamp_chip { - uint32_t clock_period; - uint32_t jitter; - uint32_t init_period; + u32 clock_period; + u32 jitter; + u32 init_period; }; /** @@ -24,8 +24,8 @@ struct inv_sensors_timestamp_chip { * @up: interval upper bound */ struct inv_sensors_timestamp_interval { - int64_t lo; - int64_t up; + s64 lo; + s64 up; }; /** @@ -35,9 +35,9 @@ struct inv_sensors_timestamp_interval { * @values: table of all measured values, use for computing the mean */ struct inv_sensors_timestamp_acc { - uint32_t val; + u32 val; size_t idx; - uint32_t values[32]; + u32 values[32]; }; /** @@ -46,6 +46,8 @@ struct inv_sensors_timestamp_acc { * @min_period: minimal acceptable clock period * @max_period: maximal acceptable clock period * @it: interrupts interval timestamps + * @delta: interval timestamps between several interrupts + * @delta_counter: number of data samples in the delta interval * @timestamp: store last timestamp for computing next data timestamp * @mult: current internal period multiplier * @new_mult: new set internal period multiplier (not yet effective) @@ -54,13 +56,15 @@ struct inv_sensors_timestamp_acc { */ struct inv_sensors_timestamp { struct inv_sensors_timestamp_chip chip; - uint32_t min_period; - uint32_t max_period; + u32 min_period; + u32 max_period; struct inv_sensors_timestamp_interval it; - int64_t timestamp; - uint32_t mult; - uint32_t new_mult; - uint32_t period; + struct inv_sensors_timestamp_interval delta; + u32 delta_counter; + s64 timestamp; + u32 mult; + u32 new_mult; + u32 period; struct inv_sensors_timestamp_acc chip_period; }; @@ -68,19 +72,19 @@ void inv_sensors_timestamp_init(struct inv_sensors_timestamp *ts, const struct inv_sensors_timestamp_chip *chip); int inv_sensors_timestamp_update_odr(struct inv_sensors_timestamp *ts, - uint32_t period, bool fifo); + u32 period, bool fifo); void inv_sensors_timestamp_interrupt(struct inv_sensors_timestamp *ts, - size_t sample_nb, int64_t timestamp); + size_t sample_nb, s64 timestamp); -static inline int64_t inv_sensors_timestamp_pop(struct inv_sensors_timestamp *ts) +static inline s64 inv_sensors_timestamp_pop(struct inv_sensors_timestamp *ts) { ts->timestamp += ts->period; return ts->timestamp; } void inv_sensors_timestamp_apply_odr(struct inv_sensors_timestamp *ts, - uint32_t fifo_period, size_t fifo_nb, + u32 fifo_period, size_t fifo_nb, unsigned int fifo_no); static inline void inv_sensors_timestamp_reset(struct inv_sensors_timestamp *ts) @@ -88,6 +92,8 @@ static inline void inv_sensors_timestamp_reset(struct inv_sensors_timestamp *ts) const struct inv_sensors_timestamp_interval interval_init = {0LL, 0LL}; ts->it = interval_init; + ts->delta = interval_init; + ts->delta_counter = 0; ts->timestamp = 0; } diff --git a/include/linux/iio/common/st_sensors.h b/include/linux/iio/common/st_sensors.h index f9ae5cdd884f..1ba496f0fea5 100644 --- a/include/linux/iio/common/st_sensors.h +++ b/include/linux/iio/common/st_sensors.h @@ -160,12 +160,12 @@ struct st_sensor_int_drdy { /** * struct st_sensor_data_ready_irq - ST sensor device data-ready interrupt - * struct int1 - data-ready configuration register for INT1 pin. - * struct int2 - data-ready configuration register for INT2 pin. + * @int1: data-ready configuration register for INT1 pin. + * @int2: data-ready configuration register for INT2 pin. * @addr_ihl: address to enable/disable active low on the INT lines. * @mask_ihl: mask to enable/disable active low on the INT lines. - * struct stat_drdy - status register of DRDY (data ready) interrupt. - * struct ig1 - represents the Interrupt Generator 1 of sensors. + * @stat_drdy: status register of DRDY (data ready) interrupt. + * @ig1: represents the Interrupt Generator 1 of sensors. * @en_addr: address of the enable ig1 register. * @en_mask: mask to write the on/off value for enable. */ @@ -190,6 +190,7 @@ struct st_sensor_data_ready_irq { * @wai_addr: The address of WhoAmI register. * @sensors_supported: List of supported sensors by struct itself. * @ch: IIO channels for the sensor. + * @num_ch: Number of IIO channels in @ch * @odr: Output data rate register and ODR list available. * @pw: Power register of the sensor. * @enable_axis: Enable one or more axis of the sensor. @@ -228,7 +229,7 @@ struct st_sensor_settings { * @regmap: Pointer to specific sensor regmap configuration. * @enabled: Status of the sensor (false->off, true->on). * @odr: Output data rate of the sensor [Hz]. - * num_data_channels: Number of data channels used in buffer. + * @num_data_channels: Number of data channels used in buffer. * @drdy_int_pin: Redirect DRDY on pin 1 (1) or pin 2 (2). * @int_pin_open_drain: Set the interrupt/DRDY to open drain. * @irq: the IRQ number. diff --git a/include/linux/iio/iio.h b/include/linux/iio/iio.h index a9ecff191bd9..711c00f67371 100644 --- a/include/linux/iio/iio.h +++ b/include/linux/iio/iio.h @@ -177,8 +177,31 @@ struct iio_event_spec { }; /** + * define IIO_SCAN_FORMAT_SIGNED_INT - signed integer data format + * + * &iio_scan_type.format value for signed integers (two's complement). + */ +#define IIO_SCAN_FORMAT_SIGNED_INT 's' + +/** + * define IIO_SCAN_FORMAT_UNSIGNED_INT - unsigned integer data format + * + * &iio_scan_type.format value for unsigned integers. + */ +#define IIO_SCAN_FORMAT_UNSIGNED_INT 'u' + +/** + * define IIO_SCAN_FORMAT_FLOAT - floating-point data format + * + * &iio_scan_type.format value for IEEE 754 floating-point numbers. + */ +#define IIO_SCAN_FORMAT_FLOAT 'f' + +/** * struct iio_scan_type - specification for channel data format in buffer - * @sign: 's' or 'u' to specify signed or unsigned + * @sign: Deprecated, use @format instead. + * @format: Data format, can have any of the IIO_SCAN_FORMAT_* + * values. * @realbits: Number of valid bits of data * @storagebits: Realbits + padding * @shift: Shift right by this before masking out realbits. @@ -189,7 +212,10 @@ struct iio_event_spec { * @endianness: little or big endian */ struct iio_scan_type { - char sign; + union { + char sign; + char format; + }; u8 realbits; u8 storagebits; u8 shift; @@ -327,15 +353,15 @@ static inline bool iio_channel_has_available(const struct iio_chan_spec *chan, (chan->info_mask_shared_by_all_available & BIT(type)); } -#define IIO_CHAN_SOFT_TIMESTAMP(_si) { \ +#define IIO_CHAN_SOFT_TIMESTAMP(_si) (struct iio_chan_spec) { \ .type = IIO_TIMESTAMP, \ .channel = -1, \ .scan_index = _si, \ .scan_type = { \ .sign = 's', \ - .realbits = 64, \ + .realbits = 64, \ .storagebits = 64, \ - }, \ + }, \ } s64 iio_get_time_ns(const struct iio_dev *indio_dev); @@ -584,6 +610,8 @@ struct iio_buffer_setup_ops { * and owner * @buffer: [DRIVER] any buffer present * @scan_bytes: [INTERN] num bytes captured to be fed to buffer demux + * @scan_timestamp_offset: [INTERN] cache of the offset (in bytes) for the + * timestamp in the scan buffer * @available_scan_masks: [DRIVER] optional array of allowed bitmasks. Sort the * array in order of preference, the most preferred * masks first. @@ -610,6 +638,7 @@ struct iio_dev { struct iio_buffer *buffer; int scan_bytes; + unsigned int __private scan_timestamp_offset; const unsigned long *available_scan_masks; unsigned int __private masklength; @@ -931,6 +960,18 @@ static inline void *iio_device_get_drvdata(const struct iio_dev *indio_dev) #define IIO_DECLARE_DMA_BUFFER_WITH_TS(type, name, count) \ __IIO_DECLARE_BUFFER_WITH_TS(type, name, count) __aligned(IIO_DMA_MINALIGN) +/** + * IIO_DECLARE_QUATERNION() - Declare a quaternion element + * @type: element type of the individual vectors + * @name: identifier name + * + * Quaternions are a vector composed of 4 elements (W, X, Y, Z). Use this macro + * to declare a quaternion element in a struct to ensure proper alignment in + * an IIO buffer. + */ +#define IIO_DECLARE_QUATERNION(type, name) \ + type name[4] __aligned(sizeof(type) * 4) + struct iio_dev *iio_device_alloc(struct device *parent, int sizeof_priv); /* The information at the returned address is guaranteed to be cacheline aligned */ diff --git a/include/linux/iio/types.h b/include/linux/iio/types.h index 34eebad12d2c..d8944c9e5e90 100644 --- a/include/linux/iio/types.h +++ b/include/linux/iio/types.h @@ -7,6 +7,9 @@ #ifndef _IIO_TYPES_H_ #define _IIO_TYPES_H_ +#include <linux/types.h> +#include <linux/wordpart.h> + #include <uapi/linux/iio/types.h> enum iio_event_info { @@ -21,6 +24,7 @@ enum iio_event_info { IIO_EV_INFO_TAP2_MIN_DELAY, IIO_EV_INFO_RUNNING_PERIOD, IIO_EV_INFO_RUNNING_COUNT, + IIO_EV_INFO_SCALE, }; #define IIO_VAL_INT 1 @@ -33,6 +37,24 @@ enum iio_event_info { #define IIO_VAL_FRACTIONAL_LOG2 11 #define IIO_VAL_CHAR 12 +#define IIO_VAL_DECIMAL64_BASE 32 +#define IIO_VAL_DECIMAL64_MILLI (IIO_VAL_DECIMAL64_BASE + 3) +#define IIO_VAL_DECIMAL64_MICRO (IIO_VAL_DECIMAL64_BASE + 6) +#define IIO_VAL_DECIMAL64_NANO (IIO_VAL_DECIMAL64_BASE + 9) +#define IIO_VAL_DECIMAL64_PICO (IIO_VAL_DECIMAL64_BASE + 12) +#define IIO_VAL_DECIMAL64_FEMTO (IIO_VAL_DECIMAL64_BASE + 15) + +static inline s64 iio_val_s64_compose(s32 val0, s32 val1) +{ + return (s64)(((u64)val1 << 32) | (u32)val0); +} + +static inline void iio_val_s64_decompose(s64 dec64, s32 *val0, s32 *val1) +{ + *val0 = lower_32_bits(dec64); + *val1 = upper_32_bits(dec64); +} + enum iio_available_type { IIO_AVAIL_LIST, IIO_AVAIL_RANGE, diff --git a/include/linux/ima.h b/include/linux/ima.h index abf8923f8fc5..8e08baf16c2f 100644 --- a/include/linux/ima.h +++ b/include/linux/ima.h @@ -11,6 +11,7 @@ #include <linux/fs.h> #include <linux/security.h> #include <linux/kexec.h> +#include <linux/secure_boot.h> #include <crypto/hash_info.h> struct linux_binprm; @@ -73,14 +74,8 @@ int ima_validate_range(phys_addr_t phys, size_t size); #endif #ifdef CONFIG_IMA_SECURE_AND_OR_TRUSTED_BOOT -extern bool arch_ima_get_secureboot(void); extern const char * const *arch_get_ima_policy(void); #else -static inline bool arch_ima_get_secureboot(void) -{ - return false; -} - static inline const char * const *arch_get_ima_policy(void) { return NULL; diff --git a/include/linux/indirect_call_wrapper.h b/include/linux/indirect_call_wrapper.h index dc272b514a01..0e4340ecd857 100644 --- a/include/linux/indirect_call_wrapper.h +++ b/include/linux/indirect_call_wrapper.h @@ -57,7 +57,7 @@ * builtin, this macro simplify dealing with indirect calls with only ipv4/ipv6 * alternatives */ -#if IS_BUILTIN(CONFIG_IPV6) +#if IS_ENABLED(CONFIG_IPV6) #define INDIRECT_CALL_INET(f, f2, f1, ...) \ INDIRECT_CALL_2(f, f2, f1, __VA_ARGS__) #elif IS_ENABLED(CONFIG_INET) diff --git a/include/linux/inetdevice.h b/include/linux/inetdevice.h index dccbeb25f701..6032eea2539a 100644 --- a/include/linux/inetdevice.h +++ b/include/linux/inetdevice.h @@ -293,6 +293,11 @@ static inline void in_dev_put(struct in_device *idev) #define __in_dev_put(idev) refcount_dec(&(idev)->refcnt) #define in_dev_hold(idev) refcount_inc(&(idev)->refcnt) +static inline bool in_dev_hold_safe(struct in_device *idev) +{ + return refcount_inc_not_zero(&idev->refcnt); +} + #endif /* __KERNEL__ */ static __inline__ __be32 inet_make_mask(int logmask) diff --git a/include/linux/init.h b/include/linux/init.h index 40331923b9f4..6326c61e2332 100644 --- a/include/linux/init.h +++ b/include/linux/init.h @@ -47,7 +47,7 @@ #define __initdata __section(".init.data") #define __initconst __section(".init.rodata") #define __exitdata __section(".exit.data") -#define __exit_call __used __section(".exitcall.exit") +#define __exit_call __maybe_unused __section(".exitcall.exit") /* * modpost check for section mismatches during the kernel build. diff --git a/include/linux/init_task.h b/include/linux/init_task.h index a6cb241ea00c..61536be773f5 100644 --- a/include/linux/init_task.h +++ b/include/linux/init_task.h @@ -24,6 +24,7 @@ extern struct files_struct init_files; extern struct fs_struct init_fs; +extern struct fs_struct *userspace_init_fs; extern struct nsproxy init_nsproxy; #ifndef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE diff --git a/include/linux/input.h b/include/linux/input.h index 7d7cb0593a63..3381608127f7 100644 --- a/include/linux/input.h +++ b/include/linux/input.h @@ -19,7 +19,7 @@ #include <linux/device.h> #include <linux/fs.h> #include <linux/timer.h> -#include <linux/mod_devicetable.h> +#include <linux/device-id/input.h> struct input_dev_poller; @@ -117,7 +117,8 @@ enum input_clock_type { * user opens device and dev->close() is called when the very * last user closes the device * @going_away: marks devices that are in a middle of unregistering and - * causes input_open_device*() fail with -ENODEV. + * causes input_open_device() and input_inhibit/uninhibit_device() + * to fail with -ENODEV. * @dev: driver model's view of this device * @h_list: list of input handles associated with the device. When * accessing the list dev->mutex must be held @@ -128,11 +129,14 @@ enum input_clock_type { * @devres_managed: indicates that devices is managed with devres framework * and needs not be explicitly unregistered or freed. * @timestamp: storage for a timestamp set by input_set_timestamp called - * by a driver + * by a driver * @inhibited: indicates that the input device is inhibited. If that is - * the case then input core ignores any events generated by the device. - * Device's close() is called when it is being inhibited and its open() - * is called when it is being uninhibited. + * the case then input core ignores any events generated by the device. + * Device's close() is called when it is being inhibited and its open() + * is called when it is being uninhibited. + * @ready: indicates that the device has been successfully opened and is + * prepared to process events (like LEDs or sounds) sent from the + * input core. */ struct input_dev { const char *name; @@ -209,6 +213,7 @@ struct input_dev { ktime_t timestamp[INPUT_CLK_MAX]; bool inhibited; + bool ready; }; #define to_input_dev(d) container_of(d, struct input_dev, dev) @@ -284,8 +289,9 @@ struct input_handle; * @connect: called when attaching a handler to an input device * @disconnect: disconnects a handler from input device * @start: starts handler for given handle. This function is called by - * input core right after connect() method and also when a process - * that "grabbed" a device releases it + * input core when device is open and ready to process events, + * and also when device is uninhibited or when a process that "grabbed" + * a device releases it * @passive_observer: set to %true by drivers only interested in observing * data stream from devices if there are other users present. Such * drivers will not result in starting underlying hardware device @@ -517,6 +523,10 @@ INPUT_GENERATE_ABS_ACCESSORS(res, resolution) int input_scancode_to_scalar(const struct input_keymap_entry *ke, unsigned int *scancode); +int input_default_setkeycode(struct input_dev *dev, + const struct input_keymap_entry *ke, + unsigned int *old_keycode); + int input_get_keycode(struct input_dev *dev, struct input_keymap_entry *ke); int input_set_keycode(struct input_dev *dev, const struct input_keymap_entry *ke); @@ -539,6 +549,8 @@ extern const struct class input_class; * @set_autocenter: Called to auto-center device * @destroy: called by input core when parent input device is being * destroyed + * @stop: called by input core when parent input device is being + * unregistered * @private: driver-specific data, will be freed automatically * @ffbit: bitmap of force feedback capabilities truly supported by * device (not emulated like ones in input_dev->ffbit) @@ -567,6 +579,7 @@ struct ff_device { void (*set_autocenter)(struct input_dev *dev, u16 magnitude); void (*destroy)(struct ff_device *); + void (*stop)(struct ff_device *); void *private; diff --git a/include/linux/input/elan-i2c-ids.h b/include/linux/input/elan-i2c-ids.h index 51cca17ee94c..874bf0ab500c 100644 --- a/include/linux/input/elan-i2c-ids.h +++ b/include/linux/input/elan-i2c-ids.h @@ -18,7 +18,7 @@ #ifndef __ELAN_I2C_IDS_H #define __ELAN_I2C_IDS_H -#include <linux/mod_devicetable.h> +#include <linux/device-id/acpi.h> static const struct acpi_device_id elan_acpi_id[] = { { "ELAN0000", 0 }, diff --git a/include/linux/instruction_pointer.h b/include/linux/instruction_pointer.h index aa0b3ffea935..ea5bc756bd99 100644 --- a/include/linux/instruction_pointer.h +++ b/include/linux/instruction_pointer.h @@ -8,6 +8,30 @@ #ifndef _THIS_IP_ #define _THIS_IP_ ({ __label__ __here; __here: (unsigned long)&&__here; }) +/* + * The current generic definition of _THIS_IP_ is considered broken by GCC [1] + * and Clang [2]. In particular, the address of a label is only expected to be + * used with a computed goto. + * + * [1] https://gcc.gnu.org/bugzilla/show_bug.cgi?id=120071 + * [2] https://github.com/llvm/llvm-project/issues/138272 + * + * Mark it as broken, so that appropriate fallback options can be implemented + * for architectures that do not define their own _THIS_IP_. + */ +#define HAS_BROKEN_THIS_IP +#endif + +/* + * _CODE_LOCATION_ provides a unique identifier for the current code location. + * When _THIS_IP_ is broken (generic version), we fall back to a static marker + * which guarantees uniqueness and resolves to a constant address at link time, + * avoiding runtime overhead and compiler optimizations breaking it. + */ +#ifdef HAS_BROKEN_THIS_IP +#define _CODE_LOCATION_ ({ static const char __here; (unsigned long)&__here; }) +#else +#define _CODE_LOCATION_ _THIS_IP_ #endif #endif /* _LINUX_INSTRUCTION_POINTER_H */ diff --git a/include/linux/intel-ish-client-if.h b/include/linux/intel-ish-client-if.h index 2cd4f65aaa37..a07d952a2b29 100644 --- a/include/linux/intel-ish-client-if.h +++ b/include/linux/intel-ish-client-if.h @@ -9,7 +9,7 @@ #define _INTEL_ISH_CLIENT_IF_H_ #include <linux/device.h> -#include <linux/mod_devicetable.h> +#include <linux/device-id/ishtp.h> struct ishtp_cl_device; struct ishtp_device; diff --git a/include/linux/intel_rapl.h b/include/linux/intel_rapl.h index fa1f328d6712..328004f605c3 100644 --- a/include/linux/intel_rapl.h +++ b/include/linux/intel_rapl.h @@ -77,7 +77,6 @@ enum rapl_primitives { PSYS_TIME_WINDOW1, PSYS_TIME_WINDOW2, /* below are not raw primitive data */ - AVERAGE_POWER, NR_RAPL_PRIMITIVES, }; @@ -128,6 +127,46 @@ struct reg_action { int err; }; +struct rapl_defaults { + u8 floor_freq_reg_addr; + int (*check_unit)(struct rapl_domain *rd); + void (*set_floor_freq)(struct rapl_domain *rd, bool mode); + u64 (*compute_time_window)(struct rapl_domain *rd, u64 val, bool to_raw); + unsigned int dram_domain_energy_unit; + unsigned int psys_domain_energy_unit; + bool spr_psys_bits; + bool msr_pl4_support; + bool msr_pmu_support; +}; + +#define PRIMITIVE_INFO_INIT(p, m, s, i, u, f) { \ + .name = #p, \ + .mask = m, \ + .shift = s, \ + .id = i, \ + .unit = u, \ + .flag = f \ + } + +enum unit_type { + ARBITRARY_UNIT, /* no translation */ + POWER_UNIT, + ENERGY_UNIT, + TIME_UNIT, +}; + +/* per domain data. used to describe individual knobs such that access function + * can be consolidated into one instead of many inline functions. + */ +struct rapl_primitive_info { + const char *name; + u64 mask; + int shift; + enum rapl_domain_reg_id id; + enum unit_type unit; + u32 flag; +}; + /** * struct rapl_if_priv: private data for different RAPL interfaces * @control_type: Each RAPL interface must have its own powercap @@ -142,8 +181,8 @@ struct reg_action { * registers. * @write_raw: Callback for writing RAPL interface specific * registers. - * @defaults: internal pointer to interface default settings - * @rpi: internal pointer to interface primitive info + * @defaults: pointer to default settings + * @rpi: pointer to interface primitive info */ struct rapl_if_priv { enum rapl_if_type type; @@ -154,8 +193,8 @@ struct rapl_if_priv { int limits[RAPL_DOMAIN_MAX]; int (*read_raw)(int id, struct reg_action *ra, bool pmu_ctx); int (*write_raw)(int id, struct reg_action *ra); - void *defaults; - void *rpi; + const struct rapl_defaults *defaults; + struct rapl_primitive_info *rpi; }; #ifdef CONFIG_PERF_EVENTS @@ -211,6 +250,9 @@ void rapl_remove_package_cpuslocked(struct rapl_package *rp); struct rapl_package *rapl_find_package_domain(int id, struct rapl_if_priv *priv, bool id_is_cpu); struct rapl_package *rapl_add_package(int id, struct rapl_if_priv *priv, bool id_is_cpu); void rapl_remove_package(struct rapl_package *rp); +int rapl_default_check_unit(struct rapl_domain *rd); +void rapl_default_set_floor_freq(struct rapl_domain *rd, bool mode); +u64 rapl_default_compute_time_window(struct rapl_domain *rd, u64 value, bool to_raw); #ifdef CONFIG_PERF_EVENTS int rapl_package_add_pmu(struct rapl_package *rp); diff --git a/include/linux/intel_tpmi.h b/include/linux/intel_tpmi.h index 94c06bf214fb..15f02422e9ca 100644 --- a/include/linux/intel_tpmi.h +++ b/include/linux/intel_tpmi.h @@ -28,6 +28,12 @@ enum intel_tpmi_id { TPMI_INFO_ID = 0x81, /* Special ID for PCI BDF and Package ID information */ }; +#define TPMI_CORE_INIT 0 +#define TPMI_CORE_EXIT 1 + +int tpmi_register_notifier(struct notifier_block *nb); +int tpmi_unregister_notifier(struct notifier_block *nb); + struct oobmsm_plat_info *tpmi_get_platform_data(struct auxiliary_device *auxdev); struct resource *tpmi_get_resource_at_index(struct auxiliary_device *auxdev, int index); int tpmi_get_resource_count(struct auxiliary_device *auxdev); diff --git a/include/linux/intel_vsec.h b/include/linux/intel_vsec.h index 1a0f357c2427..843cda8f8644 100644 --- a/include/linux/intel_vsec.h +++ b/include/linux/intel_vsec.h @@ -28,10 +28,17 @@ #define INTEL_DVSEC_TABLE_BAR(x) ((x) & GENMASK(2, 0)) #define INTEL_DVSEC_TABLE_OFFSET(x) ((x) & GENMASK(31, 3)) #define TABLE_OFFSET_SHIFT 3 +#define PMT_DISC_DWORDS 4 +struct device; struct pci_dev; struct resource; +enum intel_vsec_disc_source { + INTEL_VSEC_DISC_PCI, /* PCI, default */ + INTEL_VSEC_DISC_ACPI, /* ACPI */ +}; + enum intel_vsec_id { VSEC_ID_TELEMETRY = 2, VSEC_ID_WATCHER = 3, @@ -82,14 +89,14 @@ enum intel_vsec_quirks { * struct pmt_callbacks - Callback infrastructure for PMT devices * @read_telem: when specified, called by client driver to access PMT * data (instead of direct copy). - * * pdev: PCI device reference for the callback's use + * * dev: device reference for the callback's use * * guid: ID of data to acccss * * data: buffer for the data to be copied * * off: offset into the requested buffer * * count: size of buffer */ struct pmt_callbacks { - int (*read_telem)(struct pci_dev *pdev, u32 guid, u64 *data, loff_t off, u32 count); + int (*read_telem)(struct device *dev, u32 guid, u64 *data, loff_t off, u32 count); }; struct vsec_feature_dependency { @@ -102,6 +109,10 @@ struct vsec_feature_dependency { * @parent: parent device in the auxbus chain * @headers: list of headers to define the PMT client devices to create * @deps: array of feature dependencies + * @acpi_disc: ACPI discovery tables, each entry is two QWORDs + * in little-endian format as defined by the PMT ACPI spec. + * Valid only when @provider == INTEL_VSEC_DISC_ACPI. + * @src: source of discovery table data * @priv_data: private data, usable by parent devices, currently a callback * @caps: bitmask of PMT capabilities for the given headers * @quirks: bitmask of VSEC device quirks @@ -112,6 +123,8 @@ struct intel_vsec_platform_info { struct device *parent; struct intel_vsec_header **headers; const struct vsec_feature_dependency *deps; + u32 (*acpi_disc)[PMT_DISC_DWORDS]; + enum intel_vsec_disc_source src; void *priv_data; unsigned long caps; unsigned long quirks; @@ -122,8 +135,11 @@ struct intel_vsec_platform_info { /** * struct intel_vsec_device - Auxbus specific device information * @auxdev: auxbus device struct for auxbus access - * @pcidev: pci device associated with the device - * @resource: any resources shared by the parent + * @dev: struct device associated with the device + * @acpi_disc: ACPI discovery tables, each entry is two QWORDs + * in little-endian format as defined by the PMT ACPI spec. + * Valid only when @src == INTEL_VSEC_DISC_ACPI. + * @src: source of discovery table data * @ida: id reference * @num_resources: number of resources * @id: xarray id @@ -132,11 +148,14 @@ struct intel_vsec_platform_info { * @quirks: specified quirks * @base_addr: base address of entries (if specified) * @cap_id: the enumerated id of the vsec feature + * @resource: PCI discovery resources (BAR windows), one per discovery + * instance. Valid only when @src == INTEL_VSEC_DISC_PCI */ struct intel_vsec_device { struct auxiliary_device auxdev; - struct pci_dev *pcidev; - struct resource *resource; + struct device *dev; + u32 (*acpi_disc)[PMT_DISC_DWORDS]; + enum intel_vsec_disc_source src; struct ida *ida; int num_resources; int id; /* xa */ @@ -145,6 +164,7 @@ struct intel_vsec_device { unsigned long quirks; u64 base_addr; unsigned long cap_id; + struct resource resource[] __counted_by(num_resources); }; /** @@ -184,7 +204,7 @@ struct pmt_feature_group { struct telemetry_region regions[]; }; -int intel_vsec_add_aux(struct pci_dev *pdev, struct device *parent, +int intel_vsec_add_aux(struct device *parent, struct intel_vsec_device *intel_vsec_dev, const char *name); @@ -199,14 +219,14 @@ static inline struct intel_vsec_device *auxdev_to_ivdev(struct auxiliary_device } #if IS_ENABLED(CONFIG_INTEL_VSEC) -int intel_vsec_register(struct pci_dev *pdev, - struct intel_vsec_platform_info *info); +int intel_vsec_register(struct device *dev, + const struct intel_vsec_platform_info *info); int intel_vsec_set_mapping(struct oobmsm_plat_info *plat_info, struct intel_vsec_device *vsec_dev); struct oobmsm_plat_info *intel_vsec_get_mapping(struct pci_dev *pdev); #else -static inline int intel_vsec_register(struct pci_dev *pdev, - struct intel_vsec_platform_info *info) +static inline int intel_vsec_register(struct device *dev, + const struct intel_vsec_platform_info *info) { return -ENODEV; } diff --git a/include/linux/interconnect.h b/include/linux/interconnect.h index 4b12821528a6..75a32ad0482e 100644 --- a/include/linux/interconnect.h +++ b/include/linux/interconnect.h @@ -47,6 +47,7 @@ struct icc_path *of_icc_get(struct device *dev, const char *name); struct icc_path *devm_of_icc_get(struct device *dev, const char *name); int devm_of_icc_bulk_get(struct device *dev, int num_paths, struct icc_bulk_data *paths); struct icc_path *of_icc_get_by_index(struct device *dev, int idx); +struct icc_path *devm_of_icc_get_by_index(struct device *dev, int idx); void icc_put(struct icc_path *path); int icc_enable(struct icc_path *path); int icc_disable(struct icc_path *path); @@ -79,6 +80,11 @@ static inline struct icc_path *of_icc_get_by_index(struct device *dev, int idx) return NULL; } +static inline struct icc_path *devm_of_icc_get_by_index(struct device *dev, int idx) +{ + return NULL; +} + static inline void icc_put(struct icc_path *path) { } diff --git a/include/linux/interrupt.h b/include/linux/interrupt.h index 6cd26ffb0505..3bf969ad8fe0 100644 --- a/include/linux/interrupt.h +++ b/include/linux/interrupt.h @@ -864,6 +864,7 @@ static inline void init_irq_proc(void) struct seq_file; int show_interrupts(struct seq_file *p, void *v); int arch_show_interrupts(struct seq_file *p, int prec); +void irq_proc_emit_counts(struct seq_file *p, unsigned int __percpu *cnts); extern int early_irq_init(void); extern int arch_probe_nr_irqs(void); diff --git a/include/linux/interrupt_rc.h b/include/linux/interrupt_rc.h new file mode 100644 index 000000000000..a9ed937a80e7 --- /dev/null +++ b/include/linux/interrupt_rc.h @@ -0,0 +1,82 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __LINUX_INTERRUPT_RC_H +#define __LINUX_INTERRUPT_RC_H + +/* + * include/linux/interrupt_rc.h - refcounted local processor interrupt + * management. + * + * Since the implementation of this API currently depends on + * local_irq_save()/local_irq_restore(), we split this into its own header to + * make it easier to include without hitting circular header dependencies. + */ + +#include <linux/irqflags.h> +#include <linux/preempt.h> +#include <linux/processor.h> +#include <linux/smp.h> + +#ifndef MODULE +/* Per-CPU interrupt disabling state for local_interrupt_{disable,enable}(). */ +DECLARE_PER_CPU(unsigned long, local_interrupt_disable_state); + +static __always_inline void __local_interrupt_save_state(unsigned long flags) +{ + raw_cpu_write(local_interrupt_disable_state, flags); +} + +static __always_inline void __local_interrupt_enable(void) +{ + unsigned long flags = raw_cpu_read(local_interrupt_disable_state); + + local_irq_restore(flags); +} + +#ifndef INSTANTIATE_EXPORTED_INTERRUPT_DISABLE +static __always_inline void _local_interrupt_save_state(unsigned long flags) +{ + __local_interrupt_save_state(flags); +} + +static __always_inline void _local_interrupt_enable(void) +{ + __local_interrupt_enable(); +} +#else +extern void _local_interrupt_save_state(unsigned long flags); +extern void _local_interrupt_enable(void); +#endif + +#else /* !MODULE */ +extern void _local_interrupt_save_state(unsigned long flags); +extern void _local_interrupt_enable(void); +#endif /* !MODULE */ + +#define hardirq_disable_enter() __preempt_count_add_return(HARDIRQ_DISABLE_OFFSET) +#define hardirq_disable_exit() __preempt_count_sub_return(HARDIRQ_DISABLE_OFFSET) + +static inline void local_interrupt_disable(void) +{ + int new_count; + unsigned long flags; + + WARN_ON_ONCE(in_nmi()); + + local_irq_save(flags); + new_count = hardirq_disable_enter(); + + if ((new_count & HARDIRQ_DISABLE_MASK) == HARDIRQ_DISABLE_OFFSET) + _local_interrupt_save_state(flags); +} + +static inline void local_interrupt_enable(void) +{ + int new_count; + + new_count = hardirq_disable_exit(); + + if ((new_count & HARDIRQ_DISABLE_MASK) == 0) + _local_interrupt_enable(); +} + +#endif /* !__LINUX_INTERRUPT_RC_H */ diff --git a/include/linux/io_uring/cmd.h b/include/linux/io_uring/cmd.h index 331dcbefe72f..42801f0b6456 100644 --- a/include/linux/io_uring/cmd.h +++ b/include/linux/io_uring/cmd.h @@ -91,6 +91,15 @@ struct io_br_sel io_uring_cmd_buffer_select(struct io_uring_cmd *ioucmd, bool io_uring_mshot_cmd_post_cqe(struct io_uring_cmd *ioucmd, struct io_br_sel *sel, unsigned int issue_flags); +int io_buffer_register_request(struct io_uring_cmd *cmd, struct request *rq, + void (*release)(void *), unsigned int index, + unsigned int issue_flags); +int io_buffer_register_bvec(struct io_uring_cmd *cmd, const struct bio_vec *bvs, + unsigned int nr_bvecs, void (*release)(void *), + void *priv, u8 dir, unsigned int index, + unsigned int issue_flags); +int io_buffer_unregister(struct io_uring_cmd *cmd, unsigned int index, + unsigned int issue_flags); #else static inline int io_uring_cmd_import_fixed(u64 ubuf, unsigned long len, int rw, @@ -133,6 +142,29 @@ static inline bool io_uring_mshot_cmd_post_cqe(struct io_uring_cmd *ioucmd, { return true; } +static inline int io_buffer_register_request(struct io_uring_cmd *cmd, + struct request *rq, + void (*release)(void *), + unsigned int index, + unsigned int issue_flags) +{ + return -EOPNOTSUPP; +} +static inline int io_buffer_register_bvec(struct io_uring_cmd *cmd, + const struct bio_vec *bvs, + unsigned int nr_bvecs, + void (*release)(void *), void *priv, + u8 dir, unsigned int index, + unsigned int issue_flags) +{ + return -EOPNOTSUPP; +} +static inline int io_buffer_unregister(struct io_uring_cmd *cmd, + unsigned int index, + unsigned int issue_flags) +{ + return -EOPNOTSUPP; +} #endif static inline struct io_uring_cmd *io_uring_cmd_from_tw(struct io_tw_req tw_req) @@ -182,10 +214,4 @@ static inline void io_uring_cmd_done32(struct io_uring_cmd *ioucmd, s32 ret, return __io_uring_cmd_done(ioucmd, ret, res2, issue_flags, true); } -int io_buffer_register_bvec(struct io_uring_cmd *cmd, struct request *rq, - void (*release)(void *), unsigned int index, - unsigned int issue_flags); -int io_buffer_unregister_bvec(struct io_uring_cmd *cmd, unsigned int index, - unsigned int issue_flags); - #endif /* _LINUX_IO_URING_CMD_H */ diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h index 214fdbd49052..39629ee77b91 100644 --- a/include/linux/io_uring_types.h +++ b/include/linux/io_uring_types.h @@ -1,13 +1,17 @@ #ifndef IO_URING_TYPES_H #define IO_URING_TYPES_H -#include <linux/blkdev.h> +#include <linux/blk_plug.h> #include <linux/hashtable.h> #include <linux/task_work.h> #include <linux/bitmap.h> #include <linux/llist.h> +#include <linux/uio.h> #include <uapi/linux/io_uring.h> +struct iou_loop_params; +struct io_uring_bpf_ops; + enum { /* * A hint to not wake right away but delay until there are enough of @@ -17,6 +21,14 @@ enum { * It's also ignored unless IORING_SETUP_DEFER_TASKRUN is set. */ IOU_F_TWQ_LAZY_WAKE = 1, + + /* + * Set when task_work is queued from a waitqueue wakeup handler, where + * an arbitrary provider waitqueue lock is held. Signaling the CQ ring + * eventfd inline from there can recurse back into that lock through + * epoll, so the eventfd signal must be deferred. + */ + IOU_F_TWQ_IN_WAKE = 2, }; enum io_uring_cmd_flags { @@ -41,6 +53,13 @@ enum io_uring_cmd_flags { IO_URING_F_COMPAT = (1 << 12), }; +enum { + IO_BUF_DEST = 1 << ITER_DEST, + IO_BUF_SOURCE = 1 << ITER_SOURCE, +}; + +struct iou_loop_params; + struct io_wq_work_node { struct io_wq_work_node *next; }; @@ -50,6 +69,18 @@ struct io_wq_work_list { struct io_wq_work_node *last; }; +/* + * Lockless multi-producer, single-consumer FIFO queue, see + * io_uring/mpscq.h for the implementation and rules. Defined here so + * that it can be embedded in io_ring_ctx. This is the producer side + * only - the consumer cursor is kept separately, on a cacheline that + * isn't dirtied by the producers. + */ +struct mpscq { + struct llist_node *tail; /* producers */ + struct llist_node stub; +}; + struct io_wq_work { struct io_wq_work_node list; atomic_t flags; @@ -114,6 +145,11 @@ struct io_uring_task { const struct io_ring_ctx *last; struct task_struct *task; struct io_wq *io_wq; + /* + * Consumer cursor for ->task_list. Only popped by the task itself, + * or by ->fallback_work once the task can no longer run task_work. + */ + struct llist_node *task_head; struct file *registered_rings[IO_RINGFD_REG_MAX]; struct xarray xa; @@ -122,8 +158,11 @@ struct io_uring_task { atomic_t inflight_tracked; struct percpu_counter inflight; + /* drains ->task_list once the task can no longer run task_work */ + struct work_struct fallback_work; + struct { /* task_work */ - struct llist_head task_list; + struct mpscq task_list; struct callback_head task_work; } ____cacheline_aligned_in_smp; }; @@ -268,24 +307,32 @@ struct io_alloc_cache { unsigned int init_clear; }; +enum { + IO_RING_F_DRAIN_NEXT = BIT(0), + IO_RING_F_OP_RESTRICTED = BIT(1), + IO_RING_F_REG_RESTRICTED = BIT(2), + IO_RING_F_OFF_TIMEOUT_USED = BIT(3), + IO_RING_F_DRAIN_ACTIVE = BIT(4), + IO_RING_F_HAS_EVFD = BIT(5), + /* all CQEs should be posted only by the submitter task */ + IO_RING_F_TASK_COMPLETE = BIT(6), + IO_RING_F_LOCKLESS_CQ = BIT(7), + IO_RING_F_SYSCALL_IOPOLL = BIT(8), + IO_RING_F_POLL_ACTIVATED = BIT(9), + IO_RING_F_DRAIN_DISABLED = BIT(10), + IO_RING_F_COMPAT = BIT(11), + IO_RING_F_IOWQ_LIMITS_SET = BIT(12), +}; + +struct iou_ctx {}; + struct io_ring_ctx { /* const or read-mostly hot data */ struct { + /* ring setup flags */ unsigned int flags; - unsigned int drain_next: 1; - unsigned int op_restricted: 1; - unsigned int reg_restricted: 1; - unsigned int off_timeout_used: 1; - unsigned int drain_active: 1; - unsigned int has_evfd: 1; - /* all CQEs should be posted only by the submitter task */ - unsigned int task_complete: 1; - unsigned int lockless_cq: 1; - unsigned int syscall_iopoll: 1; - unsigned int poll_activated: 1; - unsigned int drain_disabled: 1; - unsigned int compat: 1; - unsigned int iowq_limits_set : 1; + /* internal state flags IO_RING_F_* flags , mostly read-only */ + unsigned int int_flags; struct task_struct *submitter_task; struct io_rings *rings; @@ -335,6 +382,14 @@ struct io_ring_ctx { bool poll_multi_queue; struct list_head iopoll_list; + /* + * Consumer cursor for ->work_list, protected by ->uring_lock. + * Deliberately kept away from the producer side of the queue, + * as it's written for every popped entry, and the producer + * cacheline is contended enough as it is. + */ + struct llist_node *work_head; + struct io_file_table file_table; struct io_rsrc_data buf_table; struct io_alloc_cache node_cache; @@ -355,6 +410,9 @@ struct io_ring_ctx { struct io_alloc_cache rw_cache; struct io_alloc_cache cmd_cache; + int (*loop_step)(struct iou_ctx *, + struct iou_loop_params *); + /* * Any cancelable uring_cmd is added to this list in * ->uring_cmd() by io_uring_cmd_insert_cancelable() @@ -389,8 +447,7 @@ struct io_ring_ctx { */ struct { struct io_rings __rcu *rings_rcu; - struct llist_head work_llist; - struct llist_head retry_llist; + struct mpscq work_list; unsigned long check_cq; atomic_t cq_wait_nr; atomic_t cq_timeouts; @@ -432,6 +489,9 @@ struct io_ring_ctx { /* Stores zcrx object pointers of type struct io_zcrx_ifq */ struct xarray zcrx_ctxs; + /* Used for accounting references on pages in registered buffers */ + struct xarray hpage_acct; + u32 pers_next; struct xarray personalities; @@ -450,8 +510,6 @@ struct io_ring_ctx { struct mutex tctx_lock; /* ctx exit and cancelation */ - struct llist_head fallback_llist; - struct delayed_work fallback_work; struct work_struct exit_work; struct completion ref_comp; @@ -477,6 +535,8 @@ struct io_ring_ctx { DECLARE_HASHTABLE(napi_ht, 4); #endif + struct io_uring_bpf_ops *bpf_ops; + /* * Protection for resize vs mmap races - both the mmap and resize * side will need to grab this lock, to prevent either side from @@ -488,6 +548,8 @@ struct io_ring_ctx { struct io_mapped_region ring_region; /* used for optimised request parameter and wait argument passing */ struct io_mapped_region param_region; + + struct kcov_common_handle_id kcov_handle; }; /* @@ -545,6 +607,7 @@ enum { REQ_F_HAS_METADATA_BIT, REQ_F_IMPORT_BUFFER_BIT, REQ_F_SQE_COPIED_BIT, + REQ_F_IOPOLL_BIT, /* not a real bit, just to check we're not overflowing the space */ __REQ_F_LAST_BIT, @@ -638,6 +701,8 @@ enum { REQ_F_IMPORT_BUFFER = IO_REQ_FLAG(REQ_F_IMPORT_BUFFER_BIT), /* ->sqe_copy() has been called, if necessary */ REQ_F_SQE_COPIED = IO_REQ_FLAG(REQ_F_SQE_COPIED_BIT), + /* request must be iopolled to completion (set in ->issue()) */ + REQ_F_IOPOLL = IO_REQ_FLAG(REQ_F_IOPOLL_BIT), }; struct io_tw_req { @@ -706,8 +771,6 @@ struct io_kiocb { */ u16 buf_index; - unsigned nr_tw; - /* REQ_F_* flags */ io_req_flags_t flags; diff --git a/include/linux/iomap.h b/include/linux/iomap.h index 99b7209dabd7..bc7ae6327dbf 100644 --- a/include/linux/iomap.h +++ b/include/linux/iomap.h @@ -9,7 +9,8 @@ #include <linux/types.h> #include <linux/mm_types.h> #include <linux/blkdev.h> -#include <linux/pagevec.h> +#include <linux/folio_batch.h> +#include <linux/pagemap.h> struct address_space; struct fiemap_extent_info; @@ -65,6 +66,11 @@ struct vm_fault; * * IOMAP_F_ATOMIC_BIO indicates that (write) I/O will be issued as an atomic * bio, i.e. set REQ_ATOMIC. + * + * IOMAP_F_INTEGRITY indicates that the filesystems handles integrity metadata. + * + * IOMAP_F_ZERO_TAIL indicates the remainder of the block after the data + * written should be zeroed. */ #define IOMAP_F_NEW (1U << 0) #define IOMAP_F_DIRTY (1U << 1) @@ -79,6 +85,20 @@ struct vm_fault; #define IOMAP_F_BOUNDARY (1U << 6) #define IOMAP_F_ANON_WRITE (1U << 7) #define IOMAP_F_ATOMIC_BIO (1U << 8) +#ifdef CONFIG_BLK_DEV_INTEGRITY +#define IOMAP_F_INTEGRITY (1U << 9) +#else +#define IOMAP_F_INTEGRITY 0 +#endif /* CONFIG_BLK_DEV_INTEGRITY */ +#define IOMAP_F_ZERO_TAIL (1U << 10) + +/* + * Indicates reads and writes of fsverity metadata. + * + * Fsverity metadata is stored after the regular file data and thus beyond + * i_size. + */ +#define IOMAP_F_FSVERITY (1U << 11) /* * Flag reserved for file system specific usage @@ -136,16 +156,6 @@ static inline void *iomap_inline_data(const struct iomap *iomap, loff_t pos) } /* - * Check if the mapping's length is within the valid range for inline data. - * This is used to guard against accessing data beyond the page inline_data - * points at. - */ -static inline bool iomap_inline_data_valid(const struct iomap *iomap) -{ - return iomap->length <= PAGE_SIZE - offset_in_page(iomap->inline_data); -} - -/* * When get_folio succeeds, put_folio will always be called to do any * cleanup work necessary. put_folio is responsible for unlocking and putting * @folio. @@ -203,24 +213,36 @@ struct iomap_write_ops { #define IOMAP_ATOMIC (1 << 9) /* torn-write protection */ #define IOMAP_DONTCACHE (1 << 10) -struct iomap_ops { - /* - * Return the existing mapping at pos, or reserve space starting at - * pos for up to length, as long as we can do it as a single mapping. - * The actual length is returned in iomap->length. - */ - int (*iomap_begin)(struct inode *inode, loff_t pos, loff_t length, - unsigned flags, struct iomap *iomap, - struct iomap *srcmap); +/* + * Return the existing mapping at pos, or reserve space starting at pos for up + * to length, as long as we can do it as a single mapping. + * The actual length is returned in iomap->length. + */ +typedef int (*iomap_iter_begin_fn)(struct inode *inode, loff_t pos, + loff_t length, unsigned flags, struct iomap *iomap, + struct iomap *srcmap); - /* - * Commit and/or unreserve space previous allocated using iomap_begin. - * Written indicates the length of the successful write operation which - * needs to be commited, while the rest needs to be unreserved. - * Written might be zero if no data was written. - */ - int (*iomap_end)(struct inode *inode, loff_t pos, loff_t length, - ssize_t written, unsigned flags, struct iomap *iomap); +/* + * Commit and/or unreserve space previously allocated by iomap_iter_begin_fn. + * Written indicates the length of the successful write operation which needs + * to be committed, while the rest needs to be unreserved. + * Written might be zero if no data was written. + */ +typedef int (*iomap_iter_end_fn)(struct inode *inode, loff_t pos, loff_t length, + ssize_t written, unsigned flags, struct iomap *iomap); + +/* + * Produce the next mapping (finishing the previous one if needed). + * Return 1 to continue iterating, 0 if the range is fully consumed, or a + * negative error on failure. + */ +typedef int (*iomap_iter_next_fn)(const struct iomap_iter *iter, + struct iomap *iomap, struct iomap *srcmap); + +struct iomap_ops { + iomap_iter_begin_fn iomap_begin; + iomap_iter_end_fn iomap_end; + iomap_iter_next_fn iomap_next; }; /** @@ -308,6 +330,71 @@ static inline const struct iomap *iomap_iter_srcmap(const struct iomap_iter *i) return &i->iomap; } +int iomap_iter_continue(const struct iomap_iter *iter, struct iomap *iomap, + struct iomap *srcmap, int ret); + +/** + * iomap_iter_next - finish the previous mapping and produce the next one + * @iter: iteration structure + * @iomap: mapping to finish and then repopulate + * @srcmap: source mapping to finish and then repopulate + * @begin: callback that produces a mapping for the current position + * @end: optional callback that finishes the previous mapping, or NULL + * + * Inline helper that implements the common body of an ->iomap_next() + * callback: it finishes the previous mapping via @end (if present), decides + * via iomap_iter_continue() whether to keep going, and obtains the next + * mapping via @begin. + * + * This helper is marked __always_inline so that when a caller passes + * compile-time-constant @begin and @end callbacks, the compiler can call them + * directly, avoiding the indirect-call overhead. + * + * Returns 1 to continue iterating, 0 once the range is fully consumed, or a + * negative errno on error. + */ +static __always_inline int iomap_iter_next(const struct iomap_iter *iter, + struct iomap *iomap, struct iomap *srcmap, + iomap_iter_begin_fn begin, iomap_iter_end_fn end) +{ + int ret = 0; + + if (iomap->length) { + if (end) { + /* + * Calculate how far the iter was advanced and the + * original length bytes for end(). + */ + ssize_t advanced = iter->pos - iter->iter_start_pos; + loff_t len; + + len = iomap_length_trim(iter, iter->iter_start_pos, + iter->len + advanced); + + ret = end(iter->inode, iter->iter_start_pos, len, + advanced, iter->flags, iomap); + } + ret = iomap_iter_continue(iter, iomap, srcmap, ret); + if (ret <= 0) + return ret; + } + + ret = begin(iter->inode, iter->pos, iter->len, iter->flags, iomap, + srcmap); + + return ret < 0 ? ret : 1; +} + +#define DEFINE_IOMAP_ITER_NEXT_END(name, begin_fn, end_fn) \ +int name(const struct iomap_iter *iter, struct iomap *iomap, \ + struct iomap *srcmap) \ +{ \ + return iomap_iter_next(iter, iomap, srcmap, begin_fn, end_fn); \ +} + +#define DEFINE_IOMAP_ITER_NEXT(name, begin_fn) \ + DEFINE_IOMAP_ITER_NEXT_END(name, begin_fn, NULL) + /* * Return the file offset for the first unchanged block after a short write. * @@ -344,6 +431,9 @@ static inline bool iomap_want_unshare_iter(const struct iomap_iter *iter) ssize_t iomap_file_buffered_write(struct kiocb *iocb, struct iov_iter *from, const struct iomap_ops *ops, const struct iomap_write_ops *write_ops, void *private); +int iomap_fsverity_write(struct file *file, loff_t pos, size_t length, + const void *buf, const struct iomap_ops *ops, + const struct iomap_write_ops *write_ops); void iomap_read_folio(const struct iomap_ops *ops, struct iomap_read_folio_ctx *ctx, void *private); void iomap_readahead(const struct iomap_ops *ops, @@ -353,6 +443,7 @@ struct folio *iomap_get_folio(struct iomap_iter *iter, loff_t pos, size_t len); bool iomap_release_folio(struct folio *folio, gfp_t gfp_flags); void iomap_invalidate_folio(struct folio *folio, size_t offset, size_t len); bool iomap_dirty_folio(struct address_space *mapping, struct folio *folio); +void iomap_folio_mark_uptodate(struct folio *folio); int iomap_file_unshare(struct inode *inode, loff_t pos, loff_t len, const struct iomap_ops *ops, const struct iomap_write_ops *write_ops); @@ -392,16 +483,13 @@ sector_t iomap_bmap(struct address_space *mapping, sector_t bno, #define IOMAP_IOEND_BOUNDARY (1U << 2) /* is direct I/O */ #define IOMAP_IOEND_DIRECT (1U << 3) -/* is DONTCACHE I/O */ -#define IOMAP_IOEND_DONTCACHE (1U << 4) /* * Flags that if set on either ioend prevent the merge of two ioends. * (IOMAP_IOEND_BOUNDARY also prevents merges, but only one-way) */ #define IOMAP_IOEND_NOMERGE_FLAGS \ - (IOMAP_IOEND_SHARED | IOMAP_IOEND_UNWRITTEN | IOMAP_IOEND_DIRECT | \ - IOMAP_IOEND_DONTCACHE) + (IOMAP_IOEND_SHARED | IOMAP_IOEND_UNWRITTEN | IOMAP_IOEND_DIRECT) /* * Structure for writeback I/O completions. @@ -420,6 +508,7 @@ struct iomap_ioend { loff_t io_offset; /* offset in the file */ sector_t io_sector; /* start sector of ioend */ void *io_private; /* file system private data */ + struct fsverity_info *io_vi; /* fsverity info */ struct bio io_bio; /* MUST BE LAST! */ }; @@ -493,6 +582,8 @@ struct iomap_read_folio_ctx { struct folio *cur_folio; struct readahead_control *rac; void *read_ctx; + loff_t read_ctx_file_offset; + struct fsverity_info *vi; }; struct iomap_read_ops { @@ -512,7 +603,14 @@ struct iomap_read_ops { * * This is optional. */ - void (*submit_read)(struct iomap_read_folio_ctx *ctx); + void (*submit_read)(const struct iomap_iter *iter, + struct iomap_read_folio_ctx *ctx); + + /* + * Optional, allows filesystem to specify own bio_set, so new bio's + * can be allocated from the provided bio_set. + */ + struct bio_set *bio_set; }; /* @@ -584,6 +682,71 @@ struct iomap_dio *__iomap_dio_rw(struct kiocb *iocb, struct iov_iter *iter, ssize_t iomap_dio_complete(struct iomap_dio *dio); void iomap_dio_bio_end_io(struct bio *bio); +/* + * Fast path for small, block-aligned direct I/Os that map to a single + * contiguous on-disk extent. + * + * @iter must describe a non-empty READ no larger than the inode block size: + * writes, zero-length I/O, and larger requests need the generic iomap direct + * I/O path. + * + * Does not support iomap_dio_ops, dio_flags, done_before or private data. + * The range must also stay within i_size and encrypted inodes must use the + * generic iomap direct I/O path. + * + * -ENOTBLK indicates the generic path must be used by the caller instead. + * Any other errno is a real result and is propagated as-is, in particular + * -EAGAIN for IOCB_NOWAIT must reach the caller. + * + * The caller can only provide an iomap begin handler, and the iterator + * is never advanced. + */ +ssize_t __iomap_dio_read_simple(struct kiocb *iocb, struct iov_iter *iter, + struct iomap_iter *iomi); +static __always_inline ssize_t iomap_dio_read_simple(struct kiocb *iocb, + struct iov_iter *iter, iomap_iter_begin_fn begin) +{ + struct iomap_iter iomi = { + .inode = file_inode(iocb->ki_filp), + .pos = iocb->ki_pos, + .len = iov_iter_count(iter), + .flags = IOMAP_DIRECT, + }; + ssize_t ret; + + if (!iomi.len) + return 0; + + /* + * Simple dio is an optimization for small IO. Filter out large IO + * early as it's the most common case to fail for typical direct IO + * workloads. + */ + if (iomi.len > iomi.inode->i_sb->s_blocksize) + return -ENOTBLK; + if (iocb->ki_pos + iomi.len > i_size_read(iomi.inode)) + return -ENOTBLK; + if (IS_ENCRYPTED(iomi.inode)) + return -ENOTBLK; + + ret = kiocb_write_and_wait(iocb, iomi.len); + if (ret) + return ret; + + if (iocb->ki_flags & IOCB_NOWAIT) + iomi.flags |= IOMAP_NOWAIT; + + inode_dio_begin(iomi.inode); + ret = begin(iomi.inode, iomi.pos, iomi.len, iomi.flags, &iomi.iomap, + &iomi.srcmap); + if (ret) { + inode_dio_end(iomi.inode); + return ret; + } + + return __iomap_dio_read_simple(iocb, iter, &iomi); +} + #ifdef CONFIG_SWAP struct file; struct swap_info_struct; @@ -598,6 +761,11 @@ int iomap_swapfile_activate(struct swap_info_struct *sis, extern struct bio_set iomap_ioend_bioset; #ifdef CONFIG_BLOCK +int iomap_bio_read_folio_range(const struct iomap_iter *iter, + struct iomap_read_folio_ctx *ctx, size_t plen); +void iomap_bio_submit_read_endio(const struct iomap_iter *iter, + struct iomap_read_folio_ctx *ctx, bio_end_io_t end_io); + extern const struct iomap_read_ops iomap_bio_read_ops; static inline void iomap_bio_read_folio(struct folio *folio, diff --git a/include/linux/iommu-dma.h b/include/linux/iommu-dma.h index a92b3ff9b934..060f6e23ab3c 100644 --- a/include/linux/iommu-dma.h +++ b/include/linux/iommu-dma.h @@ -7,12 +7,13 @@ #ifndef _LINUX_IOMMU_DMA_H #define _LINUX_IOMMU_DMA_H +#include <linux/device.h> #include <linux/dma-direction.h> #ifdef CONFIG_IOMMU_DMA static inline bool use_dma_iommu(struct device *dev) { - return dev->dma_iommu; + return dev_dma_iommu(dev); } #else static inline bool use_dma_iommu(struct device *dev) diff --git a/include/linux/iommu.h b/include/linux/iommu.h index 54b8b48c762e..ac43b8b93f14 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -223,6 +223,7 @@ enum iommu_domain_cookie_type { struct iommu_domain { unsigned type; enum iommu_domain_cookie_type cookie_type; + bool is_iommupt; const struct iommu_domain_ops *ops; const struct iommu_dirty_ops *dirty_ops; const struct iommu_ops *owner; /* Whose domain_alloc we came from */ @@ -271,6 +272,8 @@ enum iommu_cap { */ IOMMU_CAP_DEFERRED_FLUSH, IOMMU_CAP_DIRTY_TRACKING, /* IOMMU supports dirty tracking */ + /* ATS is supported and may be enabled for this device */ + IOMMU_CAP_PCI_ATS_SUPPORTED, }; /* These are the possible reserved region types */ @@ -342,12 +345,6 @@ struct iommu_pages_list { /** * struct iommu_iotlb_gather - Range information for a pending IOTLB flush * - * @start: IOVA representing the start of the range to be flushed - * @end: IOVA representing the end of the range to be flushed (inclusive) - * @pgsize: The interval at which to perform the flush - * @freelist: Removed pages to free after sync - * @queued: Indicates that the flush will be queued - * * This structure is intended to be updated by multiple calls to the * ->unmap() function in struct iommu_ops before eventually being passed * into ->iotlb_sync(). Drivers can add pages to @freelist to be freed after @@ -356,10 +353,44 @@ struct iommu_pages_list { * later instead of ->iotlb_sync(), so drivers may optimise accordingly. */ struct iommu_iotlb_gather { + /** @start: IOVA representing the start of the range to be flushed */ unsigned long start; + /** + * @end: IOVA representing the end of the range to be + * flushed (inclusive) + */ unsigned long end; - size_t pgsize; + + union { + /** + * @pgsize: The interval at which to perform the flush, only + * used by arm-smmu-v3 + */ + size_t pgsize; + struct { + /** + * @pt.leaf_levels_bitmap: Bitmap of generic_pt + * levels where leaf entries were unmapped. Bit 0 + * means the leaf only level. If 0 no leafs + * were unmapped. + */ + u8 leaf_levels_bitmap; + /** + * @pt.table_levels_bitmap: Bitmap of generic_pt levels + * of table entries that were removed. Bit 0 is never + * set, bit 1 means a table of all leafs was removed. + * When freelist is empty this must be 0. + */ + u8 table_levels_bitmap; + } pt; + }; + + /** + * @freelist: Removed pages to free after sync, only used by + * iommupt + */ struct iommu_pages_list freelist; + /** @queued: True if the gather will be completed with a flush all */ bool queued; }; @@ -544,6 +575,7 @@ iommu_copy_struct_from_full_user_array(void *kdst, size_t kdst_entry_size, user_array->entry_num * user_array->entry_len)) return -EFAULT; + return 0; } /* Copy item by item */ @@ -980,7 +1012,8 @@ static inline void iommu_flush_iotlb_all(struct iommu_domain *domain) static inline void iommu_iotlb_sync(struct iommu_domain *domain, struct iommu_iotlb_gather *iotlb_gather) { - if (domain->ops->iotlb_sync) + if (domain->ops->iotlb_sync && + likely(iotlb_gather->start < iotlb_gather->end)) domain->ops->iotlb_sync(domain, iotlb_gather); iommu_iotlb_gather_init(iotlb_gather); @@ -1671,6 +1704,7 @@ void iopf_free_group(struct iopf_group *group); int iommu_report_device_fault(struct device *dev, struct iopf_fault *evt); void iopf_group_response(struct iopf_group *group, enum iommu_page_response_code status); +void iopf_group_dequeue(struct iopf_group *group); #else static inline int iopf_queue_add_device(struct iopf_queue *queue, struct device *dev) @@ -1716,5 +1750,9 @@ static inline void iopf_group_response(struct iopf_group *group, enum iommu_page_response_code status) { } + +static inline void iopf_group_dequeue(struct iopf_group *group) +{ +} #endif /* CONFIG_IOMMU_IOPF */ #endif /* __LINUX_IOMMU_H */ diff --git a/include/linux/iopoll.h b/include/linux/iopoll.h index bdd2e0652bc3..53edd69acb9b 100644 --- a/include/linux/iopoll.h +++ b/include/linux/iopoll.h @@ -159,7 +159,7 @@ * * This macro does not rely on timekeeping. Hence it is safe to call even when * timekeeping is suspended, at the expense of an underestimation of wall clock - * time, which is rather minimal with a non-zero delay_us. + * time, which is rather minimal with a non-zero @delay_us. * * When available, you'll probably want to use one of the specialized * macros defined below rather than this macro directly. @@ -167,9 +167,9 @@ * Returns: 0 on success and -ETIMEDOUT upon a timeout. In either * case, the last read value at @args is stored in @val. */ -#define read_poll_timeout_atomic(op, val, cond, sleep_us, timeout_us, \ - sleep_before_read, args...) \ - poll_timeout_us_atomic((val) = op(args), cond, sleep_us, timeout_us, sleep_before_read) +#define read_poll_timeout_atomic(op, val, cond, delay_us, timeout_us, \ + delay_before_read, args...) \ + poll_timeout_us_atomic((val) = op(args), cond, delay_us, timeout_us, delay_before_read) /** * readx_poll_timeout - Periodically poll an address until a condition is met or a timeout occurs diff --git a/include/linux/ioport.h b/include/linux/ioport.h index 5533a5debf3f..f7930b3dfd0a 100644 --- a/include/linux/ioport.h +++ b/include/linux/ioport.h @@ -202,6 +202,7 @@ enum { * typedef resource_alignf - Resource alignment callback * @data: Private data used by the callback * @res: Resource candidate range (an empty resource space) + * @empty_res: Empty resource range without alignment applied * @size: The minimum size of the empty space * @align: Alignment from the constraints * @@ -212,6 +213,7 @@ enum { */ typedef resource_size_t (*resource_alignf)(void *data, const struct resource *res, + const struct resource *empty_res, resource_size_t size, resource_size_t align); @@ -259,7 +261,7 @@ extern int allocate_resource(struct resource *root, struct resource *new, struct resource *lookup_resource(struct resource *root, resource_size_t start); int adjust_resource(struct resource *res, resource_size_t start, resource_size_t size); -resource_size_t resource_alignment(struct resource *res); +resource_size_t resource_alignment(const struct resource *res); /** * resource_set_size - Calculate resource end address from size and start @@ -304,14 +306,28 @@ static inline unsigned long resource_ext_type(const struct resource *res) { return res->flags & IORESOURCE_EXT_TYPE_BITS; } -/* True iff r1 completely contains r2 */ -static inline bool resource_contains(const struct resource *r1, const struct resource *r2) + +/* + * For checking if @r1 completely contains @r2 for resources that have real + * addresses but are not yet crafted into the resource tree. Normally + * resource_contains() should be used instead of this function as it checks + * also IORESOURCE_UNSET flag. + */ +static inline bool __resource_contains_unbound(const struct resource *r1, + const struct resource *r2) { if (resource_type(r1) != resource_type(r2)) return false; + + return r1->start <= r2->start && r1->end >= r2->end; +} +/* True iff r1 completely contains r2 */ +static inline bool resource_contains(const struct resource *r1, const struct resource *r2) +{ if (r1->flags & IORESOURCE_UNSET || r2->flags & IORESOURCE_UNSET) return false; - return r1->start <= r2->start && r1->end >= r2->end; + + return __resource_contains_unbound(r1, r2); } /* True if any part of r1 overlaps r2 */ diff --git a/include/linux/ipack.h b/include/linux/ipack.h index 455f6c2a1903..7edbf9267338 100644 --- a/include/linux/ipack.h +++ b/include/linux/ipack.h @@ -6,7 +6,7 @@ * Author: Samuel Iglesias Gonsalvez <siglesias@igalia.com> */ -#include <linux/mod_devicetable.h> +#include <linux/device-id/ipack.h> #include <linux/device.h> #include <linux/interrupt.h> diff --git a/include/linux/irq-entry-common.h b/include/linux/irq-entry-common.h index d26d1b1bcbfb..0bb6c03481fa 100644 --- a/include/linux/irq-entry-common.h +++ b/include/linux/irq-entry-common.h @@ -3,6 +3,7 @@ #define __LINUX_IRQENTRYCOMMON_H #include <linux/context_tracking.h> +#include <linux/hrtimer_rearm.h> #include <linux/kmsan.h> #include <linux/rseq_entry.h> #include <linux/static_call_types.h> @@ -33,6 +34,14 @@ _TIF_PATCH_PENDING | _TIF_NOTIFY_SIGNAL | _TIF_RSEQ | \ ARCH_EXIT_TO_USER_MODE_WORK) +#ifdef CONFIG_HRTIMER_REARM_DEFERRED +# define EXIT_TO_USER_MODE_WORK_SYSCALL (EXIT_TO_USER_MODE_WORK) +# define EXIT_TO_USER_MODE_WORK_IRQ (EXIT_TO_USER_MODE_WORK | _TIF_HRTIMER_REARM) +#else +# define EXIT_TO_USER_MODE_WORK_SYSCALL (EXIT_TO_USER_MODE_WORK) +# define EXIT_TO_USER_MODE_WORK_IRQ (EXIT_TO_USER_MODE_WORK) +#endif + /** * arch_enter_from_user_mode - Architecture specific sanity check for user mode regs * @regs: Pointer to currents pt_regs @@ -40,9 +49,9 @@ * Defaults to an empty implementation. Can be replaced by architecture * specific code. * - * Invoked from syscall_enter_from_user_mode() in the non-instrumentable - * section. Use __always_inline so the compiler cannot push it out of line - * and make it instrumentable. + * Invoked from enter_from_user_mode() in the non-instrumentable section. Use + * __always_inline so the compiler cannot push it out of line and make it + * instrumentable. */ static __always_inline void arch_enter_from_user_mode(struct pt_regs *regs); @@ -101,37 +110,6 @@ static __always_inline void enter_from_user_mode(struct pt_regs *regs) } /** - * local_irq_enable_exit_to_user - Exit to user variant of local_irq_enable() - * @ti_work: Cached TIF flags gathered with interrupts disabled - * - * Defaults to local_irq_enable(). Can be supplied by architecture specific - * code. - */ -static inline void local_irq_enable_exit_to_user(unsigned long ti_work); - -#ifndef local_irq_enable_exit_to_user -static __always_inline void local_irq_enable_exit_to_user(unsigned long ti_work) -{ - local_irq_enable(); -} -#endif - -/** - * local_irq_disable_exit_to_user - Exit to user variant of local_irq_disable() - * - * Defaults to local_irq_disable(). Can be supplied by architecture specific - * code. - */ -static inline void local_irq_disable_exit_to_user(void); - -#ifndef local_irq_disable_exit_to_user -static __always_inline void local_irq_disable_exit_to_user(void) -{ - local_irq_disable(); -} -#endif - -/** * arch_exit_to_user_mode_work - Architecture specific TIF work for exit * to user mode. * @regs: Pointer to currents pt_regs @@ -203,6 +181,7 @@ unsigned long exit_to_user_mode_loop(struct pt_regs *regs, unsigned long ti_work /** * __exit_to_user_mode_prepare - call exit_to_user_mode_loop() if required * @regs: Pointer to pt_regs on entry stack + * @work_mask: Which TIF bits need to be evaluated * * 1) check that interrupts are disabled * 2) call tick_nohz_user_enter_prepare() @@ -212,7 +191,8 @@ unsigned long exit_to_user_mode_loop(struct pt_regs *regs, unsigned long ti_work * * Don't invoke directly, use the syscall/irqentry_ prefixed variants below */ -static __always_inline void __exit_to_user_mode_prepare(struct pt_regs *regs) +static __always_inline void __exit_to_user_mode_prepare(struct pt_regs *regs, + const unsigned long work_mask) { unsigned long ti_work; @@ -222,8 +202,10 @@ static __always_inline void __exit_to_user_mode_prepare(struct pt_regs *regs) tick_nohz_user_enter_prepare(); ti_work = read_thread_flags(); - if (unlikely(ti_work & EXIT_TO_USER_MODE_WORK)) - ti_work = exit_to_user_mode_loop(regs, ti_work); + if (unlikely(ti_work & work_mask)) { + if (!hrtimer_rearm_deferred_user_irq(&ti_work, work_mask)) + ti_work = exit_to_user_mode_loop(regs, ti_work); + } arch_exit_to_user_mode_prepare(regs, ti_work); } @@ -236,14 +218,6 @@ static __always_inline void __exit_to_user_mode_validate(void) lockdep_sys_exit(); } -/* Temporary workaround to keep ARM64 alive */ -static __always_inline void exit_to_user_mode_prepare_legacy(struct pt_regs *regs) -{ - __exit_to_user_mode_prepare(regs); - rseq_exit_to_user_mode_legacy(); - __exit_to_user_mode_validate(); -} - /** * syscall_exit_to_user_mode_prepare - call exit_to_user_mode_loop() if required * @regs: Pointer to pt_regs on entry stack @@ -253,7 +227,7 @@ static __always_inline void exit_to_user_mode_prepare_legacy(struct pt_regs *reg */ static __always_inline void syscall_exit_to_user_mode_prepare(struct pt_regs *regs) { - __exit_to_user_mode_prepare(regs); + __exit_to_user_mode_prepare(regs, EXIT_TO_USER_MODE_WORK_SYSCALL); rseq_syscall_exit_to_user_mode(); __exit_to_user_mode_validate(); } @@ -267,7 +241,7 @@ static __always_inline void syscall_exit_to_user_mode_prepare(struct pt_regs *re */ static __always_inline void irqentry_exit_to_user_mode_prepare(struct pt_regs *regs) { - __exit_to_user_mode_prepare(regs); + __exit_to_user_mode_prepare(regs, EXIT_TO_USER_MODE_WORK_IRQ); rseq_irqentry_exit_to_user_mode(); __exit_to_user_mode_validate(); } @@ -335,6 +309,8 @@ static __always_inline void irqentry_enter_from_user_mode(struct pt_regs *regs) */ static __always_inline void irqentry_exit_to_user_mode(struct pt_regs *regs) { + lockdep_assert_irqs_disabled(); + instrumentation_begin(); irqentry_exit_to_user_mode_prepare(regs); instrumentation_end(); @@ -366,6 +342,207 @@ typedef struct irqentry_state { #endif /** + * irqentry_exit_cond_resched - Conditionally reschedule on return from interrupt + * + * Conditional reschedule with additional sanity checks. + */ +void raw_irqentry_exit_cond_resched(void); + +#ifdef CONFIG_PREEMPT_DYNAMIC +#if defined(CONFIG_HAVE_PREEMPT_DYNAMIC_CALL) +#define irqentry_exit_cond_resched_dynamic_enabled raw_irqentry_exit_cond_resched +#define irqentry_exit_cond_resched_dynamic_disabled NULL +DECLARE_STATIC_CALL(irqentry_exit_cond_resched, raw_irqentry_exit_cond_resched); +#define irqentry_exit_cond_resched() static_call(irqentry_exit_cond_resched)() +#elif defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY) +DECLARE_STATIC_KEY_TRUE(sk_dynamic_irqentry_exit_cond_resched); +void dynamic_irqentry_exit_cond_resched(void); +#define irqentry_exit_cond_resched() dynamic_irqentry_exit_cond_resched() +#endif +#else /* CONFIG_PREEMPT_DYNAMIC */ +#define irqentry_exit_cond_resched() raw_irqentry_exit_cond_resched() +#endif /* CONFIG_PREEMPT_DYNAMIC */ + +/** + * irqentry_enter_from_kernel_mode - Establish state before invoking the irq handler + * @regs: Pointer to currents pt_regs + * + * Invoked from architecture specific entry code with interrupts disabled. + * Can only be called when the interrupt entry came from kernel mode. The + * calling code must be non-instrumentable. When the function returns all + * state is correct and the subsequent functions can be instrumented. + * + * The function establishes state (lockdep, RCU (context tracking), tracing) and + * is provided for architectures which require a strict split between entry from + * kernel and user mode and therefore cannot use irqentry_enter() which handles + * both entry modes. + * + * Returns: An opaque object that must be passed to irqentry_exit_to_kernel_mode(). + */ +static __always_inline irqentry_state_t irqentry_enter_from_kernel_mode(struct pt_regs *regs) +{ + irqentry_state_t ret = { + .exit_rcu = false, + }; + + /* + * If this entry hit the idle task invoke ct_irq_enter() whether + * RCU is watching or not. + * + * Interrupts can nest when the first interrupt invokes softirq + * processing on return which enables interrupts. + * + * Scheduler ticks in the idle task can mark quiescent state and + * terminate a grace period, if and only if the timer interrupt is + * not nested into another interrupt. + * + * Checking for rcu_is_watching() here would prevent the nesting + * interrupt to invoke ct_irq_enter(). If that nested interrupt is + * the tick then rcu_flavor_sched_clock_irq() would wrongfully + * assume that it is the first interrupt and eventually claim + * quiescent state and end grace periods prematurely. + * + * Unconditionally invoke ct_irq_enter() so RCU state stays + * consistent. + * + * TINY_RCU does not support EQS, so let the compiler eliminate + * this part when enabled. + */ + if (!IS_ENABLED(CONFIG_TINY_RCU) && + (is_idle_task(current) || arch_in_rcu_eqs())) { + /* + * If RCU is not watching then the same careful + * sequence vs. lockdep and tracing is required + * as in irqentry_enter_from_user_mode(). + */ + lockdep_hardirqs_off(CALLER_ADDR0); + ct_irq_enter(); + instrumentation_begin(); + kmsan_unpoison_entry_regs(regs); + trace_hardirqs_off_finish(); + instrumentation_end(); + + ret.exit_rcu = true; + return ret; + } + + /* + * If RCU is watching then RCU only wants to check whether it needs + * to restart the tick in NOHZ mode. rcu_irq_enter_check_tick() + * already contains a warning when RCU is not watching, so no point + * in having another one here. + */ + lockdep_hardirqs_off(CALLER_ADDR0); + instrumentation_begin(); + kmsan_unpoison_entry_regs(regs); + rcu_irq_enter_check_tick(); + trace_hardirqs_off_finish(); + instrumentation_end(); + + return ret; +} + +/** + * irqentry_exit_to_kernel_mode_preempt - Run preempt checks on return to kernel mode + * @regs: Pointer to current's pt_regs + * @state: Return value from matching call to irqentry_enter_from_kernel_mode() + * + * This is to be invoked before irqentry_exit_to_kernel_mode_after_preempt() to + * allow kernel preemption on return from interrupt. + * + * Must be invoked with interrupts disabled and CPU state which allows kernel + * preemption. + * + * After returning from this function, the caller can modify CPU state before + * invoking irqentry_exit_to_kernel_mode_after_preempt(), which is required to + * re-establish the tracing, lockdep and RCU state for returning to the + * interrupted context. + */ +static inline void irqentry_exit_to_kernel_mode_preempt(struct pt_regs *regs, + irqentry_state_t state) +{ + if (regs_irqs_disabled(regs) || state.exit_rcu) + return; + + if (IS_ENABLED(CONFIG_PREEMPTION)) + irqentry_exit_cond_resched(); +} + +/** + * irqentry_exit_to_kernel_mode_after_preempt - Establish trace, lockdep and RCU state + * @regs: Pointer to current's pt_regs + * @state: Return value from matching call to irqentry_enter_from_kernel_mode() + * + * This is to be invoked after irqentry_exit_to_kernel_mode_preempt() and before + * actually returning to the interrupted context. + * + * There are no requirements for the CPU state other than being able to complete + * the tracing, lockdep and RCU state transitions. After this function returns + * the caller must return directly to the interrupted context. + */ +static __always_inline void +irqentry_exit_to_kernel_mode_after_preempt(struct pt_regs *regs, irqentry_state_t state) +{ + if (!regs_irqs_disabled(regs)) { + /* + * If RCU was not watching on entry this needs to be done + * carefully and needs the same ordering of lockdep/tracing + * and RCU as the return to user mode path. + */ + if (state.exit_rcu) { + instrumentation_begin(); + hrtimer_rearm_deferred(); + /* Tell the tracer that IRET will enable interrupts */ + trace_hardirqs_on_prepare(); + lockdep_hardirqs_on_prepare(); + instrumentation_end(); + ct_irq_exit(); + lockdep_hardirqs_on(CALLER_ADDR0); + return; + } + + instrumentation_begin(); + hrtimer_rearm_deferred(); + /* Covers both tracing and lockdep */ + trace_hardirqs_on(); + instrumentation_end(); + } else { + /* + * IRQ flags state is correct already. Just tell RCU if it + * was not watching on entry. + */ + if (state.exit_rcu) + ct_irq_exit(); + } +} + +/** + * irqentry_exit_to_kernel_mode - Run preempt checks and establish state after + * invoking the interrupt handler + * @regs: Pointer to current's pt_regs + * @state: Return value from matching call to irqentry_enter_from_kernel_mode() + * + * This is the counterpart of irqentry_enter_from_kernel_mode() and combines + * the calls to irqentry_exit_to_kernel_mode_preempt() and + * irqentry_exit_to_kernel_mode_after_preempt(). + * + * The requirement for the CPU state is that it can schedule. After the function + * returns the tracing, lockdep and RCU state transitions are completed and the + * caller must return directly to the interrupted context. + */ +static __always_inline void irqentry_exit_to_kernel_mode(struct pt_regs *regs, + irqentry_state_t state) +{ + lockdep_assert_irqs_disabled(); + + instrumentation_begin(); + irqentry_exit_to_kernel_mode_preempt(regs, state); + instrumentation_end(); + + irqentry_exit_to_kernel_mode_after_preempt(regs, state); +} + +/** * irqentry_enter - Handle state tracking on ordinary interrupt entries * @regs: Pointer to pt_regs of interrupted context * @@ -394,33 +571,11 @@ typedef struct irqentry_state { * establish the proper context for NOHZ_FULL. Otherwise scheduling on exit * would not be possible. * - * Returns: An opaque object that must be passed to idtentry_exit() + * Returns: An opaque object that must be passed to irqentry_exit() */ irqentry_state_t noinstr irqentry_enter(struct pt_regs *regs); /** - * irqentry_exit_cond_resched - Conditionally reschedule on return from interrupt - * - * Conditional reschedule with additional sanity checks. - */ -void raw_irqentry_exit_cond_resched(void); - -#ifdef CONFIG_PREEMPT_DYNAMIC -#if defined(CONFIG_HAVE_PREEMPT_DYNAMIC_CALL) -#define irqentry_exit_cond_resched_dynamic_enabled raw_irqentry_exit_cond_resched -#define irqentry_exit_cond_resched_dynamic_disabled NULL -DECLARE_STATIC_CALL(irqentry_exit_cond_resched, raw_irqentry_exit_cond_resched); -#define irqentry_exit_cond_resched() static_call(irqentry_exit_cond_resched)() -#elif defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY) -DECLARE_STATIC_KEY_TRUE(sk_dynamic_irqentry_exit_cond_resched); -void dynamic_irqentry_exit_cond_resched(void); -#define irqentry_exit_cond_resched() dynamic_irqentry_exit_cond_resched() -#endif -#else /* CONFIG_PREEMPT_DYNAMIC */ -#define irqentry_exit_cond_resched() raw_irqentry_exit_cond_resched() -#endif /* CONFIG_PREEMPT_DYNAMIC */ - -/** * irqentry_exit - Handle return from exception that used irqentry_enter() * @regs: Pointer to pt_regs (exception entry regs) * @state: Return value from matching call to irqentry_enter() diff --git a/include/linux/irq.h b/include/linux/irq.h index 951acbdb9f84..f485369b1b4f 100644 --- a/include/linux/irq.h +++ b/include/linux/irq.h @@ -35,6 +35,10 @@ enum irqchip_irq_state; * * Bits 0-7 are the same as the IRQF_* bits in linux/interrupt.h * + * Note that the first 6 definitions are shadowed by C preprocessor definitions + * in include/dt-bindings/interrupt-controller/irq.h. This is not an issue, as + * the actual values must be the same, due to being part of the stable DT ABI. + * * IRQ_TYPE_NONE - default, unspecified type * IRQ_TYPE_EDGE_RISING - rising edge triggered * IRQ_TYPE_EDGE_FALLING - falling edge triggered @@ -99,6 +103,7 @@ enum { IRQ_DISABLE_UNLAZY = (1 << 19), IRQ_HIDDEN = (1 << 20), IRQ_NO_DEBUG = (1 << 21), + IRQ_RESERVED = (1 << 22), }; #define IRQF_MODIFY_MASK \ diff --git a/include/linux/irqchip/arm-gic-v3.h b/include/linux/irqchip/arm-gic-v3.h index 0225121f3013..ea5fd2374ebe 100644 --- a/include/linux/irqchip/arm-gic-v3.h +++ b/include/linux/irqchip/arm-gic-v3.h @@ -604,7 +604,7 @@ #include <asm/arch_gicv3.h> -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ /* * We need a value to serve as a irq-type for LPIs. Choose one that will diff --git a/include/linux/irqchip/arm-gic-v5.h b/include/linux/irqchip/arm-gic-v5.h index b78488df6c98..2c2fb39f049c 100644 --- a/include/linux/irqchip/arm-gic-v5.h +++ b/include/linux/irqchip/arm-gic-v5.h @@ -6,6 +6,7 @@ #define __LINUX_IRQCHIP_ARM_GIC_V5_H #include <linux/iopoll.h> +#include <linux/ioport.h> #include <asm/cacheflush.h> #include <asm/smp.h> @@ -25,6 +26,28 @@ #define GICV5_HWIRQ_TYPE_SPI UL(0x3) /* + * Architected PPIs + */ +#define GICV5_ARCH_PPI_S_DB_PPI 0x0 +#define GICV5_ARCH_PPI_RL_DB_PPI 0x1 +#define GICV5_ARCH_PPI_NS_DB_PPI 0x2 +#define GICV5_ARCH_PPI_SW_PPI 0x3 +#define GICV5_ARCH_PPI_HACDBSIRQ 0xf +#define GICV5_ARCH_PPI_CNTHVS 0x13 +#define GICV5_ARCH_PPI_CNTHPS 0x14 +#define GICV5_ARCH_PPI_PMBIRQ 0x15 +#define GICV5_ARCH_PPI_COMMIRQ 0x16 +#define GICV5_ARCH_PPI_PMUIRQ 0x17 +#define GICV5_ARCH_PPI_CTIIRQ 0x18 +#define GICV5_ARCH_PPI_GICMNT 0x19 +#define GICV5_ARCH_PPI_CNTHP 0x1a +#define GICV5_ARCH_PPI_CNTV 0x1b +#define GICV5_ARCH_PPI_CNTHV 0x1c +#define GICV5_ARCH_PPI_CNTPS 0x1d +#define GICV5_ARCH_PPI_CNTP 0x1e +#define GICV5_ARCH_PPI_TRBIRQ 0x1f + +/* * Tables attributes */ #define GICV5_NO_READ_ALLOC 0b0 @@ -302,6 +325,7 @@ struct gicv5_irs_chip_data { struct list_head entry; struct fwnode_handle *fwnode; void __iomem *irs_base; + struct resource res; u32 flags; u32 spi_min; u32 spi_range; @@ -365,6 +389,11 @@ int gicv5_spi_irq_set_type(struct irq_data *d, unsigned int type); int gicv5_irs_iste_alloc(u32 lpi); void gicv5_irs_syncr(void); +/* Embedded in kvm.arch */ +struct gicv5_vpe { + bool resident; +}; + struct gicv5_its_devtab_cfg { union { struct { @@ -398,9 +427,6 @@ struct gicv5_its_itt_cfg { void gicv5_init_lpis(u32 max); void gicv5_deinit_lpis(void); -int gicv5_alloc_lpi(void); -void gicv5_free_lpi(u32 lpi); - void __init gicv5_its_of_probe(struct device_node *parent); void __init gicv5_its_acpi_probe(void); #endif diff --git a/include/linux/irqchip/arm-gic.h b/include/linux/irqchip/arm-gic.h index d45fa19f9e47..849386dc5ec8 100644 --- a/include/linux/irqchip/arm-gic.h +++ b/include/linux/irqchip/arm-gic.h @@ -131,7 +131,7 @@ #define GICV_PMR_PRIORITY_SHIFT 3 #define GICV_PMR_PRIORITY_MASK (0x1f << GICV_PMR_PRIORITY_SHIFT) -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #include <linux/irqdomain.h> @@ -162,5 +162,5 @@ int gic_get_cpu_id(unsigned int cpu); void gic_migrate_target(unsigned int new_cpu_id); unsigned long gic_get_sgir_physaddr(void); -#endif /* __ASSEMBLY */ +#endif /* __ASSEMBLER__ */ #endif diff --git a/include/linux/irqchip/irq-renesas-rzv2h.h b/include/linux/irqchip/irq-renesas-rzv2h.h index 618a60d2eac0..4ffa898eaaf2 100644 --- a/include/linux/irqchip/irq-renesas-rzv2h.h +++ b/include/linux/irqchip/irq-renesas-rzv2h.h @@ -11,13 +11,18 @@ #include <linux/platform_device.h> #define RZV2H_ICU_DMAC_REQ_NO_DEFAULT 0x3ff +#define RZV2H_ICU_DMAC_ACK_NO_DEFAULT 0x7f #ifdef CONFIG_RENESAS_RZV2H_ICU void rzv2h_icu_register_dma_req(struct platform_device *icu_dev, u8 dmac_index, u8 dmac_channel, u16 req_no); +void rzv2h_icu_register_dma_ack(struct platform_device *icu_dev, u8 dmac_index, + u8 dmac_channel, u16 ack_no); #else static inline void rzv2h_icu_register_dma_req(struct platform_device *icu_dev, u8 dmac_index, u8 dmac_channel, u16 req_no) { } +static inline void rzv2h_icu_register_dma_ack(struct platform_device *icu_dev, u8 dmac_index, + u8 dmac_channel, u16 ack_no) { } #endif #endif /* __LINUX_IRQ_RENESAS_RZV2H */ diff --git a/include/linux/irqchip/riscv-imsic.h b/include/linux/irqchip/riscv-imsic.h index 4b348836de7a..61af3a5bea09 100644 --- a/include/linux/irqchip/riscv-imsic.h +++ b/include/linux/irqchip/riscv-imsic.h @@ -40,6 +40,9 @@ struct imsic_local_config { phys_addr_t msi_pa; void __iomem *msi_va; + + /* Number of guest interrupt files per-HART */ + u32 nr_guest_files; }; struct imsic_global_config { @@ -68,7 +71,7 @@ struct imsic_global_config { /* Number of guest interrupt identities */ u32 nr_guest_ids; - /* Number of guest interrupt files per core */ + /* Number of guest interrupt files across all HARTs */ u32 nr_guest_files; /* Per-CPU IMSIC addresses */ diff --git a/include/linux/irqdesc.h b/include/linux/irqdesc.h index dae9a9b93665..779d6023c5c2 100644 --- a/include/linux/irqdesc.h +++ b/include/linux/irqdesc.h @@ -52,8 +52,8 @@ struct irq_redirect { * @depth: disable-depth, for nested irq_disable() calls * @wake_depth: enable depth, for multiple irq_set_irq_wake() callers * @tot_count: stats field for non-percpu irqs - * @irq_count: stats field to detect stalled irqs * @last_unhandled: aging timer for unhandled count + * @irq_count: stats field to detect stalled irqs * @irqs_unhandled: stats field for spurious unhandled interrupts * @threads_handled: stats field for deferred spurious detection of threaded handlers * @threads_handled_last: comparator field for deferred spurious detection of threaded handlers @@ -70,6 +70,7 @@ struct irq_redirect { * IRQF_NO_SUSPEND set * @force_resume_depth: number of irqactions on a irq descriptor with * IRQF_FORCE_RESUME set + * @refcnt: Reference count mainly for /proc/interrupts * @rcu: rcu head for delayed free * @kobj: kobject used to represent this struct in sysfs * @request_mutex: mutex to protect request/free before locking desc->lock @@ -87,9 +88,9 @@ struct irq_desc { unsigned int core_internal_state__do_not_mess_with_it; unsigned int depth; /* nested irq disables */ unsigned int wake_depth; /* nested wake enables */ - unsigned int tot_count; - unsigned int irq_count; /* For detecting broken IRQs */ + unsigned long tot_count; unsigned long last_unhandled; /* Aging timer for unhandled count */ + unsigned int irq_count; /* For detecting broken IRQs */ unsigned int irqs_unhandled; atomic_t threads_handled; int threads_handled_last; @@ -119,6 +120,7 @@ struct irq_desc { struct dentry *debugfs_file; const char *dev_name; #endif + rcuref_t refcnt; #ifdef CONFIG_SPARSE_IRQ struct rcu_head rcu; struct kobject kobj; @@ -144,7 +146,7 @@ extern struct irq_desc irq_desc[NR_IRQS]; static inline unsigned int irq_desc_kstat_cpu(struct irq_desc *desc, unsigned int cpu) { - return desc->kstat_irqs ? per_cpu(desc->kstat_irqs->cnt, cpu) : 0; + return per_cpu(desc->kstat_irqs->cnt, cpu); } static inline struct irq_desc *irq_data_to_desc(struct irq_data *data) diff --git a/include/linux/irqdomain.h b/include/linux/irqdomain.h index 73c25d40846c..3ba75a4ed3da 100644 --- a/include/linux/irqdomain.h +++ b/include/linux/irqdomain.h @@ -752,24 +752,6 @@ static inline void msi_device_domain_free_wired(struct irq_domain *domain, unsig } #endif -static inline struct irq_domain *irq_domain_add_linear(struct device_node *of_node, - unsigned int size, - const struct irq_domain_ops *ops, - void *host_data) -{ - struct irq_domain_info info = { - .fwnode = of_fwnode_handle(of_node), - .size = size, - .hwirq_max = size, - .ops = ops, - .host_data = host_data, - }; - struct irq_domain *d; - - d = irq_domain_instantiate(&info); - return IS_ERR(d) ? NULL : d; -} - #else /* CONFIG_IRQ_DOMAIN */ static inline void irq_dispose_mapping(unsigned int virq) { } static inline struct irq_domain *irq_find_matching_fwnode(struct fwnode_handle *fwnode, diff --git a/include/linux/irqdomain_defs.h b/include/linux/irqdomain_defs.h index 36653e2ee1c9..3a03bdfeeee9 100644 --- a/include/linux/irqdomain_defs.h +++ b/include/linux/irqdomain_defs.h @@ -17,7 +17,6 @@ enum irq_domain_bus_token { DOMAIN_BUS_PLATFORM_MSI, DOMAIN_BUS_NEXUS, DOMAIN_BUS_IPI, - DOMAIN_BUS_FSL_MC_MSI, DOMAIN_BUS_TI_SCI_INTA_MSI, DOMAIN_BUS_WAKEUP, DOMAIN_BUS_VMD_MSI, diff --git a/include/linux/isapnp.h b/include/linux/isapnp.h index dba18c95844b..8f5a85ca6c1f 100644 --- a/include/linux/isapnp.h +++ b/include/linux/isapnp.h @@ -28,7 +28,7 @@ */ #ifdef __KERNEL__ -#include <linux/mod_devicetable.h> +#include <linux/device-id/isapnp.h> #define DEVICE_COUNT_COMPATIBLE 4 diff --git a/include/linux/isdn/capilli.h b/include/linux/isdn/capilli.h deleted file mode 100644 index 12be09b6883b..000000000000 --- a/include/linux/isdn/capilli.h +++ /dev/null @@ -1,95 +0,0 @@ -/* $Id: capilli.h,v 1.1.2.2 2004/01/16 21:09:27 keil Exp $ - * - * Kernel CAPI 2.0 Driver Interface for Linux - * - * Copyright 1999 by Carsten Paeth <calle@calle.de> - * - * This software may be used and distributed according to the terms - * of the GNU General Public License, incorporated herein by reference. - * - */ - -#ifndef __CAPILLI_H__ -#define __CAPILLI_H__ - -#include <linux/kernel.h> -#include <linux/list.h> -#include <linux/capi.h> -#include <linux/kernelcapi.h> - -typedef struct capiloaddatapart { - int user; /* data in userspace ? */ - int len; - unsigned char *data; -} capiloaddatapart; - -typedef struct capiloaddata { - capiloaddatapart firmware; - capiloaddatapart configuration; -} capiloaddata; - -typedef struct capicardparams { - unsigned int port; - unsigned irq; - int cardtype; - int cardnr; - unsigned int membase; -} capicardparams; - -struct capi_ctr { - /* filled in before calling attach_capi_ctr */ - struct module *owner; - void *driverdata; /* driver specific */ - char name[32]; /* name of controller */ - char *driver_name; /* name of driver */ - int (*load_firmware)(struct capi_ctr *, capiloaddata *); - void (*reset_ctr)(struct capi_ctr *); - void (*register_appl)(struct capi_ctr *, u16 appl, - capi_register_params *); - void (*release_appl)(struct capi_ctr *, u16 appl); - u16 (*send_message)(struct capi_ctr *, struct sk_buff *skb); - - char *(*procinfo)(struct capi_ctr *); - int (*proc_show)(struct seq_file *, void *); - - /* filled in before calling ready callback */ - u8 manu[CAPI_MANUFACTURER_LEN]; /* CAPI_GET_MANUFACTURER */ - capi_version version; /* CAPI_GET_VERSION */ - capi_profile profile; /* CAPI_GET_PROFILE */ - u8 serial[CAPI_SERIAL_LEN]; /* CAPI_GET_SERIAL */ - - /* management information for kcapi */ - - unsigned long nrecvctlpkt; - unsigned long nrecvdatapkt; - unsigned long nsentctlpkt; - unsigned long nsentdatapkt; - - int cnr; /* controller number */ - unsigned short state; /* controller state */ - int blocked; /* output blocked */ - int traceflag; /* capi trace */ - - struct proc_dir_entry *procent; - char procfn[128]; -}; - -int attach_capi_ctr(struct capi_ctr *); -int detach_capi_ctr(struct capi_ctr *); - -void capi_ctr_ready(struct capi_ctr * card); -void capi_ctr_down(struct capi_ctr * card); -void capi_ctr_handle_message(struct capi_ctr * card, u16 appl, struct sk_buff *skb); - -// --------------------------------------------------------------------------- -// needed for AVM capi drivers - -struct capi_driver { - char name[32]; /* driver name */ - char revision[32]; - - /* management information for kcapi */ - struct list_head list; -}; - -#endif /* __CAPILLI_H__ */ diff --git a/include/linux/isdn/capiutil.h b/include/linux/isdn/capiutil.h deleted file mode 100644 index 953fd500dff7..000000000000 --- a/include/linux/isdn/capiutil.h +++ /dev/null @@ -1,60 +0,0 @@ -/* $Id: capiutil.h,v 1.5.6.2 2001/09/23 22:24:33 kai Exp $ - * - * CAPI 2.0 defines & types - * - * From CAPI 2.0 Development Kit AVM 1995 (msg.c) - * Rewritten for Linux 1996 by Carsten Paeth <calle@calle.de> - * - * This software may be used and distributed according to the terms - * of the GNU General Public License, incorporated herein by reference. - * - */ - -#ifndef __CAPIUTIL_H__ -#define __CAPIUTIL_H__ - -#include <asm/types.h> - -#define CAPIMSG_BASELEN 8 -#define CAPIMSG_U8(m, off) (m[off]) -#define CAPIMSG_U16(m, off) (m[off]|(m[(off)+1]<<8)) -#define CAPIMSG_U32(m, off) (m[off]|(m[(off)+1]<<8)|(m[(off)+2]<<16)|(m[(off)+3]<<24)) -#define CAPIMSG_LEN(m) CAPIMSG_U16(m,0) -#define CAPIMSG_APPID(m) CAPIMSG_U16(m,2) -#define CAPIMSG_COMMAND(m) CAPIMSG_U8(m,4) -#define CAPIMSG_SUBCOMMAND(m) CAPIMSG_U8(m,5) -#define CAPIMSG_CMD(m) (((m[4])<<8)|(m[5])) -#define CAPIMSG_MSGID(m) CAPIMSG_U16(m,6) -#define CAPIMSG_CONTROLLER(m) (m[8] & 0x7f) -#define CAPIMSG_CONTROL(m) CAPIMSG_U32(m, 8) -#define CAPIMSG_NCCI(m) CAPIMSG_CONTROL(m) -#define CAPIMSG_DATALEN(m) CAPIMSG_U16(m,16) /* DATA_B3_REQ */ - -static inline void capimsg_setu8(void *m, int off, __u8 val) -{ - ((__u8 *)m)[off] = val; -} - -static inline void capimsg_setu16(void *m, int off, __u16 val) -{ - ((__u8 *)m)[off] = val & 0xff; - ((__u8 *)m)[off+1] = (val >> 8) & 0xff; -} - -static inline void capimsg_setu32(void *m, int off, __u32 val) -{ - ((__u8 *)m)[off] = val & 0xff; - ((__u8 *)m)[off+1] = (val >> 8) & 0xff; - ((__u8 *)m)[off+2] = (val >> 16) & 0xff; - ((__u8 *)m)[off+3] = (val >> 24) & 0xff; -} - -#define CAPIMSG_SETLEN(m, len) capimsg_setu16(m, 0, len) -#define CAPIMSG_SETAPPID(m, applid) capimsg_setu16(m, 2, applid) -#define CAPIMSG_SETCOMMAND(m,cmd) capimsg_setu8(m, 4, cmd) -#define CAPIMSG_SETSUBCOMMAND(m, cmd) capimsg_setu8(m, 5, cmd) -#define CAPIMSG_SETMSGID(m, msgid) capimsg_setu16(m, 6, msgid) -#define CAPIMSG_SETCONTROL(m, contr) capimsg_setu32(m, 8, contr) -#define CAPIMSG_SETDATALEN(m, len) capimsg_setu16(m, 16, len) - -#endif /* __CAPIUTIL_H__ */ diff --git a/include/linux/jbd2.h b/include/linux/jbd2.h index a53a00d36228..1b42fe47c26b 100644 --- a/include/linux/jbd2.h +++ b/include/linux/jbd2.h @@ -63,9 +63,6 @@ void __jbd2_debug(int level, const char *file, const char *func, #define jbd2_debug(n, fmt, a...) no_printk(fmt, ##a) #endif -extern void *jbd2_alloc(size_t size, gfp_t flags); -extern void jbd2_free(void *ptr, size_t size); - #define JBD2_MIN_JOURNAL_BLOCKS 1024 #define JBD2_DEFAULT_FAST_COMMIT_BLOCKS 256 @@ -429,22 +426,46 @@ struct jbd2_inode { unsigned long i_flags; /** - * @i_dirty_start: + * @i_dirty_start_page: + * + * Dirty range start in PAGE_SIZE units. + * + * The dirty range is empty if @i_dirty_start_page is greater than or + * equal to @i_dirty_end_page. * - * Offset in bytes where the dirty range for this inode starts. * [j_list_lock] */ - loff_t i_dirty_start; + pgoff_t i_dirty_start_page; /** - * @i_dirty_end: + * @i_dirty_end_page: * - * Inclusive offset in bytes where the dirty range for this inode - * ends. [j_list_lock] + * Dirty range end in PAGE_SIZE units (exclusive). + * + * [j_list_lock] */ - loff_t i_dirty_end; + pgoff_t i_dirty_end_page; }; +/* + * Lockless readers treat start_page >= end_page as an empty range. + * Writers publish a new non-empty range by storing i_dirty_end_page before + * i_dirty_start_page. + */ +static inline bool jbd2_jinode_get_dirty_range(const struct jbd2_inode *jinode, + loff_t *start, loff_t *end) +{ + pgoff_t start_page = READ_ONCE(jinode->i_dirty_start_page); + pgoff_t end_page = READ_ONCE(jinode->i_dirty_end_page); + + if (start_page >= end_page) + return false; + + *start = (loff_t)start_page << PAGE_SHIFT; + *end = ((loff_t)end_page << PAGE_SHIFT) - 1; + return true; +} + struct jbd2_revoke_table_s; /** @@ -489,11 +510,12 @@ struct jbd2_journal_handle int h_err; /* Flags [no locking] */ - unsigned int h_sync: 1; - unsigned int h_reserved: 1; - unsigned int h_aborted: 1; - unsigned int h_type: 8; - unsigned int h_line_no: 16; + unsigned char h_sync: 1; + unsigned char h_reserved: 1; + unsigned char h_aborted: 1; + unsigned char h_invalid: 1; + unsigned char h_type; + unsigned short h_line_no; unsigned long h_start_jiffies; unsigned int h_requested_credits; diff --git a/include/linux/jiffies.h b/include/linux/jiffies.h index d1c3d4941854..bbd57061802c 100644 --- a/include/linux/jiffies.h +++ b/include/linux/jiffies.h @@ -67,10 +67,6 @@ extern void register_refined_jiffies(long clock_tick_rate); /* USER_TICK_USEC is the time between ticks in usec assuming fake USER_HZ */ #define USER_TICK_USEC ((1000000UL + USER_HZ/2) / USER_HZ) -#ifndef __jiffy_arch_data -#define __jiffy_arch_data -#endif - /* * The 64-bit value is not atomic on 32-bit systems - you MUST NOT read it * without sampling the sequence number in jiffies_lock. @@ -83,7 +79,7 @@ extern void register_refined_jiffies(long clock_tick_rate); * See arch/ARCH/kernel/vmlinux.lds.S */ extern u64 __cacheline_aligned_in_smp jiffies_64; -extern unsigned long volatile __cacheline_aligned_in_smp __jiffy_arch_data jiffies; +extern unsigned long volatile __cacheline_aligned_in_smp jiffies; #if (BITS_PER_LONG < 64) u64 get_jiffies_64(void); diff --git a/include/linux/jump_label.h b/include/linux/jump_label.h index fdb79dd1ebd8..c95350a10987 100644 --- a/include/linux/jump_label.h +++ b/include/linux/jump_label.h @@ -71,7 +71,7 @@ * Additional babbling in: Documentation/staging/static-keys.rst */ -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #include <linux/types.h> #include <linux/compiler.h> @@ -87,13 +87,6 @@ struct static_key { atomic_t enabled; #ifdef CONFIG_JUMP_LABEL /* - * Note: - * To make anonymous unions work with old compilers, the static - * initialization of them requires brackets. This creates a dependency - * on the order of the struct with the initializers. If any fields - * are added, STATIC_KEY_INIT_TRUE and STATIC_KEY_INIT_FALSE may need - * to be modified. - * * bit 0 => 1 if key is initially true * 0 if initially false * bit 1 => 1 if points to struct static_key_mod @@ -107,12 +100,12 @@ struct static_key { #endif /* CONFIG_JUMP_LABEL */ }; -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ #ifdef CONFIG_JUMP_LABEL #include <asm/jump_label.h> -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #ifdef CONFIG_HAVE_ARCH_JUMP_LABEL_RELATIVE struct jump_entry { @@ -187,7 +180,7 @@ static inline int jump_entry_size(struct jump_entry *entry) #endif #endif -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ enum jump_label_type { JUMP_LABEL_NOP = 0, @@ -238,19 +231,12 @@ extern void static_key_enable_cpuslocked(struct static_key *key); extern void static_key_disable_cpuslocked(struct static_key *key); extern enum jump_label_type jump_label_init_type(struct jump_entry *entry); -/* - * We should be using ATOMIC_INIT() for initializing .enabled, but - * the inclusion of atomic.h is problematic for inclusion of jump_label.h - * in 'low-level' headers. Thus, we are initializing .enabled with a - * raw value, but have added a BUILD_BUG_ON() to catch any issues in - * jump_label_init() see: kernel/jump_label.c. - */ #define STATIC_KEY_INIT_TRUE \ - { .enabled = { 1 }, \ - { .type = JUMP_TYPE_TRUE } } + { .enabled = ATOMIC_INIT(1), \ + .type = JUMP_TYPE_TRUE } #define STATIC_KEY_INIT_FALSE \ - { .enabled = { 0 }, \ - { .type = JUMP_TYPE_FALSE } } + { .enabled = ATOMIC_INIT(0), \ + .type = JUMP_TYPE_FALSE } #else /* !CONFIG_JUMP_LABEL */ @@ -538,6 +524,6 @@ extern bool ____wrong_branch_error(void); #define static_branch_enable_cpuslocked(x) static_key_enable_cpuslocked(&(x)->key) #define static_branch_disable_cpuslocked(x) static_key_disable_cpuslocked(&(x)->key) -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ #endif /* _LINUX_JUMP_LABEL_H */ diff --git a/include/linux/jz4740-adc.h b/include/linux/jz4740-adc.h deleted file mode 100644 index 19d995c8bf06..000000000000 --- a/include/linux/jz4740-adc.h +++ /dev/null @@ -1,33 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ - -#ifndef __LINUX_JZ4740_ADC -#define __LINUX_JZ4740_ADC - -struct device; - -/* - * jz4740_adc_set_config - Configure a JZ4740 adc device - * @dev: Pointer to a jz4740-adc device - * @mask: Mask for the config value to be set - * @val: Value to be set - * - * This function can be used by the JZ4740 ADC mfd cells to configure their - * options in the shared config register. -*/ -int jz4740_adc_set_config(struct device *dev, uint32_t mask, uint32_t val); - -#define JZ_ADC_CONFIG_SPZZ BIT(31) -#define JZ_ADC_CONFIG_EX_IN BIT(30) -#define JZ_ADC_CONFIG_DNUM_MASK (0x7 << 16) -#define JZ_ADC_CONFIG_DMA_ENABLE BIT(15) -#define JZ_ADC_CONFIG_XYZ_MASK (0x2 << 13) -#define JZ_ADC_CONFIG_SAMPLE_NUM_MASK (0x7 << 10) -#define JZ_ADC_CONFIG_CLKDIV_MASK (0xf << 5) -#define JZ_ADC_CONFIG_BAT_MB BIT(4) - -#define JZ_ADC_CONFIG_DNUM(dnum) ((dnum) << 16) -#define JZ_ADC_CONFIG_XYZ_OFFSET(dnum) ((xyz) << 13) -#define JZ_ADC_CONFIG_SAMPLE_NUM(x) ((x) << 10) -#define JZ_ADC_CONFIG_CLKDIV(div) ((div) << 5) - -#endif diff --git a/include/linux/kasan.h b/include/linux/kasan.h index 338a1921a50a..bf233bde68c7 100644 --- a/include/linux/kasan.h +++ b/include/linux/kasan.h @@ -352,8 +352,8 @@ bool __kasan_mempool_poison_object(void *ptr, unsigned long ip); * kasan_mempool_unpoison_object(). * * This function operates on all slab allocations including large kmalloc - * allocations (the ones returned by kmalloc_large() or by kmalloc() with the - * size > KMALLOC_MAX_SIZE). + * allocations (i.e. the ones backed directly by the buddy allocator rather + * than kmalloc slab caches). * * Return: true if the allocation can be safely reused; false otherwise. */ @@ -381,8 +381,8 @@ void __kasan_mempool_unpoison_object(void *ptr, size_t size, unsigned long ip); * original tags based on the pointer value. * * This function operates on all slab allocations including large kmalloc - * allocations (the ones returned by kmalloc_large() or by kmalloc() with the - * size > KMALLOC_MAX_SIZE). + * allocations (i.e. the ones backed directly by the buddy allocator rather + * than kmalloc slab caches). */ static __always_inline void kasan_mempool_unpoison_object(void *ptr, size_t size) diff --git a/include/linux/kcov.h b/include/linux/kcov.h index 0143358874b0..895b761b2db1 100644 --- a/include/linux/kcov.h +++ b/include/linux/kcov.h @@ -21,8 +21,6 @@ enum kcov_mode { KCOV_MODE_TRACE_PC = 2, /* Collecting comparison operands mode. */ KCOV_MODE_TRACE_CMP = 3, - /* The process owns a KCOV remote reference. */ - KCOV_MODE_REMOTE = 4, }; #define KCOV_IN_CTXSW (1 << 30) @@ -43,11 +41,11 @@ do { \ /* See Documentation/dev-tools/kcov.rst for usage details. */ void kcov_remote_start(u64 handle); void kcov_remote_stop(void); -u64 kcov_common_handle(void); +struct kcov_common_handle_id kcov_common_handle(void); -static inline void kcov_remote_start_common(u64 id) +static inline void kcov_remote_start_common(struct kcov_common_handle_id id) { - kcov_remote_start(kcov_remote_handle(KCOV_SUBSYSTEM_COMMON, id)); + kcov_remote_start(kcov_remote_handle(KCOV_SUBSYSTEM_COMMON, id.val)); } static inline void kcov_remote_start_usb(u64 id) @@ -99,11 +97,11 @@ static inline void kcov_prepare_switch(struct task_struct *t) {} static inline void kcov_finish_switch(struct task_struct *t) {} static inline void kcov_remote_start(u64 handle) {} static inline void kcov_remote_stop(void) {} -static inline u64 kcov_common_handle(void) +static inline struct kcov_common_handle_id kcov_common_handle(void) { - return 0; + return (struct kcov_common_handle_id){}; } -static inline void kcov_remote_start_common(u64 id) {} +static inline void kcov_remote_start_common(struct kcov_common_handle_id id) {} static inline void kcov_remote_start_usb(u64 id) {} static inline void kcov_remote_start_usb_softirq(u64 id) {} static inline void kcov_remote_stop_softirq(void) {} diff --git a/include/linux/kcsan-checks.h b/include/linux/kcsan-checks.h index 92f3843d9ebb..e135dacaa90f 100644 --- a/include/linux/kcsan-checks.h +++ b/include/linux/kcsan-checks.h @@ -282,7 +282,7 @@ static inline void __kcsan_disable_current(void) { } * @size: size of access */ #define __kcsan_check_write(ptr, size) \ - __kcsan_check_access(ptr, size, KCSAN_ACCESS_WRITE) + __kcsan_check_access(absolute_pointer(ptr), size, KCSAN_ACCESS_WRITE) /** * __kcsan_check_read_write - check regular read-write access for races @@ -308,7 +308,7 @@ static inline void __kcsan_disable_current(void) { } * @size: size of access */ #define kcsan_check_write(ptr, size) \ - kcsan_check_access(ptr, size, KCSAN_ACCESS_WRITE) + kcsan_check_access(absolute_pointer(ptr), size, KCSAN_ACCESS_WRITE) /** * kcsan_check_read_write - check regular read-write access for races @@ -331,7 +331,7 @@ static inline void __kcsan_disable_current(void) { } #define kcsan_check_atomic_read(ptr, size) \ kcsan_check_access(ptr, size, KCSAN_ACCESS_ATOMIC) #define kcsan_check_atomic_write(ptr, size) \ - kcsan_check_access(ptr, size, KCSAN_ACCESS_ATOMIC | KCSAN_ACCESS_WRITE) + kcsan_check_access(absolute_pointer(ptr), size, KCSAN_ACCESS_ATOMIC | KCSAN_ACCESS_WRITE) #define kcsan_check_atomic_read_write(ptr, size) \ kcsan_check_access(ptr, size, KCSAN_ACCESS_ATOMIC | KCSAN_ACCESS_WRITE | KCSAN_ACCESS_COMPOUND) #endif diff --git a/include/linux/kernel.h b/include/linux/kernel.h index e5570a16cbb1..24414c79e59a 100644 --- a/include/linux/kernel.h +++ b/include/linux/kernel.h @@ -72,7 +72,7 @@ extern int dynamic_might_resched(void); #ifdef CONFIG_DEBUG_ATOMIC_SLEEP extern void __might_resched(const char *file, int line, unsigned int offsets); extern void __might_sleep(const char *file, int line); -extern void __cant_sleep(const char *file, int line, int preempt_offset); +extern void __cant_sleep(const char *file, int line); extern void __cant_migrate(const char *file, int line); /** @@ -95,7 +95,7 @@ extern void __cant_migrate(const char *file, int line); * this macro will print a stack trace if it is executed with preemption enabled */ # define cant_sleep() \ - do { __cant_sleep(__FILE__, __LINE__, 0); } while (0) + do { __cant_sleep(__FILE__, __LINE__); } while (0) # define sched_annotate_sleep() (current->task_state_change = 0) /** diff --git a/include/linux/kernel_stat.h b/include/linux/kernel_stat.h index b97ce2df376f..9ca6c2259dfe 100644 --- a/include/linux/kernel_stat.h +++ b/include/linux/kernel_stat.h @@ -34,7 +34,14 @@ enum cpu_usage_stat { }; struct kernel_cpustat { - u64 cpustat[NR_STATS]; +#ifdef CONFIG_NO_HZ_COMMON + bool idle_dyntick; + bool idle_elapse; + seqcount_t idle_sleeptime_seq; + u64 idle_entrytime; + u64 idle_stealtime[2]; +#endif + u64 cpustat[NR_STATS]; }; struct kernel_stat { @@ -99,23 +106,95 @@ static inline unsigned long kstat_cpu_irqs_sum(unsigned int cpu) return kstat_cpu(cpu).irqs_sum; } +#ifdef CONFIG_NO_HZ_COMMON + +#ifdef CONFIG_HAVE_VIRT_CPU_ACCOUNTING_IDLE + +static inline void kcpustat_dyntick_start(u64 now) { } +static inline void kcpustat_dyntick_stop(u64 now) { } +static inline void kcpustat_irq_enter(u64 now) { } +static inline void kcpustat_irq_exit(u64 now) { } +static inline bool kcpustat_idle_dyntick(void) { return false; } + +extern u64 arch_kcpustat_field_idle(int cpu); +extern u64 arch_kcpustat_field_iowait(int cpu); + +static inline u64 kcpustat_field_idle(int cpu) +{ + return arch_kcpustat_field_idle(cpu); +} + +static inline u64 kcpustat_field_iowait(int cpu) +{ + return arch_kcpustat_field_iowait(cpu); +} + +#else /* !CONFIG_HAVE_VIRT_CPU_ACCOUNTING_IDLE */ + +extern void kcpustat_dyntick_start(u64 now); +extern void kcpustat_dyntick_stop(u64 now); +extern void kcpustat_irq_enter(u64 now); +extern void kcpustat_irq_exit(u64 now); +extern u64 kcpustat_field_idle(int cpu); +extern u64 kcpustat_field_iowait(int cpu); + +static inline bool kcpustat_idle_dyntick(void) +{ + return __this_cpu_read(kernel_cpustat.idle_dyntick); +} + +#endif /* !CONFIG_HAVE_VIRT_CPU_ACCOUNTING_IDLE */ + +#else +static inline u64 kcpustat_field_idle(int cpu) +{ + return kcpustat_cpu(cpu).cpustat[CPUTIME_IDLE]; +} +static inline u64 kcpustat_field_iowait(int cpu) +{ + return kcpustat_cpu(cpu).cpustat[CPUTIME_IOWAIT]; +} + +static inline bool kcpustat_idle_dyntick(void) +{ + return false; +} +#endif /* CONFIG_NO_HZ_COMMON */ + +extern u64 get_cpu_idle_time_us(int cpu, u64 *last_update_time); +extern u64 get_cpu_iowait_time_us(int cpu, u64 *last_update_time); + +/* Fetch cputime values when vtime is disabled on a CPU */ +static inline u64 kcpustat_field_default(enum cpu_usage_stat usage, int cpu) +{ + if (usage == CPUTIME_IDLE) + return kcpustat_field_idle(cpu); + if (usage == CPUTIME_IOWAIT) + return kcpustat_field_iowait(cpu); + return kcpustat_cpu(cpu).cpustat[usage]; +} + +static inline void kcpustat_cpu_fetch_default(struct kernel_cpustat *dst, int cpu) +{ + *dst = kcpustat_cpu(cpu); + dst->cpustat[CPUTIME_IDLE] = kcpustat_field_idle(cpu); + dst->cpustat[CPUTIME_IOWAIT] = kcpustat_field_iowait(cpu); +} + #ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN -extern u64 kcpustat_field(struct kernel_cpustat *kcpustat, - enum cpu_usage_stat usage, int cpu); +extern u64 kcpustat_field(enum cpu_usage_stat usage, int cpu); extern void kcpustat_cpu_fetch(struct kernel_cpustat *dst, int cpu); #else -static inline u64 kcpustat_field(struct kernel_cpustat *kcpustat, - enum cpu_usage_stat usage, int cpu) +static inline u64 kcpustat_field(enum cpu_usage_stat usage, int cpu) { - return kcpustat->cpustat[usage]; + return kcpustat_field_default(usage, cpu); } static inline void kcpustat_cpu_fetch(struct kernel_cpustat *dst, int cpu) { - *dst = kcpustat_cpu(cpu); + kcpustat_cpu_fetch_default(dst, cpu); } - -#endif +#endif /* !CONFIG_VIRT_CPU_ACCOUNTING_GEN */ extern void account_user_time(struct task_struct *, u64); extern void account_guest_time(struct task_struct *, u64); @@ -124,19 +203,17 @@ extern void account_system_index_time(struct task_struct *, u64, enum cpu_usage_stat); extern void account_steal_time(u64); extern void account_idle_time(u64); -extern u64 get_idle_time(struct kernel_cpustat *kcs, int cpu); #ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE static inline void account_process_tick(struct task_struct *tsk, int user) { - vtime_flush(tsk); + if (!kcpustat_idle_dyntick()) + vtime_flush(tsk); } #else extern void account_process_tick(struct task_struct *, int user); #endif -extern void account_idle_ticks(unsigned long ticks); - #ifdef CONFIG_SCHED_CORE extern void __account_forceidle_time(struct task_struct *tsk, u64 delta); #endif diff --git a/include/linux/kernelcapi.h b/include/linux/kernelcapi.h deleted file mode 100644 index 94ba42bf9da1..000000000000 --- a/include/linux/kernelcapi.h +++ /dev/null @@ -1,45 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * $Id: kernelcapi.h,v 1.8.6.2 2001/02/07 11:31:31 kai Exp $ - * - * Kernel CAPI 2.0 Interface for Linux - * - * (c) Copyright 1997 by Carsten Paeth (calle@calle.in-berlin.de) - * - */ -#ifndef __KERNELCAPI_H__ -#define __KERNELCAPI_H__ - -#include <linux/list.h> -#include <linux/skbuff.h> -#include <linux/workqueue.h> -#include <linux/notifier.h> -#include <uapi/linux/kernelcapi.h> - -#define CAPI_NOERROR 0x0000 - -#define CAPI_TOOMANYAPPLS 0x1001 -#define CAPI_LOGBLKSIZETOSMALL 0x1002 -#define CAPI_BUFFEXECEEDS64K 0x1003 -#define CAPI_MSGBUFSIZETOOSMALL 0x1004 -#define CAPI_ANZLOGCONNNOTSUPPORTED 0x1005 -#define CAPI_REGRESERVED 0x1006 -#define CAPI_REGBUSY 0x1007 -#define CAPI_REGOSRESOURCEERR 0x1008 -#define CAPI_REGNOTINSTALLED 0x1009 -#define CAPI_REGCTRLERNOTSUPPORTEXTEQUIP 0x100a -#define CAPI_REGCTRLERONLYSUPPORTEXTEQUIP 0x100b - -#define CAPI_ILLAPPNR 0x1101 -#define CAPI_ILLCMDORSUBCMDORMSGTOSMALL 0x1102 -#define CAPI_SENDQUEUEFULL 0x1103 -#define CAPI_RECEIVEQUEUEEMPTY 0x1104 -#define CAPI_RECEIVEOVERFLOW 0x1105 -#define CAPI_UNKNOWNNOTPAR 0x1106 -#define CAPI_MSGBUSY 0x1107 -#define CAPI_MSGOSRESOURCEERR 0x1108 -#define CAPI_MSGNOTINSTALLED 0x1109 -#define CAPI_MSGCTRLERNOTSUPPORTEXTEQUIP 0x110a -#define CAPI_MSGCTRLERONLYSUPPORTEXTEQUIP 0x110b - -#endif /* __KERNELCAPI_H__ */ diff --git a/include/linux/kernfs.h b/include/linux/kernfs.h index b5a5f32fdfd1..351a5101c862 100644 --- a/include/linux/kernfs.h +++ b/include/linux/kernfs.h @@ -23,6 +23,7 @@ struct file; struct dentry; struct iattr; +struct ns_common; struct seq_file; struct vm_area_struct; struct vm_operations_struct; @@ -75,20 +76,25 @@ struct kernfs_iattrs; * kernfs_open_file. * * kernfs_open_files are chained at kernfs_open_node->files, which is - * protected by kernfs_global_locks.open_file_mutex[i]. + * protected by kernfs_global_locks.node_mutex[i]. * * To reduce possible contention in sysfs access, arising due to single - * locks, use an array of locks (e.g. open_file_mutex) and use kernfs_node + * locks, use an array of locks (e.g. node_mutex) and use kernfs_node * object address as hash keys to get the index of these locks. * * Hashed mutexes are safe to use here because operations using these don't * rely on global exclusion. * + * The hashed mutex array protects per-node data: the kernfs_open_node for + * open file management, and kernfs_node xattr operations (necessary because + * multiple superblocks with different namespaces can share the same + * kernfs_node, making per-inode locking insufficient). + * * In future we intend to replace other global locks with hashed ones as well. * kernfs_global_locks acts as a holder for all such hash tables. */ struct kernfs_global_locks { - struct mutex open_file_mutex[NR_KERNFS_LOCKS]; + struct mutex node_mutex[NR_KERNFS_LOCKS]; }; enum kernfs_node_type { @@ -99,8 +105,6 @@ enum kernfs_node_type { #define KERNFS_TYPE_MASK 0x000f #define KERNFS_FLAG_MASK ~KERNFS_TYPE_MASK -#define KERNFS_MAX_USER_XATTRS 128 -#define KERNFS_USER_XATTR_SIZE_LIMIT (128 << 10) enum kernfs_node_flag { KERNFS_ACTIVATED = 0x0010, @@ -209,7 +213,7 @@ struct kernfs_node { struct rb_node rb; - const void *ns; /* namespace tag */ + const struct ns_common *ns; /* namespace tag */ unsigned int hash; /* ns + name hash */ unsigned short flags; umode_t mode; @@ -331,7 +335,7 @@ struct kernfs_ops { */ struct kernfs_fs_context { struct kernfs_root *root; /* Root of the hierarchy being mounted */ - void *ns_tag; /* Namespace tag of the mount (or NULL) */ + struct ns_common *ns_tag; /* Namespace tag of the mount (or NULL) */ unsigned long magic; /* File system specific magic number */ /* The following are set/used by kernfs_mount() */ @@ -406,9 +410,11 @@ void pr_cont_kernfs_name(struct kernfs_node *kn); void pr_cont_kernfs_path(struct kernfs_node *kn); struct kernfs_node *kernfs_get_parent(struct kernfs_node *kn); struct kernfs_node *kernfs_find_and_get_ns(struct kernfs_node *parent, - const char *name, const void *ns); + const char *name, + const struct ns_common *ns); struct kernfs_node *kernfs_walk_and_get_ns(struct kernfs_node *parent, - const char *path, const void *ns); + const char *path, + const struct ns_common *ns); void kernfs_get(struct kernfs_node *kn); void kernfs_put(struct kernfs_node *kn); @@ -426,7 +432,8 @@ unsigned int kernfs_root_flags(struct kernfs_node *kn); struct kernfs_node *kernfs_create_dir_ns(struct kernfs_node *parent, const char *name, umode_t mode, kuid_t uid, kgid_t gid, - void *priv, const void *ns); + void *priv, + const struct ns_common *ns); struct kernfs_node *kernfs_create_empty_dir(struct kernfs_node *parent, const char *name); struct kernfs_node *__kernfs_create_file(struct kernfs_node *parent, @@ -434,7 +441,8 @@ struct kernfs_node *__kernfs_create_file(struct kernfs_node *parent, kuid_t uid, kgid_t gid, loff_t size, const struct kernfs_ops *ops, - void *priv, const void *ns, + void *priv, + const struct ns_common *ns, struct lock_class_key *key); struct kernfs_node *kernfs_create_link(struct kernfs_node *parent, const char *name, @@ -446,9 +454,9 @@ void kernfs_break_active_protection(struct kernfs_node *kn); void kernfs_unbreak_active_protection(struct kernfs_node *kn); bool kernfs_remove_self(struct kernfs_node *kn); int kernfs_remove_by_name_ns(struct kernfs_node *parent, const char *name, - const void *ns); + const struct ns_common *ns); int kernfs_rename_ns(struct kernfs_node *kn, struct kernfs_node *new_parent, - const char *new_name, const void *new_ns); + const char *new_name, const struct ns_common *new_ns); int kernfs_setattr(struct kernfs_node *kn, const struct iattr *iattr); __poll_t kernfs_generic_poll(struct kernfs_open_file *of, struct poll_table_struct *pt); @@ -459,7 +467,7 @@ int kernfs_xattr_get(struct kernfs_node *kn, const char *name, int kernfs_xattr_set(struct kernfs_node *kn, const char *name, const void *value, size_t size, int flags); -const void *kernfs_super_ns(struct super_block *sb); +const struct ns_common *kernfs_super_ns(struct super_block *sb); int kernfs_get_tree(struct fs_context *fc); void kernfs_free_fs_context(struct fs_context *fc); void kernfs_kill_sb(struct super_block *sb); @@ -494,11 +502,11 @@ static inline struct kernfs_node *kernfs_get_parent(struct kernfs_node *kn) static inline struct kernfs_node * kernfs_find_and_get_ns(struct kernfs_node *parent, const char *name, - const void *ns) + const struct ns_common *ns) { return NULL; } static inline struct kernfs_node * kernfs_walk_and_get_ns(struct kernfs_node *parent, const char *path, - const void *ns) + const struct ns_common *ns) { return NULL; } static inline void kernfs_get(struct kernfs_node *kn) { } @@ -526,14 +534,15 @@ static inline unsigned int kernfs_root_flags(struct kernfs_node *kn) static inline struct kernfs_node * kernfs_create_dir_ns(struct kernfs_node *parent, const char *name, umode_t mode, kuid_t uid, kgid_t gid, - void *priv, const void *ns) + void *priv, const struct ns_common *ns) { return ERR_PTR(-ENOSYS); } static inline struct kernfs_node * __kernfs_create_file(struct kernfs_node *parent, const char *name, umode_t mode, kuid_t uid, kgid_t gid, loff_t size, const struct kernfs_ops *ops, - void *priv, const void *ns, struct lock_class_key *key) + void *priv, const struct ns_common *ns, + struct lock_class_key *key) { return ERR_PTR(-ENOSYS); } static inline struct kernfs_node * @@ -549,12 +558,14 @@ static inline bool kernfs_remove_self(struct kernfs_node *kn) { return false; } static inline int kernfs_remove_by_name_ns(struct kernfs_node *kn, - const char *name, const void *ns) + const char *name, + const struct ns_common *ns) { return -ENOSYS; } static inline int kernfs_rename_ns(struct kernfs_node *kn, struct kernfs_node *new_parent, - const char *new_name, const void *new_ns) + const char *new_name, + const struct ns_common *new_ns) { return -ENOSYS; } static inline int kernfs_setattr(struct kernfs_node *kn, @@ -575,7 +586,7 @@ static inline int kernfs_xattr_set(struct kernfs_node *kn, const char *name, const void *value, size_t size, int flags) { return -ENOSYS; } -static inline const void *kernfs_super_ns(struct super_block *sb) +static inline const struct ns_common *kernfs_super_ns(struct super_block *sb) { return NULL; } static inline int kernfs_get_tree(struct fs_context *fc) diff --git a/include/linux/kexec.h b/include/linux/kexec.h index 8a22bc9b8c6c..0af8ae4fdd08 100644 --- a/include/linux/kexec.h +++ b/include/linux/kexec.h @@ -13,7 +13,7 @@ #define IND_SOURCE (1 << IND_SOURCE_BIT) #define IND_FLAGS (IND_DESTINATION | IND_INDIRECTION | IND_DONE | IND_SOURCE) -#if !defined(__ASSEMBLY__) +#if !defined(__ASSEMBLER__) #include <linux/vmcore_info.h> #include <linux/crash_reserve.h> diff --git a/include/linux/kexec_handover.h b/include/linux/kexec_handover.h index ac4129d1d741..46de86dc343e 100644 --- a/include/linux/kexec_handover.h +++ b/include/linux/kexec_handover.h @@ -5,11 +5,8 @@ #include <linux/err.h> #include <linux/errno.h> #include <linux/types.h> - -struct kho_scratch { - phys_addr_t addr; - phys_addr_t size; -}; +#include <linux/mm.h> +#include <asm-generic/kexec_handover.h> struct kho_vmalloc; @@ -32,14 +29,25 @@ void kho_restore_free(void *mem); struct folio *kho_restore_folio(phys_addr_t phys); struct page *kho_restore_pages(phys_addr_t phys, unsigned long nr_pages); void *kho_restore_vmalloc(const struct kho_vmalloc *preservation); -int kho_add_subtree(const char *name, void *fdt); -void kho_remove_subtree(void *fdt); -int kho_retrieve_subtree(const char *name, phys_addr_t *phys); +int kho_add_subtree(const char *name, void *blob, size_t size); +void kho_remove_subtree(void *blob); +int kho_retrieve_subtree(const char *name, phys_addr_t *phys, size_t *size); void kho_memory_init(void); +void kho_memory_init_early(void); void kho_populate(phys_addr_t fdt_phys, u64 fdt_len, phys_addr_t scratch_phys, u64 scratch_len); + +bool kho_scratch_overlap(phys_addr_t phys, size_t size); + +static inline enum migratetype kho_scratch_migratetype(unsigned long pfn, + enum migratetype mt) +{ + if (kho_scratch_overlap(PFN_PHYS(pfn), pageblock_nr_pages << PAGE_SHIFT)) + return MIGRATE_CMA; + return mt; +} #else static inline bool kho_is_enabled(void) { @@ -97,24 +105,38 @@ static inline void *kho_restore_vmalloc(const struct kho_vmalloc *preservation) return NULL; } -static inline int kho_add_subtree(const char *name, void *fdt) +static inline int kho_add_subtree(const char *name, void *blob, size_t size) { return -EOPNOTSUPP; } -static inline void kho_remove_subtree(void *fdt) { } +static inline void kho_remove_subtree(void *blob) { } -static inline int kho_retrieve_subtree(const char *name, phys_addr_t *phys) +static inline int kho_retrieve_subtree(const char *name, phys_addr_t *phys, + size_t *size) { return -EOPNOTSUPP; } static inline void kho_memory_init(void) { } +static inline void kho_memory_init_early(void) { } + static inline void kho_populate(phys_addr_t fdt_phys, u64 fdt_len, phys_addr_t scratch_phys, u64 scratch_len) { } + +static inline bool kho_scratch_overlap(phys_addr_t phys, size_t size) +{ + return false; +} + +static inline enum migratetype kho_scratch_migratetype(unsigned long pfn, + enum migratetype mt) +{ + return mt; +} #endif /* CONFIG_KEXEC_HANDOVER */ #endif /* LINUX_KEXEC_HANDOVER_H */ diff --git a/include/linux/kgdb.h b/include/linux/kgdb.h index 22b3f3839f30..6c46591a2eac 100644 --- a/include/linux/kgdb.h +++ b/include/linux/kgdb.h @@ -202,9 +202,10 @@ extern void kgdb_call_nmi_hook(void *ignored); * * On SMP systems, we need to get the attention of the other CPUs * and get them into a known state. This should do what is needed - * to get the other CPUs to call kgdb_wait(). Note that on some arches, - * the NMI approach is not used for rounding up all the CPUs. Normally - * those architectures can just not implement this and get the default. + * to get the other CPUs to call kgdb_handle_exception(). Note that + * on some arches, the NMI approach is not used for rounding up all + * the CPUs. Normally those architectures can just not implement + * this and get the default. * * On non-SMP systems, this is not called. */ diff --git a/include/linux/kho/abi/block.h b/include/linux/kho/abi/block.h new file mode 100644 index 000000000000..d06d64b963be --- /dev/null +++ b/include/linux/kho/abi/block.h @@ -0,0 +1,54 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2026, Google LLC. + * Pasha Tatashin <pasha.tatashin@soleen.com> + */ + +/** + * DOC: KHO Serialization Blocks ABI + * + * Subsystems using the KHO Serialization Blocks framework rely on the stable + * Application Binary Interface defined below to pass serialized state from a + * pre-update kernel to a post-update kernel. + * + * This interface is a contract. Any modification to the structure fields, + * compatible strings, or the layout of the `__packed` serialization + * structures defined here constitutes a breaking change. Such changes require + * incrementing the version number in the `KHO_FDT_COMPATIBLE` string to + * prevent a new kernel from misinterpreting data from an old kernel. + * + * Changes are allowed provided the compatibility version is incremented; + * however, backward/forward compatibility is only guaranteed for kernels + * supporting the same ABI version. + */ + +#ifndef _LINUX_KHO_ABI_BLOCK_H +#define _LINUX_KHO_ABI_BLOCK_H + +#include <asm/page.h> +#include <linux/types.h> + +/** + * KHO_BLOCK_SIZE - The size of each serialization block. + * + * This is defined as PAGE_SIZE. PAGE_SIZE is ABI compliant because live + * update between kernels with different page sizes is not supported by KHO. + */ +#define KHO_BLOCK_SIZE PAGE_SIZE + +/** + * struct kho_block_header_ser - Header for the serialized data block. + * @next: Physical address of the next struct kho_block_header_ser. + * @count: The number of entries that immediately follow this header in the + * memory block. + * + * This structure is located at the beginning of a block of physical memory + * preserved across a kexec. It provides the necessary metadata to interpret + * the array of entries that follow. + */ +struct kho_block_header_ser { + u64 next; + u64 count; +} __packed; + +#endif /* _LINUX_KHO_ABI_BLOCK_H */ diff --git a/include/linux/kho/abi/kexec_handover.h b/include/linux/kho/abi/kexec_handover.h index 2201a0d2c159..2f4fb9c63942 100644 --- a/include/linux/kho/abi/kexec_handover.h +++ b/include/linux/kho/abi/kexec_handover.h @@ -10,8 +10,13 @@ #ifndef _LINUX_KHO_ABI_KEXEC_HANDOVER_H #define _LINUX_KHO_ABI_KEXEC_HANDOVER_H +#include <linux/bits.h> +#include <linux/log2.h> +#include <linux/math.h> #include <linux/types.h> +#include <asm/page.h> + /** * DOC: Kexec Handover ABI * @@ -29,34 +34,37 @@ * compatibility is only guaranteed for kernels supporting the same ABI version. * * FDT Structure Overview: - * The FDT serves as a central registry for physical - * addresses of preserved data structures and sub-FDTs. The first kernel - * populates this FDT with references to memory regions and other FDTs that - * need to persist across the kexec transition. The subsequent kernel then - * parses this FDT to locate and restore the preserved data.:: + * The FDT serves as a central registry for physical addresses of preserved + * data structures. The first kernel populates this FDT with references to + * memory regions and other metadata that need to persist across the kexec + * transition. The subsequent kernel then parses this FDT to locate and + * restore the preserved data.:: * * / { - * compatible = "kho-v1"; + * compatible = "kho-v3"; * * preserved-memory-map = <0x...>; * * <subnode-name-1> { - * fdt = <0x...>; + * preserved-data = <0x...>; + * blob-size = <0x...>; * }; * * <subnode-name-2> { - * fdt = <0x...>; + * preserved-data = <0x...>; + * blob-size = <0x...>; * }; * ... ... * <subnode-name-N> { - * fdt = <0x...>; + * preserved-data = <0x...>; + * blob-size = <0x...>; * }; * }; * * Root KHO Node (/): - * - compatible: "kho-v1" + * - compatible: "kho-v3" * - * Indentifies the overall KHO ABI version. + * Identifies the overall KHO ABI version. * * - preserved-memory-map: u64 * @@ -69,20 +77,29 @@ * is provided by the subsystem that uses KHO for preserving its * data. * - * - fdt: u64 + * - preserved-data: u64 * - * Physical address pointing to a subnode FDT blob that is also + * Physical address pointing to a subnode data blob that is also * being preserved. + * + * - blob-size: u64 + * + * Size in bytes of the preserved data blob. This is needed because + * blobs may use arbitrary formats (not just FDT), so the size + * cannot be determined from the blob content alone. */ /* The compatible string for the KHO FDT root node. */ -#define KHO_FDT_COMPATIBLE "kho-v1" +#define KHO_FDT_COMPATIBLE "kho-v4" /* The FDT property for the preserved memory map. */ #define KHO_FDT_MEMORY_MAP_PROP_NAME "preserved-memory-map" -/* The FDT property for sub-FDTs. */ -#define KHO_FDT_SUB_TREE_PROP_NAME "fdt" +/* The FDT property for preserved data blobs. */ +#define KHO_SUB_TREE_PROP_NAME "preserved-data" + +/* The FDT property for the size of preserved data blobs. */ +#define KHO_SUB_TREE_SIZE_PROP_NAME "blob-size" /** * DOC: Kexec Handover ABI for vmalloc Preservation @@ -160,4 +177,110 @@ struct kho_vmalloc { unsigned short order; }; +/** + * DOC: KHO persistent memory tracker + * + * KHO tracks preserved memory using a radix tree data structure. Each node of + * the tree is exactly a single page. The leaf nodes are bitmaps where each set + * bit is a preserved page of any order. The intermediate nodes are tables of + * physical addresses that point to a lower level node. + * + * The tree hierarchy is shown below:: + * + * root + * +-------------------+ + * | Level 5 | (struct kho_radix_node) + * +-------------------+ + * | + * v + * +-------------------+ + * | Level 4 | (struct kho_radix_node) + * +-------------------+ + * | + * | ... (intermediate levels) + * | + * v + * +-------------------+ + * | Level 0 | (struct kho_radix_leaf) + * +-------------------+ + * + * The tree is traversed using a key that encodes the page's physical address + * (pa) and its order into a single unsigned long value. The encoded key value + * is composed of two parts: the 'order bit' in the upper part and the + * 'shifted physical address' in the lower part.:: + * + * +------------+-----------------------------+--------------------------+ + * | Page Order | Order Bit | Shifted Physical Address | + * +------------+-----------------------------+--------------------------+ + * | 0 | ...000100 ... (at bit 52) | pa >> (PAGE_SHIFT + 0) | + * | 1 | ...000010 ... (at bit 51) | pa >> (PAGE_SHIFT + 1) | + * | 2 | ...000001 ... (at bit 50) | pa >> (PAGE_SHIFT + 2) | + * | ... | ... | ... | + * +------------+-----------------------------+--------------------------+ + * + * Shifted Physical Address: + * The 'shifted physical address' is the physical address normalized for its + * order. It effectively represents the PFN shifted right by the order. + * + * Order Bit: + * The 'order bit' encodes the page order by setting a single bit at a + * specific position. The position of this bit itself represents the order. + * + * For instance, on a 64-bit system with 4KB pages (PAGE_SHIFT = 12), the + * maximum range for the shifted physical address (for order 0) is 52 bits + * (64 - 12). This address occupies bits [0-51]. For order 0, the order bit is + * set at position 52. + * + * The following diagram illustrates how the encoded key value is split into + * indices for the tree levels, with PAGE_SIZE of 4KB:: + * + * 63:60 59:51 50:42 41:33 32:24 23:15 14:0 + * +---------+--------+--------+--------+--------+--------+-----------------+ + * | 0 | Lv 5 | Lv 4 | Lv 3 | Lv 2 | Lv 1 | Lv 0 (bitmap) | + * +---------+--------+--------+--------+--------+--------+-----------------+ + * + * The radix tree stores pages of all orders in a single 6-level hierarchy. It + * efficiently shares higher tree levels, especially due to common zero top + * address bits, allowing a single, efficient algorithm to manage all + * pages. This bitmap approach also offers memory efficiency; for example, a + * 512KB bitmap can cover a 16GB memory range for 0-order pages with PAGE_SIZE = + * 4KB. + * + * The data structures defined here are part of the KHO ABI. Any modification + * to these structures that breaks backward compatibility must be accompanied by + * an update to the "compatible" string. This ensures that a newer kernel can + * correctly interpret the data passed by an older kernel. + */ + +/* + * Defines constants for the KHO radix tree structure, used to track preserved + * memory. These constants govern the indexing, sizing, and depth of the tree. + */ +enum kho_radix_consts { + /* Need to store the PFN, plus one bit for order. */ + KHO_RADIX_KEY_WIDTH = 64 - PAGE_SHIFT + 1, + + /* Size of the table in kho_radix_node, in log2 */ + KHO_TABLE_SIZE_LOG2 = const_ilog2(PAGE_SIZE / sizeof(phys_addr_t)), + + /* Number of bits in the kho_radix_leaf bitmap, in log2 */ + KHO_BITMAP_SIZE_LOG2 = PAGE_SHIFT + const_ilog2(BITS_PER_BYTE), + + /* + * The total tree depth is the number of intermediate levels + * and 1 bitmap level. + */ + KHO_TREE_MAX_DEPTH = + DIV_ROUND_UP(KHO_RADIX_KEY_WIDTH - KHO_BITMAP_SIZE_LOG2, + KHO_TABLE_SIZE_LOG2) + 1, +}; + +struct kho_radix_node { + u64 table[1 << KHO_TABLE_SIZE_LOG2]; +}; + +struct kho_radix_leaf { + DECLARE_BITMAP(bitmap, 1 << KHO_BITMAP_SIZE_LOG2); +}; + #endif /* _LINUX_KHO_ABI_KEXEC_HANDOVER_H */ diff --git a/include/linux/kho/abi/kexec_metadata.h b/include/linux/kho/abi/kexec_metadata.h new file mode 100644 index 000000000000..e9e3f7e38a7c --- /dev/null +++ b/include/linux/kho/abi/kexec_metadata.h @@ -0,0 +1,46 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +/** + * DOC: Kexec Metadata ABI + * + * The "kexec-metadata" subtree stores optional metadata about the kexec chain. + * It is registered via kho_add_subtree(), keeping it independent from the core + * KHO ABI. This allows the metadata format to evolve without affecting other + * KHO consumers. + * + * The metadata is stored as a plain C struct rather than FDT format for + * simplicity and direct field access. + * + * Copyright (c) 2026 Meta Platforms, Inc. and affiliates. + * Copyright (c) 2026 Breno Leitao <leitao@debian.org> + */ + +#ifndef _LINUX_KHO_ABI_KEXEC_METADATA_H +#define _LINUX_KHO_ABI_KEXEC_METADATA_H + +#include <linux/types.h> +#include <linux/utsname.h> + +#define KHO_KEXEC_METADATA_VERSION 1 + +/** + * struct kho_kexec_metadata - Kexec metadata passed between kernels + * @version: ABI version of this struct (must be first field) + * @previous_release: Kernel version string that initiated the kexec + * @kexec_count: Number of kexec boots since last cold boot + * + * This structure is preserved across kexec and allows the new kernel to + * identify which kernel it was booted from and how many kexec reboots + * have occurred. + * + * __NEW_UTS_LEN is part of uABI, so it safe to use it in here. + */ +struct kho_kexec_metadata { + u32 version; + char previous_release[__NEW_UTS_LEN + 1]; + u32 kexec_count; +} __packed; + +#define KHO_METADATA_NODE_NAME "kexec-metadata" + +#endif /* _LINUX_KHO_ABI_KEXEC_METADATA_H */ diff --git a/include/linux/kho/abi/luo.h b/include/linux/kho/abi/luo.h index 46750a0ddf88..288076de6d4a 100644 --- a/include/linux/kho/abi/luo.h +++ b/include/linux/kho/abi/luo.h @@ -10,11 +10,11 @@ * * Live Update Orchestrator uses the stable Application Binary Interface * defined below to pass state from a pre-update kernel to a post-update - * kernel. The ABI is built upon the Kexec HandOver framework and uses a - * Flattened Device Tree to describe the preserved data. + * kernel. The ABI is built upon the Kexec HandOver framework and registers + * the central `struct luo_ser` via the KHO raw subtree API. * - * This interface is a contract. Any modification to the FDT structure, node - * properties, compatible strings, or the layout of the `__packed` serialization + * This interface is a contract. Any modification to the structure fields, + * compatible strings, or the layout of the `__packed` serialization * structures defined here constitutes a breaking change. Such changes require * incrementing the version number in the relevant `_COMPATIBLE` string to * prevent a new kernel from misinterpreting data from an old kernel. @@ -23,68 +23,20 @@ * however, backward/forward compatibility is only guaranteed for kernels * supporting the same ABI version. * - * FDT Structure Overview: + * KHO Structure Overview: * The entire LUO state is encapsulated within a single KHO entry named "LUO". - * This entry contains an FDT with the following layout: - * - * .. code-block:: none - * - * / { - * compatible = "luo-v1"; - * liveupdate-number = <...>; - * - * luo-session { - * compatible = "luo-session-v1"; - * luo-session-header = <phys_addr_of_session_header_ser>; - * }; - * - * luo-flb { - * compatible = "luo-flb-v1"; - * luo-flb-header = <phys_addr_of_flb_header_ser>; - * }; - * }; - * - * Main LUO Node (/): - * - * - compatible: "luo-v1" - * Identifies the overall LUO ABI version. - * - liveupdate-number: u64 - * A counter tracking the number of successful live updates performed. - * - * Session Node (luo-session): - * This node describes all preserved user-space sessions. - * - * - compatible: "luo-session-v1" - * Identifies the session ABI version. - * - luo-session-header: u64 - * The physical address of a `struct luo_session_header_ser`. This structure - * is the header for a contiguous block of memory containing an array of - * `struct luo_session_ser`, one for each preserved session. - * - * File-Lifecycle-Bound Node (luo-flb): - * This node describes all preserved global objects whose lifecycle is bound - * to that of the preserved files (e.g., shared IOMMU state). - * - * - compatible: "luo-flb-v1" - * Identifies the FLB ABI version. - * - luo-flb-header: u64 - * The physical address of a `struct luo_flb_header_ser`. This structure is - * the header for a contiguous block of memory containing an array of - * `struct luo_flb_ser`, one for each preserved global object. + * This entry contains the `struct luo_ser` structure. * * Serialization Structures: - * The FDT properties point to memory regions containing arrays of simple, - * `__packed` structures. These structures contain the actual preserved state. - * - * - struct luo_session_header_ser: - * Header for the session array. Contains the total page count of the - * preserved memory block and the number of `struct luo_session_ser` - * entries that follow. + * - struct luo_ser: + * The central ABI structure that contains the overall state of the LUO. + * It includes the compatibility string, the liveupdate-number, and pointers + * to sessions and FLBs. * * - struct luo_session_ser: * Metadata for a single session, including its name and a physical pointer - * to another preserved memory block containing an array of - * `struct luo_file_ser` for all files in that session. + * to the first `struct kho_block_header_ser` for all files in that session. + * Multiple blocks are linked via the `next` field in the header. * * - struct luo_file_ser: * Metadata for a single preserved file. Contains the `compatible` string to @@ -105,17 +57,32 @@ #ifndef _LINUX_KHO_ABI_LUO_H #define _LINUX_KHO_ABI_LUO_H +#include <linux/align.h> +#include <linux/kho/abi/block.h> #include <uapi/linux/liveupdate.h> /* - * The LUO FDT hooks all LUO state for sessions, fds, etc. - * In the root it also carries "liveupdate-number" 64-bit property that - * corresponds to the number of live-updates performed on this machine. + * The LUO state is registered under this KHO entry name. */ -#define LUO_FDT_SIZE PAGE_SIZE -#define LUO_FDT_KHO_ENTRY_NAME "LUO" -#define LUO_FDT_COMPATIBLE "luo-v1" -#define LUO_FDT_LIVEUPDATE_NUM "liveupdate-number" +#define LUO_KHO_ENTRY_NAME "LUO" +#define LUO_ABI_COMPATIBLE "luo-v5" +#define LUO_ABI_COMPAT_LEN ALIGN(sizeof(LUO_ABI_COMPATIBLE), 8) + +/** + * struct luo_ser - Centralized LUO ABI header. + * @compatible: Compatibility string identifying the LUO ABI version. + * @liveupdate_num: A counter tracking the number of successful live updates. + * @sessions_pa: Physical address of the first session block header. + * @flbs_pa: Physical address of the FLB header. + * + * This structure is the root of all preserved LUO state. + */ +struct luo_ser { + char compatible[LUO_ABI_COMPAT_LEN]; + u64 liveupdate_num; + u64 sessions_pa; + u64 flbs_pa; +} __packed; #define LIVEUPDATE_HNDL_COMPAT_LENGTH 48 @@ -125,7 +92,7 @@ * @data: Private data * @token: User provided token for this file * - * If this structure is modified, LUO_SESSION_COMPATIBLE must be updated. + * If this structure is modified, `LUO_ABI_COMPATIBLE` must be updated. */ struct luo_file_ser { char compatible[LIVEUPDATE_HNDL_COMPAT_LENGTH]; @@ -135,9 +102,10 @@ struct luo_file_ser { /** * struct luo_file_set_ser - Represents the serialized metadata for file set - * @files: The physical address of a contiguous memory block that holds - * the serialized state of files (array of luo_file_ser) in this file - * set. + * @files: The physical address of the first `struct kho_block_header_ser`. + * This structure is the header for a block of memory containing + * an array of `struct luo_file_ser` entries. Multiple blocks are + * linked via the `next` field in the header. * @count: The total number of files that were part of this session during * serialization. Used for iteration and validation during * restoration. @@ -147,30 +115,6 @@ struct luo_file_set_ser { u64 count; } __packed; -/* - * LUO FDT session node - * LUO_FDT_SESSION_HEADER: is a u64 physical address of struct - * luo_session_header_ser - */ -#define LUO_FDT_SESSION_NODE_NAME "luo-session" -#define LUO_FDT_SESSION_COMPATIBLE "luo-session-v2" -#define LUO_FDT_SESSION_HEADER "luo-session-header" - -/** - * struct luo_session_header_ser - Header for the serialized session data block. - * @count: The number of `struct luo_session_ser` entries that immediately - * follow this header in the memory block. - * - * This structure is located at the beginning of a contiguous block of - * physical memory preserved across the kexec. It provides the necessary - * metadata to interpret the array of session entries that follow. - * - * If this structure is modified, `LUO_FDT_SESSION_COMPATIBLE` must be updated. - */ -struct luo_session_header_ser { - u64 count; -} __packed; - /** * struct luo_session_ser - Represents the serialized metadata for a LUO session. * @name: The unique name of the session, provided by the userspace at @@ -182,7 +126,7 @@ struct luo_session_header_ser { * session) is created and passed to the new kernel, allowing it to reconstruct * the session context. * - * If this structure is modified, `LUO_FDT_SESSION_COMPATIBLE` must be updated. + * If this structure is modified, `LUO_ABI_COMPATIBLE` must be updated. */ struct luo_session_ser { char name[LIVEUPDATE_SESSION_NAME_LENGTH]; @@ -192,10 +136,6 @@ struct luo_session_ser { /* The max size is set so it can be reliably used during in serialization */ #define LIVEUPDATE_FLB_COMPAT_LENGTH 48 -#define LUO_FDT_FLB_NODE_NAME "luo-flb" -#define LUO_FDT_FLB_COMPATIBLE "luo-flb-v1" -#define LUO_FDT_FLB_HEADER "luo-flb-header" - /** * struct luo_flb_header_ser - Header for the serialized FLB data block. * @pgcnt: The total number of pages occupied by the entire preserved memory @@ -205,11 +145,9 @@ struct luo_session_ser { * in the memory block. * * This structure is located at the physical address specified by the - * `LUO_FDT_FLB_HEADER` FDT property. It provides the new kernel with the - * necessary information to find and iterate over the array of preserved - * File-Lifecycle-Bound objects and to manage the underlying memory. + * flbs_pa in luo_ser. * - * If this structure is modified, LUO_FDT_FLB_COMPATIBLE must be updated. + * If this structure is modified, `LUO_ABI_COMPATIBLE` must be updated. */ struct luo_flb_header_ser { u64 pgcnt; @@ -231,7 +169,7 @@ struct luo_flb_header_ser { * passed to the new kernel. Each entry allows the LUO core to restore one * global, shared object. * - * If this structure is modified, LUO_FDT_FLB_COMPATIBLE must be updated. + * If this structure is modified, `LUO_ABI_COMPATIBLE` must be updated. */ struct luo_flb_ser { char name[LIVEUPDATE_FLB_COMPAT_LENGTH]; diff --git a/include/linux/kho/abi/memfd.h b/include/linux/kho/abi/memfd.h index 68cb6303b846..08b10fea2afc 100644 --- a/include/linux/kho/abi/memfd.h +++ b/include/linux/kho/abi/memfd.h @@ -56,10 +56,24 @@ struct memfd_luo_folio_ser { u64 index; } __packed; +/* + * The set of seals this version supports preserving. If support for any new + * seals is needed, add it here and bump version. + */ +#define MEMFD_LUO_ALL_SEALS (F_SEAL_SEAL | \ + F_SEAL_SHRINK | \ + F_SEAL_GROW | \ + F_SEAL_WRITE | \ + F_SEAL_FUTURE_WRITE | \ + F_SEAL_EXEC) + /** * struct memfd_luo_ser - Main serialization structure for a memfd. * @pos: The file's current position (f_pos). * @size: The total size of the file in bytes (i_size). + * @seals: The seals present on the memfd. The seals are uABI so it is safe + * to directly use them in the ABI. + * @flags: Flags for the file. Unused flag bits must be set to 0. * @nr_folios: Number of folios in the folios array. * @folios: KHO vmalloc descriptor pointing to the array of * struct memfd_luo_folio_ser. @@ -67,11 +81,13 @@ struct memfd_luo_folio_ser { struct memfd_luo_ser { u64 pos; u64 size; + u32 seals; + u32 flags; u64 nr_folios; struct kho_vmalloc folios; } __packed; /* The compatibility string for memfd file handler */ -#define MEMFD_LUO_FH_COMPATIBLE "memfd-v1" +#define MEMFD_LUO_FH_COMPATIBLE "memfd-v2" #endif /* _LINUX_KHO_ABI_MEMFD_H */ diff --git a/include/linux/kho_block.h b/include/linux/kho_block.h new file mode 100644 index 000000000000..93a7cc2be5f5 --- /dev/null +++ b/include/linux/kho_block.h @@ -0,0 +1,106 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2026, Google LLC. + * Pasha Tatashin <pasha.tatashin@soleen.com> + */ + +#ifndef _LINUX_KHO_BLOCK_H +#define _LINUX_KHO_BLOCK_H + +#include <linux/list.h> +#include <linux/types.h> +#include <linux/kho/abi/block.h> + +/** + * struct kho_block - Internal representation of a serialization block. + * @list: List head for linking blocks in memory. + * @ser: Pointer to the serialized header in preserved memory. + */ +struct kho_block { + struct list_head list; + struct kho_block_header_ser *ser; +}; + +/** + * struct kho_block_set - A set of blocks containing serialized entries of the same type. + * @blocks: The list of serialization blocks (struct kho_block). + * @nblocks: The number of allocated serialization blocks. + * @head_pa: Physical address of the first block header. + * @entry_size: The size of each entry in the blocks. + * @count_per_block: The maximum number of entries each block can hold. + * @incoming: True if this block set was restored from the previous kernel. + * + * Note: Synchronization and locking are the responsibility of the caller. + * The block set structure itself is not internally synchronized. + */ +struct kho_block_set { + struct list_head blocks; + long nblocks; + u64 head_pa; + size_t entry_size; + u64 count_per_block; + bool incoming; +}; + +/** + * struct kho_block_set_it - Iterator for serializing entries into blocks. + * @bs: The block set being iterated. + * @block: The current block. + * @i: The current entry index within @block. + */ +struct kho_block_set_it { + struct kho_block_set *bs; + struct kho_block *block; + u64 i; +}; + +/** + * KHO_BLOCK_SET_INIT - Initialize a static kho_block_set. + * @_name: Name of the kho_block_set variable. + * @_entry_size: The size of each entry in the block set. + */ +#define KHO_BLOCK_SET_INIT(_name, _entry_size) { \ + .blocks = LIST_HEAD_INIT((_name).blocks), \ + .entry_size = _entry_size, \ + .count_per_block = (KHO_BLOCK_SIZE - \ + sizeof(struct kho_block_header_ser)) / \ + (_entry_size), \ +} + +void kho_block_set_init(struct kho_block_set *bs, size_t entry_size); + +int kho_block_set_grow(struct kho_block_set *bs, u64 count); +void kho_block_set_shrink(struct kho_block_set *bs, u64 count); + +int kho_block_set_restore(struct kho_block_set *bs, u64 head_pa); +void kho_block_set_destroy(struct kho_block_set *bs); +void kho_block_set_clear(struct kho_block_set *bs); + +/** + * kho_block_set_head_pa - Get the physical address of the first block header. + * @bs: The block set. + * + * Return: The physical address of the first block header, or 0 if empty. + */ +static inline u64 kho_block_set_head_pa(struct kho_block_set *bs) +{ + return bs->head_pa; +} + +/** + * kho_block_set_is_empty - Check if the block set has no allocated blocks. + * @bs: The block set. + * + * Return: True if there are no blocks in the set, false otherwise. + */ +static inline bool kho_block_set_is_empty(struct kho_block_set *bs) +{ + return list_empty(&bs->blocks); +} + +void kho_block_set_it_init(struct kho_block_set_it *it, struct kho_block_set *bs); +void *kho_block_set_it_reserve_entry(struct kho_block_set_it *it); +void *kho_block_set_it_read_entry(struct kho_block_set_it *it); +void *kho_block_set_it_prev(struct kho_block_set_it *it); + +#endif /* _LINUX_KHO_BLOCK_H */ diff --git a/include/linux/kho_radix_tree.h b/include/linux/kho_radix_tree.h new file mode 100644 index 000000000000..fcd69639721b --- /dev/null +++ b/include/linux/kho_radix_tree.h @@ -0,0 +1,90 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef _LINUX_KHO_RADIX_TREE_H +#define _LINUX_KHO_RADIX_TREE_H + +#include <linux/err.h> +#include <linux/errno.h> +#include <linux/mutex_types.h> +#include <linux/types.h> + +/** + * DOC: Kexec Handover Radix Tree + * + * This is a radix tree implementation for tracking numeric keys across kexec + * transitions. It was developed for the KHO preserved memory map but is + * designed for broader use by any subsystem that needs to track keys. + * Conceptually speaking, the data structure is similar to a set. It tracks the + * presence or absence of numeric keys. + * + * The radix tree is a multi-level tree where leaf nodes are bitmaps + * representing individual keys. + * + * For the KHO preserved memory map, to allow pages of different sizes (orders) + * to be stored efficiently in a single tree, it uses a unique key encoding + * scheme. Each key is an unsigned long that combines a page's physical address + * and its order. + * + * Client code is responsible for allocating the root node of the tree, + * initializing the mutex lock, and managing its lifecycle. It must use the + * tree data structures defined in the KHO ABI, + * `include/linux/kho/abi/kexec_handover.h`. + */ + +struct kho_radix_node; + +struct kho_radix_tree { + struct kho_radix_node *root; + struct mutex lock; /* protects the tree's structure and root pointer */ +}; + +/** + * struct kho_radix_walk_cb - Callbacks for KHO radix tree walk. + * @leaf: Called on each present key in the radix tree. + * @node: Called on each node of the radix tree itself. Receives the + * physical address of the page containing the node. + * + * For each callback, a return value of 0 continues the walk and a non-zero + * return value is directly returned to the caller. + */ +struct kho_radix_walk_cb { + int (*leaf)(unsigned long key, void *data); + int (*node)(phys_addr_t phys, void *data); +}; + +#ifdef CONFIG_KEXEC_HANDOVER + +int kho_radix_add_key(struct kho_radix_tree *tree, unsigned long key); +void kho_radix_del_key(struct kho_radix_tree *tree, unsigned long key); +int kho_radix_walk_tree(struct kho_radix_tree *tree, + const struct kho_radix_walk_cb *cb, void *data); +int kho_radix_init_tree(struct kho_radix_tree *tree, struct kho_radix_node *root); +void kho_radix_destroy_tree(struct kho_radix_tree *tree); + +#else /* #ifdef CONFIG_KEXEC_HANDOVER */ + +static inline int kho_radix_add_key(struct kho_radix_tree *tree, unsigned long key) +{ + return -EOPNOTSUPP; +} + +static inline void kho_radix_del_key(struct kho_radix_tree *tree, + unsigned long key) { } + +static inline int kho_radix_walk_tree(struct kho_radix_tree *tree, + const struct kho_radix_walk_cb *cb, void *data) +{ + return -EOPNOTSUPP; +} + +static inline int kho_radix_init_tree(struct kho_radix_tree *tree, + struct kho_radix_node *root) +{ + return 0; +} + +static inline void kho_radix_destroy_tree(struct kho_radix_tree *tree) { } + +#endif /* #ifdef CONFIG_KEXEC_HANDOVER */ + +#endif /* _LINUX_KHO_RADIX_TREE_H */ diff --git a/include/linux/kobject.h b/include/linux/kobject.h index c8219505a79f..55e37a5d405e 100644 --- a/include/linux/kobject.h +++ b/include/linux/kobject.h @@ -109,7 +109,7 @@ struct kobject *kobject_get(struct kobject *kobj); struct kobject * __must_check kobject_get_unless_zero(struct kobject *kobj); void kobject_put(struct kobject *kobj); -const void *kobject_namespace(const struct kobject *kobj); +const struct ns_common *kobject_namespace(const struct kobject *kobj); void kobject_get_ownership(const struct kobject *kobj, kuid_t *uid, kgid_t *gid); char *kobject_get_path(const struct kobject *kobj, gfp_t flag); @@ -118,7 +118,7 @@ struct kobj_type { const struct sysfs_ops *sysfs_ops; const struct attribute_group **default_groups; const struct kobj_ns_type_operations *(*child_ns_type)(const struct kobject *kobj); - const void *(*namespace)(const struct kobject *kobj); + const struct ns_common *(*namespace)(const struct kobject *kobj); void (*get_ownership)(const struct kobject *kobj, kuid_t *uid, kgid_t *gid); }; @@ -138,12 +138,79 @@ struct kset_uevent_ops { struct kobj_attribute { struct attribute attr; - ssize_t (*show)(struct kobject *kobj, struct kobj_attribute *attr, - char *buf); - ssize_t (*store)(struct kobject *kobj, struct kobj_attribute *attr, - const char *buf, size_t count); + __SYSFS_FUNCTION_ALTERNATIVE( + ssize_t (*show)(struct kobject *kobj, struct kobj_attribute *attr, char *buf); + ssize_t (*show_const)(struct kobject *kobj, const struct kobj_attribute *attr, + char *buf); + ); + __SYSFS_FUNCTION_ALTERNATIVE( + ssize_t (*store)(struct kobject *kobj, struct kobj_attribute *attr, + const char *buf, size_t count); + ssize_t (*store_const)(struct kobject *kobj, const struct kobj_attribute *attr, + const char *buf, size_t count); + ); }; +typedef ssize_t __kobj_show_handler_const(struct kobject *kobj, const struct kobj_attribute *attr, + char *buf); +typedef ssize_t __kobj_store_handler_const(struct kobject *kobj, const struct kobj_attribute *attr, + const char *buf, size_t count); + +#ifdef CONFIG_CFI + +#define __KOBJ_ATTR_SHOW_STORE(_show, _store) \ + .show = _Generic(_show, \ + __kobj_show_handler_const * : NULL, \ + default : _show \ + ), \ + .show_const = _Generic(_show, \ + __kobj_show_handler_const * : _show, \ + default : NULL \ + ), \ + .store = _Generic(_store, \ + __kobj_store_handler_const * : NULL, \ + default : _store \ + ), \ + .store_const = _Generic(_store, \ + __kobj_store_handler_const * : _store, \ + default : NULL \ + ), + +#else + +#define __KOBJ_ATTR_SHOW_STORE(_show, _store) \ + .show = _Generic(_show, \ + __kobj_show_handler_const * : (void *)_show, \ + default : _show \ + ), \ + .store = _Generic(_store, \ + __kobj_store_handler_const * : (void *)_store, \ + default : _store \ + ), \ + +#endif + +#define __KOBJ_ATTR(_name, _mode, _show, _store) { \ + .attr = { .name = __stringify(_name), \ + .mode = VERIFY_OCTAL_PERMISSIONS(_mode) }, \ + __KOBJ_ATTR_SHOW_STORE(_show, _store) \ +} + +#define __KOBJ_ATTR_RO_MODE(_name, _mode) \ + __KOBJ_ATTR(_name, _mode, _name##_show, NULL) + +#define __KOBJ_ATTR_RO(_name) \ + __KOBJ_ATTR_RO_MODE(_name, 0444) + +#define __KOBJ_ATTR_RW_MODE(_name, _mode) \ + __KOBJ_ATTR(_name, _mode, _name##_show, _name##_store) + +#define __KOBJ_ATTR_WO(_name) \ + __KOBJ_ATTR(_name, 0200, NULL, _name##_store) + +#define __KOBJ_ATTR_RW(_name) \ + __KOBJ_ATTR(_name, 0644, _name##_show, _name##_store) + extern const struct sysfs_ops kobj_sysfs_ops; struct sock; diff --git a/include/linux/kobject_ns.h b/include/linux/kobject_ns.h index 150fe2ae1b6b..4f0990e09b93 100644 --- a/include/linux/kobject_ns.h +++ b/include/linux/kobject_ns.h @@ -16,6 +16,7 @@ #ifndef _LINUX_KOBJECT_NS_H #define _LINUX_KOBJECT_NS_H +struct ns_common; struct sock; struct kobject; @@ -39,10 +40,10 @@ enum kobj_ns_type { struct kobj_ns_type_operations { enum kobj_ns_type type; bool (*current_may_mount)(void); - void *(*grab_current_ns)(void); - const void *(*netlink_ns)(struct sock *sk); - const void *(*initial_ns)(void); - void (*drop_ns)(void *); + struct ns_common *(*grab_current_ns)(void); + const struct ns_common *(*netlink_ns)(struct sock *sk); + const struct ns_common *(*initial_ns)(void); + void (*drop_ns)(struct ns_common *); }; int kobj_ns_type_register(const struct kobj_ns_type_operations *ops); @@ -51,7 +52,7 @@ const struct kobj_ns_type_operations *kobj_child_ns_ops(const struct kobject *pa const struct kobj_ns_type_operations *kobj_ns_ops(const struct kobject *kobj); bool kobj_ns_current_may_mount(enum kobj_ns_type type); -void *kobj_ns_grab_current(enum kobj_ns_type type); -void kobj_ns_drop(enum kobj_ns_type type, void *ns); +struct ns_common *kobj_ns_grab_current(enum kobj_ns_type type); +void kobj_ns_drop(enum kobj_ns_type type, struct ns_common *ns); #endif /* _LINUX_KOBJECT_NS_H */ diff --git a/include/linux/kprobes.h b/include/linux/kprobes.h index 8c4f3bb24429..e6de7ae55bda 100644 --- a/include/linux/kprobes.h +++ b/include/linux/kprobes.h @@ -181,6 +181,7 @@ struct kprobe_blacklist_entry { struct list_head list; unsigned long start_addr; unsigned long end_addr; + struct rcu_head rcu; }; #ifdef CONFIG_KPROBES diff --git a/include/linux/ks8851_mll.h b/include/linux/ks8851_mll.h deleted file mode 100644 index 57c0a39ed796..000000000000 --- a/include/linux/ks8851_mll.h +++ /dev/null @@ -1,21 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * ks8861_mll platform data struct definition - * Copyright (c) 2012 BTicino S.p.A. - */ - -#ifndef _LINUX_KS8851_MLL_H -#define _LINUX_KS8851_MLL_H - -#include <linux/if_ether.h> - -/** - * struct ks8851_mll_platform_data - Platform data of the KS8851_MLL network driver - * @macaddr: The MAC address of the device, set to all 0:s to use the on in - * the chip. - */ -struct ks8851_mll_platform_data { - u8 mac_addr[ETH_ALEN]; -}; - -#endif diff --git a/include/linux/ksm.h b/include/linux/ksm.h index c982694c987b..d39d0d5483a2 100644 --- a/include/linux/ksm.h +++ b/include/linux/ksm.h @@ -17,8 +17,8 @@ #ifdef CONFIG_KSM int ksm_madvise(struct vm_area_struct *vma, unsigned long start, unsigned long end, int advice, vm_flags_t *vm_flags); -vm_flags_t ksm_vma_flags(struct mm_struct *mm, const struct file *file, - vm_flags_t vm_flags); +vma_flags_t ksm_vma_flags(struct mm_struct *mm, const struct file *file, + vma_flags_t vma_flags); int ksm_enable_merge_any(struct mm_struct *mm); int ksm_disable_merge_any(struct mm_struct *mm); int ksm_disable(struct mm_struct *mm); @@ -103,10 +103,10 @@ bool ksm_process_mergeable(struct mm_struct *mm); #else /* !CONFIG_KSM */ -static inline vm_flags_t ksm_vma_flags(struct mm_struct *mm, - const struct file *file, vm_flags_t vm_flags) +static inline vma_flags_t ksm_vma_flags(struct mm_struct *mm, + const struct file *file, vma_flags_t vma_flags) { - return vm_flags; + return vma_flags; } static inline int ksm_disable(struct mm_struct *mm) diff --git a/include/linux/kstrtox.h b/include/linux/kstrtox.h index 6ea897222af1..b41b9736886f 100644 --- a/include/linux/kstrtox.h +++ b/include/linux/kstrtox.h @@ -97,6 +97,9 @@ int __must_check kstrtou8(const char *s, unsigned int base, u8 *res); int __must_check kstrtos8(const char *s, unsigned int base, s8 *res); int __must_check kstrtobool(const char *s, bool *res); +int __must_check kstrtoudec64(const char *s, unsigned int scale, u64 *res); +int __must_check kstrtodec64(const char *s, unsigned int scale, s64 *res); + int __must_check kstrtoull_from_user(const char __user *s, size_t count, unsigned int base, unsigned long long *res); int __must_check kstrtoll_from_user(const char __user *s, size_t count, unsigned int base, long long *res); int __must_check kstrtoul_from_user(const char __user *s, size_t count, unsigned int base, unsigned long *res); @@ -142,10 +145,9 @@ static inline int __must_check kstrtos32_from_user(const char __user *s, size_t * Keep in mind above caveat. */ -extern unsigned long simple_strtoul(const char *,char **,unsigned int); -extern unsigned long simple_strntoul(const char *,char **,unsigned int,size_t); -extern long simple_strtol(const char *,char **,unsigned int); -extern unsigned long long simple_strtoull(const char *,char **,unsigned int); -extern long long simple_strtoll(const char *,char **,unsigned int); +unsigned long simple_strtoul(const char *cp, char **endp, unsigned int base); +long simple_strtol(const char *cp, char **endp, unsigned int base); +unsigned long long simple_strtoull(const char *cp, char **endp, unsigned int base); +long long simple_strtoll(const char *cp, char **endp, unsigned int base); #endif /* _LINUX_KSTRTOX_H */ diff --git a/include/linux/ksysfs.h b/include/linux/ksysfs.h new file mode 100644 index 000000000000..c7dc6e18f28e --- /dev/null +++ b/include/linux/ksysfs.h @@ -0,0 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef _KSYSFS_H_ +#define _KSYSFS_H_ + +void ksysfs_init(void); + +#endif /* _KSYSFS_H_ */ diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index 6b76e7a6f4c2..03bfc92864b6 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -318,7 +318,8 @@ static inline bool kvm_vcpu_can_poll(ktime_t cur, ktime_t stop) struct kvm_mmio_fragment { gpa_t gpa; void *data; - unsigned len; + u64 val; + unsigned int len; }; struct kvm_vcpu { @@ -790,6 +791,7 @@ struct kvm { /* The current active memslot set for each address space */ struct kvm_memslots __rcu *memslots[KVM_MAX_NR_ADDRESS_SPACES]; struct xarray vcpu_array; + DECLARE_BITMAP(vcpu_ids, KVM_MAX_VCPU_IDS); /* * Protected by slots_lock, but can be read outside if an * incorrect answer is acceptable. @@ -988,6 +990,13 @@ static inline struct kvm_io_bus *kvm_get_bus(struct kvm *kvm, enum kvm_bus idx) lockdep_is_held(&kvm->slots_lock)); } +static inline void kvm_lockdep_assert_vcpu_is_locked_or_unreachable(struct kvm_vcpu *vcpu) +{ + lockdep_assert_once(lockdep_is_held(&vcpu->mutex) || + vcpu->vcpu_idx < 0 || + !refcount_read(&vcpu->kvm->users_count)); +} + static inline struct kvm_vcpu *kvm_get_vcpu(struct kvm *kvm, int i) { int num_vcpus = atomic_read(&kvm->online_vcpus); @@ -1029,6 +1038,13 @@ static inline struct kvm_vcpu *kvm_get_vcpu_by_id(struct kvm *kvm, int id) return NULL; } +static inline bool kvm_is_vcpu_creation_in_progress(struct kvm *kvm) +{ + lockdep_assert_held(&kvm->lock); + + return kvm->created_vcpus != atomic_read(&kvm->online_vcpus); +} + void kvm_destroy_vcpus(struct kvm *kvm); int kvm_trylock_all_vcpus(struct kvm *kvm); @@ -1382,20 +1398,20 @@ void mark_page_dirty_in_slot(struct kvm *kvm, const struct kvm_memory_slot *mems void mark_page_dirty(struct kvm *kvm, gfn_t gfn); void kvm_vcpu_mark_page_dirty(struct kvm_vcpu *vcpu, gfn_t gfn); -int __kvm_vcpu_map(struct kvm_vcpu *vcpu, gpa_t gpa, struct kvm_host_map *map, +int __kvm_vcpu_map(struct kvm_vcpu *vcpu, gfn_t gfn, struct kvm_host_map *map, bool writable); void kvm_vcpu_unmap(struct kvm_vcpu *vcpu, struct kvm_host_map *map); -static inline int kvm_vcpu_map(struct kvm_vcpu *vcpu, gpa_t gpa, +static inline int kvm_vcpu_map(struct kvm_vcpu *vcpu, gfn_t gfn, struct kvm_host_map *map) { - return __kvm_vcpu_map(vcpu, gpa, map, true); + return __kvm_vcpu_map(vcpu, gfn, map, true); } -static inline int kvm_vcpu_map_readonly(struct kvm_vcpu *vcpu, gpa_t gpa, +static inline int kvm_vcpu_map_readonly(struct kvm_vcpu *vcpu, gfn_t gfn, struct kvm_host_map *map) { - return __kvm_vcpu_map(vcpu, gpa, map, false); + return __kvm_vcpu_map(vcpu, gfn, map, false); } static inline void kvm_vcpu_map_mark_dirty(struct kvm_vcpu *vcpu, @@ -1405,6 +1421,25 @@ static inline void kvm_vcpu_map_mark_dirty(struct kvm_vcpu *vcpu, kvm_vcpu_mark_page_dirty(vcpu, map->gfn); } +typedef struct { + struct kvm_vcpu *vcpu; + struct kvm_host_map map; + int ret; +} kvm_vcpu_local_map_t; + +#define DEFINE_VCPU_MAP_CLASS(ro) \ +DEFINE_CLASS(kvm_vcpu_map_local##ro, kvm_vcpu_local_map_t, \ + if (!_T.ret) kvm_vcpu_unmap(_T.vcpu, &_T.map), \ + ({ \ + kvm_vcpu_local_map_t m = { .vcpu = vcpu }; \ + \ + m.ret = kvm_vcpu_map##ro(vcpu, gfn, &m.map); \ + \ + m; \ + }), struct kvm_vcpu *vcpu, gfn_t gfn); +DEFINE_VCPU_MAP_CLASS(); +DEFINE_VCPU_MAP_CLASS(_readonly); + unsigned long kvm_vcpu_gfn_to_hva(struct kvm_vcpu *vcpu, gfn_t gfn); unsigned long kvm_vcpu_gfn_to_hva_prot(struct kvm_vcpu *vcpu, gfn_t gfn, bool *writable); int kvm_vcpu_read_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn, void *data, int offset, @@ -1554,7 +1589,7 @@ void kvm_mmu_free_memory_cache(struct kvm_mmu_memory_cache *mc); void *kvm_mmu_memory_cache_alloc(struct kvm_mmu_memory_cache *mc); #endif -void kvm_mmu_invalidate_begin(struct kvm *kvm); +void kvm_mmu_invalidate_start(struct kvm *kvm); void kvm_mmu_invalidate_range_add(struct kvm *kvm, gfn_t start, gfn_t end); void kvm_mmu_invalidate_end(struct kvm *kvm); bool kvm_mmu_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range); @@ -1628,6 +1663,13 @@ static inline void kvm_create_vcpu_debugfs(struct kvm_vcpu *vcpu) {} #ifdef CONFIG_KVM_GENERIC_HARDWARE_ENABLING /* + * kvm_arch_shutdown() is invoked immediately prior to forcefully disabling + * hardware virtualization on all CPUs via IPI function calls (in preparation + * for shutdown or reboot), e.g. to allow arch code to prepare for disabling + * virtualization while KVM may be actively running vCPUs. + */ +void kvm_arch_shutdown(void); +/* * kvm_arch_{enable,disable}_virtualization() are called on one CPU, under * kvm_usage_lock, immediately after/before 0=>1 and 1=>0 transitions of * kvm_usage_count, i.e. at the beginning of the generic hardware enabling @@ -1800,6 +1842,11 @@ void kvm_unregister_irq_ack_notifier(struct kvm *kvm, struct kvm_irq_ack_notifier *kian); bool kvm_arch_irqfd_allowed(struct kvm *kvm, struct kvm_irqfd *args); +static inline bool is_gfn_in_memslot(const struct kvm_memory_slot *slot, gfn_t gfn) +{ + return gfn >= slot->base_gfn && gfn < slot->base_gfn + slot->npages; +} + /* * Returns a pointer to the memslot if it contains gfn. * Otherwise returns NULL. @@ -1810,7 +1857,7 @@ try_get_memslot(struct kvm_memory_slot *slot, gfn_t gfn) if (!slot) return NULL; - if (gfn >= slot->base_gfn && gfn < slot->base_gfn + slot->npages) + if (is_gfn_in_memslot(slot, gfn)) return slot; else return NULL; @@ -2300,7 +2347,6 @@ static inline bool kvm_check_request(int req, struct kvm_vcpu *vcpu) #ifdef CONFIG_KVM_GENERIC_HARDWARE_ENABLING extern bool enable_virt_at_load; -extern bool kvm_rebooting; #endif extern unsigned int halt_poll_ns; @@ -2366,6 +2412,7 @@ void kvm_unregister_device_ops(u32 type); extern struct kvm_device_ops kvm_mpic_ops; extern struct kvm_device_ops kvm_arm_vgic_v2_ops; extern struct kvm_device_ops kvm_arm_vgic_v3_ops; +extern struct kvm_device_ops kvm_arm_vgic_v5_ops; #ifdef CONFIG_HAVE_KVM_CPU_RELAX_INTERCEPT @@ -2552,8 +2599,9 @@ static inline int kvm_gmem_get_pfn(struct kvm *kvm, } #endif /* CONFIG_KVM_GUEST_MEMFD */ -#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_PREPARE -int kvm_arch_gmem_prepare(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, int max_order); +#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT +int kvm_arch_gmem_make_private(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, + kvm_pfn_t nr_pages); #endif #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_POPULATE @@ -2581,12 +2629,17 @@ int kvm_arch_gmem_prepare(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, int max_ord typedef int (*kvm_gmem_populate_cb)(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, struct page *page, void *opaque); -long kvm_gmem_populate(struct kvm *kvm, gfn_t gfn, void __user *src, long npages, +long kvm_gmem_populate(struct kvm *kvm, gfn_t start_gfn, void __user *src, + long npages, bool may_writeback_src, kvm_gmem_populate_cb post_populate, void *opaque); #endif +#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_RECLAIM +void kvm_arch_gmem_reclaim(kvm_pfn_t pfn, kvm_pfn_t nr_pages); +#endif + #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_INVALIDATE -void kvm_arch_gmem_invalidate(kvm_pfn_t start, kvm_pfn_t end); +void kvm_arch_gmem_invalidate_range(struct kvm *kvm, struct kvm_gfn_range *range); #endif #ifdef CONFIG_KVM_GENERIC_PRE_FAULT_MEMORY @@ -2594,12 +2647,4 @@ long kvm_arch_vcpu_pre_fault_memory(struct kvm_vcpu *vcpu, struct kvm_pre_fault_memory *range); #endif -#ifdef CONFIG_KVM_GENERIC_HARDWARE_ENABLING -int kvm_enable_virtualization(void); -void kvm_disable_virtualization(void); -#else -static inline int kvm_enable_virtualization(void) { return 0; } -static inline void kvm_disable_virtualization(void) { } -#endif - #endif diff --git a/include/linux/landlock.h b/include/linux/landlock.h new file mode 100644 index 000000000000..004cbd0b9298 --- /dev/null +++ b/include/linux/landlock.h @@ -0,0 +1,56 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Landlock - Public types and definitions + * + * Copyright © 2016-2026 Mickaël Salaün <mic@digikod.net> + * Copyright © 2026 Cloudflare, Inc. + */ + +#ifndef _LINUX_LANDLOCK_H +#define _LINUX_LANDLOCK_H + +#include <linux/types.h> +#include <uapi/linux/landlock.h> + +/* + * Access-right and scope names, shared between the audit records (get_blocker() + * in security/landlock/audit.c) and the trace events + * (include/trace/events/landlock.h). A consumer defines + * _LANDLOCK_NAME_ENTRY(mask, name) before expanding a list and undefines it + * afterwards: audit maps each entry to a "[bit] = name" slot for O(1) lookup, + * the trace events map it to a __print_flags() { mask, name } pair. The bit + * value lives only in the LANDLOCK_* UAPI constant each entry references. + * Names are unprefixed; audit prepends the "fs."/"net."/"scope." category. + */ +#define _LANDLOCK_ACCESS_FS_NAMES \ + _LANDLOCK_NAME_ENTRY(LANDLOCK_ACCESS_FS_EXECUTE, "execute"), \ + _LANDLOCK_NAME_ENTRY(LANDLOCK_ACCESS_FS_WRITE_FILE, "write_file"), \ + _LANDLOCK_NAME_ENTRY(LANDLOCK_ACCESS_FS_READ_FILE, "read_file"), \ + _LANDLOCK_NAME_ENTRY(LANDLOCK_ACCESS_FS_READ_DIR, "read_dir"), \ + _LANDLOCK_NAME_ENTRY(LANDLOCK_ACCESS_FS_REMOVE_DIR, "remove_dir"), \ + _LANDLOCK_NAME_ENTRY(LANDLOCK_ACCESS_FS_REMOVE_FILE, "remove_file"), \ + _LANDLOCK_NAME_ENTRY(LANDLOCK_ACCESS_FS_MAKE_CHAR, "make_char"), \ + _LANDLOCK_NAME_ENTRY(LANDLOCK_ACCESS_FS_MAKE_DIR, "make_dir"), \ + _LANDLOCK_NAME_ENTRY(LANDLOCK_ACCESS_FS_MAKE_REG, "make_reg"), \ + _LANDLOCK_NAME_ENTRY(LANDLOCK_ACCESS_FS_MAKE_SOCK, "make_sock"), \ + _LANDLOCK_NAME_ENTRY(LANDLOCK_ACCESS_FS_MAKE_FIFO, "make_fifo"), \ + _LANDLOCK_NAME_ENTRY(LANDLOCK_ACCESS_FS_MAKE_BLOCK, "make_block"), \ + _LANDLOCK_NAME_ENTRY(LANDLOCK_ACCESS_FS_MAKE_SYM, "make_sym"), \ + _LANDLOCK_NAME_ENTRY(LANDLOCK_ACCESS_FS_REFER, "refer"), \ + _LANDLOCK_NAME_ENTRY(LANDLOCK_ACCESS_FS_TRUNCATE, "truncate"), \ + _LANDLOCK_NAME_ENTRY(LANDLOCK_ACCESS_FS_IOCTL_DEV, "ioctl_dev"), \ + _LANDLOCK_NAME_ENTRY(LANDLOCK_ACCESS_FS_RESOLVE_UNIX, "resolve_unix") + +#define _LANDLOCK_ACCESS_NET_NAMES \ + _LANDLOCK_NAME_ENTRY(LANDLOCK_ACCESS_NET_BIND_TCP, "bind_tcp"), \ + _LANDLOCK_NAME_ENTRY(LANDLOCK_ACCESS_NET_CONNECT_TCP, "connect_tcp"), \ + _LANDLOCK_NAME_ENTRY(LANDLOCK_ACCESS_NET_BIND_UDP, "bind_udp"), \ + _LANDLOCK_NAME_ENTRY(LANDLOCK_ACCESS_NET_CONNECT_SEND_UDP, \ + "connect_send_udp") + +#define _LANDLOCK_SCOPE_NAMES \ + _LANDLOCK_NAME_ENTRY(LANDLOCK_SCOPE_ABSTRACT_UNIX_SOCKET, \ + "abstract_unix_socket"), \ + _LANDLOCK_NAME_ENTRY(LANDLOCK_SCOPE_SIGNAL, "signal") + +#endif /* _LINUX_LANDLOCK_H */ diff --git a/include/linux/leafops.h b/include/linux/leafops.h index a9ff94b744f2..4c1476ae3234 100644 --- a/include/linux/leafops.h +++ b/include/linux/leafops.h @@ -81,7 +81,7 @@ static inline pte_t softleaf_to_pte(softleaf_t entry) return swp_entry_to_pte(entry); } -#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifdef CONFIG_ARCH_HAS_PMD_SOFTLEAVES /** * softleaf_from_pmd() - Obtain a leaf entry from a PMD entry. * @pmd: PMD entry. @@ -100,14 +100,29 @@ static inline softleaf_t softleaf_from_pmd(pmd_t pmd) if (pmd_swp_soft_dirty(pmd)) pmd = pmd_swp_clear_soft_dirty(pmd); - if (pmd_swp_uffd_wp(pmd)) - pmd = pmd_swp_clear_uffd_wp(pmd); + if (pmd_swp_uffd(pmd)) + pmd = pmd_swp_clear_uffd(pmd); arch_entry = __pmd_to_swp_entry(pmd); /* Temporary until swp_entry_t eliminated. */ return swp_entry(__swp_type(arch_entry), __swp_offset(arch_entry)); } +/** + * softleaf_to_pmd() - Obtain a PMD entry from a leaf entry. + * @entry: Leaf entry. + * + * This generates an architecture-specific PMD entry that can be utilised to + * encode the metadata the leaf entry encodes. + * + * Returns: Architecture-specific PMD entry encoding leaf entry. + */ +static inline pmd_t softleaf_to_pmd(softleaf_t entry) +{ + /* Temporary until swp_entry_t eliminated. */ + return swp_entry_to_pmd(entry); +} + #else static inline softleaf_t softleaf_from_pmd(pmd_t pmd) @@ -115,6 +130,11 @@ static inline softleaf_t softleaf_from_pmd(pmd_t pmd) return softleaf_mk_none(); } +static inline pmd_t softleaf_to_pmd(softleaf_t entry) +{ + return __pmd(0); +} + #endif /** @@ -363,6 +383,23 @@ static inline unsigned long softleaf_to_pfn(softleaf_t entry) return swp_offset(entry) & SWP_PFN_MASK; } +static inline void softleaf_migration_sync(softleaf_t entry, + struct folio *folio) +{ + /* + * Ensure we do not race with split, which might alter tail pages into new + * folios and thus result in observing an unlocked folio. + * This matches the write barrier in __split_folio_to_order(). + */ + smp_rmb(); + + /* + * Any use of migration entries may only occur while the + * corresponding page is locked + */ + VM_WARN_ON_ONCE(!folio_test_locked(folio)); +} + /** * softleaf_to_page() - Obtains struct page for PFN encoded within leaf entry. * @entry: Leaf entry, softleaf_has_pfn(@entry) must return true. @@ -374,11 +411,8 @@ static inline struct page *softleaf_to_page(softleaf_t entry) struct page *page = pfn_to_page(softleaf_to_pfn(entry)); VM_WARN_ON_ONCE(!softleaf_has_pfn(entry)); - /* - * Any use of migration entries may only occur while the - * corresponding page is locked - */ - VM_WARN_ON_ONCE(softleaf_is_migration(entry) && !PageLocked(page)); + if (softleaf_is_migration(entry)) + softleaf_migration_sync(entry, page_folio(page)); return page; } @@ -394,12 +428,8 @@ static inline struct folio *softleaf_to_folio(softleaf_t entry) struct folio *folio = pfn_folio(softleaf_to_pfn(entry)); VM_WARN_ON_ONCE(!softleaf_has_pfn(entry)); - /* - * Any use of migration entries may only occur while the - * corresponding folio is locked. - */ - VM_WARN_ON_ONCE(softleaf_is_migration(entry) && - !folio_test_locked(folio)); + if (softleaf_is_migration(entry)) + softleaf_migration_sync(entry, folio); return folio; } @@ -557,7 +587,7 @@ static inline bool pte_is_uffd_marker(pte_t pte) return false; } -#if defined(CONFIG_ZONE_DEVICE) && defined(CONFIG_ARCH_ENABLE_THP_MIGRATION) +#if defined(CONFIG_ZONE_DEVICE) && defined(CONFIG_ARCH_HAS_PMD_SOFTLEAVES) /** * pmd_is_device_private_entry() - Check if PMD contains a device private swap @@ -576,14 +606,14 @@ static inline bool pmd_is_device_private_entry(pmd_t pmd) return softleaf_is_device_private(softleaf_from_pmd(pmd)); } -#else /* CONFIG_ZONE_DEVICE && CONFIG_ARCH_ENABLE_THP_MIGRATION */ +#else /* CONFIG_ZONE_DEVICE && CONFIG_ARCH_HAS_PMD_SOFTLEAVES */ static inline bool pmd_is_device_private_entry(pmd_t pmd) { return false; } -#endif /* CONFIG_ZONE_DEVICE && CONFIG_ARCH_ENABLE_THP_MIGRATION */ +#endif /* CONFIG_ZONE_DEVICE && CONFIG_ARCH_HAS_PMD_SOFTLEAVES */ /** * pmd_is_migration_entry() - Does this PMD entry encode a migration entry? @@ -597,7 +627,20 @@ static inline bool pmd_is_migration_entry(pmd_t pmd) } /** - * pmd_is_valid_softleaf() - Is this PMD entry a valid leaf entry? + * softleaf_is_valid_pmd_entry() - Is the specified softleaf entry obtained from + * a PMD one that we support at PMD level? + * @entry: Entry to check. + * Returns: true if the softleaf entry is valid at PMD, otherwise false. + */ +static inline bool softleaf_is_valid_pmd_entry(softleaf_t entry) +{ + /* Only device private, migration entries valid for PMD. */ + return softleaf_is_device_private(entry) || + softleaf_is_migration(entry); +} + +/** + * pmd_is_valid_softleaf() - Is this PMD entry a valid softleaf entry? * @pmd: PMD entry. * * PMD leaf entries are valid only if they are device private or migration @@ -610,9 +653,27 @@ static inline bool pmd_is_valid_softleaf(pmd_t pmd) { const softleaf_t entry = softleaf_from_pmd(pmd); - /* Only device private, migration entries valid for PMD. */ - return softleaf_is_device_private(entry) || - softleaf_is_migration(entry); + return softleaf_is_valid_pmd_entry(entry); +} + +/** + * pmd_to_softleaf_folio() - Convert the PMD entry to a folio. + * @pmd: PMD entry. + * + * The PMD entry is expected to be a valid PMD softleaf entry. + * + * Returns: the folio the softleaf entry references if this is a valid softleaf + * entry, otherwise NULL. + */ +static inline struct folio *pmd_to_softleaf_folio(pmd_t pmd) +{ + const softleaf_t entry = softleaf_from_pmd(pmd); + + if (!softleaf_is_valid_pmd_entry(entry)) { + VM_WARN_ON_ONCE(true); + return NULL; + } + return softleaf_to_folio(entry); } #endif /* CONFIG_MMU */ diff --git a/include/linux/led-class-multicolor.h b/include/linux/led-class-multicolor.h index db9f34c6736e..8a05836cae25 100644 --- a/include/linux/led-class-multicolor.h +++ b/include/linux/led-class-multicolor.h @@ -9,10 +9,31 @@ #include <linux/leds.h> #include <dt-bindings/leds/common.h> +/** + * struct mc_subled - Color component description. + * @color_index: Color ID. + * @brightness: Scaled intensity. + * @intensity: Current intensity. + * @max_intensity: Maximum supported intensity value. + * @channel: Channel index. + * + * Describes a color component of a multicolor LED. Many multicolor LEDs + * do not support global brightness control in hardware, so they use + * the brightness field in connection with led_mc_calc_color_components() + * to perform the intensity scaling in software. + * Such drivers should set max_intensity to 0 to signal the multicolor LED core + * that the maximum global brightness of the LED class device should be used for + * limiting incoming intensity values. + * + * Multicolor LEDs that do support global brightness control in hardware + * should instead set max_intensity to the maximum intensity value supported + * by the hardware for a given color component. + */ struct mc_subled { unsigned int color_index; unsigned int brightness; unsigned int intensity; + unsigned int max_intensity; unsigned int channel; }; @@ -53,7 +74,14 @@ int led_classdev_multicolor_register_ext(struct device *parent, */ void led_classdev_multicolor_unregister(struct led_classdev_mc *mcled_cdev); -/* Calculate brightness for the monochrome LED cluster */ +/** + * led_mc_calc_color_components() - Calculates component brightness values of a LED cluster. + * @mcled_cdev - Multicolor LED class device of the LED cluster. + * @brightness - Global brightness of the LED cluster. + * + * Calculates the brightness values for each color component of a monochrome LED cluster, + * see Documentation/leds/leds-class-multicolor.rst for details. + */ int led_mc_calc_color_components(struct led_classdev_mc *mcled_cdev, enum led_brightness brightness); diff --git a/include/linux/leds.h b/include/linux/leds.h index b16b803cc1ac..a515f075c29a 100644 --- a/include/linux/leds.h +++ b/include/linux/leds.h @@ -607,6 +607,10 @@ enum led_trigger_netdev_modes { TRIGGER_NETDEV_LINK_2500, TRIGGER_NETDEV_LINK_5000, TRIGGER_NETDEV_LINK_10000, + TRIGGER_NETDEV_LINK_25000, + TRIGGER_NETDEV_LINK_40000, + TRIGGER_NETDEV_LINK_50000, + TRIGGER_NETDEV_LINK_100000, TRIGGER_NETDEV_HALF_DUPLEX, TRIGGER_NETDEV_FULL_DUPLEX, TRIGGER_NETDEV_TX, @@ -676,8 +680,10 @@ typedef int (*gpio_blink_set_t)(struct gpio_desc *desc, int state, struct gpio_led { const char *name; const char *default_trigger; +#ifdef CONFIG_GPIOLIB_LEGACY unsigned gpio; unsigned active_low : 1; +#endif unsigned retain_state_suspended : 1; unsigned panic_indicator : 1; unsigned default_state : 2; diff --git a/include/linux/libata.h b/include/linux/libata.h index 00346ce3af5e..313e96173b19 100644 --- a/include/linux/libata.h +++ b/include/linux/libata.h @@ -121,6 +121,55 @@ enum { ATA_QUIRK_NO_FUA = BIT_ULL(__ATA_QUIRK_NO_FUA), }; +/* + * struct ata_device flags + */ +enum { + ATA_DFLAG_LBA = (1UL << 0), /* device supports LBA */ + ATA_DFLAG_LBA48 = (1UL << 1), /* device supports LBA48 */ + ATA_DFLAG_CDB_INTR = (1UL << 2), /* device asserts INTRQ when ready for CDB */ + ATA_DFLAG_NCQ = (1UL << 3), /* device supports NCQ */ + ATA_DFLAG_FLUSH_EXT = (1UL << 4), /* do FLUSH_EXT instead of FLUSH */ + ATA_DFLAG_ACPI_PENDING = (1UL << 5), /* ACPI resume action pending */ + ATA_DFLAG_ACPI_FAILED = (1UL << 6), /* ACPI on devcfg has failed */ + ATA_DFLAG_AN = (1UL << 7), /* AN configured */ + ATA_DFLAG_TRUSTED = (1UL << 8), /* device supports trusted send/recv */ + ATA_DFLAG_FUA = (1UL << 9), /* device supports FUA */ + ATA_DFLAG_DMADIR = (1UL << 10), /* device requires DMADIR */ + ATA_DFLAG_NCQ_SEND_RECV = (1UL << 11), /* device supports NCQ SEND and RECV */ + ATA_DFLAG_NCQ_PRIO = (1UL << 12), /* device supports NCQ priority */ + ATA_DFLAG_CDL = (1UL << 13), /* supports cmd duration limits */ + ATA_DFLAG_DEPOP = (1UL << 14), /* supports depopulation capability */ + ATA_DFLAG_DEPOP_RESTORE = (1UL << 15), /* supports depopulation restoration */ + ATA_DFLAG_DEPOP_MODIFY = (1UL << 16), /* supports zoned depopulation */ + ATA_DFLAG_CFG_MASK = (1UL << 17) - 1, + + ATA_DFLAG_PIO = (1UL << 17), /* device limited to PIO mode */ + ATA_DFLAG_NCQ_OFF = (1UL << 18), /* device limited to non-NCQ mode */ + ATA_DFLAG_SLEEPING = (1UL << 19), /* device is sleeping */ + ATA_DFLAG_DUBIOUS_XFER = (1UL << 20), /* data transfer not verified */ + ATA_DFLAG_NO_UNLOAD = (1UL << 21), /* device doesn't support unload */ + ATA_DFLAG_UNLOCK_HPA = (1UL << 22), /* unlock HPA */ + ATA_DFLAG_INIT_MASK = (1UL << 23) - 1, + + ATA_DFLAG_NCQ_PRIO_ENABLED = (1UL << 23), /* Priority cmds sent to dev */ + ATA_DFLAG_CDL_ENABLED = (1UL << 24), /* cmd duration limits is enabled */ + ATA_DFLAG_RESUMING = (1UL << 25), /* Device is resuming */ + ATA_DFLAG_DETACH = (1UL << 26), + ATA_DFLAG_DETACHED = (1UL << 27), + ATA_DFLAG_DA = (1UL << 28), /* device supports Device Attention */ + ATA_DFLAG_DEVSLP = (1UL << 29), /* device supports Device Sleep */ + ATA_DFLAG_ACPI_DISABLED = (1UL << 30), /* ACPI for the device is disabled */ + ATA_DFLAG_D_SENSE = (1UL << 31), /* Descriptor sense requested */ + + ATA_DFLAG_FEATURES_MASK = (ATA_DFLAG_TRUSTED | ATA_DFLAG_DA | \ + ATA_DFLAG_DEVSLP | ATA_DFLAG_NCQ_SEND_RECV | \ + ATA_DFLAG_NCQ_PRIO | ATA_DFLAG_FUA | \ + ATA_DFLAG_CDL | ATA_DFLAG_DEPOP | \ + ATA_DFLAG_DEPOP_RESTORE | + ATA_DFLAG_DEPOP_MODIFY) +}; + enum { /* various global constants */ LIBATA_MAX_PRD = ATA_MAX_PRD / 2, @@ -146,46 +195,7 @@ enum { ATA_TFLAG_FUA = (1 << 5), /* enable FUA */ ATA_TFLAG_POLLING = (1 << 6), /* set nIEN to 1 and use polling */ - /* struct ata_device stuff */ - ATA_DFLAG_LBA = (1 << 0), /* device supports LBA */ - ATA_DFLAG_LBA48 = (1 << 1), /* device supports LBA48 */ - ATA_DFLAG_CDB_INTR = (1 << 2), /* device asserts INTRQ when ready for CDB */ - ATA_DFLAG_NCQ = (1 << 3), /* device supports NCQ */ - ATA_DFLAG_FLUSH_EXT = (1 << 4), /* do FLUSH_EXT instead of FLUSH */ - ATA_DFLAG_ACPI_PENDING = (1 << 5), /* ACPI resume action pending */ - ATA_DFLAG_ACPI_FAILED = (1 << 6), /* ACPI on devcfg has failed */ - ATA_DFLAG_AN = (1 << 7), /* AN configured */ - ATA_DFLAG_TRUSTED = (1 << 8), /* device supports trusted send/recv */ - ATA_DFLAG_FUA = (1 << 9), /* device supports FUA */ - ATA_DFLAG_DMADIR = (1 << 10), /* device requires DMADIR */ - ATA_DFLAG_NCQ_SEND_RECV = (1 << 11), /* device supports NCQ SEND and RECV */ - ATA_DFLAG_NCQ_PRIO = (1 << 12), /* device supports NCQ priority */ - ATA_DFLAG_CDL = (1 << 13), /* supports cmd duration limits */ - ATA_DFLAG_CFG_MASK = (1 << 14) - 1, - - ATA_DFLAG_PIO = (1 << 14), /* device limited to PIO mode */ - ATA_DFLAG_NCQ_OFF = (1 << 15), /* device limited to non-NCQ mode */ - ATA_DFLAG_SLEEPING = (1 << 16), /* device is sleeping */ - ATA_DFLAG_DUBIOUS_XFER = (1 << 17), /* data transfer not verified */ - ATA_DFLAG_NO_UNLOAD = (1 << 18), /* device doesn't support unload */ - ATA_DFLAG_UNLOCK_HPA = (1 << 19), /* unlock HPA */ - ATA_DFLAG_INIT_MASK = (1 << 20) - 1, - - ATA_DFLAG_NCQ_PRIO_ENABLED = (1 << 20), /* Priority cmds sent to dev */ - ATA_DFLAG_CDL_ENABLED = (1 << 21), /* cmd duration limits is enabled */ - ATA_DFLAG_RESUMING = (1 << 22), /* Device is resuming */ - ATA_DFLAG_DETACH = (1 << 24), - ATA_DFLAG_DETACHED = (1 << 25), - ATA_DFLAG_DA = (1 << 26), /* device supports Device Attention */ - ATA_DFLAG_DEVSLP = (1 << 27), /* device supports Device Sleep */ - ATA_DFLAG_ACPI_DISABLED = (1 << 28), /* ACPI for the device is disabled */ - ATA_DFLAG_D_SENSE = (1 << 29), /* Descriptor sense requested */ - - ATA_DFLAG_FEATURES_MASK = (ATA_DFLAG_TRUSTED | ATA_DFLAG_DA | \ - ATA_DFLAG_DEVSLP | ATA_DFLAG_NCQ_SEND_RECV | \ - ATA_DFLAG_NCQ_PRIO | ATA_DFLAG_FUA | \ - ATA_DFLAG_CDL), - + /* sturct ata_device class. */ ATA_DEV_UNKNOWN = 0, /* unknown device */ ATA_DEV_ATA = 1, /* ATA device */ ATA_DEV_ATA_UNSUP = 2, /* ATA device (unsupported) */ @@ -371,6 +381,7 @@ enum { /* return values for ->qc_defer */ ATA_DEFER_LINK = 1, ATA_DEFER_PORT = 2, + ATA_DEFER_LINK_EXCL = 3, /* desc_len for ata_eh_info and context */ ATA_EH_DESC_LEN = 80, @@ -424,7 +435,7 @@ enum { /* This should match the actual table size of * ata_eh_cmd_timeout_table in libata-eh.c. */ - ATA_EH_CMD_TIMEOUT_TABLE_SIZE = 8, + ATA_EH_CMD_TIMEOUT_TABLE_SIZE = 9, /* User visible DMA mask for DMA control. DO NOT renumber. */ ATA_DMA_MASK_ATA = (1 << 0), /* DMA on ATA Disk */ @@ -854,6 +865,9 @@ struct ata_link { unsigned int sata_spd; /* current SATA PHY speed */ enum ata_lpm_policy lpm_policy; + struct work_struct deferred_qc_work; + struct ata_queued_cmd *deferred_qc; + /* record runtime error info, protected by host_set lock */ struct ata_eh_info eh_info; /* EH context */ @@ -899,9 +913,6 @@ struct ata_port { u64 qc_active; int nr_active_links; /* #links with active qcs */ - struct work_struct deferred_qc_work; - struct ata_queued_cmd *deferred_qc; - struct ata_link link; /* host default link */ struct ata_link *slave_link; /* see ata_slave_link_init() */ @@ -983,7 +994,8 @@ struct ata_port_operations { void (*thaw)(struct ata_port *ap); struct ata_reset_operations reset; struct ata_reset_operations pmp_reset; - void (*error_handler)(struct ata_port *ap); + void (*error_handler)(struct ata_port *ap) + __must_hold(&ap->host->eh_mutex); void (*lost_interrupt)(struct ata_port *ap); void (*post_internal_cmd)(struct ata_queued_cmd *qc); void (*sched_eh)(struct ata_port *ap); @@ -1151,6 +1163,9 @@ extern int ata_scsi_ioctl(struct scsi_device *dev, unsigned int cmd, #endif extern enum scsi_qc_status ata_scsi_queuecmd(struct Scsi_Host *h, struct scsi_cmnd *cmd); +enum scsi_timeout_action ata_scsi_retry_deferred_qc(struct ata_port *ap, + struct scsi_cmnd *scmd); +enum scsi_timeout_action ata_scsi_eh_timed_out(struct scsi_cmnd *cmd); #if IS_REACHABLE(CONFIG_ATA) bool ata_scsi_dma_need_drain(struct request *rq); #else @@ -1205,7 +1220,6 @@ extern unsigned int ata_do_dev_read_id(struct ata_device *dev, struct ata_taskfile *tf, __le16 *id); extern void ata_qc_complete(struct ata_queued_cmd *qc); extern u64 ata_qc_get_active(struct ata_port *ap); -extern void ata_scsi_simulate(struct ata_device *dev, struct scsi_cmnd *cmd); extern int ata_std_bios_param(struct scsi_device *sdev, struct gendisk *unused, sector_t capacity, int geom[]); @@ -1226,7 +1240,8 @@ extern int ata_ncq_prio_enable(struct ata_port *ap, struct scsi_device *sdev, extern struct ata_device *ata_dev_pair(struct ata_device *adev); int ata_set_mode(struct ata_link *link, struct ata_device **r_failed_dev); extern void ata_scsi_port_error_handler(struct Scsi_Host *host, struct ata_port *ap); -extern void ata_scsi_cmd_error_handler(struct Scsi_Host *host, struct ata_port *ap, struct list_head *eh_q); +int ata_scsi_cmd_error_handler(struct Scsi_Host *host, struct ata_port *ap, + struct list_head *eh_q); /* * SATA specific code - drivers/ata/libata-sata.c @@ -1313,7 +1328,8 @@ extern int ata_tport_add(struct device *parent, struct ata_port *ap); extern void ata_tport_delete(struct ata_port *ap); int ata_sas_sdev_configure(struct scsi_device *sdev, struct queue_limits *lim, struct ata_port *ap); -extern int ata_sas_queuecmd(struct scsi_cmnd *cmd, struct ata_port *ap); +extern int ata_sas_queuecmd(struct scsi_cmnd *cmd, struct ata_port *ap) + __must_hold(ap->lock); extern void ata_tf_to_fis(const struct ata_taskfile *tf, u8 pmp, int is_cmd, u8 *fis); extern void ata_tf_from_fis(const u8 *fis, struct ata_taskfile *tf); @@ -1415,10 +1431,8 @@ extern int ata_port_freeze(struct ata_port *ap); extern void ata_eh_freeze_port(struct ata_port *ap); extern void ata_eh_thaw_port(struct ata_port *ap); -extern void ata_eh_qc_complete(struct ata_queued_cmd *qc); -extern void ata_eh_qc_retry(struct ata_queued_cmd *qc); - -extern void ata_std_error_handler(struct ata_port *ap); +extern void ata_std_error_handler(struct ata_port *ap) + __must_hold(&ap->host->eh_mutex); extern void ata_std_sched_eh(struct ata_port *ap); extern void ata_std_end_eh(struct ata_port *ap); extern int ata_link_nr_enabled(struct ata_link *link); @@ -1460,6 +1474,7 @@ extern const struct attribute_group *ata_common_sdev_groups[]; .ioctl = ata_scsi_ioctl, \ ATA_SCSI_COMPAT_IOCTL \ .queuecommand = ata_scsi_queuecmd, \ + .eh_timed_out = ata_scsi_eh_timed_out, \ .dma_need_drain = ata_scsi_dma_need_drain, \ .this_id = ATA_SHT_THIS_ID, \ .emulated = ATA_SHT_EMULATED, \ @@ -1998,7 +2013,8 @@ extern void ata_timing_merge(const struct ata_timing *, extern const struct ata_port_operations sata_pmp_port_ops; extern int sata_pmp_qc_defer_cmd_switch(struct ata_queued_cmd *qc); -extern void sata_pmp_error_handler(struct ata_port *ap); +extern void sata_pmp_error_handler(struct ata_port *ap) + __must_hold(&ap->host->eh_mutex); #else /* CONFIG_SATA_PMP */ @@ -2062,7 +2078,8 @@ extern int sata_sff_hardreset(struct ata_link *link, unsigned int *class, unsigned long deadline); extern void ata_sff_postreset(struct ata_link *link, unsigned int *classes); extern void ata_sff_drain_fifo(struct ata_queued_cmd *qc); -extern void ata_sff_error_handler(struct ata_port *ap); +extern void ata_sff_error_handler(struct ata_port *ap) + __must_hold(&ap->host->eh_mutex); extern void ata_sff_std_ports(struct ata_ioports *ioaddr); #ifdef CONFIG_PCI extern int ata_pci_sff_init_host(struct ata_host *host); @@ -2092,7 +2109,8 @@ extern enum ata_completion_errors ata_bmdma_dumb_qc_prep(struct ata_queued_cmd * extern unsigned int ata_bmdma_port_intr(struct ata_port *ap, struct ata_queued_cmd *qc); extern irqreturn_t ata_bmdma_interrupt(int irq, void *dev_instance); -extern void ata_bmdma_error_handler(struct ata_port *ap); +extern void ata_bmdma_error_handler(struct ata_port *ap) + __must_hold(&ap->host->eh_mutex); extern void ata_bmdma_post_internal_cmd(struct ata_queued_cmd *qc); extern void ata_bmdma_irq_clear(struct ata_port *ap); extern void ata_bmdma_setup(struct ata_queued_cmd *qc); diff --git a/include/linux/libnvdimm.h b/include/linux/libnvdimm.h index 28f086c4a187..d929d83abf3b 100644 --- a/include/linux/libnvdimm.h +++ b/include/linux/libnvdimm.h @@ -126,6 +126,15 @@ struct nd_mapping_desc { struct bio; struct resource; struct nd_region; + +/* + * Provider flush callback return values: + * 0: flush completed synchronously + * <0: flush failed + * >0: flush completion was queued and @bio will be completed later + */ +#define NVDIMM_FLUSH_ASYNC 1 + struct nd_region_desc { struct resource *res; struct nd_mapping_desc *mapping; diff --git a/include/linux/linear_range.h b/include/linux/linear_range.h index 2e4f4c3539c0..0f3037f1a94f 100644 --- a/include/linux/linear_range.h +++ b/include/linux/linear_range.h @@ -57,5 +57,8 @@ void linear_range_get_selector_within(const struct linear_range *r, int linear_range_get_selector_low_array(const struct linear_range *r, int ranges, unsigned int val, unsigned int *selector, bool *found); +int linear_range_get_selector_high_array(const struct linear_range *r, + int ranges, unsigned int val, + unsigned int *selector, bool *found); #endif diff --git a/include/linux/linkage.h b/include/linux/linkage.h index b11660b706c5..53fe1f48fd28 100644 --- a/include/linux/linkage.h +++ b/include/linux/linkage.h @@ -39,6 +39,10 @@ #define __page_aligned_data __section(".data..page_aligned") __aligned(PAGE_SIZE) #define __page_aligned_bss __section(".bss..page_aligned") __aligned(PAGE_SIZE) +#ifndef __bss_pgtbl +#define __bss_pgtbl __page_aligned_bss +#endif + /* * For assembly routines. * diff --git a/include/linux/lis3lv02d.h b/include/linux/lis3lv02d.h index b72b8cdba765..feb60ba4e30e 100644 --- a/include/linux/lis3lv02d.h +++ b/include/linux/lis3lv02d.h @@ -30,8 +30,8 @@ * @default_rate: Default sampling rate. 0 means reset default * @setup_resources: Interrupt line setup call back function * @release_resources: Interrupt line release call back function - * @st_min_limits[3]: Selftest acceptance minimum values - * @st_max_limits[3]: Selftest acceptance maximum values + * @st_min_limits: Selftest acceptance minimum values (x, y, z) + * @st_max_limits: Selftest acceptance maximum values (x, y, z) * @irq2: Irq line 2 number * * Platform data is used to setup the sensor chip. Meaning of the different diff --git a/include/linux/list.h b/include/linux/list.h index 00ea8e5fb88b..77fb62f79928 100644 --- a/include/linux/list.h +++ b/include/linux/list.h @@ -34,6 +34,14 @@ struct list_head name = LIST_HEAD_INIT(name) /** + * LIST_HEAD_GUARDED - define a &struct list_head annotated with __guarded_by() + * @name: name of the list_head + * @lock: lock protecting the list + */ +#define LIST_HEAD_GUARDED(name, lock) \ + __guarded_by(&(lock)) LIST_HEAD(name) + +/** * INIT_LIST_HEAD - Initialize a list_head structure * @list: list_head structure to be initialized. * @@ -150,10 +158,13 @@ static inline bool __list_del_entry_valid(struct list_head *entry) * * This is only for internal list manipulation where we know * the prev/next entries already! + * + * Must be inlined to ensure it can be safely called + * with initdata arguments. */ -static inline void __list_add(struct list_head *new, - struct list_head *prev, - struct list_head *next) +static __always_inline void __list_add(struct list_head *new, + struct list_head *prev, + struct list_head *next) { if (!__list_add_valid(new, prev, next)) return; @@ -171,8 +182,12 @@ static inline void __list_add(struct list_head *new, * * Insert a new entry after the specified head. * This is good for implementing stacks. + * + * Must be inlined to ensure it can be safely called + * with initdata arguments. */ -static inline void list_add(struct list_head *new, struct list_head *head) +static __always_inline void list_add(struct list_head *new, + struct list_head *head) { __list_add(new, head, head->next); } @@ -191,6 +206,29 @@ static inline void list_add_tail(struct list_head *new, struct list_head *head) __list_add(new, head->prev, head); } +/** + * list_add_tail_release - add a new entry with release barrier + * @new: new entry to be added + * @head: list head to add it before + * + * Insert a new entry before the specified head, using a release barrier to set + * the ->next pointer that points to it. This is useful for implementing + * queues, in particular one that the elements will be walked through forwards + * locklessly. + */ +static inline void list_add_tail_release(struct list_head *new, + struct list_head *head) +{ + struct list_head *prev = head->prev; + + if (__list_add_valid(new, prev, head)) { + new->next = head; + new->prev = prev; + head->prev = new; + smp_store_release(&prev->next, new); + } +} + /* * Delete a list entry by making the prev/next entries * point to each other. @@ -413,6 +451,7 @@ static inline void list_del_init_careful(struct list_head *entry) * if another CPU could re-list_add() it. */ static inline int list_empty_careful(const struct list_head *head) + __context_unsafe(/* intentional lockless access to @head */) { struct list_head *next = smp_load_acquire(&head->next); return list_is_head(next, head) && (next == READ_ONCE(head->prev)); @@ -645,6 +684,20 @@ static inline void list_splice_tail_init(struct list_head *list, }) /** + * list_first_entry_or_null_acquire - get the first element from a list with barrier + * @ptr: the list head to take the element from. + * @type: the type of the struct this is embedded in. + * @member: the name of the list_head within the struct. + * + * Note that if the list is empty, it returns NULL. + */ +#define list_first_entry_or_null_acquire(ptr, type, member) ({ \ + struct list_head *head__ = (ptr); \ + struct list_head *pos__ = smp_load_acquire(&head__->next); \ + pos__ != head__ ? list_entry(pos__, type, member) : NULL; \ +}) + +/** * list_last_entry_or_null - get the last element from a list * @ptr: the list head to take the element from. * @type: the type of the struct this is embedded in. diff --git a/include/linux/list_lru.h b/include/linux/list_lru.h index fe739d35a864..a450fffe1550 100644 --- a/include/linux/list_lru.h +++ b/include/linux/list_lru.h @@ -81,9 +81,76 @@ static inline int list_lru_init_memcg_key(struct list_lru *lru, struct shrinker int memcg_list_lru_alloc(struct mem_cgroup *memcg, struct list_lru *lru, gfp_t gfp); + +#ifdef CONFIG_MEMCG +/** + * folio_memcg_list_lru_alloc - allocate list_lru heads for shrinkable folio + * @folio: the newly allocated & charged folio + * @lru: the list_lru this might be queued on + * @gfp: gfp mask + * + * Allocate list_lru heads (per-memcg, per-node) needed to queue this + * particular folio down the line. + * + * This does memcg_list_lru_alloc(), but on the memcg that @folio is + * associated with. Handles folio_memcg() access rules in the fast + * path (list_lru heads allocated) and the allocation slowpath. + * + * Returns 0 on success, a negative error value otherwise. + */ +int folio_memcg_list_lru_alloc(struct folio *folio, struct list_lru *lru, + gfp_t gfp); +#else +static inline int folio_memcg_list_lru_alloc(struct folio *folio, + struct list_lru *lru, gfp_t gfp) +{ + return 0; +} +#endif + void memcg_reparent_list_lrus(struct mem_cgroup *memcg, struct mem_cgroup *parent); /** + * list_lru_lock: lock the sublist for the given node and memcg + * @lru: the lru pointer + * @nid: the node id of the sublist to lock. + * @memcg: pointer to the cgroup of the sublist to lock. On return, + * updated to the cgroup whose sublist was actually locked, + * which may be an ancestor if the original memcg was dying. + * + * Returns the locked list_lru_one sublist. The caller must call + * list_lru_unlock() when done. + * + * You must ensure that the memcg is not freed during this call (e.g., with + * rcu or by taking a css refcnt). + * + * Return: the locked list_lru_one, or NULL on failure + */ +struct list_lru_one *list_lru_lock(struct list_lru *lru, int nid, + struct mem_cgroup **memcg); + +/** + * list_lru_unlock: unlock a sublist locked by list_lru_lock() + * @l: the list_lru_one to unlock + */ +void list_lru_unlock(struct list_lru_one *l); + +struct list_lru_one *list_lru_lock_irq(struct list_lru *lru, int nid, + struct mem_cgroup **memcg); +void list_lru_unlock_irq(struct list_lru_one *l); + +struct list_lru_one *list_lru_lock_irqsave(struct list_lru *lru, int nid, + struct mem_cgroup **memcg, unsigned long *irq_flags); +void list_lru_unlock_irqrestore(struct list_lru_one *l, + unsigned long *irq_flags); + +/* Caller-locked variants, see list_lru_add() etc for documentation */ +bool __list_lru_add(struct list_lru *lru, struct list_lru_one *l, + struct list_head *item, int nid, struct mem_cgroup *memcg); +bool __list_lru_del(struct list_lru *lru, struct list_lru_one *l, + struct list_head *item, int nid); + +/** * list_lru_add: add an element to the lru list's tail * @lru: the lru pointer * @item: the item to be added. @@ -115,6 +182,9 @@ void memcg_reparent_list_lrus(struct mem_cgroup *memcg, struct mem_cgroup *paren bool list_lru_add(struct list_lru *lru, struct list_head *item, int nid, struct mem_cgroup *memcg); +bool list_lru_add_irq(struct list_lru *lru, struct list_head *item, int nid, + struct mem_cgroup *memcg); + /** * list_lru_add_obj: add an element to the lru list's tail * @lru: the lru pointer diff --git a/include/linux/liveupdate.h b/include/linux/liveupdate.h index dd11fdc76a5f..63ea5417de84 100644 --- a/include/linux/liveupdate.h +++ b/include/linux/liveupdate.h @@ -12,6 +12,8 @@ #include <linux/kho/abi/luo.h> #include <linux/list.h> #include <linux/mutex.h> +#include <linux/refcount.h> +#include <linux/rwsem.h> #include <linux/types.h> #include <uapi/linux/liveupdate.h> @@ -63,6 +65,7 @@ struct liveupdate_file_op_args { * finish, in order to do successful finish calls for all * resources in the session. * @finish: Required. Final cleanup in the new kernel. + * @get_id: Optional. Returns a unique identifier for the file. * @owner: Module reference * * All operations (except can_preserve) receive a pointer to a @@ -78,6 +81,7 @@ struct liveupdate_file_ops { int (*retrieve)(struct liveupdate_file_op_args *args); bool (*can_finish)(struct liveupdate_file_op_args *args); void (*finish)(struct liveupdate_file_op_args *args); + unsigned long (*get_id)(struct file *file); struct module *owner; }; @@ -169,15 +173,17 @@ struct liveupdate_flb_ops { * @lock: A mutex that protects all fields within this structure, providing * the synchronization service for the FLB's ops. * @finished: True once the FLB's finish() callback has run. - * @retrieved: True once the FLB's retrieve() callback has run. + * @retrieve_status: Status code indicating whether retrieve() has been + * attempted. 0 means not attempted, 1 means successful, + * and negative value means it failed with that error code. */ struct luo_flb_private_state { - long count; + refcount_t count; u64 data; void *obj; struct mutex lock; bool finished; - bool retrieved; + int retrieve_status; }; /* @@ -228,15 +234,18 @@ bool liveupdate_enabled(void); int liveupdate_reboot(void); int liveupdate_register_file_handler(struct liveupdate_file_handler *fh); -int liveupdate_unregister_file_handler(struct liveupdate_file_handler *fh); +void liveupdate_unregister_file_handler(struct liveupdate_file_handler *fh); int liveupdate_register_flb(struct liveupdate_file_handler *fh, struct liveupdate_flb *flb); -int liveupdate_unregister_flb(struct liveupdate_file_handler *fh, - struct liveupdate_flb *flb); +void liveupdate_unregister_flb(struct liveupdate_file_handler *fh, + struct liveupdate_flb *flb); int liveupdate_flb_get_incoming(struct liveupdate_flb *flb, void **objp); +void liveupdate_flb_put_incoming(struct liveupdate_flb *flb); + int liveupdate_flb_get_outgoing(struct liveupdate_flb *flb, void **objp); +void liveupdate_flb_put_outgoing(struct liveupdate_flb *flb); #else /* CONFIG_LIVEUPDATE */ @@ -255,9 +264,8 @@ static inline int liveupdate_register_file_handler(struct liveupdate_file_handle return -EOPNOTSUPP; } -static inline int liveupdate_unregister_file_handler(struct liveupdate_file_handler *fh) +static inline void liveupdate_unregister_file_handler(struct liveupdate_file_handler *fh) { - return -EOPNOTSUPP; } static inline int liveupdate_register_flb(struct liveupdate_file_handler *fh, @@ -266,10 +274,9 @@ static inline int liveupdate_register_flb(struct liveupdate_file_handler *fh, return -EOPNOTSUPP; } -static inline int liveupdate_unregister_flb(struct liveupdate_file_handler *fh, - struct liveupdate_flb *flb) +static inline void liveupdate_unregister_flb(struct liveupdate_file_handler *fh, + struct liveupdate_flb *flb) { - return -EOPNOTSUPP; } static inline int liveupdate_flb_get_incoming(struct liveupdate_flb *flb, @@ -278,11 +285,19 @@ static inline int liveupdate_flb_get_incoming(struct liveupdate_flb *flb, return -EOPNOTSUPP; } +static inline void liveupdate_flb_put_incoming(struct liveupdate_flb *flb) +{ +} + static inline int liveupdate_flb_get_outgoing(struct liveupdate_flb *flb, void **objp) { return -EOPNOTSUPP; } +static inline void liveupdate_flb_put_outgoing(struct liveupdate_flb *flb) +{ +} + #endif /* CONFIG_LIVEUPDATE */ #endif /* _LINUX_LIVEUPDATE_H */ diff --git a/include/linux/llc.h b/include/linux/llc.h index b965314d017f..944e9e213112 100644 --- a/include/linux/llc.h +++ b/include/linux/llc.h @@ -1,14 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * IEEE 802.2 User Interface SAPs for Linux, data structures and indicators. * * Copyright (c) 2001 by Jay Schulist <jschlst@samba.org> - * - * This program can be redistributed or modified under the terms of the - * GNU General Public License as published by the Free Software Foundation. - * This program is distributed without any warranty or implied warranty - * of merchantability or fitness for a particular purpose. - * - * See the GNU General Public License for more details. */ #ifndef __LINUX_LLC_H #define __LINUX_LLC_H diff --git a/include/linux/llist.h b/include/linux/llist.h index 607b2360c938..413249764f7b 100644 --- a/include/linux/llist.h +++ b/include/linux/llist.h @@ -26,8 +26,8 @@ * * | add | del_first | del_all * add | - | - | - - * del_first | | L | L - * del_all | | | - + * del_first | - | L | L + * del_all | - | - | - * * Where, a particular row's operation can happen concurrently with a column's * operation, with "-" being no lock needed, while "L" being lock is needed. @@ -66,7 +66,7 @@ struct llist_node { /** * init_llist_head - initialize lock-less list head - * @head: the head for your lock-less list + * @list: the head for your lock-less list */ static inline void init_llist_head(struct llist_head *list) { diff --git a/include/linux/lockd/bind.h b/include/linux/lockd/bind.h index c53c81242e72..db8207d4059f 100644 --- a/include/linux/lockd/bind.h +++ b/include/linux/lockd/bind.h @@ -10,30 +10,29 @@ #ifndef LINUX_LOCKD_BIND_H #define LINUX_LOCKD_BIND_H -#include <linux/lockd/nlm.h> -/* need xdr-encoded error codes too, so... */ -#include <linux/lockd/xdr.h> -#ifdef CONFIG_LOCKD_V4 -#include <linux/lockd/xdr4.h> -#endif - -/* Dummy declarations */ +struct file_lock; +struct nfs_fh; struct svc_rqst; struct rpc_task; struct rpc_clnt; +struct super_block; +struct module; -/* - * This is the set of functions for lockd->nfsd communication +/** + * struct nlmsvc_binding - lockd -> nfsd callback table + * @owner: module that provides this binding. + * @fopen: open a file by NFS file handle on behalf of an NLM request. + * @fclose: close a file that was previously opened via @fopen. + * Implementations MUST be semantically equivalent to fput(). */ struct nlmsvc_binding { - __be32 (*fopen)(struct svc_rqst *, - struct nfs_fh *, - struct file **, - int mode); - void (*fclose)(struct file *); + struct module *owner; + int (*fopen)(struct svc_rqst *rqstp, struct nfs_fh *f, + struct file **filp, int flags); + void (*fclose)(struct file *filp); }; -extern const struct nlmsvc_binding *nlmsvc_ops; +extern const struct nlmsvc_binding __rcu *nlmsvc_ops; /* * Similar to nfs_client_initdata, but without the NFS-specific @@ -58,6 +57,7 @@ struct nlmclnt_initdata { extern struct nlm_host *nlmclnt_init(const struct nlmclnt_initdata *nlm_init); extern void nlmclnt_done(struct nlm_host *host); extern struct rpc_clnt *nlmclnt_rpc_clnt(struct nlm_host *host); +extern void nlmclnt_shutdown_rpc_clnt(struct nlm_host *host); /* * NLM client operations provide a means to modify RPC processing of NLM @@ -82,4 +82,10 @@ extern int nlmclnt_proc(struct nlm_host *host, int cmd, struct file_lock *fl, vo extern int lockd_up(struct net *net, const struct cred *cred); extern void lockd_down(struct net *net); +/* + * Cluster failover support + */ +int nlmsvc_unlock_all_by_sb(struct super_block *sb); +int nlmsvc_unlock_all_by_ip(struct sockaddr *server_addr); + #endif /* LINUX_LOCKD_BIND_H */ diff --git a/include/linux/lockd/debug.h b/include/linux/lockd/debug.h deleted file mode 100644 index eede2ab5246f..000000000000 --- a/include/linux/lockd/debug.h +++ /dev/null @@ -1,40 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * linux/include/linux/lockd/debug.h - * - * Debugging stuff. - * - * Copyright (C) 1996 Olaf Kirch <okir@monad.swb.de> - */ - -#ifndef LINUX_LOCKD_DEBUG_H -#define LINUX_LOCKD_DEBUG_H - -#include <linux/sunrpc/debug.h> - -/* - * Enable lockd debugging. - * Requires RPC_DEBUG. - */ -#undef ifdebug -#if IS_ENABLED(CONFIG_SUNRPC_DEBUG) -# define ifdebug(flag) if (unlikely(nlm_debug & NLMDBG_##flag)) -#else -# define ifdebug(flag) if (0) -#endif - -/* - * Debug flags - */ -#define NLMDBG_SVC 0x0001 -#define NLMDBG_CLIENT 0x0002 -#define NLMDBG_CLNTLOCK 0x0004 -#define NLMDBG_SVCLOCK 0x0008 -#define NLMDBG_MONITOR 0x0010 -#define NLMDBG_CLNTSUBS 0x0020 -#define NLMDBG_SVCSUBS 0x0040 -#define NLMDBG_HOSTCACHE 0x0080 -#define NLMDBG_XDR 0x0100 -#define NLMDBG_ALL 0x7fff - -#endif /* LINUX_LOCKD_DEBUG_H */ diff --git a/include/linux/lockd/lockd.h b/include/linux/lockd/lockd.h deleted file mode 100644 index 330e38776bb2..000000000000 --- a/include/linux/lockd/lockd.h +++ /dev/null @@ -1,395 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * linux/include/linux/lockd/lockd.h - * - * General-purpose lockd include file. - * - * Copyright (C) 1996 Olaf Kirch <okir@monad.swb.de> - */ - -#ifndef LINUX_LOCKD_LOCKD_H -#define LINUX_LOCKD_LOCKD_H - -/* XXX: a lot of this should really be under fs/lockd. */ - -#include <linux/exportfs.h> -#include <linux/in.h> -#include <linux/in6.h> -#include <net/ipv6.h> -#include <linux/fs.h> -#include <linux/kref.h> -#include <linux/refcount.h> -#include <linux/utsname.h> -#include <linux/lockd/bind.h> -#include <linux/lockd/xdr.h> -#ifdef CONFIG_LOCKD_V4 -#include <linux/lockd/xdr4.h> -#endif -#include <linux/lockd/debug.h> -#include <linux/sunrpc/svc.h> - -/* - * Version string - */ -#define LOCKD_VERSION "0.5" - -/* - * Default timeout for RPC calls (seconds) - */ -#define LOCKD_DFLT_TIMEO 10 - -/* - * Lockd host handle (used both by the client and server personality). - */ -struct nlm_host { - struct hlist_node h_hash; /* doubly linked list */ - struct sockaddr_storage h_addr; /* peer address */ - size_t h_addrlen; - struct sockaddr_storage h_srcaddr; /* our address (optional) */ - size_t h_srcaddrlen; - struct rpc_clnt *h_rpcclnt; /* RPC client to talk to peer */ - char *h_name; /* remote hostname */ - u32 h_version; /* interface version */ - unsigned short h_proto; /* transport proto */ - unsigned short h_reclaiming : 1, - h_server : 1, /* server side, not client side */ - h_noresvport : 1, - h_inuse : 1; - wait_queue_head_t h_gracewait; /* wait while reclaiming */ - struct rw_semaphore h_rwsem; /* Reboot recovery lock */ - u32 h_state; /* pseudo-state counter */ - u32 h_nsmstate; /* true remote NSM state */ - u32 h_pidcount; /* Pseudopids */ - refcount_t h_count; /* reference count */ - struct mutex h_mutex; /* mutex for pmap binding */ - unsigned long h_nextrebind; /* next portmap call */ - unsigned long h_expires; /* eligible for GC */ - struct list_head h_lockowners; /* Lockowners for the client */ - spinlock_t h_lock; - struct list_head h_granted; /* Locks in GRANTED state */ - struct list_head h_reclaim; /* Locks in RECLAIM state */ - struct nsm_handle *h_nsmhandle; /* NSM status handle */ - char *h_addrbuf; /* address eyecatcher */ - struct net *net; /* host net */ - const struct cred *h_cred; - char nodename[UNX_MAXNODENAME + 1]; - const struct nlmclnt_operations *h_nlmclnt_ops; /* Callback ops for NLM users */ -}; - -/* - * The largest string sm_addrbuf should hold is a full-size IPv6 address - * (no "::" anywhere) with a scope ID. The buffer size is computed to - * hold eight groups of colon-separated four-hex-digit numbers, a - * percent sign, a scope id (at most 32 bits, in decimal), and NUL. - */ -#define NSM_ADDRBUF ((8 * 4 + 7) + (1 + 10) + 1) - -struct nsm_handle { - struct list_head sm_link; - refcount_t sm_count; - char *sm_mon_name; - char *sm_name; - struct sockaddr_storage sm_addr; - size_t sm_addrlen; - unsigned int sm_monitored : 1, - sm_sticky : 1; /* don't unmonitor */ - struct nsm_private sm_priv; - char sm_addrbuf[NSM_ADDRBUF]; -}; - -/* - * Rigorous type checking on sockaddr type conversions - */ -static inline struct sockaddr *nlm_addr(const struct nlm_host *host) -{ - return (struct sockaddr *)&host->h_addr; -} - -static inline struct sockaddr *nlm_srcaddr(const struct nlm_host *host) -{ - return (struct sockaddr *)&host->h_srcaddr; -} - -/* - * Map an fl_owner_t into a unique 32-bit "pid" - */ -struct nlm_lockowner { - struct list_head list; - refcount_t count; - - struct nlm_host *host; - fl_owner_t owner; - uint32_t pid; -}; - -/* - * This is the representation of a blocked client lock. - */ -struct nlm_wait { - struct list_head b_list; /* linked list */ - wait_queue_head_t b_wait; /* where to wait on */ - struct nlm_host *b_host; - struct file_lock *b_lock; /* local file lock */ - __be32 b_status; /* grant callback status */ -}; - -/* - * Memory chunk for NLM client RPC request. - */ -#define NLMCLNT_OHSIZE ((__NEW_UTS_LEN) + 10u) -struct nlm_rqst { - refcount_t a_count; - unsigned int a_flags; /* initial RPC task flags */ - struct nlm_host * a_host; /* host handle */ - struct nlm_args a_args; /* arguments */ - struct nlm_res a_res; /* result */ - struct nlm_block * a_block; - unsigned int a_retries; /* Retry count */ - u8 a_owner[NLMCLNT_OHSIZE]; - void * a_callback_data; /* sent to nlmclnt_operations callbacks */ -}; - -/* - * This struct describes a file held open by lockd on behalf of - * an NFS client. - */ -struct nlm_file { - struct hlist_node f_list; /* linked list */ - struct nfs_fh f_handle; /* NFS file handle */ - struct file * f_file[2]; /* VFS file pointers, - indexed by O_ flags */ - struct nlm_share * f_shares; /* DOS shares */ - struct list_head f_blocks; /* blocked locks */ - unsigned int f_locks; /* guesstimate # of locks */ - unsigned int f_count; /* reference count */ - struct mutex f_mutex; /* avoid concurrent access */ -}; - -/* - * This is a server block (i.e. a lock requested by some client which - * couldn't be granted because of a conflicting lock). - */ -#define NLM_NEVER (~(unsigned long) 0) -/* timeout on non-blocking call: */ -#define NLM_TIMEOUT (7 * HZ) - -struct nlm_block { - struct kref b_count; /* Reference count */ - struct list_head b_list; /* linked list of all blocks */ - struct list_head b_flist; /* linked list (per file) */ - struct nlm_rqst * b_call; /* RPC args & callback info */ - struct svc_serv * b_daemon; /* NLM service */ - struct nlm_host * b_host; /* host handle for RPC clnt */ - unsigned long b_when; /* next re-xmit */ - unsigned int b_id; /* block id */ - unsigned char b_granted; /* VFS granted lock */ - struct nlm_file * b_file; /* file in question */ - struct cache_req * b_cache_req; /* deferred request handling */ - struct cache_deferred_req * b_deferred_req; - unsigned int b_flags; /* block flags */ -#define B_QUEUED 1 /* lock queued */ -#define B_GOT_CALLBACK 2 /* got lock or conflicting lock */ -#define B_TIMED_OUT 4 /* filesystem too slow to respond */ -}; - -/* - * Global variables - */ -extern const struct rpc_program nlm_program; -extern const struct svc_procedure nlmsvc_procedures[24]; -#ifdef CONFIG_LOCKD_V4 -extern const struct svc_procedure nlmsvc_procedures4[24]; -#endif -extern int nlmsvc_grace_period; -extern unsigned long nlm_timeout; -extern bool nsm_use_hostnames; -extern u32 nsm_local_state; - -extern struct timer_list nlmsvc_retry; - -/* - * Lockd client functions - */ -struct nlm_rqst * nlm_alloc_call(struct nlm_host *host); -int nlm_async_call(struct nlm_rqst *, u32, const struct rpc_call_ops *); -int nlm_async_reply(struct nlm_rqst *, u32, const struct rpc_call_ops *); -void nlmclnt_release_call(struct nlm_rqst *); -void nlmclnt_prepare_block(struct nlm_wait *block, struct nlm_host *host, - struct file_lock *fl); -void nlmclnt_queue_block(struct nlm_wait *block); -__be32 nlmclnt_dequeue_block(struct nlm_wait *block); -int nlmclnt_wait(struct nlm_wait *block, struct nlm_rqst *req, long timeout); -__be32 nlmclnt_grant(const struct sockaddr *addr, - const struct nlm_lock *lock); -void nlmclnt_recovery(struct nlm_host *); -int nlmclnt_reclaim(struct nlm_host *, struct file_lock *, - struct nlm_rqst *); -void nlmclnt_next_cookie(struct nlm_cookie *); - -/* - * Host cache - */ -struct nlm_host *nlmclnt_lookup_host(const struct sockaddr *sap, - const size_t salen, - const unsigned short protocol, - const u32 version, - const char *hostname, - int noresvport, - struct net *net, - const struct cred *cred); -void nlmclnt_release_host(struct nlm_host *); -struct nlm_host *nlmsvc_lookup_host(const struct svc_rqst *rqstp, - const char *hostname, - const size_t hostname_len); -void nlmsvc_release_host(struct nlm_host *); -struct rpc_clnt * nlm_bind_host(struct nlm_host *); -void nlm_rebind_host(struct nlm_host *); -struct nlm_host * nlm_get_host(struct nlm_host *); -void nlm_shutdown_hosts(void); -void nlm_shutdown_hosts_net(struct net *net); -void nlm_host_rebooted(const struct net *net, - const struct nlm_reboot *); - -/* - * Host monitoring - */ -int nsm_monitor(const struct nlm_host *host); -void nsm_unmonitor(const struct nlm_host *host); - -struct nsm_handle *nsm_get_handle(const struct net *net, - const struct sockaddr *sap, - const size_t salen, - const char *hostname, - const size_t hostname_len); -struct nsm_handle *nsm_reboot_lookup(const struct net *net, - const struct nlm_reboot *info); -void nsm_release(struct nsm_handle *nsm); - -/* - * This is used in garbage collection and resource reclaim - * A return value != 0 means destroy the lock/block/share - */ -typedef int (*nlm_host_match_fn_t)(void *cur, struct nlm_host *ref); - -/* - * Server-side lock handling - */ -int lock_to_openmode(struct file_lock *); -__be32 nlmsvc_lock(struct svc_rqst *, struct nlm_file *, - struct nlm_host *, struct nlm_lock *, int, - struct nlm_cookie *, int); -__be32 nlmsvc_unlock(struct net *net, struct nlm_file *, struct nlm_lock *); -__be32 nlmsvc_testlock(struct svc_rqst *rqstp, struct nlm_file *file, - struct nlm_host *host, struct nlm_lock *lock, - struct nlm_lock *conflock); -__be32 nlmsvc_cancel_blocked(struct net *net, struct nlm_file *, struct nlm_lock *); -void nlmsvc_retry_blocked(struct svc_rqst *rqstp); -void nlmsvc_traverse_blocks(struct nlm_host *, struct nlm_file *, - nlm_host_match_fn_t match); -void nlmsvc_grant_reply(struct nlm_cookie *, __be32); -void nlmsvc_release_call(struct nlm_rqst *); -void nlmsvc_locks_init_private(struct file_lock *, struct nlm_host *, pid_t); - -/* - * File handling for the server personality - */ -__be32 nlm_lookup_file(struct svc_rqst *, struct nlm_file **, - struct nlm_lock *); -void nlm_release_file(struct nlm_file *); -void nlmsvc_put_lockowner(struct nlm_lockowner *); -void nlmsvc_release_lockowner(struct nlm_lock *); -void nlmsvc_mark_resources(struct net *); -void nlmsvc_free_host_resources(struct nlm_host *); -void nlmsvc_invalidate_all(void); - -/* - * Cluster failover support - */ -int nlmsvc_unlock_all_by_sb(struct super_block *sb); -int nlmsvc_unlock_all_by_ip(struct sockaddr *server_addr); - -static inline struct file *nlmsvc_file_file(const struct nlm_file *file) -{ - return file->f_file[O_RDONLY] ? - file->f_file[O_RDONLY] : file->f_file[O_WRONLY]; -} - -static inline struct inode *nlmsvc_file_inode(struct nlm_file *file) -{ - return file_inode(nlmsvc_file_file(file)); -} - -static inline bool -nlmsvc_file_cannot_lock(const struct nlm_file *file) -{ - return exportfs_cannot_lock(nlmsvc_file_file(file)->f_path.dentry->d_sb->s_export_op); -} - -static inline int __nlm_privileged_request4(const struct sockaddr *sap) -{ - const struct sockaddr_in *sin = (struct sockaddr_in *)sap; - - if (ntohs(sin->sin_port) > 1023) - return 0; - - return ipv4_is_loopback(sin->sin_addr.s_addr); -} - -#if IS_ENABLED(CONFIG_IPV6) -static inline int __nlm_privileged_request6(const struct sockaddr *sap) -{ - const struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)sap; - - if (ntohs(sin6->sin6_port) > 1023) - return 0; - - if (ipv6_addr_type(&sin6->sin6_addr) & IPV6_ADDR_MAPPED) - return ipv4_is_loopback(sin6->sin6_addr.s6_addr32[3]); - - return ipv6_addr_type(&sin6->sin6_addr) & IPV6_ADDR_LOOPBACK; -} -#else /* IS_ENABLED(CONFIG_IPV6) */ -static inline int __nlm_privileged_request6(const struct sockaddr *sap) -{ - return 0; -} -#endif /* IS_ENABLED(CONFIG_IPV6) */ - -/* - * Ensure incoming requests are from local privileged callers. - * - * Return TRUE if sender is local and is connecting via a privileged port; - * otherwise return FALSE. - */ -static inline int nlm_privileged_requester(const struct svc_rqst *rqstp) -{ - const struct sockaddr *sap = svc_addr(rqstp); - - switch (sap->sa_family) { - case AF_INET: - return __nlm_privileged_request4(sap); - case AF_INET6: - return __nlm_privileged_request6(sap); - default: - return 0; - } -} - -/* - * Compare two NLM locks. - * When the second lock is of type F_UNLCK, this acts like a wildcard. - */ -static inline int nlm_compare_locks(const struct file_lock *fl1, - const struct file_lock *fl2) -{ - return file_inode(fl1->c.flc_file) == file_inode(fl2->c.flc_file) - && fl1->c.flc_pid == fl2->c.flc_pid - && fl1->c.flc_owner == fl2->c.flc_owner - && fl1->fl_start == fl2->fl_start - && fl1->fl_end == fl2->fl_end - &&(fl1->c.flc_type == fl2->c.flc_type || fl2->c.flc_type == F_UNLCK); -} - -extern const struct lock_manager_operations nlmsvc_lock_operations; - -#endif /* LINUX_LOCKD_LOCKD_H */ diff --git a/include/linux/lockd/nlm.h b/include/linux/lockd/nlm.h deleted file mode 100644 index 6e343ef760dc..000000000000 --- a/include/linux/lockd/nlm.h +++ /dev/null @@ -1,58 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * linux/include/linux/lockd/nlm.h - * - * Declarations for the Network Lock Manager protocol. - * - * Copyright (C) 1996, Olaf Kirch <okir@monad.swb.de> - */ - -#ifndef LINUX_LOCKD_NLM_H -#define LINUX_LOCKD_NLM_H - - -/* Maximum file offset in file_lock.fl_end */ -# define NLM_OFFSET_MAX ((s32) 0x7fffffff) -# define NLM4_OFFSET_MAX ((s64) ((~(u64)0) >> 1)) - -/* Return states for NLM */ -enum { - NLM_LCK_GRANTED = 0, - NLM_LCK_DENIED = 1, - NLM_LCK_DENIED_NOLOCKS = 2, - NLM_LCK_BLOCKED = 3, - NLM_LCK_DENIED_GRACE_PERIOD = 4, -#ifdef CONFIG_LOCKD_V4 - NLM_DEADLCK = 5, - NLM_ROFS = 6, - NLM_STALE_FH = 7, - NLM_FBIG = 8, - NLM_FAILED = 9, -#endif -}; - -#define NLM_PROGRAM 100021 - -#define NLMPROC_NULL 0 -#define NLMPROC_TEST 1 -#define NLMPROC_LOCK 2 -#define NLMPROC_CANCEL 3 -#define NLMPROC_UNLOCK 4 -#define NLMPROC_GRANTED 5 -#define NLMPROC_TEST_MSG 6 -#define NLMPROC_LOCK_MSG 7 -#define NLMPROC_CANCEL_MSG 8 -#define NLMPROC_UNLOCK_MSG 9 -#define NLMPROC_GRANTED_MSG 10 -#define NLMPROC_TEST_RES 11 -#define NLMPROC_LOCK_RES 12 -#define NLMPROC_CANCEL_RES 13 -#define NLMPROC_UNLOCK_RES 14 -#define NLMPROC_GRANTED_RES 15 -#define NLMPROC_NSM_NOTIFY 16 /* statd callback */ -#define NLMPROC_SHARE 20 -#define NLMPROC_UNSHARE 21 -#define NLMPROC_NM_LOCK 22 -#define NLMPROC_FREE_ALL 23 - -#endif /* LINUX_LOCKD_NLM_H */ diff --git a/include/linux/lockd/share.h b/include/linux/lockd/share.h deleted file mode 100644 index 1f18a9faf645..000000000000 --- a/include/linux/lockd/share.h +++ /dev/null @@ -1,32 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * linux/include/linux/lockd/share.h - * - * DOS share management for lockd. - * - * Copyright (C) 1996, Olaf Kirch <okir@monad.swb.de> - */ - -#ifndef LINUX_LOCKD_SHARE_H -#define LINUX_LOCKD_SHARE_H - -/* - * DOS share for a specific file - */ -struct nlm_share { - struct nlm_share * s_next; /* linked list */ - struct nlm_host * s_host; /* client host */ - struct nlm_file * s_file; /* shared file */ - struct xdr_netobj s_owner; /* owner handle */ - u32 s_access; /* access mode */ - u32 s_mode; /* deny mode */ -}; - -__be32 nlmsvc_share_file(struct nlm_host *, struct nlm_file *, - struct nlm_args *); -__be32 nlmsvc_unshare_file(struct nlm_host *, struct nlm_file *, - struct nlm_args *); -void nlmsvc_traverse_shares(struct nlm_host *, struct nlm_file *, - nlm_host_match_fn_t); - -#endif /* LINUX_LOCKD_SHARE_H */ diff --git a/include/linux/lockd/xdr.h b/include/linux/lockd/xdr.h deleted file mode 100644 index 17d53165d9f2..000000000000 --- a/include/linux/lockd/xdr.h +++ /dev/null @@ -1,115 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * linux/include/linux/lockd/xdr.h - * - * XDR types for the NLM protocol - * - * Copyright (C) 1996 Olaf Kirch <okir@monad.swb.de> - */ - -#ifndef LOCKD_XDR_H -#define LOCKD_XDR_H - -#include <linux/fs.h> -#include <linux/filelock.h> -#include <linux/nfs.h> -#include <linux/sunrpc/xdr.h> - -#define SM_MAXSTRLEN 1024 -#define SM_PRIV_SIZE 16 - -struct nsm_private { - unsigned char data[SM_PRIV_SIZE]; -}; - -struct svc_rqst; - -#define NLM_MAXCOOKIELEN 32 -#define NLM_MAXSTRLEN 1024 - -#define nlm_granted cpu_to_be32(NLM_LCK_GRANTED) -#define nlm_lck_denied cpu_to_be32(NLM_LCK_DENIED) -#define nlm_lck_denied_nolocks cpu_to_be32(NLM_LCK_DENIED_NOLOCKS) -#define nlm_lck_blocked cpu_to_be32(NLM_LCK_BLOCKED) -#define nlm_lck_denied_grace_period cpu_to_be32(NLM_LCK_DENIED_GRACE_PERIOD) - -#define nlm_drop_reply cpu_to_be32(30000) - -/* Lock info passed via NLM */ -struct nlm_lock { - char * caller; - unsigned int len; /* length of "caller" */ - struct nfs_fh fh; - struct xdr_netobj oh; - u32 svid; - u64 lock_start; - u64 lock_len; - struct file_lock fl; -}; - -/* - * NLM cookies. Technically they can be 1K, but Linux only uses 8 bytes. - * FreeBSD uses 16, Apple Mac OS X 10.3 uses 20. Therefore we set it to - * 32 bytes. - */ - -struct nlm_cookie -{ - unsigned char data[NLM_MAXCOOKIELEN]; - unsigned int len; -}; - -/* - * Generic lockd arguments for all but sm_notify - */ -struct nlm_args { - struct nlm_cookie cookie; - struct nlm_lock lock; - u32 block; - u32 reclaim; - u32 state; - u32 monitor; - u32 fsm_access; - u32 fsm_mode; -}; - -/* - * Generic lockd result - */ -struct nlm_res { - struct nlm_cookie cookie; - __be32 status; - struct nlm_lock lock; -}; - -/* - * statd callback when client has rebooted - */ -struct nlm_reboot { - char *mon; - unsigned int len; - u32 state; - struct nsm_private priv; -}; - -/* - * Contents of statd callback when monitored host rebooted - */ -#define NLMSVC_XDRSIZE sizeof(struct nlm_args) - -bool nlmsvc_decode_void(struct svc_rqst *rqstp, struct xdr_stream *xdr); -bool nlmsvc_decode_testargs(struct svc_rqst *rqstp, struct xdr_stream *xdr); -bool nlmsvc_decode_lockargs(struct svc_rqst *rqstp, struct xdr_stream *xdr); -bool nlmsvc_decode_cancargs(struct svc_rqst *rqstp, struct xdr_stream *xdr); -bool nlmsvc_decode_unlockargs(struct svc_rqst *rqstp, struct xdr_stream *xdr); -bool nlmsvc_decode_res(struct svc_rqst *rqstp, struct xdr_stream *xdr); -bool nlmsvc_decode_reboot(struct svc_rqst *rqstp, struct xdr_stream *xdr); -bool nlmsvc_decode_shareargs(struct svc_rqst *rqstp, struct xdr_stream *xdr); -bool nlmsvc_decode_notify(struct svc_rqst *rqstp, struct xdr_stream *xdr); - -bool nlmsvc_encode_testres(struct svc_rqst *rqstp, struct xdr_stream *xdr); -bool nlmsvc_encode_res(struct svc_rqst *rqstp, struct xdr_stream *xdr); -bool nlmsvc_encode_void(struct svc_rqst *rqstp, struct xdr_stream *xdr); -bool nlmsvc_encode_shareres(struct svc_rqst *rqstp, struct xdr_stream *xdr); - -#endif /* LOCKD_XDR_H */ diff --git a/include/linux/lockd/xdr4.h b/include/linux/lockd/xdr4.h deleted file mode 100644 index 72831e35dca3..000000000000 --- a/include/linux/lockd/xdr4.h +++ /dev/null @@ -1,43 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * linux/include/linux/lockd/xdr4.h - * - * XDR types for the NLM protocol - * - * Copyright (C) 1996 Olaf Kirch <okir@monad.swb.de> - */ - -#ifndef LOCKD_XDR4_H -#define LOCKD_XDR4_H - -#include <linux/fs.h> -#include <linux/nfs.h> -#include <linux/sunrpc/xdr.h> -#include <linux/lockd/xdr.h> - -/* error codes new to NLMv4 */ -#define nlm4_deadlock cpu_to_be32(NLM_DEADLCK) -#define nlm4_rofs cpu_to_be32(NLM_ROFS) -#define nlm4_stale_fh cpu_to_be32(NLM_STALE_FH) -#define nlm4_fbig cpu_to_be32(NLM_FBIG) -#define nlm4_failed cpu_to_be32(NLM_FAILED) - -void nlm4svc_set_file_lock_range(struct file_lock *fl, u64 off, u64 len); -bool nlm4svc_decode_void(struct svc_rqst *rqstp, struct xdr_stream *xdr); -bool nlm4svc_decode_testargs(struct svc_rqst *rqstp, struct xdr_stream *xdr); -bool nlm4svc_decode_lockargs(struct svc_rqst *rqstp, struct xdr_stream *xdr); -bool nlm4svc_decode_cancargs(struct svc_rqst *rqstp, struct xdr_stream *xdr); -bool nlm4svc_decode_unlockargs(struct svc_rqst *rqstp, struct xdr_stream *xdr); -bool nlm4svc_decode_res(struct svc_rqst *rqstp, struct xdr_stream *xdr); -bool nlm4svc_decode_reboot(struct svc_rqst *rqstp, struct xdr_stream *xdr); -bool nlm4svc_decode_shareargs(struct svc_rqst *rqstp, struct xdr_stream *xdr); -bool nlm4svc_decode_notify(struct svc_rqst *rqstp, struct xdr_stream *xdr); - -bool nlm4svc_encode_testres(struct svc_rqst *rqstp, struct xdr_stream *xdr); -bool nlm4svc_encode_res(struct svc_rqst *rqstp, struct xdr_stream *xdr); -bool nlm4svc_encode_void(struct svc_rqst *rqstp, struct xdr_stream *xdr); -bool nlm4svc_encode_shareres(struct svc_rqst *rqstp, struct xdr_stream *xdr); - -extern const struct rpc_version nlm_version4; - -#endif /* LOCKD_XDR4_H */ diff --git a/include/linux/lockdep.h b/include/linux/lockdep.h index 621566345406..a6451ecbbe9a 100644 --- a/include/linux/lockdep.h +++ b/include/linux/lockdep.h @@ -273,6 +273,9 @@ extern struct pin_cookie lock_pin_lock(struct lockdep_map *lock); extern void lock_repin_lock(struct lockdep_map *lock, struct pin_cookie); extern void lock_unpin_lock(struct lockdep_map *lock, struct pin_cookie); +extern u32 lock_sequence(struct lockdep_map *lock); +#define lockdep_sequence(lock) lock_sequence(&(lock)->dep_map) + #define lockdep_depth(tsk) (debug_locks ? (tsk)->lockdep_depth : 0) #define lockdep_assert(cond) \ diff --git a/include/linux/lockdep_types.h b/include/linux/lockdep_types.h index eae115a26488..55c4b152fedf 100644 --- a/include/linux/lockdep_types.h +++ b/include/linux/lockdep_types.h @@ -253,7 +253,8 @@ struct held_lock { unsigned int hardirqs_off:1; unsigned int sync:1; unsigned int references:11; /* 32 bits */ - unsigned int pin_count; + unsigned int pin_count:24; + unsigned int seq_count:8; }; #else /* !CONFIG_LOCKDEP */ diff --git a/include/linux/lockref.h b/include/linux/lockref.h index 6ded24cdb4a8..ddfb7d3b8cec 100644 --- a/include/linux/lockref.h +++ b/include/linux/lockref.h @@ -34,6 +34,8 @@ struct lockref { }; }; +#define __LOCKREF_DEAD_VAL -128 + /** * lockref_init - Initialize a lockref * @lockref: pointer to lockref structure @@ -55,9 +57,15 @@ void lockref_mark_dead(struct lockref *lockref); bool lockref_get_not_dead(struct lockref *lockref); /* Must be called under spinlock for reliable results */ -static inline bool __lockref_is_dead(const struct lockref *l) +static inline bool lockref_is_dead(const struct lockref *l) +{ + return (READ_ONCE(l->count) == __LOCKREF_DEAD_VAL); +} + +static inline bool lockref_is_dead_or_zero(const struct lockref *l) { - return ((int)l->count < 0); + int count = READ_ONCE(l->count); + return (count == __LOCKREF_DEAD_VAL || count == 0); } #endif /* __LINUX_LOCKREF_H */ diff --git a/include/linux/lsm_audit.h b/include/linux/lsm_audit.h index 382c56a97bba..584db296e43b 100644 --- a/include/linux/lsm_audit.h +++ b/include/linux/lsm_audit.h @@ -94,7 +94,7 @@ struct common_audit_data { #endif char *kmod_name; struct lsm_ioctlop_audit *op; - struct file *file; + const struct file *file; struct lsm_ibpkey_audit *ibpkey; struct lsm_ibendport_audit *ibendport; int reason; diff --git a/include/linux/lsm_hook_defs.h b/include/linux/lsm_hook_defs.h index 8c42b4bde09c..65c9609ec207 100644 --- a/include/linux/lsm_hook_defs.h +++ b/include/linux/lsm_hook_defs.h @@ -176,8 +176,8 @@ LSM_HOOK(int, -EOPNOTSUPP, inode_getsecurity, struct mnt_idmap *idmap, struct inode *inode, const char *name, void **buffer, bool alloc) LSM_HOOK(int, -EOPNOTSUPP, inode_setsecurity, struct inode *inode, const char *name, const void *value, size_t size, int flags) -LSM_HOOK(int, 0, inode_listsecurity, struct inode *inode, char *buffer, - size_t buffer_size) +LSM_HOOK(int, 0, inode_listsecurity, struct inode *inode, char **buffer, + ssize_t *remaining_size) LSM_HOOK(void, LSM_RET_VOID, inode_getlsmprop, struct inode *inode, struct lsm_prop *prop) LSM_HOOK(int, 0, inode_copy_up, struct dentry *src, struct cred **new) @@ -191,6 +191,9 @@ LSM_HOOK(int, 0, file_permission, struct file *file, int mask) LSM_HOOK(int, 0, file_alloc_security, struct file *file) LSM_HOOK(void, LSM_RET_VOID, file_release, struct file *file) LSM_HOOK(void, LSM_RET_VOID, file_free_security, struct file *file) +LSM_HOOK(int, 0, backing_file_alloc, struct file *backing_file, + const struct file *user_file) +LSM_HOOK(void, LSM_RET_VOID, backing_file_free, struct file *backing_file) LSM_HOOK(int, 0, file_ioctl, struct file *file, unsigned int cmd, unsigned long arg) LSM_HOOK(int, 0, file_ioctl_compat, struct file *file, unsigned int cmd, @@ -198,6 +201,8 @@ LSM_HOOK(int, 0, file_ioctl_compat, struct file *file, unsigned int cmd, LSM_HOOK(int, 0, mmap_addr, unsigned long addr) LSM_HOOK(int, 0, mmap_file, struct file *file, unsigned long reqprot, unsigned long prot, unsigned long flags) +LSM_HOOK(int, 0, mmap_backing_file, struct vm_area_struct *vma, + struct file *backing_file, struct file *user_file) LSM_HOOK(int, 0, file_mprotect, struct vm_area_struct *vma, unsigned long reqprot, unsigned long prot) LSM_HOOK(int, 0, file_lock, struct file *file, unsigned int cmd) @@ -317,6 +322,11 @@ LSM_HOOK(int, 0, post_notification, const struct cred *w_cred, LSM_HOOK(int, 0, watch_key, struct key *key) #endif /* CONFIG_SECURITY && CONFIG_KEY_NOTIFICATIONS */ +#if defined(CONFIG_SECURITY_NETWORK) && defined(CONFIG_SECURITY_PATH) +LSM_HOOK(int, 0, unix_find, const struct path *path, struct sock *other, + int flags) +#endif /* CONFIG_SECURITY_NETWORK && CONFIG_SECURITY_PATH */ + #ifdef CONFIG_SECURITY_NETWORK LSM_HOOK(int, 0, unix_stream_connect, struct sock *sock, struct sock *other, struct sock *newsk) diff --git a/include/linux/lsm_hooks.h b/include/linux/lsm_hooks.h index d48bf0ad26f4..c4488c4a6d8a 100644 --- a/include/linux/lsm_hooks.h +++ b/include/linux/lsm_hooks.h @@ -104,6 +104,7 @@ struct security_hook_list { struct lsm_blob_sizes { unsigned int lbs_cred; unsigned int lbs_file; + unsigned int lbs_backing_file; unsigned int lbs_ib; unsigned int lbs_inode; unsigned int lbs_sock; @@ -166,6 +167,7 @@ enum lsm_order { * @initcall_fs: LSM callback for fs_initcall setup, optional * @initcall_device: LSM callback for device_initcall() setup, optional * @initcall_late: LSM callback for late_initcall() setup, optional + * @initcall_late_sync: LSM callback for late_initcall_sync() setup, optional */ struct lsm_info { const struct lsm_id *id; @@ -181,6 +183,7 @@ struct lsm_info { int (*initcall_fs)(void); int (*initcall_device)(void); int (*initcall_late)(void); + int (*initcall_late_sync)(void); }; #define DEFINE_LSM(lsm) \ diff --git a/include/linux/mISDNdsp.h b/include/linux/mISDNdsp.h deleted file mode 100644 index 00758f45fddc..000000000000 --- a/include/linux/mISDNdsp.h +++ /dev/null @@ -1,40 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef __mISDNdsp_H__ -#define __mISDNdsp_H__ - -struct mISDN_dsp_element_arg { - char *name; - char *def; - char *desc; -}; - -struct mISDN_dsp_element { - char *name; - void *(*new)(const char *arg); - void (*free)(void *p); - void (*process_tx)(void *p, unsigned char *data, int len); - void (*process_rx)(void *p, unsigned char *data, int len, - unsigned int txlen); - int num_args; - struct mISDN_dsp_element_arg - *args; -}; - -extern int mISDN_dsp_element_register(struct mISDN_dsp_element *elem); -extern void mISDN_dsp_element_unregister(struct mISDN_dsp_element *elem); - -struct dsp_features { - int hfc_id; /* unique id to identify the chip (or -1) */ - int hfc_dtmf; /* set if HFCmulti card supports dtmf */ - int hfc_conf; /* set if HFCmulti card supports conferences */ - int hfc_loops; /* set if card supports tone loops */ - int hfc_echocanhw; /* set if card supports echocancelation*/ - int pcm_id; /* unique id to identify the pcm bus (or -1) */ - int pcm_slots; /* number of slots on the pcm bus */ - int pcm_banks; /* number of IO banks of pcm bus */ - int unclocked; /* data is not clocked (has jitter/loss) */ - int unordered; /* data is unordered (packets have index) */ -}; - -#endif - diff --git a/include/linux/mISDNhw.h b/include/linux/mISDNhw.h deleted file mode 100644 index ef4f8eb02eac..000000000000 --- a/include/linux/mISDNhw.h +++ /dev/null @@ -1,192 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * - * Author Karsten Keil <kkeil@novell.com> - * - * Basic declarations for the mISDN HW channels - * - * Copyright 2008 by Karsten Keil <kkeil@novell.com> - */ - -#ifndef MISDNHW_H -#define MISDNHW_H -#include <linux/mISDNif.h> -#include <linux/timer.h> - -/* - * HW DEBUG 0xHHHHGGGG - * H - hardware driver specific bits - * G - for all drivers - */ - -#define DEBUG_HW 0x00000001 -#define DEBUG_HW_OPEN 0x00000002 -#define DEBUG_HW_DCHANNEL 0x00000100 -#define DEBUG_HW_DFIFO 0x00000200 -#define DEBUG_HW_BCHANNEL 0x00001000 -#define DEBUG_HW_BFIFO 0x00002000 - -#define MAX_DFRAME_LEN_L1 300 -#define MAX_MON_FRAME 32 -#define MAX_LOG_SPACE 2048 -#define MISDN_COPY_SIZE 32 - -/* channel->Flags bit field */ -#define FLG_TX_BUSY 0 /* tx_buf in use */ -#define FLG_TX_NEXT 1 /* next_skb in use */ -#define FLG_L1_BUSY 2 /* L1 is permanent busy */ -#define FLG_L2_ACTIVATED 3 /* activated from L2 */ -#define FLG_OPEN 5 /* channel is in use */ -#define FLG_ACTIVE 6 /* channel is activated */ -#define FLG_BUSY_TIMER 7 -/* channel type */ -#define FLG_DCHANNEL 8 /* channel is D-channel */ -#define FLG_BCHANNEL 9 /* channel is B-channel */ -#define FLG_ECHANNEL 10 /* channel is E-channel */ -#define FLG_TRANSPARENT 12 /* channel use transparent data */ -#define FLG_HDLC 13 /* channel use hdlc data */ -#define FLG_L2DATA 14 /* channel use L2 DATA primitivs */ -#define FLG_ORIGIN 15 /* channel is on origin site */ -/* channel specific stuff */ -#define FLG_FILLEMPTY 16 /* fill fifo on first frame (empty) */ -/* arcofi specific */ -#define FLG_ARCOFI_TIMER 17 -#define FLG_ARCOFI_ERROR 18 -/* isar specific */ -#define FLG_INITIALIZED 17 -#define FLG_DLEETX 18 -#define FLG_LASTDLE 19 -#define FLG_FIRST 20 -#define FLG_LASTDATA 21 -#define FLG_NMD_DATA 22 -#define FLG_FTI_RUN 23 -#define FLG_LL_OK 24 -#define FLG_LL_CONN 25 -#define FLG_DTMFSEND 26 -#define FLG_TX_EMPTY 27 -/* stop sending received data upstream */ -#define FLG_RX_OFF 28 -/* workq events */ -#define FLG_RECVQUEUE 30 -#define FLG_PHCHANGE 31 - -#define schedule_event(s, ev) do { \ - test_and_set_bit(ev, &((s)->Flags)); \ - schedule_work(&((s)->workq)); \ - } while (0) - -struct dchannel { - struct mISDNdevice dev; - u_long Flags; - struct work_struct workq; - void (*phfunc) (struct dchannel *); - u_int state; - void *l1; - void *hw; - int slot; /* multiport card channel slot */ - struct timer_list timer; - /* receive data */ - struct sk_buff *rx_skb; - int maxlen; - /* send data */ - struct sk_buff_head squeue; - struct sk_buff_head rqueue; - struct sk_buff *tx_skb; - int tx_idx; - int debug; - /* statistics */ - int err_crc; - int err_tx; - int err_rx; -}; - -typedef int (dchannel_l1callback)(struct dchannel *, u_int); -extern int create_l1(struct dchannel *, dchannel_l1callback *); - -/* private L1 commands */ -#define INFO0 0x8002 -#define INFO1 0x8102 -#define INFO2 0x8202 -#define INFO3_P8 0x8302 -#define INFO3_P10 0x8402 -#define INFO4_P8 0x8502 -#define INFO4_P10 0x8602 -#define LOSTFRAMING 0x8702 -#define ANYSIGNAL 0x8802 -#define HW_POWERDOWN 0x8902 -#define HW_RESET_REQ 0x8a02 -#define HW_POWERUP_REQ 0x8b02 -#define HW_DEACT_REQ 0x8c02 -#define HW_ACTIVATE_REQ 0x8e02 -#define HW_D_NOBLOCKED 0x8f02 -#define HW_RESET_IND 0x9002 -#define HW_POWERUP_IND 0x9102 -#define HW_DEACT_IND 0x9202 -#define HW_ACTIVATE_IND 0x9302 -#define HW_DEACT_CNF 0x9402 -#define HW_TESTLOOP 0x9502 -#define HW_TESTRX_RAW 0x9602 -#define HW_TESTRX_HDLC 0x9702 -#define HW_TESTRX_OFF 0x9802 -#define HW_TIMER3_IND 0x9902 -#define HW_TIMER3_VALUE 0x9a00 -#define HW_TIMER3_VMASK 0x00FF - -struct layer1; -extern int l1_event(struct layer1 *, u_int); - -#define MISDN_BCH_FILL_SIZE 4 - -struct bchannel { - struct mISDNchannel ch; - int nr; - u_long Flags; - struct work_struct workq; - u_int state; - void *hw; - int slot; /* multiport card channel slot */ - struct timer_list timer; - /* receive data */ - u8 fill[MISDN_BCH_FILL_SIZE]; - struct sk_buff *rx_skb; - unsigned short maxlen; - unsigned short init_maxlen; /* initial value */ - unsigned short next_maxlen; /* pending value */ - unsigned short minlen; /* for transparent data */ - unsigned short init_minlen; /* initial value */ - unsigned short next_minlen; /* pending value */ - /* send data */ - struct sk_buff *next_skb; - struct sk_buff *tx_skb; - struct sk_buff_head rqueue; - int rcount; - int tx_idx; - int debug; - /* statistics */ - int err_crc; - int err_tx; - int err_rx; - int dropcnt; -}; - -extern int mISDN_initdchannel(struct dchannel *, int, void *); -extern int mISDN_initbchannel(struct bchannel *, unsigned short, - unsigned short); -extern int mISDN_freedchannel(struct dchannel *); -extern void mISDN_clear_bchannel(struct bchannel *); -extern void mISDN_freebchannel(struct bchannel *); -extern int mISDN_ctrl_bchannel(struct bchannel *, struct mISDN_ctrl_req *); -extern void queue_ch_frame(struct mISDNchannel *, u_int, - int, struct sk_buff *); -extern int dchannel_senddata(struct dchannel *, struct sk_buff *); -extern int bchannel_senddata(struct bchannel *, struct sk_buff *); -extern int bchannel_get_rxbuf(struct bchannel *, int); -extern void recv_Dchannel(struct dchannel *); -extern void recv_Echannel(struct dchannel *, struct dchannel *); -extern void recv_Bchannel(struct bchannel *, unsigned int, bool); -extern void recv_Dchannel_skb(struct dchannel *, struct sk_buff *); -extern void recv_Bchannel_skb(struct bchannel *, struct sk_buff *); -extern int get_next_bframe(struct bchannel *); -extern int get_next_dframe(struct dchannel *); - -#endif diff --git a/include/linux/mISDNif.h b/include/linux/mISDNif.h deleted file mode 100644 index 7aab4a769736..000000000000 --- a/include/linux/mISDNif.h +++ /dev/null @@ -1,603 +0,0 @@ -/* - * - * Author Karsten Keil <kkeil@novell.com> - * - * Copyright 2008 by Karsten Keil <kkeil@novell.com> - * - * This code is free software; you can redistribute it and/or modify - * it under the terms of the GNU LESSER GENERAL PUBLIC LICENSE - * version 2.1 as published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU LESSER GENERAL PUBLIC LICENSE for more details. - * - */ - -#ifndef mISDNIF_H -#define mISDNIF_H - -#include <linux/types.h> -#include <linux/errno.h> -#include <linux/socket.h> - -/* - * ABI Version 32 bit - * - * <8 bit> Major version - * - changed if any interface become backwards incompatible - * - * <8 bit> Minor version - * - changed if any interface is extended but backwards compatible - * - * <16 bit> Release number - * - should be incremented on every checkin - */ -#define MISDN_MAJOR_VERSION 1 -#define MISDN_MINOR_VERSION 1 -#define MISDN_RELEASE 29 - -/* primitives for information exchange - * generell format - * <16 bit 0 > - * <8 bit command> - * BIT 8 = 1 LAYER private - * BIT 7 = 1 answer - * BIT 6 = 1 DATA - * <8 bit target layer mask> - * - * Layer = 00 is reserved for general commands - Layer = 01 L2 -> HW - Layer = 02 HW -> L2 - Layer = 04 L3 -> L2 - Layer = 08 L2 -> L3 - * Layer = FF is reserved for broadcast commands - */ - -#define MISDN_CMDMASK 0xff00 -#define MISDN_LAYERMASK 0x00ff - -/* generell commands */ -#define OPEN_CHANNEL 0x0100 -#define CLOSE_CHANNEL 0x0200 -#define CONTROL_CHANNEL 0x0300 -#define CHECK_DATA 0x0400 - -/* layer 2 -> layer 1 */ -#define PH_ACTIVATE_REQ 0x0101 -#define PH_DEACTIVATE_REQ 0x0201 -#define PH_DATA_REQ 0x2001 -#define MPH_ACTIVATE_REQ 0x0501 -#define MPH_DEACTIVATE_REQ 0x0601 -#define MPH_INFORMATION_REQ 0x0701 -#define PH_CONTROL_REQ 0x0801 - -/* layer 1 -> layer 2 */ -#define PH_ACTIVATE_IND 0x0102 -#define PH_ACTIVATE_CNF 0x4102 -#define PH_DEACTIVATE_IND 0x0202 -#define PH_DEACTIVATE_CNF 0x4202 -#define PH_DATA_IND 0x2002 -#define PH_DATA_E_IND 0x3002 -#define MPH_ACTIVATE_IND 0x0502 -#define MPH_DEACTIVATE_IND 0x0602 -#define MPH_INFORMATION_IND 0x0702 -#define PH_DATA_CNF 0x6002 -#define PH_CONTROL_IND 0x0802 -#define PH_CONTROL_CNF 0x4802 - -/* layer 3 -> layer 2 */ -#define DL_ESTABLISH_REQ 0x1004 -#define DL_RELEASE_REQ 0x1104 -#define DL_DATA_REQ 0x3004 -#define DL_UNITDATA_REQ 0x3104 -#define DL_INFORMATION_REQ 0x0004 - -/* layer 2 -> layer 3 */ -#define DL_ESTABLISH_IND 0x1008 -#define DL_ESTABLISH_CNF 0x5008 -#define DL_RELEASE_IND 0x1108 -#define DL_RELEASE_CNF 0x5108 -#define DL_DATA_IND 0x3008 -#define DL_UNITDATA_IND 0x3108 -#define DL_INFORMATION_IND 0x0008 - -/* intern layer 2 management */ -#define MDL_ASSIGN_REQ 0x1804 -#define MDL_ASSIGN_IND 0x1904 -#define MDL_REMOVE_REQ 0x1A04 -#define MDL_REMOVE_IND 0x1B04 -#define MDL_STATUS_UP_IND 0x1C04 -#define MDL_STATUS_DOWN_IND 0x1D04 -#define MDL_STATUS_UI_IND 0x1E04 -#define MDL_ERROR_IND 0x1F04 -#define MDL_ERROR_RSP 0x5F04 - -/* intern layer 2 */ -#define DL_TIMER200_IND 0x7004 -#define DL_TIMER203_IND 0x7304 -#define DL_INTERN_MSG 0x7804 - -/* DL_INFORMATION_IND types */ -#define DL_INFO_L2_CONNECT 0x0001 -#define DL_INFO_L2_REMOVED 0x0002 - -/* PH_CONTROL types */ -/* TOUCH TONE IS 0x20XX XX "0"..."9", "A","B","C","D","*","#" */ -#define DTMF_TONE_VAL 0x2000 -#define DTMF_TONE_MASK 0x007F -#define DTMF_TONE_START 0x2100 -#define DTMF_TONE_STOP 0x2200 -#define DTMF_HFC_COEF 0x4000 -#define DSP_CONF_JOIN 0x2403 -#define DSP_CONF_SPLIT 0x2404 -#define DSP_RECEIVE_OFF 0x2405 -#define DSP_RECEIVE_ON 0x2406 -#define DSP_ECHO_ON 0x2407 -#define DSP_ECHO_OFF 0x2408 -#define DSP_MIX_ON 0x2409 -#define DSP_MIX_OFF 0x240a -#define DSP_DELAY 0x240b -#define DSP_JITTER 0x240c -#define DSP_TXDATA_ON 0x240d -#define DSP_TXDATA_OFF 0x240e -#define DSP_TX_DEJITTER 0x240f -#define DSP_TX_DEJ_OFF 0x2410 -#define DSP_TONE_PATT_ON 0x2411 -#define DSP_TONE_PATT_OFF 0x2412 -#define DSP_VOL_CHANGE_TX 0x2413 -#define DSP_VOL_CHANGE_RX 0x2414 -#define DSP_BF_ENABLE_KEY 0x2415 -#define DSP_BF_DISABLE 0x2416 -#define DSP_BF_ACCEPT 0x2416 -#define DSP_BF_REJECT 0x2417 -#define DSP_PIPELINE_CFG 0x2418 -#define HFC_VOL_CHANGE_TX 0x2601 -#define HFC_VOL_CHANGE_RX 0x2602 -#define HFC_SPL_LOOP_ON 0x2603 -#define HFC_SPL_LOOP_OFF 0x2604 -/* for T30 FAX and analog modem */ -#define HW_MOD_FRM 0x4000 -#define HW_MOD_FRH 0x4001 -#define HW_MOD_FTM 0x4002 -#define HW_MOD_FTH 0x4003 -#define HW_MOD_FTS 0x4004 -#define HW_MOD_CONNECT 0x4010 -#define HW_MOD_OK 0x4011 -#define HW_MOD_NOCARR 0x4012 -#define HW_MOD_FCERROR 0x4013 -#define HW_MOD_READY 0x4014 -#define HW_MOD_LASTDATA 0x4015 - -/* DSP_TONE_PATT_ON parameter */ -#define TONE_OFF 0x0000 -#define TONE_GERMAN_DIALTONE 0x0001 -#define TONE_GERMAN_OLDDIALTONE 0x0002 -#define TONE_AMERICAN_DIALTONE 0x0003 -#define TONE_GERMAN_DIALPBX 0x0004 -#define TONE_GERMAN_OLDDIALPBX 0x0005 -#define TONE_AMERICAN_DIALPBX 0x0006 -#define TONE_GERMAN_RINGING 0x0007 -#define TONE_GERMAN_OLDRINGING 0x0008 -#define TONE_AMERICAN_RINGPBX 0x000b -#define TONE_GERMAN_RINGPBX 0x000c -#define TONE_GERMAN_OLDRINGPBX 0x000d -#define TONE_AMERICAN_RINGING 0x000e -#define TONE_GERMAN_BUSY 0x000f -#define TONE_GERMAN_OLDBUSY 0x0010 -#define TONE_AMERICAN_BUSY 0x0011 -#define TONE_GERMAN_HANGUP 0x0012 -#define TONE_GERMAN_OLDHANGUP 0x0013 -#define TONE_AMERICAN_HANGUP 0x0014 -#define TONE_SPECIAL_INFO 0x0015 -#define TONE_GERMAN_GASSENBESETZT 0x0016 -#define TONE_GERMAN_AUFSCHALTTON 0x0016 - -/* MPH_INFORMATION_IND */ -#define L1_SIGNAL_LOS_OFF 0x0010 -#define L1_SIGNAL_LOS_ON 0x0011 -#define L1_SIGNAL_AIS_OFF 0x0012 -#define L1_SIGNAL_AIS_ON 0x0013 -#define L1_SIGNAL_RDI_OFF 0x0014 -#define L1_SIGNAL_RDI_ON 0x0015 -#define L1_SIGNAL_SLIP_RX 0x0020 -#define L1_SIGNAL_SLIP_TX 0x0021 - -/* - * protocol ids - * D channel 1-31 - * B channel 33 - 63 - */ - -#define ISDN_P_NONE 0 -#define ISDN_P_BASE 0 -#define ISDN_P_TE_S0 0x01 -#define ISDN_P_NT_S0 0x02 -#define ISDN_P_TE_E1 0x03 -#define ISDN_P_NT_E1 0x04 -#define ISDN_P_TE_UP0 0x05 -#define ISDN_P_NT_UP0 0x06 - -#define IS_ISDN_P_TE(p) ((p == ISDN_P_TE_S0) || (p == ISDN_P_TE_E1) || \ - (p == ISDN_P_TE_UP0) || (p == ISDN_P_LAPD_TE)) -#define IS_ISDN_P_NT(p) ((p == ISDN_P_NT_S0) || (p == ISDN_P_NT_E1) || \ - (p == ISDN_P_NT_UP0) || (p == ISDN_P_LAPD_NT)) -#define IS_ISDN_P_S0(p) ((p == ISDN_P_TE_S0) || (p == ISDN_P_NT_S0)) -#define IS_ISDN_P_E1(p) ((p == ISDN_P_TE_E1) || (p == ISDN_P_NT_E1)) -#define IS_ISDN_P_UP0(p) ((p == ISDN_P_TE_UP0) || (p == ISDN_P_NT_UP0)) - - -#define ISDN_P_LAPD_TE 0x10 -#define ISDN_P_LAPD_NT 0x11 - -#define ISDN_P_B_MASK 0x1f -#define ISDN_P_B_START 0x20 - -#define ISDN_P_B_RAW 0x21 -#define ISDN_P_B_HDLC 0x22 -#define ISDN_P_B_X75SLP 0x23 -#define ISDN_P_B_L2DTMF 0x24 -#define ISDN_P_B_L2DSP 0x25 -#define ISDN_P_B_L2DSPHDLC 0x26 -#define ISDN_P_B_T30_FAX 0x27 -#define ISDN_P_B_MODEM_ASYNC 0x28 - -#define OPTION_L2_PMX 1 -#define OPTION_L2_PTP 2 -#define OPTION_L2_FIXEDTEI 3 -#define OPTION_L2_CLEANUP 4 -#define OPTION_L1_HOLD 5 - -/* should be in sync with linux/kobject.h:KOBJ_NAME_LEN */ -#define MISDN_MAX_IDLEN 20 - -struct mISDNhead { - unsigned int prim; - unsigned int id; -} __packed; - -#define MISDN_HEADER_LEN sizeof(struct mISDNhead) -#define MAX_DATA_SIZE 2048 -#define MAX_DATA_MEM (MAX_DATA_SIZE + MISDN_HEADER_LEN) -#define MAX_DFRAME_LEN 260 - -#define MISDN_ID_ADDR_MASK 0xFFFF -#define MISDN_ID_TEI_MASK 0xFF00 -#define MISDN_ID_SAPI_MASK 0x00FF -#define MISDN_ID_TEI_ANY 0x7F00 - -#define MISDN_ID_ANY 0xFFFF -#define MISDN_ID_NONE 0xFFFE - -#define GROUP_TEI 127 -#define TEI_SAPI 63 -#define CTRL_SAPI 0 - -#define MISDN_MAX_CHANNEL 127 -#define MISDN_CHMAP_SIZE ((MISDN_MAX_CHANNEL + 1) >> 3) - -#define SOL_MISDN 0 - -struct sockaddr_mISDN { - sa_family_t family; - unsigned char dev; - unsigned char channel; - unsigned char sapi; - unsigned char tei; -}; - -struct mISDNversion { - unsigned char major; - unsigned char minor; - unsigned short release; -}; - -struct mISDN_devinfo { - u_int id; - u_int Dprotocols; - u_int Bprotocols; - u_int protocol; - u_char channelmap[MISDN_CHMAP_SIZE]; - u_int nrbchan; - char name[MISDN_MAX_IDLEN]; -}; - -struct mISDN_devrename { - u_int id; - char name[MISDN_MAX_IDLEN]; /* new name */ -}; - -/* MPH_INFORMATION_REQ payload */ -struct ph_info_ch { - __u32 protocol; - __u64 Flags; -}; - -struct ph_info_dch { - struct ph_info_ch ch; - __u16 state; - __u16 num_bch; -}; - -struct ph_info { - struct ph_info_dch dch; - struct ph_info_ch bch[]; -}; - -/* timer device ioctl */ -#define IMADDTIMER _IOR('I', 64, int) -#define IMDELTIMER _IOR('I', 65, int) - -/* socket ioctls */ -#define IMGETVERSION _IOR('I', 66, int) -#define IMGETCOUNT _IOR('I', 67, int) -#define IMGETDEVINFO _IOR('I', 68, int) -#define IMCTRLREQ _IOR('I', 69, int) -#define IMCLEAR_L2 _IOR('I', 70, int) -#define IMSETDEVNAME _IOR('I', 71, struct mISDN_devrename) -#define IMHOLD_L1 _IOR('I', 72, int) - -static inline int -test_channelmap(u_int nr, u_char *map) -{ - if (nr <= MISDN_MAX_CHANNEL) - return map[nr >> 3] & (1 << (nr & 7)); - else - return 0; -} - -static inline void -set_channelmap(u_int nr, u_char *map) -{ - map[nr >> 3] |= (1 << (nr & 7)); -} - -static inline void -clear_channelmap(u_int nr, u_char *map) -{ - map[nr >> 3] &= ~(1 << (nr & 7)); -} - -/* CONTROL_CHANNEL parameters */ -#define MISDN_CTRL_GETOP 0x0000 -#define MISDN_CTRL_LOOP 0x0001 -#define MISDN_CTRL_CONNECT 0x0002 -#define MISDN_CTRL_DISCONNECT 0x0004 -#define MISDN_CTRL_RX_BUFFER 0x0008 -#define MISDN_CTRL_PCMCONNECT 0x0010 -#define MISDN_CTRL_PCMDISCONNECT 0x0020 -#define MISDN_CTRL_SETPEER 0x0040 -#define MISDN_CTRL_UNSETPEER 0x0080 -#define MISDN_CTRL_RX_OFF 0x0100 -#define MISDN_CTRL_FILL_EMPTY 0x0200 -#define MISDN_CTRL_GETPEER 0x0400 -#define MISDN_CTRL_L1_TIMER3 0x0800 -#define MISDN_CTRL_HW_FEATURES_OP 0x2000 -#define MISDN_CTRL_HW_FEATURES 0x2001 -#define MISDN_CTRL_HFC_OP 0x4000 -#define MISDN_CTRL_HFC_PCM_CONN 0x4001 -#define MISDN_CTRL_HFC_PCM_DISC 0x4002 -#define MISDN_CTRL_HFC_CONF_JOIN 0x4003 -#define MISDN_CTRL_HFC_CONF_SPLIT 0x4004 -#define MISDN_CTRL_HFC_RECEIVE_OFF 0x4005 -#define MISDN_CTRL_HFC_RECEIVE_ON 0x4006 -#define MISDN_CTRL_HFC_ECHOCAN_ON 0x4007 -#define MISDN_CTRL_HFC_ECHOCAN_OFF 0x4008 -#define MISDN_CTRL_HFC_WD_INIT 0x4009 -#define MISDN_CTRL_HFC_WD_RESET 0x400A - -/* special RX buffer value for MISDN_CTRL_RX_BUFFER request.p1 is the minimum - * buffer size request.p2 the maximum. Using MISDN_CTRL_RX_SIZE_IGNORE will - * not change the value, but still read back the actual stetting. - */ -#define MISDN_CTRL_RX_SIZE_IGNORE -1 - -/* socket options */ -#define MISDN_TIME_STAMP 0x0001 - -struct mISDN_ctrl_req { - int op; - int channel; - int p1; - int p2; -}; - -/* muxer options */ -#define MISDN_OPT_ALL 1 -#define MISDN_OPT_TEIMGR 2 - -#ifdef __KERNEL__ -#include <linux/list.h> -#include <linux/skbuff.h> -#include <linux/net.h> -#include <net/sock.h> -#include <linux/completion.h> - -#define DEBUG_CORE 0x000000ff -#define DEBUG_CORE_FUNC 0x00000002 -#define DEBUG_SOCKET 0x00000004 -#define DEBUG_MANAGER 0x00000008 -#define DEBUG_SEND_ERR 0x00000010 -#define DEBUG_MSG_THREAD 0x00000020 -#define DEBUG_QUEUE_FUNC 0x00000040 -#define DEBUG_L1 0x0000ff00 -#define DEBUG_L1_FSM 0x00000200 -#define DEBUG_L2 0x00ff0000 -#define DEBUG_L2_FSM 0x00020000 -#define DEBUG_L2_CTRL 0x00040000 -#define DEBUG_L2_RECV 0x00080000 -#define DEBUG_L2_TEI 0x00100000 -#define DEBUG_L2_TEIFSM 0x00200000 -#define DEBUG_TIMER 0x01000000 -#define DEBUG_CLOCK 0x02000000 - -#define mISDN_HEAD_P(s) ((struct mISDNhead *)&s->cb[0]) -#define mISDN_HEAD_PRIM(s) (((struct mISDNhead *)&s->cb[0])->prim) -#define mISDN_HEAD_ID(s) (((struct mISDNhead *)&s->cb[0])->id) - -/* socket states */ -#define MISDN_OPEN 1 -#define MISDN_BOUND 2 -#define MISDN_CLOSED 3 - -struct mISDNchannel; -struct mISDNdevice; -struct mISDNstack; -struct mISDNclock; - -struct channel_req { - u_int protocol; - struct sockaddr_mISDN adr; - struct mISDNchannel *ch; -}; - -typedef int (ctrl_func_t)(struct mISDNchannel *, u_int, void *); -typedef int (send_func_t)(struct mISDNchannel *, struct sk_buff *); -typedef int (create_func_t)(struct channel_req *); - -struct Bprotocol { - struct list_head list; - char *name; - u_int Bprotocols; - create_func_t *create; -}; - -struct mISDNchannel { - struct list_head list; - u_int protocol; - u_int nr; - u_long opt; - u_int addr; - struct mISDNstack *st; - struct mISDNchannel *peer; - send_func_t *send; - send_func_t *recv; - ctrl_func_t *ctrl; -}; - -struct mISDN_sock_list { - struct hlist_head head; - rwlock_t lock; -}; - -struct mISDN_sock { - struct sock sk; - struct mISDNchannel ch; - u_int cmask; - struct mISDNdevice *dev; -}; - - - -struct mISDNdevice { - struct mISDNchannel D; - u_int id; - u_int Dprotocols; - u_int Bprotocols; - u_int nrbchan; - u_char channelmap[MISDN_CHMAP_SIZE]; - struct list_head bchannels; - struct mISDNchannel *teimgr; - struct device dev; -}; - -struct mISDNstack { - u_long status; - struct mISDNdevice *dev; - struct task_struct *thread; - struct completion *notify; - wait_queue_head_t workq; - struct sk_buff_head msgq; - struct list_head layer2; - struct mISDNchannel *layer1; - struct mISDNchannel own; - struct mutex lmutex; /* protect lists */ - struct mISDN_sock_list l1sock; -#ifdef MISDN_MSG_STATS - u_int msg_cnt; - u_int sleep_cnt; - u_int stopped_cnt; -#endif -}; - -typedef int (clockctl_func_t)(void *, int); - -struct mISDNclock { - struct list_head list; - char name[64]; - int pri; - clockctl_func_t *ctl; - void *priv; -}; - -/* global alloc/queue functions */ - -static inline struct sk_buff * -mI_alloc_skb(unsigned int len, gfp_t gfp_mask) -{ - struct sk_buff *skb; - - skb = alloc_skb(len + MISDN_HEADER_LEN, gfp_mask); - if (likely(skb)) - skb_reserve(skb, MISDN_HEADER_LEN); - return skb; -} - -static inline struct sk_buff * -_alloc_mISDN_skb(u_int prim, u_int id, u_int len, void *dp, gfp_t gfp_mask) -{ - struct sk_buff *skb = mI_alloc_skb(len, gfp_mask); - struct mISDNhead *hh; - - if (!skb) - return NULL; - if (len) - skb_put_data(skb, dp, len); - hh = mISDN_HEAD_P(skb); - hh->prim = prim; - hh->id = id; - return skb; -} - -static inline void -_queue_data(struct mISDNchannel *ch, u_int prim, - u_int id, u_int len, void *dp, gfp_t gfp_mask) -{ - struct sk_buff *skb; - - if (!ch->peer) - return; - skb = _alloc_mISDN_skb(prim, id, len, dp, gfp_mask); - if (!skb) - return; - if (ch->recv(ch->peer, skb)) - dev_kfree_skb(skb); -} - -/* global register/unregister functions */ - -extern int mISDN_register_device(struct mISDNdevice *, - struct device *parent, char *name); -extern void mISDN_unregister_device(struct mISDNdevice *); -extern int mISDN_register_Bprotocol(struct Bprotocol *); -extern void mISDN_unregister_Bprotocol(struct Bprotocol *); -extern struct mISDNclock *mISDN_register_clock(char *, int, clockctl_func_t *, - void *); -extern void mISDN_unregister_clock(struct mISDNclock *); - -static inline struct mISDNdevice *dev_to_mISDN(const struct device *dev) -{ - if (dev) - return dev_get_drvdata(dev); - else - return NULL; -} - -extern void set_channel_address(struct mISDNchannel *, u_int, u_int); -extern void mISDN_clock_update(struct mISDNclock *, int, ktime_t *); -extern unsigned short mISDN_clock_get(void); -extern const char *mISDNDevName4ch(struct mISDNchannel *); - -#endif /* __KERNEL__ */ -#endif /* mISDNIF_H */ diff --git a/include/linux/mailbox_client.h b/include/linux/mailbox_client.h index c6eea9afb943..e5997120f45c 100644 --- a/include/linux/mailbox_client.h +++ b/include/linux/mailbox_client.h @@ -45,6 +45,7 @@ int mbox_send_message(struct mbox_chan *chan, void *mssg); int mbox_flush(struct mbox_chan *chan, unsigned long timeout); void mbox_client_txdone(struct mbox_chan *chan, int r); /* atomic */ bool mbox_client_peek_data(struct mbox_chan *chan); /* atomic */ +unsigned int mbox_chan_tx_slots_available(struct mbox_chan *chan); /* atomic */ void mbox_free_channel(struct mbox_chan *chan); /* may sleep */ #endif /* __MAILBOX_CLIENT_H */ diff --git a/include/linux/mailbox_controller.h b/include/linux/mailbox_controller.h index 80a427c7ca29..26a238a6f941 100644 --- a/include/linux/mailbox_controller.h +++ b/include/linux/mailbox_controller.h @@ -3,6 +3,7 @@ #ifndef __MAILBOX_CONTROLLER_H #define __MAILBOX_CONTROLLER_H +#include <linux/bits.h> #include <linux/completion.h> #include <linux/device.h> #include <linux/hrtimer.h> @@ -11,6 +12,13 @@ struct mbox_chan; +/* Sentinel value distinguishing "no active request" from "NULL message data" */ +#define MBOX_NO_MSG ((void *)-1) + +#define MBOX_TXDONE_BY_IRQ BIT(0) /* controller has remote RTR irq */ +#define MBOX_TXDONE_BY_POLL BIT(1) /* controller can read status of last TX */ +#define MBOX_TXDONE_BY_ACK BIT(2) /* S/W ACK received by Client ticks the TX */ + /** * struct mbox_chan_ops - methods to control mailbox channels * @send_data: The API asks the MBOX controller driver, in atomic @@ -54,10 +62,10 @@ struct mbox_chan_ops { /** * struct mbox_controller - Controller of a class of communication channels - * @dev: Device backing this controller - * @ops: Operators that work on each communication chan - * @chans: Array of channels - * @num_chans: Number of channels in the 'chans' array. + * @dev: Device backing this controller. Required. + * @ops: Operators that work on each communication chan. Required. + * @chans: Array of channels. Required. + * @num_chans: Number of channels in the 'chans' array. Required. * @txdone_irq: Indicates if the controller can report to API when * the last transmitted data was read by the remote. * Eg, if it has some TX ACK irq. @@ -70,6 +78,7 @@ struct mbox_chan_ops { * @of_xlate: Controller driver specific mapping of channel via DT * @poll_hrt: API private. hrtimer used to poll for TXDONE on all * channels. + * @poll_hrt_lock: API private. Lock protecting access to poll_hrt. * @node: API private. To hook into list of controllers. */ struct mbox_controller { @@ -111,6 +120,7 @@ struct mbox_controller { * @txdone_method: Way to detect TXDone chosen by the API * @cl: Pointer to the current owner of this channel * @tx_complete: Transmission completion + * @tx_status: Transmission status * @active_req: Currently active request hook * @msg_count: No. of mssg currently queued * @msg_free: Index of next available mssg slot @@ -123,6 +133,7 @@ struct mbox_chan { unsigned txdone_method; struct mbox_client *cl; struct completion tx_complete; + int tx_status; void *active_req; unsigned msg_count, msg_free; void *msg_data[MBOX_TX_QUEUE_LEN]; diff --git a/include/linux/maple_tree.h b/include/linux/maple_tree.h index 7b8aad47121e..e595ae5cd0ee 100644 --- a/include/linux/maple_tree.h +++ b/include/linux/maple_tree.h @@ -4,14 +4,13 @@ /* * Maple Tree - An RCU-safe adaptive tree for storing ranges * Copyright (c) 2018-2022 Oracle - * Authors: Liam R. Howlett <Liam.Howlett@Oracle.com> + * Authors: Liam R. Howlett <liam@infradead.org> * Matthew Wilcox <willy@infradead.org> */ #include <linux/kernel.h> #include <linux/rcupdate.h> #include <linux/spinlock.h> -/* #define CONFIG_MAPLE_RCU_DISABLED */ /* * Allocated nodes are mutable until they have been inserted into the tree, @@ -29,13 +28,11 @@ #define MAPLE_NODE_SLOTS 31 /* 256 bytes including ->parent */ #define MAPLE_RANGE64_SLOTS 16 /* 256 bytes */ #define MAPLE_ARANGE64_SLOTS 10 /* 240 bytes */ -#define MAPLE_ALLOC_SLOTS (MAPLE_NODE_SLOTS - 1) #else /* 32bit sizes */ #define MAPLE_NODE_SLOTS 63 /* 256 bytes including ->parent */ #define MAPLE_RANGE64_SLOTS 32 /* 256 bytes */ #define MAPLE_ARANGE64_SLOTS 21 /* 240 bytes */ -#define MAPLE_ALLOC_SLOTS (MAPLE_NODE_SLOTS - 2) #endif /* defined(CONFIG_64BIT) || defined(BUILD_VDSO32_64) */ #define MAPLE_NODE_MASK 255UL @@ -139,6 +136,7 @@ enum maple_type { maple_leaf_64, maple_range_64, maple_arange_64, + maple_copy, }; enum store_type { @@ -154,6 +152,46 @@ enum store_type { wr_slot_store, }; +struct maple_copy { + /* + * min, max, and pivots are values + * start, end, split are indexes into arrays + * data is a size + */ + + struct { + struct maple_node *node; + unsigned long max; + enum maple_type mt; + } dst[3]; + struct { + struct maple_node *node; + unsigned long max; + unsigned char start; + unsigned char end; + enum maple_type mt; + } src[4]; + /* Simulated node */ + void __rcu *slot[3]; + unsigned long gap[3]; + unsigned long min; + union { + unsigned long pivot[3]; + struct { + void *_pad[2]; + unsigned long max; + }; + }; + unsigned char end; + + /*Avoid passing these around */ + unsigned char s_count; + unsigned char d_count; + unsigned char split; + unsigned char data; + unsigned char height; +}; + /** * DOC: Maple tree flags * @@ -299,6 +337,7 @@ struct maple_node { }; struct maple_range_64 mr64; struct maple_arange_64 ma64; + struct maple_copy cp; }; }; @@ -443,6 +482,12 @@ struct ma_state { unsigned char mas_flags; unsigned char end; /* The end of the node */ enum store_type store_type; /* The type of store needed for this operation */ +#ifdef CONFIG_LOCKDEP + u32 ld_seq; +#ifdef CONFIG_RCU_STRICT_GRACE_PERIOD + unsigned long rcu_gp; +#endif /* CONFIG_RCU_STRICT_GRACE_PERIOD */ +#endif /* CONFIG_LOCKDEP */ }; struct ma_wr_state { @@ -525,12 +570,14 @@ int mas_alloc_cyclic(struct ma_state *mas, unsigned long *startp, unsigned long *next, gfp_t gfp); bool mas_nomem(struct ma_state *mas, gfp_t gfp); +bool mas_nomem_nofail(struct ma_state *mas, unsigned long index, + unsigned long last); void mas_pause(struct ma_state *mas); void maple_tree_init(void); void mas_destroy(struct ma_state *mas); void *mas_prev(struct ma_state *mas, unsigned long min); -void *mas_prev_range(struct ma_state *mas, unsigned long max); +void *mas_prev_range(struct ma_state *mas, unsigned long min); void *mas_next(struct ma_state *mas, unsigned long max); void *mas_next_range(struct ma_state *mas, unsigned long max); @@ -822,9 +869,6 @@ static inline void mt_init(struct maple_tree *mt) static inline bool mt_in_rcu(struct maple_tree *mt) { -#ifdef CONFIG_MAPLE_RCU_DISABLED - return false; -#endif return mt->ma_flags & MT_FLAGS_USE_RCU; } diff --git a/include/linux/math.h b/include/linux/math.h index 6dc1d1d32fbc..1e8fb3efbc8c 100644 --- a/include/linux/math.h +++ b/include/linux/math.h @@ -89,23 +89,7 @@ } \ ) -/* - * Divide positive or negative dividend by positive or negative divisor - * and round to closest integer. Result is undefined for negative - * divisors if the dividend variable type is unsigned and for negative - * dividends if the divisor variable type is unsigned. - */ -#define DIV_ROUND_CLOSEST(x, divisor)( \ -{ \ - typeof(x) __x = x; \ - typeof(divisor) __d = divisor; \ - (((typeof(x))-1) > 0 || \ - ((typeof(divisor))-1) > 0 || \ - (((__x) > 0) == ((__d) > 0))) ? \ - (((__x) + ((__d) / 2)) / (__d)) : \ - (((__x) - ((__d) / 2)) / (__d)); \ -} \ -) +#define DIV_ROUND_CLOSEST __KERNEL_DIV_ROUND_CLOSEST /* * Same as above but for u64 dividends. divisor must be a 32-bit * number. diff --git a/include/linux/math64.h b/include/linux/math64.h index cc305206d89f..99189410d4bb 100644 --- a/include/linux/math64.h +++ b/include/linux/math64.h @@ -58,6 +58,20 @@ static inline u64 div64_u64_rem(u64 dividend, u64 divisor, u64 *remainder) } /** + * div64_s64_rem - signed 64bit divide with 64bit divisor and remainder + * @dividend: signed 64bit dividend + * @divisor: signed 64bit divisor + * @remainder: pointer to signed 64bit remainder + * + * Return: sets ``*remainder``, then returns dividend / divisor + */ +static inline s64 div64_s64_rem(s64 dividend, s64 divisor, s64 *remainder) +{ + *remainder = dividend % divisor; + return dividend / divisor; +} + +/** * div64_u64 - unsigned 64bit divide with 64bit divisor * @dividend: unsigned 64bit dividend * @divisor: unsigned 64bit divisor @@ -102,6 +116,10 @@ extern s64 div_s64_rem(s64 dividend, s32 divisor, s32 *remainder); extern u64 div64_u64_rem(u64 dividend, u64 divisor, u64 *remainder); #endif +#ifndef div64_s64_rem +extern s64 div64_s64_rem(s64 dividend, s64 divisor, s64 *remainder); +#endif + #ifndef div64_u64 extern u64 div64_u64(u64 dividend, u64 divisor); #endif diff --git a/include/linux/mcb.h b/include/linux/mcb.h index 4ab2691f51a6..874118765d0f 100644 --- a/include/linux/mcb.h +++ b/include/linux/mcb.h @@ -8,7 +8,7 @@ #ifndef _LINUX_MCB_H #define _LINUX_MCB_H -#include <linux/mod_devicetable.h> +#include <linux/device-id/mcb.h> #include <linux/device.h> #include <linux/irqreturn.h> diff --git a/include/linux/mdio-gpio.h b/include/linux/mdio-gpio.h deleted file mode 100644 index cea443a672cb..000000000000 --- a/include/linux/mdio-gpio.h +++ /dev/null @@ -1,9 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef __LINUX_MDIO_GPIO_H -#define __LINUX_MDIO_GPIO_H - -#define MDIO_GPIO_MDC 0 -#define MDIO_GPIO_MDIO 1 -#define MDIO_GPIO_MDO 2 - -#endif diff --git a/include/linux/mdio.h b/include/linux/mdio.h index 5d1203b9af20..a7d9e3ae362a 100644 --- a/include/linux/mdio.h +++ b/include/linux/mdio.h @@ -8,7 +8,6 @@ #include <uapi/linux/mdio.h> #include <linux/bitfield.h> -#include <linux/mod_devicetable.h> struct gpio_desc; struct mii_bus; @@ -87,6 +86,11 @@ static inline void *mdiodev_get_drvdata(struct mdio_device *mdio) return dev_get_drvdata(&mdio->dev); } +static inline bool mdiodev_has_reset(struct mdio_device *mdio) +{ + return (mdio->reset_gpio || mdio->reset_ctrl); +} + void mdio_device_free(struct mdio_device *mdiodev); struct mdio_device *mdio_device_create(struct mii_bus *bus, int addr); int mdio_device_register(struct mdio_device *mdiodev); @@ -688,8 +692,6 @@ static inline int mdiodev_c45_write(struct mdio_device *mdiodev, u32 devad, val); } -int mdiobus_register_device(struct mdio_device *mdiodev); -int mdiobus_unregister_device(struct mdio_device *mdiodev); bool mdiobus_is_registered_device(struct mii_bus *bus, int addr); struct phy_device *mdiobus_get_phy(struct mii_bus *bus, int addr); diff --git a/include/linux/mei_cl_bus.h b/include/linux/mei_cl_bus.h index a82755e1fc40..d5d29451eabf 100644 --- a/include/linux/mei_cl_bus.h +++ b/include/linux/mei_cl_bus.h @@ -7,7 +7,7 @@ #include <linux/device.h> #include <linux/uuid.h> -#include <linux/mod_devicetable.h> +#include <linux/device-id/mei_cl.h> struct mei_cl_device; struct mei_device; @@ -112,6 +112,7 @@ int mei_cldev_register_rx_cb(struct mei_cl_device *cldev, mei_cldev_cb_t rx_cb); int mei_cldev_register_notif_cb(struct mei_cl_device *cldev, mei_cldev_cb_t notif_cb); +const uuid_le *mei_cldev_uuid(const struct mei_cl_device *cldev); u8 mei_cldev_ver(const struct mei_cl_device *cldev); size_t mei_cldev_mtu(const struct mei_cl_device *cldev); diff --git a/include/linux/memblock.h b/include/linux/memblock.h index 6ec5e9ac0699..d62db9e776cf 100644 --- a/include/linux/memblock.h +++ b/include/linux/memblock.h @@ -51,6 +51,7 @@ extern unsigned long long max_possible_pfn; * memory reservations yet, so we get scratch memory from the previous * kernel that we know is good to use. It is the only memory that * allocations may happen from in this phase. + * @MEMBLOCK_RSRV_HUGETLB: memory is reserved for hugetlb pages */ enum memblock_flags { MEMBLOCK_NONE = 0x0, /* No special request */ @@ -61,6 +62,7 @@ enum memblock_flags { MEMBLOCK_RSRV_NOINIT = 0x10, /* don't initialize struct pages */ MEMBLOCK_RSRV_KERN = 0x20, /* memory reserved for kernel use */ MEMBLOCK_KHO_SCRATCH = 0x40, /* scratch memory for kexec handover */ + MEMBLOCK_RSRV_HUGETLB = 0x80, /* memory reserved for hugetlb pages */ }; /** @@ -155,6 +157,7 @@ int memblock_mark_mirror(phys_addr_t base, phys_addr_t size); int memblock_mark_nomap(phys_addr_t base, phys_addr_t size); int memblock_clear_nomap(phys_addr_t base, phys_addr_t size); int memblock_reserved_mark_noinit(phys_addr_t base, phys_addr_t size); +int memblock_reserved_mark_kern(phys_addr_t base, phys_addr_t size); int memblock_mark_kho_scratch(phys_addr_t base, phys_addr_t size); int memblock_clear_kho_scratch(phys_addr_t base, phys_addr_t size); @@ -172,8 +175,6 @@ void __next_mem_range_rev(u64 *idx, int nid, enum memblock_flags flags, struct memblock_type *type_b, phys_addr_t *out_start, phys_addr_t *out_end, int *out_nid); -void memblock_free_late(phys_addr_t base, phys_addr_t size); - #ifdef CONFIG_HAVE_MEMBLOCK_PHYS_MAP static inline void __next_physmem_range(u64 *idx, struct memblock_type *type, phys_addr_t *out_start, @@ -421,6 +422,7 @@ void *memblock_alloc_try_nid_raw(phys_addr_t size, phys_addr_t align, void *memblock_alloc_try_nid(phys_addr_t size, phys_addr_t align, phys_addr_t min_addr, phys_addr_t max_addr, int nid); +void *memblock_alloc_hugetlb(phys_addr_t size, int nid, bool exact_nid); static __always_inline void *memblock_alloc(phys_addr_t size, phys_addr_t align) { @@ -485,6 +487,7 @@ static inline __init_memblock bool memblock_bottom_up(void) phys_addr_t memblock_phys_mem_size(void); phys_addr_t memblock_reserved_size(void); phys_addr_t memblock_reserved_kern_size(phys_addr_t limit, int nid); +phys_addr_t memblock_reserved_hugetlb_size(phys_addr_t limit, int nid); unsigned long memblock_estimated_nr_free_pages(void); phys_addr_t memblock_start_of_DRAM(void); phys_addr_t memblock_end_of_DRAM(void); @@ -614,11 +617,9 @@ static inline void memtest_report_meminfo(struct seq_file *m) { } #ifdef CONFIG_MEMBLOCK_KHO_SCRATCH void memblock_set_kho_scratch_only(void); void memblock_clear_kho_scratch_only(void); -void memmap_init_kho_scratch_pages(void); #else static inline void memblock_set_kho_scratch_only(void) { } static inline void memblock_clear_kho_scratch_only(void) { } -static inline void memmap_init_kho_scratch_pages(void) {} #endif #endif /* _LINUX_MEMBLOCK_H */ diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index 70b685a85bf4..7d1c0ce189a8 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -29,16 +29,17 @@ struct obj_cgroup; struct page; struct mm_struct; struct kmem_cache; +struct swap_cluster_info; /* Cgroup-specific page state, on top of universal node page state */ enum memcg_stat_item { MEMCG_SWAP = NR_VM_NODE_STAT_ITEMS, MEMCG_SOCK, MEMCG_PERCPU_B, - MEMCG_VMALLOC, MEMCG_KMEM, MEMCG_ZSWAP_B, MEMCG_ZSWAPPED, + MEMCG_ZSWAP_INCOMP, MEMCG_NR_STAT, }; @@ -115,6 +116,16 @@ struct mem_cgroup_per_node { unsigned long lru_zone_size[MAX_NR_ZONES][NR_LRU_LISTS]; struct mem_cgroup_reclaim_iter iter; + /* + * objcg is wiped out as a part of the objcg repaprenting process. + * orig_objcg preserves a pointer (and a reference) to the original + * objcg until the end of live of memcg. + */ + struct obj_cgroup __rcu *objcg; + struct obj_cgroup *orig_objcg; + /* list of inherited objcgs, protected by objcg_lock */ + struct list_head objcg_list; + #ifdef CONFIG_MEMCG_NMI_SAFETY_REQUIRES_ATOMIC /* slab stats for nmi context */ atomic_t slab_reclaimable; @@ -179,6 +190,7 @@ struct obj_cgroup { struct list_head list; /* protected by objcg_lock */ struct rcu_head rcu; }; + bool is_root; }; /* @@ -227,8 +239,6 @@ struct mem_cgroup { */ bool oom_group; - int swappiness; - /* memory.events and memory.events.local */ struct cgroup_file events_file; struct cgroup_file events_local_file; @@ -257,26 +267,18 @@ struct mem_cgroup { seqlock_t socket_pressure_seqlock; #endif int kmemcg_id; - /* - * memcg->objcg is wiped out as a part of the objcg repaprenting - * process. memcg->orig_objcg preserves a pointer (and a reference) - * to the original objcg until the end of live of memcg. - */ - struct obj_cgroup __rcu *objcg; - struct obj_cgroup *orig_objcg; - /* list of inherited objcgs, protected by objcg_lock */ - struct list_head objcg_list; - - struct memcg_vmstats_percpu __percpu *vmstats_percpu; #ifdef CONFIG_CGROUP_WRITEBACK struct list_head cgwb_list; - struct wb_domain cgwb_domain; - struct memcg_cgwb_frn cgwb_frn[MEMCG_CGWB_FRN_CNT]; #endif -#ifdef CONFIG_TRANSPARENT_HUGEPAGE - struct deferred_split deferred_split_queue; + /* Keep the hot per-CPU stats pointer away from memory event counters. */ + struct memcg_vmstats_percpu __percpu *vmstats_percpu + ____cacheline_aligned_in_smp; + +#ifdef CONFIG_CGROUP_WRITEBACK + struct wb_domain cgwb_domain; + struct memcg_cgwb_frn cgwb_frn[MEMCG_CGWB_FRN_CNT]; #endif #ifdef CONFIG_LRU_GEN_WALKS_MMU @@ -319,6 +321,8 @@ struct mem_cgroup { /* List of events which userspace want to receive */ struct list_head event_list; spinlock_t event_list_lock; + + int swappiness; #endif /* CONFIG_MEMCG_V1 */ struct mem_cgroup_per_node *nodeinfo[]; @@ -367,9 +371,6 @@ enum objext_flags { #define OBJEXTS_FLAGS_MASK (__NR_OBJEXTS_FLAGS - 1) #ifdef CONFIG_MEMCG - -static inline bool folio_memcg_kmem(struct folio *folio); - /* * After the initialization objcg->memcg is always pointing at * a valid memcg, but can be atomically swapped to the parent memcg. @@ -383,43 +384,19 @@ static inline struct mem_cgroup *obj_cgroup_memcg(struct obj_cgroup *objcg) } /* - * __folio_memcg - Get the memory cgroup associated with a non-kmem folio - * @folio: Pointer to the folio. - * - * Returns a pointer to the memory cgroup associated with the folio, - * or NULL. This function assumes that the folio is known to have a - * proper memory cgroup pointer. It's not safe to call this function - * against some type of folios, e.g. slab folios or ex-slab folios or - * kmem folios. - */ -static inline struct mem_cgroup *__folio_memcg(struct folio *folio) -{ - unsigned long memcg_data = folio->memcg_data; - - VM_BUG_ON_FOLIO(folio_test_slab(folio), folio); - VM_BUG_ON_FOLIO(memcg_data & MEMCG_DATA_OBJEXTS, folio); - VM_BUG_ON_FOLIO(memcg_data & MEMCG_DATA_KMEM, folio); - - return (struct mem_cgroup *)(memcg_data & ~OBJEXTS_FLAGS_MASK); -} - -/* - * __folio_objcg - get the object cgroup associated with a kmem folio. + * folio_objcg - get the object cgroup associated with a folio. * @folio: Pointer to the folio. * * Returns a pointer to the object cgroup associated with the folio, * or NULL. This function assumes that the folio is known to have a - * proper object cgroup pointer. It's not safe to call this function - * against some type of folios, e.g. slab folios or ex-slab folios or - * LRU folios. + * proper object cgroup pointer. */ -static inline struct obj_cgroup *__folio_objcg(struct folio *folio) +static inline struct obj_cgroup *folio_objcg(struct folio *folio) { unsigned long memcg_data = folio->memcg_data; VM_BUG_ON_FOLIO(folio_test_slab(folio), folio); VM_BUG_ON_FOLIO(memcg_data & MEMCG_DATA_OBJEXTS, folio); - VM_BUG_ON_FOLIO(!(memcg_data & MEMCG_DATA_KMEM), folio); return (struct obj_cgroup *)(memcg_data & ~OBJEXTS_FLAGS_MASK); } @@ -433,21 +410,30 @@ static inline struct obj_cgroup *__folio_objcg(struct folio *folio) * proper memory cgroup pointer. It's not safe to call this function * against some type of folios, e.g. slab folios or ex-slab folios. * - * For a non-kmem folio any of the following ensures folio and memcg binding - * stability: + * For a folio any of the following ensures folio and objcg binding stability: * * - the folio lock * - LRU isolation * - exclusive reference * - * For a kmem folio a caller should hold an rcu read lock to protect memcg - * associated with a kmem folio from being released. + * Based on the stable binding of folio and objcg, for a folio any of the + * following ensures folio and memcg binding stability: + * + * - cgroup_mutex + * - the lruvec lock + * + * If the caller only want to ensure that the page counters of memcg are + * updated correctly, ensure that the binding stability of folio and objcg + * is sufficient. + * + * Note: The caller should hold an rcu read lock or cgroup_mutex to protect + * memcg associated with a folio from being released. */ static inline struct mem_cgroup *folio_memcg(struct folio *folio) { - if (folio_memcg_kmem(folio)) - return obj_cgroup_memcg(__folio_objcg(folio)); - return __folio_memcg(folio); + struct obj_cgroup *objcg = folio_objcg(folio); + + return objcg ? obj_cgroup_memcg(objcg) : NULL; } /* @@ -471,15 +457,10 @@ static inline bool folio_memcg_charged(struct folio *folio) * has an associated memory cgroup pointer or an object cgroups vector or * an object cgroup. * - * For a non-kmem folio any of the following ensures folio and memcg binding - * stability: - * - * - the folio lock - * - LRU isolation - * - exclusive reference + * The page and objcg or memcg binding rules can refer to folio_memcg(). * - * For a kmem folio a caller should hold an rcu read lock to protect memcg - * associated with a kmem folio from being released. + * A caller should hold an rcu read lock to protect memcg associated with a + * page from being released. */ static inline struct mem_cgroup *folio_memcg_check(struct folio *folio) { @@ -488,18 +469,14 @@ static inline struct mem_cgroup *folio_memcg_check(struct folio *folio) * for slabs, READ_ONCE() should be used here. */ unsigned long memcg_data = READ_ONCE(folio->memcg_data); + struct obj_cgroup *objcg; if (memcg_data & MEMCG_DATA_OBJEXTS) return NULL; - if (memcg_data & MEMCG_DATA_KMEM) { - struct obj_cgroup *objcg; + objcg = (void *)(memcg_data & ~OBJEXTS_FLAGS_MASK); - objcg = (void *)(memcg_data & ~OBJEXTS_FLAGS_MASK); - return obj_cgroup_memcg(objcg); - } - - return (struct mem_cgroup *)(memcg_data & ~OBJEXTS_FLAGS_MASK); + return objcg ? obj_cgroup_memcg(objcg) : NULL; } static inline struct mem_cgroup *page_memcg_check(struct page *page) @@ -548,6 +525,27 @@ static inline bool mem_cgroup_is_root(struct mem_cgroup *memcg) return (memcg == root_mem_cgroup); } +/** + * mem_cgroup_shrink_is_root - is this a global or root-memcg shrink invocation? + * @sc: shrink_control describing the current shrinker call + * + * Returns true when @sc represents a global reclaim shrink (sc->memcg == NULL) + * or a root-memcg shrink, i.e. not a per-memcg iteration of + * shrink_slab_memcg(). Filesystems whose ->nr_cached_objects()/ + * ->free_cached_objects() implementations operate on filesystem-global state + * and do not honour sc->memcg can use this to early-return 0 in per-memcg + * contexts. + */ +static inline bool mem_cgroup_shrink_is_root(struct shrink_control *sc) +{ + return !sc->memcg || mem_cgroup_is_root(sc->memcg); +} + +static inline bool obj_cgroup_is_root(const struct obj_cgroup *objcg) +{ + return objcg->is_root; +} + static inline bool mem_cgroup_disabled(void) { return !cgroup_subsys_enabled(memory_cgrp_subsys); @@ -666,8 +664,8 @@ static inline int mem_cgroup_charge(struct folio *folio, struct mm_struct *mm, int mem_cgroup_charge_hugetlb(struct folio* folio, gfp_t gfp); -int mem_cgroup_swapin_charge_folio(struct folio *folio, struct mm_struct *mm, - gfp_t gfp, swp_entry_t entry); +int mem_cgroup_swapin_charge_folio(struct folio *folio, unsigned short id, + struct mm_struct *mm, gfp_t gfp); void __mem_cgroup_uncharge(struct folio *folio); @@ -735,7 +733,15 @@ out: * folio_lruvec - return lruvec for isolating/putting an LRU folio * @folio: Pointer to the folio. * - * This function relies on folio->mem_cgroup being stable. + * Call with rcu_read_lock() held to ensure the lifetime of the returned lruvec. + * Note that this alone will NOT guarantee the stability of the folio->lruvec + * association; the folio can be reparented to an ancestor if this races with + * cgroup deletion. + * + * Use folio_lruvec_lock() to ensure both lifetime and stability of the binding. + * Once a lruvec is locked, folio_lruvec() can be called on other folios, and + * their binding is stable if the returned lruvec matches the one the caller has + * locked. Useful for lock batching. */ static inline struct lruvec *folio_lruvec(struct folio *folio) { @@ -758,15 +764,6 @@ struct lruvec *folio_lruvec_lock_irq(struct folio *folio); struct lruvec *folio_lruvec_lock_irqsave(struct folio *folio, unsigned long *flags); -#ifdef CONFIG_DEBUG_VM -void lruvec_memcg_debug(struct lruvec *lruvec, struct folio *folio); -#else -static inline -void lruvec_memcg_debug(struct lruvec *lruvec, struct folio *folio) -{ -} -#endif - static inline struct mem_cgroup *mem_cgroup_from_css(struct cgroup_subsys_state *css){ return css ? container_of(css, struct mem_cgroup, css) : NULL; @@ -774,23 +771,26 @@ struct mem_cgroup *mem_cgroup_from_css(struct cgroup_subsys_state *css){ static inline bool obj_cgroup_tryget(struct obj_cgroup *objcg) { + if (obj_cgroup_is_root(objcg)) + return true; return percpu_ref_tryget(&objcg->refcnt); } -static inline void obj_cgroup_get(struct obj_cgroup *objcg) +static inline void obj_cgroup_get_many(struct obj_cgroup *objcg, + unsigned long nr) { - percpu_ref_get(&objcg->refcnt); + if (!obj_cgroup_is_root(objcg)) + percpu_ref_get_many(&objcg->refcnt, nr); } -static inline void obj_cgroup_get_many(struct obj_cgroup *objcg, - unsigned long nr) +static inline void obj_cgroup_get(struct obj_cgroup *objcg) { - percpu_ref_get_many(&objcg->refcnt, nr); + obj_cgroup_get_many(objcg, 1); } static inline void obj_cgroup_put(struct obj_cgroup *objcg) { - if (objcg) + if (objcg && !obj_cgroup_is_root(objcg)) percpu_ref_put(&objcg->refcnt); } @@ -885,7 +885,7 @@ static inline bool mm_match_cgroup(struct mm_struct *mm, return match; } -struct cgroup_subsys_state *mem_cgroup_css_from_folio(struct folio *folio); +struct cgroup_subsys_state *get_mem_cgroup_css_from_folio(struct folio *folio); ino_t page_cgroup_ino(struct page *page); static inline bool mem_cgroup_online(struct mem_cgroup *memcg) @@ -896,7 +896,7 @@ static inline bool mem_cgroup_online(struct mem_cgroup *memcg) } void mem_cgroup_update_lru_size(struct lruvec *lruvec, enum lru_list lru, - int zid, int nr_pages); + int zid, long nr_pages); static inline unsigned long mem_cgroup_get_zone_lru_size(struct lruvec *lruvec, @@ -952,6 +952,8 @@ unsigned long memcg_page_state_output(struct mem_cgroup *memcg, int item); bool memcg_stat_item_valid(int idx); bool memcg_vm_event_item_valid(enum vm_event_item idx); unsigned long lruvec_page_state(struct lruvec *lruvec, enum node_stat_item idx); +unsigned long lruvec_page_state_monotonic(struct lruvec *lruvec, + enum node_stat_item idx); unsigned long lruvec_page_state_local(struct lruvec *lruvec, enum node_stat_item idx); @@ -966,10 +968,15 @@ void count_memcg_events(struct mem_cgroup *memcg, enum vm_event_item idx, static inline void count_memcg_folio_events(struct folio *folio, enum vm_event_item idx, unsigned long nr) { - struct mem_cgroup *memcg = folio_memcg(folio); + struct mem_cgroup *memcg; - if (memcg) - count_memcg_events(memcg, idx, nr); + if (!folio_memcg_charged(folio)) + return; + + rcu_read_lock(); + memcg = folio_memcg(folio); + count_memcg_events(memcg, idx, nr); + rcu_read_unlock(); } static inline void count_memcg_events_mm(struct mm_struct *mm, @@ -1087,6 +1094,16 @@ static inline bool mem_cgroup_is_root(struct mem_cgroup *memcg) return true; } +static inline bool mem_cgroup_shrink_is_root(struct shrink_control *sc) +{ + return true; +} + +static inline bool obj_cgroup_is_root(const struct obj_cgroup *objcg) +{ + return true; +} + static inline bool mem_cgroup_disabled(void) { return true; @@ -1145,7 +1162,7 @@ static inline int mem_cgroup_charge_hugetlb(struct folio* folio, gfp_t gfp) } static inline int mem_cgroup_swapin_charge_folio(struct folio *folio, - struct mm_struct *mm, gfp_t gfp, swp_entry_t entry) + unsigned short id, struct mm_struct *mm, gfp_t gfp) { return 0; } @@ -1179,11 +1196,6 @@ static inline struct lruvec *folio_lruvec(struct folio *folio) return &pgdat->__lruvec; } -static inline -void lruvec_memcg_debug(struct lruvec *lruvec, struct folio *folio) -{ -} - static inline struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *memcg) { return NULL; @@ -1242,6 +1254,7 @@ static inline struct lruvec *folio_lruvec_lock(struct folio *folio) { struct pglist_data *pgdat = folio_pgdat(folio); + rcu_read_lock(); spin_lock(&pgdat->__lruvec.lru_lock); return &pgdat->__lruvec; } @@ -1250,6 +1263,7 @@ static inline struct lruvec *folio_lruvec_lock_irq(struct folio *folio) { struct pglist_data *pgdat = folio_pgdat(folio); + rcu_read_lock(); spin_lock_irq(&pgdat->__lruvec.lru_lock); return &pgdat->__lruvec; } @@ -1259,6 +1273,7 @@ static inline struct lruvec *folio_lruvec_lock_irqsave(struct folio *folio, { struct pglist_data *pgdat = folio_pgdat(folio); + rcu_read_lock(); spin_lock_irqsave(&pgdat->__lruvec.lru_lock, *flagsp); return &pgdat->__lruvec; } @@ -1391,6 +1406,12 @@ static inline unsigned long lruvec_page_state(struct lruvec *lruvec, return node_page_state(lruvec_pgdat(lruvec), idx); } +static inline unsigned long lruvec_page_state_monotonic(struct lruvec *lruvec, + enum node_stat_item idx) +{ + return node_page_state_monotonic(lruvec_pgdat(lruvec), idx); +} + static inline unsigned long lruvec_page_state_local(struct lruvec *lruvec, enum node_stat_item idx) { @@ -1453,19 +1474,6 @@ static inline void mem_cgroup_flush_workqueue(void) { } static inline int mem_cgroup_init(void) { return 0; } #endif /* CONFIG_MEMCG */ -/* - * Extended information for slab objects stored as an array in page->memcg_data - * if MEMCG_DATA_OBJEXTS is set. - */ -struct slabobj_ext { -#ifdef CONFIG_MEMCG - struct obj_cgroup *objcg; -#endif -#ifdef CONFIG_MEM_ALLOC_PROFILING - union codetag_ref ref; -#endif -} __aligned(8); - static inline struct lruvec *parent_lruvec(struct lruvec *lruvec) { struct mem_cgroup *memcg; @@ -1479,20 +1487,53 @@ static inline struct lruvec *parent_lruvec(struct lruvec *lruvec) return mem_cgroup_lruvec(memcg, lruvec_pgdat(lruvec)); } -static inline void unlock_page_lruvec(struct lruvec *lruvec) +static inline void lruvec_lock_irq(struct lruvec *lruvec) +{ + rcu_read_lock(); + spin_lock_irq(&lruvec->lru_lock); +} + +static inline struct lruvec *lruvec_live_lock_irq(struct lruvec *lruvec) +{ +#ifdef CONFIG_MEMCG + struct pglist_data *pgdat = lruvec_pgdat(lruvec); + struct mem_cgroup *memcg = lruvec_memcg(lruvec); + + rcu_read_lock(); + + /* + * The memcg can be NULL when the memory controller is disabled. + * Otherwise, the caller keeps the memcg owning @lruvec alive. + */ + while (unlikely(memcg && css_is_dying(&memcg->css))) { + memcg = parent_mem_cgroup(memcg); + lruvec = mem_cgroup_lruvec(memcg, pgdat); + } + + spin_lock_irq(&lruvec->lru_lock); +#else + lruvec_lock_irq(lruvec); +#endif + + return lruvec; +} + +static inline void lruvec_unlock(struct lruvec *lruvec) { spin_unlock(&lruvec->lru_lock); + rcu_read_unlock(); } -static inline void unlock_page_lruvec_irq(struct lruvec *lruvec) +static inline void lruvec_unlock_irq(struct lruvec *lruvec) { spin_unlock_irq(&lruvec->lru_lock); + rcu_read_unlock(); } -static inline void unlock_page_lruvec_irqrestore(struct lruvec *lruvec, - unsigned long flags) +static inline void lruvec_unlock_irqrestore(struct lruvec *lruvec, unsigned long flags) { spin_unlock_irqrestore(&lruvec->lru_lock, flags); + rcu_read_unlock(); } /* Test requires a stable folio->memcg binding, see folio_memcg() */ @@ -1511,7 +1552,7 @@ static inline struct lruvec *folio_lruvec_relock_irq(struct folio *folio, if (folio_matches_lruvec(folio, locked_lruvec)) return locked_lruvec; - unlock_page_lruvec_irq(locked_lruvec); + lruvec_unlock_irq(locked_lruvec); } return folio_lruvec_lock_irq(folio); @@ -1525,7 +1566,7 @@ static inline void folio_lruvec_relock_irqsave(struct folio *folio, if (folio_matches_lruvec(folio, *lruvecp)) return; - unlock_page_lruvec_irqrestore(*lruvecp, *flags); + lruvec_unlock_irqrestore(*lruvecp, *flags); } *lruvecp = folio_lruvec_lock_irqsave(folio, flags); @@ -1549,9 +1590,14 @@ static inline void mem_cgroup_track_foreign_dirty(struct folio *folio, if (mem_cgroup_disabled()) return; + if (!folio_memcg_charged(folio)) + return; + + rcu_read_lock(); memcg = folio_memcg(folio); - if (unlikely(memcg && &memcg->css != wb->memcg_css)) + if (unlikely(&memcg->css != wb->memcg_css)) mem_cgroup_track_foreign_dirty_slowpath(folio, wb); + rcu_read_unlock(); } void mem_cgroup_flush_foreign(struct bdi_writeback *wb); @@ -1896,9 +1942,6 @@ static inline void mem_cgroup_exit_user_fault(void) current->in_user_fault = 0; } -void memcg1_swapout(struct folio *folio, swp_entry_t entry); -void memcg1_swapin(swp_entry_t entry, unsigned int nr_pages); - #else /* CONFIG_MEMCG_V1 */ static inline unsigned long memcg1_soft_limit_reclaim(pg_data_t *pgdat, int order, @@ -1926,14 +1969,23 @@ static inline void mem_cgroup_exit_user_fault(void) { } -static inline void memcg1_swapout(struct folio *folio, swp_entry_t entry) +#endif /* CONFIG_MEMCG_V1 */ + +#if defined(CONFIG_MEMCG_V1) && defined(CONFIG_SWAP) + +void __memcg1_swapout(struct folio *folio, struct swap_cluster_info *ci); +void memcg1_swapin(struct folio *folio); + +#else + +static inline void __memcg1_swapout(struct folio *folio, + struct swap_cluster_info *ci) { } -static inline void memcg1_swapin(swp_entry_t entry, unsigned int nr_pages) +static inline void memcg1_swapin(struct folio *folio) { } - -#endif /* CONFIG_MEMCG_V1 */ +#endif #endif /* _LINUX_MEMCONTROL_H */ diff --git a/include/linux/memfd.h b/include/linux/memfd.h index c328a7b356d0..c159e40e3f34 100644 --- a/include/linux/memfd.h +++ b/include/linux/memfd.h @@ -14,10 +14,12 @@ struct folio *memfd_alloc_folio(struct file *memfd, pgoff_t idx); * to sealing, or 0 otherwise. * * We also update VMA flags if appropriate by manipulating the VMA flags pointed - * to by vm_flags_ptr. + * to by vma_flags_ptr. */ -int memfd_check_seals_mmap(struct file *file, vm_flags_t *vm_flags_ptr); +int memfd_check_seals_mmap(struct file *file, vma_flags_t *vma_flags_ptr); struct file *memfd_alloc_file(const char *name, unsigned int flags); +int memfd_get_seals(struct file *file); +int memfd_add_seals(struct file *file, unsigned int seals); #else static inline long memfd_fcntl(struct file *f, unsigned int c, unsigned int a) { @@ -28,7 +30,7 @@ static inline struct folio *memfd_alloc_folio(struct file *memfd, pgoff_t idx) return ERR_PTR(-EINVAL); } static inline int memfd_check_seals_mmap(struct file *file, - vm_flags_t *vm_flags_ptr) + vma_flags_t *vma_flags_ptr) { return 0; } @@ -37,6 +39,16 @@ static inline struct file *memfd_alloc_file(const char *name, unsigned int flags { return ERR_PTR(-EINVAL); } + +static inline int memfd_get_seals(struct file *file) +{ + return -EINVAL; +} + +static inline int memfd_add_seals(struct file *file, unsigned int seals) +{ + return -EINVAL; +} #endif #endif /* __LINUX_MEMFD_H */ diff --git a/include/linux/memory-tiers.h b/include/linux/memory-tiers.h index 96987d9d95a8..7999c58629ee 100644 --- a/include/linux/memory-tiers.h +++ b/include/linux/memory-tiers.h @@ -52,7 +52,7 @@ int mt_perf_to_adistance(struct access_coordinate *perf, int *adist); struct memory_dev_type *mt_find_alloc_memory_type(int adist, struct list_head *memory_types); void mt_put_memory_types(struct list_head *memory_types); -#ifdef CONFIG_MIGRATION +#ifdef CONFIG_NUMA_MIGRATION int next_demotion_node(int node, const nodemask_t *allowed_mask); void node_get_allowed_targets(pg_data_t *pgdat, nodemask_t *targets); bool node_is_toptier(int node); diff --git a/include/linux/memory.h b/include/linux/memory.h index faeaa921e55b..1783299073e4 100644 --- a/include/linux/memory.h +++ b/include/linux/memory.h @@ -19,6 +19,8 @@ #include <linux/node.h> #include <linux/compiler.h> #include <linux/mutex.h> +#include <linux/memory_hotplug.h> +#include <linux/range.h> #define MIN_MEMORY_BLOCK_SIZE (1UL << SECTION_SIZE_BITS) @@ -77,7 +79,7 @@ enum memory_block_state { struct memory_block { unsigned long start_section_nr; enum memory_block_state state; /* serialized by the dev->lock */ - int online_type; /* for passing data to online routine */ + enum mmop online_type; /* for passing data to online routine */ int nid; /* NID for this memory block */ /* * The single zone of this memory block if all PFNs of this memory block @@ -99,6 +101,32 @@ int arch_get_memory_phys_device(unsigned long start_pfn); unsigned long memory_block_size_bytes(void); int set_memory_block_size_order(unsigned int order); +/** + * memory_block_aligned_range - align a physical address range to memory blocks + * @range: the input range to align + * + * Aligns the start address up and the end address down to memory block + * boundaries. This is required for memory hotplug operations which must + * operate on memory-block aligned ranges. + * + * Returns the aligned range. Callers should check that the returned + * range is valid (aligned.start < aligned.end) before using it. + */ +static inline struct range memory_block_aligned_range(const struct range *range) +{ + struct range aligned; + + aligned.start = ALIGN(range->start, memory_block_size_bytes()); + aligned.end = ALIGN_DOWN(range->end + 1, memory_block_size_bytes()); + /* No whole block fits (e.g. range below the first boundary): empty. */ + if (aligned.end <= aligned.start) + aligned.start = aligned.end; + else + aligned.end -= 1; + + return aligned; +} + struct memory_notify { unsigned long start_pfn; unsigned long nr_pages; @@ -157,7 +185,11 @@ int create_memory_block_devices(unsigned long start, unsigned long size, void remove_memory_block_devices(unsigned long start, unsigned long size); extern void memory_dev_init(void); extern int memory_notify(enum memory_block_state state, void *v); -extern struct memory_block *find_memory_block(unsigned long section_nr); +struct memory_block *memory_block_get(unsigned long block_id); +static inline void memory_block_put(struct memory_block *mem) +{ + put_device(&mem->dev); +} typedef int (*walk_memory_blocks_func_t)(struct memory_block *, void *); extern int walk_memory_blocks(unsigned long start, unsigned long size, void *arg, walk_memory_blocks_func_t func); @@ -170,7 +202,6 @@ struct memory_group *memory_group_find_by_id(int mgid); typedef int (*walk_memory_groups_func_t)(struct memory_group *, void *); int walk_dynamic_memory_groups(int nid, walk_memory_groups_func_t func, struct memory_group *excluded, void *arg); -struct memory_block *find_memory_block_by_id(unsigned long block_id); #define hotplug_memory_notifier(fn, pri) ({ \ static __meminitdata struct notifier_block fn##_mem_nb =\ { .notifier_call = fn, .priority = pri };\ diff --git a/include/linux/memory_hotplug.h b/include/linux/memory_hotplug.h index f2f16cdd73ee..b39605d30896 100644 --- a/include/linux/memory_hotplug.h +++ b/include/linux/memory_hotplug.h @@ -16,11 +16,8 @@ struct resource; struct vmem_altmap; struct dev_pagemap; -#ifdef CONFIG_MEMORY_HOTPLUG -struct page *pfn_to_online_page(unsigned long pfn); - /* Types for control the zone type of onlined and offlined memory */ -enum { +enum mmop { /* Offline the memory. */ MMOP_OFFLINE = 0, /* Online the memory. Zone depends, see default_zone_for_pfn(). */ @@ -31,6 +28,9 @@ enum { MMOP_ONLINE_MOVABLE, }; +#ifdef CONFIG_MEMORY_HOTPLUG +struct page *pfn_to_online_page(unsigned long pfn); + /* Flags for add_memory() and friends to specify memory hotplug details. */ typedef int __bitwise mhp_t; @@ -127,6 +127,7 @@ extern int arch_add_memory(int nid, u64 start, u64 size, extern u64 max_mem_size; extern int mhp_online_type_from_str(const char *str); +const char *mhp_online_type_to_str(int online_type); /* If movable_node boot option specified */ extern bool movable_node_enabled; @@ -135,9 +136,10 @@ static inline bool movable_node_is_enabled(void) return movable_node_enabled; } -extern void arch_remove_memory(u64 start, u64 size, struct vmem_altmap *altmap); +extern void arch_remove_memory(u64 start, u64 size, struct vmem_altmap *altmap, + struct dev_pagemap *pgmap); extern void __remove_pages(unsigned long start_pfn, unsigned long nr_pages, - struct vmem_altmap *altmap); + struct vmem_altmap *altmap, struct dev_pagemap *pgmap); /* reasonably generic interface to expand the physical pages */ extern int __add_pages(int nid, unsigned long start_pfn, unsigned long nr_pages, @@ -266,6 +268,8 @@ extern int offline_pages(unsigned long start_pfn, unsigned long nr_pages, extern int remove_memory(u64 start, u64 size); extern void __remove_memory(u64 start, u64 size); extern int offline_and_remove_memory(u64 start, u64 size); +int offline_and_remove_memory_ranges(const struct range *ranges, + unsigned int nr_ranges); #else static inline void try_offline_node(int nid) {} @@ -282,17 +286,26 @@ static inline int remove_memory(u64 start, u64 size) } static inline void __remove_memory(u64 start, u64 size) {} + +static inline int offline_and_remove_memory_ranges(const struct range *ranges, + unsigned int nr_ranges) +{ + return -EBUSY; +} #endif /* CONFIG_MEMORY_HOTREMOVE */ #ifdef CONFIG_MEMORY_HOTPLUG /* Default online_type (MMOP_*) when new memory blocks are added. */ -extern int mhp_get_default_online_type(void); -extern void mhp_set_default_online_type(int online_type); -extern void __ref free_area_init_core_hotplug(struct pglist_data *pgdat); +extern enum mmop mhp_get_default_online_type(void); +extern void mhp_set_default_online_type(enum mmop online_type); +int __ref free_area_init_core_hotplug(struct pglist_data *pgdat); extern int __add_memory(int nid, u64 start, u64 size, mhp_t mhp_flags); extern int add_memory(int nid, u64 start, u64 size, mhp_t mhp_flags); extern int add_memory_resource(int nid, struct resource *resource, mhp_t mhp_flags); +int __add_memory_driver_managed(int nid, u64 start, u64 size, + const char *resource_name, mhp_t mhp_flags, + enum mmop online_type); extern int add_memory_driver_managed(int nid, u64 start, u64 size, const char *resource_name, mhp_t mhp_flags); @@ -307,15 +320,16 @@ extern int sparse_add_section(int nid, unsigned long pfn, unsigned long nr_pages, struct vmem_altmap *altmap, struct dev_pagemap *pgmap); extern void sparse_remove_section(unsigned long pfn, unsigned long nr_pages, - struct vmem_altmap *altmap); -extern struct page *sparse_decode_mem_map(unsigned long coded_mem_map, - unsigned long pnum); -extern struct zone *zone_for_pfn_range(int online_type, int nid, - struct memory_group *group, unsigned long start_pfn, + struct vmem_altmap *altmap, + struct dev_pagemap *pgmap); +extern struct zone *zone_for_pfn_range(enum mmop online_type, + int nid, struct memory_group *group, unsigned long start_pfn, unsigned long nr_pages); extern int arch_create_linear_mapping(int nid, u64 start, u64 size, struct mhp_params *params); void arch_remove_linear_mapping(u64 start, u64 size); +#else +static inline enum mmop mhp_get_default_online_type(void) { return MMOP_OFFLINE; } #endif /* CONFIG_MEMORY_HOTPLUG */ #endif /* __LINUX_MEMORY_HOTPLUG_H */ diff --git a/include/linux/mempolicy.h b/include/linux/mempolicy.h index 0fe96f3ab3ef..65c732d440d2 100644 --- a/include/linux/mempolicy.h +++ b/include/linux/mempolicy.h @@ -55,6 +55,7 @@ struct mempolicy { nodemask_t cpuset_mems_allowed; /* relative to these nodes */ nodemask_t user_nodemask; /* nodemask passed by user */ } w; + struct rcu_head rcu; }; /* diff --git a/include/linux/mempool.h b/include/linux/mempool.h index e8e440e04a06..a0fa6d43e0dc 100644 --- a/include/linux/mempool.h +++ b/include/linux/mempool.h @@ -66,7 +66,7 @@ void *mempool_alloc_noprof(struct mempool *pool, gfp_t gfp_mask) __malloc; #define mempool_alloc(...) \ alloc_hooks(mempool_alloc_noprof(__VA_ARGS__)) int mempool_alloc_bulk_noprof(struct mempool *pool, void **elem, - unsigned int count, unsigned int allocated); + unsigned int count); #define mempool_alloc_bulk(...) \ alloc_hooks(mempool_alloc_bulk_noprof(__VA_ARGS__)) diff --git a/include/linux/mfd/88pm886.h b/include/linux/mfd/88pm886.h index 38892ba7b8a4..2c24dd3032ab 100644 --- a/include/linux/mfd/88pm886.h +++ b/include/linux/mfd/88pm886.h @@ -11,6 +11,7 @@ #define PM886_PAGE_OFFSET_REGULATORS 1 #define PM886_PAGE_OFFSET_GPADC 2 +#define PM886_PAGE_OFFSET_BATTERY 3 #define PM886_REG_ID 0x00 @@ -128,9 +129,13 @@ #define PM886_GPADC_BIAS_LEVELS 16 #define PM886_GPADC_INDEX_TO_BIAS_uA(i) (1 + (i) * 5) +/* Battery block register definitions */ +#define PM886_REG_CLS_CONFIG1 0x71 + struct pm886_chip { struct i2c_client *client; unsigned int chip_id; struct regmap *regmap; + struct regmap *regmap_battery; }; #endif /* __MFD_88PM886_H */ diff --git a/include/linux/mfd/arizona/pdata.h b/include/linux/mfd/arizona/pdata.h index f72e6d4b14a7..d465dcd8c90a 100644 --- a/include/linux/mfd/arizona/pdata.h +++ b/include/linux/mfd/arizona/pdata.h @@ -117,11 +117,6 @@ struct arizona_pdata { /** Check for line output with HPDET method */ bool hpdet_acc_id_line; -#ifdef CONFIG_GPIOLIB_LEGACY - /** GPIO used for mic isolation with HPDET */ - int hpdet_id_gpio; -#endif - /** Channel to use for headphone detection */ unsigned int hpdet_channel; @@ -131,11 +126,6 @@ struct arizona_pdata { /** Extra debounce timeout used during initial mic detection (ms) */ unsigned int micd_detect_debounce; -#ifdef CONFIG_GPIOLIB_LEGACY - /** GPIO for mic detection polarity */ - int micd_pol_gpio; -#endif - /** Mic detect ramp rate */ unsigned int micd_bias_start_time; diff --git a/include/linux/mfd/atmel-hlcdc.h b/include/linux/mfd/atmel-hlcdc.h index 80d675a03b39..07c2081867fd 100644 --- a/include/linux/mfd/atmel-hlcdc.h +++ b/include/linux/mfd/atmel-hlcdc.h @@ -75,6 +75,7 @@ */ struct atmel_hlcdc { struct regmap *regmap; + struct clk *lvds_pll_clk; struct clk *periph_clk; struct clk *sys_clk; struct clk *slow_clk; diff --git a/include/linux/mfd/bcm2835-pm.h b/include/linux/mfd/bcm2835-pm.h index f70a810c55f7..d2e17ab1dbfc 100644 --- a/include/linux/mfd/bcm2835-pm.h +++ b/include/linux/mfd/bcm2835-pm.h @@ -5,11 +5,18 @@ #include <linux/regmap.h> +enum bcm2835_soc { + BCM2835_PM_SOC_BCM2835, + BCM2835_PM_SOC_BCM2711, + BCM2835_PM_SOC_BCM2712, +}; + struct bcm2835_pm { struct device *dev; void __iomem *base; void __iomem *asb; void __iomem *rpivid_asb; + enum bcm2835_soc soc; }; #endif /* BCM2835_MFD_PM_H */ diff --git a/include/linux/mfd/bq257xx.h b/include/linux/mfd/bq257xx.h index 1d6ddc7fb09f..379ef4ee8291 100644 --- a/include/linux/mfd/bq257xx.h +++ b/include/linux/mfd/bq257xx.h @@ -98,7 +98,433 @@ #define BQ25703_EN_OTG_MASK BIT(12) +#define BQ25792_REG00_MIN_SYS_VOLTAGE 0x00 +#define BQ25792_REG01_CHARGE_VOLTAGE_LIMIT 0x01 +#define BQ25792_REG03_CHARGE_CURRENT_LIMIT 0x03 +#define BQ25792_REG05_INPUT_VOLTAGE_LIMIT 0x05 +#define BQ25792_REG06_INPUT_CURRENT_LIMIT 0x06 +#define BQ25792_REG08_PRECHARGE_CONTROL 0x08 +#define BQ25792_REG09_TERMINATION_CONTROL 0x09 +#define BQ25792_REG0A_RECHARGE_CONTROL 0x0a +#define BQ25792_REG0B_VOTG_REGULATION 0x0b +#define BQ25792_REG0D_IOTG_REGULATION 0x0d +#define BQ25792_REG0E_TIMER_CONTROL 0x0e +#define BQ25792_REG0F_CHARGER_CONTROL_0 0x0f +#define BQ25792_REG10_CHARGER_CONTROL_1 0x10 +#define BQ25792_REG11_CHARGER_CONTROL_2 0x11 +#define BQ25792_REG12_CHARGER_CONTROL_3 0x12 +#define BQ25792_REG13_CHARGER_CONTROL_4 0x13 +#define BQ25792_REG14_CHARGER_CONTROL_5 0x14 +/* REG15 reserved */ +#define BQ25792_REG16_TEMPERATURE_CONTROL 0x16 +#define BQ25792_REG17_NTC_CONTROL_0 0x17 +#define BQ25792_REG18_NTC_CONTROL_1 0x18 +#define BQ25792_REG19_ICO_CURRENT_LIMIT 0x19 +#define BQ25792_REG1B_CHARGER_STATUS_0 0x1b +#define BQ25792_REG1C_CHARGER_STATUS_1 0x1c +#define BQ25792_REG1D_CHARGER_STATUS_2 0x1d +#define BQ25792_REG1E_CHARGER_STATUS_3 0x1e +#define BQ25792_REG1F_CHARGER_STATUS_4 0x1f +#define BQ25792_REG20_FAULT_STATUS_0 0x20 +#define BQ25792_REG21_FAULT_STATUS_1 0x21 +#define BQ25792_REG22_CHARGER_FLAG_0 0x22 +#define BQ25792_REG23_CHARGER_FLAG_1 0x23 +#define BQ25792_REG24_CHARGER_FLAG_2 0x24 +#define BQ25792_REG25_CHARGER_FLAG_3 0x25 +#define BQ25792_REG26_FAULT_FLAG_0 0x26 +#define BQ25792_REG27_FAULT_FLAG_1 0x27 +#define BQ25792_REG28_CHARGER_MASK_0 0x28 +#define BQ25792_REG29_CHARGER_MASK_1 0x29 +#define BQ25792_REG2A_CHARGER_MASK_2 0x2a +#define BQ25792_REG2B_CHARGER_MASK_3 0x2b +#define BQ25792_REG2C_FAULT_MASK_0 0x2c +#define BQ25792_REG2D_FAULT_MASK_1 0x2d +#define BQ25792_REG2E_ADC_CONTROL 0x2e +#define BQ25792_REG2F_ADC_FUNCTION_DISABLE_0 0x2f +#define BQ25792_REG30_ADC_FUNCTION_DISABLE_1 0x30 +#define BQ25792_REG31_IBUS_ADC 0x31 +#define BQ25792_REG33_IBAT_ADC 0x33 +#define BQ25792_REG35_VBUS_ADC 0x35 +#define BQ25792_REG37_VAC1_ADC 0x37 +#define BQ25792_REG39_VAC2_ADC 0x39 +#define BQ25792_REG3B_VBAT_ADC 0x3b +#define BQ25792_REG3D_VSYS_ADC 0x3d +#define BQ25792_REG3F_TS_ADC 0x3f +#define BQ25792_REG41_TDIE_ADC 0x41 +#define BQ25792_REG43_DP_ADC 0x43 +#define BQ25792_REG45_DM_ADC 0x45 +#define BQ25792_REG47_DPDM_DRIVER 0x47 +#define BQ25792_REG48_PART_INFORMATION 0x48 + +/* Minimal System Voltage */ +#define BQ25792_REG00_VSYSMIN_MASK GENMASK(5, 0) + +#define BQ25792_MINVSYS_MIN_UV 2500000 +#define BQ25792_MINVSYS_STEP_UV 250000 +#define BQ25792_MINVSYS_MAX_UV 16000000 + +/* Charge Voltage Limit */ +#define BQ25792_REG01_VREG_MASK GENMASK(10, 0) + +#define BQ25792_VBATREG_MIN_UV 3000000 +#define BQ25792_VBATREG_STEP_UV 10000 +#define BQ25792_VBATREG_MAX_UV 18800000 + +/* Charge Current Limit */ +#define BQ25792_REG03_ICHG_MASK GENMASK(8, 0) + +#define BQ25792_ICHG_MIN_UA 50000 +#define BQ25792_ICHG_STEP_UA 10000 +#define BQ25792_ICHG_MAX_UA 5000000 + +/* Input Voltage Limit */ +#define BQ25792_REG05_VINDPM_MASK GENMASK(7, 0) + +/* Input Current Limit */ +#define BQ25792_REG06_IINDPM_MASK GENMASK(8, 0) +#define BQ25792_IINDPM_DEFAULT_UA 3000000 +#define BQ25792_IINDPM_STEP_UA 10000 +#define BQ25792_IINDPM_MIN_UA 100000 +#define BQ25792_IINDPM_MAX_UA 3300000 + +/* Precharge Control */ +#define BQ25792_REG08_VBAT_LOWV_MASK GENMASK(7, 6) +#define BQ25792_REG08_IPRECHG_MASK GENMASK(5, 0) + +/* Termination Control */ +#define BQ25792_REG09_REG_RST BIT(6) +#define BQ25792_REG09_ITERM_MASK GENMASK(4, 0) + +/* Re-charge Control */ +#define BQ25792_REG0A_CELL_MASK GENMASK(7, 6) +#define BQ25792_REG0A_TRECHG_MASK GENMASK(5, 4) +#define BQ25792_REG0A_VRECHG_MASK GENMASK(3, 0) + +#define BQ25792_CELL_1S 0 +#define BQ25792_CELL_2S 1 +#define BQ25792_CELL_3S 2 +#define BQ25792_CELL_4S 3 + +#define BQ25792_TRECHG_64MS 0 +#define BQ25792_TRECHG_256MS 1 +#define BQ25792_TRECHG_1024MS 2 +#define BQ25792_TRECHG_2048MS 3 + +#define BQ25792_VRECHG_MIN_UV 50000 +#define BQ25792_VRECHG_STEP_UV 50000 +#define BQ25792_VRECHG_MAX_UV 800000 + +/* VOTG regulation */ +#define BQ25792_REG0B_VOTG_MASK GENMASK(10, 0) + +#define BQ25792_OTG_VOLT_MIN_UV 2800000 +#define BQ25792_OTG_VOLT_STEP_UV 10000 +#define BQ25792_OTG_VOLT_MAX_UV 22000000 +#define BQ25792_OTG_VOLT_NUM_VOLT ((BQ25792_OTG_VOLT_MAX_UV \ + - BQ25792_OTG_VOLT_MIN_UV) \ + / BQ25792_OTG_VOLT_STEP_UV + 1) + +/* IOTG regulation */ +#define BQ25792_REG0D_PRECHG_TMR BIT(7) +#define BQ25792_REG0D_IOTG_MASK GENMASK(6, 0) + +#define BQ25792_OTG_CUR_MIN_UA 120000 +#define BQ25792_OTG_CUR_STEP_UA 40000 +#define BQ25792_OTG_CUR_MAX_UA 3320000 + +/* Timer Control */ +#define BQ25792_REG0E_TOPOFF_TMR_MASK GENMASK(7, 6) +#define BQ25792_REG0E_EN_TRICHG_TMR BIT(5) +#define BQ25792_REG0E_EN_PRECHG_TMR BIT(4) +#define BQ25792_REG0E_EN_CHG_TMR BIT(3) +#define BQ25792_REG0E_CHG_TMR_MASK GENMASK(2, 1) +#define BQ25792_REG0E_TMR2X_EN BIT(0) + +/* Charger Control 0 */ +#define BQ25792_REG0F_EN_AUTO_IBATDIS BIT(7) +#define BQ25792_REG0F_FORCE_IBATDIS BIT(6) +#define BQ25792_REG0F_EN_CHG BIT(5) +#define BQ25792_REG0F_EN_ICO BIT(4) +#define BQ25792_REG0F_FORCE_ICO BIT(3) +#define BQ25792_REG0F_EN_HIZ BIT(2) +#define BQ25792_REG0F_EN_TERM BIT(1) +/* bit0 reserved */ + +/* Charger Control 1 */ +#define BQ25792_REG10_VAC_OVP_MASK GENMASK(5, 4) +#define BQ25792_REG10_WD_RST BIT(3) +#define BQ25792_REG10_WATCHDOG_MASK GENMASK(2, 0) + +/* Charger Control 2 */ +#define BQ25792_REG11_FORCE_INDET BIT(7) +#define BQ25792_REG11_AUTO_INDET_EN BIT(6) +#define BQ25792_REG11_EN_12V BIT(5) +#define BQ25792_REG11_EN_9V BIT(4) +#define BQ25792_REG11_HVDCP_EN BIT(3) +#define BQ25792_REG11_SDRV_CTRL_MASK GENMASK(2, 1) +#define BQ25792_REG11_SDRV_DLY BIT(0) + +/* Charger Control 3 */ +#define BQ25792_REG12_DIS_ACDRV BIT(7) +#define BQ25792_REG12_EN_OTG BIT(6) +#define BQ25792_REG12_PFM_OTG_DIS BIT(5) +#define BQ25792_REG12_PFM_FWD_DIS BIT(4) +#define BQ25792_REG12_WKUP_DLY BIT(3) +#define BQ25792_REG12_DIS_LDO BIT(2) +#define BQ25792_REG12_DIS_OTG_OOA BIT(1) +#define BQ25792_REG12_DIS_FWD_OOA BIT(0) + +/* Charger Control 4 */ +#define BQ25792_REG13_EN_ACDRV2 BIT(7) +#define BQ25792_REG13_EN_ACDRV1 BIT(6) +#define BQ25792_REG13_PWM_FREQ BIT(5) +#define BQ25792_REG13_DIS_STAT BIT(4) +#define BQ25792_REG13_DIS_VSYS_SHORT BIT(3) +#define BQ25792_REG13_DIS_VOTG_UVP BIT(2) +#define BQ25792_REG13_FORCE_VINDPM_DET BIT(1) +#define BQ25792_REG13_EN_IBUS_OCP BIT(0) + +/* Charger Control 5 */ +#define BQ25792_REG14_SFET_PRESENT BIT(7) +/* bit6 reserved */ +#define BQ25792_REG14_EN_IBAT BIT(5) +#define BQ25792_REG14_IBAT_REG_MASK GENMASK(4, 3) +#define BQ25792_REG14_EN_IINDPM BIT(2) +#define BQ25792_REG14_EN_EXTILIM BIT(1) +#define BQ25792_REG14_EN_BATOC BIT(0) + +#define BQ25792_IBAT_3A FIELD_PREP(BQ25792_REG14_IBAT_REG_MASK, 0) +#define BQ25792_IBAT_4A FIELD_PREP(BQ25792_REG14_IBAT_REG_MASK, 1) +#define BQ25792_IBAT_5A FIELD_PREP(BQ25792_REG14_IBAT_REG_MASK, 2) +#define BQ25792_IBAT_UNLIM FIELD_PREP(BQ25792_REG14_IBAT_REG_MASK, 3) + +/* Temperature Control */ +#define BQ25792_REG16_TREG_MASK GENMASK(7, 6) +#define BQ25792_REG16_TSHUT_MASK GENMASK(5, 4) +#define BQ25792_REG16_VBUS_PD_EN BIT(3) +#define BQ25792_REG16_VAC1_PD_EN BIT(2) +#define BQ25792_REG16_VAC2_PD_EN BIT(1) + +/* NTC Control 0 */ +#define BQ25792_REG17_JEITA_VSET_MASK GENMASK(7, 5) +#define BQ25792_REG17_JEITA_ISETH_MASK GENMASK(4, 3) +#define BQ25792_REG17_JEITA_ISETC_MASK GENMASK(2, 1) + +/* NTC Control 1 */ +#define BQ25792_REG18_TS_COOL_MASK GENMASK(7, 6) +#define BQ25792_REG18_TS_WARM_MASK GENMASK(5, 4) +#define BQ25792_REG18_BHOT_MASK GENMASK(3, 2) +#define BQ25792_REG18_BCOLD BIT(1) +#define BQ25792_REG18_TS_IGNORE BIT(0) + +/* ICO Current Limit */ +#define BQ25792_REG19_ICO_ILIM_MASK GENMASK(8, 0) + +/* Charger Status 0 */ +#define BQ25792_REG1B_IINDPM_STAT BIT(7) +#define BQ25792_REG1B_VINDPM_STAT BIT(6) +#define BQ25792_REG1B_WD_STAT BIT(5) +#define BQ25792_REG1B_POORSRC_STAT BIT(4) +#define BQ25792_REG1B_PG_STAT BIT(3) +#define BQ25792_REG1B_AC2_PRESENT_STAT BIT(2) +#define BQ25792_REG1B_AC1_PRESENT_STAT BIT(1) +#define BQ25792_REG1B_VBUS_PRESENT_STAT BIT(0) + +/* Charger Status 1 */ +#define BQ25792_REG1C_CHG_STAT_MASK GENMASK(7, 5) +#define BQ25792_REG1C_VBUS_STAT_MASK GENMASK(4, 1) +#define BQ25792_REG1C_BC12_DONE_STAT BIT(0) + +/* Charger Status 2 */ +#define BQ25792_REG1D_ICO_STAT_MASK GENMASK(7, 6) +#define BQ25792_REG1D_TREG_STAT BIT(2) +#define BQ25792_REG1D_DPDM_STAT BIT(1) +#define BQ25792_REG1D_VBAT_PRESENT_STAT BIT(0) + +/* Charger Status 3 */ +#define BQ25792_REG1E_ACRB2_STAT BIT(7) +#define BQ25792_REG1E_ACRB1_STAT BIT(6) +#define BQ25792_REG1E_ADC_DONE_STAT BIT(5) +#define BQ25792_REG1E_VSYS_STAT BIT(4) +#define BQ25792_REG1E_CHG_TMR_STAT BIT(3) +#define BQ25792_REG1E_TRICHG_TMR_STAT BIT(2) +#define BQ25792_REG1E_PRECHG_TMR_STAT BIT(1) + +/* Charger Status 4 */ +#define BQ25792_REG1F_VBATOTG_LOW_STAT BIT(4) +#define BQ25792_REG1F_TS_COLD_STAT BIT(3) +#define BQ25792_REG1F_TS_COOL_STAT BIT(2) +#define BQ25792_REG1F_TS_WARM_STAT BIT(1) +#define BQ25792_REG1F_TS_HOT_STAT BIT(0) + +/* FAULT Status 0 */ +#define BQ25792_REG20_IBAT_REG_STAT BIT(7) +#define BQ25792_REG20_VBUS_OVP_STAT BIT(6) +#define BQ25792_REG20_VBAT_OVP_STAT BIT(5) +#define BQ25792_REG20_IBUS_OCP_STAT BIT(4) +#define BQ25792_REG20_IBAT_OCP_STAT BIT(3) +#define BQ25792_REG20_CONV_OCP_STAT BIT(2) +#define BQ25792_REG20_VAC2_OVP_STAT BIT(1) +#define BQ25792_REG20_VAC1_OVP_STAT BIT(0) + +#define BQ25792_REG20_OVERVOLTAGE_MASK (BQ25792_REG20_VBAT_OVP_STAT | \ + BQ25792_REG20_VAC2_OVP_STAT | \ + BQ25792_REG20_VAC1_OVP_STAT) +#define BQ25792_REG20_OVERCURRENT_MASK (BQ25792_REG20_IBAT_OCP_STAT | \ + BQ25792_REG20_CONV_OCP_STAT) + +/* FAULT Status 1 */ +#define BQ25792_REG21_VSYS_SHORT_STAT BIT(7) +#define BQ25792_REG21_VSYS_OVP_STAT BIT(6) +#define BQ25792_REG21_OTG_OVP_STAT BIT(5) +#define BQ25792_REG21_OTG_UVP_STAT BIT(4) +#define BQ25792_REG21_TSHUT_STAT BIT(2) + + +/* Charger Flag 0 */ +#define BQ25792_REG22_IINDPM_FLAG BIT(7) +#define BQ25792_REG22_VINDPM_FLAG BIT(6) +#define BQ25792_REG22_WD_FLAG BIT(5) +#define BQ25792_REG22_POORSRC_FLAG BIT(4) +#define BQ25792_REG22_PG_FLAG BIT(3) +#define BQ25792_REG22_AC2_PRESENT_FLAG BIT(2) +#define BQ25792_REG22_AC1_PRESENT_FLAG BIT(1) +#define BQ25792_REG22_VBUS_PRESENT_FLAG BIT(0) + +/* Charger Flag 1 */ +#define BQ25792_REG23_CHG_FLAG BIT(7) +#define BQ25792_REG23_ICO_FLAG BIT(6) +#define BQ25792_REG23_VBUS_FLAG BIT(4) +#define BQ25792_REG23_TREG_FLAG BIT(2) +#define BQ25792_REG23_VBAT_PRESENT_FLAG BIT(1) +#define BQ25792_REG23_BC12_DONE_FLAG BIT(0) + +/* Charger Flag 2 */ +#define BQ25792_REG24_DPDM_DONE_FLAG BIT(6) +#define BQ25792_REG24_ADC_DONE_FLAG BIT(5) +#define BQ25792_REG24_VSYS_FLAG BIT(4) +#define BQ25792_REG24_CHG_TMR_FLAG BIT(3) +#define BQ25792_REG24_TRICHG_TMR_FLAG BIT(2) +#define BQ25792_REG24_PRECHG_TMR_FLAG BIT(1) +#define BQ25792_REG24_TOPOFF_TMR_FLAG BIT(0) + +/* Charger Flag 3 */ +#define BQ25792_REG25_VBATOTG_LOW_FLAG BIT(4) +#define BQ25792_REG25_TS_COLD_FLAG BIT(3) +#define BQ25792_REG25_TS_COOL_FLAG BIT(2) +#define BQ25792_REG25_TS_WARM_FLAG BIT(1) +#define BQ25792_REG25_TS_HOT_FLAG BIT(0) + +/* FAULT Flag 0 */ +#define BQ25792_REG26_IBAT_REG_FLAG BIT(7) +#define BQ25792_REG26_VBUS_OVP_FLAG BIT(6) +#define BQ25792_REG26_VBAT_OVP_FLAG BIT(5) +#define BQ25792_REG26_IBUS_OCP_FLAG BIT(4) +#define BQ25792_REG26_IBAT_OCP_FLAG BIT(3) +#define BQ25792_REG26_CONV_OCP_FLAG BIT(2) +#define BQ25792_REG26_VAC2_OVP_FLAG BIT(1) +#define BQ25792_REG26_VAC1_OVP_FLAG BIT(0) + +/* FAULT Flag 1 */ +#define BQ25792_REG27_VSYS_SHORT_FLAG BIT(7) +#define BQ25792_REG27_VSYS_OVP_FLAG BIT(6) +#define BQ25792_REG27_OTG_OVP_FLAG BIT(5) +#define BQ25792_REG27_OTG_UVP_FLAG BIT(4) +#define BQ25792_REG27_TSHUT_FLAG BIT(2) + +/* Charger Mask 0 */ +#define BQ25792_REG28_IINDPM_MASK BIT(7) +#define BQ25792_REG28_VINDPM_MASK BIT(6) +#define BQ25792_REG28_WD_MASK BIT(5) +#define BQ25792_REG28_POORSRC_MASK BIT(4) +#define BQ25792_REG28_PG_MASK BIT(3) +#define BQ25792_REG28_AC2_PRESENT_MASK BIT(2) +#define BQ25792_REG28_AC1_PRESENT_MASK BIT(1) +#define BQ25792_REG28_VBUS_PRESENT_MASK BIT(0) + +/* Charger Mask 1 */ +#define BQ25792_REG29_CHG_MASK BIT(7) +#define BQ25792_REG29_ICO_MASK BIT(6) +#define BQ25792_REG29_VBUS_MASK BIT(4) +#define BQ25792_REG29_TREG_MASK BIT(2) +#define BQ25792_REG29_VBAT_PRESENT_MASK BIT(1) +#define BQ25792_REG29_BC12_DONE_MASK BIT(0) + +/* Charger Mask 2 */ +#define BQ25792_REG2A_DPDM_DONE_MASK BIT(6) +#define BQ25792_REG2A_ADC_DONE_MASK BIT(5) +#define BQ25792_REG2A_VSYS_MASK BIT(4) +#define BQ25792_REG2A_CHG_TMR_MASK BIT(3) +#define BQ25792_REG2A_TRICHG_TMR_MASK BIT(2) +#define BQ25792_REG2A_PRECHG_TMR_MASK BIT(1) +#define BQ25792_REG2A_TOPOFF_TMR_MASK BIT(0) + +/* Charger Mask 3 */ +#define BQ25792_REG2B_VBATOTG_LOW_MASK BIT(4) +#define BQ25792_REG2B_TS_COLD_MASK BIT(3) +#define BQ25792_REG2B_TS_COOL_MASK BIT(2) +#define BQ25792_REG2B_TS_WARM_MASK BIT(1) +#define BQ25792_REG2B_TS_HOT_MASK BIT(0) + +/* FAULT Mask 0 */ +#define BQ25792_REG2C_IBAT_REG_MASK BIT(7) +#define BQ25792_REG2C_VBUS_OVP_MASK BIT(6) +#define BQ25792_REG2C_VBAT_OVP_MASK BIT(5) +#define BQ25792_REG2C_IBUS_OCP_MASK BIT(4) +#define BQ25792_REG2C_IBAT_OCP_MASK BIT(3) +#define BQ25792_REG2C_CONV_OCP_MASK BIT(2) +#define BQ25792_REG2C_VAC2_OVP_MASK BIT(1) +#define BQ25792_REG2C_VAC1_OVP_MASK BIT(0) + +/* FAULT Mask 1 */ +#define BQ25792_REG2D_VSYS_SHORT_MASK BIT(7) +#define BQ25792_REG2D_VSYS_OVP_MASK BIT(6) +#define BQ25792_REG2D_OTG_OVP_MASK BIT(5) +#define BQ25792_REG2D_OTG_UVP_MASK BIT(4) +#define BQ25792_REG2D_TSHUT_MASK BIT(2) + +/* ADC Control */ +#define BQ25792_REG2E_ADC_EN BIT(7) +#define BQ25792_REG2E_ADC_RATE BIT(6) +#define BQ25792_REG2E_ADC_SAMPLE_MASK GENMASK(5, 4) +#define BQ25792_REG2E_ADC_AVG BIT(3) +#define BQ25792_REG2E_ADC_AVG_INIT BIT(2) + +/* ADC Function Disable 0 */ +#define BQ25792_REG2F_IBUS_ADC_DIS BIT(7) +#define BQ25792_REG2F_IBAT_ADC_DIS BIT(6) +#define BQ25792_REG2F_VBUS_ADC_DIS BIT(5) +#define BQ25792_REG2F_VBAT_ADC_DIS BIT(4) +#define BQ25792_REG2F_VSYS_ADC_DIS BIT(3) +#define BQ25792_REG2F_TS_ADC_DIS BIT(2) +#define BQ25792_REG2F_TDIE_ADC_DIS BIT(1) + +/* ADC Function Disable 1 */ +#define BQ25792_REG30_DP_ADC_DIS BIT(7) +#define BQ25792_REG30_DM_ADC_DIS BIT(6) +#define BQ25792_REG30_VAC2_ADC_DIS BIT(5) +#define BQ25792_REG30_VAC1_ADC_DIS BIT(4) + +/* 0x31-0x45: ADC result registers (16-bit, RO): single full-width field */ + +#define BQ25792_ADCVSYSVBAT_STEP_UV 1000 +#define BQ25792_ADCIBAT_STEP_UA 1000 + +/* DPDM Driver */ +#define BQ25792_REG47_DPLUS_DAC_MASK GENMASK(7, 5) +#define BQ25792_REG47_DMINUS_DAC_MASK GENMASK(4, 2) + +/* Part Information */ +#define BQ25792_REG48_PN_MASK GENMASK(5, 3) +#define BQ25792_REG48_DEV_REV_MASK GENMASK(2, 0) + +enum bq257xx_type { + BQ25703A = 1, + BQ25792, +}; + struct bq257xx_device { struct i2c_client *client; struct regmap *regmap; + enum bq257xx_type type; }; diff --git a/include/linux/mfd/cgbc.h b/include/linux/mfd/cgbc.h index badbec4c7033..91f501e76c8f 100644 --- a/include/linux/mfd/cgbc.h +++ b/include/linux/mfd/cgbc.h @@ -26,8 +26,8 @@ struct cgbc_version { * @io_cmd: Pointer to the command IO memory * @session: Session id returned by the Board Controller * @dev: Pointer to kernel device structure - * @cgbc_version: Board Controller version structure - * @mutex: Board Controller mutex + * @version: Board Controller version structure + * @lock: Board Controller mutex */ struct cgbc_device_data { void __iomem *io_session; diff --git a/include/linux/mfd/cs42l43-regs.h b/include/linux/mfd/cs42l43-regs.h index c39a49269cb7..68831f113589 100644 --- a/include/linux/mfd/cs42l43-regs.h +++ b/include/linux/mfd/cs42l43-regs.h @@ -1181,4 +1181,80 @@ /* CS42L43_FW_MISSION_CTRL_MM_MCU_CFG_REG */ #define CS42L43_FW_MISSION_CTRL_MM_MCU_CFG_DISABLE_VAL 0xF05AA50F +/* CS42L43B VARIANT REGISTERS */ +#define CS42L43B_DEVID_VAL 0x0042A43B + +#define CS42L43B_DECIM_VOL_CTRL_CH1_CH2 0x00008280 +#define CS42L43B_DECIM_VOL_CTRL_CH3_CH4 0x00008284 + +#define CS42L43B_DECIM_VOL_CTRL_CH5_CH6 0x00008290 +#define CS42L43B_DECIM_VOL_CTRL_UPDATE 0x0000829C + +#define CS42L43B_DECIM_HPF_WNF_CTRL5 0x000082A0 +#define CS42L43B_DECIM_HPF_WNF_CTRL6 0x000082A4 + +#define CS42L43B_SWIRE_DP3_CH3_INPUT 0x0000C320 +#define CS42L43B_SWIRE_DP3_CH4_INPUT 0x0000C330 +#define CS42L43B_SWIRE_DP4_CH3_INPUT 0x0000C340 +#define CS42L43B_SWIRE_DP4_CH4_INPUT 0x0000C350 + +#define CS42L43B_ISRC1DEC3_INPUT1 0x0000C780 +#define CS42L43B_ISRC1DEC4_INPUT1 0x0000C790 +#define CS42L43B_ISRC2DEC3_INPUT1 0x0000C7A0 +#define CS42L43B_ISRC2DEC4_INPUT1 0x0000C7B0 + +#define CS42L43B_FW_MISSION_CTRL_NEED_CONFIGS 0x00117E00 +#define CS42L43B_FW_MISSION_CTRL_HAVE_CONFIGS 0x00117E04 +#define CS42L43B_FW_MISSION_CTRL_PATCH_START_ADDR_REG 0x00117E08 +#define CS42L43B_FW_MISSION_CTRL_MM_CTRL_SELECTION 0x00117E0C +#define CS42L43B_FW_MISSION_CTRL_MM_MCU_CFG_REG 0x00117E10 + +#define CS42L43B_MCU_SW_REV 0x00117314 +#define CS42L43B_PATCH_START_ADDR 0x00117318 +#define CS42L43B_CONFIG_SELECTION 0x0011731C +#define CS42L43B_NEED_CONFIGS 0x00117320 +#define CS42L43B_BOOT_STATUS 0x00117330 + +#define CS42L43B_FW_MISSION_CTRL_NEED_CONFIGS 0x00117E00 +#define CS42L43B_FW_MISSION_CTRL_HAVE_CONFIGS 0x00117E04 +#define CS42L43B_FW_MISSION_CTRL_PATCH_START_ADDR_REG 0x00117E08 +#define CS42L43B_FW_MISSION_CTRL_MM_CTRL_SELECTION 0x00117E0C +#define CS42L43B_FW_MISSION_CTRL_MM_MCU_CFG_REG 0x00117E10 + +#define CS42L43B_MCU_RAM_MAX 0x00117FFF + +/* CS42L43B_DECIM_DECIM_VOL_CTRL_CH5_CH6 */ +#define CS42L43B_DECIM6_MUTE_MASK 0x80000000 +#define CS42L43B_DECIM6_MUTE_SHIFT 31 +#define CS42L43B_DECIM6_VOL_MASK 0x3FC00000 +#define CS42L43B_DECIM6_VOL_SHIFT 22 +#define CS42L43B_DECIM6_PATH1_VOL_FALL_RATE_MASK 0x00380000 +#define CS42L43B_DECIM6_PATH1_VOL_FALL_RATE_SHIFT 19 +#define CS42L43B_DECIM6_PATH1_VOL_RISE_RATE_MASK 0x00070000 +#define CS42L43B_DECIM6_PATH1_VOL_RISE_RATE_SHIFT 16 +#define CS42L43B_DECIM5_MUTE_MASK 0x00008000 +#define CS42L43B_DECIM5_MUTE_SHIFT 15 +#define CS42L43B_DECIM5_VOL_MASK 0x00003FC0 +#define CS42L43B_DECIM5_VOL_SHIFT 6 +#define CS42L43B_DECIM5_PATH1_VOL_FALL_RATE_MASK 0x00000038 +#define CS42L43B_DECIM5_PATH1_VOL_FALL_RATE_SHIFT 3 +#define CS42L43B_DECIM5_PATH1_VOL_RISE_RATE_MASK 0x00000007 +#define CS42L43B_DECIM5_PATH1_VOL_RISE_RATE_SHIFT 0 + +/* CS42L43B_DECIM_VOL_CTRL_UPDATE */ +#define CS42L43B_DECIM6_PATH1_VOL_TRIG_MASK 0x00000800 +#define CS42L43B_DECIM6_PATH1_VOL_TRIG_SHIFT 11 +#define CS42L43B_DECIM5_PATH1_VOL_TRIG_MASK 0x00000100 +#define CS42L43B_DECIM5_PATH1_VOL_TRIG_SHIFT 8 +#define CS42L43B_DECIM4_VOL_UPDATE_MASK 0x00000020 +#define CS42L43B_DECIM4_VOL_UPDATE_SHIFT 5 + +/* CS42L43_ISRC1_CTRL..CS42L43_ISRC2_CTRL */ +#define CS42L43B_ISRC_DEC4_EN_MASK 0x00000008 +#define CS42L43B_ISRC_DEC4_EN_SHIFT 3 +#define CS42L43B_ISRC_DEC4_EN_WIDTH 1 +#define CS42L43B_ISRC_DEC3_EN_MASK 0x00000004 +#define CS42L43B_ISRC_DEC3_EN_SHIFT 2 +#define CS42L43B_ISRC_DEC3_EN_WIDTH 1 + #endif /* CS42L43_CORE_REGS_H */ diff --git a/include/linux/mfd/cs42l43.h b/include/linux/mfd/cs42l43.h index 2239d8585e78..8e993fb535e6 100644 --- a/include/linux/mfd/cs42l43.h +++ b/include/linux/mfd/cs42l43.h @@ -86,7 +86,6 @@ struct cs42l43 { struct regmap_irq_chip_data *irq_data; struct work_struct boot_work; - struct completion device_attach; struct completion device_detach; struct completion firmware_download; int firmware_error; @@ -96,8 +95,8 @@ struct cs42l43 { struct mutex pll_lock; bool sdw_pll_active; - bool attached; bool hw_lock; + long variant_id; }; #endif /* CS42L43_CORE_EXT_H */ diff --git a/include/linux/mfd/cs5535.h b/include/linux/mfd/cs5535.h new file mode 100644 index 000000000000..2e4ebf5d06af --- /dev/null +++ b/include/linux/mfd/cs5535.h @@ -0,0 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#ifndef __MFD_CS5535_H__ +#define __MFD_CS5535_H__ + +extern const struct software_node cs5535_gpio_swnode; + +#endif /* __MFD_CS5535_H__ */ diff --git a/include/linux/mfd/db8500-prcmu.h b/include/linux/mfd/db8500-prcmu.h index a62de3d155ed..c939c9a1170a 100644 --- a/include/linux/mfd/db8500-prcmu.h +++ b/include/linux/mfd/db8500-prcmu.h @@ -12,6 +12,9 @@ #include <linux/interrupt.h> #include <linux/bitops.h> +#include <linux/err.h> + +#include <dt-bindings/mfd/dbx500-prcmu.h> /* For clock identifiers */ /* * Registers @@ -24,6 +27,38 @@ #define DB8500_PRCM_DSI_SW_RESET_DSI1_SW_RESETN BIT(1) #define DB8500_PRCM_DSI_SW_RESET_DSI2_SW_RESETN BIT(2) +/* Offset for the firmware version within the TCPM */ +#define DB8500_PRCMU_FW_VERSION_OFFSET 0xA4 + +#define DB8500_PRCMU_LEGACY_OFFSET 0xDD4 + +/* + * CLKOUT sources + */ +#define PRCMU_CLKSRC_CLK38M 0x00 +#define PRCMU_CLKSRC_ACLK 0x01 +#define PRCMU_CLKSRC_SYSCLK 0x02 +#define PRCMU_CLKSRC_LCDCLK 0x03 +#define PRCMU_CLKSRC_SDMMCCLK 0x04 +#define PRCMU_CLKSRC_TVCLK 0x05 +#define PRCMU_CLKSRC_TIMCLK 0x06 +#define PRCMU_CLKSRC_CLK009 0x07 +/* These are only valid for CLKOUT1: */ +#define PRCMU_CLKSRC_SIAMMDSPCLK 0x40 +#define PRCMU_CLKSRC_I2CCLK 0x41 +#define PRCMU_CLKSRC_MSP02CLK 0x42 +#define PRCMU_CLKSRC_ARMPLL_OBSCLK 0x43 +#define PRCMU_CLKSRC_HSIRXCLK 0x44 +#define PRCMU_CLKSRC_HSITXCLK 0x45 +#define PRCMU_CLKSRC_ARMCLKFIX 0x46 +#define PRCMU_CLKSRC_HDMICLK 0x47 + +/* + * Definitions for controlling ESRAM0 in deep sleep. + */ +#define ESRAM0_DEEP_SLEEP_STATE_OFF 1 +#define ESRAM0_DEEP_SLEEP_STATE_RET 2 + /* This portion previously known as <mach/prcmu-fw-defs_v1.h> */ /** @@ -451,10 +486,173 @@ enum prcmu_power_status { PRCMU_ARMPENDINGIT_ER = 0x93, }; +/* PRCMU Wakeup defines */ +enum prcmu_wakeup_index { + PRCMU_WAKEUP_INDEX_RTC, + PRCMU_WAKEUP_INDEX_RTT0, + PRCMU_WAKEUP_INDEX_RTT1, + PRCMU_WAKEUP_INDEX_HSI0, + PRCMU_WAKEUP_INDEX_HSI1, + PRCMU_WAKEUP_INDEX_USB, + PRCMU_WAKEUP_INDEX_ABB, + PRCMU_WAKEUP_INDEX_ABB_FIFO, + PRCMU_WAKEUP_INDEX_ARM, + PRCMU_WAKEUP_INDEX_CD_IRQ, + NUM_PRCMU_WAKEUP_INDICES +}; + +#define PRCMU_WAKEUP(_name) (BIT(PRCMU_WAKEUP_INDEX_##_name)) + +/** + * enum prcmu_wdog_id - PRCMU watchdog IDs + * @PRCMU_WDOG_ALL: use all timers + * @PRCMU_WDOG_CPU1: use first CPU timer only + * @PRCMU_WDOG_CPU2: use second CPU timer conly + */ +enum prcmu_wdog_id { + PRCMU_WDOG_ALL = 0x00, + PRCMU_WDOG_CPU1 = 0x01, + PRCMU_WDOG_CPU2 = 0x02, +}; + +/** + * enum ape_opp - APE OPP states definition + * @APE_OPP_INIT: + * @APE_NO_CHANGE: The APE operating point is unchanged + * @APE_100_OPP: The new APE operating point is ape100opp + * @APE_50_OPP: 50% + * @APE_50_PARTLY_25_OPP: 50%, except some clocks at 25%. + */ +enum ape_opp { + APE_OPP_INIT = 0x00, + APE_NO_CHANGE = 0x01, + APE_100_OPP = 0x02, + APE_50_OPP = 0x03, + APE_50_PARTLY_25_OPP = 0xFF, +}; + +/** + * enum arm_opp - ARM OPP states definition + * @ARM_OPP_INIT: + * @ARM_NO_CHANGE: The ARM operating point is unchanged + * @ARM_100_OPP: The new ARM operating point is arm100opp + * @ARM_50_OPP: The new ARM operating point is arm50opp + * @ARM_MAX_OPP: Operating point is "max" (more than 100) + * @ARM_MAX_FREQ100OPP: Set max opp if available, else 100 + * @ARM_EXTCLK: The new ARM operating point is armExtClk + */ +enum arm_opp { + ARM_OPP_INIT = 0x00, + ARM_NO_CHANGE = 0x01, + ARM_100_OPP = 0x02, + ARM_50_OPP = 0x03, + ARM_MAX_OPP = 0x04, + ARM_MAX_FREQ100OPP = 0x05, + ARM_EXTCLK = 0x07 +}; + +/** + * enum ddr_opp - DDR OPP states definition + * @DDR_100_OPP: The new DDR operating point is ddr100opp + * @DDR_50_OPP: The new DDR operating point is ddr50opp + * @DDR_25_OPP: The new DDR operating point is ddr25opp + */ +enum ddr_opp { + DDR_100_OPP = 0x00, + DDR_50_OPP = 0x01, + DDR_25_OPP = 0x02, +}; + +/** + * enum ddr_pwrst - DDR power states definition + * @DDR_PWR_STATE_UNCHANGED: SDRAM and DDR controller state is unchanged + * @DDR_PWR_STATE_ON: + * @DDR_PWR_STATE_OFFLOWLAT: + * @DDR_PWR_STATE_OFFHIGHLAT: + */ +enum ddr_pwrst { + DDR_PWR_STATE_UNCHANGED = 0x00, + DDR_PWR_STATE_ON = 0x01, + DDR_PWR_STATE_OFFLOWLAT = 0x02, + DDR_PWR_STATE_OFFHIGHLAT = 0x03 +}; + /* * Definitions for autonomous power management configuration. */ +/* EPOD (power domain) IDs */ + +/* + * DB8500 EPODs + * - EPOD_ID_SVAMMDSP: power domain for SVA MMDSP + * - EPOD_ID_SVAPIPE: power domain for SVA pipe + * - EPOD_ID_SIAMMDSP: power domain for SIA MMDSP + * - EPOD_ID_SIAPIPE: power domain for SIA pipe + * - EPOD_ID_SGA: power domain for SGA + * - EPOD_ID_B2R2_MCDE: power domain for B2R2 and MCDE + * - EPOD_ID_ESRAM12: power domain for ESRAM 1 and 2 + * - EPOD_ID_ESRAM34: power domain for ESRAM 3 and 4 + * - NUM_EPOD_ID: number of power domains + * + * TODO: These should be prefixed. + */ +#define EPOD_ID_SVAMMDSP 0 +#define EPOD_ID_SVAPIPE 1 +#define EPOD_ID_SIAMMDSP 2 +#define EPOD_ID_SIAPIPE 3 +#define EPOD_ID_SGA 4 +#define EPOD_ID_B2R2_MCDE 5 +#define EPOD_ID_ESRAM12 6 +#define EPOD_ID_ESRAM34 7 +#define NUM_EPOD_ID 8 + +/* + * state definition for EPOD (power domain) + * - EPOD_STATE_NO_CHANGE: The EPOD should remain unchanged + * - EPOD_STATE_OFF: The EPOD is switched off + * - EPOD_STATE_RAMRET: The EPOD is switched off with its internal RAM in + * retention + * - EPOD_STATE_ON_CLK_OFF: The EPOD is switched on, clock is still off + * - EPOD_STATE_ON: Same as above, but with clock enabled + */ +#define EPOD_STATE_NO_CHANGE 0x00 +#define EPOD_STATE_OFF 0x01 +#define EPOD_STATE_RAMRET 0x02 +#define EPOD_STATE_ON_CLK_OFF 0x03 +#define EPOD_STATE_ON 0x04 + +#define PRCMU_FW_PROJECT_U8500 2 +#define PRCMU_FW_PROJECT_U8400 3 +#define PRCMU_FW_PROJECT_U9500 4 /* Customer specific */ +#define PRCMU_FW_PROJECT_U8500_MBB 5 +#define PRCMU_FW_PROJECT_U8500_C1 6 +#define PRCMU_FW_PROJECT_U8500_C2 7 +#define PRCMU_FW_PROJECT_U8500_C3 8 +#define PRCMU_FW_PROJECT_U8500_C4 9 +#define PRCMU_FW_PROJECT_U9500_MBL 10 +#define PRCMU_FW_PROJECT_U8500_SSG1 11 /* Samsung specific */ +#define PRCMU_FW_PROJECT_U8500_MBL2 12 /* Customer specific */ +#define PRCMU_FW_PROJECT_U8520 13 +#define PRCMU_FW_PROJECT_U8420 14 +#define PRCMU_FW_PROJECT_U8500_SSG2 15 /* Samsung specific */ +#define PRCMU_FW_PROJECT_U8420_SYSCLK 17 +#define PRCMU_FW_PROJECT_A9420 20 +/* [32..63] 9540 and derivatives */ +#define PRCMU_FW_PROJECT_U9540 32 +/* [64..95] 8540 and derivatives */ +#define PRCMU_FW_PROJECT_L8540 64 +/* [96..126] 8580 and derivatives */ +#define PRCMU_FW_PROJECT_L8580 96 + +#define PRCMU_FW_PROJECT_NAME_LEN 20 + +/* PRCMU QoS APE OPP class */ +#define PRCMU_QOS_APE_OPP 1 +#define PRCMU_QOS_DDR_OPP 2 +#define PRCMU_QOS_ARM_OPP 3 +#define PRCMU_QOS_DEFAULT_VALUE -1 + #define PRCMU_AUTO_PM_OFF 0 #define PRCMU_AUTO_PM_ON 1 @@ -469,6 +667,14 @@ enum prcmu_auto_pm_policy { PRCMU_AUTO_PM_POLICY_DSP_CLK_OFF_HWP_CLK_OFF, }; +struct prcmu_fw_version { + u32 project; /* Notice, project shifted with 8 on ux540 */ + u8 api_version; + u8 func_version; + u8 errata; + char project_name[PRCMU_FW_PROJECT_NAME_LEN]; +}; + /** * struct prcmu_auto_pm_config - Autonomous power management configuration. * @sia_auto_pm_enable: SIA autonomous pm enable. (PRCMU_AUTO_PM_{OFF,ON}) @@ -501,6 +707,9 @@ void prcmu_configure_auto_pm(struct prcmu_auto_pm_config *sleep, bool prcmu_is_auto_pm_enabled(void); int prcmu_config_clkout(u8 clkout, u8 source, u8 div); +unsigned long prcmu_clock_rate(u8 clock); +long prcmu_round_clock_rate(u8 clock, unsigned long rate); +int prcmu_set_clock_rate(u8 clock, unsigned long rate); int prcmu_set_clock_divider(u8 clock, u8 divider); int db8500_prcmu_config_hotdog(u8 threshold); int db8500_prcmu_config_hotmon(u8 low, u8 high); @@ -508,6 +717,8 @@ int db8500_prcmu_start_temp_sense(u16 cycles32k); int db8500_prcmu_stop_temp_sense(void); int prcmu_abb_read(u8 slave, u8 reg, u8 *value, u8 size); int prcmu_abb_write(u8 slave, u8 reg, u8 *value, u8 size); +int prcmu_abb_write_masked(u8 slave, u8 reg, u8 *value, + u8 *mask, u8 size); int prcmu_ac_wake_req(void); void prcmu_ac_sleep_req(void); @@ -610,6 +821,21 @@ static inline int prcmu_config_clkout(u8 clkout, u8 source, u8 div) return 0; } +static inline unsigned long prcmu_clock_rate(u8 clock) +{ + return 0; +} + +static inline long prcmu_round_clock_rate(u8 clock, unsigned long rate) +{ + return 0; +} + +static inline int prcmu_set_clock_rate(u8 clock, unsigned long rate) +{ + return 0; +} + static inline int prcmu_set_clock_divider(u8 clock, u8 divider) { return 0; @@ -637,12 +863,18 @@ static inline int db8500_prcmu_stop_temp_sense(void) static inline int prcmu_abb_read(u8 slave, u8 reg, u8 *value, u8 size) { - return -ENOSYS; + return -EINVAL; } static inline int prcmu_abb_write(u8 slave, u8 reg, u8 *value, u8 size) { - return -ENOSYS; + return -EINVAL; +} + +static inline int prcmu_abb_write_masked(u8 slave, u8 reg, + u8 *value, u8 *mask, u8 size) +{ + return -EINVAL; } static inline int prcmu_ac_wake_req(void) @@ -745,4 +977,20 @@ static inline void db8500_prcmu_write_masked(unsigned int reg, u32 mask, #endif /* !CONFIG_MFD_DB8500_PRCMU */ +static inline int prcmu_qos_add_requirement(int prcmu_qos_class, + char *name, s32 value) +{ + return 0; +} + +static inline int prcmu_qos_update_requirement(int prcmu_qos_class, + char *name, s32 new_value) +{ + return 0; +} + +static inline void prcmu_qos_remove_requirement(int prcmu_qos_class, char *name) +{ +} + #endif /* __MFD_DB8500_PRCMU_H */ diff --git a/include/linux/mfd/dbx500-prcmu.h b/include/linux/mfd/dbx500-prcmu.h deleted file mode 100644 index 828362b7860c..000000000000 --- a/include/linux/mfd/dbx500-prcmu.h +++ /dev/null @@ -1,575 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright (C) ST Ericsson SA 2011 - * - * STE Ux500 PRCMU API - */ -#ifndef __MACH_PRCMU_H -#define __MACH_PRCMU_H - -#include <linux/interrupt.h> -#include <linux/notifier.h> -#include <linux/err.h> - -#include <dt-bindings/mfd/dbx500-prcmu.h> /* For clock identifiers */ - -/* Offset for the firmware version within the TCPM */ -#define DB8500_PRCMU_FW_VERSION_OFFSET 0xA4 -#define DBX540_PRCMU_FW_VERSION_OFFSET 0xA8 - -/* PRCMU Wakeup defines */ -enum prcmu_wakeup_index { - PRCMU_WAKEUP_INDEX_RTC, - PRCMU_WAKEUP_INDEX_RTT0, - PRCMU_WAKEUP_INDEX_RTT1, - PRCMU_WAKEUP_INDEX_HSI0, - PRCMU_WAKEUP_INDEX_HSI1, - PRCMU_WAKEUP_INDEX_USB, - PRCMU_WAKEUP_INDEX_ABB, - PRCMU_WAKEUP_INDEX_ABB_FIFO, - PRCMU_WAKEUP_INDEX_ARM, - PRCMU_WAKEUP_INDEX_CD_IRQ, - NUM_PRCMU_WAKEUP_INDICES -}; -#define PRCMU_WAKEUP(_name) (BIT(PRCMU_WAKEUP_INDEX_##_name)) - -/* EPOD (power domain) IDs */ - -/* - * DB8500 EPODs - * - EPOD_ID_SVAMMDSP: power domain for SVA MMDSP - * - EPOD_ID_SVAPIPE: power domain for SVA pipe - * - EPOD_ID_SIAMMDSP: power domain for SIA MMDSP - * - EPOD_ID_SIAPIPE: power domain for SIA pipe - * - EPOD_ID_SGA: power domain for SGA - * - EPOD_ID_B2R2_MCDE: power domain for B2R2 and MCDE - * - EPOD_ID_ESRAM12: power domain for ESRAM 1 and 2 - * - EPOD_ID_ESRAM34: power domain for ESRAM 3 and 4 - * - NUM_EPOD_ID: number of power domains - * - * TODO: These should be prefixed. - */ -#define EPOD_ID_SVAMMDSP 0 -#define EPOD_ID_SVAPIPE 1 -#define EPOD_ID_SIAMMDSP 2 -#define EPOD_ID_SIAPIPE 3 -#define EPOD_ID_SGA 4 -#define EPOD_ID_B2R2_MCDE 5 -#define EPOD_ID_ESRAM12 6 -#define EPOD_ID_ESRAM34 7 -#define NUM_EPOD_ID 8 - -/* - * state definition for EPOD (power domain) - * - EPOD_STATE_NO_CHANGE: The EPOD should remain unchanged - * - EPOD_STATE_OFF: The EPOD is switched off - * - EPOD_STATE_RAMRET: The EPOD is switched off with its internal RAM in - * retention - * - EPOD_STATE_ON_CLK_OFF: The EPOD is switched on, clock is still off - * - EPOD_STATE_ON: Same as above, but with clock enabled - */ -#define EPOD_STATE_NO_CHANGE 0x00 -#define EPOD_STATE_OFF 0x01 -#define EPOD_STATE_RAMRET 0x02 -#define EPOD_STATE_ON_CLK_OFF 0x03 -#define EPOD_STATE_ON 0x04 - -/* - * CLKOUT sources - */ -#define PRCMU_CLKSRC_CLK38M 0x00 -#define PRCMU_CLKSRC_ACLK 0x01 -#define PRCMU_CLKSRC_SYSCLK 0x02 -#define PRCMU_CLKSRC_LCDCLK 0x03 -#define PRCMU_CLKSRC_SDMMCCLK 0x04 -#define PRCMU_CLKSRC_TVCLK 0x05 -#define PRCMU_CLKSRC_TIMCLK 0x06 -#define PRCMU_CLKSRC_CLK009 0x07 -/* These are only valid for CLKOUT1: */ -#define PRCMU_CLKSRC_SIAMMDSPCLK 0x40 -#define PRCMU_CLKSRC_I2CCLK 0x41 -#define PRCMU_CLKSRC_MSP02CLK 0x42 -#define PRCMU_CLKSRC_ARMPLL_OBSCLK 0x43 -#define PRCMU_CLKSRC_HSIRXCLK 0x44 -#define PRCMU_CLKSRC_HSITXCLK 0x45 -#define PRCMU_CLKSRC_ARMCLKFIX 0x46 -#define PRCMU_CLKSRC_HDMICLK 0x47 - -/** - * enum prcmu_wdog_id - PRCMU watchdog IDs - * @PRCMU_WDOG_ALL: use all timers - * @PRCMU_WDOG_CPU1: use first CPU timer only - * @PRCMU_WDOG_CPU2: use second CPU timer conly - */ -enum prcmu_wdog_id { - PRCMU_WDOG_ALL = 0x00, - PRCMU_WDOG_CPU1 = 0x01, - PRCMU_WDOG_CPU2 = 0x02, -}; - -/** - * enum ape_opp - APE OPP states definition - * @APE_OPP_INIT: - * @APE_NO_CHANGE: The APE operating point is unchanged - * @APE_100_OPP: The new APE operating point is ape100opp - * @APE_50_OPP: 50% - * @APE_50_PARTLY_25_OPP: 50%, except some clocks at 25%. - */ -enum ape_opp { - APE_OPP_INIT = 0x00, - APE_NO_CHANGE = 0x01, - APE_100_OPP = 0x02, - APE_50_OPP = 0x03, - APE_50_PARTLY_25_OPP = 0xFF, -}; - -/** - * enum arm_opp - ARM OPP states definition - * @ARM_OPP_INIT: - * @ARM_NO_CHANGE: The ARM operating point is unchanged - * @ARM_100_OPP: The new ARM operating point is arm100opp - * @ARM_50_OPP: The new ARM operating point is arm50opp - * @ARM_MAX_OPP: Operating point is "max" (more than 100) - * @ARM_MAX_FREQ100OPP: Set max opp if available, else 100 - * @ARM_EXTCLK: The new ARM operating point is armExtClk - */ -enum arm_opp { - ARM_OPP_INIT = 0x00, - ARM_NO_CHANGE = 0x01, - ARM_100_OPP = 0x02, - ARM_50_OPP = 0x03, - ARM_MAX_OPP = 0x04, - ARM_MAX_FREQ100OPP = 0x05, - ARM_EXTCLK = 0x07 -}; - -/** - * enum ddr_opp - DDR OPP states definition - * @DDR_100_OPP: The new DDR operating point is ddr100opp - * @DDR_50_OPP: The new DDR operating point is ddr50opp - * @DDR_25_OPP: The new DDR operating point is ddr25opp - */ -enum ddr_opp { - DDR_100_OPP = 0x00, - DDR_50_OPP = 0x01, - DDR_25_OPP = 0x02, -}; - -/* - * Definitions for controlling ESRAM0 in deep sleep. - */ -#define ESRAM0_DEEP_SLEEP_STATE_OFF 1 -#define ESRAM0_DEEP_SLEEP_STATE_RET 2 - -/** - * enum ddr_pwrst - DDR power states definition - * @DDR_PWR_STATE_UNCHANGED: SDRAM and DDR controller state is unchanged - * @DDR_PWR_STATE_ON: - * @DDR_PWR_STATE_OFFLOWLAT: - * @DDR_PWR_STATE_OFFHIGHLAT: - */ -enum ddr_pwrst { - DDR_PWR_STATE_UNCHANGED = 0x00, - DDR_PWR_STATE_ON = 0x01, - DDR_PWR_STATE_OFFLOWLAT = 0x02, - DDR_PWR_STATE_OFFHIGHLAT = 0x03 -}; - -#define DB8500_PRCMU_LEGACY_OFFSET 0xDD4 - -#define PRCMU_FW_PROJECT_U8500 2 -#define PRCMU_FW_PROJECT_U8400 3 -#define PRCMU_FW_PROJECT_U9500 4 /* Customer specific */ -#define PRCMU_FW_PROJECT_U8500_MBB 5 -#define PRCMU_FW_PROJECT_U8500_C1 6 -#define PRCMU_FW_PROJECT_U8500_C2 7 -#define PRCMU_FW_PROJECT_U8500_C3 8 -#define PRCMU_FW_PROJECT_U8500_C4 9 -#define PRCMU_FW_PROJECT_U9500_MBL 10 -#define PRCMU_FW_PROJECT_U8500_SSG1 11 /* Samsung specific */ -#define PRCMU_FW_PROJECT_U8500_MBL2 12 /* Customer specific */ -#define PRCMU_FW_PROJECT_U8520 13 -#define PRCMU_FW_PROJECT_U8420 14 -#define PRCMU_FW_PROJECT_U8500_SSG2 15 /* Samsung specific */ -#define PRCMU_FW_PROJECT_U8420_SYSCLK 17 -#define PRCMU_FW_PROJECT_A9420 20 -/* [32..63] 9540 and derivatives */ -#define PRCMU_FW_PROJECT_U9540 32 -/* [64..95] 8540 and derivatives */ -#define PRCMU_FW_PROJECT_L8540 64 -/* [96..126] 8580 and derivatives */ -#define PRCMU_FW_PROJECT_L8580 96 - -#define PRCMU_FW_PROJECT_NAME_LEN 20 -struct prcmu_fw_version { - u32 project; /* Notice, project shifted with 8 on ux540 */ - u8 api_version; - u8 func_version; - u8 errata; - char project_name[PRCMU_FW_PROJECT_NAME_LEN]; -}; - -#include <linux/mfd/db8500-prcmu.h> - -#if defined(CONFIG_UX500_SOC_DB8500) - -static inline void __init prcmu_early_init(void) -{ - db8500_prcmu_early_init(); -} - -static inline int prcmu_set_power_state(u8 state, bool keep_ulp_clk, - bool keep_ap_pll) -{ - return db8500_prcmu_set_power_state(state, keep_ulp_clk, - keep_ap_pll); -} - -static inline u8 prcmu_get_power_state_result(void) -{ - return db8500_prcmu_get_power_state_result(); -} - -static inline int prcmu_set_epod(u16 epod_id, u8 epod_state) -{ - return db8500_prcmu_set_epod(epod_id, epod_state); -} - -static inline void prcmu_enable_wakeups(u32 wakeups) -{ - db8500_prcmu_enable_wakeups(wakeups); -} - -static inline void prcmu_disable_wakeups(void) -{ - prcmu_enable_wakeups(0); -} - -static inline void prcmu_config_abb_event_readout(u32 abb_events) -{ - db8500_prcmu_config_abb_event_readout(abb_events); -} - -static inline void prcmu_get_abb_event_buffer(void __iomem **buf) -{ - db8500_prcmu_get_abb_event_buffer(buf); -} - -int prcmu_abb_read(u8 slave, u8 reg, u8 *value, u8 size); -int prcmu_abb_write(u8 slave, u8 reg, u8 *value, u8 size); -int prcmu_abb_write_masked(u8 slave, u8 reg, u8 *value, u8 *mask, u8 size); - -int prcmu_config_clkout(u8 clkout, u8 source, u8 div); - -static inline int prcmu_request_clock(u8 clock, bool enable) -{ - return db8500_prcmu_request_clock(clock, enable); -} - -unsigned long prcmu_clock_rate(u8 clock); -long prcmu_round_clock_rate(u8 clock, unsigned long rate); -int prcmu_set_clock_rate(u8 clock, unsigned long rate); - -static inline int prcmu_get_ddr_opp(void) -{ - return db8500_prcmu_get_ddr_opp(); -} - -static inline int prcmu_set_arm_opp(u8 opp) -{ - return db8500_prcmu_set_arm_opp(opp); -} - -static inline int prcmu_get_arm_opp(void) -{ - return db8500_prcmu_get_arm_opp(); -} - -static inline int prcmu_set_ape_opp(u8 opp) -{ - return db8500_prcmu_set_ape_opp(opp); -} - -static inline int prcmu_get_ape_opp(void) -{ - return db8500_prcmu_get_ape_opp(); -} - -static inline int prcmu_request_ape_opp_100_voltage(bool enable) -{ - return db8500_prcmu_request_ape_opp_100_voltage(enable); -} - -static inline void prcmu_system_reset(u16 reset_code) -{ - db8500_prcmu_system_reset(reset_code); -} - -static inline u16 prcmu_get_reset_code(void) -{ - return db8500_prcmu_get_reset_code(); -} - -int prcmu_ac_wake_req(void); -void prcmu_ac_sleep_req(void); -static inline void prcmu_modem_reset(void) -{ - db8500_prcmu_modem_reset(); -} - -static inline bool prcmu_is_ac_wake_requested(void) -{ - return db8500_prcmu_is_ac_wake_requested(); -} - -static inline int prcmu_config_esram0_deep_sleep(u8 state) -{ - return db8500_prcmu_config_esram0_deep_sleep(state); -} - -static inline int prcmu_config_hotdog(u8 threshold) -{ - return db8500_prcmu_config_hotdog(threshold); -} - -static inline int prcmu_config_hotmon(u8 low, u8 high) -{ - return db8500_prcmu_config_hotmon(low, high); -} - -static inline int prcmu_start_temp_sense(u16 cycles32k) -{ - return db8500_prcmu_start_temp_sense(cycles32k); -} - -static inline int prcmu_stop_temp_sense(void) -{ - return db8500_prcmu_stop_temp_sense(); -} - -static inline u32 prcmu_read(unsigned int reg) -{ - return db8500_prcmu_read(reg); -} - -static inline void prcmu_write(unsigned int reg, u32 value) -{ - db8500_prcmu_write(reg, value); -} - -static inline void prcmu_write_masked(unsigned int reg, u32 mask, u32 value) -{ - db8500_prcmu_write_masked(reg, mask, value); -} - -static inline int prcmu_enable_a9wdog(u8 id) -{ - return db8500_prcmu_enable_a9wdog(id); -} - -static inline int prcmu_disable_a9wdog(u8 id) -{ - return db8500_prcmu_disable_a9wdog(id); -} - -static inline int prcmu_kick_a9wdog(u8 id) -{ - return db8500_prcmu_kick_a9wdog(id); -} - -static inline int prcmu_load_a9wdog(u8 id, u32 timeout) -{ - return db8500_prcmu_load_a9wdog(id, timeout); -} - -static inline int prcmu_config_a9wdog(u8 num, bool sleep_auto_off) -{ - return db8500_prcmu_config_a9wdog(num, sleep_auto_off); -} -#else - -static inline void prcmu_early_init(void) {} - -static inline int prcmu_set_power_state(u8 state, bool keep_ulp_clk, - bool keep_ap_pll) -{ - return 0; -} - -static inline int prcmu_set_epod(u16 epod_id, u8 epod_state) -{ - return 0; -} - -static inline void prcmu_enable_wakeups(u32 wakeups) {} - -static inline void prcmu_disable_wakeups(void) {} - -static inline int prcmu_abb_read(u8 slave, u8 reg, u8 *value, u8 size) -{ - return -ENOSYS; -} - -static inline int prcmu_abb_write(u8 slave, u8 reg, u8 *value, u8 size) -{ - return -ENOSYS; -} - -static inline int prcmu_abb_write_masked(u8 slave, u8 reg, u8 *value, u8 *mask, - u8 size) -{ - return -ENOSYS; -} - -static inline int prcmu_config_clkout(u8 clkout, u8 source, u8 div) -{ - return 0; -} - -static inline int prcmu_request_clock(u8 clock, bool enable) -{ - return 0; -} - -static inline long prcmu_round_clock_rate(u8 clock, unsigned long rate) -{ - return 0; -} - -static inline int prcmu_set_clock_rate(u8 clock, unsigned long rate) -{ - return 0; -} - -static inline unsigned long prcmu_clock_rate(u8 clock) -{ - return 0; -} - -static inline int prcmu_set_ape_opp(u8 opp) -{ - return 0; -} - -static inline int prcmu_get_ape_opp(void) -{ - return APE_100_OPP; -} - -static inline int prcmu_request_ape_opp_100_voltage(bool enable) -{ - return 0; -} - -static inline int prcmu_set_arm_opp(u8 opp) -{ - return 0; -} - -static inline int prcmu_get_arm_opp(void) -{ - return ARM_100_OPP; -} - -static inline int prcmu_get_ddr_opp(void) -{ - return DDR_100_OPP; -} - -static inline void prcmu_system_reset(u16 reset_code) {} - -static inline u16 prcmu_get_reset_code(void) -{ - return 0; -} - -static inline int prcmu_ac_wake_req(void) -{ - return 0; -} - -static inline void prcmu_ac_sleep_req(void) {} - -static inline void prcmu_modem_reset(void) {} - -static inline bool prcmu_is_ac_wake_requested(void) -{ - return false; -} - -static inline int prcmu_config_esram0_deep_sleep(u8 state) -{ - return 0; -} - -static inline void prcmu_config_abb_event_readout(u32 abb_events) {} - -static inline void prcmu_get_abb_event_buffer(void __iomem **buf) -{ - *buf = NULL; -} - -static inline int prcmu_config_hotdog(u8 threshold) -{ - return 0; -} - -static inline int prcmu_config_hotmon(u8 low, u8 high) -{ - return 0; -} - -static inline int prcmu_start_temp_sense(u16 cycles32k) -{ - return 0; -} - -static inline int prcmu_stop_temp_sense(void) -{ - return 0; -} - -static inline u32 prcmu_read(unsigned int reg) -{ - return 0; -} - -static inline void prcmu_write(unsigned int reg, u32 value) {} - -static inline void prcmu_write_masked(unsigned int reg, u32 mask, u32 value) {} - -#endif - -static inline void prcmu_set(unsigned int reg, u32 bits) -{ - prcmu_write_masked(reg, bits, bits); -} - -static inline void prcmu_clear(unsigned int reg, u32 bits) -{ - prcmu_write_masked(reg, bits, 0); -} - -/* PRCMU QoS APE OPP class */ -#define PRCMU_QOS_APE_OPP 1 -#define PRCMU_QOS_DDR_OPP 2 -#define PRCMU_QOS_ARM_OPP 3 -#define PRCMU_QOS_DEFAULT_VALUE -1 - -static inline int prcmu_qos_add_requirement(int prcmu_qos_class, - char *name, s32 value) -{ - return 0; -} - -static inline int prcmu_qos_update_requirement(int prcmu_qos_class, - char *name, s32 new_value) -{ - return 0; -} - -static inline void prcmu_qos_remove_requirement(int prcmu_qos_class, char *name) -{ -} - -#endif /* __MACH_PRCMU_H */ diff --git a/include/linux/mfd/ezx-pcap.h b/include/linux/mfd/ezx-pcap.h deleted file mode 100644 index ea51b1cdca5a..000000000000 --- a/include/linux/mfd/ezx-pcap.h +++ /dev/null @@ -1,253 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * Copyright 2009 Daniel Ribeiro <drwyrm@gmail.com> - * - * For further information, please see http://wiki.openezx.org/PCAP2 - */ - -#ifndef EZX_PCAP_H -#define EZX_PCAP_H - -struct pcap_subdev { - int id; - const char *name; - void *platform_data; -}; - -struct pcap_platform_data { - unsigned int irq_base; - unsigned int config; - int gpio; - void (*init) (void *); /* board specific init */ - int num_subdevs; - struct pcap_subdev *subdevs; -}; - -struct pcap_chip; - -int ezx_pcap_write(struct pcap_chip *, u8, u32); -int ezx_pcap_read(struct pcap_chip *, u8, u32 *); -int ezx_pcap_set_bits(struct pcap_chip *, u8, u32, u32); -int pcap_to_irq(struct pcap_chip *, int); -int irq_to_pcap(struct pcap_chip *, int); -int pcap_adc_async(struct pcap_chip *, u8, u32, u8[], void *, void *); -void pcap_set_ts_bits(struct pcap_chip *, u32); - -#define PCAP_SECOND_PORT 1 -#define PCAP_CS_AH 2 - -#define PCAP_REGISTER_WRITE_OP_BIT 0x80000000 -#define PCAP_REGISTER_READ_OP_BIT 0x00000000 - -#define PCAP_REGISTER_VALUE_MASK 0x01ffffff -#define PCAP_REGISTER_ADDRESS_MASK 0x7c000000 -#define PCAP_REGISTER_ADDRESS_SHIFT 26 -#define PCAP_REGISTER_NUMBER 32 -#define PCAP_CLEAR_INTERRUPT_REGISTER 0x01ffffff -#define PCAP_MASK_ALL_INTERRUPT 0x01ffffff - -/* registers accessible by both pcap ports */ -#define PCAP_REG_ISR 0x0 /* Interrupt Status */ -#define PCAP_REG_MSR 0x1 /* Interrupt Mask */ -#define PCAP_REG_PSTAT 0x2 /* Processor Status */ -#define PCAP_REG_VREG2 0x6 /* Regulator Bank 2 Control */ -#define PCAP_REG_AUXVREG 0x7 /* Auxiliary Regulator Control */ -#define PCAP_REG_BATT 0x8 /* Battery Control */ -#define PCAP_REG_ADC 0x9 /* AD Control */ -#define PCAP_REG_ADR 0xa /* AD Result */ -#define PCAP_REG_CODEC 0xb /* Audio Codec Control */ -#define PCAP_REG_RX_AMPS 0xc /* RX Audio Amplifiers Control */ -#define PCAP_REG_ST_DAC 0xd /* Stereo DAC Control */ -#define PCAP_REG_BUSCTRL 0x14 /* Connectivity Control */ -#define PCAP_REG_PERIPH 0x15 /* Peripheral Control */ -#define PCAP_REG_LOWPWR 0x18 /* Regulator Low Power Control */ -#define PCAP_REG_TX_AMPS 0x1a /* TX Audio Amplifiers Control */ -#define PCAP_REG_GP 0x1b /* General Purpose */ -#define PCAP_REG_TEST1 0x1c -#define PCAP_REG_TEST2 0x1d -#define PCAP_REG_VENDOR_TEST1 0x1e -#define PCAP_REG_VENDOR_TEST2 0x1f - -/* registers accessible by pcap port 1 only (a1200, e2 & e6) */ -#define PCAP_REG_INT_SEL 0x3 /* Interrupt Select */ -#define PCAP_REG_SWCTRL 0x4 /* Switching Regulator Control */ -#define PCAP_REG_VREG1 0x5 /* Regulator Bank 1 Control */ -#define PCAP_REG_RTC_TOD 0xe /* RTC Time of Day */ -#define PCAP_REG_RTC_TODA 0xf /* RTC Time of Day Alarm */ -#define PCAP_REG_RTC_DAY 0x10 /* RTC Day */ -#define PCAP_REG_RTC_DAYA 0x11 /* RTC Day Alarm */ -#define PCAP_REG_MTRTMR 0x12 /* AD Monitor Timer */ -#define PCAP_REG_PWR 0x13 /* Power Control */ -#define PCAP_REG_AUXVREG_MASK 0x16 /* Auxiliary Regulator Mask */ -#define PCAP_REG_VENDOR_REV 0x17 -#define PCAP_REG_PERIPH_MASK 0x19 /* Peripheral Mask */ - -/* PCAP2 Interrupts */ -#define PCAP_NIRQS 23 -#define PCAP_IRQ_ADCDONE 0 /* ADC done port 1 */ -#define PCAP_IRQ_TS 1 /* Touch Screen */ -#define PCAP_IRQ_1HZ 2 /* 1HZ timer */ -#define PCAP_IRQ_WH 3 /* ADC above high limit */ -#define PCAP_IRQ_WL 4 /* ADC below low limit */ -#define PCAP_IRQ_TODA 5 /* Time of day alarm */ -#define PCAP_IRQ_USB4V 6 /* USB above 4V */ -#define PCAP_IRQ_ONOFF 7 /* On/Off button */ -#define PCAP_IRQ_ONOFF2 8 /* On/Off button 2 */ -#define PCAP_IRQ_USB1V 9 /* USB above 1V */ -#define PCAP_IRQ_MOBPORT 10 -#define PCAP_IRQ_MIC 11 /* Mic attach/HS button */ -#define PCAP_IRQ_HS 12 /* Headset attach */ -#define PCAP_IRQ_ST 13 -#define PCAP_IRQ_PC 14 /* Power Cut */ -#define PCAP_IRQ_WARM 15 -#define PCAP_IRQ_EOL 16 /* Battery End Of Life */ -#define PCAP_IRQ_CLK 17 -#define PCAP_IRQ_SYSRST 18 /* System Reset */ -#define PCAP_IRQ_DUMMY 19 -#define PCAP_IRQ_ADCDONE2 20 /* ADC done port 2 */ -#define PCAP_IRQ_SOFTRESET 21 -#define PCAP_IRQ_MNEXB 22 - -/* voltage regulators */ -#define V1 0 -#define V2 1 -#define V3 2 -#define V4 3 -#define V5 4 -#define V6 5 -#define V7 6 -#define V8 7 -#define V9 8 -#define V10 9 -#define VAUX1 10 -#define VAUX2 11 -#define VAUX3 12 -#define VAUX4 13 -#define VSIM 14 -#define VSIM2 15 -#define VVIB 16 -#define SW1 17 -#define SW2 18 -#define SW3 19 -#define SW1S 20 -#define SW2S 21 - -#define PCAP_BATT_DAC_MASK 0x000000ff -#define PCAP_BATT_DAC_SHIFT 0 -#define PCAP_BATT_B_FDBK (1 << 8) -#define PCAP_BATT_EXT_ISENSE (1 << 9) -#define PCAP_BATT_V_COIN_MASK 0x00003c00 -#define PCAP_BATT_V_COIN_SHIFT 10 -#define PCAP_BATT_I_COIN (1 << 14) -#define PCAP_BATT_COIN_CH_EN (1 << 15) -#define PCAP_BATT_EOL_SEL_MASK 0x000e0000 -#define PCAP_BATT_EOL_SEL_SHIFT 17 -#define PCAP_BATT_EOL_CMP_EN (1 << 20) -#define PCAP_BATT_BATT_DET_EN (1 << 21) -#define PCAP_BATT_THERMBIAS_CTRL (1 << 22) - -#define PCAP_ADC_ADEN (1 << 0) -#define PCAP_ADC_RAND (1 << 1) -#define PCAP_ADC_AD_SEL1 (1 << 2) -#define PCAP_ADC_AD_SEL2 (1 << 3) -#define PCAP_ADC_ADA1_MASK 0x00000070 -#define PCAP_ADC_ADA1_SHIFT 4 -#define PCAP_ADC_ADA2_MASK 0x00000380 -#define PCAP_ADC_ADA2_SHIFT 7 -#define PCAP_ADC_ATO_MASK 0x00003c00 -#define PCAP_ADC_ATO_SHIFT 10 -#define PCAP_ADC_ATOX (1 << 14) -#define PCAP_ADC_MTR1 (1 << 15) -#define PCAP_ADC_MTR2 (1 << 16) -#define PCAP_ADC_TS_M_MASK 0x000e0000 -#define PCAP_ADC_TS_M_SHIFT 17 -#define PCAP_ADC_TS_REF_LOWPWR (1 << 20) -#define PCAP_ADC_TS_REFENB (1 << 21) -#define PCAP_ADC_BATT_I_POLARITY (1 << 22) -#define PCAP_ADC_BATT_I_ADC (1 << 23) - -#define PCAP_ADC_BANK_0 0 -#define PCAP_ADC_BANK_1 1 -/* ADC bank 0 */ -#define PCAP_ADC_CH_COIN 0 -#define PCAP_ADC_CH_BATT 1 -#define PCAP_ADC_CH_BPLUS 2 -#define PCAP_ADC_CH_MOBPORTB 3 -#define PCAP_ADC_CH_TEMPERATURE 4 -#define PCAP_ADC_CH_CHARGER_ID 5 -#define PCAP_ADC_CH_AD6 6 -/* ADC bank 1 */ -#define PCAP_ADC_CH_AD7 0 -#define PCAP_ADC_CH_AD8 1 -#define PCAP_ADC_CH_AD9 2 -#define PCAP_ADC_CH_TS_X1 3 -#define PCAP_ADC_CH_TS_X2 4 -#define PCAP_ADC_CH_TS_Y1 5 -#define PCAP_ADC_CH_TS_Y2 6 - -#define PCAP_ADC_T_NOW 0 -#define PCAP_ADC_T_IN_BURST 1 -#define PCAP_ADC_T_OUT_BURST 2 - -#define PCAP_ADC_ATO_IN_BURST 6 -#define PCAP_ADC_ATO_OUT_BURST 0 - -#define PCAP_ADC_TS_M_XY 1 -#define PCAP_ADC_TS_M_PRESSURE 2 -#define PCAP_ADC_TS_M_PLATE_X 3 -#define PCAP_ADC_TS_M_PLATE_Y 4 -#define PCAP_ADC_TS_M_STANDBY 5 -#define PCAP_ADC_TS_M_NONTS 6 - -#define PCAP_ADR_ADD1_MASK 0x000003ff -#define PCAP_ADR_ADD1_SHIFT 0 -#define PCAP_ADR_ADD2_MASK 0x000ffc00 -#define PCAP_ADR_ADD2_SHIFT 10 -#define PCAP_ADR_ADINC1 (1 << 20) -#define PCAP_ADR_ADINC2 (1 << 21) -#define PCAP_ADR_ASC (1 << 22) -#define PCAP_ADR_ONESHOT (1 << 23) - -#define PCAP_BUSCTRL_FSENB (1 << 0) -#define PCAP_BUSCTRL_USB_SUSPEND (1 << 1) -#define PCAP_BUSCTRL_USB_PU (1 << 2) -#define PCAP_BUSCTRL_USB_PD (1 << 3) -#define PCAP_BUSCTRL_VUSB_EN (1 << 4) -#define PCAP_BUSCTRL_USB_PS (1 << 5) -#define PCAP_BUSCTRL_VUSB_MSTR_EN (1 << 6) -#define PCAP_BUSCTRL_VBUS_PD_ENB (1 << 7) -#define PCAP_BUSCTRL_CURRLIM (1 << 8) -#define PCAP_BUSCTRL_RS232ENB (1 << 9) -#define PCAP_BUSCTRL_RS232_DIR (1 << 10) -#define PCAP_BUSCTRL_SE0_CONN (1 << 11) -#define PCAP_BUSCTRL_USB_PDM (1 << 12) -#define PCAP_BUSCTRL_BUS_PRI_ADJ (1 << 24) - -/* leds */ -#define PCAP_LED0 0 -#define PCAP_LED1 1 -#define PCAP_BL0 2 -#define PCAP_BL1 3 -#define PCAP_LED_3MA 0 -#define PCAP_LED_4MA 1 -#define PCAP_LED_5MA 2 -#define PCAP_LED_9MA 3 -#define PCAP_LED_T_MASK 0xf -#define PCAP_LED_C_MASK 0x3 -#define PCAP_BL_MASK 0x1f -#define PCAP_BL0_SHIFT 0 -#define PCAP_LED0_EN (1 << 5) -#define PCAP_LED1_EN (1 << 6) -#define PCAP_LED0_T_SHIFT 7 -#define PCAP_LED1_T_SHIFT 11 -#define PCAP_LED0_C_SHIFT 15 -#define PCAP_LED1_C_SHIFT 17 -#define PCAP_BL1_SHIFT 20 - -/* RTC */ -#define PCAP_RTC_DAY_MASK 0x3fff -#define PCAP_RTC_TOD_MASK 0xffff -#define PCAP_RTC_PC_MASK 0x7 -#define SEC_PER_DAY 86400 - -#endif diff --git a/include/linux/mfd/kempld.h b/include/linux/mfd/kempld.h index 643c096b93ac..5d75071eaaea 100644 --- a/include/linux/mfd/kempld.h +++ b/include/linux/mfd/kempld.h @@ -37,6 +37,7 @@ #define KEMPLD_SPEC_GET_MINOR(x) (x & 0x0f) #define KEMPLD_SPEC_GET_MAJOR(x) ((x >> 4) & 0x0f) #define KEMPLD_IRQ_GPIO 0x35 +#define KEMPLD_IRQ_GPIO_MASK 0x0f #define KEMPLD_IRQ_I2C 0x36 #define KEMPLD_CFG 0x37 #define KEMPLD_CFG_GPIO_I2C_MUX (1 << 0) @@ -97,10 +98,10 @@ struct kempld_device_data { /** * struct kempld_platform_data - PLD hardware configuration structure * @pld_clock: PLD clock frequency - * @gpio_base GPIO base pin number + * @gpio_base: GPIO base pin number * @ioresource: IO addresses of the PLD - * @get_mutex: PLD specific get_mutex callback - * @release_mutex: PLD specific release_mutex callback + * @get_hardware_mutex: PLD specific get_mutex callback + * @release_hardware_mutex: PLD specific release_mutex callback * @get_info: PLD specific get_info callback * @register_cells: PLD specific register_cells callback */ diff --git a/include/linux/mfd/lp3943.h b/include/linux/mfd/lp3943.h index 402f01078fcc..5d2d172d3598 100644 --- a/include/linux/mfd/lp3943.h +++ b/include/linux/mfd/lp3943.h @@ -10,7 +10,7 @@ #ifndef __MFD_LP3943_H__ #define __MFD_LP3943_H__ -#include <linux/gpio.h> +#include <linux/gpio/consumer.h> #include <linux/regmap.h> /* Registers */ diff --git a/include/linux/mfd/lpc_ich.h b/include/linux/mfd/lpc_ich.h index 1fbda1f8967d..1819aa743c5c 100644 --- a/include/linux/mfd/lpc_ich.h +++ b/include/linux/mfd/lpc_ich.h @@ -37,4 +37,6 @@ struct lpc_ich_info { u8 use_gpio; }; +extern const struct software_node lpc_ich_gpio_swnode; + #endif diff --git a/include/linux/mfd/max14577-private.h b/include/linux/mfd/max14577-private.h index dd51a37fa37f..5957e15b568e 100644 --- a/include/linux/mfd/max14577-private.h +++ b/include/linux/mfd/max14577-private.h @@ -350,6 +350,9 @@ enum max77836_pmic_reg { #define MAX77836_CNFG1_LDO_PWRMD_SHIFT 6 #define MAX77836_CNFG1_LDO_TV_SHIFT 0 #define MAX77836_CNFG1_LDO_PWRMD_MASK (0x3 << MAX77836_CNFG1_LDO_PWRMD_SHIFT) +#define MAX77836_CNFG1_LDO_PWRMD_OFF (0x0 << MAX77836_CNFG1_LDO_PWRMD_SHIFT) +#define MAX77836_CNFG1_LDO_PWRMD_LPM (0x1 << MAX77836_CNFG1_LDO_PWRMD_SHIFT) +#define MAX77836_CNFG1_LDO_PWRMD_NORMAL (0x3 << MAX77836_CNFG1_LDO_PWRMD_SHIFT) #define MAX77836_CNFG1_LDO_TV_MASK (0x3f << MAX77836_CNFG1_LDO_TV_SHIFT) /* LDO1/LDO2 CONFIG2 register */ diff --git a/include/linux/mfd/max77759.h b/include/linux/mfd/max77759.h index c6face34e385..7c0b13219d51 100644 --- a/include/linux/mfd/max77759.h +++ b/include/linux/mfd/max77759.h @@ -59,35 +59,65 @@ #define MAX77759_MAXQ_REG_AP_DATAIN0 0xb1 #define MAX77759_MAXQ_REG_UIC_SWRST 0xe0 -#define MAX77759_CHGR_REG_CHG_INT 0xb0 -#define MAX77759_CHGR_REG_CHG_INT2 0xb1 -#define MAX77759_CHGR_REG_CHG_INT_MASK 0xb2 -#define MAX77759_CHGR_REG_CHG_INT2_MASK 0xb3 -#define MAX77759_CHGR_REG_CHG_INT_OK 0xb4 -#define MAX77759_CHGR_REG_CHG_DETAILS_00 0xb5 -#define MAX77759_CHGR_REG_CHG_DETAILS_01 0xb6 -#define MAX77759_CHGR_REG_CHG_DETAILS_02 0xb7 -#define MAX77759_CHGR_REG_CHG_DETAILS_03 0xb8 -#define MAX77759_CHGR_REG_CHG_CNFG_00 0xb9 -#define MAX77759_CHGR_REG_CHG_CNFG_01 0xba -#define MAX77759_CHGR_REG_CHG_CNFG_02 0xbb -#define MAX77759_CHGR_REG_CHG_CNFG_03 0xbc -#define MAX77759_CHGR_REG_CHG_CNFG_04 0xbd -#define MAX77759_CHGR_REG_CHG_CNFG_05 0xbe -#define MAX77759_CHGR_REG_CHG_CNFG_06 0xbf -#define MAX77759_CHGR_REG_CHG_CNFG_07 0xc0 -#define MAX77759_CHGR_REG_CHG_CNFG_08 0xc1 -#define MAX77759_CHGR_REG_CHG_CNFG_09 0xc2 -#define MAX77759_CHGR_REG_CHG_CNFG_10 0xc3 -#define MAX77759_CHGR_REG_CHG_CNFG_11 0xc4 -#define MAX77759_CHGR_REG_CHG_CNFG_12 0xc5 -#define MAX77759_CHGR_REG_CHG_CNFG_13 0xc6 -#define MAX77759_CHGR_REG_CHG_CNFG_14 0xc7 -#define MAX77759_CHGR_REG_CHG_CNFG_15 0xc8 -#define MAX77759_CHGR_REG_CHG_CNFG_16 0xc9 -#define MAX77759_CHGR_REG_CHG_CNFG_17 0xca -#define MAX77759_CHGR_REG_CHG_CNFG_18 0xcb -#define MAX77759_CHGR_REG_CHG_CNFG_19 0xcc +#define MAX77759_CHGR_REG_CHG_INT 0xb0 +#define MAX77759_CHGR_REG_CHG_INT_AICL BIT(7) +#define MAX77759_CHGR_REG_CHG_INT_CHGIN BIT(6) +#define MAX77759_CHGR_REG_CHG_INT_WCIN BIT(5) +#define MAX77759_CHGR_REG_CHG_INT_CHG BIT(4) +#define MAX77759_CHGR_REG_CHG_INT_BAT BIT(3) +#define MAX77759_CHGR_REG_CHG_INT_INLIM BIT(2) +#define MAX77759_CHGR_REG_CHG_INT_THM2 BIT(1) +#define MAX77759_CHGR_REG_CHG_INT_BYP BIT(0) +#define MAX77759_CHGR_REG_CHG_INT2 0xb1 +#define MAX77759_CHGR_REG_CHG_INT2_INSEL BIT(7) +#define MAX77759_CHGR_REG_CHG_INT2_SYS_UVLO1 BIT(6) +#define MAX77759_CHGR_REG_CHG_INT2_SYS_UVLO2 BIT(5) +#define MAX77759_CHGR_REG_CHG_INT2_BAT_OILO BIT(4) +#define MAX77759_CHGR_REG_CHG_INT2_CHG_STA_CC BIT(3) +#define MAX77759_CHGR_REG_CHG_INT2_CHG_STA_CV BIT(2) +#define MAX77759_CHGR_REG_CHG_INT2_CHG_STA_TO BIT(1) +#define MAX77759_CHGR_REG_CHG_INT2_CHG_STA_DONE BIT(0) +#define MAX77759_CHGR_REG_CHG_INT_MASK 0xb2 +#define MAX77759_CHGR_REG_CHG_INT2_MASK 0xb3 +#define MAX77759_CHGR_REG_CHG_INT_OK 0xb4 +#define MAX77759_CHGR_REG_CHG_DETAILS_00 0xb5 +#define MAX77759_CHGR_REG_CHG_DETAILS_00_CHGIN_DTLS GENMASK(6, 5) +#define MAX77759_CHGR_REG_CHG_DETAILS_01 0xb6 +#define MAX77759_CHGR_REG_CHG_DETAILS_01_BAT_DTLS GENMASK(6, 4) +#define MAX77759_CHGR_REG_CHG_DETAILS_01_CHG_DTLS GENMASK(3, 0) +#define MAX77759_CHGR_REG_CHG_DETAILS_02 0xb7 +#define MAX77759_CHGR_REG_CHG_DETAILS_02_CHGIN_STS BIT(5) +#define MAX77759_CHGR_REG_CHG_DETAILS_03 0xb8 +#define MAX77759_CHGR_REG_CHG_CNFG_00 0xb9 +#define MAX77759_CHGR_REG_CHG_CNFG_00_MODE GENMASK(3, 0) +#define MAX77759_CHGR_REG_CHG_CNFG_01 0xba +#define MAX77759_CHGR_REG_CHG_CNFG_02 0xbb +#define MAX77759_CHGR_REG_CHG_CNFG_02_CHGCC GENMASK(5, 0) +#define MAX77759_CHGR_REG_CHG_CNFG_03 0xbc +#define MAX77759_CHGR_REG_CHG_CNFG_04 0xbd +#define MAX77759_CHGR_REG_CHG_CNFG_04_CHG_CV_PRM GENMASK(5, 0) +#define MAX77759_CHGR_REG_CHG_CNFG_05 0xbe +#define MAX77759_CHGR_REG_CHG_CNFG_06 0xbf +#define MAX77759_CHGR_REG_CHG_CNFG_06_CHGPROT GENMASK(3, 2) +#define MAX77759_CHGR_REG_CHG_CNFG_07 0xc0 +#define MAX77759_CHGR_REG_CHG_CNFG_08 0xc1 +#define MAX77759_CHGR_REG_CHG_CNFG_09 0xc2 +#define MAX77759_CHGR_REG_CHG_CNFG_09_CHGIN_ILIM GENMASK(6, 0) +#define MAX77759_CHGR_REG_CHG_CNFG_10 0xc3 +#define MAX77759_CHGR_REG_CHG_CNFG_11 0xc4 +#define MAX77759_CHGR_REG_CHG_CNFG_12 0xc5 +/* Setting this enables the Wireless Charging input channel. */ +#define MAX77759_CHGR_REG_CHG_CNFG_12_WCINSEL BIT(6) +/* Setting this enables the CHGIN/USB input channel. */ +#define MAX77759_CHGR_REG_CHG_CNFG_12_CHGINSEL BIT(5) +#define MAX77759_CHGR_REG_CHG_CNFG_13 0xc6 +#define MAX77759_CHGR_REG_CHG_CNFG_14 0xc7 +#define MAX77759_CHGR_REG_CHG_CNFG_15 0xc8 +#define MAX77759_CHGR_REG_CHG_CNFG_16 0xc9 +#define MAX77759_CHGR_REG_CHG_CNFG_17 0xca +#define MAX77759_CHGR_REG_CHG_CNFG_18 0xcb +#define MAX77759_CHGR_REG_CHG_CNFG_18_WDTEN BIT(0) +#define MAX77759_CHGR_REG_CHG_CNFG_19 0xcc /* MaxQ opcodes for max77759_maxq_command() */ #define MAX77759_MAXQ_OPCODE_MAXLENGTH (MAX77759_MAXQ_REG_AP_DATAOUT32 - \ @@ -102,6 +132,84 @@ #define MAX77759_MAXQ_OPCODE_USER_SPACE_WRITE 0x82 /** + * enum max77759_chgr_chgin_dtls_status - Charger Input Status + * @MAX77759_CHGR_CHGIN_DTLS_VBUS_UNDERVOLTAGE: + * Charger input voltage (Vchgin) < Under Voltage Threshold (Vuvlo) + * @MAX77759_CHGR_CHGIN_DTLS_VBUS_MARGINAL_VOLTAGE: + * Vchgin > Vuvlo and Vchgin < (Battery Voltage (Vbatt) + system voltage (Vsys)) + * @MAX77759_CHGR_CHGIN_DTLS_VBUS_OVERVOLTAGE: + * Vchgin > Over Voltage threshold (Vovlo) + * @MAX77759_CHGR_CHGIN_DTLS_VBUS_VALID: + * Vchgin > Vuvlo, Vchgin < Vovlo and Vchgin > (Vsys + Vbatt) + */ +enum max77759_chgr_chgin_dtls_status { + MAX77759_CHGR_CHGIN_DTLS_VBUS_UNDERVOLTAGE, + MAX77759_CHGR_CHGIN_DTLS_VBUS_MARGINAL_VOLTAGE, + MAX77759_CHGR_CHGIN_DTLS_VBUS_OVERVOLTAGE, + MAX77759_CHGR_CHGIN_DTLS_VBUS_VALID, +}; + +/** + * enum max77759_chgr_bat_dtls_states - Battery Details + * @MAX77759_CHGR_BAT_DTLS_NO_BATT_CHG_SUSP: No battery and the charger suspended + * @MAX77759_CHGR_BAT_DTLS_DEAD_BATTERY: Vbatt < Vtrickle + * @MAX77759_CHGR_BAT_DTLS_BAT_CHG_TIMER_FAULT: Charging suspended due to timer fault + * @MAX77759_CHGR_BAT_DTLS_BAT_OKAY: Battery okay and Vbatt > Min Sys Voltage (Vsysmin) + * @MAX77759_CHGR_BAT_DTLS_BAT_UNDERVOLTAGE: Battery is okay. Vtrickle < Vbatt < Vsysmin + * @MAX77759_CHGR_BAT_DTLS_BAT_OVERVOLTAGE: Battery voltage > Overvoltage threshold + * @MAX77759_CHGR_BAT_DTLS_BAT_OVERCURRENT: Battery current exceeds overcurrent threshold + * @MAX77759_CHGR_BAT_DTLS_BAT_ONLY_MODE: Battery only mode and battery level not available + */ +enum max77759_chgr_bat_dtls_states { + MAX77759_CHGR_BAT_DTLS_NO_BATT_CHG_SUSP, + MAX77759_CHGR_BAT_DTLS_DEAD_BATTERY, + MAX77759_CHGR_BAT_DTLS_BAT_CHG_TIMER_FAULT, + MAX77759_CHGR_BAT_DTLS_BAT_OKAY, + MAX77759_CHGR_BAT_DTLS_BAT_UNDERVOLTAGE, + MAX77759_CHGR_BAT_DTLS_BAT_OVERVOLTAGE, + MAX77759_CHGR_BAT_DTLS_BAT_OVERCURRENT, + MAX77759_CHGR_BAT_DTLS_BAT_ONLY_MODE, +}; + +/** + * enum max77759_chgr_chg_dtls_states - Charger Details + * @MAX77759_CHGR_CHG_DTLS_PREQUAL: Charger in prequalification mode + * @MAX77759_CHGR_CHG_DTLS_CC: Charger in fast charge const curr mode + * @MAX77759_CHGR_CHG_DTLS_CV: Charger in fast charge const voltage mode + * @MAX77759_CHGR_CHG_DTLS_TO: Charger is in top off mode + * @MAX77759_CHGR_CHG_DTLS_DONE: Charger is done + * @MAX77759_CHGR_CHG_DTLS_RSVD_1: Reserved + * @MAX77759_CHGR_CHG_DTLS_TIMER_FAULT: Charger is in timer fault mode + * @MAX77759_CHGR_CHG_DTLS_SUSP_BATT_THM: Charger is suspended as battery removal detected + * @MAX77759_CHGR_CHG_DTLS_OFF: Charger is off. Input invalid or charger disabled + * @MAX77759_CHGR_CHG_DTLS_RSVD_2: Reserved + * @MAX77759_CHGR_CHG_DTLS_RSVD_3: Reserved + * @MAX77759_CHGR_CHG_DTLS_OFF_WDOG_TIMER: Charger is off as watchdog timer expired + * @MAX77759_CHGR_CHG_DTLS_SUSP_JEITA: Charger is in JEITA control mode + */ +enum max77759_chgr_chg_dtls_states { + MAX77759_CHGR_CHG_DTLS_PREQUAL, + MAX77759_CHGR_CHG_DTLS_CC, + MAX77759_CHGR_CHG_DTLS_CV, + MAX77759_CHGR_CHG_DTLS_TO, + MAX77759_CHGR_CHG_DTLS_DONE, + MAX77759_CHGR_CHG_DTLS_RSVD_1, + MAX77759_CHGR_CHG_DTLS_TIMER_FAULT, + MAX77759_CHGR_CHG_DTLS_SUSP_BATT_THM, + MAX77759_CHGR_CHG_DTLS_OFF, + MAX77759_CHGR_CHG_DTLS_RSVD_2, + MAX77759_CHGR_CHG_DTLS_RSVD_3, + MAX77759_CHGR_CHG_DTLS_OFF_WDOG_TIMER, + MAX77759_CHGR_CHG_DTLS_SUSP_JEITA, +}; + +enum max77759_chgr_mode { + MAX77759_CHGR_MODE_OFF = 0x0, + MAX77759_CHGR_MODE_CHG_BUCK_ON = 0x5, + MAX77759_CHGR_MODE_OTG_BOOST_ON = 0xA, +}; + +/** * struct max77759 - core max77759 internal data structure * * @regmap_top: Regmap for accessing TOP registers diff --git a/include/linux/mfd/mt6397/core.h b/include/linux/mfd/mt6397/core.h index b774c3a4bb62..340fc72e22aa 100644 --- a/include/linux/mfd/mt6397/core.h +++ b/include/linux/mfd/mt6397/core.h @@ -12,9 +12,9 @@ enum chip_id { MT6323_CHIP_ID = 0x23, - MT6328_CHIP_ID = 0x30, - MT6331_CHIP_ID = 0x20, - MT6332_CHIP_ID = 0x20, + MT6328_CHIP_ID = 0x28, + MT6331_CHIP_ID = 0x31, + MT6332_CHIP_ID = 0x32, MT6357_CHIP_ID = 0x57, MT6358_CHIP_ID = 0x58, MT6359_CHIP_ID = 0x59, diff --git a/include/linux/mfd/rohm-bd72720.h b/include/linux/mfd/rohm-bd72720.h index ae7343bcab06..d8ddbf232bb3 100644 --- a/include/linux/mfd/rohm-bd72720.h +++ b/include/linux/mfd/rohm-bd72720.h @@ -21,7 +21,6 @@ enum { BD72720_BUCK8, BD72720_BUCK9, BD72720_BUCK10, - BD72720_BUCK11, BD72720_LDO1, BD72720_LDO2, BD72720_LDO3, diff --git a/include/linux/mfd/rsmu.h b/include/linux/mfd/rsmu.h index 0379aa207428..2f27386a7122 100644 --- a/include/linux/mfd/rsmu.h +++ b/include/linux/mfd/rsmu.h @@ -19,7 +19,6 @@ enum rsmu_type { }; /** - * * struct rsmu_ddata - device data structure for sub devices. * * @dev: i2c/spi device. diff --git a/include/linux/mfd/samsung/core.h b/include/linux/mfd/samsung/core.h index 4480c631110a..6191f409de94 100644 --- a/include/linux/mfd/samsung/core.h +++ b/include/linux/mfd/samsung/core.h @@ -47,6 +47,7 @@ enum sec_device_type { S2MPS15X, S2MPU02, S2MPU05, + S2MU005, }; /** diff --git a/include/linux/mfd/samsung/irq.h b/include/linux/mfd/samsung/irq.h index 6eab95de6fa8..19d0f0e12944 100644 --- a/include/linux/mfd/samsung/irq.h +++ b/include/linux/mfd/samsung/irq.h @@ -408,6 +408,72 @@ enum s2mpu05_irq { #define S2MPU05_IRQ_INT140C_MASK BIT(1) #define S2MPU05_IRQ_TSD_MASK BIT(2) +enum s2mu005_irq { + S2MU005_IRQ_CHGR_DETBAT, + S2MU005_IRQ_CHGR_BAT, + S2MU005_IRQ_CHGR_IVR, + S2MU005_IRQ_CHGR_EVENT, + S2MU005_IRQ_CHGR_CHG, + S2MU005_IRQ_CHGR_VMID, + S2MU005_IRQ_CHGR_WCIN, + S2MU005_IRQ_CHGR_VBUS, + + S2MU005_IRQ_FLED_LBPROT, + S2MU005_IRQ_FLED_OPENCH2, + S2MU005_IRQ_FLED_OPENCH1, + S2MU005_IRQ_FLED_SHORTCH2, + S2MU005_IRQ_FLED_SHORTCH1, + + S2MU005_IRQ_MUIC_ATTACH, + S2MU005_IRQ_MUIC_DETACH, + S2MU005_IRQ_MUIC_KP, + S2MU005_IRQ_MUIC_LKP, + S2MU005_IRQ_MUIC_LKR, + S2MU005_IRQ_MUIC_RIDCHG, + + S2MU005_IRQ_MUIC_VBUSON, + S2MU005_IRQ_MUIC_RSVD, + S2MU005_IRQ_MUIC_ADC, + S2MU005_IRQ_MUIC_STUCK, + S2MU005_IRQ_MUIC_STUCKRCV, + S2MU005_IRQ_MUIC_MHDL, + S2MU005_IRQ_MUIC_AVCHG, + S2MU005_IRQ_MUIC_VBUSOFF, + + S2MU005_IRQ_NR, +}; + +#define S2MU005_IRQ_CHGR_DETBAT_MASK BIT(0) +#define S2MU005_IRQ_CHGR_BAT_MASK BIT(1) +#define S2MU005_IRQ_CHGR_IVR_MASK BIT(2) +#define S2MU005_IRQ_CHGR_EVENT_MASK BIT(3) +#define S2MU005_IRQ_CHGR_CHG_MASK BIT(4) +#define S2MU005_IRQ_CHGR_VMID_MASK BIT(5) +#define S2MU005_IRQ_CHGR_WCIN_MASK BIT(6) +#define S2MU005_IRQ_CHGR_VBUS_MASK BIT(7) + +#define S2MU005_IRQ_FLED_LBPROT_MASK BIT(2) +#define S2MU005_IRQ_FLED_OPENCH2_MASK BIT(4) +#define S2MU005_IRQ_FLED_OPENCH1_MASK BIT(5) +#define S2MU005_IRQ_FLED_SHORTCH2_MASK BIT(6) +#define S2MU005_IRQ_FLED_SHORTCH1_MASK BIT(7) + +#define S2MU005_IRQ_MUIC_ATTACH_MASK BIT(0) +#define S2MU005_IRQ_MUIC_DETACH_MASK BIT(1) +#define S2MU005_IRQ_MUIC_KP_MASK BIT(2) +#define S2MU005_IRQ_MUIC_LKP_MASK BIT(3) +#define S2MU005_IRQ_MUIC_LKR_MASK BIT(4) +#define S2MU005_IRQ_MUIC_RIDCHG_MASK BIT(5) + +#define S2MU005_IRQ_MUIC_VBUSON_MASK BIT(0) +#define S2MU005_IRQ_MUIC_RSVD_MASK BIT(1) +#define S2MU005_IRQ_MUIC_ADC_MASK BIT(2) +#define S2MU005_IRQ_MUIC_STUCK_MASK BIT(3) +#define S2MU005_IRQ_MUIC_STUCKRCV_MASK BIT(4) +#define S2MU005_IRQ_MUIC_MHDL_MASK BIT(5) +#define S2MU005_IRQ_MUIC_AVCHG_MASK BIT(6) +#define S2MU005_IRQ_MUIC_VBUSOFF_MASK BIT(7) + enum s5m8767_irq { S5M8767_IRQ_PWRR, S5M8767_IRQ_PWRF, diff --git a/include/linux/mfd/samsung/s2mu005.h b/include/linux/mfd/samsung/s2mu005.h new file mode 100644 index 000000000000..46e7759545af --- /dev/null +++ b/include/linux/mfd/samsung/s2mu005.h @@ -0,0 +1,332 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* + * Copyright (c) 2015 Samsung Electronics Co., Ltd + * Copyright (c) 2026 Kaustabh Chakraborty <kauschluss@disroot.org> + * Copyright (c) 2026 Łukasz Lebiedziński <kernel@lvkasz.us> + */ + +#ifndef __LINUX_MFD_S2MU005_H +#define __LINUX_MFD_S2MU005_H + +#include <linux/bitfield.h> +#include <linux/bits.h> + +/* S2MU005 registers */ +enum s2mu005_reg { + S2MU005_REG_CHGR_INT1, + S2MU005_REG_CHGR_INT1M, + + S2MU005_REG_FLED_INT1, + S2MU005_REG_FLED_INT1M, + + S2MU005_REG_MUIC_INT1, + S2MU005_REG_MUIC_INT2, + S2MU005_REG_MUIC_INT1M, + S2MU005_REG_MUIC_INT2M, + + S2MU005_REG_CHGR_STATUS0, + S2MU005_REG_CHGR_STATUS1, + S2MU005_REG_CHGR_STATUS2, + S2MU005_REG_CHGR_STATUS3, + S2MU005_REG_CHGR_STATUS4, + S2MU005_REG_CHGR_STATUS5, + S2MU005_REG_CHGR_CTRL0, + S2MU005_REG_CHGR_CTRL1, + S2MU005_REG_CHGR_CTRL2, + S2MU005_REG_CHGR_CTRL3, + S2MU005_REG_CHGR_CTRL4, + S2MU005_REG_CHGR_CTRL5, + S2MU005_REG_CHGR_CTRL6, + S2MU005_REG_CHGR_CTRL7, + S2MU005_REG_CHGR_CTRL8, + S2MU005_REG_CHGR_CTRL9, + S2MU005_REG_CHGR_CTRL10, + S2MU005_REG_CHGR_CTRL11, + S2MU005_REG_CHGR_CTRL12, + S2MU005_REG_CHGR_CTRL13, + S2MU005_REG_CHGR_CTRL14, + S2MU005_REG_CHGR_CTRL15, + S2MU005_REG_CHGR_CTRL16, + S2MU005_REG_CHGR_CTRL17, + S2MU005_REG_CHGR_CTRL18, + S2MU005_REG_CHGR_CTRL19, + S2MU005_REG_CHGR_TEST0, + S2MU005_REG_CHGR_TEST1, + S2MU005_REG_CHGR_TEST2, + S2MU005_REG_CHGR_TEST3, + S2MU005_REG_CHGR_TEST4, + S2MU005_REG_CHGR_TEST5, + S2MU005_REG_CHGR_TEST6, + S2MU005_REG_CHGR_TEST7, + S2MU005_REG_CHGR_TEST8, + S2MU005_REG_CHGR_TEST9, + S2MU005_REG_CHGR_TEST10, + + S2MU005_REG_FLED_STATUS, + S2MU005_REG_FLED_CH0_CTRL0, + S2MU005_REG_FLED_CH0_CTRL1, + S2MU005_REG_FLED_CH0_CTRL2, + S2MU005_REG_FLED_CH0_CTRL3, + S2MU005_REG_FLED_CH1_CTRL0, + S2MU005_REG_FLED_CH1_CTRL1, + S2MU005_REG_FLED_CH1_CTRL2, + S2MU005_REG_FLED_CH1_CTRL3, + S2MU005_REG_FLED_CTRL0, + S2MU005_REG_FLED_CTRL1, + S2MU005_REG_FLED_CTRL2, + S2MU005_REG_FLED_CTRL3, + S2MU005_REG_FLED_CTRL4, + S2MU005_REG_FLED_CTRL5, + S2MU005_REG_FLED_CTRL6, + + S2MU005_REG_RGB_EN, + S2MU005_REG_RGB_CH0_CTRL, + S2MU005_REG_RGB_CH1_CTRL, + S2MU005_REG_RGB_CH2_CTRL, + S2MU005_REG_RGB_CH0_RAMP, + S2MU005_REG_RGB_CH0_STAY, + S2MU005_REG_RGB_CH1_RAMP, + S2MU005_REG_RGB_CH1_STAY, + S2MU005_REG_RGB_CH2_RAMP, + S2MU005_REG_RGB_CH2_STAY, + S2MU005_REG_RGB_TEST0, + S2MU005_REG_RGB_CTRL0, + + S2MU005_REG_MUIC_ADC, + S2MU005_REG_MUIC_DEV1, + S2MU005_REG_MUIC_DEV2, + S2MU005_REG_MUIC_DEV3, + S2MU005_REG_MUIC_BUTTON1, + S2MU005_REG_MUIC_BUTTON2, + S2MU005_REG_MUIC_RESET, + S2MU005_REG_MUIC_CHGTYPE, + S2MU005_REG_MUIC_DEVAPPLE, + S2MU005_REG_MUIC_BCDRESCAN, + S2MU005_REG_MUIC_TEST1, + S2MU005_REG_MUIC_TEST2, + S2MU005_REG_MUIC_TEST3, + + S2MU005_REG_ID = 0x73, + + S2MU005_REG_MUIC_CTRL1 = 0xb2, + S2MU005_REG_MUIC_TIMERSET1, + S2MU005_REG_MUIC_TIMERSET2, + S2MU005_REG_MUIC_SWCTRL, + S2MU005_REG_MUIC_TIMERSET3, + S2MU005_REG_MUIC_CTRL2, + S2MU005_REG_MUIC_CTRL3, + + S2MU005_REG_MUIC_LDOADC_L = 0xbf, + S2MU005_REG_MUIC_LDOADC_H, +}; + +#define S2MU005_REG_FLED_CH_CTRL0(x) (S2MU005_REG_FLED_CH0_CTRL0 + 4 * (x)) +#define S2MU005_REG_FLED_CH_CTRL1(x) (S2MU005_REG_FLED_CH0_CTRL1 + 4 * (x)) +#define S2MU005_REG_FLED_CH_CTRL2(x) (S2MU005_REG_FLED_CH0_CTRL2 + 4 * (x)) +#define S2MU005_REG_FLED_CH_CTRL3(x) (S2MU005_REG_FLED_CH0_CTRL3 + 4 * (x)) + +#define S2MU005_REG_RGB_CH_CTRL(x) (S2MU005_REG_RGB_CH0_CTRL + 1 * (x)) +#define S2MU005_REG_RGB_CH_RAMP(x) (S2MU005_REG_RGB_CH0_RAMP + 2 * (x)) +#define S2MU005_REG_RGB_CH_STAY(x) (S2MU005_REG_RGB_CH0_STAY + 2 * (x)) + +/* S2MU005_REG_CHGR_STATUS0 */ +#define S2MU005_CHGR_VBUS BIT(7) +#define S2MU005_CHGR_WCIN BIT(6) +#define S2MU005_CHGR_VMID BIT(5) +#define S2MU005_CHGR_CHG BIT(4) +#define S2MU005_CHGR_STAT GENMASK(3, 0) + +#define S2MU005_CHGR_STAT_DONE 8 +#define S2MU005_CHGR_STAT_TOPOFF 7 +#define S2MU005_CHGR_STAT_DONE_FLAG 6 +#define S2MU005_CHGR_STAT_CV 5 +#define S2MU005_CHGR_STAT_CC 4 +#define S2MU005_CHGR_STAT_COOL_CHG 3 +#define S2MU005_CHGR_STAT_PRE_CHG 2 + +/* S2MU005_REG_CHGR_STATUS1 */ +#define S2MU005_CHGR_DETBAT BIT(7) +#define S2MU005_CHGR_VBUS_OVP GENMASK(6, 4) + +#define S2MU005_CHGR_VBUS_OVP_OVERVOLT 2 + +/* S2MU005_REG_CHGR_STATUS2 */ +#define S2MU005_CHGR_BAT GENMASK(6, 4) + +#define S2MU005_CHGR_BAT_VOLT_DET 7 +#define S2MU005_CHGR_BAT_FAST_CHG_DET 6 +#define S2MU005_CHGR_BAT_COOL_CHG_DET 5 +#define S2MU005_CHGR_BAT_LOW_CHG 2 +#define S2MU005_CHGR_BAT_SELF_DISCHG 1 +#define S2MU005_CHGR_BAT_OVP_DET 0 + +/* S2MU005_REG_CHGR_STATUS3 */ +#define S2MU005_CHGR_EVT GENMASK(3, 0) + +#define S2MU005_CHGR_EVT_WDT_RST 6 +#define S2MU005_CHGR_EVT_WDT_SUSP 5 +#define S2MU005_CHGR_EVT_VSYS_VUVLO 4 +#define S2MU005_CHGR_EVT_VSYS_VOVP 3 +#define S2MU005_CHGR_EVT_THERM_FOLDBACK 2 +#define S2MU005_CHGR_EVT_THERM_SHUTDOWN 1 + +/* S2MU005_REG_CHGR_CTRL0 */ +#define S2MU005_CHGR_CHG_EN BIT(4) +#define S2MU005_CHGR_OP_MODE GENMASK(2, 0) + +#define S2MU005_CHGR_OP_MODE_OTG BIT(2) +#define S2MU005_CHGR_OP_MODE_CHG BIT(1) + +/* S2MU005_REG_CHGR_CTRL1 */ +#define S2MU005_CHGR_VIN_DROP GENMASK(6, 4) + +/* S2MU005_REG_CHGR_CTRL2 */ +#define S2MU005_CHGR_IN_CURR_LIM GENMASK(5, 0) + +/* S2MU005_REG_CHGR_CTRL4 */ +#define S2MU005_CHGR_OTG_OCP_ON BIT(5) +#define S2MU005_CHGR_OTG_OCP_OFF BIT(4) +#define S2MU005_CHGR_OTG_OCP GENMASK(3, 2) +#define S2MU005_CHGR_OTG_OCP_1P5A 0x3 + +/* S2MU005_REG_CHGR_CTRL5 */ +#define S2MU005_CHGR_VMID_BOOST GENMASK(4, 0) +#define S2MU005_CHGR_VMID_BOOST_5P1V 0x16 + +/* S2MU005_REG_CHGR_CTRL6 */ +#define S2MU005_CHGR_COOL_CHG_CURR GENMASK(5, 0) + +/* S2MU005_REG_CHGR_CTRL7 */ +#define S2MU005_CHGR_FAST_CHG_CURR GENMASK(5, 0) + +/* S2MU005_REG_CHGR_CTRL8 */ +#define S2MU005_CHGR_VF_VBAT GENMASK(6, 1) + +/* S2MU005_REG_CHGR_CTRL10 */ +#define S2MU005_CHGR_TOPOFF_CURR(x) (GENMASK(3, 0) << 4 * (x)) + +/* S2MU005_REG_CHGR_CTRL11 */ +#define S2MU005_CHGR_OSC_BOOST GENMASK(6, 5) +#define S2MU005_CHGR_OSC_BUCK GENMASK(4, 3) +#define S2MU005_CHGR_OSC_BOOST_2MHZ 0x3 + +/* S2MU005_REG_CHGR_CTRL12 */ +#define S2MU005_CHGR_WDT GENMASK(2, 0) + +#define S2MU005_CHGR_WDT_ON BIT(2) +#define S2MU005_CHGR_WDT_OFF BIT(1) + +/* S2MU005_REG_CHGR_CTRL15 */ +#define S2MU005_CHGR_OTG_EN GENMASK(3, 2) +#define S2MU005_CHGR_OTG_EN_ON 0x3 + +/* S2MU005_REG_FLED_STATUS */ +#define S2MU005_FLED_FLASH_STATUS(x) (BIT(7) >> 2 * (x)) +#define S2MU005_FLED_TORCH_STATUS(x) (BIT(6) >> 2 * (x)) + +/* S2MU005_REG_FLED_CHx_CTRL0 */ +#define S2MU005_FLED_FLASH_IOUT GENMASK(3, 0) + +/* S2MU005_REG_FLED_CHx_CTRL1 */ +#define S2MU005_FLED_TORCH_IOUT GENMASK(3, 0) + +/* S2MU005_REG_FLED_CHx_CTRL2 */ +#define S2MU005_FLED_TORCH_TIMEOUT GENMASK(3, 0) + +/* S2MU005_REG_FLED_CHx_CTRL3 */ +#define S2MU005_FLED_FLASH_TIMEOUT GENMASK(3, 0) + +/* S2MU005_REG_FLED_CTRL1 */ +#define S2MU005_FLED_CH_EN BIT(7) + +/* + * S2MU005_REG_FLED_CTRL4 - Rev. EVT0 + * S2MU005_REG_FLED_CTRL6 - Rev. EVT1 and later + */ +#define S2MU005_FLED_FLASH_EN(x) (GENMASK(7, 6) >> 4 * (x)) +#define S2MU005_FLED_TORCH_EN(x) (GENMASK(5, 4) >> 4 * (x)) + +/* S2MU005_REG_RGB_EN */ +#define S2MU005_RGB_RESET BIT(6) +#define S2MU005_RGB_SLOPE GENMASK(5, 0) + +#define S2MU005_RGB_SLOPE_CONST (BIT(4) | BIT(2) | BIT(0)) +#define S2MU005_RGB_SLOPE_SMOOTH (BIT(5) | BIT(3) | BIT(1)) + +/* S2MU005_REG_RGB_CHx_RAMP */ +#define S2MU005_RGB_CH_RAMP_UP GENMASK(7, 4) +#define S2MU005_RGB_CH_RAMP_DN GENMASK(3, 0) + +/* S2MU005_REG_RGB_CHx_STAY */ +#define S2MU005_RGB_CH_STAY_HI GENMASK(7, 4) +#define S2MU005_RGB_CH_STAY_LO GENMASK(3, 0) + +/* S2MU005_REG_MUIC_DEV1 */ +#define S2MU005_MUIC_OTG BIT(7) +#define S2MU005_MUIC_DCP BIT(6) +#define S2MU005_MUIC_CDP BIT(5) +#define S2MU005_MUIC_T1_T2_CHG BIT(4) +#define S2MU005_MUIC_UART BIT(3) +#define S2MU005_MUIC_SDP BIT(2) +#define S2MU005_MUIC_LANHUB BIT(1) +#define S2MU005_MUIC_AUDIO BIT(0) + +/* S2MU005_REG_MUIC_DEV2 */ +#define S2MU005_MUIC_SDP_1P8S BIT(7) +#define S2MU005_MUIC_AV BIT(6) +#define S2MU005_MUIC_TTY BIT(5) +#define S2MU005_MUIC_PPD BIT(4) +#define S2MU005_MUIC_JIG_UART_OFF BIT(3) +#define S2MU005_MUIC_JIG_UART_ON BIT(2) +#define S2MU005_MUIC_JIG_USB_OFF BIT(1) +#define S2MU005_MUIC_JIG_USB_ON BIT(0) + +/* S2MU005_REG_MUIC_DEV3 */ +#define S2MU005_MUIC_U200_CHG BIT(7) +#define S2MU005_MUIC_VBUS_AV BIT(4) +#define S2MU005_MUIC_VBUS_R255 BIT(1) +#define S2MU005_MUIC_MHL BIT(0) + +/* S2MU005_REG_MUIC_DEVAPPLE */ +#define S2MU005_MUIC_APPLE_CHG_0P5A BIT(7) +#define S2MU005_MUIC_APPLE_CHG_1P0A BIT(6) +#define S2MU005_MUIC_APPLE_CHG_2P0A BIT(5) +#define S2MU005_MUIC_APPLE_CHG_2P4A BIT(4) +#define S2MU005_MUIC_SDP_DCD_OUT BIT(3) +#define S2MU005_MUIC_RID_WAKEUP BIT(2) +#define S2MU005_MUIC_VBUS_WAKEUP BIT(1) +#define S2MU005_MUIC_BCV1P2_OR_OPEN BIT(0) + +/* S2MU005_REG_ID */ +#define S2MU005_ID_MASK GENMASK(3, 0) + +/* S2MU005_REG_MUIC_SWCTRL */ +#define S2MU005_MUIC_DM_DP GENMASK(7, 2) +#define S2MU005_MUIC_JIG BIT(0) + +#define S2MU005_MUIC_DM_DP_UART 0x12 +#define S2MU005_MUIC_DM_DP_USB 0x09 + +/* S2MU005_REG_MUIC_CTRL1 */ +#define S2MU005_MUIC_OPEN BIT(4) +#define S2MU005_MUIC_RAW_DATA BIT(3) +#define S2MU005_MUIC_MAN_SW BIT(2) +#define S2MU005_MUIC_WAIT BIT(1) +#define S2MU005_MUIC_IRQ BIT(0) + +/* S2MU005_REG_MUIC_CTRL3 */ +#define S2MU005_MUIC_ONESHOT_ADC BIT(2) + +/* S2MU005_REG_MUIC_LDOADC_L and S2MU005_REG_MUIC_LDOADC_H */ +#define S2MU005_MUIC_VSET GENMASK(4, 0) + +#define S2MU005_MUIC_VSET_3P0V 0x1f +#define S2MU005_MUIC_VSET_2P6V 0x0e +#define S2MU005_MUIC_VSET_2P4V 0x0c +#define S2MU005_MUIC_VSET_2P2V 0x0a +#define S2MU005_MUIC_VSET_2P0V 0x08 +#define S2MU005_MUIC_VSET_1P5V 0x03 +#define S2MU005_MUIC_VSET_1P4V 0x02 +#define S2MU005_MUIC_VSET_1P2V 0x00 + +#endif /* __LINUX_MFD_S2MU005_H */ diff --git a/include/linux/mfd/si476x-core.h b/include/linux/mfd/si476x-core.h index dd95c37ca134..d9e3a322134c 100644 --- a/include/linux/mfd/si476x-core.h +++ b/include/linux/mfd/si476x-core.h @@ -14,6 +14,7 @@ #include <linux/kfifo.h> #include <linux/atomic.h> +#include <linux/gpio/consumer.h> #include <linux/i2c.h> #include <linux/regmap.h> #include <linux/mutex.h> @@ -77,6 +78,7 @@ enum si476x_power_state { * underlying "core" device which all the MFD cell-devices use. * * @client: Actual I2C client used to transfer commands to the chip. + * @regmap: Regmap for accessing the device registers * @chip_id: Last digit of the chip model(E.g. "1" for SI4761) * @cells: MFD cell devices created by this driver. * @cmd_lock: Mutex used to serialize all the requests to the core @@ -100,16 +102,18 @@ enum si476x_power_state { * @stc: Similar to @cts, but for the STC bit of the status value. * @power_up_parameters: Parameters used as argument for POWER_UP * command when the device is started. - * @state: Current power state of the device. - * @supplues: Structure containing handles to all power supplies used + * @power_state: Current power state of the device. + * @supplies: Structure containing handles to all power supplies used * by the device (NULL ones are ignored). - * @gpio_reset: GPIO pin connectet to the RSTB pin of the chip. + * @reset: GPIO connected to the RSTB pin of the chip. * @pinmux: Chip's configurable pins configuration. * @diversity_mode: Chips role when functioning in diversity mode. + * @is_alive: Chip is initialized and active. * @status_monitor: Polling worker used in polling use case scenarion * (when IRQ is not avalible). * @revision: Chip's running firmware revision number(Used for correct * command set support). + * @rds_fifo_depth: RDS FIFO size: 20 for IRQ mode or 5 for polling mode. */ struct si476x_core { @@ -139,7 +143,7 @@ struct si476x_core { struct regulator_bulk_data supplies[4]; - int gpio_reset; + struct gpio_desc *reset; struct si476x_pinmux pinmux; enum si476x_phase_diversity_mode diversity_mode; @@ -166,6 +170,7 @@ static inline struct si476x_core *i2c_mfd_cell_to_core(struct device *dev) /** * si476x_core_lock() - lock the core device to get an exclusive access * to it. + * @core: Core device structure */ static inline void si476x_core_lock(struct si476x_core *core) { @@ -175,6 +180,7 @@ static inline void si476x_core_lock(struct si476x_core *core) /** * si476x_core_unlock() - unlock the core device to relinquish an * exclusive access to it. + * @core: Core device structure */ static inline void si476x_core_unlock(struct si476x_core *core) { @@ -246,9 +252,10 @@ static inline int si476x_to_v4l2(struct si476x_core *core, u16 freq) * struct si476x_func_info - structure containing result of the * FUNC_INFO command. * + * @firmware: Firmware version numbers. * @firmware.major: Firmware major number. * @firmware.minor[...]: Firmware minor numbers. - * @patch_id: + * @patch_id: Firmware patch level. * @func: Mode tuner is working in. */ struct si476x_func_info { @@ -318,8 +325,9 @@ enum si476x_smoothmetrics { * @tp: Current channel's TP flag. * @pty: Current channel's PTY code. * @pi: Current channel's PI code. - * @rdsfifoused: Number of blocks remaining in the RDS FIFO (0 if - * empty). + * @rdsfifoused: Number of blocks remaining in the RDS FIFO (0 if empty). + * @ble: + * @rds: RDS data descriptor */ struct si476x_rds_status_report { bool rdstpptyint, rdspiint, rdssyncint, rdsfifoint; diff --git a/include/linux/mfd/si476x-platform.h b/include/linux/mfd/si476x-platform.h index cb99e16ca947..f9e1f6b27277 100644 --- a/include/linux/mfd/si476x-platform.h +++ b/include/linux/mfd/si476x-platform.h @@ -246,8 +246,6 @@ enum si476x_phase_diversity_mode { * Platform dependent definition */ struct si476x_platform_data { - int gpio_reset; /* < 0 if not used */ - struct si476x_power_up_args power_up_parameters; enum si476x_phase_diversity_mode diversity_mode; diff --git a/include/linux/mfd/ti-lmu.h b/include/linux/mfd/ti-lmu.h index 0bc0e8199798..2089ec5124e8 100644 --- a/include/linux/mfd/ti-lmu.h +++ b/include/linux/mfd/ti-lmu.h @@ -10,7 +10,6 @@ #ifndef __MFD_TI_LMU_H__ #define __MFD_TI_LMU_H__ -#include <linux/gpio.h> #include <linux/notifier.h> #include <linux/regmap.h> #include <linux/gpio/consumer.h> diff --git a/include/linux/mfd/tps65910.h b/include/linux/mfd/tps65910.h index f67ef0a4e041..3813fc9c2b55 100644 --- a/include/linux/mfd/tps65910.h +++ b/include/linux/mfd/tps65910.h @@ -12,7 +12,7 @@ #ifndef __LINUX_MFD_TPS65910_H #define __LINUX_MFD_TPS65910_H -#include <linux/gpio.h> +#include <linux/gpio/consumer.h> #include <linux/regmap.h> /* TPS chip id list */ diff --git a/include/linux/mfd/ucb1x00.h b/include/linux/mfd/ucb1x00.h index ede237384723..66ecad4667c9 100644 --- a/include/linux/mfd/ucb1x00.h +++ b/include/linux/mfd/ucb1x00.h @@ -9,7 +9,7 @@ #include <linux/device.h> #include <linux/mfd/mcp.h> -#include <linux/gpio.h> +#include <linux/gpio/consumer.h> #include <linux/gpio/driver.h> #include <linux/mutex.h> @@ -103,6 +103,9 @@ #define UCB_MODE_DYN_VFLAG_ENA (1 << 12) #define UCB_MODE_AUD_OFF_CAN (1 << 13) +struct software_node; +extern const struct software_node ucb1x00_gpiochip_node; + enum ucb1x00_reset { UCB_RST_PROBE, UCB_RST_RESUME, diff --git a/include/linux/mfd/viperboard.h b/include/linux/mfd/viperboard.h index 0557667fe544..5761ebdc81b3 100644 --- a/include/linux/mfd/viperboard.h +++ b/include/linux/mfd/viperboard.h @@ -47,7 +47,7 @@ struct vprbrd_i2c_write_hdr { u8 cmd; - u16 addr; + __le16 addr; u8 len1; u8 len2; u8 last; @@ -57,15 +57,15 @@ struct vprbrd_i2c_write_hdr { struct vprbrd_i2c_read_hdr { u8 cmd; - u16 addr; + __le16 addr; u8 len0; u8 len1; u8 len2; u8 len3; u8 len4; u8 len5; - u16 tf1; /* transfer 1 length */ - u16 tf2; /* transfer 2 length */ + __le16 tf1; /* transfer 1 length */ + __le16 tf2; /* transfer 2 length */ } __packed; struct vprbrd_i2c_status { @@ -89,7 +89,7 @@ struct vprbrd_i2c_addr_msg { u8 cmd; u8 addr; u8 unknown1; - u16 len; + __le16 len; u8 unknown2; u8 unknown3; } __packed; diff --git a/include/linux/mfd/wm8994/pdata.h b/include/linux/mfd/wm8994/pdata.h index 6e2962ef5b81..b95a56a338c3 100644 --- a/include/linux/mfd/wm8994/pdata.h +++ b/include/linux/mfd/wm8994/pdata.h @@ -226,11 +226,6 @@ struct wm8994_pdata { * lines is mastered. */ int max_channels_clocked[WM8994_NUM_AIF]; - - /** - * GPIO for the IRQ pin if host only supports edge triggering - */ - int irq_gpio; }; #endif diff --git a/include/linux/mhi.h b/include/linux/mhi.h index 88ccb3e14f48..8616bacd8675 100644 --- a/include/linux/mhi.h +++ b/include/linux/mhi.h @@ -14,6 +14,7 @@ #include <linux/spinlock.h> #include <linux/wait.h> #include <linux/workqueue.h> +#include <linux/device-id/mhi.h> #define MHI_MAX_OEM_PK_HASH_SEGMENTS 16 @@ -86,22 +87,38 @@ enum mhi_ch_type { }; /** + * struct mhi_buf - MHI Buffer description + * @buf: Virtual address of the buffer + * @name: Buffer label. For offload channel, configurations name must be: + * ECA - Event context array data + * CCA - Channel context array data + * @dma_addr: IOMMU address of the buffer + * @len: # of bytes + */ +struct mhi_buf { + void *buf; + const char *name; + dma_addr_t dma_addr; + size_t len; +}; + +/** * struct image_info - Firmware and RDDM table * @mhi_buf: Buffer for firmware and RDDM table * @entries: # of entries in table */ struct image_info { - struct mhi_buf *mhi_buf; /* private: from internal.h */ struct bhi_vec_entry *bhi_vec; /* public: */ u32 entries; + struct mhi_buf mhi_buf[] __counted_by(entries); }; /** * struct mhi_link_info - BW requirement - * target_link_speed - Link speed as defined by TLS bits in LinkControl reg - * target_link_width - Link width as defined by NLW bits in LinkStatus reg + * @target_link_speed: Link speed as defined by TLS bits in LinkControl reg + * @target_link_width: Link width as defined by NLW bits in LinkStatus reg */ struct mhi_link_info { unsigned int target_link_speed; @@ -156,6 +173,7 @@ enum mhi_state { MHI_STATE_M3_FAST = 0x6, MHI_STATE_BHI = 0x7, MHI_STATE_SYS_ERR = 0xFF, + /* private: */ MHI_STATE_MAX, }; @@ -210,12 +228,12 @@ enum mhi_db_brst_mode { * @type: Channel type * @ee_mask: Execution Environment mask for this channel * @pollcfg: Polling configuration for burst mode. 0 is default. milliseconds - for UL channels, multiple of 8 ring elements for DL channels + * for UL channels, multiple of 8 ring elements for DL channels * @doorbell: Doorbell mode * @lpm_notify: The channel master requires low power mode notifications * @offload_channel: The client manages the channel completely * @doorbell_mode_switch: Channel switches to doorbell mode on M0 transition - * @wake-capable: Channel capable of waking up the system + * @wake_capable: Channel capable of waking up the system */ struct mhi_channel_config { char *name; @@ -333,7 +351,9 @@ struct mhi_controller_config { * @dev_state: MHI device state * @dev_wake: Device wakeup count * @pending_pkts: Pending packets for the controller - * @M0, M2, M3: Counters to track number of device MHI state changes + * @M0: Counter to track number of device MHI state changes + * @M2: Counter to track number of device MHI state changes + * @M3: Counter to track number of device MHI state changes * @transition_list: List of MHI state transitions * @transition_lock: Lock for protecting MHI state transition list * @wlock: Lock for protecting device wakeup @@ -358,6 +378,7 @@ struct mhi_controller_config { * @bounce_buf: Use of bounce buffer * @fbc_download: MHI host needs to do complete image transfer (optional) * @wake_set: Device wakeup set flag + * @no_m3: Device doesn't support M3 state * @irq_flags: irq flags passed to request_irq (optional) * @mru: the default MRU for the MHI device * @@ -443,6 +464,7 @@ struct mhi_controller { bool bounce_buf; bool fbc_download; bool wake_set; + bool no_m3; unsigned long irq_flags; u32 mru; }; @@ -489,23 +511,8 @@ struct mhi_result { }; /** - * struct mhi_buf - MHI Buffer description - * @buf: Virtual address of the buffer - * @name: Buffer label. For offload channel, configurations name must be: - * ECA - Event context array data - * CCA - Channel context array data - * @dma_addr: IOMMU address of the buffer - * @len: # of bytes - */ -struct mhi_buf { - void *buf; - const char *name; - dma_addr_t dma_addr; - size_t len; -}; - -/** * struct mhi_driver - Structure representing a MHI client driver + * @id_table: table of MHI channel names that a driver supports * @probe: CB function for client driver probe function * @remove: CB function for client driver remove function * @ul_xfer_cb: CB function for UL data transfer @@ -537,6 +544,7 @@ struct mhi_controller *mhi_alloc_controller(void); /** * mhi_free_controller - Free the MHI Controller structure + * @mhi_cntrl: MHI controller to free * Free the mhi_controller structure which was previously allocated */ void mhi_free_controller(struct mhi_controller *mhi_cntrl); diff --git a/include/linux/mhi_ep.h b/include/linux/mhi_ep.h index 7b40fc8cbe77..f6383a57a872 100644 --- a/include/linux/mhi_ep.h +++ b/include/linux/mhi_ep.h @@ -107,6 +107,7 @@ struct mhi_ep_buf_info { * @write_sync: CB function for writing to host memory synchronously * @read_async: CB function for reading from host memory asynchronously * @write_async: CB function for writing to host memory asynchronously + * @flush_async: CB function for flushing asynchronous read/writes * @mhi_state: MHI Endpoint state * @max_chan: Maximum channels supported by the endpoint controller * @mru: MRU (Maximum Receive Unit) value of the endpoint controller @@ -164,6 +165,7 @@ struct mhi_ep_cntrl { int (*write_sync)(struct mhi_ep_cntrl *mhi_cntrl, struct mhi_ep_buf_info *buf_info); int (*read_async)(struct mhi_ep_cntrl *mhi_cntrl, struct mhi_ep_buf_info *buf_info); int (*write_async)(struct mhi_ep_cntrl *mhi_cntrl, struct mhi_ep_buf_info *buf_info); + void (*flush_async)(struct mhi_ep_cntrl *mhi_cntrl); enum mhi_state mhi_state; diff --git a/include/linux/micrel_phy.h b/include/linux/micrel_phy.h index ca691641788b..9c6f9817383f 100644 --- a/include/linux/micrel_phy.h +++ b/include/linux/micrel_phy.h @@ -33,6 +33,7 @@ #define PHY_ID_LAN8804 0x00221670 #define PHY_ID_LAN8841 0x00221650 #define PHY_ID_LAN8842 0x002216C0 +#define PHY_ID_LAN9645X 0x002216D0 #define PHY_ID_KSZ886X 0x00221430 #define PHY_ID_KSZ8863 0x00221435 diff --git a/include/linux/microchipphy.h b/include/linux/microchipphy.h index 517288da19fd..7da956c666a0 100644 --- a/include/linux/microchipphy.h +++ b/include/linux/microchipphy.h @@ -61,6 +61,11 @@ /* Registers specific to the LAN7800/LAN7850 embedded phy */ #define LAN78XX_PHY_LED_MODE_SELECT (0x1D) +/* PHY Control 3 register (page 1) */ +#define LAN78XX_PHY_CTRL3 (0x14) +#define LAN78XX_PHY_CTRL3_AUTO_DOWNSHIFT BIT(4) +#define LAN78XX_PHY_CTRL3_DOWNSHIFT_CTRL_MASK GENMASK(3, 2) + /* DSP registers */ #define PHY_ARDENNES_MMD_DEV_3_PHY_CFG (0x806A) #define PHY_ARDENNES_MMD_DEV_3_PHY_CFG_ZD_DLY_EN_ (0x2000) diff --git a/include/linux/migrate.h b/include/linux/migrate.h index d5af2b7f577b..78424b3824c2 100644 --- a/include/linux/migrate.h +++ b/include/linux/migrate.h @@ -57,8 +57,9 @@ void putback_movable_pages(struct list_head *l); int migrate_folio(struct address_space *mapping, struct folio *dst, struct folio *src, enum migrate_mode mode); int migrate_pages(struct list_head *l, new_folio_t new, free_folio_t free, - unsigned long private, enum migrate_mode mode, int reason, - unsigned int *ret_succeeded); + unsigned long private, enum migrate_mode mode, + enum migrate_reason reason, + unsigned int *ret_succeeded); struct folio *alloc_migration_target(struct folio *src, unsigned long private); bool isolate_movable_ops_page(struct page *page, isolate_mode_t mode); bool isolate_folio_to_list(struct folio *folio, struct list_head *list); @@ -77,7 +78,8 @@ int set_movable_ops(const struct movable_operations *ops, enum pagetype type); static inline void putback_movable_pages(struct list_head *l) {} static inline int migrate_pages(struct list_head *l, new_folio_t new, free_folio_t free, unsigned long private, - enum migrate_mode mode, int reason, unsigned int *ret_succeeded) + enum migrate_mode mode, enum migrate_reason reason, + unsigned int *ret_succeeded) { return -ENOSYS; } static inline struct folio *alloc_migration_target(struct folio *src, unsigned long private) diff --git a/include/linux/migrate_mode.h b/include/linux/migrate_mode.h index 265c4328b36a..05102d4d2490 100644 --- a/include/linux/migrate_mode.h +++ b/include/linux/migrate_mode.h @@ -25,6 +25,7 @@ enum migrate_reason { MR_LONGTERM_PIN, MR_DEMOTION, MR_DAMON, + MR_NEVER, /* page has never been migrated */ MR_TYPES }; diff --git a/include/linux/misc/tsi.h b/include/linux/misc/tsi.h new file mode 100644 index 000000000000..5273c44688f0 --- /dev/null +++ b/include/linux/misc/tsi.h @@ -0,0 +1,72 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * AMD SBTSI shared data structure and auxiliary bus definitions. + * + * Copyright (C) 2026 Advanced Micro Devices, Inc. + */ + +#ifndef _LINUX_MISC_TSI_H_ +#define _LINUX_MISC_TSI_H_ + +#include <linux/cleanup.h> +#include <linux/i2c.h> +#include <linux/i3c/device.h> +#include <linux/kref.h> +#include <linux/miscdevice.h> +#include <linux/mutex.h> +#include <linux/types.h> + +/** + * struct sbtsi_data - driver private data for an AMD SB-TSI device + * @client: underlying I2C client + * @i3cdev: underlying I3C device (when using I3C bus) + * @sbtsi_misc_dev: miscdevice exposing ioctl interface at /dev/sbtsi-<addr> + * @lock: mutex protecting concurrent access to the device + * @kref: reference count; keeps @sbtsi_data alive while misc fds are open + * @dev_addr: I2C/I3C device address, used as the auxiliary device instance id + * and name the misc device node + * @ext_range_mode: sensor uses extended temperature range + * @read_order: if set, decimal part must be read before integer part + * @is_i3c: true when the device is accessed over I3C + * @detached: set on driver unbind; open/ioctl return -ENODEV afterward + */ +struct sbtsi_data { + union { + struct i2c_client *client; + struct i3c_device *i3cdev; + }; + struct miscdevice sbtsi_misc_dev; + struct mutex lock; /* protects concurrent access to the device */ + struct kref kref; + u8 dev_addr; + bool ext_range_mode; + bool read_order; + bool is_i3c; + bool detached; +}; + +DEFINE_GUARD(sbtsi, struct sbtsi_data *, mutex_lock(&_T->lock), + mutex_unlock(&_T->lock)) + +/* + * Name of the auxiliary device published on the auxiliary bus by the core + * driver. The full device name is "amd-sbtsi.temp-sensor.<id>". where + * <id> is the auxiliary device instance id. + */ +#define AMD_SBTSI_ADEV "amd-sbtsi" +#define AMD_SBTSI_AUX_HWMON "temp-sensor" + +/** + * sbtsi_xfer - Perform a register read or write transfer on an AMD SB-TSI device. + * + * @data: Pointer to the sbtsi_data structure containing the device context + * @reg: Register address to access. + * @val: Pointer to the value to read into or write from. + * @is_read: If true, performs a read transfer and stores the result in @val. + * If false, performs a write transfer using the value in @val. + * + * Returns 0 on success, or a negative error code on failure. + */ +int sbtsi_xfer(struct sbtsi_data *data, u8 reg, u8 *val, bool is_read); + +#endif /* _LINUX_MISC_TSI_H_ */ diff --git a/include/linux/mlx5/device.h b/include/linux/mlx5/device.h index b37fe39cef27..8cb321a9fb3d 100644 --- a/include/linux/mlx5/device.h +++ b/include/linux/mlx5/device.h @@ -293,6 +293,7 @@ enum { MLX5_UMR_INLINE = (1 << 7), }; +#define MLX5_UMR_ALIGN (2048) #define MLX5_UMR_FLEX_ALIGNMENT 0x40 #define MLX5_UMR_MTT_NUM_ENTRIES_ALIGNMENT (MLX5_UMR_FLEX_ALIGNMENT / sizeof(struct mlx5_mtt)) #define MLX5_UMR_KLM_NUM_ENTRIES_ALIGNMENT (MLX5_UMR_FLEX_ALIGNMENT / sizeof(struct mlx5_klm)) @@ -1170,6 +1171,7 @@ enum { MLX5_MATCH_MISC_PARAMETERS_3 = 1 << 4, MLX5_MATCH_MISC_PARAMETERS_4 = 1 << 5, MLX5_MATCH_MISC_PARAMETERS_5 = 1 << 6, + MLX5_MATCH_MISC_PARAMETERS_6 = 1 << 7, }; enum { @@ -1259,6 +1261,7 @@ enum mlx5_cap_type { MLX5_CAP_PORT_SELECTION = 0x25, MLX5_CAP_ADV_VIRTUALIZATION = 0x26, MLX5_CAP_ADV_RDMA = 0x28, + MLX5_CAP_TLP_EMULATION = 0x2a, /* NUM OF CAP Types */ MLX5_CAP_NUM }; @@ -1481,6 +1484,14 @@ enum mlx5_qcam_feature_groups { MLX5_GET64(virtio_emulation_cap, \ (mdev)->caps.hca[MLX5_CAP_VDPA_EMULATION]->cur, cap) +#define MLX5_CAP_DEV_TLP_EMULATION(mdev, cap)\ + MLX5_GET(tlp_dev_emu_capabilities, \ + (mdev)->caps.hca[MLX5_CAP_TLP_EMULATION]->cur, cap) + +#define MLX5_CAP64_DEV_TLP_EMULATION(mdev, cap)\ + MLX5_GET64(tlp_dev_emu_capabilities, \ + (mdev)->caps.hca[MLX5_CAP_TLP_EMULATION]->cur, cap) + #define MLX5_CAP_IPSEC(mdev, cap)\ MLX5_GET(ipsec_cap, (mdev)->caps.hca[MLX5_CAP_IPSEC]->cur, cap) diff --git a/include/linux/mlx5/driver.h b/include/linux/mlx5/driver.h index 04dcd09f7517..83d0a83bbfbc 100644 --- a/include/linux/mlx5/driver.h +++ b/include/linux/mlx5/driver.h @@ -343,9 +343,11 @@ struct mlx5_cmd_mailbox { struct mlx5_cmd_mailbox *next; }; +struct mlx5_dma_pool_page; struct mlx5_buf_list { void *buf; dma_addr_t map; + struct mlx5_dma_pool_page *frag_page; }; struct mlx5_frag_buf { @@ -545,19 +547,30 @@ struct mlx5_debugfs_entries { struct dentry *eq_debugfs; struct dentry *cq_debugfs; struct dentry *cmdif_debugfs; + struct dentry *frag_buf_dma_pools_debugfs; + struct dentry *db_dma_pools_debugfs; struct dentry *pages_debugfs; struct dentry *lag_debugfs; }; enum mlx5_func_type { - MLX5_PF, + MLX5_SELF, MLX5_VF, MLX5_SF, MLX5_HOST_PF, + MLX5_SPF, MLX5_EC_VF, MLX5_FUNC_TYPE_NUM, + MLX5_FUNC_TYPE_NONE = MLX5_FUNC_TYPE_NUM, }; +enum mlx5_page_mgt_mode { + MLX5_PAGE_MGT_MODE_FUNC_ID, + MLX5_PAGE_MGT_MODE_VHCA_ID, +}; + +struct mlx5_frag_buf_node_pools; +struct mlx5_dma_pool; struct mlx5_ft_pool; struct mlx5_priv { /* IRQ table valid only for real pci devices PF or VF */ @@ -575,20 +588,21 @@ struct mlx5_priv { u32 fw_pages_alloc_failed; u32 give_pages_dropped; u32 reclaim_pages_discard; + enum mlx5_page_mgt_mode page_mgt_mode; struct mlx5_core_health health; struct list_head traps; struct mlx5_debugfs_entries dbg; - /* start: alloc staff */ + /* start: alloc stuff */ /* protect buffer allocation according to numa node */ struct mutex alloc_mutex; int numa_node; - struct mutex pgdir_mutex; - struct list_head pgdir_list; - /* end: alloc staff */ + struct mlx5_frag_buf_node_pools **frag_buf_node_pools; + struct mlx5_dma_pool **db_node_pools; + /* end: alloc stuff */ struct mlx5_adev **adev; int adev_idx; @@ -705,23 +719,12 @@ struct mlx5_st; enum { MLX5_PROF_MASK_QP_SIZE = (u64)1 << 0, - MLX5_PROF_MASK_MR_CACHE = (u64)1 << 1, -}; - -enum { - MKEY_CACHE_LAST_STD_ENTRY = 20, - MLX5_IMR_KSM_CACHE_ENTRY, - MAX_MKEY_CACHE_ENTRIES }; struct mlx5_profile { u64 mask; u8 log_max_qp; u8 num_cmd_caches; - struct { - int size; - int limit; - } mr_cache[MAX_MKEY_CACHE_ENTRIES]; }; struct mlx5_hca_cap { @@ -755,7 +758,6 @@ struct mlx5_core_dev { } caps; struct mlx5_timeouts *timeouts; u64 sys_image_guid; - phys_addr_t iseg_base; struct mlx5_init_seg __iomem *iseg; phys_addr_t bar_addr; enum mlx5_device_state state; @@ -798,12 +800,13 @@ struct mlx5_core_dev { enum mlx5_wc_state wc_state; /* sync write combining state */ struct mutex wc_state_lock; + struct devlink *shd; }; struct mlx5_db { __be32 *db; union { - struct mlx5_db_pgdir *pgdir; + struct mlx5_dma_pool_page *pool_page; struct mlx5_ib_user_db_page *user_page; } u; dma_addr_t dma; @@ -1045,6 +1048,8 @@ void mlx5_pagealloc_start(struct mlx5_core_dev *dev); void mlx5_pagealloc_stop(struct mlx5_core_dev *dev); void mlx5_pages_debugfs_init(struct mlx5_core_dev *dev); void mlx5_pages_debugfs_cleanup(struct mlx5_core_dev *dev); +void mlx5_pages_by_func_type_debugfs_init(struct mlx5_core_dev *dev); +void mlx5_pages_by_func_type_debugfs_cleanup(struct mlx5_core_dev *dev); int mlx5_satisfy_startup_pages(struct mlx5_core_dev *dev, int boot); int mlx5_reclaim_startup_pages(struct mlx5_core_dev *dev); void mlx5_register_debugfs(void); diff --git a/include/linux/mlx5/eswitch.h b/include/linux/mlx5/eswitch.h index 67256e776566..a0dd162baa78 100644 --- a/include/linux/mlx5/eswitch.h +++ b/include/linux/mlx5/eswitch.h @@ -63,7 +63,13 @@ struct mlx5_eswitch_rep { void mlx5_eswitch_register_vport_reps(struct mlx5_eswitch *esw, const struct mlx5_eswitch_rep_ops *ops, u8 rep_type); +void +mlx5_eswitch_register_vport_reps_nested(struct mlx5_eswitch *esw, + const struct mlx5_eswitch_rep_ops *ops, + u8 rep_type); void mlx5_eswitch_unregister_vport_reps(struct mlx5_eswitch *esw, u8 rep_type); +void mlx5_eswitch_unregister_vport_reps_nested(struct mlx5_eswitch *esw, + u8 rep_type); void *mlx5_eswitch_get_proto_dev(struct mlx5_eswitch *esw, u16 vport_num, u8 rep_type); @@ -217,7 +223,7 @@ static inline bool is_mdev_switchdev_mode(struct mlx5_core_dev *dev) static inline u16 mlx5_eswitch_manager_vport(struct mlx5_core_dev *dev) { return mlx5_core_is_ecpf_esw_manager(dev) ? - MLX5_VPORT_ECPF : MLX5_VPORT_PF; + MLX5_VPORT_ECPF : MLX5_VPORT_HOST_PF; } #endif diff --git a/include/linux/mlx5/fs.h b/include/linux/mlx5/fs.h index 9cadb1d5e6df..d8f3b7ef319e 100644 --- a/include/linux/mlx5/fs.h +++ b/include/linux/mlx5/fs.h @@ -55,6 +55,7 @@ enum mlx5_flow_destination_type { MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE_NUM, MLX5_FLOW_DESTINATION_TYPE_RANGE, MLX5_FLOW_DESTINATION_TYPE_TABLE_TYPE, + MLX5_FLOW_DESTINATION_TYPE_VHCA_RX, }; enum { @@ -190,6 +191,9 @@ struct mlx5_flow_destination { struct mlx5_flow_table *ft; struct mlx5_fc *counter; struct { + u16 id; + } vhca; + struct { u16 num; u16 vhca_id; struct mlx5_pkt_reformat *pkt_reformat; @@ -248,9 +252,9 @@ mlx5_create_auto_grouped_flow_table(struct mlx5_flow_namespace *ns, struct mlx5_flow_table * mlx5_create_vport_flow_table(struct mlx5_flow_namespace *ns, struct mlx5_flow_table_attr *ft_attr, u16 vport); -struct mlx5_flow_table *mlx5_create_lag_demux_flow_table( - struct mlx5_flow_namespace *ns, - int prio, u32 level); +struct mlx5_flow_table * +mlx5_create_lag_demux_flow_table(struct mlx5_flow_namespace *ns, + struct mlx5_flow_table_attr *ft_attr); int mlx5_destroy_flow_table(struct mlx5_flow_table *ft); /* inbox should be set with the following values: diff --git a/include/linux/mlx5/lag.h b/include/linux/mlx5/lag.h new file mode 100644 index 000000000000..ab9f754664e5 --- /dev/null +++ b/include/linux/mlx5/lag.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ +/* Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. */ + +#ifndef __MLX5_LAG_API_H__ +#define __MLX5_LAG_API_H__ + +#include <linux/types.h> + +struct mlx5_core_dev; +struct mlx5_flow_table; +struct mlx5_flow_table_attr; + +int mlx5_lag_demux_init(struct mlx5_core_dev *dev, + struct mlx5_flow_table_attr *ft_attr); +void mlx5_lag_demux_cleanup(struct mlx5_core_dev *dev); +int mlx5_lag_demux_rule_add(struct mlx5_core_dev *dev, u16 vport_num, + int vport_index); +void mlx5_lag_demux_rule_del(struct mlx5_core_dev *dev, int vport_index); +int mlx5_lag_get_dev_seq(struct mlx5_core_dev *dev); + +#endif /* __MLX5_LAG_API_H__ */ diff --git a/include/linux/mlx5/mlx5_ifc.h b/include/linux/mlx5/mlx5_ifc.h index 775cb0c56865..12b9a81e5621 100644 --- a/include/linux/mlx5/mlx5_ifc.h +++ b/include/linux/mlx5/mlx5_ifc.h @@ -469,7 +469,8 @@ struct mlx5_ifc_flow_table_prop_layout_bits { u8 table_miss_action_domain[0x1]; u8 termination_table[0x1]; u8 reformat_and_fwd_to_table[0x1]; - u8 reserved_at_1a[0x2]; + u8 forward_vhca_rx[0x1]; + u8 reserved_at_1b[0x1]; u8 ipsec_encrypt[0x1]; u8 ipsec_decrypt[0x1]; u8 sw_owner_v2[0x1]; @@ -507,7 +508,8 @@ struct mlx5_ifc_flow_table_prop_layout_bits { u8 reformat_l2_to_l3_audp_tunnel[0x1]; u8 reformat_l3_audp_tunnel_to_l2[0x1]; u8 ignore_flow_level_rtc_valid[0x1]; - u8 reserved_at_70[0x8]; + u8 reserved_at_70[0x7]; + u8 reformat_del_psp_transport[0x1]; u8 log_max_ft_num[0x8]; u8 reserved_at_80[0x10]; @@ -797,6 +799,15 @@ struct mlx5_ifc_fte_match_set_misc5_bits { u8 reserved_at_100[0x100]; }; +struct mlx5_ifc_fte_match_set_misc6_bits { + u8 reserved_at_0[0x1a]; + u8 psp_version[0x4]; + u8 reserved_at_1e[0x2]; + + u8 reserved_at_20[0x1e0]; +}; + + struct mlx5_ifc_cmd_pas_bits { u8 pa_h[0x20]; @@ -1041,20 +1052,6 @@ struct mlx5_ifc_wqe_based_flow_table_cap_bits { u8 reserved_at_1c1[0x1f]; }; -struct mlx5_ifc_esw_cap_bits { - u8 reserved_at_0[0x1d]; - u8 merged_eswitch[0x1]; - u8 reserved_at_1e[0x2]; - - u8 reserved_at_20[0x40]; - - u8 esw_manager_vport_number_valid[0x1]; - u8 reserved_at_61[0xf]; - u8 esw_manager_vport_number[0x10]; - - u8 reserved_at_80[0x780]; -}; - enum { MLX5_COUNTER_SOURCE_ESWITCH = 0x0, MLX5_COUNTER_FLOW_ESWITCH = 0x1, @@ -1095,7 +1092,11 @@ struct mlx5_ifc_e_switch_cap_bits { u8 log_max_esw_sf[0x5]; u8 esw_sf_base_id[0x10]; - u8 reserved_at_60[0x7a0]; + u8 esw_manager_vport_number_valid[0x1]; + u8 reserved_at_61[0xf]; + u8 esw_manager_vport_number[0x10]; + + u8 reserved_at_80[0x780]; }; @@ -1115,7 +1116,10 @@ struct mlx5_ifc_qos_cap_bits { u8 log_esw_max_sched_depth[0x4]; u8 reserved_at_10[0x10]; - u8 reserved_at_20[0x9]; + u8 reserved_at_20[0x2]; + u8 packet_pacing_req_ud[0x1]; + u8 packet_pacing_req_uc[0x1]; + u8 reserved_at_24[0x5]; u8 esw_cross_esw_sched[0x1]; u8 reserved_at_2a[0x1]; u8 log_max_qos_nic_queue_group[0x5]; @@ -1389,6 +1393,26 @@ struct mlx5_ifc_virtio_emulation_cap_bits { u8 reserved_at_1c0[0x640]; }; +struct mlx5_ifc_tlp_dev_emu_capabilities_bits { + u8 reserved_at_0[0x20]; + + u8 reserved_at_20[0x13]; + u8 log_tlp_rsp_gw_page_stride[0x5]; + u8 reserved_at_38[0x8]; + + u8 reserved_at_40[0xc0]; + + u8 reserved_at_100[0xc]; + u8 tlp_rsp_gw_num_pages[0x4]; + u8 reserved_at_110[0x10]; + + u8 reserved_at_120[0xa0]; + + u8 tlp_rsp_gw_pages_bar_offset[0x40]; + + u8 reserved_at_200[0x600]; +}; + enum { MLX5_ATOMIC_CAPS_ATOMIC_SIZE_QP_1_BYTE = 0x0, MLX5_ATOMIC_CAPS_ATOMIC_SIZE_QP_2_BYTES = 0x2, @@ -1633,6 +1657,11 @@ enum { MLX5_STEERING_FORMAT_CONNECTX_8 = 3, }; +enum { + MLX5_ID_MODE_FUNCTION_INDEX = 0, + MLX5_ID_MODE_FUNCTION_VHCA_ID = 1, +}; + struct mlx5_ifc_cmd_hca_cap_bits { u8 reserved_at_0[0x6]; u8 page_request_disable[0x1]; @@ -1895,7 +1924,8 @@ struct mlx5_ifc_cmd_hca_cap_bits { u8 reserved_at_280[0x10]; u8 max_wqe_sz_sq[0x10]; - u8 reserved_at_2a0[0x7]; + u8 icm_mng_function_id_mode[0x1]; + u8 reserved_at_2a1[0x6]; u8 mkey_pcie_tph[0x1]; u8 reserved_at_2a8[0x1]; u8 tis_tir_td_order[0x1]; @@ -1908,7 +1938,8 @@ struct mlx5_ifc_cmd_hca_cap_bits { u8 max_flow_counter_31_16[0x10]; u8 max_wqe_sz_sq_dc[0x10]; - u8 reserved_at_2e0[0x7]; + u8 query_host_net_function_num_max[0x5]; + u8 reserved_at_2e5[0x2]; u8 max_qp_mcg[0x19]; u8 reserved_at_300[0x10]; @@ -1947,7 +1978,8 @@ struct mlx5_ifc_cmd_hca_cap_bits { u8 reserved_at_360[0x3]; u8 log_max_rq[0x5]; - u8 reserved_at_368[0x3]; + u8 ft_alias_sw_vhca_id[0x1]; + u8 reserved_at_369[0x2]; u8 log_max_sq[0x5]; u8 reserved_at_370[0x3]; u8 log_max_tir[0x5]; @@ -1957,11 +1989,12 @@ struct mlx5_ifc_cmd_hca_cap_bits { u8 basic_cyclic_rcv_wqe[0x1]; u8 reserved_at_381[0x2]; u8 log_max_rmp[0x5]; - u8 reserved_at_388[0x3]; + u8 sd_group_size[0x1]; + u8 reserved_at_389[0x2]; u8 log_max_rqt[0x5]; u8 reserved_at_390[0x3]; u8 log_max_rqt_size[0x5]; - u8 reserved_at_398[0x1]; + u8 tlp_device_emulation_manager[0x1]; u8 vnic_env_cnt_bar_uar_access[0x1]; u8 vnic_env_cnt_odp_page_fault[0x1]; u8 log_max_tis_per_sq[0x5]; @@ -1972,7 +2005,8 @@ struct mlx5_ifc_cmd_hca_cap_bits { u8 log_max_stride_sz_rq[0x5]; u8 reserved_at_3a8[0x3]; u8 log_min_stride_sz_rq[0x5]; - u8 reserved_at_3b0[0x3]; + u8 reserved_at_3b0[0x2]; + u8 qp_latency_sensitive_disable[0x1]; u8 log_max_stride_sz_sq[0x5]; u8 reserved_at_3b8[0x3]; u8 log_min_stride_sz_sq[0x5]; @@ -1992,12 +2026,14 @@ struct mlx5_ifc_cmd_hca_cap_bits { u8 disable_local_lb_mc[0x1]; u8 log_min_hairpin_wq_data_sz[0x5]; u8 reserved_at_3e8[0x1]; - u8 silent_mode[0x1]; + u8 silent_mode_set[0x1]; u8 vhca_state[0x1]; u8 log_max_vlan_list[0x5]; u8 reserved_at_3f0[0x3]; u8 log_max_current_mc_list[0x5]; - u8 reserved_at_3f8[0x3]; + u8 reserved_at_3f8[0x1]; + u8 silent_mode_query[0x1]; + u8 query_host_net_function_v1[0x1]; u8 log_max_current_uc_list[0x5]; u8 general_obj_types[0x40]; @@ -2173,7 +2209,8 @@ struct mlx5_ifc_cmd_hca_cap_2_bits { u8 sf_eq_usage[0x1]; u8 reserved_at_d3[0x5]; u8 multiplane[0x1]; - u8 reserved_at_d9[0x7]; + u8 migration_state[0x1]; + u8 reserved_at_da[0x6]; u8 cross_vhca_object_to_object_supported[0x20]; @@ -2259,6 +2296,7 @@ enum mlx5_ifc_flow_destination_type { MLX5_IFC_FLOW_DESTINATION_TYPE_VPORT = 0x0, MLX5_IFC_FLOW_DESTINATION_TYPE_FLOW_TABLE = 0x1, MLX5_IFC_FLOW_DESTINATION_TYPE_TIR = 0x2, + MLX5_IFC_FLOW_DESTINATION_TYPE_VHCA_RX = 0x4, MLX5_IFC_FLOW_DESTINATION_TYPE_FLOW_SAMPLER = 0x6, MLX5_IFC_FLOW_DESTINATION_TYPE_UPLINK = 0x8, MLX5_IFC_FLOW_DESTINATION_TYPE_TABLE_TYPE = 0xA, @@ -2315,7 +2353,7 @@ struct mlx5_ifc_fte_match_param_bits { struct mlx5_ifc_fte_match_set_misc5_bits misc_parameters_5; - u8 reserved_at_e00[0x200]; + struct mlx5_ifc_fte_match_set_misc6_bits misc_parameters_6; }; enum { @@ -3675,7 +3713,8 @@ struct mlx5_ifc_qpc_bits { u8 cur_retry_count[0x3]; u8 reserved_at_39b[0x5]; - u8 reserved_at_3a0[0x20]; + u8 reserved_at_3a0[0x10]; + u8 packet_pacing_rate_limit_index[0x10]; u8 reserved_at_3c0[0x8]; u8 next_send_psn[0x18]; @@ -3821,7 +3860,6 @@ union mlx5_ifc_hca_cap_union_bits { struct mlx5_ifc_flow_table_nic_cap_bits flow_table_nic_cap; struct mlx5_ifc_flow_table_eswitch_cap_bits flow_table_eswitch_cap; struct mlx5_ifc_wqe_based_flow_table_cap_bits wqe_based_flow_table_cap; - struct mlx5_ifc_esw_cap_bits esw_cap; struct mlx5_ifc_e_switch_cap_bits e_switch_cap; struct mlx5_ifc_port_selection_cap_bits port_selection_cap; struct mlx5_ifc_qos_cap_bits qos_cap; @@ -3830,6 +3868,7 @@ union mlx5_ifc_hca_cap_union_bits { struct mlx5_ifc_tls_cap_bits tls_cap; struct mlx5_ifc_device_mem_cap_bits device_mem_cap; struct mlx5_ifc_virtio_emulation_cap_bits virtio_emulation_cap; + struct mlx5_ifc_tlp_dev_emu_capabilities_bits tlp_dev_emu_capabilities; struct mlx5_ifc_macsec_cap_bits macsec_cap; struct mlx5_ifc_crypto_cap_bits crypto_cap; struct mlx5_ifc_ipsec_cap_bits ipsec_cap; @@ -4435,7 +4474,9 @@ struct mlx5_ifc_nic_vport_context_bits { u8 reserved_at_100[0x1]; u8 sd_group[0x3]; - u8 reserved_at_104[0x1c]; + u8 reserved_at_104[0x4]; + u8 sd_group_size[0x8]; + u8 reserved_at_110[0x10]; u8 reserved_at_120[0x10]; u8 mtu[0x10]; @@ -4453,8 +4494,8 @@ struct mlx5_ifc_nic_vport_context_bits { u8 promisc_all[0x1]; u8 reserved_at_783[0x2]; u8 allowed_list_type[0x3]; - u8 reserved_at_788[0xc]; - u8 allowed_list_size[0xc]; + u8 reserved_at_788[0x8]; + u8 allowed_list_size[0x10]; struct mlx5_ifc_mac_address_layout_bits permanent_address; @@ -6244,7 +6285,9 @@ struct mlx5_ifc_query_l2_table_entry_out_bits { u8 reserved_at_40[0xa0]; - u8 reserved_at_e0[0x13]; + u8 reserved_at_e0[0x11]; + u8 silent_mode[0x1]; + u8 reserved_at_f2[0x1]; u8 vlan_valid[0x1]; u8 vlan[0xc]; @@ -6260,7 +6303,10 @@ struct mlx5_ifc_query_l2_table_entry_in_bits { u8 reserved_at_20[0x10]; u8 op_mod[0x10]; - u8 reserved_at_40[0x60]; + u8 reserved_at_40[0x40]; + + u8 silent_mode_query[0x1]; + u8 reserved_at_81[0x1f]; u8 reserved_at_a0[0x8]; u8 table_index[0x18]; @@ -6927,7 +6973,9 @@ struct mlx5_ifc_create_match_definer_out_bits { struct mlx5_ifc_alias_context_bits { u8 vhca_id_to_be_accessed[0x10]; - u8 reserved_at_10[0xd]; + u8 reserved_at_10[0xb]; + u8 vhca_id_type[0x1]; + u8 reserved_at_1c[0x1]; u8 status[0x3]; u8 object_id_to_be_accessed[0x20]; u8 reserved_at_40[0x40]; @@ -6951,6 +6999,7 @@ enum { MLX5_QUERY_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_MISC_PARAMETERS_3 = 0x4, MLX5_QUERY_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_MISC_PARAMETERS_4 = 0x5, MLX5_QUERY_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_MISC_PARAMETERS_5 = 0x6, + MLX5_QUERY_FLOW_GROUP_IN_MATCH_CRITERIA_ENABLE_MISC_PARAMETERS_6 = 0x7, }; struct mlx5_ifc_query_flow_group_out_bits { @@ -7212,6 +7261,7 @@ enum mlx5_reformat_ctx_type { MLX5_REFORMAT_TYPE_REMOVE_HDR = 0x10, MLX5_REFORMAT_TYPE_ADD_MACSEC = 0x11, MLX5_REFORMAT_TYPE_DEL_MACSEC = 0x12, + MLX5_REFORMAT_TYPE_REMOVE_PSP_TRANSPORT = 0x16, }; struct mlx5_ifc_alloc_packet_reformat_context_in_bits { @@ -7335,6 +7385,7 @@ enum { MLX5_ACTION_IN_FIELD_OUT_EMD_47_32 = 0x6F, MLX5_ACTION_IN_FIELD_OUT_EMD_31_0 = 0x70, MLX5_ACTION_IN_FIELD_PSP_SYNDROME = 0x71, + MLX5_ACTION_IN_FIELD_PSP_HEADER_1 = 0x78, }; struct mlx5_ifc_alloc_modify_header_context_out_bits { @@ -8412,7 +8463,9 @@ struct mlx5_ifc_enable_hca_in_bits { u8 op_mod[0x10]; u8 embedded_cpu_function[0x1]; - u8 reserved_at_41[0xf]; + u8 reserved_at_41[0x2]; + u8 function_id_type[0x1]; + u8 reserved_at_44[0xc]; u8 function_id[0x10]; u8 reserved_at_60[0x20]; @@ -8457,7 +8510,9 @@ struct mlx5_ifc_disable_hca_in_bits { u8 op_mod[0x10]; u8 embedded_cpu_function[0x1]; - u8 reserved_at_41[0xf]; + u8 reserved_at_41[0x2]; + u8 function_id_type[0x1]; + u8 reserved_at_44[0xc]; u8 function_id[0x10]; u8 reserved_at_60[0x20]; @@ -10824,7 +10879,9 @@ struct mlx5_ifc_pcam_enhanced_features_bits { u8 fec_200G_per_lane_in_pplm[0x1]; u8 reserved_at_1e[0x2a]; u8 fec_100G_per_lane_in_pplm[0x1]; - u8 reserved_at_49[0xa]; + u8 reserved_at_49[0x2]; + u8 shp_pbmc_pbsr_support[0x1]; + u8 reserved_at_4c[0x7]; u8 buffer_ownership[0x1]; u8 resereved_at_54[0x14]; u8 fec_50G_per_lane_in_pplm[0x1]; @@ -12069,8 +12126,9 @@ struct mlx5_ifc_pbmc_reg_bits { u8 port_buffer_size[0x10]; struct mlx5_ifc_bufferx_reg_bits buffer[10]; + struct mlx5_ifc_bufferx_reg_bits shared_headroom_pool; - u8 reserved_at_2e0[0x80]; + u8 reserved_at_320[0x40]; }; struct mlx5_ifc_sbpr_reg_bits { @@ -12160,18 +12218,7 @@ struct mlx5_ifc_mcia_reg_bits { u8 reserved_at_60[0x20]; - u8 dword_0[0x20]; - u8 dword_1[0x20]; - u8 dword_2[0x20]; - u8 dword_3[0x20]; - u8 dword_4[0x20]; - u8 dword_5[0x20]; - u8 dword_6[0x20]; - u8 dword_7[0x20]; - u8 dword_8[0x20]; - u8 dword_9[0x20]; - u8 dword_10[0x20]; - u8 dword_11[0x20]; + u8 dwords[0x400]; }; struct mlx5_ifc_dcbx_param_bits { @@ -12657,6 +12704,54 @@ struct mlx5_ifc_host_params_context_bits { u8 reserved_at_80[0x180]; }; +enum mlx5_ifc_vhca_state { + MLX5_VHCA_STATE_INVALID = 0x0, + MLX5_VHCA_STATE_ALLOCATED = 0x1, + MLX5_VHCA_STATE_ACTIVE = 0x2, + MLX5_VHCA_STATE_IN_USE = 0x3, + MLX5_VHCA_STATE_TEARDOWN_REQUEST = 0x4, +}; + +enum { + MLX5_PCI_PF_TYPE_EXTERNAL_HOST_PF = 0x0, + MLX5_PCI_PF_TYPE_SATELLITE_PF = 0x1, +}; + +struct mlx5_ifc_network_function_params_bits { + u8 host_number[0x8]; + u8 pci_pf_type[0x4]; + u8 reserved_at_c[0x4]; + u8 pci_num_vfs[0x10]; + + u8 pci_total_vfs[0x10]; + u8 pci_bus[0x8]; + u8 pci_device_function[0x8]; + + u8 vhca_id[0x10]; + u8 vhca_state[0x4]; + u8 reserved_at_54[0xc]; + + u8 reserved_at_60[0xa]; + u8 esw_vport_manual[0x1]; + u8 pci_bus_assigned[0x1]; + u8 pci_vf_info_valid[0x1]; + u8 reserved_at_6d[0x13]; + + u8 pci_vf_stride[0x10]; + u8 pci_first_vf_offset[0x10]; + + u8 reserved_at_a0[0x160]; +}; + +union mlx5_ifc_net_function_params_bits { + struct mlx5_ifc_host_params_context_bits host_params_context; + struct mlx5_ifc_network_function_params_bits network_function_params; +}; + +enum { + MLX5_QUERY_ESW_FUNC_OP_MOD_LAYOUT_V1 = BIT(14), +}; + struct mlx5_ifc_query_esw_functions_in_bits { u8 opcode[0x10]; u8 reserved_at_10[0x10]; @@ -12673,12 +12768,16 @@ struct mlx5_ifc_query_esw_functions_out_bits { u8 syndrome[0x20]; - u8 reserved_at_40[0x40]; + u8 reserved_at_40[0x20]; - struct mlx5_ifc_host_params_context_bits host_params_context; + u8 net_function_num[0x8]; + u8 reserved_at_68[0x18]; - u8 reserved_at_280[0x180]; - u8 host_sf_enable[][0x40]; + union { + u8 reserved_at_80[0x380]; + DECLARE_FLEX_ARRAY(union mlx5_ifc_net_function_params_bits, + net_function_params); + }; }; struct mlx5_ifc_sf_partition_bits { @@ -13280,13 +13379,24 @@ struct mlx5_ifc_query_vhca_migration_state_in_bits { u8 reserved_at_60[0x20]; }; +enum { + MLX5_QUERY_VHCA_MIG_STATE_UNINITIALIZED = 0x0, + MLX5_QUERY_VHCA_MIG_STATE_OPER_MIGRATION_IDLE = 0x1, + MLX5_QUERY_VHCA_MIG_STATE_OPER_MIGRATION_READY = 0x2, + MLX5_QUERY_VHCA_MIG_STATE_OPER_MIGRATION_DIRTY = 0x3, + MLX5_QUERY_VHCA_MIG_STATE_OPER_MIGRATION_INIT = 0x4, +}; + struct mlx5_ifc_query_vhca_migration_state_out_bits { u8 status[0x8]; u8 reserved_at_8[0x18]; u8 syndrome[0x20]; - u8 reserved_at_40[0x40]; + u8 reserved_at_40[0x20]; + + u8 migration_state[0x4]; + u8 reserved_at_64[0x1c]; u8 required_umem_size[0x20]; diff --git a/include/linux/mlx5/qp.h b/include/linux/mlx5/qp.h index d67aedc6ea68..40f889403b07 100644 --- a/include/linux/mlx5/qp.h +++ b/include/linux/mlx5/qp.h @@ -72,6 +72,7 @@ enum mlx5_qp_optpar { MLX5_QP_OPTPAR_CQN_RCV = 1 << 19, MLX5_QP_OPTPAR_DC_HS = 1 << 20, MLX5_QP_OPTPAR_DC_KEY = 1 << 21, + MLX5_QP_OPTPAR_PP_INDEX = 1 << 22, MLX5_QP_OPTPAR_COUNTER_SET_ID = 1 << 25, }; diff --git a/include/linux/mlx5/vport.h b/include/linux/mlx5/vport.h index dfa2fe32217a..57c6b5bacedb 100644 --- a/include/linux/mlx5/vport.h +++ b/include/linux/mlx5/vport.h @@ -51,8 +51,8 @@ enum { /* Vport number for each function must keep unchanged */ enum { - MLX5_VPORT_PF = 0x0, - MLX5_VPORT_FIRST_VF = 0x1, + MLX5_VPORT_HOST_PF = 0x0, + MLX5_VPORT_FIRST_HOST_VF = 0x1, MLX5_VPORT_ECPF = 0xfffe, MLX5_VPORT_UPLINK = 0xffff }; @@ -61,7 +61,8 @@ u8 mlx5_query_vport_state(struct mlx5_core_dev *mdev, u8 opmod, u16 vport); int mlx5_modify_vport_admin_state(struct mlx5_core_dev *mdev, u8 opmod, u16 vport, u8 other_vport, u8 state); int mlx5_query_vport_max_tx_speed(struct mlx5_core_dev *mdev, u8 op_mod, - u16 vport, u8 other_vport, u32 *max_tx_speed); + u16 vport, u8 other_vport, + u32 *max_tx_speed, u8 *state); int mlx5_modify_vport_max_tx_speed(struct mlx5_core_dev *mdev, u8 opmod, u16 vport, u8 other_vport, u16 max_tx_speed); int mlx5_query_nic_vport_mac_address(struct mlx5_core_dev *mdev, @@ -78,7 +79,8 @@ int mlx5_query_nic_vport_mtu(struct mlx5_core_dev *mdev, u16 *mtu); int mlx5_modify_nic_vport_mtu(struct mlx5_core_dev *mdev, u16 mtu); int mlx5_query_nic_vport_system_image_guid(struct mlx5_core_dev *mdev, u64 *system_image_guid); -int mlx5_query_nic_vport_sd_group(struct mlx5_core_dev *mdev, u8 *sd_group); +int mlx5_query_nic_vport_sd_group(struct mlx5_core_dev *mdev, u8 *sd_group, + u8 *sd_group_size); int mlx5_query_nic_vport_node_guid(struct mlx5_core_dev *mdev, u16 vport, bool other_vport, u64 *node_guid); int mlx5_modify_nic_vport_node_guid(struct mlx5_core_dev *mdev, @@ -102,8 +104,8 @@ int mlx5_query_hca_vport_node_guid(struct mlx5_core_dev *dev, int mlx5_query_nic_vport_mac_list(struct mlx5_core_dev *dev, u16 vport, enum mlx5_list_type list_type, - u8 addr_list[][ETH_ALEN], - int *list_size); + u8 (**mac_list)[ETH_ALEN], + int *mac_list_size); int mlx5_modify_nic_vport_mac_list(struct mlx5_core_dev *dev, enum mlx5_list_type list_type, u8 addr_list[][ETH_ALEN], diff --git a/include/linux/mm.h b/include/linux/mm.h index abb4963c1f06..dd09c438fa23 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -27,7 +27,6 @@ #include <linux/page-flags.h> #include <linux/page_ref.h> #include <linux/overflow.h> -#include <linux/sizes.h> #include <linux/sched.h> #include <linux/pgtable.h> #include <linux/kasan.h> @@ -38,6 +37,7 @@ #include <linux/bitmap.h> #include <linux/bitops.h> #include <linux/iommu-debug-pagealloc.h> +#include <linux/kcsan-checks.h> struct mempolicy; struct anon_vma; @@ -208,8 +208,6 @@ static inline void __mm_zero_struct_page(struct page *page) #define MAPCOUNT_ELF_CORE_MARGIN (5) #define DEFAULT_MAX_MAP_COUNT (USHRT_MAX - MAPCOUNT_ELF_CORE_MARGIN) -extern int sysctl_max_map_count; - extern unsigned long sysctl_user_reserve_kbytes; extern unsigned long sysctl_admin_reserve_kbytes; @@ -349,13 +347,14 @@ enum { * if KVM does not lock down the memory type. */ DECLARE_VMA_BIT(ALLOW_ANY_UNCACHED, 39), -#ifdef CONFIG_PPC32 +#if defined(CONFIG_PPC32) DECLARE_VMA_BIT_ALIAS(DROPPABLE, ARCH_1), -#else +#elif defined(CONFIG_64BIT) DECLARE_VMA_BIT(DROPPABLE, 40), #endif DECLARE_VMA_BIT(UFFD_MINOR, 41), DECLARE_VMA_BIT(SEALED, 42), + DECLARE_VMA_BIT(UFFD_RWP, 43), /* Flags that reuse flags above. */ DECLARE_VMA_BIT_ALIAS(PKEY_BIT0, HIGH_ARCH_0), DECLARE_VMA_BIT_ALIAS(PKEY_BIT1, HIGH_ARCH_1), @@ -443,8 +442,10 @@ enum { #define VM_STACK INIT_VM_FLAG(STACK) #ifdef CONFIG_STACK_GROWSUP #define VM_STACK_EARLY INIT_VM_FLAG(STACK_EARLY) +#define VMA_STACK_EARLY mk_vma_flags(VMA_STACK_EARLY_BIT) #else #define VM_STACK_EARLY VM_NONE +#define VMA_STACK_EARLY EMPTY_VMA_FLAGS #endif #ifdef CONFIG_ARCH_HAS_PKEYS #define VM_PKEY_SHIFT ((__force int)VMA_HIGH_ARCH_0_BIT) @@ -466,13 +467,18 @@ enum { #if defined(CONFIG_X86_USER_SHADOW_STACK) || defined(CONFIG_ARM64_GCS) || \ defined(CONFIG_RISCV_USER_CFI) #define VM_SHADOW_STACK INIT_VM_FLAG(SHADOW_STACK) +#define VMA_SHADOW_STACK mk_vma_flags(VMA_SHADOW_STACK_BIT) +#define VMA_STARTGAP_FLAGS mk_vma_flags(VMA_GROWSDOWN_BIT, VMA_SHADOW_STACK_BIT) #else #define VM_SHADOW_STACK VM_NONE +#define VMA_SHADOW_STACK EMPTY_VMA_FLAGS +#define VMA_STARTGAP_FLAGS mk_vma_flags(VMA_GROWSDOWN_BIT) #endif #if defined(CONFIG_PPC64) #define VM_SAO INIT_VM_FLAG(SAO) #elif defined(CONFIG_PARISC) #define VM_GROWSUP INIT_VM_FLAG(GROWSUP) +#define VMA_GROWSUP mk_vma_flags(VMA_GROWSUP_BIT) #elif defined(CONFIG_SPARC64) #define VM_SPARC_ADI INIT_VM_FLAG(SPARC_ADI) #define VM_ARCH_CLEAR INIT_VM_FLAG(ARCH_CLEAR) @@ -484,6 +490,7 @@ enum { #endif #ifndef VM_GROWSUP #define VM_GROWSUP VM_NONE +#define VMA_GROWSUP EMPTY_VMA_FLAGS #endif #ifdef CONFIG_ARM64_MTE #define VM_MTE INIT_VM_FLAG(MTE) @@ -497,6 +504,31 @@ enum { #else #define VM_UFFD_MINOR VM_NONE #endif +#ifdef CONFIG_USERFAULTFD_RWP +#define VM_UFFD_RWP INIT_VM_FLAG(UFFD_RWP) +#else +#define VM_UFFD_RWP VM_NONE +#endif + +/* + * vma_flags_t masks for the userfaultfd VMA flags. The two high-bit modes are + * gated on the same configs as their VM_* flags above -- both of which imply + * 64BIT -- so an out-of-range bit is never fed to mk_vma_flags() on a build + * whose bitmap cannot hold it. + */ +#define VMA_UFFD_MISSING mk_vma_flags(VMA_UFFD_MISSING_BIT) +#define VMA_UFFD_WP mk_vma_flags(VMA_UFFD_WP_BIT) +#ifdef CONFIG_HAVE_ARCH_USERFAULTFD_MINOR +#define VMA_UFFD_MINOR mk_vma_flags(VMA_UFFD_MINOR_BIT) +#else +#define VMA_UFFD_MINOR EMPTY_VMA_FLAGS +#endif +#ifdef CONFIG_USERFAULTFD_RWP +#define VMA_UFFD_RWP mk_vma_flags(VMA_UFFD_RWP_BIT) +#else +#define VMA_UFFD_RWP EMPTY_VMA_FLAGS +#endif + #ifdef CONFIG_64BIT #define VM_ALLOW_ANY_UNCACHED INIT_VM_FLAG(ALLOW_ANY_UNCACHED) #define VM_SEALED INIT_VM_FLAG(SEALED) @@ -506,32 +538,43 @@ enum { #endif #if defined(CONFIG_64BIT) || defined(CONFIG_PPC32) #define VM_DROPPABLE INIT_VM_FLAG(DROPPABLE) +#define VMA_DROPPABLE mk_vma_flags(VMA_DROPPABLE_BIT) #else #define VM_DROPPABLE VM_NONE +#define VMA_DROPPABLE EMPTY_VMA_FLAGS #endif /* Bits set in the VMA until the stack is in its final location */ #define VM_STACK_INCOMPLETE_SETUP (VM_RAND_READ | VM_SEQ_READ | VM_STACK_EARLY) +#define VMA_STACK_INCOMPLETE_SETUP append_vma_flags( \ + VMA_STACK_EARLY, VMA_RAND_READ_BIT, VMA_SEQ_READ_BIT) -#define TASK_EXEC ((current->personality & READ_IMPLIES_EXEC) ? VM_EXEC : 0) +#define TASK_EXEC_BIT ((current->personality & READ_IMPLIES_EXEC) ? \ + VMA_EXEC_BIT : VMA_READ_BIT) /* Common data flag combinations */ -#define VM_DATA_FLAGS_TSK_EXEC (VM_READ | VM_WRITE | TASK_EXEC | \ - VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC) -#define VM_DATA_FLAGS_NON_EXEC (VM_READ | VM_WRITE | VM_MAYREAD | \ - VM_MAYWRITE | VM_MAYEXEC) -#define VM_DATA_FLAGS_EXEC (VM_READ | VM_WRITE | VM_EXEC | \ - VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC) - -#ifndef VM_DATA_DEFAULT_FLAGS /* arch can override this */ -#define VM_DATA_DEFAULT_FLAGS VM_DATA_FLAGS_EXEC +#define VMA_DATA_FLAGS_TSK_EXEC mk_vma_flags(VMA_READ_BIT, VMA_WRITE_BIT, \ + TASK_EXEC_BIT, VMA_MAYREAD_BIT, VMA_MAYWRITE_BIT, \ + VMA_MAYEXEC_BIT) +#define VMA_DATA_FLAGS_NON_EXEC mk_vma_flags(VMA_READ_BIT, VMA_WRITE_BIT, \ + VMA_MAYREAD_BIT, VMA_MAYWRITE_BIT, VMA_MAYEXEC_BIT) +#define VMA_DATA_FLAGS_EXEC mk_vma_flags(VMA_READ_BIT, VMA_WRITE_BIT, \ + VMA_EXEC_BIT, VMA_MAYREAD_BIT, VMA_MAYWRITE_BIT, \ + VMA_MAYEXEC_BIT) + +#ifndef VMA_DATA_DEFAULT_FLAGS /* arch can override this */ +#define VMA_DATA_DEFAULT_FLAGS VMA_DATA_FLAGS_EXEC #endif -#ifndef VM_STACK_DEFAULT_FLAGS /* arch can override this */ -#define VM_STACK_DEFAULT_FLAGS VM_DATA_DEFAULT_FLAGS +#ifndef VMA_STACK_DEFAULT_FLAGS /* arch can override this */ +#define VMA_STACK_DEFAULT_FLAGS VMA_DATA_DEFAULT_FLAGS #endif -#define VM_STARTGAP_FLAGS (VM_GROWSDOWN | VM_SHADOW_STACK) +#define VMA_STACK_FLAGS append_vma_flags(VMA_STACK_DEFAULT_FLAGS, \ + VMA_STACK_BIT, VMA_ACCOUNT_BIT) + +/* Temporary until VMA flags conversion complete. */ +#define VM_STACK_FLAGS vma_flags_to_legacy(VMA_STACK_FLAGS) #ifdef CONFIG_MSEAL_SYSTEM_MAPPINGS #define VM_SEALED_SYSMAP VM_SEALED @@ -539,15 +582,17 @@ enum { #define VM_SEALED_SYSMAP VM_NONE #endif -#define VM_STACK_FLAGS (VM_STACK | VM_STACK_DEFAULT_FLAGS | VM_ACCOUNT) - /* VMA basic access permission flags */ #define VM_ACCESS_FLAGS (VM_READ | VM_WRITE | VM_EXEC) +#define VMA_ACCESS_FLAGS mk_vma_flags(VMA_READ_BIT, VMA_WRITE_BIT, VMA_EXEC_BIT) /* * Special vmas that are non-mergable, non-mlock()able. */ -#define VM_SPECIAL (VM_IO | VM_DONTEXPAND | VM_PFNMAP | VM_MIXEDMAP) + +#define VMA_SPECIAL_FLAGS mk_vma_flags(VMA_IO_BIT, VMA_DONTEXPAND_BIT, \ + VMA_PFNMAP_BIT, VMA_MIXEDMAP_BIT) +#define VM_SPECIAL vma_flags_to_legacy(VMA_SPECIAL_FLAGS) /* * Physically remapped pages are special. Tell the @@ -574,6 +619,8 @@ enum { /* This mask represents all the VMA flag bits used by mlock */ #define VM_LOCKED_MASK (VM_LOCKED | VM_LOCKONFAULT) +#define VMA_LOCKED_MASK mk_vma_flags(VMA_LOCKED_BIT, VMA_LOCKONFAULT_BIT) + /* These flags can be updated atomically via VMA/mmap read lock. */ #define VM_ATOMIC_SET_ALLOWED VM_MAYBE_GUARD @@ -588,27 +635,32 @@ enum { * possesses it but the other does not, the merged VMA should nonetheless have * applied to it: * - * VM_SOFTDIRTY - if a VMA is marked soft-dirty, that is has not had its - * references cleared via /proc/$pid/clear_refs, any merged VMA - * should be considered soft-dirty also as it operates at a VMA - * granularity. + * VMA_SOFTDIRTY_BIT - if a VMA is marked soft-dirty, that is has not had its + * references cleared via /proc/$pid/clear_refs, any + * merged VMA should be considered soft-dirty also as it + * operates at a VMA granularity. * - * VM_MAYBE_GUARD - If a VMA may have guard regions in place it implies that - * mapped page tables may contain metadata not described by the - * VMA and thus any merged VMA may also contain this metadata, - * and thus we must make this flag sticky. + * VMA_MAYBE_GUARD_BIT - If a VMA may have guard regions in place it implies + * that mapped page tables may contain metadata not + * described by the VMA and thus any merged VMA may also + * contain this metadata, and thus we must make this flag + * sticky. */ -#define VM_STICKY (VM_SOFTDIRTY | VM_MAYBE_GUARD) +#ifdef CONFIG_MEM_SOFT_DIRTY +#define VMA_STICKY_FLAGS mk_vma_flags(VMA_SOFTDIRTY_BIT, VMA_MAYBE_GUARD_BIT) +#else +#define VMA_STICKY_FLAGS mk_vma_flags(VMA_MAYBE_GUARD_BIT) +#endif /* * VMA flags we ignore for the purposes of merge, i.e. one VMA possessing one * of these flags and the other not does not preclude a merge. * - * VM_STICKY - When merging VMAs, VMA flags must match, unless they are - * 'sticky'. If any sticky flags exist in either VMA, we simply - * set all of them on the merged VMA. + * VMA_STICKY_FLAGS - When merging VMAs, VMA flags must match, unless they + * are 'sticky'. If any sticky flags exist in either VMA, + * we simply set all of them on the merged VMA. */ -#define VM_IGNORE_MERGE VM_STICKY +#define VMA_IGNORE_MERGE_FLAGS VMA_STICKY_FLAGS /* * Flags which should result in page tables being copied on fork. These are @@ -616,29 +668,32 @@ enum { * reconsistuted upon page fault, so necessitate page table copying upon fork. * * Note that these flags should be compared with the DESTINATION VMA not the - * source, as VM_UFFD_WP may not be propagated to destination, while all other - * flags will be. + * source: VM_UFFD_WP and VM_UFFD_RWP may be cleared on the destination + * (dup_userfaultfd() -> userfaultfd_reset_ctx() when the parent context did + * not negotiate UFFD_FEATURE_EVENT_FORK), while all other flags propagate. * * VM_PFNMAP / VM_MIXEDMAP - These contain kernel-mapped data which cannot be * reasonably reconstructed on page fault. * * VM_UFFD_WP - Encodes metadata about an installed uffd - * write protect handler, which cannot be - * reconstructed on page fault. + * VM_UFFD_RWP write- or read-write-protect handler, which + * cannot be reconstructed on page fault. * - * We always copy pgtables when dst_vma has uffd-wp - * enabled even if it's file-backed - * (e.g. shmem). Because when uffd-wp is enabled, - * pgtable contains uffd-wp protection information, - * that's something we can't retrieve from page cache, - * and skip copying will lose those info. + * We always copy pgtables when dst_vma has the + * uffd PTE bit in use even if it's file-backed + * (e.g. shmem). Because when the uffd bit is + * in use, the pgtable contains the protection + * information, that's something we can't + * retrieve from page cache, and skip copying + * will lose those info. * * VM_MAYBE_GUARD - Could contain page guard region markers which * by design are a property of the page tables * only and thus cannot be reconstructed on page * fault. */ -#define VM_COPY_ON_FORK (VM_PFNMAP | VM_MIXEDMAP | VM_UFFD_WP | VM_MAYBE_GUARD) +#define VM_COPY_ON_FORK (VM_PFNMAP | VM_MIXEDMAP | VM_UFFD_WP | VM_UFFD_RWP | \ + VM_MAYBE_GUARD) /* * mapping from the currently active vm_flags protection bits (the @@ -741,21 +796,45 @@ struct vm_fault { */ }; +struct vm_uffd_ops; + /* * These are the virtual MM functions - opening of an area, closing and * unmapping it (needed to keep files on disk up-to-date etc), pointer * to the functions called when a no-page or a wp-page exception occurs. */ struct vm_operations_struct { - void (*open)(struct vm_area_struct * area); + /** + * @open: Called when a VMA is remapped, split or forked. Not called + * upon first mapping a VMA. + * Context: User context. May sleep. Caller holds mmap_lock. + */ + void (*open)(struct vm_area_struct *vma); /** * @close: Called when the VMA is being removed from the MM. * Context: User context. May sleep. Caller holds mmap_lock. */ - void (*close)(struct vm_area_struct * area); + void (*close)(struct vm_area_struct *vma); + /** + * @mapped: Called when the VMA is first mapped in the MM. Not called if + * the new VMA is merged with an adjacent VMA. + * + * The @vm_private_data field is an output field allowing the user to + * modify vma->vm_private_data as necessary. + * + * ONLY valid if set from f_op->mmap_prepare. Will result in an error if + * set from f_op->mmap. + * + * Returns %0 on success, or an error otherwise. On error, the VMA will + * be unmapped. + * + * Context: User context. May sleep. Caller holds mmap_lock. + */ + int (*mapped)(unsigned long start, unsigned long end, pgoff_t pgoff, + const struct file *file, void **vm_private_data); /* Called any time before splitting to check if it's allowed */ - int (*may_split)(struct vm_area_struct *area, unsigned long addr); - int (*mremap)(struct vm_area_struct *area); + int (*may_split)(struct vm_area_struct *vma, unsigned long addr); + int (*mremap)(struct vm_area_struct *vma); /* * Called by mprotect() to make driver-specific permission * checks before mprotect() is finalised. The VMA must not @@ -767,7 +846,7 @@ struct vm_operations_struct { vm_fault_t (*huge_fault)(struct vm_fault *vmf, unsigned int order); vm_fault_t (*map_pages)(struct vm_fault *vmf, pgoff_t start_pgoff, pgoff_t end_pgoff); - unsigned long (*pagesize)(struct vm_area_struct * area); + unsigned long (*pagesize)(struct vm_area_struct *vma); /* notification that a previously read-only page is about to become * writable, if an error is returned it will cause a SIGBUS */ @@ -826,6 +905,9 @@ struct vm_operations_struct { struct page *(*find_normal_page)(struct vm_area_struct *vma, unsigned long addr); #endif /* CONFIG_FIND_NORMAL_PAGE */ +#ifdef CONFIG_USERFAULTFD + const struct vm_uffd_ops *uffd_ops; +#endif }; #ifdef CONFIG_NUMA_BALANCING @@ -937,22 +1019,20 @@ static inline void vm_flags_reset(struct vm_area_struct *vma, vm_flags_init(vma, flags); } -static inline void vm_flags_reset_once(struct vm_area_struct *vma, - vm_flags_t flags) +static inline void vma_flags_reset_once(struct vm_area_struct *vma, + vma_flags_t *flags) { - vma_assert_write_locked(vma); - /* - * If VMA flags exist beyond the first system word, also clear these. It - * is assumed the write once behaviour is required only for the first - * system word. - */ + const unsigned long word = flags->__vma_flags[0]; + + /* It is assumed only the first system word must be written once. */ + vma_flags_overwrite_word_once(&vma->flags, word); + /* The remainder can be copied normally. */ if (NUM_VMA_FLAG_BITS > BITS_PER_LONG) { - unsigned long *bitmap = vma->flags.__vma_flags; + unsigned long *dst = &vma->flags.__vma_flags[1]; + const unsigned long *src = &flags->__vma_flags[1]; - bitmap_zero(&bitmap[1], NUM_VMA_FLAG_BITS - BITS_PER_LONG); + bitmap_copy(dst, src, NUM_VMA_FLAG_BITS - BITS_PER_LONG); } - - vma_flags_overwrite_word_once(&vma->flags, flags); } static inline void vm_flags_set(struct vm_area_struct *vma, @@ -991,7 +1071,8 @@ static inline void vm_flags_mod(struct vm_area_struct *vma, __vm_flags_mod(vma, set, clear); } -static inline bool __vma_atomic_valid_flag(struct vm_area_struct *vma, vma_flag_t bit) +static __always_inline bool __vma_atomic_valid_flag(struct vm_area_struct *vma, + vma_flag_t bit) { const vm_flags_t mask = BIT((__force int)bit); @@ -1006,7 +1087,8 @@ static inline bool __vma_atomic_valid_flag(struct vm_area_struct *vma, vma_flag_ * Set VMA flag atomically. Requires only VMA/mmap read lock. Only specific * valid flags are allowed to do this. */ -static inline void vma_set_atomic_flag(struct vm_area_struct *vma, vma_flag_t bit) +static __always_inline void vma_set_atomic_flag(struct vm_area_struct *vma, + vma_flag_t bit) { unsigned long *bitmap = vma->flags.__vma_flags; @@ -1022,7 +1104,8 @@ static inline void vma_set_atomic_flag(struct vm_area_struct *vma, vma_flag_t bi * This is necessarily racey, so callers must ensure that serialisation is * achieved through some other means, or that races are permissible. */ -static inline bool vma_test_atomic_flag(struct vm_area_struct *vma, vma_flag_t bit) +static __always_inline bool vma_test_atomic_flag(struct vm_area_struct *vma, + vma_flag_t bit) { if (__vma_atomic_valid_flag(vma, bit)) return test_bit((__force int)bit, &vma->vm_flags); @@ -1031,21 +1114,21 @@ static inline bool vma_test_atomic_flag(struct vm_area_struct *vma, vma_flag_t b } /* Set an individual VMA flag in flags, non-atomically. */ -static inline void vma_flag_set(vma_flags_t *flags, vma_flag_t bit) +static __always_inline void vma_flags_set_flag(vma_flags_t *flags, + vma_flag_t bit) { unsigned long *bitmap = flags->__vma_flags; __set_bit((__force int)bit, bitmap); } -static inline vma_flags_t __mk_vma_flags(size_t count, const vma_flag_t *bits) +static __always_inline vma_flags_t __mk_vma_flags(vma_flags_t flags, + size_t count, const vma_flag_t *bits) { - vma_flags_t flags; int i; - vma_flags_clear_all(&flags); for (i = 0; i < count; i++) - vma_flag_set(&flags, bits[i]); + vma_flags_set_flag(&flags, bits[i]); return flags; } @@ -1054,16 +1137,73 @@ static inline vma_flags_t __mk_vma_flags(size_t count, const vma_flag_t *bits) * vma_flags_t bitmap value. E.g.: * * vma_flags_t flags = mk_vma_flags(VMA_IO_BIT, VMA_PFNMAP_BIT, - * VMA_DONTEXPAND_BIT, VMA_DONTDUMP_BIT); + * VMA_DONTEXPAND_BIT, VMA_DONTDUMP_BIT); * * The compiler cleverly optimises away all of the work and this ends up being * equivalent to aggregating the values manually. */ -#define mk_vma_flags(...) __mk_vma_flags(COUNT_ARGS(__VA_ARGS__), \ - (const vma_flag_t []){__VA_ARGS__}) +#define mk_vma_flags(...) __mk_vma_flags(EMPTY_VMA_FLAGS, \ + COUNT_ARGS(__VA_ARGS__), (const vma_flag_t []){__VA_ARGS__}) + +/* + * Helper macro which acts like mk_vma_flags, only appending to a copy of the + * specified flags rather than establishing new flags. E.g.: + * + * vma_flags_t flags = append_vma_flags(VMA_STACK_DEFAULT_FLAGS, VMA_STACK_BIT, + * VMA_ACCOUNT_BIT); + */ +#define append_vma_flags(flags, ...) __mk_vma_flags(flags, \ + COUNT_ARGS(__VA_ARGS__), (const vma_flag_t []){__VA_ARGS__}) + +/* Calculates the number of set bits in the specified VMA flags. */ +static __always_inline int vma_flags_count(const vma_flags_t *flags) +{ + const unsigned long *bitmap = flags->__vma_flags; + + return bitmap_weight(bitmap, NUM_VMA_FLAG_BITS); +} + +/* + * Test whether a specific VMA flag is set, e.g.: + * + * if (vma_flags_test(flags, VMA_READ_BIT)) { ... } + */ +static __always_inline bool vma_flags_test(const vma_flags_t *flags, + vma_flag_t bit) +{ + const unsigned long *bitmap = flags->__vma_flags; + + return test_bit((__force int)bit, bitmap); +} + +/* + * Obtain a set of VMA flags which contain the overlapping flags contained + * within flags and to_and. + */ +static __always_inline vma_flags_t vma_flags_and_mask(const vma_flags_t *flags, + vma_flags_t to_and) +{ + vma_flags_t dst; + unsigned long *bitmap_dst = dst.__vma_flags; + const unsigned long *bitmap = flags->__vma_flags; + const unsigned long *bitmap_to_and = to_and.__vma_flags; + + bitmap_and(bitmap_dst, bitmap, bitmap_to_and, NUM_VMA_FLAG_BITS); + return dst; +} + +/* + * Obtain a set of VMA flags which contains the specified overlapping flags, + * e.g.: + * + * vma_flags_t read_flags = vma_flags_and(&flags, VMA_READ_BIT, + * VMA_MAY_READ_BIT); + */ +#define vma_flags_and(flags, ...) \ + vma_flags_and_mask(flags, mk_vma_flags(__VA_ARGS__)) /* Test each of to_test flags in flags, non-atomically. */ -static __always_inline bool vma_flags_test_mask(const vma_flags_t *flags, +static __always_inline bool vma_flags_test_any_mask(const vma_flags_t *flags, vma_flags_t to_test) { const unsigned long *bitmap = flags->__vma_flags; @@ -1075,10 +1215,10 @@ static __always_inline bool vma_flags_test_mask(const vma_flags_t *flags, /* * Test whether any specified VMA flag is set, e.g.: * - * if (vma_flags_test(flags, VMA_READ_BIT, VMA_MAYREAD_BIT)) { ... } + * if (vma_flags_test_any(flags, VMA_READ_BIT, VMA_MAYREAD_BIT)) { ... } */ -#define vma_flags_test(flags, ...) \ - vma_flags_test_mask(flags, mk_vma_flags(__VA_ARGS__)) +#define vma_flags_test_any(flags, ...) \ + vma_flags_test_any_mask(flags, mk_vma_flags(__VA_ARGS__)) /* Test that ALL of the to_test flags are set, non-atomically. */ static __always_inline bool vma_flags_test_all_mask(const vma_flags_t *flags, @@ -1098,8 +1238,29 @@ static __always_inline bool vma_flags_test_all_mask(const vma_flags_t *flags, #define vma_flags_test_all(flags, ...) \ vma_flags_test_all_mask(flags, mk_vma_flags(__VA_ARGS__)) +/* + * Helper to test that a flag mask of type vma_flags_t has a SINGLE flag set + * (returning false if flagmask has no flags set). + * + * This is defined to make the semantics clearer when testing an optionally + * defined VMA flags mask, e.g.: + * + * if (vma_flags_test_single_mask(&flags, VMA_DROPPABLE)) { ... } + * + * When VMA_DROPPABLE is defined if available, or set to EMPTY_VMA_FLAGS + * otherwise. + */ +static __always_inline bool vma_flags_test_single_mask(const vma_flags_t *flags, + vma_flags_t flagmask) +{ + VM_WARN_ON_ONCE(vma_flags_count(&flagmask) > 1); + + return vma_flags_test_any_mask(flags, flagmask); +} + /* Set each of the to_set flags in flags, non-atomically. */ -static __always_inline void vma_flags_set_mask(vma_flags_t *flags, vma_flags_t to_set) +static __always_inline void vma_flags_set_mask(vma_flags_t *flags, + vma_flags_t to_set) { unsigned long *bitmap = flags->__vma_flags; const unsigned long *bitmap_to_set = to_set.__vma_flags; @@ -1115,8 +1276,33 @@ static __always_inline void vma_flags_set_mask(vma_flags_t *flags, vma_flags_t t #define vma_flags_set(flags, ...) \ vma_flags_set_mask(flags, mk_vma_flags(__VA_ARGS__)) +static __always_inline vma_flags_t __mk_vma_flags_from_masks(size_t count, + const vma_flags_t *masks) +{ + vma_flags_t flags = EMPTY_VMA_FLAGS; + size_t i; + + for (i = 0; i < count; i++) + vma_flags_set_mask(&flags, masks[i]); + return flags; +} + +/* + * Combine pre-computed vma_flags_t masks into one value, e.g.: + * + * vma_flags_t flags = mk_vma_flags_from_masks(VMA_UFFD_WP, VMA_UFFD_MINOR); + * + * Unlike mk_vma_flags(), which takes bit numbers, this takes whole masks -- + * each of which may be EMPTY_VMA_FLAGS when its feature is unavailable -- so a + * bit that does not exist on the current build is never materialised. + */ +#define mk_vma_flags_from_masks(...) \ + __mk_vma_flags_from_masks(COUNT_ARGS(__VA_ARGS__), \ + (const vma_flags_t []){__VA_ARGS__}) + /* Clear all of the to-clear flags in flags, non-atomically. */ -static __always_inline void vma_flags_clear_mask(vma_flags_t *flags, vma_flags_t to_clear) +static __always_inline void vma_flags_clear_mask(vma_flags_t *flags, + vma_flags_t to_clear) { unsigned long *bitmap = flags->__vma_flags; const unsigned long *bitmap_to_clear = to_clear.__vma_flags; @@ -1133,13 +1319,85 @@ static __always_inline void vma_flags_clear_mask(vma_flags_t *flags, vma_flags_t vma_flags_clear_mask(flags, mk_vma_flags(__VA_ARGS__)) /* + * Obtain a VMA flags value containing those flags that are present in flags or + * flags_other but not in both. + */ +static __always_inline vma_flags_t vma_flags_diff_pair(const vma_flags_t *flags, + const vma_flags_t *flags_other) +{ + vma_flags_t dst; + const unsigned long *bitmap_other = flags_other->__vma_flags; + const unsigned long *bitmap = flags->__vma_flags; + unsigned long *bitmap_dst = dst.__vma_flags; + + bitmap_xor(bitmap_dst, bitmap, bitmap_other, NUM_VMA_FLAG_BITS); + return dst; +} + +/* Determine if flags and flags_other have precisely the same flags set. */ +static __always_inline bool vma_flags_same_pair(const vma_flags_t *flags, + const vma_flags_t *flags_other) +{ + const unsigned long *bitmap = flags->__vma_flags; + const unsigned long *bitmap_other = flags_other->__vma_flags; + + return bitmap_equal(bitmap, bitmap_other, NUM_VMA_FLAG_BITS); +} + +/* Determine if flags and flags_other have precisely the same flags set. */ +static __always_inline bool vma_flags_same_mask(const vma_flags_t *flags, + vma_flags_t flags_other) +{ + const unsigned long *bitmap = flags->__vma_flags; + const unsigned long *bitmap_other = flags_other.__vma_flags; + + return bitmap_equal(bitmap, bitmap_other, NUM_VMA_FLAG_BITS); +} + +/* + * Helper macro to determine if only the specific flags are set, e.g.: + * + * if (vma_flags_same(&flags, VMA_WRITE_BIT) { ... } + */ +#define vma_flags_same(flags, ...) \ + vma_flags_same_mask(flags, mk_vma_flags(__VA_ARGS__)) + +/* + * Test whether a specific flag in the VMA is set, e.g.: + * + * if (vma_test(vma, VMA_READ_BIT)) { ... } + */ +static __always_inline bool vma_test(const struct vm_area_struct *vma, + vma_flag_t bit) +{ + return vma_flags_test(&vma->flags, bit); +} + +/* Helper to test any VMA flags in a VMA . */ +static __always_inline bool vma_test_any_mask(const struct vm_area_struct *vma, + vma_flags_t flags) +{ + return vma_flags_test_any_mask(&vma->flags, flags); +} + +/* + * Helper macro for testing whether any VMA flags are set in a VMA, + * e.g.: + * + * if (vma_test_any(vma, VMA_IO_BIT, VMA_PFNMAP_BIT, + * VMA_DONTEXPAND_BIT, VMA_DONTDUMP_BIT)) { ... } + */ +#define vma_test_any(vma, ...) \ + vma_test_any_mask(vma, mk_vma_flags(__VA_ARGS__)) + +/* * Helper to test that ALL specified flags are set in a VMA. * * Note: appropriate locks must be held, this function does not acquire them for * you. */ -static inline bool vma_test_all_flags_mask(const struct vm_area_struct *vma, - vma_flags_t flags) +static __always_inline bool vma_test_all_mask(const struct vm_area_struct *vma, + vma_flags_t flags) { return vma_flags_test_all_mask(&vma->flags, flags); } @@ -1147,10 +1405,28 @@ static inline bool vma_test_all_flags_mask(const struct vm_area_struct *vma, /* * Helper macro for checking that ALL specified flags are set in a VMA, e.g.: * - * if (vma_test_all_flags(vma, VMA_READ_BIT, VMA_MAYREAD_BIT) { ... } + * if (vma_test_all(vma, VMA_READ_BIT, VMA_MAYREAD_BIT) { ... } */ -#define vma_test_all_flags(vma, ...) \ - vma_test_all_flags_mask(vma, mk_vma_flags(__VA_ARGS__)) +#define vma_test_all(vma, ...) \ + vma_test_all_mask(vma, mk_vma_flags(__VA_ARGS__)) + +/* + * Helper to test that a flag mask of type vma_flags_t has a SINGLE flag set + * (returning false if flagmask has no flags set). + * + * This is useful when a flag needs to be either defined or not depending upon + * kernel configuration, e.g.: + * + * if (vma_test_single_mask(vma, VMA_DROPPABLE)) { ... } + * + * When VMA_DROPPABLE is defined if available, or set to EMPTY_VMA_FLAGS + * otherwise. + */ +static __always_inline bool +vma_test_single_mask(const struct vm_area_struct *vma, vma_flags_t flagmask) +{ + return vma_flags_test_single_mask(&vma->flags, flagmask); +} /* * Helper to set all VMA flags in a VMA. @@ -1158,8 +1434,8 @@ static inline bool vma_test_all_flags_mask(const struct vm_area_struct *vma, * Note: appropriate locks must be held, this function does not acquire them for * you. */ -static inline void vma_set_flags_mask(struct vm_area_struct *vma, - vma_flags_t flags) +static __always_inline void vma_set_flags_mask(struct vm_area_struct *vma, + vma_flags_t flags) { vma_flags_set_mask(&vma->flags, flags); } @@ -1176,26 +1452,69 @@ static inline void vma_set_flags_mask(struct vm_area_struct *vma, #define vma_set_flags(vma, ...) \ vma_set_flags_mask(vma, mk_vma_flags(__VA_ARGS__)) -/* Helper to test all VMA flags in a VMA descriptor. */ -static inline bool vma_desc_test_flags_mask(const struct vm_area_desc *desc, - vma_flags_t flags) +/* Helper to clear all VMA flags in a VMA. */ +static __always_inline void vma_clear_flags_mask(struct vm_area_struct *vma, + vma_flags_t flags) { - return vma_flags_test_mask(&desc->vma_flags, flags); + vma_flags_clear_mask(&vma->flags, flags); } /* - * Helper macro for testing VMA flags for an input pointer to a struct - * vm_area_desc object describing a proposed VMA, e.g.: + * Helper macro for clearing VMA flags, e.g.: * - * if (vma_desc_test_flags(desc, VMA_IO_BIT, VMA_PFNMAP_BIT, + * vma_clear_flags(vma, VMA_IO_BIT, VMA_PFNMAP_BIT, VMA_DONTEXPAND_BIT, + * VMA_DONTDUMP_BIT); + */ +#define vma_clear_flags(vma, ...) \ + vma_clear_flags_mask(vma, mk_vma_flags(__VA_ARGS__)) + +/* + * Test whether a specific VMA flag is set in a VMA descriptor, e.g.: + * + * if (vma_desc_test(desc, VMA_READ_BIT)) { ... } + */ +static __always_inline bool vma_desc_test(const struct vm_area_desc *desc, + vma_flag_t bit) +{ + return vma_flags_test(&desc->vma_flags, bit); +} + +/* Helper to test any VMA flags in a VMA descriptor. */ +static __always_inline bool vma_desc_test_any_mask(const struct vm_area_desc *desc, + vma_flags_t flags) +{ + return vma_flags_test_any_mask(&desc->vma_flags, flags); +} + +/* + * Helper macro for testing whether any VMA flags are set in a VMA descriptor, + * e.g.: + * + * if (vma_desc_test_any(desc, VMA_IO_BIT, VMA_PFNMAP_BIT, * VMA_DONTEXPAND_BIT, VMA_DONTDUMP_BIT)) { ... } */ -#define vma_desc_test_flags(desc, ...) \ - vma_desc_test_flags_mask(desc, mk_vma_flags(__VA_ARGS__)) +#define vma_desc_test_any(desc, ...) \ + vma_desc_test_any_mask(desc, mk_vma_flags(__VA_ARGS__)) + +/* Helper to test all VMA flags in a VMA descriptor. */ +static __always_inline bool vma_desc_test_all_mask(const struct vm_area_desc *desc, + vma_flags_t flags) +{ + return vma_flags_test_all_mask(&desc->vma_flags, flags); +} + +/* + * Helper macro for testing whether ALL VMA flags are set in a VMA descriptor, + * e.g.: + * + * if (vma_desc_test_all(desc, VMA_READ_BIT, VMA_MAYREAD_BIT)) { ... } + */ +#define vma_desc_test_all(desc, ...) \ + vma_desc_test_all_mask(desc, mk_vma_flags(__VA_ARGS__)) /* Helper to set all VMA flags in a VMA descriptor. */ -static inline void vma_desc_set_flags_mask(struct vm_area_desc *desc, - vma_flags_t flags) +static __always_inline void vma_desc_set_flags_mask(struct vm_area_desc *desc, + vma_flags_t flags) { vma_flags_set_mask(&desc->vma_flags, flags); } @@ -1211,8 +1530,8 @@ static inline void vma_desc_set_flags_mask(struct vm_area_desc *desc, vma_desc_set_flags_mask(desc, mk_vma_flags(__VA_ARGS__)) /* Helper to clear all VMA flags in a VMA descriptor. */ -static inline void vma_desc_clear_flags_mask(struct vm_area_desc *desc, - vma_flags_t flags) +static __always_inline void vma_desc_clear_flags_mask(struct vm_area_desc *desc, + vma_flags_t flags) { vma_flags_clear_mask(&desc->vma_flags, flags); } @@ -1232,7 +1551,12 @@ static inline void vma_set_anonymous(struct vm_area_struct *vma) vma->vm_ops = NULL; } -static inline bool vma_is_anonymous(struct vm_area_struct *vma) +static inline void vma_desc_set_anonymous(struct vm_area_desc *desc) +{ + desc->vm_ops = NULL; +} + +static inline bool vma_is_anonymous(const struct vm_area_struct *vma) { return !vma->vm_ops; } @@ -1262,11 +1586,24 @@ static inline bool vma_is_initial_stack(const struct vm_area_struct *vma) vma->vm_end >= vma->vm_mm->start_stack; } -static inline bool vma_is_temporary_stack(const struct vm_area_struct *vma) +static inline bool vma_flags_can_grow(const vma_flags_t *flags) { - int maybe_stack = vma->vm_flags & (VM_GROWSDOWN | VM_GROWSUP); + if (vma_flags_test_single_mask(flags, VMA_GROWSUP)) + return true; + if (vma_flags_test(flags, VMA_GROWSDOWN_BIT)) + return true; - if (!maybe_stack) + return false; +} + +static inline bool vma_can_grow(const struct vm_area_struct *vma) +{ + return vma_flags_can_grow(&vma->flags); +} + +static inline bool vma_is_temporary_stack(const struct vm_area_struct *vma) +{ + if (!vma_can_grow(vma)) return false; if ((vma->vm_flags & VM_STACK_INCOMPLETE_SETUP) == @@ -1292,12 +1629,6 @@ static inline bool vma_is_accessible(const struct vm_area_struct *vma) return vma->vm_flags & VM_ACCESS_FLAGS; } -static inline bool is_shared_maywrite_vm_flags(vm_flags_t vm_flags) -{ - return (vm_flags & (VM_SHARED | VM_MAYWRITE)) == - (VM_SHARED | VM_MAYWRITE); -} - static inline bool is_shared_maywrite(const vma_flags_t *flags) { return vma_flags_test_all(flags, VMA_SHARED_BIT, VMA_MAYWRITE_BIT); @@ -1308,6 +1639,28 @@ static inline bool vma_is_shared_maywrite(const struct vm_area_struct *vma) return is_shared_maywrite(&vma->flags); } +/** + * vma_kernel_pagesize - Default page size granularity for this VMA. + * @vma: The user mapping. + * + * The kernel page size specifies in which granularity VMA modifications + * can be performed. Folios in this VMA will be aligned to, and at least + * the size of the number of bytes returned by this function. + * + * The default kernel page size is not affected by Transparent Huge Pages + * being in effect. + * + * Return: The default page size granularity for this VMA. + */ +static inline unsigned long vma_kernel_pagesize(struct vm_area_struct *vma) +{ + if (unlikely(vma->vm_ops && vma->vm_ops->pagesize)) + return vma->vm_ops->pagesize(vma); + return PAGE_SIZE; +} + +unsigned long vma_mmu_pagesize(struct vm_area_struct *vma); + static inline struct vm_area_struct *vma_find(struct vma_iterator *vmi, unsigned long max) { @@ -1507,7 +1860,7 @@ static inline int folio_put_testzero(struct folio *folio) */ static inline bool get_page_unless_zero(struct page *page) { - return page_ref_add_unless(page, 1, 0); + return page_ref_add_unless_zero(page, 1); } static inline struct folio *folio_get_nontail_page(struct page *page) @@ -1615,16 +1968,6 @@ static inline bool folio_mapped(const struct folio *folio) return folio_mapcount(folio) >= 1; } -/* - * Return true if this page is mapped into pagetables. - * For compound page it returns true if any sub-page of compound page is mapped, - * even if this particular sub-page is not itself mapped by any PTE or PMD. - */ -static inline bool page_mapped(const struct page *page) -{ - return folio_mapped(page_folio(page)); -} - static inline struct page *virt_to_head_page(const void *x) { struct page *page = virt_to_page(x); @@ -1928,17 +2271,76 @@ void unpin_user_pages(struct page **pages, unsigned long npages); void unpin_user_folio(struct folio *folio, unsigned long npages); void unpin_folios(struct folio **folios, unsigned long nfolios); -static inline bool is_cow_mapping(vm_flags_t flags) +/** + * vma_flags_is_cow_mapping() - Do these VMA flags imply a CoW mapping? + * @flags: The VMA flags to check. + * + * Mappings which could be CoW'd (subject to Copy-On-Write faults) are + * described as CoW mappings. + * + * All mappings backed by anonymous folios (all anonymous mappings and most + * MAP_PRIVATE-file backed ranges) are CoW mappings. + * + * All other mappings (including all MAP_SHARED mappings) are non-CoW. + * + * The criteria are !VMA_SHARED_BIT, VMA_MAYWRITE_BIT. + * + * VMA_MAYWRITE_BIT is checked instead of VMA_WRITE_BIT to account for both + * future mprotect() calls which can render a read-only mapping writable, and + * GUP with FOLL_FORCE (e.g. ptrace) which can CoW a read-only mapping. + * + * - No anonymous mapping can ever clear VMA_MAYWRITE_BIT. + * + * - Writes to anonymous mappings do not immediately result in CoW faults but + * may do so after the process is forked or if a read is followed by a + * write. + * + * - Writes to MAP_PRIVATE file-backed mappings result in CoW faults and may + * do so again after fork. + * + * - MAP_SHARED mappings of a file opened read-only are transformed into + * VMA_MAYSHARE_BIT, !VMA_SHARED_BIT, !VMA_MAYWRITE_BIT mappings, so remain + * non-CoW. + * + * - Drivers may clear VMA_MAYWRITE_BIT but do so at mmap() time and cannot + * mark themselves anonymous. Having cleared this flag it is not valid for + * them to leave the VMA_WRITE_BIT flag set. + * + * As a consequence, the anonymous reverse mapping only tracks CoW mappings. + * + * Returns: true if the flags indicate a CoW mapping, otherwise false. + */ +static inline bool vma_flags_is_cow_mapping(const vma_flags_t *flags) { - return (flags & (VM_SHARED | VM_MAYWRITE)) == VM_MAYWRITE; + return vma_flags_test(flags, VMA_MAYWRITE_BIT) && + !vma_flags_test(flags, VMA_SHARED_BIT); } -static inline bool vma_desc_is_cow_mapping(struct vm_area_desc *desc) +/** + * vma_is_cow_mapping() - Is this VMA a CoW mapping? + * @vma: The VMA to check. + * + * See vma_flags_is_cow_mapping() for details. + * + * Returns: true if the VMA is a CoW mapping, otherwise false. + */ +static inline bool vma_is_cow_mapping(const struct vm_area_struct *vma) { - const vma_flags_t *flags = &desc->vma_flags; + return vma_flags_is_cow_mapping(&vma->flags); +} - return vma_flags_test(flags, VMA_MAYWRITE_BIT) && - !vma_flags_test(flags, VMA_SHARED_BIT); +/** + * vma_desc_is_cow_mapping() - Is this VMA descriptor a CoW mapping? + * @desc: The VMA descriptor to check. + * + * See vma_flags_is_cow_mapping() for details. + * + * Returns: true if the VMA descriptor describes a CoW mapping, otherwise + * false. + */ +static inline bool vma_desc_is_cow_mapping(struct vm_area_desc *desc) +{ + return vma_flags_is_cow_mapping(&desc->vma_flags); } #ifndef CONFIG_MMU @@ -1957,7 +2359,7 @@ static inline bool is_nommu_shared_mapping(vm_flags_t flags) static inline bool is_nommu_shared_vma_flags(const vma_flags_t *flags) { - return vma_flags_test(flags, VMA_MAYSHARE_BIT, VMA_MAYOVERLAY_BIT); + return vma_flags_test_any(flags, VMA_MAYSHARE_BIT, VMA_MAYOVERLAY_BIT); } #endif @@ -1979,22 +2381,30 @@ static inline int page_zone_id(struct page *page) } #ifdef NODE_NOT_IN_PAGE_FLAGS -int memdesc_nid(memdesc_flags_t mdf); +int memdesc_nid(const memdesc_flags_t *mdf); #else -static inline int memdesc_nid(memdesc_flags_t mdf) +#ifdef CONFIG_NUMA +static inline int memdesc_nid(const memdesc_flags_t *mdf) { - return (mdf.f >> NODES_PGSHIFT) & NODES_MASK; + ASSERT_EXCLUSIVE_BITS(mdf->f, NODES_MASK << NODES_PGSHIFT); + return (mdf->f >> NODES_PGSHIFT) & NODES_MASK; } +#else +static inline int memdesc_nid(const memdesc_flags_t *mdf) +{ + return 0; +} +#endif #endif static inline int page_to_nid(const struct page *page) { - return memdesc_nid(PF_POISONED_CHECK(page)->flags); + return memdesc_nid(&(PF_POISONED_CHECK(page)->flags)); } static inline int folio_nid(const struct folio *folio) { - return memdesc_nid(folio->flags); + return memdesc_nid(&folio->flags); } #ifdef CONFIG_NUMA_BALANCING @@ -2234,12 +2644,13 @@ static inline void set_page_section(struct page *page, unsigned long section) page->flags.f |= (section & SECTIONS_MASK) << SECTIONS_PGSHIFT; } -static inline unsigned long memdesc_section(memdesc_flags_t mdf) +static inline unsigned long memdesc_section(const memdesc_flags_t *mdf) { - return (mdf.f >> SECTIONS_PGSHIFT) & SECTIONS_MASK; + ASSERT_EXCLUSIVE_BITS(mdf->f, SECTIONS_MASK << SECTIONS_PGSHIFT); + return (mdf->f >> SECTIONS_PGSHIFT) & SECTIONS_MASK; } #else /* !SECTION_IN_PAGE_FLAGS */ -static inline unsigned long memdesc_section(memdesc_flags_t mdf) +static inline unsigned long memdesc_section(const memdesc_flags_t *mdf) { return 0; } @@ -2479,36 +2890,6 @@ static inline unsigned long folio_nr_pages(const struct folio *folio) return folio_large_nr_pages(folio); } -#if !defined(CONFIG_HAVE_GIGANTIC_FOLIOS) -/* - * We don't expect any folios that exceed buddy sizes (and consequently - * memory sections). - */ -#define MAX_FOLIO_ORDER MAX_PAGE_ORDER -#elif defined(CONFIG_SPARSEMEM) && !defined(CONFIG_SPARSEMEM_VMEMMAP) -/* - * Only pages within a single memory section are guaranteed to be - * contiguous. By limiting folios to a single memory section, all folio - * pages are guaranteed to be contiguous. - */ -#define MAX_FOLIO_ORDER PFN_SECTION_SHIFT -#elif defined(CONFIG_HUGETLB_PAGE) -/* - * There is no real limit on the folio size. We limit them to the maximum we - * currently expect (see CONFIG_HAVE_GIGANTIC_FOLIOS): with hugetlb, we expect - * no folios larger than 16 GiB on 64bit and 1 GiB on 32bit. - */ -#define MAX_FOLIO_ORDER get_order(IS_ENABLED(CONFIG_64BIT) ? SZ_16G : SZ_1G) -#else -/* - * Without hugetlb, gigantic folios that are bigger than a single PUD are - * currently impossible. - */ -#define MAX_FOLIO_ORDER PUD_ORDER -#endif - -#define MAX_FOLIO_NR_PAGES (1UL << MAX_FOLIO_ORDER) - /* * compound_nr() returns the number of pages in this potentially compound * page. compound_nr() can be called on a tail page, and is defined to @@ -2667,7 +3048,7 @@ static inline bool folio_maybe_mapped_shared(struct folio *folio) * The caller must add any reference (e.g., from folio_try_get()) it might be * holding itself to the result. * - * Returns the expected folio refcount. + * Returns: the expected folio refcount. */ static inline int folio_expected_ref_count(const struct folio *folio) { @@ -2798,8 +3179,9 @@ extern void pagefault_out_of_memory(void); */ struct zap_details { struct folio *single_folio; /* Locked folio to be unmapped */ - bool even_cows; /* Zap COWed private pages too? */ + bool skip_cows; /* Do not zap COWed private pages */ bool reclaim_pt; /* Need reclaim page tables? */ + bool reaping; /* Reaping, do not block. */ zap_flags_t zap_flags; /* Extra flags for zapping */ }; @@ -2832,14 +3214,17 @@ struct page *vm_normal_page_pmd(struct vm_area_struct *vma, unsigned long addr, struct page *vm_normal_page_pud(struct vm_area_struct *vma, unsigned long addr, pud_t pud); -void zap_vma_ptes(struct vm_area_struct *vma, unsigned long address, +void zap_special_vma_range(struct vm_area_struct *vma, unsigned long address, unsigned long size); -void zap_page_range_single(struct vm_area_struct *vma, unsigned long address, - unsigned long size, struct zap_details *details); -static inline void zap_vma_pages(struct vm_area_struct *vma) +void zap_vma_range(struct vm_area_struct *vma, unsigned long address, + unsigned long size); +/** + * zap_vma - zap all page table entries in a vma + * @vma: The vma to zap. + */ +static inline void zap_vma(struct vm_area_struct *vma) { - zap_page_range_single(vma, vma->vm_start, - vma->vm_end - vma->vm_start, NULL); + zap_vma_range(vma, vma->vm_start, vma->vm_end - vma->vm_start); } struct mmu_notifier_range; @@ -3036,6 +3421,11 @@ int get_cmdline(struct task_struct *task, char *buffer, int buflen); #define MM_CP_UFFD_WP_RESOLVE (1UL << 3) /* Resolve wp */ #define MM_CP_UFFD_WP_ALL (MM_CP_UFFD_WP | \ MM_CP_UFFD_WP_RESOLVE) +/* Whether this change is for uffd RWP */ +#define MM_CP_UFFD_RWP (1UL << 4) /* do rwp */ +#define MM_CP_UFFD_RWP_RESOLVE (1UL << 5) /* resolve rwp */ +#define MM_CP_UFFD_RWP_ALL (MM_CP_UFFD_RWP | \ + MM_CP_UFFD_RWP_RESOLVE) bool can_change_pte_writable(struct vm_area_struct *vma, unsigned long addr, pte_t pte); @@ -3686,11 +4076,12 @@ extern unsigned long free_reserved_area(void *start, void *end, extern void adjust_managed_page_count(struct page *page, long count); -extern void reserve_bootmem_region(phys_addr_t start, - phys_addr_t end, int nid); +void free_reserved_pages(struct page *page, unsigned int order); -/* Free the reserved page into the buddy system, so it gets managed. */ -void free_reserved_page(struct page *page); +static inline void free_reserved_page(struct page *page) +{ + free_reserved_pages(page, 0); +} static inline void mark_page_reserved(struct page *page) { @@ -3764,7 +4155,7 @@ extern int __meminit early_pfn_to_nid(unsigned long pfn); extern void mem_init(void); extern void __init mmap_init(void); -extern void __show_mem(unsigned int flags, nodemask_t *nodemask, int max_zone_idx); +extern void __show_mem(unsigned int flags, const nodemask_t *nodemask, int max_zone_idx); static inline void show_mem(void) { __show_mem(0, NULL, MAX_NR_ZONES - 1); @@ -3774,7 +4165,7 @@ extern void si_meminfo(struct sysinfo * val); extern void si_meminfo_node(struct sysinfo *val, int nid); extern __printf(3, 4) -void warn_alloc(gfp_t gfp_mask, nodemask_t *nodemask, const char *fmt, ...); +void warn_alloc(gfp_t gfp_mask, const nodemask_t *nodemask, const char *fmt, ...); extern void setup_per_cpu_pageset(void); @@ -3783,44 +4174,46 @@ extern atomic_long_t mmap_pages_allocated; extern int nommu_shrink_inode_mappings(struct inode *, size_t, size_t); /* interval_tree.c */ -void vma_interval_tree_insert(struct vm_area_struct *node, - struct rb_root_cached *root); -void vma_interval_tree_insert_after(struct vm_area_struct *node, +void mapping_rmap_tree_insert(struct vm_area_struct *vma, + struct address_space *mapping); +void mapping_rmap_tree_insert_after(struct vm_area_struct *vma, struct vm_area_struct *prev, - struct rb_root_cached *root); -void vma_interval_tree_remove(struct vm_area_struct *node, - struct rb_root_cached *root); -struct vm_area_struct *vma_interval_tree_subtree_search(struct vm_area_struct *node, - unsigned long start, unsigned long last); -struct vm_area_struct *vma_interval_tree_iter_first(struct rb_root_cached *root, - unsigned long start, unsigned long last); -struct vm_area_struct *vma_interval_tree_iter_next(struct vm_area_struct *node, - unsigned long start, unsigned long last); - -#define vma_interval_tree_foreach(vma, root, start, last) \ - for (vma = vma_interval_tree_iter_first(root, start, last); \ - vma; vma = vma_interval_tree_iter_next(vma, start, last)) - -void anon_vma_interval_tree_insert(struct anon_vma_chain *node, - struct rb_root_cached *root); -void anon_vma_interval_tree_remove(struct anon_vma_chain *node, - struct rb_root_cached *root); + struct address_space *mapping); +void mapping_rmap_tree_remove(struct vm_area_struct *vma, + struct address_space *mapping); +struct vm_area_struct * +mapping_rmap_tree_iter_first(struct address_space *mapping, + pgoff_t pgoff_start, pgoff_t pgoff_last); +struct vm_area_struct * +mapping_rmap_tree_iter_next(struct vm_area_struct *vma, + pgoff_t pgoff_start, pgoff_t pgoff_last); + +#define mapping_rmap_tree_foreach(vma, mapping, pgoff_start, pgoff_last) \ + for (vma = mapping_rmap_tree_iter_first(mapping, pgoff_start, \ + pgoff_last); \ + vma; vma = mapping_rmap_tree_iter_next(vma, pgoff_start, \ + pgoff_last)) + +void anon_rmap_tree_insert(struct anon_vma_chain *avc, + struct anon_vma *anon_vma); +void anon_rmap_tree_remove(struct anon_vma_chain *avc, + struct anon_vma *anon_vma); +struct anon_vma_chain * +anon_rmap_tree_iter_first(struct anon_vma *anon_vma, + pgoff_t pgoff_start, pgoff_t pgoff_last); struct anon_vma_chain * -anon_vma_interval_tree_iter_first(struct rb_root_cached *root, - unsigned long start, unsigned long last); -struct anon_vma_chain *anon_vma_interval_tree_iter_next( - struct anon_vma_chain *node, unsigned long start, unsigned long last); +anon_rmap_tree_iter_next(struct anon_vma_chain *avc, + pgoff_t pgoff_start, pgoff_t pgoff_last); #ifdef CONFIG_DEBUG_VM_RB -void anon_vma_interval_tree_verify(struct anon_vma_chain *node); +void anon_rmap_tree_verify(struct anon_vma_chain *avc); #endif -#define anon_vma_interval_tree_foreach(avc, root, start, last) \ - for (avc = anon_vma_interval_tree_iter_first(root, start, last); \ - avc; avc = anon_vma_interval_tree_iter_next(avc, start, last)) +#define anon_rmap_tree_foreach(avc, anon_vma, pgoff_start, pgoff_last) \ + for (avc = anon_rmap_tree_iter_first(anon_vma, pgoff_start, pgoff_last); \ + avc; avc = anon_rmap_tree_iter_next(avc, pgoff_start, pgoff_last)) /* mmap.c */ extern int __vm_enough_memory(const struct mm_struct *mm, long pages, int cap_sys_admin); -extern int insert_vm_struct(struct mm_struct *, struct vm_area_struct *); extern void exit_mmap(struct mm_struct *); bool mmap_read_lock_maybe_expand(struct mm_struct *mm, struct vm_area_struct *vma, unsigned long addr, bool write); @@ -3847,7 +4240,6 @@ extern int replace_mm_exe_file(struct mm_struct *mm, struct file *new_exe_file); extern struct file *get_mm_exe_file(struct mm_struct *mm); extern struct file *get_task_exe_file(struct task_struct *task); -extern bool may_expand_vm(struct mm_struct *, vm_flags_t, unsigned long npages); extern void vm_stat_account(struct mm_struct *, vm_flags_t, long npages); extern bool vma_is_special_mapping(const struct vm_area_struct *vma, @@ -3862,18 +4254,20 @@ unsigned long randomize_page(unsigned long start, unsigned long range); unsigned long __get_unmapped_area(struct file *file, unsigned long addr, unsigned long len, - unsigned long pgoff, unsigned long flags, vm_flags_t vm_flags); + unsigned long pgoff, unsigned long flags, + vma_flags_t vma_flags); static inline unsigned long get_unmapped_area(struct file *file, unsigned long addr, unsigned long len, unsigned long pgoff, unsigned long flags) { - return __get_unmapped_area(file, addr, len, pgoff, flags, 0); + return __get_unmapped_area(file, addr, len, pgoff, flags, + EMPTY_VMA_FLAGS); } -extern unsigned long do_mmap(struct file *file, unsigned long addr, +unsigned long do_mmap(struct file *file, unsigned long addr, unsigned long len, unsigned long prot, unsigned long flags, - vm_flags_t vm_flags, unsigned long pgoff, unsigned long *populate, + vma_flags_t vma_flags, unsigned long pgoff, unsigned long *populate, struct list_head *uf); extern int do_vmi_munmap(struct vma_iterator *vmi, struct mm_struct *mm, unsigned long start, size_t len, struct list_head *uf, @@ -3898,11 +4292,13 @@ static inline void mm_populate(unsigned long addr, unsigned long len) {} #endif /* This takes the mm semaphore itself */ -extern int __must_check vm_brk_flags(unsigned long, unsigned long, unsigned long); -extern int vm_munmap(unsigned long, size_t); -extern unsigned long __must_check vm_mmap(struct file *, unsigned long, - unsigned long, unsigned long, - unsigned long, unsigned long); +int __must_check vm_brk_flags(unsigned long addr, unsigned long request, bool is_exec); +int vm_munmap(unsigned long start, size_t len); +unsigned long __must_check vm_mmap(struct file *file, unsigned long addr, + unsigned long len, unsigned long prot, + unsigned long flag, unsigned long offset); +unsigned long __must_check vm_mmap_shadow_stack(unsigned long addr, + unsigned long len, unsigned long flags); struct vm_unmapped_area_info { #define VM_UNMAPPED_AREA_TOPDOWN 1 @@ -3999,6 +4395,122 @@ static inline unsigned long vma_pages(const struct vm_area_struct *vma) return (vma->vm_end - vma->vm_start) >> PAGE_SHIFT; } +/** + * vma_start_pgoff() - Get the page offset of the start of @vma + * @vma: The VMA whose page offset is required. + * + * If the VMA is file-backed, this is the page offset into the file. + * + * If @vma is anonymous, this is the virtual page offset of the start of the + * VMA - if unfaulted, then vma->vm_start >> PAGE_SHIFT, if faulted then the + * virtual page offset at the time of first fault. + * + * If @vma is a MAP_PRIVATE file-backed mapping, then this returns the + * page offset within the file. + * + * Edge cases: nommu does not abide by these, MAP_PRIVATE-/dev/zero satisfies + * vma_is_anonymous() but has file-backed page offset, and MAP_PRIVATE-pfnmap + * regions have their page offset set to the first PFN in the range. + * + * Returns: The page offset of the start of @vma. + */ +static inline pgoff_t vma_start_pgoff(const struct vm_area_struct *vma) +{ + return vma->vm_pgoff; +} + +/** + * vma_end_pgoff() - Get the page offset of the exclusive end of @vma + * @vma: The VMA whose end page offset is required. + * + * This returns the exclusive end page offset of @vma, which is useful for + * expressing page offset ranges. + * + * See the description of vma_start_pgoff() for a description of VMA page + * offsets. + * + * Returns: The exclusive end page offset of @vma. + */ +static inline pgoff_t vma_end_pgoff(const struct vm_area_struct *vma) +{ + return vma_start_pgoff(vma) + vma_pages(vma); +} + +/** + * vma_last_pgoff() - Get the page offset of the last page in @vma + * @vma: The VMA whose last page offset is required. + * + * This returns the last page offset contained within @vma. + * + * See the description of vma_start_pgoff() for a description of VMA page + * offsets. + * + * Returns: The last page offset of @vma. + */ +static inline pgoff_t vma_last_pgoff(const struct vm_area_struct *vma) +{ + return vma_end_pgoff(vma) - 1; +} + +/** + * vma_start_anon_pgoff() - Get the anonymous page offset of the start of @vma + * @vma: The VMA whose anonymous page offset is required. + * + * If unfaulted, then this is vma->vm_start >> PAGE_SHIFT, if faulted then the + * anonymous page offset at the time of first fault. + * + * If the VMA is anonymous, this returns the same value as vma_start_pgoff(). + * + * This value is used for tracking MAP_PRIVATE file-backed mappings by their + * anonymous page offset. + * + * Returns: The anonymous page offset of the start of @vma. + */ +static inline pgoff_t vma_start_anon_pgoff(const struct vm_area_struct *vma) +{ + pgoff_t pgoff = 0; + +#ifdef CONFIG_64BIT + pgoff += vma->__vm_anon_pgoff_hi; + pgoff <<= 32; +#endif + pgoff += vma->__vm_anon_pgoff_lo; + return pgoff; +} + +/** + * vma_end_anon_pgoff() - Get the anonymous page offset of the exclusive end of + * @vma. + * @vma: The VMA whose end anonymous page offset is required. + * + * This returns the anonymous exclusive end page offset of @vma, which is useful + * for expressing page offset ranges. + * + * See the description of vma_start_anon_pgoff() for a description of VMA + * anonymous page offsets. + * + * Returns: The exclusive end anonymous page offset of @vma. + */ +static inline pgoff_t vma_end_anon_pgoff(const struct vm_area_struct *vma) +{ + return vma_start_anon_pgoff(vma) + vma_pages(vma); +} + +/** + * vma_last_anon_pgoff() - Get the anonymous page offset of the last page in + * @vma. + * @vma: The VMA whose last anonymous page offset is required. + * + * See the description of vma_start_anon_pgoff() for a description of VMA + * anonymous page offsets. + * + * Returns: The last anonymous page offset of @vma. + */ +static inline pgoff_t vma_last_anon_pgoff(const struct vm_area_struct *vma) +{ + return vma_end_anon_pgoff(vma) - 1; +} + static inline unsigned long vma_desc_size(const struct vm_area_desc *desc) { return desc->end - desc->start; @@ -4073,15 +4585,75 @@ static inline void mmap_action_ioremap(struct vm_area_desc *desc, * @start_pfn: The first PFN in the range to remap. */ static inline void mmap_action_ioremap_full(struct vm_area_desc *desc, - unsigned long start_pfn) + unsigned long start_pfn) { mmap_action_ioremap(desc, desc->start, start_pfn, vma_desc_size(desc)); } -void mmap_action_prepare(struct mmap_action *action, - struct vm_area_desc *desc); -int mmap_action_complete(struct mmap_action *action, - struct vm_area_struct *vma); +/** + * mmap_action_simple_ioremap - helper for mmap_prepare hook to specify that the + * physical range in [start_phys_addr, start_phys_addr + size) should be I/O + * remapped. + * @desc: The VMA descriptor for the VMA requiring remap. + * @start_phys_addr: Start of the physical memory to be mapped. + * @size: Size of the area to map. + * + * NOTE: Some drivers might want to tweak desc->page_prot for purposes of + * write-combine or similar. + */ +static inline void mmap_action_simple_ioremap(struct vm_area_desc *desc, + phys_addr_t start_phys_addr, + unsigned long size) +{ + struct mmap_action *action = &desc->action; + + action->simple_ioremap.start_phys_addr = start_phys_addr; + action->simple_ioremap.size = size; + action->type = MMAP_SIMPLE_IO_REMAP; +} + +/** + * mmap_action_map_kernel_pages - helper for mmap_prepare hook to specify that + * @num kernel pages contained in the @pages array should be mapped to userland + * starting at virtual address @start. + * @desc: The VMA descriptor for the VMA requiring kernel pags to be mapped. + * @start: The virtual address from which to map them. + * @pages: An array of struct page pointers describing the memory to map. + * @nr_pages: The number of entries in the @pages aray. + */ +static inline void mmap_action_map_kernel_pages(struct vm_area_desc *desc, + unsigned long start, struct page **pages, + unsigned long nr_pages) +{ + struct mmap_action *action = &desc->action; + + action->type = MMAP_MAP_KERNEL_PAGES; + action->map_kernel.start = start; + action->map_kernel.pages = pages; + action->map_kernel.nr_pages = nr_pages; + action->map_kernel.pgoff = desc->pgoff; +} + +/** + * mmap_action_map_kernel_pages_full - helper for mmap_prepare hook to specify that + * kernel pages contained in the @pages array should be mapped to userland + * from @desc->start to @desc->end. + * @desc: The VMA descriptor for the VMA requiring kernel pags to be mapped. + * @pages: An array of struct page pointers describing the memory to map. + * + * The caller must ensure that @pages contains sufficient entries to cover the + * entire range described by @desc. + */ +static inline void mmap_action_map_kernel_pages_full(struct vm_area_desc *desc, + struct page **pages) +{ + mmap_action_map_kernel_pages(desc, desc->start, pages, + vma_desc_pages(desc)); +} + +int mmap_action_prepare(struct vm_area_desc *desc); +int mmap_action_complete(struct vm_area_struct *vma, + struct mmap_action *action, bool is_compat); /* Look up the first VMA which exactly match the interval vm_start ... vm_end */ static inline struct vm_area_struct *find_exact_vma(struct mm_struct *mm, @@ -4095,23 +4667,93 @@ static inline struct vm_area_struct *find_exact_vma(struct mm_struct *mm, return vma; } +/** + * range_is_subset - Is the specified inner range a subset of the outer range? + * @outer_start: The start of the outer range. + * @outer_end: The exclusive end of the outer range. + * @inner_start: The start of the inner range. + * @inner_end: The exclusive end of the inner range. + * + * Returns: %true if [inner_start, inner_end) is a subset of [outer_start, + * outer_end), otherwise %false. + */ +static inline bool range_is_subset(unsigned long outer_start, + unsigned long outer_end, + unsigned long inner_start, + unsigned long inner_end) +{ + return outer_start <= inner_start && inner_end <= outer_end; +} + +/** + * range_in_vma - is the specified [@start, @end) range a subset of the VMA? + * @vma: The VMA against which we want to check [@start, @end). + * @start: The start of the range we wish to check. + * @end: The exclusive end of the range we wish to check. + * + * Returns: %true if [@start, @end) is a subset of [@vma->vm_start, + * @vma->vm_end), %false otherwise. + */ static inline bool range_in_vma(const struct vm_area_struct *vma, unsigned long start, unsigned long end) { - return (vma && vma->vm_start <= start && end <= vma->vm_end); + if (!vma) + return false; + + return range_is_subset(vma->vm_start, vma->vm_end, start, end); +} + +/** + * range_in_vma_desc - is the specified [@start, @end) range a subset of the VMA + * described by @desc, a VMA descriptor? + * @desc: The VMA descriptor against which we want to check [@start, @end). + * @start: The start of the range we wish to check. + * @end: The exclusive end of the range we wish to check. + * + * Returns: %true if [@start, @end) is a subset of [@desc->start, @desc->end), + * %false otherwise. + */ +static inline bool range_in_vma_desc(const struct vm_area_desc *desc, + unsigned long start, unsigned long end) +{ + if (!desc) + return false; + + return range_is_subset(desc->start, desc->end, start, end); } #ifdef CONFIG_MMU pgprot_t vm_get_page_prot(vm_flags_t vm_flags); + +static inline pgprot_t vma_flags_to_page_prot(vma_flags_t vma_flags) +{ + const vm_flags_t vm_flags = vma_flags_to_legacy(vma_flags); + + return vm_get_page_prot(vm_flags); +} + +static inline pgprot_t vma_get_page_prot(const struct vm_area_struct *vma) +{ + return vma_flags_to_page_prot(vma->flags); +} + void vma_set_page_prot(struct vm_area_struct *vma); #else static inline pgprot_t vm_get_page_prot(vm_flags_t vm_flags) { return __pgprot(0); } +static inline pgprot_t vma_flags_to_page_prot(vma_flags_t vma_flags) +{ + return __pgprot(0); +} +static inline pgprot_t vma_get_page_prot(const struct vm_area_struct *vma) +{ + return __pgprot(0); +} static inline void vma_set_page_prot(struct vm_area_struct *vma) { - vma->vm_page_prot = vm_get_page_prot(vma->vm_flags); + vma->vm_page_prot = vma_get_page_prot(vma); } #endif @@ -4130,6 +4772,9 @@ int remap_pfn_range(struct vm_area_struct *vma, unsigned long addr, int vm_insert_page(struct vm_area_struct *, unsigned long addr, struct page *); int vm_insert_pages(struct vm_area_struct *vma, unsigned long addr, struct page **pages, unsigned long *num); +int map_kernel_pages_prepare(struct vm_area_desc *desc); +int map_kernel_pages_complete(struct vm_area_struct *vma, + struct mmap_action *action); int vm_map_pages(struct vm_area_struct *vma, struct page **pages, unsigned long num); int vm_map_pages_zero(struct vm_area_struct *vma, struct page **pages, @@ -4217,12 +4862,26 @@ static inline int vm_fault_to_errno(vm_fault_t vm_fault, int foll_flags) /* * Indicates whether GUP can follow a PROT_NONE mapped page, or whether - * a (NUMA hinting) fault is required. + * a (NUMA hinting or userfaultfd RWP) fault is required. */ static inline bool gup_can_follow_protnone(const struct vm_area_struct *vma, unsigned int flags) { /* + * VM_UFFD_RWP uses protnone as an access-tracking marker, not for + * NUMA hinting. GUP must always take a fault so the access is + * delivered to userfaultfd, regardless of FOLL_HONOR_NUMA_FAULT. + * + * Only do so while the VMA is accessible. If it has been made + * inaccessible (e.g. mprotect(PROT_NONE)), fall through to the guard + * below: forcing a fault there would loop, as handle_mm_fault() makes + * no progress on protnone in an inaccessible VMA, and the access is + * denied regardless of RWP anyway. + */ + if (vma_test_single_mask(vma, VMA_UFFD_RWP) && vma_is_accessible(vma)) + return false; + + /* * If callers don't want to honor NUMA hinting faults, no need to * determine if we would actually have to trigger a NUMA hinting fault. */ @@ -4481,18 +5140,10 @@ static inline void print_vma_addr(char *prefix, unsigned long rip) } #endif -void *sparse_buffer_alloc(unsigned long size); unsigned long section_map_size(void); struct page * __populate_section_memmap(unsigned long pfn, unsigned long nr_pages, int nid, struct vmem_altmap *altmap, struct dev_pagemap *pgmap); -pgd_t *vmemmap_pgd_populate(unsigned long addr, int node); -p4d_t *vmemmap_p4d_populate(pgd_t *pgd, unsigned long addr, int node); -pud_t *vmemmap_pud_populate(p4d_t *p4d, unsigned long addr, int node); -pmd_t *vmemmap_pmd_populate(pud_t *pud, unsigned long addr, int node); -pte_t *vmemmap_pte_populate(pmd_t *pmd, unsigned long addr, int node, - struct vmem_altmap *altmap, unsigned long ptpfn, - unsigned long flags); void *vmemmap_alloc_block(unsigned long size, int node); struct vmem_altmap; void *vmemmap_alloc_block_buf(unsigned long size, int node, @@ -4508,10 +5159,9 @@ int vmemmap_populate_hugepages(unsigned long start, unsigned long end, int node, struct vmem_altmap *altmap); int vmemmap_populate(unsigned long start, unsigned long end, int node, struct vmem_altmap *altmap); -int vmemmap_populate_hvo(unsigned long start, unsigned long end, int node, +int vmemmap_populate_hvo(unsigned long start, unsigned long end, + unsigned int order, struct zone *zone, unsigned long headsize); -int vmemmap_undo_hvo(unsigned long start, unsigned long end, int node, - unsigned long headsize); void vmemmap_wrprotect_hvo(unsigned long start, unsigned long end, int node, unsigned long headsize); void vmemmap_populate_print_last(void); @@ -4602,8 +5252,6 @@ extern int soft_offline_page(unsigned long pfn, int flags); */ extern const struct attribute_group memory_failure_attr_group; extern void memory_failure_queue(unsigned long pfn, int flags); -extern int __get_huge_page_for_hwpoison(unsigned long pfn, int flags, - bool *migratable_cleared); void num_poisoned_pages_inc(unsigned long pfn); void num_poisoned_pages_sub(unsigned long pfn, long i); #else @@ -4611,12 +5259,6 @@ static inline void memory_failure_queue(unsigned long pfn, int flags) { } -static inline int __get_huge_page_for_hwpoison(unsigned long pfn, int flags, - bool *migratable_cleared) -{ - return 0; -} - static inline void num_poisoned_pages_inc(unsigned long pfn) { } @@ -4697,22 +5339,6 @@ long copy_folio_from_user(struct folio *dst_folio, const void __user *usr_src, bool allow_pagefault); -/** - * vma_is_special_huge - Are transhuge page-table entries considered special? - * @vma: Pointer to the struct vm_area_struct to consider - * - * Whether transhuge page-table entries are considered "special" following - * the definition in vm_normal_page(). - * - * Return: true if transhuge page-table entries should be considered special, - * false otherwise. - */ -static inline bool vma_is_special_huge(const struct vm_area_struct *vma) -{ - return vma_is_dax(vma) || (vma->vm_file && - (vma->vm_flags & (VM_PFNMAP | VM_MIXEDMAP))); -} - #endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_HUGETLBFS */ #if MAX_NUMNODES > 1 @@ -4783,13 +5409,9 @@ int reserve_mem_find_by_name(const char *name, phys_addr_t *start, phys_addr_t * int reserve_mem_release_by_name(const char *name); #ifdef CONFIG_64BIT -int do_mseal(unsigned long start, size_t len_in, unsigned long flags); +void mseal_mmap_page_zero(void); #else -static inline int do_mseal(unsigned long start, size_t len_in, unsigned long flags) -{ - /* noop on 32 bit */ - return 0; -} +static inline void mseal_mmap_page_zero(void) {} #endif /* @@ -4894,4 +5516,8 @@ static inline bool snapshot_page_is_faithful(const struct page_snapshot *ps) void snapshot_page(struct page_snapshot *ps, const struct page *page); +void map_anon_folio_pte_nopf(struct folio *folio, pte_t *pte, + struct vm_area_struct *vma, unsigned long addr, + bool uffd_wp); + #endif /* _LINUX_MM_H */ diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h index fa2d6ba811b5..621c8653d8f7 100644 --- a/include/linux/mm_inline.h +++ b/include/linux/mm_inline.h @@ -30,11 +30,6 @@ static inline int folio_is_file_lru(const struct folio *folio) return !folio_test_swapbacked(folio); } -static inline int page_is_file_lru(struct page *page) -{ - return folio_is_file_lru(page_folio(page)); -} - static __always_inline void __update_lru_size(struct lruvec *lruvec, enum lru_list lru, enum zone_type zid, long nr_pages) @@ -102,6 +97,12 @@ static __always_inline enum lru_list folio_lru_list(const struct folio *folio) #ifdef CONFIG_LRU_GEN +static inline bool lru_gen_switching(void) +{ + DECLARE_STATIC_KEY_FALSE(lru_switch); + + return static_branch_unlikely(&lru_switch); +} #ifdef CONFIG_LRU_GEN_ENABLED static inline bool lru_gen_enabled(void) { @@ -246,7 +247,7 @@ static inline unsigned long lru_gen_folio_seq(const struct lruvec *lruvec, (folio_test_dirty(folio) || folio_test_writeback(folio)))) gen = MIN_NR_GENS; else - gen = MAX_NR_GENS - folio_test_workingset(folio); + gen = MAX_NR_GENS - (folio_test_workingset(folio) || folio_test_referenced(folio)); return max(READ_ONCE(lrugen->max_seq) - gen + 1, READ_ONCE(lrugen->min_seq[type])); } @@ -316,6 +317,11 @@ static inline bool lru_gen_enabled(void) return false; } +static inline bool lru_gen_switching(void) +{ + return false; +} + static inline bool lru_gen_in_fault(void) { return false; @@ -342,6 +348,8 @@ void lruvec_add_folio(struct lruvec *lruvec, struct folio *folio) { enum lru_list lru = folio_lru_list(folio); + VM_WARN_ON_ONCE_FOLIO(!folio_matches_lruvec(folio, lruvec), folio); + if (lru_gen_add_folio(lruvec, folio, false)) return; @@ -356,6 +364,8 @@ void lruvec_add_folio_tail(struct lruvec *lruvec, struct folio *folio) { enum lru_list lru = folio_lru_list(folio); + VM_WARN_ON_ONCE_FOLIO(!folio_matches_lruvec(folio, lruvec), folio); + if (lru_gen_add_folio(lruvec, folio, true)) return; @@ -370,6 +380,8 @@ void lruvec_del_folio(struct lruvec *lruvec, struct folio *folio) { enum lru_list lru = folio_lru_list(folio); + VM_WARN_ON_ONCE_FOLIO(!folio_matches_lruvec(folio, lruvec), folio); + if (lru_gen_del_folio(lruvec, folio, false)) return; @@ -554,59 +566,6 @@ static inline pte_marker copy_pte_marker( return dstm; } -/* - * If this pte is wr-protected by uffd-wp in any form, arm the special pte to - * replace a none pte. NOTE! This should only be called when *pte is already - * cleared so we will never accidentally replace something valuable. Meanwhile - * none pte also means we are not demoting the pte so tlb flushed is not needed. - * E.g., when pte cleared the caller should have taken care of the tlb flush. - * - * Must be called with pgtable lock held so that no thread will see the none - * pte, and if they see it, they'll fault and serialize at the pgtable lock. - * - * Returns true if an uffd-wp pte was installed, false otherwise. - */ -static inline bool -pte_install_uffd_wp_if_needed(struct vm_area_struct *vma, unsigned long addr, - pte_t *pte, pte_t pteval) -{ - bool arm_uffd_pte = false; - - if (!uffd_supports_wp_marker()) - return false; - - /* The current status of the pte should be "cleared" before calling */ - WARN_ON_ONCE(!pte_none(ptep_get(pte))); - - /* - * NOTE: userfaultfd_wp_unpopulated() doesn't need this whole - * thing, because when zapping either it means it's dropping the - * page, or in TTU where the present pte will be quickly replaced - * with a swap pte. There's no way of leaking the bit. - */ - if (vma_is_anonymous(vma) || !userfaultfd_wp(vma)) - return false; - - /* A uffd-wp wr-protected normal pte */ - if (unlikely(pte_present(pteval) && pte_uffd_wp(pteval))) - arm_uffd_pte = true; - - /* - * A uffd-wp wr-protected swap pte. Note: this should even cover an - * existing pte marker with uffd-wp bit set. - */ - if (unlikely(pte_swp_uffd_wp_any(pteval))) - arm_uffd_pte = true; - - if (unlikely(arm_uffd_pte)) { - set_pte_at(vma->vm_mm, addr, pte, - make_pte_marker(PTE_MARKER_UFFD_WP)); - return true; - } - - return false; -} - static inline bool vma_has_recency(const struct vm_area_struct *vma) { if (vma->vm_flags & (VM_SEQ_READ | VM_RAND_READ)) @@ -638,7 +597,7 @@ static inline bool vma_has_recency(const struct vm_area_struct *vma) static inline size_t num_pages_contiguous(struct page **pages, size_t nr_pages) { struct page *cur_page = pages[0]; - unsigned long section = memdesc_section(cur_page->flags); + unsigned long section = memdesc_section(&cur_page->flags); size_t i; for (i = 1; i < nr_pages; i++) { @@ -648,7 +607,7 @@ static inline size_t num_pages_contiguous(struct page **pages, size_t nr_pages) * In unproblematic kernel configs, page_to_section() == 0 and * the whole check will get optimized out. */ - if (memdesc_section(cur_page->flags) != section) + if (memdesc_section(&cur_page->flags) != section) break; } diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index 3cc8ae722886..6d815f6440c9 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -20,6 +20,7 @@ #include <linux/seqlock.h> #include <linux/percpu_counter.h> #include <linux/types.h> +#include <linux/futex_types.h> #include <linux/rseq_types.h> #include <linux/bitmap.h> @@ -126,14 +127,14 @@ struct page { atomic_long_t pp_ref_count; }; struct { /* Tail pages of compound page */ - unsigned long compound_head; /* Bit zero is set */ + unsigned long compound_info; /* Bit zero is set */ }; struct { /* ZONE_DEVICE pages */ /* - * The first word is used for compound_head or folio + * The first word is used for compound_info or folio * pgmap */ - void *_unused_pgmap_compound_head; + void *_unused_pgmap_compound_info; void *zone_device_data; /* * ZONE_DEVICE private pages are counted as being @@ -367,6 +368,8 @@ typedef unsigned short mm_id_t; * dax_associate_entry. * @private: Filesystem per-folio data (see folio_attach_private()). * @swap: Used for swp_entry_t if folio_test_swapcache(). + * @migrate_info: Stores migration state (anon_vma pointer and + * FOLIO_WAS_* markers). * @_mapcount: Do not access this member directly. Use folio_mapcount() to * find out how many times this folio is mapped by userspace. * @_refcount: Do not access this member directly. Use folio_ref_count() @@ -409,7 +412,7 @@ struct folio { /* private: avoid cluttering the output */ /* For the Unevictable "LRU list" slot */ struct { - /* Avoid compound_head */ + /* Avoid compound_info */ void *__filler; /* public: */ unsigned int mlock_count; @@ -426,6 +429,7 @@ struct folio { union { void *private; swp_entry_t swap; + unsigned long migrate_info; }; atomic_t _mapcount; atomic_t _refcount; @@ -510,7 +514,7 @@ struct folio { FOLIO_MATCH(flags, flags); FOLIO_MATCH(lru, lru); FOLIO_MATCH(mapping, mapping); -FOLIO_MATCH(compound_head, lru); +FOLIO_MATCH(compound_info, lru); FOLIO_MATCH(__folio_index, index); FOLIO_MATCH(private, private); FOLIO_MATCH(_mapcount, _mapcount); @@ -529,7 +533,7 @@ FOLIO_MATCH(_last_cpupid, _last_cpupid); static_assert(offsetof(struct folio, fl) == \ offsetof(struct page, pg) + sizeof(struct page)) FOLIO_MATCH(flags, _flags_1); -FOLIO_MATCH(compound_head, _head_1); +FOLIO_MATCH(compound_info, _head_1); FOLIO_MATCH(_mapcount, _mapcount_1); FOLIO_MATCH(_refcount, _refcount_1); #undef FOLIO_MATCH @@ -537,13 +541,13 @@ FOLIO_MATCH(_refcount, _refcount_1); static_assert(offsetof(struct folio, fl) == \ offsetof(struct page, pg) + 2 * sizeof(struct page)) FOLIO_MATCH(flags, _flags_2); -FOLIO_MATCH(compound_head, _head_2); +FOLIO_MATCH(compound_info, _head_2); #undef FOLIO_MATCH #define FOLIO_MATCH(pg, fl) \ static_assert(offsetof(struct folio, fl) == \ offsetof(struct page, pg) + 3 * sizeof(struct page)) FOLIO_MATCH(flags, _flags_3); -FOLIO_MATCH(compound_head, _head_3); +FOLIO_MATCH(compound_info, _head_3); #undef FOLIO_MATCH /** @@ -609,8 +613,8 @@ struct ptdesc { #define TABLE_MATCH(pg, pt) \ static_assert(offsetof(struct page, pg) == offsetof(struct ptdesc, pt)) TABLE_MATCH(flags, pt_flags); -TABLE_MATCH(compound_head, pt_list); -TABLE_MATCH(compound_head, _pt_pad_1); +TABLE_MATCH(compound_info, pt_list); +TABLE_MATCH(compound_info, _pt_pad_1); TABLE_MATCH(mapping, __page_mapping); TABLE_MATCH(__folio_index, pt_index); TABLE_MATCH(rcu_head, pt_rcu_head); @@ -814,6 +818,8 @@ enum mmap_action_type { MMAP_NOTHING, /* Mapping is complete, no further action. */ MMAP_REMAP_PFN, /* Remap PFN range. */ MMAP_IO_REMAP_PFN, /* I/O remap PFN range. */ + MMAP_SIMPLE_IO_REMAP, /* I/O remap with guardrails. */ + MMAP_MAP_KERNEL_PAGES, /* Map kernel page range from array. */ }; /* @@ -822,34 +828,30 @@ enum mmap_action_type { */ struct mmap_action { union { - /* Remap range. */ struct { unsigned long start; unsigned long start_pfn; unsigned long size; pgprot_t pgprot; } remap; + struct { + phys_addr_t start_phys_addr; + unsigned long size; + } simple_ioremap; + struct { + unsigned long start; + struct page **pages; + unsigned long nr_pages; + pgoff_t pgoff; + } map_kernel; }; enum mmap_action_type type; /* - * If specified, this hook is invoked after the selected action has been - * successfully completed. Note that the VMA write lock still held. - * - * The absolute minimum ought to be done here. - * - * Returns 0 on success, or an error code. + * If non-zero, replace errors that arise from mmap actions with this + * value instead. Only valid error codes may be specified. */ - int (*success_hook)(const struct vm_area_struct *vma); - - /* - * If specified, this hook is invoked when an error occurred when - * attempting the selected action. - * - * The hook can return an error code in order to filter the error, but - * it is not valid to clear the error here. - */ - int (*error_hook)(int err); + int error_override; /* * This should be set in rare instances where the operation required @@ -870,6 +872,14 @@ typedef struct { #define EMPTY_VMA_FLAGS ((vma_flags_t){ }) +/* Are no flags set in the specified VMA flags? */ +static __always_inline bool vma_flags_empty(const vma_flags_t *flags) +{ + const unsigned long *bitmap = flags->__vma_flags; + + return bitmap_empty(bitmap, NUM_VMA_FLAG_BITS); +} + /* * Describes a VMA that is about to be mmap()'ed. Drivers may choose to * manipulate mutable fields which will cause those fields to be updated in the @@ -879,8 +889,8 @@ typedef struct { */ struct vm_area_desc { /* Immutable state. */ - const struct mm_struct *const mm; - struct file *const file; /* May vary from vm_file in stacked callers. */ + struct mm_struct *mm; + struct file *file; /* May vary from vm_file in stacked callers. */ unsigned long start; unsigned long end; @@ -958,6 +968,11 @@ struct vm_area_struct { unsigned int vm_lock_seq; #endif /* + * Low 32-bits of anonymous page offset. + * See vma_start_anon_pgoff() comment for details. + */ + unsigned int __vm_anon_pgoff_lo; + /* * A file's MAP_PRIVATE vma can be in both i_mmap tree and anon_vma * list, after a COW of one of the file pages. A MAP_SHARED vma * can only be in the i_mmap tree. An anonymous MAP_PRIVATE, stack @@ -1032,6 +1047,13 @@ struct vm_area_struct { struct lockdep_map vmlock_dep_map; #endif #endif +#ifdef CONFIG_64BIT + /* + * High 32-bits of anonymous page offset. + * See vma_start_anon_pgoff() comment for details. + */ + unsigned int __vm_anon_pgoff_hi; +#endif /* * For areas with an address space and backing store, * linkage into the address_space->i_mmap interval tree. @@ -1056,18 +1078,31 @@ struct vm_area_struct { } __randomize_layout; /* Clears all bits in the VMA flags bitmap, non-atomically. */ -static inline void vma_flags_clear_all(vma_flags_t *flags) +static __always_inline void vma_flags_clear_all(vma_flags_t *flags) { bitmap_zero(flags->__vma_flags, NUM_VMA_FLAG_BITS); } /* + * Helper function which converts a vma_flags_t value to a legacy vm_flags_t + * value. This is only valid if the input flags value can be expressed in a + * system word. + * + * Will be removed once the conversion to VMA flags is complete. + */ +static __always_inline vm_flags_t vma_flags_to_legacy(vma_flags_t flags) +{ + return (vm_flags_t)flags.__vma_flags[0]; +} + +/* * Copy value to the first system word of VMA flags, non-atomically. * * IMPORTANT: This does not overwrite bytes past the first system word. The * caller must account for this. */ -static inline void vma_flags_overwrite_word(vma_flags_t *flags, unsigned long value) +static __always_inline void vma_flags_overwrite_word(vma_flags_t *flags, + unsigned long value) { unsigned long *bitmap = flags->__vma_flags; @@ -1075,12 +1110,27 @@ static inline void vma_flags_overwrite_word(vma_flags_t *flags, unsigned long va } /* + * Helper function which converts a legacy vm_flags_t value to a vma_flags_t + * value. + * + * Will be removed once the conversion to VMA flags is complete. + */ +static __always_inline vma_flags_t legacy_to_vma_flags(vm_flags_t flags) +{ + vma_flags_t ret = EMPTY_VMA_FLAGS; + + vma_flags_overwrite_word(&ret, flags); + return ret; +} + +/* * Copy value to the first system word of VMA flags ONCE, non-atomically. * * IMPORTANT: This does not overwrite bytes past the first system word. The * caller must account for this. */ -static inline void vma_flags_overwrite_word_once(vma_flags_t *flags, unsigned long value) +static __always_inline void vma_flags_overwrite_word_once(vma_flags_t *flags, + unsigned long value) { unsigned long *bitmap = flags->__vma_flags; @@ -1088,7 +1138,8 @@ static inline void vma_flags_overwrite_word_once(vma_flags_t *flags, unsigned lo } /* Update the first system word of VMA flags setting bits, non-atomically. */ -static inline void vma_flags_set_word(vma_flags_t *flags, unsigned long value) +static __always_inline void vma_flags_set_word(vma_flags_t *flags, + unsigned long value) { unsigned long *bitmap = flags->__vma_flags; @@ -1096,7 +1147,8 @@ static inline void vma_flags_set_word(vma_flags_t *flags, unsigned long value) } /* Update the first system word of VMA flags clearing bits, non-atomically. */ -static inline void vma_flags_clear_word(vma_flags_t *flags, unsigned long value) +static __always_inline void vma_flags_clear_word(vma_flags_t *flags, + unsigned long value) { unsigned long *bitmap = flags->__vma_flags; @@ -1173,6 +1225,8 @@ struct mm_struct { /* MM CID related storage */ struct mm_mm_cid mm_cid; + /* sched_cache related statistics */ + struct sched_cache_stat sc_stat; #ifdef CONFIG_MMU atomic_long_t pgtables_bytes; /* size of all page tables */ #endif @@ -1221,16 +1275,7 @@ struct mm_struct { */ seqcount_t mm_lock_seq; #endif -#ifdef CONFIG_FUTEX_PRIVATE_HASH - struct mutex futex_hash_lock; - struct futex_private_hash __rcu *futex_phash; - struct futex_private_hash *futex_phash_new; - /* futex-ref */ - unsigned long futex_batches; - struct rcu_head futex_rcu; - atomic_long_t futex_atomic; - unsigned int __percpu *futex_ref; -#endif + struct futex_mm_data futex; unsigned long hiwater_rss; /* High-watermark of RSS usage */ unsigned long hiwater_vm; /* High-water virtual memory usage */ @@ -1241,7 +1286,11 @@ struct mm_struct { unsigned long data_vm; /* VM_WRITE & ~VM_SHARED & ~VM_STACK */ unsigned long exec_vm; /* VM_EXEC & ~VM_WRITE & ~VM_STACK */ unsigned long stack_vm; /* VM_STACK */ - vm_flags_t def_flags; + union { + /* Temporary while VMA flags are being converted. */ + vm_flags_t def_flags; + vma_flags_t def_vma_flags; + }; /** * @write_protect_seq: Locked when any thread is write @@ -1289,7 +1338,6 @@ struct mm_struct { */ struct task_struct __rcu *owner; #endif - struct user_namespace *user_ns; /* store ref to file /proc/<pid>/exe symlink points to */ struct file __rcu *exe_file; @@ -1575,6 +1623,36 @@ static inline unsigned int mm_cid_size(void) # define MM_CID_STATIC_SIZE 0 #endif /* CONFIG_SCHED_MM_CID */ +#ifdef CONFIG_SCHED_CACHE +void mm_init_sched(struct mm_struct *mm, + struct sched_cache_time __percpu *pcpu_sched); + +static inline int mm_alloc_sched_noprof(struct mm_struct *mm) +{ + struct sched_cache_time __percpu *pcpu_sched = + alloc_percpu_noprof(struct sched_cache_time); + + if (!pcpu_sched) + return -ENOMEM; + + mm_init_sched(mm, pcpu_sched); + return 0; +} + +#define mm_alloc_sched(...) alloc_hooks(mm_alloc_sched_noprof(__VA_ARGS__)) + +static inline void mm_destroy_sched(struct mm_struct *mm) +{ + free_percpu(mm->sc_stat.pcpu_sched); + mm->sc_stat.pcpu_sched = NULL; +} +#else /* !CONFIG_SCHED_CACHE */ + +static inline int mm_alloc_sched(struct mm_struct *mm) { return 0; } +static inline void mm_destroy_sched(struct mm_struct *mm) { } + +#endif /* CONFIG_SCHED_CACHE */ + struct mmu_gather; extern void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm); extern void tlb_gather_mmu_fullmm(struct mmu_gather *tlb, struct mm_struct *mm); @@ -1640,20 +1718,20 @@ enum vm_fault_reason { VM_FAULT_SIGSEGV | VM_FAULT_HWPOISON | \ VM_FAULT_HWPOISON_LARGE | VM_FAULT_FALLBACK) -#define VM_FAULT_RESULT_TRACE \ - { VM_FAULT_OOM, "OOM" }, \ - { VM_FAULT_SIGBUS, "SIGBUS" }, \ - { VM_FAULT_MAJOR, "MAJOR" }, \ - { VM_FAULT_HWPOISON, "HWPOISON" }, \ - { VM_FAULT_HWPOISON_LARGE, "HWPOISON_LARGE" }, \ - { VM_FAULT_SIGSEGV, "SIGSEGV" }, \ - { VM_FAULT_NOPAGE, "NOPAGE" }, \ - { VM_FAULT_LOCKED, "LOCKED" }, \ - { VM_FAULT_RETRY, "RETRY" }, \ - { VM_FAULT_FALLBACK, "FALLBACK" }, \ - { VM_FAULT_DONE_COW, "DONE_COW" }, \ - { VM_FAULT_NEEDDSYNC, "NEEDDSYNC" }, \ - { VM_FAULT_COMPLETED, "COMPLETED" } +#define VM_FAULT_RESULT_TRACE \ + { (__force u32)VM_FAULT_OOM, "OOM" }, \ + { (__force u32)VM_FAULT_SIGBUS, "SIGBUS" }, \ + { (__force u32)VM_FAULT_MAJOR, "MAJOR" }, \ + { (__force u32)VM_FAULT_HWPOISON, "HWPOISON" }, \ + { (__force u32)VM_FAULT_HWPOISON_LARGE, "HWPOISON_LARGE" }, \ + { (__force u32)VM_FAULT_SIGSEGV, "SIGSEGV" }, \ + { (__force u32)VM_FAULT_NOPAGE, "NOPAGE" }, \ + { (__force u32)VM_FAULT_LOCKED, "LOCKED" }, \ + { (__force u32)VM_FAULT_RETRY, "RETRY" }, \ + { (__force u32)VM_FAULT_FALLBACK, "FALLBACK" }, \ + { (__force u32)VM_FAULT_DONE_COW, "DONE_COW" }, \ + { (__force u32)VM_FAULT_NEEDDSYNC, "NEEDDSYNC" }, \ + { (__force u32)VM_FAULT_COMPLETED, "COMPLETED" } struct vm_special_mapping { const char *name; /* The name, e.g. "[vdso]". */ @@ -1854,11 +1932,11 @@ enum { /* mm flags */ /* - * The first two bits represent core dump modes for set-user-ID, - * the modes are SUID_DUMP_* defined in linux/sched/coredump.h + * Bits 0 and 1 were dumpability; that moved to task->exec_state. Reserve + * the bits so MMF_DUMP_FILTER_* positions stay stable for the + * /proc/<pid>/coredump_filter ABI. */ #define MMF_DUMPABLE_BITS 2 -#define MMF_DUMPABLE_MASK (BIT(MMF_DUMPABLE_BITS) - 1) /* coredump filter bits */ #define MMF_DUMP_ANON_PRIVATE 2 #define MMF_DUMP_ANON_SHARED 3 @@ -1919,7 +1997,7 @@ enum { #define MMF_TOPDOWN 31 /* mm searches top down by default */ #define MMF_TOPDOWN_MASK BIT(MMF_TOPDOWN) -#define MMF_INIT_LEGACY_MASK (MMF_DUMPABLE_MASK | MMF_DUMP_FILTER_MASK |\ +#define MMF_INIT_LEGACY_MASK (MMF_DUMP_FILTER_MASK |\ MMF_DISABLE_THP_MASK | MMF_HAS_MDWE_MASK |\ MMF_VM_MERGE_ANY_MASK | MMF_TOPDOWN_MASK) diff --git a/include/linux/mman.h b/include/linux/mman.h index 0ba8a7e8b90a..389521594c69 100644 --- a/include/linux/mman.h +++ b/include/linux/mman.h @@ -170,53 +170,4 @@ static inline bool arch_memory_deny_write_exec_supported(void) } #define arch_memory_deny_write_exec_supported arch_memory_deny_write_exec_supported #endif - -/* - * Denies creating a writable executable mapping or gaining executable permissions. - * - * This denies the following: - * - * a) mmap(PROT_WRITE | PROT_EXEC) - * - * b) mmap(PROT_WRITE) - * mprotect(PROT_EXEC) - * - * c) mmap(PROT_WRITE) - * mprotect(PROT_READ) - * mprotect(PROT_EXEC) - * - * But allows the following: - * - * d) mmap(PROT_READ | PROT_EXEC) - * mmap(PROT_READ | PROT_EXEC | PROT_BTI) - * - * This is only applicable if the user has set the Memory-Deny-Write-Execute - * (MDWE) protection mask for the current process. - * - * @old specifies the VMA flags the VMA originally possessed, and @new the ones - * we propose to set. - * - * Return: false if proposed change is OK, true if not ok and should be denied. - */ -static inline bool map_deny_write_exec(unsigned long old, unsigned long new) -{ - /* If MDWE is disabled, we have nothing to deny. */ - if (!mm_flags_test(MMF_HAS_MDWE, current->mm)) - return false; - - /* If the new VMA is not executable, we have nothing to deny. */ - if (!(new & VM_EXEC)) - return false; - - /* Under MDWE we do not accept newly writably executable VMAs... */ - if (new & VM_WRITE) - return true; - - /* ...nor previously non-executable VMAs becoming executable. */ - if (!(old & VM_EXEC)) - return true; - - return false; -} - #endif /* _LINUX_MMAN_H */ diff --git a/include/linux/mmap_lock.h b/include/linux/mmap_lock.h index 93eca48bc443..b8a13b8d36a4 100644 --- a/include/linux/mmap_lock.h +++ b/include/linux/mmap_lock.h @@ -506,6 +506,8 @@ static inline __must_check int vma_start_write_killable(struct vm_area_struct *vma) { return 0; } static inline void vma_assert_write_locked(struct vm_area_struct *vma) { mmap_assert_write_locked(vma->vm_mm); } +static inline bool vma_is_attached(struct vm_area_struct *vma) + { return true; } static inline void vma_assert_attached(struct vm_area_struct *vma) {} static inline void vma_assert_detached(struct vm_area_struct *vma) {} static inline void vma_mark_attached(struct vm_area_struct *vma) {} @@ -530,6 +532,12 @@ static inline void vma_assert_stabilised(struct vm_area_struct *vma) #endif /* CONFIG_PER_VMA_LOCK */ +static inline void vma_assert_can_modify(struct vm_area_struct *vma) +{ + if (vma_is_attached(vma)) + vma_assert_write_locked(vma); +} + static inline void mmap_write_lock(struct mm_struct *mm) { __mmap_lock_trace_start_locking(mm, true); @@ -546,7 +554,7 @@ static inline void mmap_write_lock_nested(struct mm_struct *mm, int subclass) __mmap_lock_trace_acquire_returned(mm, true, true); } -static inline int mmap_write_lock_killable(struct mm_struct *mm) +static inline int __must_check mmap_write_lock_killable(struct mm_struct *mm) { int ret; @@ -593,7 +601,7 @@ static inline void mmap_read_lock(struct mm_struct *mm) __mmap_lock_trace_acquire_returned(mm, false, true); } -static inline int mmap_read_lock_killable(struct mm_struct *mm) +static inline int __must_check mmap_read_lock_killable(struct mm_struct *mm) { int ret; @@ -603,7 +611,7 @@ static inline int mmap_read_lock_killable(struct mm_struct *mm) return ret; } -static inline bool mmap_read_trylock(struct mm_struct *mm) +static inline bool __must_check mmap_read_trylock(struct mm_struct *mm) { bool ret; @@ -621,6 +629,9 @@ static inline void mmap_read_unlock(struct mm_struct *mm) DEFINE_GUARD(mmap_read_lock, struct mm_struct *, mmap_read_lock(_T), mmap_read_unlock(_T)) +DEFINE_GUARD_COND(mmap_read_lock, _try, mmap_read_trylock(_T)) +DEFINE_GUARD(mmap_write_lock, struct mm_struct *, + mmap_write_lock(_T), mmap_write_unlock(_T)) static inline void mmap_read_unlock_non_owner(struct mm_struct *mm) { diff --git a/include/linux/mmc/card.h b/include/linux/mmc/card.h index e9e964c20e53..9dc4750296af 100644 --- a/include/linux/mmc/card.h +++ b/include/linux/mmc/card.h @@ -329,6 +329,8 @@ struct mmc_card { #define MMC_QUIRK_BROKEN_CACHE_FLUSH (1<<16) /* Don't flush cache until the write has occurred */ #define MMC_QUIRK_BROKEN_SD_POWEROFF_NOTIFY (1<<17) /* Disable broken SD poweroff notify support */ #define MMC_QUIRK_NO_UHS_DDR50_TUNING (1<<18) /* Disable DDR50 tuning */ +#define MMC_QUIRK_BROKEN_MDT (1<<19) /* Wrong manufacturing year */ +#define MMC_QUIRK_FIXED_SECURE_ERASE_TRIM_TIME (1<<20) /* Secure erase/trim time is fixed regardless of size */ bool written_flag; /* Indicates eMMC has been written since power on */ bool reenable_cmdq; /* Re-enable Command Queue */ diff --git a/include/linux/mmc/sdio_func.h b/include/linux/mmc/sdio_func.h index 4534bf462aac..5d63a6465b0d 100644 --- a/include/linux/mmc/sdio_func.h +++ b/include/linux/mmc/sdio_func.h @@ -9,7 +9,7 @@ #define LINUX_MMC_SDIO_FUNC_H #include <linux/device.h> -#include <linux/mod_devicetable.h> +#include <linux/device-id/sdio.h> #include <linux/mmc/pm.h> diff --git a/include/linux/mmc/sdio_ids.h b/include/linux/mmc/sdio_ids.h index 673cbdf43453..afae93f6f245 100644 --- a/include/linux/mmc/sdio_ids.h +++ b/include/linux/mmc/sdio_ids.h @@ -111,11 +111,19 @@ #define SDIO_VENDOR_ID_MEDIATEK 0x037a #define SDIO_DEVICE_ID_MEDIATEK_MT7663 0x7663 #define SDIO_DEVICE_ID_MEDIATEK_MT7668 0x7668 +#define SDIO_DEVICE_ID_MEDIATEK_MT7902 0x790a #define SDIO_DEVICE_ID_MEDIATEK_MT7961 0x7961 #define SDIO_VENDOR_ID_MICROCHIP_WILC 0x0296 #define SDIO_DEVICE_ID_MICROCHIP_WILC1000 0x5347 +#define SDIO_VENDOR_ID_MORSEMICRO 0x325b +#define SDIO_DEVICE_ID_MORSEMICRO_MM8108 0x0809 + +#define SDIO_VENDOR_ID_NXP 0x0471 +#define SDIO_DEVICE_ID_NXP_IW61X_BASE 0x0204 +#define SDIO_DEVICE_ID_NXP_IW61X 0x0205 + #define SDIO_VENDOR_ID_REALTEK 0x024c #define SDIO_DEVICE_ID_REALTEK_RTW8723BS 0xb723 #define SDIO_DEVICE_ID_REALTEK_RTW8821BS 0xb821 diff --git a/include/linux/mmu_notifier.h b/include/linux/mmu_notifier.h index 8450e18a87c2..a11a44eef521 100644 --- a/include/linux/mmu_notifier.h +++ b/include/linux/mmu_notifier.h @@ -97,20 +97,20 @@ struct mmu_notifier_ops { * Start-end is necessary in case the secondary MMU is mapping the page * at a smaller granularity than the primary MMU. */ - int (*clear_flush_young)(struct mmu_notifier *subscription, - struct mm_struct *mm, - unsigned long start, - unsigned long end); + bool (*clear_flush_young)(struct mmu_notifier *subscription, + struct mm_struct *mm, + unsigned long start, + unsigned long end); /* * clear_young is a lightweight version of clear_flush_young. Like the * latter, it is supposed to test-and-clear the young/accessed bitflag * in the secondary pte, but it may omit flushing the secondary tlb. */ - int (*clear_young)(struct mmu_notifier *subscription, - struct mm_struct *mm, - unsigned long start, - unsigned long end); + bool (*clear_young)(struct mmu_notifier *subscription, + struct mm_struct *mm, + unsigned long start, + unsigned long end); /* * test_young is called to check the young/accessed bitflag in @@ -118,9 +118,9 @@ struct mmu_notifier_ops { * frequently used without actually clearing the flag or tearing * down the secondary mapping on the page. */ - int (*test_young)(struct mmu_notifier *subscription, - struct mm_struct *mm, - unsigned long address); + bool (*test_young)(struct mmu_notifier *subscription, + struct mm_struct *mm, + unsigned long address); /* * invalidate_range_start() and invalidate_range_end() must be @@ -134,8 +134,8 @@ struct mmu_notifier_ops { * Invalidation of multiple concurrent ranges may be * optionally permitted by the driver. Either way the * establishment of sptes is forbidden in the range passed to - * invalidate_range_begin/end for the whole duration of the - * invalidate_range_begin/end critical section. + * invalidate_range_start/end for the whole duration of the + * invalidate_range_start/end critical section. * * invalidate_range_start() is called when all pages in the * range are still mapped and have at least a refcount of one. @@ -234,15 +234,57 @@ struct mmu_notifier { }; /** + * struct mmu_interval_notifier_finish - mmu_interval_notifier two-pass abstraction + * @link: Lockless list link for the notifiers pending pass list + * @notifier: The mmu_interval_notifier for which the finish pass is called. + * + * Allocate, typically using GFP_NOWAIT in the interval notifier's start pass. + * Note that with a large number of notifiers implementing two passes, + * allocation with GFP_NOWAIT will become increasingly likely to fail, so consider + * implementing a small pool instead of using kmalloc() allocations. + * + * If the implementation needs to pass data between the start and the finish passes, + * the recommended way is to embed struct mmu_interval_notifier_finish into a larger + * structure that also contains the data needed to be shared. Keep in mind that + * a notifier callback can be invoked in parallel, and each invocation needs its + * own struct mmu_interval_notifier_finish. + * + * If allocation fails, then the &mmu_interval_notifier_ops->invalidate_start op + * needs to implements the full notifier functionality. Please refer to its + * documentation. + */ +struct mmu_interval_notifier_finish { + struct llist_node link; + struct mmu_interval_notifier *notifier; +}; + +/** * struct mmu_interval_notifier_ops - callback for range notification * @invalidate: Upon return the caller must stop using any SPTEs within this * range. This function can sleep. Return false only if sleeping * was required but mmu_notifier_range_blockable(range) is false. + * @invalidate_start: Similar to @invalidate, but intended for two-pass notifier + * callbacks where the call to @invalidate_start is the first + * pass and any struct mmu_interval_notifier_finish pointer + * returned in the @finish parameter describes the finish pass. + * If *@finish is %NULL on return, then no final pass will be + * called, and @invalidate_start needs to implement the full + * notifier, behaving like @invalidate. The value of *@finish + * is guaranteed to be %NULL at function entry. + * @invalidate_finish: Called as the second pass for any notifier that returned + * a non-NULL *@finish from @invalidate_start. The @finish + * pointer passed here is the same one returned by + * @invalidate_start. */ struct mmu_interval_notifier_ops { bool (*invalidate)(struct mmu_interval_notifier *interval_sub, const struct mmu_notifier_range *range, unsigned long cur_seq); + bool (*invalidate_start)(struct mmu_interval_notifier *interval_sub, + const struct mmu_notifier_range *range, + unsigned long cur_seq, + struct mmu_interval_notifier_finish **finish); + void (*invalidate_finish)(struct mmu_interval_notifier_finish *finish); }; struct mmu_interval_notifier { @@ -376,14 +418,12 @@ mmu_interval_check_retry(struct mmu_interval_notifier *interval_sub, extern void __mmu_notifier_subscriptions_destroy(struct mm_struct *mm); extern void __mmu_notifier_release(struct mm_struct *mm); -extern int __mmu_notifier_clear_flush_young(struct mm_struct *mm, - unsigned long start, - unsigned long end); -extern int __mmu_notifier_clear_young(struct mm_struct *mm, - unsigned long start, - unsigned long end); -extern int __mmu_notifier_test_young(struct mm_struct *mm, - unsigned long address); +bool __mmu_notifier_clear_flush_young(struct mm_struct *mm, + unsigned long start, unsigned long end); +bool __mmu_notifier_clear_young(struct mm_struct *mm, + unsigned long start, unsigned long end); +bool __mmu_notifier_test_young(struct mm_struct *mm, + unsigned long address); extern int __mmu_notifier_invalidate_range_start(struct mmu_notifier_range *r); extern void __mmu_notifier_invalidate_range_end(struct mmu_notifier_range *r); extern void __mmu_notifier_arch_invalidate_secondary_tlbs(struct mm_struct *mm, @@ -403,30 +443,28 @@ static inline void mmu_notifier_release(struct mm_struct *mm) __mmu_notifier_release(mm); } -static inline int mmu_notifier_clear_flush_young(struct mm_struct *mm, - unsigned long start, - unsigned long end) +static inline bool mmu_notifier_clear_flush_young(struct mm_struct *mm, + unsigned long start, unsigned long end) { if (mm_has_notifiers(mm)) return __mmu_notifier_clear_flush_young(mm, start, end); - return 0; + return false; } -static inline int mmu_notifier_clear_young(struct mm_struct *mm, - unsigned long start, - unsigned long end) +static inline bool mmu_notifier_clear_young(struct mm_struct *mm, + unsigned long start, unsigned long end) { if (mm_has_notifiers(mm)) return __mmu_notifier_clear_young(mm, start, end); - return 0; + return false; } -static inline int mmu_notifier_test_young(struct mm_struct *mm, - unsigned long address) +static inline bool mmu_notifier_test_young(struct mm_struct *mm, + unsigned long address) { if (mm_has_notifiers(mm)) return __mmu_notifier_test_young(mm, address); - return 0; + return false; } static inline void @@ -516,55 +554,6 @@ static inline void mmu_notifier_range_init_owner( range->owner = owner; } -#define clear_flush_young_ptes_notify(__vma, __address, __ptep, __nr) \ -({ \ - int __young; \ - struct vm_area_struct *___vma = __vma; \ - unsigned long ___address = __address; \ - unsigned int ___nr = __nr; \ - __young = clear_flush_young_ptes(___vma, ___address, __ptep, ___nr); \ - __young |= mmu_notifier_clear_flush_young(___vma->vm_mm, \ - ___address, \ - ___address + \ - ___nr * PAGE_SIZE); \ - __young; \ -}) - -#define pmdp_clear_flush_young_notify(__vma, __address, __pmdp) \ -({ \ - int __young; \ - struct vm_area_struct *___vma = __vma; \ - unsigned long ___address = __address; \ - __young = pmdp_clear_flush_young(___vma, ___address, __pmdp); \ - __young |= mmu_notifier_clear_flush_young(___vma->vm_mm, \ - ___address, \ - ___address + \ - PMD_SIZE); \ - __young; \ -}) - -#define ptep_clear_young_notify(__vma, __address, __ptep) \ -({ \ - int __young; \ - struct vm_area_struct *___vma = __vma; \ - unsigned long ___address = __address; \ - __young = ptep_test_and_clear_young(___vma, ___address, __ptep);\ - __young |= mmu_notifier_clear_young(___vma->vm_mm, ___address, \ - ___address + PAGE_SIZE); \ - __young; \ -}) - -#define pmdp_clear_young_notify(__vma, __address, __pmdp) \ -({ \ - int __young; \ - struct vm_area_struct *___vma = __vma; \ - unsigned long ___address = __address; \ - __young = pmdp_test_and_clear_young(___vma, ___address, __pmdp);\ - __young |= mmu_notifier_clear_young(___vma->vm_mm, ___address, \ - ___address + PMD_SIZE); \ - __young; \ -}) - #else /* CONFIG_MMU_NOTIFIER */ struct mmu_notifier_range { @@ -601,24 +590,22 @@ static inline void mmu_notifier_release(struct mm_struct *mm) { } -static inline int mmu_notifier_clear_flush_young(struct mm_struct *mm, - unsigned long start, - unsigned long end) +static inline bool mmu_notifier_clear_flush_young(struct mm_struct *mm, + unsigned long start, unsigned long end) { - return 0; + return false; } -static inline int mmu_notifier_clear_young(struct mm_struct *mm, - unsigned long start, - unsigned long end) +static inline bool mmu_notifier_clear_young(struct mm_struct *mm, + unsigned long start, unsigned long end) { - return 0; + return false; } -static inline int mmu_notifier_test_young(struct mm_struct *mm, - unsigned long address) +static inline bool mmu_notifier_test_young(struct mm_struct *mm, + unsigned long address) { - return 0; + return false; } static inline void @@ -652,11 +639,6 @@ static inline void mmu_notifier_subscriptions_destroy(struct mm_struct *mm) #define mmu_notifier_range_update_to_read_only(r) false -#define clear_flush_young_ptes_notify clear_flush_young_ptes -#define pmdp_clear_flush_young_notify pmdp_clear_flush_young -#define ptep_clear_young_notify ptep_test_and_clear_young -#define pmdp_clear_young_notify pmdp_test_and_clear_young - static inline void mmu_notifier_synchronize(void) { } diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index 3e51190a55e4..94f9c3ff5416 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -2,7 +2,7 @@ #ifndef _LINUX_MMZONE_H #define _LINUX_MMZONE_H -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #ifndef __GENERATING_BOUNDS_H #include <linux/spinlock.h> @@ -23,6 +23,7 @@ #include <linux/page-flags.h> #include <linux/local_lock.h> #include <linux/zswap.h> +#include <linux/sizes.h> #include <asm/page.h> /* Free memory management - zoned buddy allocator. */ @@ -61,6 +62,59 @@ */ #define PAGE_ALLOC_COSTLY_ORDER 3 +#if !defined(CONFIG_HAVE_GIGANTIC_FOLIOS) +/* + * We don't expect any folios that exceed buddy sizes (and consequently + * memory sections). + */ +#define MAX_FOLIO_ORDER MAX_PAGE_ORDER +#elif defined(CONFIG_SPARSEMEM) && !defined(CONFIG_SPARSEMEM_VMEMMAP) +/* + * Only pages within a single memory section are guaranteed to be + * contiguous. By limiting folios to a single memory section, all folio + * pages are guaranteed to be contiguous. + */ +#define MAX_FOLIO_ORDER PFN_SECTION_SHIFT +#elif defined(CONFIG_HUGETLB_PAGE) +/* + * There is no real limit on the folio size. We limit them to the maximum we + * currently expect (see CONFIG_HAVE_GIGANTIC_FOLIOS): with hugetlb, we expect + * no folios larger than 16 GiB on 64bit and 1 GiB on 32bit. + */ +#ifdef CONFIG_64BIT +#define MAX_FOLIO_ORDER (ilog2(SZ_16G) - PAGE_SHIFT) +#else +#define MAX_FOLIO_ORDER (ilog2(SZ_1G) - PAGE_SHIFT) +#endif +#else +/* + * Without hugetlb, gigantic folios that are bigger than a single PUD are + * currently impossible. + */ +#define MAX_FOLIO_ORDER (PUD_SHIFT - PAGE_SHIFT) +#endif + +#define MAX_FOLIO_NR_PAGES (1UL << MAX_FOLIO_ORDER) + +/* + * HugeTLB Vmemmap Optimization (HVO) requires struct pages of the head page to + * be naturally aligned with regard to the folio size. + * + * HVO which is only active if the size of struct page is a power of 2. + */ +#define MAX_FOLIO_VMEMMAP_ALIGN \ + (IS_ENABLED(CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP) && \ + is_power_of_2(sizeof(struct page)) ? \ + MAX_FOLIO_NR_PAGES * sizeof(struct page) : 0) + +/* + * vmemmap optimization (like HVO) is only possible for page orders that fill + * two or more pages with struct pages. + */ +#define VMEMMAP_TAIL_MIN_ORDER (ilog2(2 * PAGE_SIZE / sizeof(struct page))) +#define __NR_VMEMMAP_TAILS (MAX_FOLIO_ORDER - VMEMMAP_TAIL_MIN_ORDER + 1) +#define NR_VMEMMAP_TAILS (__NR_VMEMMAP_TAILS > 0 ? __NR_VMEMMAP_TAILS : 0) + enum migratetype { MIGRATE_UNMOVABLE, MIGRATE_MOVABLE, @@ -123,9 +177,12 @@ static inline bool migratetype_is_mergeable(int mt) return mt < MIGRATE_PCPTYPES; } -#define for_each_migratetype_order(order, type) \ - for (order = 0; order < NR_PAGE_ORDERS; order++) \ - for (type = 0; type < MIGRATE_TYPES; type++) +#define for_each_free_list(list, zone, order) \ + for (order = 0; order < NR_PAGE_ORDERS; order++) \ + for (unsigned int __type = 0; \ + __type < MIGRATE_TYPES && \ + (list = &(zone)->free_area[order].free_list[__type], 1); \ + __type++) extern int page_group_by_mobility_disabled; @@ -157,7 +214,6 @@ enum numa_stat_item { #endif enum zone_stat_item { - /* First 128 byte cacheline (assuming 64 bit words) */ NR_FREE_PAGES, NR_FREE_PAGES_BLOCKS, NR_ZONE_LRU_BASE, /* Used only for compaction and reclaim retry */ @@ -168,7 +224,6 @@ enum zone_stat_item { NR_ZONE_UNEVICTABLE, NR_ZONE_WRITE_PENDING, /* Count of dirty, writeback and unstable pages */ NR_MLOCK, /* mlock()ed pages found and moved off LRU */ - /* Second 128 byte cacheline */ #if IS_ENABLED(CONFIG_ZSMALLOC) NR_ZSPAGES, /* allocated in zsmalloc */ #endif @@ -220,6 +275,7 @@ enum node_stat_item { NR_KERNEL_MISC_RECLAIMABLE, /* reclaimable non-slab kernel pages */ NR_FOLL_PIN_ACQUIRED, /* via: pin_user_page(), gup flag: FOLL_PIN */ NR_FOLL_PIN_RELEASED, /* pages returned via unpin_user_page() */ + NR_VMALLOC, NR_KERNEL_STACK_KB, /* measured in KiB */ #if IS_ENABLED(CONFIG_SHADOW_CALL_STACK) NR_KERNEL_SCS_KB, /* measured in KiB */ @@ -255,11 +311,28 @@ enum node_stat_item { PGDEMOTE_DIRECT, PGDEMOTE_KHUGEPAGED, PGDEMOTE_PROACTIVE, + PGSTEAL_KSWAPD, + PGSTEAL_DIRECT, + PGSTEAL_KHUGEPAGED, + PGSTEAL_PROACTIVE, + PGSTEAL_ANON, + PGSTEAL_FILE, + PGSCAN_KSWAPD, + PGSCAN_DIRECT, + PGSCAN_KHUGEPAGED, + PGSCAN_PROACTIVE, + PGSCAN_ANON, + PGSCAN_FILE, + PGROTATE_ANON, + PGROTATE_FILE, + PGREFILL, #ifdef CONFIG_HUGETLB_PAGE NR_HUGETLB, #endif NR_BALLOON_PAGES, NR_KERNEL_FILE_PAGES, + NR_GPU_ACTIVE, /* Pages assigned to GPU objects */ + NR_GPU_RECLAIM, /* Pages in shrinkable GPU pools */ NR_VM_NODE_STAT_ITEMS }; @@ -616,7 +689,7 @@ struct lru_gen_memcg { void lru_gen_init_pgdat(struct pglist_data *pgdat); void lru_gen_init_lruvec(struct lruvec *lruvec); -bool lru_gen_look_around(struct page_vma_mapped_walk *pvmw); +bool lru_gen_look_around(struct page_vma_mapped_walk *pvmw, unsigned int nr); void lru_gen_init_memcg(struct mem_cgroup *memcg); void lru_gen_exit_memcg(struct mem_cgroup *memcg); @@ -624,6 +697,9 @@ void lru_gen_online_memcg(struct mem_cgroup *memcg); void lru_gen_offline_memcg(struct mem_cgroup *memcg); void lru_gen_release_memcg(struct mem_cgroup *memcg); void lru_gen_soft_reclaim(struct mem_cgroup *memcg, int nid); +void max_lru_gen_memcg(struct mem_cgroup *memcg, int nid); +bool recheck_lru_gen_max_memcg(struct mem_cgroup *memcg, int nid); +void lru_gen_reparent_memcg(struct mem_cgroup *memcg, struct mem_cgroup *parent, int nid); #else /* !CONFIG_LRU_GEN */ @@ -635,7 +711,8 @@ static inline void lru_gen_init_lruvec(struct lruvec *lruvec) { } -static inline bool lru_gen_look_around(struct page_vma_mapped_walk *pvmw) +static inline bool lru_gen_look_around(struct page_vma_mapped_walk *pvmw, + unsigned int nr) { return false; } @@ -664,8 +741,28 @@ static inline void lru_gen_soft_reclaim(struct mem_cgroup *memcg, int nid) { } +static inline void max_lru_gen_memcg(struct mem_cgroup *memcg, int nid) +{ +} + +static inline bool recheck_lru_gen_max_memcg(struct mem_cgroup *memcg, int nid) +{ + return true; +} + +static inline +void lru_gen_reparent_memcg(struct mem_cgroup *memcg, struct mem_cgroup *parent, int nid) +{ +} + #endif /* CONFIG_LRU_GEN */ +struct lru_cost { + unsigned long count; + unsigned long last_rotated; + unsigned long last_io; +}; + struct lruvec { struct list_head lists[NR_LRU_LISTS]; /* per lruvec lru_lock for memcg */ @@ -674,9 +771,12 @@ struct lruvec { * These track the cost of reclaiming one LRU - file or anon - * over the other. As the observed cost of reclaiming one LRU * increases, the reclaim scan balance tips toward the other. + * Updated and decayed at prepare_scan_control() time; cost_lock + * serialises that update. */ - unsigned long anon_cost; - unsigned long file_cost; + struct lru_cost cost[ANON_AND_FILE]; + /* Protects cost[]. */ + spinlock_t cost_lock; /* Non-resident age, driven by LRU movement */ atomic_long_t nonresident_age; /* Refaults at the time of last reclaim cycle */ @@ -1057,6 +1157,9 @@ struct zone { /* Zone statistics */ atomic_long_t vm_stat[NR_VM_ZONE_STAT_ITEMS]; atomic_long_t vm_numa_event[NR_VM_NUMA_EVENT_ITEMS]; +#ifdef CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP + struct page *vmemmap_tails[NR_VMEMMAP_TAILS]; +#endif } ____cacheline_internodealigned_in_smp; enum pgdat_flags { @@ -1180,31 +1283,33 @@ static inline bool zone_is_empty(const struct zone *zone) #define KASAN_TAG_MASK ((1UL << KASAN_TAG_WIDTH) - 1) #define ZONEID_MASK ((1UL << ZONEID_SHIFT) - 1) -static inline enum zone_type memdesc_zonenum(memdesc_flags_t flags) +static inline enum zone_type memdesc_zonenum(const memdesc_flags_t *flags) { - ASSERT_EXCLUSIVE_BITS(flags.f, ZONES_MASK << ZONES_PGSHIFT); - return (flags.f >> ZONES_PGSHIFT) & ZONES_MASK; +#if ZONES_WIDTH != 0 + ASSERT_EXCLUSIVE_BITS(flags->f, ZONES_MASK << ZONES_PGSHIFT); +#endif + return (flags->f >> ZONES_PGSHIFT) & ZONES_MASK; } static inline enum zone_type page_zonenum(const struct page *page) { - return memdesc_zonenum(page->flags); + return memdesc_zonenum(&page->flags); } static inline enum zone_type folio_zonenum(const struct folio *folio) { - return memdesc_zonenum(folio->flags); + return memdesc_zonenum(&folio->flags); } #ifdef CONFIG_ZONE_DEVICE -static inline bool memdesc_is_zone_device(memdesc_flags_t mdf) +static inline bool memdesc_is_zone_device(const memdesc_flags_t *mdf) { return memdesc_zonenum(mdf) == ZONE_DEVICE; } static inline struct dev_pagemap *page_pgmap(const struct page *page) { - VM_WARN_ON_ONCE_PAGE(!memdesc_is_zone_device(page->flags), page); + VM_WARN_ON_ONCE_PAGE(!memdesc_is_zone_device(&page->flags), page); return page_folio(page)->pgmap; } @@ -1219,9 +1324,9 @@ static inline struct dev_pagemap *page_pgmap(const struct page *page) static inline bool zone_device_pages_have_same_pgmap(const struct page *a, const struct page *b) { - if (memdesc_is_zone_device(a->flags) != memdesc_is_zone_device(b->flags)) + if (memdesc_is_zone_device(&a->flags) != memdesc_is_zone_device(&b->flags)) return false; - if (!memdesc_is_zone_device(a->flags)) + if (!memdesc_is_zone_device(&a->flags)) return true; return page_pgmap(a) == page_pgmap(b); } @@ -1229,7 +1334,7 @@ static inline bool zone_device_pages_have_same_pgmap(const struct page *a, extern void memmap_init_zone_device(struct zone *, unsigned long, unsigned long, struct dev_pagemap *); #else -static inline bool memdesc_is_zone_device(memdesc_flags_t mdf) +static inline bool memdesc_is_zone_device(const memdesc_flags_t *mdf) { return false; } @@ -1246,12 +1351,12 @@ static inline struct dev_pagemap *page_pgmap(const struct page *page) static inline bool is_zone_device_page(const struct page *page) { - return memdesc_is_zone_device(page->flags); + return memdesc_is_zone_device(&page->flags); } static inline bool folio_is_zone_device(const struct folio *folio) { - return memdesc_is_zone_device(folio->flags); + return memdesc_is_zone_device(&folio->flags); } static inline bool is_zone_movable_page(const struct page *page) @@ -1337,14 +1442,6 @@ struct zonelist { */ extern struct page *mem_map; -#ifdef CONFIG_TRANSPARENT_HUGEPAGE -struct deferred_split { - spinlock_t split_queue_lock; - struct list_head split_queue; - unsigned long split_queue_len; -}; -#endif - #ifdef CONFIG_MEMORY_FAILURE /* * Per NUMA node memory failure handling statistics. @@ -1470,10 +1567,6 @@ typedef struct pglist_data { unsigned long first_deferred_pfn; #endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */ -#ifdef CONFIG_TRANSPARENT_HUGEPAGE - struct deferred_split deferred_split_queue; -#endif - #ifdef CONFIG_NUMA_BALANCING /* start time in ms of current promote rate limit period */ unsigned int nbp_rl_start; @@ -1735,7 +1828,7 @@ static inline int zonelist_node_idx(const struct zoneref *zoneref) struct zoneref *__next_zones_zonelist(struct zoneref *z, enum zone_type highest_zoneidx, - nodemask_t *nodes); + const nodemask_t *nodes); /** * next_zones_zonelist - Returns the next zone at or below highest_zoneidx within the allowed nodemask using a cursor within a zonelist as a starting point @@ -1754,7 +1847,7 @@ struct zoneref *__next_zones_zonelist(struct zoneref *z, */ static __always_inline struct zoneref *next_zones_zonelist(struct zoneref *z, enum zone_type highest_zoneidx, - nodemask_t *nodes) + const nodemask_t *nodes) { if (likely(!nodes && zonelist_zone_idx(z) <= highest_zoneidx)) return z; @@ -1780,7 +1873,7 @@ static __always_inline struct zoneref *next_zones_zonelist(struct zoneref *z, */ static inline struct zoneref *first_zones_zonelist(struct zonelist *zonelist, enum zone_type highest_zoneidx, - nodemask_t *nodes) + const nodemask_t *nodes) { return next_zones_zonelist(zonelist->_zonerefs, highest_zoneidx, nodes); @@ -1910,15 +2003,13 @@ struct mem_section_usage { unsigned long pageblock_flags[0]; }; -void subsection_map_init(unsigned long pfn, unsigned long nr_pages); - struct page; struct page_ext; struct mem_section { /* * This is, logically, a pointer to an array of struct * pages. However, it is stored with some other magic. - * (see sparse.c::sparse_init_one_section()) + * (see sparse_init_one_section()) * * Additionally during early boot we encode node id of * the location of the section here to guide allocation. @@ -1978,24 +2069,18 @@ static inline struct mem_section *__nr_to_section(unsigned long nr) #endif return &mem_section[root][nr & SECTION_ROOT_MASK]; } -extern size_t mem_section_usage_size(void); /* - * We use the lower bits of the mem_map pointer to store - * a little bit of information. The pointer is calculated - * as mem_map - section_nr_to_pfn(pnum). The result is - * aligned to the minimum alignment of the two values: - * 1. All mem_map arrays are page-aligned. - * 2. section_nr_to_pfn() always clears PFN_SECTION_SHIFT - * lowest bits. PFN_SECTION_SHIFT is arch-specific - * (equal SECTION_SIZE_BITS - PAGE_SHIFT), and the - * worst combination is powerpc with 256k pages, - * which results in PFN_SECTION_SHIFT equal 6. - * To sum it up, at least 6 bits are available on all architectures. - * However, we can exceed 6 bits on some other architectures except - * powerpc (e.g. 15 bits are available on x86_64, 13 bits are available - * with the worst case of 64K pages on arm64) if we make sure the - * exceeded bit is not applicable to powerpc. + * We use the lower bits of the mem_map pointer to store a little bit of + * information. The pointer is calculated as mem_map - section_nr_to_pfn(). + * The result is aligned to the minimum alignment of the two values: + * + * 1. All mem_map arrays are page-aligned. + * 2. section_nr_to_pfn() always clears PFN_SECTION_SHIFT lowest bits. + * + * We always expect a single section to cover full pages. Therefore, + * we can safely assume that PFN_SECTION_SHIFT is large enough to + * accommodate SECTION_MAP_LAST_BIT. We use BUILD_BUG_ON() to ensure this. */ enum { SECTION_MARKED_PRESENT_BIT, @@ -2083,8 +2168,6 @@ static inline int preinited_vmemmap_section(const struct mem_section *section) } void sparse_vmemmap_init_nid_early(int nid); -void sparse_vmemmap_init_nid_late(int nid); - #else static inline int preinited_vmemmap_section(const struct mem_section *section) { @@ -2093,10 +2176,6 @@ static inline int preinited_vmemmap_section(const struct mem_section *section) static inline void sparse_vmemmap_init_nid_early(int nid) { } - -static inline void sparse_vmemmap_init_nid_late(int nid) -{ -} #endif static inline int online_section_nr(unsigned long nr) @@ -2300,11 +2379,8 @@ static inline unsigned long next_present_section_nr(unsigned long section_nr) #endif #else -#define sparse_index_init(_sec, _nid) do {} while (0) #define sparse_vmemmap_init_nid_early(_nid) do {} while (0) -#define sparse_vmemmap_init_nid_late(_nid) do {} while (0) #define pfn_in_present_section pfn_valid -#define subsection_map_init(_pfn, _nr_pages) do {} while (0) #endif /* CONFIG_SPARSEMEM */ /* @@ -2318,5 +2394,5 @@ static inline unsigned long next_present_section_nr(unsigned long section_nr) #endif #endif /* !__GENERATING_BOUNDS.H */ -#endif /* !__ASSEMBLY__ */ +#endif /* !__ASSEMBLER__ */ #endif /* _LINUX_MMZONE_H */ diff --git a/include/linux/mod_devicetable.h b/include/linux/mod_devicetable.h index 5b1725fe9707..a397213bedac 100644 --- a/include/linux/mod_devicetable.h +++ b/include/linux/mod_devicetable.h @@ -9,701 +9,65 @@ #define LINUX_MOD_DEVICETABLE_H #ifdef __KERNEL__ -#include <linux/mei.h> #include <linux/types.h> -#include <linux/uuid.h> -typedef unsigned long kernel_ulong_t; #endif -#define PCI_ANY_ID (~0) - -enum { - PCI_ID_F_VFIO_DRIVER_OVERRIDE = 1, -}; - -/** - * struct pci_device_id - PCI device ID structure - * @vendor: Vendor ID to match (or PCI_ANY_ID) - * @device: Device ID to match (or PCI_ANY_ID) - * @subvendor: Subsystem vendor ID to match (or PCI_ANY_ID) - * @subdevice: Subsystem device ID to match (or PCI_ANY_ID) - * @class: Device class, subclass, and "interface" to match. - * See Appendix D of the PCI Local Bus Spec or - * include/linux/pci_ids.h for a full list of classes. - * Most drivers do not need to specify class/class_mask - * as vendor/device is normally sufficient. - * @class_mask: Limit which sub-fields of the class field are compared. - * See drivers/scsi/sym53c8xx_2/ for example of usage. - * @driver_data: Data private to the driver. - * Most drivers don't need to use driver_data field. - * Best practice is to use driver_data as an index - * into a static list of equivalent device types, - * instead of using it as a pointer. - * @override_only: Match only when dev->driver_override is this driver. - */ -struct pci_device_id { - __u32 vendor, device; /* Vendor and device ID or PCI_ANY_ID*/ - __u32 subvendor, subdevice; /* Subsystem ID's or PCI_ANY_ID */ - __u32 class, class_mask; /* (class,subclass,prog-if) triplet */ - kernel_ulong_t driver_data; /* Data private to the driver */ - __u32 override_only; -}; - - -#define IEEE1394_MATCH_VENDOR_ID 0x0001 -#define IEEE1394_MATCH_MODEL_ID 0x0002 -#define IEEE1394_MATCH_SPECIFIER_ID 0x0004 -#define IEEE1394_MATCH_VERSION 0x0008 - -struct ieee1394_device_id { - __u32 match_flags; - __u32 vendor_id; - __u32 model_id; - __u32 specifier_id; - __u32 version; - kernel_ulong_t driver_data; -}; - - -/* - * Device table entry for "new style" table-driven USB drivers. - * User mode code can read these tables to choose which modules to load. - * Declare the table as a MODULE_DEVICE_TABLE. - * - * A probe() parameter will point to a matching entry from this table. - * Use the driver_info field for each match to hold information tied - * to that match: device quirks, etc. - * - * Terminate the driver's table with an all-zeroes entry. - * Use the flag values to control which fields are compared. - */ - -/** - * struct usb_device_id - identifies USB devices for probing and hotplugging - * @match_flags: Bit mask controlling which of the other fields are used to - * match against new devices. Any field except for driver_info may be - * used, although some only make sense in conjunction with other fields. - * This is usually set by a USB_DEVICE_*() macro, which sets all - * other fields in this structure except for driver_info. - * @idVendor: USB vendor ID for a device; numbers are assigned - * by the USB forum to its members. - * @idProduct: Vendor-assigned product ID. - * @bcdDevice_lo: Low end of range of vendor-assigned product version numbers. - * This is also used to identify individual product versions, for - * a range consisting of a single device. - * @bcdDevice_hi: High end of version number range. The range of product - * versions is inclusive. - * @bDeviceClass: Class of device; numbers are assigned - * by the USB forum. Products may choose to implement classes, - * or be vendor-specific. Device classes specify behavior of all - * the interfaces on a device. - * @bDeviceSubClass: Subclass of device; associated with bDeviceClass. - * @bDeviceProtocol: Protocol of device; associated with bDeviceClass. - * @bInterfaceClass: Class of interface; numbers are assigned - * by the USB forum. Products may choose to implement classes, - * or be vendor-specific. Interface classes specify behavior only - * of a given interface; other interfaces may support other classes. - * @bInterfaceSubClass: Subclass of interface; associated with bInterfaceClass. - * @bInterfaceProtocol: Protocol of interface; associated with bInterfaceClass. - * @bInterfaceNumber: Number of interface; composite devices may use - * fixed interface numbers to differentiate between vendor-specific - * interfaces. - * @driver_info: Holds information used by the driver. Usually it holds - * a pointer to a descriptor understood by the driver, or perhaps - * device flags. - * - * In most cases, drivers will create a table of device IDs by using - * USB_DEVICE(), or similar macros designed for that purpose. - * They will then export it to userspace using MODULE_DEVICE_TABLE(), - * and provide it to the USB core through their usb_driver structure. - * - * See the usb_match_id() function for information about how matches are - * performed. Briefly, you will normally use one of several macros to help - * construct these entries. Each entry you provide will either identify - * one or more specific products, or will identify a class of products - * which have agreed to behave the same. You should put the more specific - * matches towards the beginning of your table, so that driver_info can - * record quirks of specific products. - */ -struct usb_device_id { - /* which fields to match against? */ - __u16 match_flags; - - /* Used for product specific matches; range is inclusive */ - __u16 idVendor; - __u16 idProduct; - __u16 bcdDevice_lo; - __u16 bcdDevice_hi; - - /* Used for device class matches */ - __u8 bDeviceClass; - __u8 bDeviceSubClass; - __u8 bDeviceProtocol; - - /* Used for interface class matches */ - __u8 bInterfaceClass; - __u8 bInterfaceSubClass; - __u8 bInterfaceProtocol; - - /* Used for vendor-specific interface matches */ - __u8 bInterfaceNumber; - - /* not matched against */ - kernel_ulong_t driver_info - __attribute__((aligned(sizeof(kernel_ulong_t)))); -}; - -/* Some useful macros to use to create struct usb_device_id */ -#define USB_DEVICE_ID_MATCH_VENDOR 0x0001 -#define USB_DEVICE_ID_MATCH_PRODUCT 0x0002 -#define USB_DEVICE_ID_MATCH_DEV_LO 0x0004 -#define USB_DEVICE_ID_MATCH_DEV_HI 0x0008 -#define USB_DEVICE_ID_MATCH_DEV_CLASS 0x0010 -#define USB_DEVICE_ID_MATCH_DEV_SUBCLASS 0x0020 -#define USB_DEVICE_ID_MATCH_DEV_PROTOCOL 0x0040 -#define USB_DEVICE_ID_MATCH_INT_CLASS 0x0080 -#define USB_DEVICE_ID_MATCH_INT_SUBCLASS 0x0100 -#define USB_DEVICE_ID_MATCH_INT_PROTOCOL 0x0200 -#define USB_DEVICE_ID_MATCH_INT_NUMBER 0x0400 - -#define HID_ANY_ID (~0) -#define HID_BUS_ANY 0xffff -#define HID_GROUP_ANY 0x0000 - -struct hid_device_id { - __u16 bus; - __u16 group; - __u32 vendor; - __u32 product; - kernel_ulong_t driver_data; -}; - -/* s390 CCW devices */ -struct ccw_device_id { - __u16 match_flags; /* which fields to match against */ - - __u16 cu_type; /* control unit type */ - __u16 dev_type; /* device type */ - __u8 cu_model; /* control unit model */ - __u8 dev_model; /* device model */ - - kernel_ulong_t driver_info; -}; - -#define CCW_DEVICE_ID_MATCH_CU_TYPE 0x01 -#define CCW_DEVICE_ID_MATCH_CU_MODEL 0x02 -#define CCW_DEVICE_ID_MATCH_DEVICE_TYPE 0x04 -#define CCW_DEVICE_ID_MATCH_DEVICE_MODEL 0x08 - -/* s390 AP bus devices */ -struct ap_device_id { - __u16 match_flags; /* which fields to match against */ - __u8 dev_type; /* device type */ - kernel_ulong_t driver_info; -}; - -#define AP_DEVICE_ID_MATCH_CARD_TYPE 0x01 -#define AP_DEVICE_ID_MATCH_QUEUE_TYPE 0x02 - -/* s390 css bus devices (subchannels) */ -struct css_device_id { - __u8 match_flags; - __u8 type; /* subchannel type */ - kernel_ulong_t driver_data; -}; - -#define ACPI_ID_LEN 16 - -struct acpi_device_id { - __u8 id[ACPI_ID_LEN]; - kernel_ulong_t driver_data; - __u32 cls; - __u32 cls_msk; -}; - -/** - * ACPI_DEVICE_CLASS - macro used to describe an ACPI device with - * the PCI-defined class-code information - * - * @_cls : the class, subclass, prog-if triple for this device - * @_msk : the class mask for this device - * - * This macro is used to create a struct acpi_device_id that matches a - * specific PCI class. The .id and .driver_data fields will be left - * initialized with the default value. - */ -#define ACPI_DEVICE_CLASS(_cls, _msk) .cls = (_cls), .cls_msk = (_msk), - -#define PNP_ID_LEN 8 -#define PNP_MAX_DEVICES 8 - -struct pnp_device_id { - __u8 id[PNP_ID_LEN]; - kernel_ulong_t driver_data; -}; - -struct pnp_card_device_id { - __u8 id[PNP_ID_LEN]; - kernel_ulong_t driver_data; - struct { - __u8 id[PNP_ID_LEN]; - } devs[PNP_MAX_DEVICES]; -}; - - -#define SERIO_ANY 0xff - -struct serio_device_id { - __u8 type; - __u8 extra; - __u8 id; - __u8 proto; -}; - -struct hda_device_id { - __u32 vendor_id; - __u32 rev_id; - __u8 api_version; - const char *name; - unsigned long driver_data; -}; - -struct sdw_device_id { - __u16 mfg_id; - __u16 part_id; - __u8 sdw_version; - __u8 class_id; - kernel_ulong_t driver_data; -}; - -/* - * Struct used for matching a device - */ -struct of_device_id { - char name[32]; - char type[32]; - char compatible[128]; - const void *data; -}; - -/* VIO */ -struct vio_device_id { - char type[32]; - char compat[32]; -}; - -/* PCMCIA */ - -struct pcmcia_device_id { - __u16 match_flags; - - __u16 manf_id; - __u16 card_id; - - __u8 func_id; - - /* for real multi-function devices */ - __u8 function; - - /* for pseudo multi-function devices */ - __u8 device_no; - - __u32 prod_id_hash[4]; - - /* not matched against in kernelspace */ - const char * prod_id[4]; - - /* not matched against */ - kernel_ulong_t driver_info; - char * cisfile; -}; - -#define PCMCIA_DEV_ID_MATCH_MANF_ID 0x0001 -#define PCMCIA_DEV_ID_MATCH_CARD_ID 0x0002 -#define PCMCIA_DEV_ID_MATCH_FUNC_ID 0x0004 -#define PCMCIA_DEV_ID_MATCH_FUNCTION 0x0008 -#define PCMCIA_DEV_ID_MATCH_PROD_ID1 0x0010 -#define PCMCIA_DEV_ID_MATCH_PROD_ID2 0x0020 -#define PCMCIA_DEV_ID_MATCH_PROD_ID3 0x0040 -#define PCMCIA_DEV_ID_MATCH_PROD_ID4 0x0080 -#define PCMCIA_DEV_ID_MATCH_DEVICE_NO 0x0100 -#define PCMCIA_DEV_ID_MATCH_FAKE_CIS 0x0200 -#define PCMCIA_DEV_ID_MATCH_ANONYMOUS 0x0400 - -/* Input */ -#define INPUT_DEVICE_ID_EV_MAX 0x1f -#define INPUT_DEVICE_ID_KEY_MIN_INTERESTING 0x71 -#define INPUT_DEVICE_ID_KEY_MAX 0x2ff -#define INPUT_DEVICE_ID_REL_MAX 0x0f -#define INPUT_DEVICE_ID_ABS_MAX 0x3f -#define INPUT_DEVICE_ID_MSC_MAX 0x07 -#define INPUT_DEVICE_ID_LED_MAX 0x0f -#define INPUT_DEVICE_ID_SND_MAX 0x07 -#define INPUT_DEVICE_ID_FF_MAX 0x7f -#define INPUT_DEVICE_ID_SW_MAX 0x11 -#define INPUT_DEVICE_ID_PROP_MAX 0x1f - -#define INPUT_DEVICE_ID_MATCH_BUS 1 -#define INPUT_DEVICE_ID_MATCH_VENDOR 2 -#define INPUT_DEVICE_ID_MATCH_PRODUCT 4 -#define INPUT_DEVICE_ID_MATCH_VERSION 8 - -#define INPUT_DEVICE_ID_MATCH_EVBIT 0x0010 -#define INPUT_DEVICE_ID_MATCH_KEYBIT 0x0020 -#define INPUT_DEVICE_ID_MATCH_RELBIT 0x0040 -#define INPUT_DEVICE_ID_MATCH_ABSBIT 0x0080 -#define INPUT_DEVICE_ID_MATCH_MSCIT 0x0100 -#define INPUT_DEVICE_ID_MATCH_LEDBIT 0x0200 -#define INPUT_DEVICE_ID_MATCH_SNDBIT 0x0400 -#define INPUT_DEVICE_ID_MATCH_FFBIT 0x0800 -#define INPUT_DEVICE_ID_MATCH_SWBIT 0x1000 -#define INPUT_DEVICE_ID_MATCH_PROPBIT 0x2000 - -struct input_device_id { - - kernel_ulong_t flags; - - __u16 bustype; - __u16 vendor; - __u16 product; - __u16 version; - - kernel_ulong_t evbit[INPUT_DEVICE_ID_EV_MAX / BITS_PER_LONG + 1]; - kernel_ulong_t keybit[INPUT_DEVICE_ID_KEY_MAX / BITS_PER_LONG + 1]; - kernel_ulong_t relbit[INPUT_DEVICE_ID_REL_MAX / BITS_PER_LONG + 1]; - kernel_ulong_t absbit[INPUT_DEVICE_ID_ABS_MAX / BITS_PER_LONG + 1]; - kernel_ulong_t mscbit[INPUT_DEVICE_ID_MSC_MAX / BITS_PER_LONG + 1]; - kernel_ulong_t ledbit[INPUT_DEVICE_ID_LED_MAX / BITS_PER_LONG + 1]; - kernel_ulong_t sndbit[INPUT_DEVICE_ID_SND_MAX / BITS_PER_LONG + 1]; - kernel_ulong_t ffbit[INPUT_DEVICE_ID_FF_MAX / BITS_PER_LONG + 1]; - kernel_ulong_t swbit[INPUT_DEVICE_ID_SW_MAX / BITS_PER_LONG + 1]; - kernel_ulong_t propbit[INPUT_DEVICE_ID_PROP_MAX / BITS_PER_LONG + 1]; - - kernel_ulong_t driver_info; -}; - -/* EISA */ - -#define EISA_SIG_LEN 8 - -/* The EISA signature, in ASCII form, null terminated */ -struct eisa_device_id { - char sig[EISA_SIG_LEN]; - kernel_ulong_t driver_data; -}; - -#define EISA_DEVICE_MODALIAS_FMT "eisa:s%s" - -struct parisc_device_id { - __u8 hw_type; /* 5 bits used */ - __u8 hversion_rev; /* 4 bits */ - __u16 hversion; /* 12 bits */ - __u32 sversion; /* 20 bits */ -}; - -#define PA_HWTYPE_ANY_ID 0xff -#define PA_HVERSION_REV_ANY_ID 0xff -#define PA_HVERSION_ANY_ID 0xffff -#define PA_SVERSION_ANY_ID 0xffffffff - -/* SDIO */ - -#define SDIO_ANY_ID (~0) - -struct sdio_device_id { - __u8 class; /* Standard interface or SDIO_ANY_ID */ - __u16 vendor; /* Vendor or SDIO_ANY_ID */ - __u16 device; /* Device ID or SDIO_ANY_ID */ - kernel_ulong_t driver_data; /* Data private to the driver */ -}; - -/* SSB core, see drivers/ssb/ */ -struct ssb_device_id { - __u16 vendor; - __u16 coreid; - __u8 revision; - __u8 __pad; -} __attribute__((packed, aligned(2))); -#define SSB_DEVICE(_vendor, _coreid, _revision) \ - { .vendor = _vendor, .coreid = _coreid, .revision = _revision, } - -#define SSB_ANY_VENDOR 0xFFFF -#define SSB_ANY_ID 0xFFFF -#define SSB_ANY_REV 0xFF - -/* Broadcom's specific AMBA core, see drivers/bcma/ */ -struct bcma_device_id { - __u16 manuf; - __u16 id; - __u8 rev; - __u8 class; -} __attribute__((packed,aligned(2))); -#define BCMA_CORE(_manuf, _id, _rev, _class) \ - { .manuf = _manuf, .id = _id, .rev = _rev, .class = _class, } - -#define BCMA_ANY_MANUF 0xFFFF -#define BCMA_ANY_ID 0xFFFF -#define BCMA_ANY_REV 0xFF -#define BCMA_ANY_CLASS 0xFF - -struct virtio_device_id { - __u32 device; - __u32 vendor; -}; -#define VIRTIO_DEV_ANY_ID 0xffffffff - -/* - * For Hyper-V devices we use the device guid as the id. - */ -struct hv_vmbus_device_id { - guid_t guid; - kernel_ulong_t driver_data; /* Data private to the driver */ -}; - -/* rpmsg */ - -#define RPMSG_NAME_SIZE 32 -#define RPMSG_DEVICE_MODALIAS_FMT "rpmsg:%s" - -struct rpmsg_device_id { - char name[RPMSG_NAME_SIZE]; - kernel_ulong_t driver_data; -}; - -/* i2c */ - -#define I2C_NAME_SIZE 20 -#define I2C_MODULE_PREFIX "i2c:" - -struct i2c_device_id { - char name[I2C_NAME_SIZE]; - kernel_ulong_t driver_data; /* Data private to the driver */ -}; - -/* pci_epf */ - -#define PCI_EPF_NAME_SIZE 20 -#define PCI_EPF_MODULE_PREFIX "pci_epf:" - -struct pci_epf_device_id { - char name[PCI_EPF_NAME_SIZE]; - kernel_ulong_t driver_data; -}; - -/* i3c */ - -#define I3C_MATCH_DCR 0x1 -#define I3C_MATCH_MANUF 0x2 -#define I3C_MATCH_PART 0x4 -#define I3C_MATCH_EXTRA_INFO 0x8 - -struct i3c_device_id { - __u8 match_flags; - __u8 dcr; - __u16 manuf_id; - __u16 part_id; - __u16 extra_info; - - const void *data; -}; - -/* spi */ - -#define SPI_NAME_SIZE 32 -#define SPI_MODULE_PREFIX "spi:" - -struct spi_device_id { - char name[SPI_NAME_SIZE]; - kernel_ulong_t driver_data; /* Data private to the driver */ -}; - -/* SLIMbus */ - -#define SLIMBUS_NAME_SIZE 32 -#define SLIMBUS_MODULE_PREFIX "slim:" - -struct slim_device_id { - __u16 manf_id, prod_code; - __u16 dev_index, instance; - - /* Data private to the driver */ - kernel_ulong_t driver_data; -}; - -#define APR_NAME_SIZE 32 -#define APR_MODULE_PREFIX "apr:" - -struct apr_device_id { - char name[APR_NAME_SIZE]; - __u32 domain_id; - __u32 svc_id; - __u32 svc_version; - kernel_ulong_t driver_data; /* Data private to the driver */ -}; - -#define SPMI_NAME_SIZE 32 -#define SPMI_MODULE_PREFIX "spmi:" - -struct spmi_device_id { - char name[SPMI_NAME_SIZE]; - kernel_ulong_t driver_data; /* Data private to the driver */ -}; - -/* dmi */ -enum dmi_field { - DMI_NONE, - DMI_BIOS_VENDOR, - DMI_BIOS_VERSION, - DMI_BIOS_DATE, - DMI_BIOS_RELEASE, - DMI_EC_FIRMWARE_RELEASE, - DMI_SYS_VENDOR, - DMI_PRODUCT_NAME, - DMI_PRODUCT_VERSION, - DMI_PRODUCT_SERIAL, - DMI_PRODUCT_UUID, - DMI_PRODUCT_SKU, - DMI_PRODUCT_FAMILY, - DMI_BOARD_VENDOR, - DMI_BOARD_NAME, - DMI_BOARD_VERSION, - DMI_BOARD_SERIAL, - DMI_BOARD_ASSET_TAG, - DMI_CHASSIS_VENDOR, - DMI_CHASSIS_TYPE, - DMI_CHASSIS_VERSION, - DMI_CHASSIS_SERIAL, - DMI_CHASSIS_ASSET_TAG, - DMI_STRING_MAX, - DMI_OEM_STRING, /* special case - will not be in dmi_ident */ -}; - -struct dmi_strmatch { - unsigned char slot:7; - unsigned char exact_match:1; - char substr[79]; -}; - -struct dmi_system_id { - int (*callback)(const struct dmi_system_id *); - const char *ident; - struct dmi_strmatch matches[4]; - void *driver_data; -}; -/* - * struct dmi_device_id appears during expansion of - * "MODULE_DEVICE_TABLE(dmi, x)". Compiler doesn't look inside it - * but this is enough for gcc 3.4.6 to error out: - * error: storage size of '__mod_dmi_device_table' isn't known - */ -#define dmi_device_id dmi_system_id - -#define DMI_MATCH(a, b) { .slot = a, .substr = b } -#define DMI_EXACT_MATCH(a, b) { .slot = a, .substr = b, .exact_match = 1 } - -#define PLATFORM_NAME_SIZE 24 -#define PLATFORM_MODULE_PREFIX "platform:" - -struct platform_device_id { - char name[PLATFORM_NAME_SIZE]; - kernel_ulong_t driver_data; -}; - -#define MDIO_MODULE_PREFIX "mdio:" - -#define MDIO_ID_FMT "%u%u%u%u%u%u%u%u%u%u%u%u%u%u%u%u%u%u%u%u%u%u%u%u%u%u%u%u%u%u%u%u" -#define MDIO_ID_ARGS(_id) \ - ((_id)>>31) & 1, ((_id)>>30) & 1, ((_id)>>29) & 1, ((_id)>>28) & 1, \ - ((_id)>>27) & 1, ((_id)>>26) & 1, ((_id)>>25) & 1, ((_id)>>24) & 1, \ - ((_id)>>23) & 1, ((_id)>>22) & 1, ((_id)>>21) & 1, ((_id)>>20) & 1, \ - ((_id)>>19) & 1, ((_id)>>18) & 1, ((_id)>>17) & 1, ((_id)>>16) & 1, \ - ((_id)>>15) & 1, ((_id)>>14) & 1, ((_id)>>13) & 1, ((_id)>>12) & 1, \ - ((_id)>>11) & 1, ((_id)>>10) & 1, ((_id)>>9) & 1, ((_id)>>8) & 1, \ - ((_id)>>7) & 1, ((_id)>>6) & 1, ((_id)>>5) & 1, ((_id)>>4) & 1, \ - ((_id)>>3) & 1, ((_id)>>2) & 1, ((_id)>>1) & 1, (_id) & 1 - -/** - * struct mdio_device_id - identifies PHY devices on an MDIO/MII bus - * @phy_id: The result of - * (mdio_read(&MII_PHYSID1) << 16 | mdio_read(&MII_PHYSID2)) & @phy_id_mask - * for this PHY type - * @phy_id_mask: Defines the significant bits of @phy_id. A value of 0 - * is used to terminate an array of struct mdio_device_id. - */ -struct mdio_device_id { - __u32 phy_id; - __u32 phy_id_mask; -}; - -struct zorro_device_id { - __u32 id; /* Device ID or ZORRO_WILDCARD */ - kernel_ulong_t driver_data; /* Data private to the driver */ -}; - -#define ZORRO_WILDCARD (0xffffffff) /* not official */ - -#define ZORRO_DEVICE_MODALIAS_FMT "zorro:i%08X" - -#define ISAPNP_ANY_ID 0xffff -struct isapnp_device_id { - unsigned short card_vendor, card_device; - unsigned short vendor, function; - kernel_ulong_t driver_data; /* data private to the driver */ -}; - -/** - * struct amba_id - identifies a device on an AMBA bus - * @id: The significant bits if the hardware device ID - * @mask: Bitmask specifying which bits of the id field are significant when - * matching. A driver binds to a device when ((hardware device ID) & mask) - * == id. - * @data: Private data used by the driver. - */ -struct amba_id { - unsigned int id; - unsigned int mask; - void *data; -}; - -/** - * struct mips_cdmm_device_id - identifies devices in MIPS CDMM bus - * @type: Device type identifier. - */ -struct mips_cdmm_device_id { - __u8 type; -}; - -/* - * Match x86 CPUs for CPU specific drivers. - * See documentation of "x86_match_cpu" for details. - */ - -/* - * MODULE_DEVICE_TABLE expects this struct to be called x86cpu_device_id. - * Although gcc seems to ignore this error, clang fails without this define. - */ -#define x86cpu_device_id x86_cpu_id -struct x86_cpu_id { - __u16 vendor; - __u16 family; - __u16 model; - __u16 steppings; - __u16 feature; /* bit index */ - /* Solely for kernel-internal use: DO NOT EXPORT to userspace! */ - __u16 flags; - __u8 type; - kernel_ulong_t driver_data; -}; - -/* Wild cards for x86_cpu_id::vendor, family, model and feature */ -#define X86_VENDOR_ANY 0xffff -#define X86_FAMILY_ANY 0 -#define X86_MODEL_ANY 0 -#define X86_STEPPING_ANY 0 -#define X86_STEP_MIN 0 -#define X86_STEP_MAX 0xf -#define X86_FEATURE_ANY 0 /* Same as FPU, you can't test for that */ -#define X86_CPU_TYPE_ANY 0 +#include "device-id/acpi.h" +#include "device-id/amba.h" +#include "device-id/ap.h" +#include "device-id/apr.h" +#include "device-id/auxiliary.h" +#include "device-id/bcma.h" +#include "device-id/ccw.h" +#include "device-id/cdx.h" +#include "device-id/coreboot.h" +#include "device-id/css.h" +#include "device-id/dfl.h" +#include "device-id/dmi.h" +#include "device-id/eisa.h" +#include "device-id/fsl_mc.h" +#include "device-id/hda.h" +#include "device-id/hid.h" +#include "device-id/hv_vmbus.h" +#include "device-id/i2c.h" +#include "device-id/i3c.h" +#include "device-id/ieee1394.h" +#include "device-id/input.h" +#include "device-id/ipack.h" +#include "device-id/isapnp.h" +#include "device-id/ishtp.h" +#include "device-id/mcb.h" +#include "device-id/mdio.h" +#include "device-id/mei_cl.h" +#include "device-id/mhi.h" +#include "device-id/mips_cdmm.h" +#include "device-id/of.h" +#include "device-id/parisc.h" +#include "device-id/pci.h" +#include "device-id/pcmcia.h" +#include "device-id/platform.h" +#include "device-id/pnp.h" +#include "device-id/rio.h" +#include "device-id/rpmsg.h" +#include "device-id/sdio.h" +#include "device-id/sdw.h" +#include "device-id/serio.h" +#include "device-id/slim.h" +#include "device-id/spi.h" +#include "device-id/spmi.h" +#include "device-id/ssam.h" +#include "device-id/ssb.h" +#include "device-id/tb.h" +#include "device-id/tee_client.h" +#include "device-id/typec.h" +#include "device-id/ulpi.h" +#include "device-id/usb.h" +#include "device-id/vchiq.h" +#include "device-id/vio.h" +#include "device-id/virtio.h" +#include "device-id/wmi.h" +#include "device-id/x86_cpu.h" +#include "device-id/zorro.h" /* * Generic table type for matching CPU features. @@ -714,265 +78,4 @@ struct cpu_feature { __u16 feature; }; -#define IPACK_ANY_FORMAT 0xff -#define IPACK_ANY_ID (~0) -struct ipack_device_id { - __u8 format; /* Format version or IPACK_ANY_ID */ - __u32 vendor; /* Vendor ID or IPACK_ANY_ID */ - __u32 device; /* Device ID or IPACK_ANY_ID */ -}; - -#define MEI_CL_MODULE_PREFIX "mei:" -#define MEI_CL_NAME_SIZE 32 -#define MEI_CL_VERSION_ANY 0xff - -/** - * struct mei_cl_device_id - MEI client device identifier - * @name: helper name - * @uuid: client uuid - * @version: client protocol version - * @driver_info: information used by the driver. - * - * identifies mei client device by uuid and name - */ -struct mei_cl_device_id { - char name[MEI_CL_NAME_SIZE]; - uuid_le uuid; - __u8 version; - kernel_ulong_t driver_info; -}; - -/* RapidIO */ - -#define RIO_ANY_ID 0xffff - -/** - * struct rio_device_id - RIO device identifier - * @did: RapidIO device ID - * @vid: RapidIO vendor ID - * @asm_did: RapidIO assembly device ID - * @asm_vid: RapidIO assembly vendor ID - * - * Identifies a RapidIO device based on both the device/vendor IDs and - * the assembly device/vendor IDs. - */ -struct rio_device_id { - __u16 did, vid; - __u16 asm_did, asm_vid; -}; - -struct mcb_device_id { - __u16 device; - kernel_ulong_t driver_data; -}; - -struct ulpi_device_id { - __u16 vendor; - __u16 product; - kernel_ulong_t driver_data; -}; - -/** - * struct fsl_mc_device_id - MC object device identifier - * @vendor: vendor ID - * @obj_type: MC object type - * - * Type of entries in the "device Id" table for MC object devices supported by - * a MC object device driver. The last entry of the table has vendor set to 0x0 - */ -struct fsl_mc_device_id { - __u16 vendor; - const char obj_type[16]; -}; - -/** - * struct tb_service_id - Thunderbolt service identifiers - * @match_flags: Flags used to match the structure - * @protocol_key: Protocol key the service supports - * @protocol_id: Protocol id the service supports - * @protocol_version: Version of the protocol - * @protocol_revision: Revision of the protocol software - * @driver_data: Driver specific data - * - * Thunderbolt XDomain services are exposed as devices where each device - * carries the protocol information the service supports. Thunderbolt - * XDomain service drivers match against that information. - */ -struct tb_service_id { - __u32 match_flags; - char protocol_key[8 + 1]; - __u32 protocol_id; - __u32 protocol_version; - __u32 protocol_revision; - kernel_ulong_t driver_data; -}; - -#define TBSVC_MATCH_PROTOCOL_KEY 0x0001 -#define TBSVC_MATCH_PROTOCOL_ID 0x0002 -#define TBSVC_MATCH_PROTOCOL_VERSION 0x0004 -#define TBSVC_MATCH_PROTOCOL_REVISION 0x0008 - -/* USB Type-C Alternate Modes */ - -#define TYPEC_ANY_MODE 0x7 - -/** - * struct typec_device_id - USB Type-C alternate mode identifiers - * @svid: Standard or Vendor ID - * @mode: Mode index - * @driver_data: Driver specific data - */ -struct typec_device_id { - __u16 svid; - __u8 mode; - kernel_ulong_t driver_data; -}; - -/** - * struct tee_client_device_id - tee based device identifier - * @uuid: For TEE based client devices we use the device uuid as - * the identifier. - */ -struct tee_client_device_id { - uuid_t uuid; -}; - -/* WMI */ - -#define WMI_MODULE_PREFIX "wmi:" - -/** - * struct wmi_device_id - WMI device identifier - * @guid_string: 36 char string of the form fa50ff2b-f2e8-45de-83fa-65417f2f49ba - * @context: pointer to driver specific data - */ -struct wmi_device_id { - const char guid_string[UUID_STRING_LEN+1]; - const void *context; -}; - -#define MHI_DEVICE_MODALIAS_FMT "mhi:%s" -#define MHI_NAME_SIZE 32 - -#define MHI_EP_DEVICE_MODALIAS_FMT "mhi_ep:%s" - -/** - * struct mhi_device_id - MHI device identification - * @chan: MHI channel name - * @driver_data: driver data; - */ -struct mhi_device_id { - const char chan[MHI_NAME_SIZE]; - kernel_ulong_t driver_data; -}; - -#define AUXILIARY_NAME_SIZE 40 -#define AUXILIARY_MODULE_PREFIX "auxiliary:" - -struct auxiliary_device_id { - char name[AUXILIARY_NAME_SIZE]; - kernel_ulong_t driver_data; -}; - -/* Surface System Aggregator Module */ - -#define SSAM_MATCH_TARGET 0x1 -#define SSAM_MATCH_INSTANCE 0x2 -#define SSAM_MATCH_FUNCTION 0x4 - -struct ssam_device_id { - __u8 match_flags; - - __u8 domain; - __u8 category; - __u8 target; - __u8 instance; - __u8 function; - - kernel_ulong_t driver_data; -}; - -/* - * DFL (Device Feature List) - * - * DFL defines a linked list of feature headers within the device MMIO space to - * provide an extensible way of adding features. Software can walk through these - * predefined data structures to enumerate features. It is now used in the FPGA. - * See Documentation/fpga/dfl.rst for more information. - * - * The dfl bus type is introduced to match the individual feature devices (dfl - * devices) for specific dfl drivers. - */ - -/** - * struct dfl_device_id - dfl device identifier - * @type: DFL FIU type of the device. See enum dfl_id_type. - * @feature_id: feature identifier local to its DFL FIU type. - * @driver_data: driver specific data. - */ -struct dfl_device_id { - __u16 type; - __u16 feature_id; - kernel_ulong_t driver_data; -}; - -/* ISHTP (Integrated Sensor Hub Transport Protocol) */ - -#define ISHTP_MODULE_PREFIX "ishtp:" - -/** - * struct ishtp_device_id - ISHTP device identifier - * @guid: GUID of the device. - * @driver_data: pointer to driver specific data - */ -struct ishtp_device_id { - guid_t guid; - kernel_ulong_t driver_data; -}; - -#define CDX_ANY_ID (0xFFFF) - -enum { - CDX_ID_F_VFIO_DRIVER_OVERRIDE = 1, -}; - -/** - * struct cdx_device_id - CDX device identifier - * @vendor: Vendor ID - * @device: Device ID - * @subvendor: Subsystem vendor ID (or CDX_ANY_ID) - * @subdevice: Subsystem device ID (or CDX_ANY_ID) - * @class: Device class - * Most drivers do not need to specify class/class_mask - * as vendor/device is normally sufficient. - * @class_mask: Limit which sub-fields of the class field are compared. - * @override_only: Match only when dev->driver_override is this driver. - * - * Type of entries in the "device Id" table for CDX devices supported by - * a CDX device driver. - */ -struct cdx_device_id { - __u16 vendor; - __u16 device; - __u16 subvendor; - __u16 subdevice; - __u32 class; - __u32 class_mask; - __u32 override_only; -}; - -struct vchiq_device_id { - char name[32]; -}; - -/** - * struct coreboot_device_id - Identifies a coreboot table entry - * @tag: tag ID - * @driver_data: driver specific data - */ -struct coreboot_device_id { - __u32 tag; - kernel_ulong_t driver_data; -}; - #endif /* LINUX_MOD_DEVICETABLE_H */ diff --git a/include/linux/module.h b/include/linux/module.h index 14f391b186c6..96cc98568eea 100644 --- a/include/linux/module.h +++ b/include/linux/module.h @@ -413,11 +413,13 @@ struct module { struct module_attribute *modinfo_attrs; const char *version; const char *srcversion; + const char *imported_namespaces; struct kobject *holders_dir; /* Exported symbols */ const struct kernel_symbol *syms; const u32 *crcs; + const u8 *flagstab; unsigned int num_syms; #ifdef CONFIG_ARCH_USES_CFI_TRAPS @@ -433,9 +435,6 @@ struct module { unsigned int num_kp; /* GPL-only exported symbols. */ - unsigned int num_gpl_syms; - const struct kernel_symbol *gpl_syms; - const u32 *gpl_crcs; bool using_gplonly_symbols; #ifdef CONFIG_MODULE_SIG @@ -478,10 +477,6 @@ struct module { struct module_notes_attrs *notes_attrs; #endif - /* The command line arguments (may be mangled). People like - keeping pointers to this stuff */ - char *args; - #ifdef CONFIG_SMP /* Per-cpu data. */ void __percpu *percpu; diff --git a/include/linux/module_signature.h b/include/linux/module_signature.h index 7eb4b00381ac..db335d46787f 100644 --- a/include/linux/module_signature.h +++ b/include/linux/module_signature.h @@ -10,35 +10,7 @@ #define _LINUX_MODULE_SIGNATURE_H #include <linux/types.h> - -/* In stripped ARM and x86-64 modules, ~ is surprisingly rare. */ -#define MODULE_SIG_STRING "~Module signature appended~\n" - -enum pkey_id_type { - PKEY_ID_PGP, /* OpenPGP generated key ID */ - PKEY_ID_X509, /* X.509 arbitrary subjectKeyIdentifier */ - PKEY_ID_PKCS7, /* Signature in PKCS#7 message */ -}; - -/* - * Module signature information block. - * - * The constituents of the signature section are, in order: - * - * - Signer's name - * - Key identifier - * - Signature data - * - Information block - */ -struct module_signature { - u8 algo; /* Public-key crypto algorithm [0] */ - u8 hash; /* Digest algorithm [0] */ - u8 id_type; /* Key identifier type [PKEY_ID_PKCS7] */ - u8 signer_len; /* Length of signer's name [0] */ - u8 key_id_len; /* Length of key identifier [0] */ - u8 __pad[3]; - __be32 sig_len; /* Length of signature data */ -}; +#include <uapi/linux/module_signature.h> int mod_check_sig(const struct module_signature *ms, size_t file_len, const char *name); diff --git a/include/linux/module_symbol.h b/include/linux/module_symbol.h index 77c9895b9ddb..574609aced99 100644 --- a/include/linux/module_symbol.h +++ b/include/linux/module_symbol.h @@ -2,6 +2,11 @@ #ifndef _LINUX_MODULE_SYMBOL_H #define _LINUX_MODULE_SYMBOL_H +/* Kernel symbol flags bitset. */ +enum ksym_flags { + KSYM_FLAG_GPL_ONLY = 1 << 0, +}; + /* This ignores the intensely annoying "mapping symbols" found in ELF files. */ static inline bool is_mapping_symbol(const char *str) { diff --git a/include/linux/moduleparam.h b/include/linux/moduleparam.h index 7d22d4c4ea2e..075f28585074 100644 --- a/include/linux/moduleparam.h +++ b/include/linux/moduleparam.h @@ -317,8 +317,8 @@ struct kparam_array name, &__param_ops_##name, arg, perm, -1, 0) #ifdef CONFIG_SYSFS -extern void kernel_param_lock(struct module *mod); -extern void kernel_param_unlock(struct module *mod); +void kernel_param_lock(struct module *mod); +void kernel_param_unlock(struct module *mod); #else static inline void kernel_param_lock(struct module *mod) { @@ -398,7 +398,7 @@ static inline void kernel_param_unlock(struct module *mod) * Returns: true if the two parameter names are equal. * Dashes (-) are considered equal to underscores (_). */ -extern bool parameq(const char *name1, const char *name2); +bool parameq(const char *name1, const char *name2); /** * parameqn - checks if two parameter names match @@ -412,28 +412,23 @@ extern bool parameq(const char *name1, const char *name2); * are equal. * Dashes (-) are considered equal to underscores (_). */ -extern bool parameqn(const char *name1, const char *name2, size_t n); +bool parameqn(const char *name1, const char *name2, size_t n); typedef int (*parse_unknown_fn)(char *param, char *val, const char *doing, void *arg); /* Called on module insert or kernel boot */ -extern char *parse_args(const char *name, - char *args, - const struct kernel_param *params, - unsigned num, - s16 level_min, - s16 level_max, - void *arg, parse_unknown_fn unknown); +char *parse_args(const char *doing, + char *args, + const struct kernel_param *params, + unsigned int num, + s16 min_level, + s16 max_level, + void *arg, parse_unknown_fn unknown); /* Called by module remove. */ -#ifdef CONFIG_SYSFS -extern void destroy_params(const struct kernel_param *params, unsigned num); -#else -static inline void destroy_params(const struct kernel_param *params, - unsigned num) -{ -} -#endif /* !CONFIG_SYSFS */ +#ifdef CONFIG_MODULES +void module_destroy_params(const struct kernel_param *params, unsigned int num); +#endif /* All the helper functions */ /* The macros to do compile-time type checking stolen from Jakub @@ -442,78 +437,77 @@ static inline void destroy_params(const struct kernel_param *params, static inline type __always_unused *__check_##name(void) { return(p); } extern const struct kernel_param_ops param_ops_byte; -extern int param_set_byte(const char *val, const struct kernel_param *kp); -extern int param_get_byte(char *buffer, const struct kernel_param *kp); +int param_set_byte(const char *val, const struct kernel_param *kp); +int param_get_byte(char *buffer, const struct kernel_param *kp); #define param_check_byte(name, p) __param_check(name, p, unsigned char) extern const struct kernel_param_ops param_ops_short; -extern int param_set_short(const char *val, const struct kernel_param *kp); -extern int param_get_short(char *buffer, const struct kernel_param *kp); +int param_set_short(const char *val, const struct kernel_param *kp); +int param_get_short(char *buffer, const struct kernel_param *kp); #define param_check_short(name, p) __param_check(name, p, short) extern const struct kernel_param_ops param_ops_ushort; -extern int param_set_ushort(const char *val, const struct kernel_param *kp); -extern int param_get_ushort(char *buffer, const struct kernel_param *kp); +int param_set_ushort(const char *val, const struct kernel_param *kp); +int param_get_ushort(char *buffer, const struct kernel_param *kp); #define param_check_ushort(name, p) __param_check(name, p, unsigned short) extern const struct kernel_param_ops param_ops_int; -extern int param_set_int(const char *val, const struct kernel_param *kp); -extern int param_get_int(char *buffer, const struct kernel_param *kp); +int param_set_int(const char *val, const struct kernel_param *kp); +int param_get_int(char *buffer, const struct kernel_param *kp); #define param_check_int(name, p) __param_check(name, p, int) extern const struct kernel_param_ops param_ops_uint; -extern int param_set_uint(const char *val, const struct kernel_param *kp); -extern int param_get_uint(char *buffer, const struct kernel_param *kp); +int param_set_uint(const char *val, const struct kernel_param *kp); +int param_get_uint(char *buffer, const struct kernel_param *kp); int param_set_uint_minmax(const char *val, const struct kernel_param *kp, unsigned int min, unsigned int max); #define param_check_uint(name, p) __param_check(name, p, unsigned int) extern const struct kernel_param_ops param_ops_long; -extern int param_set_long(const char *val, const struct kernel_param *kp); -extern int param_get_long(char *buffer, const struct kernel_param *kp); +int param_set_long(const char *val, const struct kernel_param *kp); +int param_get_long(char *buffer, const struct kernel_param *kp); #define param_check_long(name, p) __param_check(name, p, long) extern const struct kernel_param_ops param_ops_ulong; -extern int param_set_ulong(const char *val, const struct kernel_param *kp); -extern int param_get_ulong(char *buffer, const struct kernel_param *kp); +int param_set_ulong(const char *val, const struct kernel_param *kp); +int param_get_ulong(char *buffer, const struct kernel_param *kp); #define param_check_ulong(name, p) __param_check(name, p, unsigned long) extern const struct kernel_param_ops param_ops_ullong; -extern int param_set_ullong(const char *val, const struct kernel_param *kp); -extern int param_get_ullong(char *buffer, const struct kernel_param *kp); +int param_set_ullong(const char *val, const struct kernel_param *kp); +int param_get_ullong(char *buffer, const struct kernel_param *kp); #define param_check_ullong(name, p) __param_check(name, p, unsigned long long) extern const struct kernel_param_ops param_ops_hexint; -extern int param_set_hexint(const char *val, const struct kernel_param *kp); -extern int param_get_hexint(char *buffer, const struct kernel_param *kp); +int param_set_hexint(const char *val, const struct kernel_param *kp); +int param_get_hexint(char *buffer, const struct kernel_param *kp); #define param_check_hexint(name, p) param_check_uint(name, p) extern const struct kernel_param_ops param_ops_charp; -extern int param_set_charp(const char *val, const struct kernel_param *kp); -extern int param_get_charp(char *buffer, const struct kernel_param *kp); -extern void param_free_charp(void *arg); +int param_set_charp(const char *val, const struct kernel_param *kp); +int param_get_charp(char *buffer, const struct kernel_param *kp); +void param_free_charp(void *arg); #define param_check_charp(name, p) __param_check(name, p, char *) /* We used to allow int as well as bool. We're taking that away! */ extern const struct kernel_param_ops param_ops_bool; -extern int param_set_bool(const char *val, const struct kernel_param *kp); -extern int param_get_bool(char *buffer, const struct kernel_param *kp); +int param_set_bool(const char *val, const struct kernel_param *kp); +int param_get_bool(char *buffer, const struct kernel_param *kp); #define param_check_bool(name, p) __param_check(name, p, bool) extern const struct kernel_param_ops param_ops_bool_enable_only; -extern int param_set_bool_enable_only(const char *val, - const struct kernel_param *kp); +int param_set_bool_enable_only(const char *val, const struct kernel_param *kp); /* getter is the same as for the regular bool */ #define param_check_bool_enable_only param_check_bool extern const struct kernel_param_ops param_ops_invbool; -extern int param_set_invbool(const char *val, const struct kernel_param *kp); -extern int param_get_invbool(char *buffer, const struct kernel_param *kp); +int param_set_invbool(const char *val, const struct kernel_param *kp); +int param_get_invbool(char *buffer, const struct kernel_param *kp); #define param_check_invbool(name, p) __param_check(name, p, bool) /* An int, which can only be set like a bool (though it shows as an int). */ extern const struct kernel_param_ops param_ops_bint; -extern int param_set_bint(const char *val, const struct kernel_param *kp); +int param_set_bint(const char *val, const struct kernel_param *kp); #define param_get_bint param_get_int #define param_check_bint param_check_int @@ -620,19 +614,19 @@ enum hwparam_type { extern const struct kernel_param_ops param_array_ops; extern const struct kernel_param_ops param_ops_string; -extern int param_set_copystring(const char *val, const struct kernel_param *); -extern int param_get_string(char *buffer, const struct kernel_param *kp); +int param_set_copystring(const char *val, const struct kernel_param *kp); +int param_get_string(char *buffer, const struct kernel_param *kp); /* for exporting parameters in /sys/module/.../parameters */ struct module; #if defined(CONFIG_SYSFS) && defined(CONFIG_MODULES) -extern int module_param_sysfs_setup(struct module *mod, - const struct kernel_param *kparam, - unsigned int num_params); +int module_param_sysfs_setup(struct module *mod, + const struct kernel_param *kparam, + unsigned int num_params); -extern void module_param_sysfs_remove(struct module *mod); +void module_param_sysfs_remove(struct module *mod); #else static inline int module_param_sysfs_setup(struct module *mod, const struct kernel_param *kparam, diff --git a/include/linux/most.h b/include/linux/most.h index 232e01b7f5d2..4aeed08f816e 100644 --- a/include/linux/most.h +++ b/include/linux/most.h @@ -15,7 +15,7 @@ struct module; struct interface_private; /** - * Interface type + * enum most_interface_type - Interface type */ enum most_interface_type { ITYPE_LOOPBACK = 1, @@ -30,7 +30,7 @@ enum most_interface_type { }; /** - * Channel direction. + * enum most_channel_direction - Channel direction. */ enum most_channel_direction { MOST_CH_RX = 1 << 0, @@ -38,7 +38,7 @@ enum most_channel_direction { }; /** - * Channel data type. + * enum most_channel_data_type - Channel data type. */ enum most_channel_data_type { MOST_CH_CONTROL = 1 << 0, @@ -115,6 +115,7 @@ struct most_channel_capability { * @subbuffer_size: size of a subbuffer * @packets_per_xact: number of MOST frames that are packet inside one USB * packet. This is USB specific + * @dbr_size: DBR data buffer size (MediaLB communication only) * * Describes the configuration for a MOST channel. This information is * provided from the MostCore to a HDM (like the Medusa PCIe Interface) as a @@ -189,28 +190,31 @@ struct mbo { }; /** - * Interface instance description. + * struct most_interface - Interface instance description. * * Describes an interface of a MOST device the core driver is bound to. * This structure is allocated and initialized in the HDM. MostCore may not * modify this structure. * * @dev: the actual device + * @driver_dev: struct device for calling snd_card_new() * @mod: module - * @interface Interface type. \sa most_interface_type. - * @description PRELIMINARY. + * @interface: Interface type. \sa most_interface_type. + * @description: PRELIMINARY. * Unique description of the device instance from point of view of the * interface in free text form (ASCII). * It may be a hexadecimal presentation of the memory address for the MediaLB * IP or USB device ID with USB properties for USB interface, etc. - * @num_channels Number of channels and size of the channel_vector. - * @channel_vector Properties of the channels. + * @num_channels: Number of channels and size of the channel_vector. + * @channel_vector: Properties of the channels. * Array index represents channel ID by the driver. - * @configure Callback to change data type for the channel of the + * @dma_alloc: Callback to interface driver to allocation DMA memory. + * @dma_free: Callback to interface driver to free a DMA memory allocation. + * @configure: Callback to change data type for the channel of the * interface instance. May be zero if the instance of the interface is not * configurable. Parameter channel_config describes direction and data * type for the channel, configured by the higher level. The content of - * @enqueue Delivers MBO to the HDM for processing. + * @enqueue: Delivers MBO to the HDM for processing. * After HDM completes Rx- or Tx- operation the processed MBO shall * be returned back to the MostCore using completion routine. * The reason to get the MBO delivered from the MostCore after the channel @@ -218,7 +222,7 @@ struct mbo { * In this case the HDM shall hold MBOs and service the channel as usual. * The HDM must be able to hold at least one MBO for each channel. * The callback returns a negative value on error, otherwise 0. - * @poison_channel Informs HDM about closing the channel. The HDM shall + * @poison_channel: Informs HDM about closing the channel. The HDM shall * cancel all transfers and synchronously or asynchronously return * all enqueued for this channel MBOs using the completion routine. * The callback returns a negative value on error, otherwise 0. @@ -226,7 +230,8 @@ struct mbo { * means of "Message exchange over MDP/MEP" * The call of the function request_netinfo with the parameter on_netinfo as * NULL prohibits use of the previously obtained function pointer. - * @priv Private field used by mostcore to store context information. + * @priv: Private field used by mostcore to store context information. + * @p: &struct interface_private pointer */ struct most_interface { struct device *dev; @@ -255,10 +260,12 @@ struct most_interface { * struct most_component - identifies a loadable component for the mostcore * @list: list_head * @name: component name + * @mod: owning module * @probe_channel: function for core to notify driver about channel connection * @disconnect_channel: callback function to disconnect a certain channel * @rx_completion: completion handler for received packets * @tx_completion: completion handler for transmitted packets + * @cfg_complete: setup completion function called after the device is matched */ struct most_component { struct list_head list; @@ -278,7 +285,7 @@ struct most_component { * most_register_interface - Registers instance of the interface. * @iface: Pointer to the interface instance description. * - * Returns a pointer to the kobject of the generated instance. + * Returns: a pointer to the kobject of the generated instance. * * Note: HDM has to ensure that any reference held on the kobj is * released before deregistering the interface. @@ -286,8 +293,8 @@ struct most_component { int most_register_interface(struct most_interface *iface); /** - * Deregisters instance of the interface. - * @intf_instance Pointer to the interface instance description. + * most_deregister_interface - Deregisters instance of the interface. + * @iface: Pointer to the interface instance description. */ void most_deregister_interface(struct most_interface *iface); void most_submit_mbo(struct mbo *mbo); diff --git a/include/linux/mpage.h b/include/linux/mpage.h index 1bdc39daac0a..358946990bfa 100644 --- a/include/linux/mpage.h +++ b/include/linux/mpage.h @@ -17,7 +17,14 @@ struct readahead_control; void mpage_readahead(struct readahead_control *, get_block_t get_block); int mpage_read_folio(struct folio *folio, get_block_t get_block); -int mpage_writepages(struct address_space *mapping, - struct writeback_control *wbc, get_block_t get_block); +int __mpage_writepages(struct address_space *mapping, + struct writeback_control *wbc, get_block_t get_block, + int (*write_folio)(struct folio *folio, + struct writeback_control *wbc)); +static inline int mpage_writepages(struct address_space *mapping, + struct writeback_control *wbc, get_block_t get_block) +{ + return __mpage_writepages(mapping, wbc, get_block, NULL); +} #endif diff --git a/include/linux/mroute_base.h b/include/linux/mroute_base.h index 0075f6e5c3da..4d55827e9705 100644 --- a/include/linux/mroute_base.h +++ b/include/linux/mroute_base.h @@ -76,7 +76,7 @@ static inline int mr_call_vif_notifiers(struct net *net, struct vif_device *vif, struct net_device *vif_dev, unsigned short vif_index, u32 tb_id, - unsigned int *ipmr_seq) + atomic_t *ipmr_seq) { struct vif_entry_notifier_info info = { .info = { @@ -89,7 +89,7 @@ static inline int mr_call_vif_notifiers(struct net *net, }; ASSERT_RTNL(); - (*ipmr_seq)++; + atomic_inc(ipmr_seq); return call_fib_notifiers(net, event_type, &info.info); } @@ -198,7 +198,7 @@ static inline int mr_call_mfc_notifiers(struct net *net, unsigned short family, enum fib_event_type event_type, struct mr_mfc *mfc, u32 tb_id, - unsigned int *ipmr_seq) + atomic_t *ipmr_seq) { struct mfc_entry_notifier_info info = { .info = { @@ -208,8 +208,7 @@ static inline int mr_call_mfc_notifiers(struct net *net, .tb_id = tb_id }; - ASSERT_RTNL(); - (*ipmr_seq)++; + atomic_inc(ipmr_seq); return call_fib_notifiers(net, event_type, &info.info); } @@ -227,6 +226,7 @@ struct mr_table_ops { /** * struct mr_table - a multicast routing table + * @work: used for table destruction * @list: entry within a list of multicast routing tables * @net: net where this table belongs * @ops: protocol specific operations @@ -244,6 +244,7 @@ struct mr_table_ops { * @mroute_reg_vif_num: PIM-device vif index */ struct mr_table { + struct rcu_work work; struct list_head list; possible_net_t net; struct mr_table_ops ops; @@ -255,7 +256,7 @@ struct mr_table { struct rhltable mfc_hash; struct list_head mfc_cache_list; int maxvif; - atomic_t cache_resolve_queue_len; + u32 cache_resolve_queue_len; bool mroute_do_assert; bool mroute_do_pim; bool mroute_do_wrvifwhole; @@ -275,6 +276,7 @@ void vif_device_init(struct vif_device *v, unsigned short flags, unsigned short get_iflink_mask); +void mr_table_free(struct mr_table *mrt); struct mr_table * mr_table_alloc(struct net *net, u32 id, struct mr_table_ops *ops, diff --git a/include/linux/msi.h b/include/linux/msi.h index fa41eed62868..a4613de11960 100644 --- a/include/linux/msi.h +++ b/include/linux/msi.h @@ -444,7 +444,7 @@ struct msi_domain_info; * * @domain_alloc_irqs, @domain_free_irqs can be used to override the * default allocation/free functions (__msi_domain_alloc/free_irqs). This - * is initially for a wrapper around XENs seperate MSI universe which can't + * is initially for a wrapper around XEN's separate MSI universe which can't * be wrapped into the regular irq domains concepts by mere mortals. This * allows to universally use msi_domain_alloc/free_irqs without having to * special case XEN all over the place. diff --git a/include/linux/mtd/concat.h b/include/linux/mtd/concat.h index d6f653e07426..f8d4d6ac1fc1 100644 --- a/include/linux/mtd/concat.h +++ b/include/linux/mtd/concat.h @@ -9,6 +9,18 @@ #define MTD_CONCAT_H +/* + * Our storage structure: + * Subdev points to an array of pointers to struct mtd_info objects + * which is allocated along with this structure + * + */ +struct mtd_concat { + struct mtd_info mtd; + int num_subdev; + struct mtd_info *subdev[]; +}; + struct mtd_info *mtd_concat_create( struct mtd_info *subdev[], /* subdevices to concatenate */ int num_devs, /* number of subdevices */ @@ -16,5 +28,54 @@ struct mtd_info *mtd_concat_create( void mtd_concat_destroy(struct mtd_info *mtd); -#endif +/** + * mtd_virt_concat_node_create - Create a component for concatenation + * + * Returns a positive number representing the no. of devices found for + * concatenation, or a negative error code. + * + * List all the devices for concatenations found in DT and create a + * component for concatenation. + */ +int mtd_virt_concat_node_create(void); + +/** + * mtd_virt_concat_add - add mtd_info object to the list of subdevices for concatenation + * @mtd: pointer to new MTD device info structure + * + * Returns true if the mtd_info object is added successfully else returns false. + * + * The mtd_info object is added to the list of subdevices for concatenation. + * It returns true if a match is found, and false if all subdevices have + * already been added or if the mtd_info object does not match any of the + * intended MTD devices. + */ +bool mtd_virt_concat_add(struct mtd_info *mtd); +/** + * mtd_virt_concat_create_join - Create and register the concatenated MTD device + * + * Returns 0 on succes, or a negative error code. + * + * Creates and registers the concatenated MTD device + */ +int mtd_virt_concat_create_join(void); + +/** + * mtd_virt_concat_destroy - Remove the concat that includes a specific mtd device + * as one of its components. + * @mtd: pointer to MTD device info structure. + * + * Returns 0 on succes, or a negative error code. + * + * If the mtd_info object is part of a concatenated device, all other MTD devices + * within that concat are registered individually. The concatenated device is then + * removed, along with its concatenation component. + * + */ +int mtd_virt_concat_destroy(struct mtd_info *mtd); + +void mtd_virt_concat_destroy_joins(void); +void mtd_virt_concat_destroy_items(void); + +#endif diff --git a/include/linux/mtd/nand-qpic-common.h b/include/linux/mtd/nand-qpic-common.h index e8201d1b7cf9..437448995187 100644 --- a/include/linux/mtd/nand-qpic-common.h +++ b/include/linux/mtd/nand-qpic-common.h @@ -9,6 +9,8 @@ #ifndef __MTD_NAND_QPIC_COMMON_H__ #define __MTD_NAND_QPIC_COMMON_H__ +#include <linux/mtd/rawnand.h> + /* NANDc reg offsets */ #define NAND_FLASH_CMD 0x00 #define NAND_ADDR0 0x04 @@ -394,7 +396,7 @@ struct qcom_nand_controller { const struct qcom_nandc_props *props; - struct nand_controller *controller; + struct nand_controller controller; struct qpic_spi_nand *qspi; struct list_head host_list; @@ -441,6 +443,7 @@ struct qcom_nand_controller { * @dev_cmd_reg_start - NAND_DEV_CMD_* registers starting offset * @supports_bam - whether NAND controller is using BAM * @nandc_part_of_qpic - whether NAND controller is part of qpic IP + * @has_onfi_read_op - whether ONFI param page read command is supported * @qpic_version2 - flag to indicate QPIC IP version 2 * @use_codeword_fixup - whether NAND has different layout for boot partitions */ @@ -450,6 +453,7 @@ struct qcom_nandc_props { u32 bam_offset; bool supports_bam; bool nandc_part_of_qpic; + bool has_onfi_read_op; bool qpic_version2; bool use_codeword_fixup; }; diff --git a/include/linux/mtd/spi-nor.h b/include/linux/mtd/spi-nor.h index cdcfe0fd2e7d..4b92494827b1 100644 --- a/include/linux/mtd/spi-nor.h +++ b/include/linux/mtd/spi-nor.h @@ -21,8 +21,8 @@ /* Flash opcodes. */ #define SPINOR_OP_WRDI 0x04 /* Write disable */ #define SPINOR_OP_WREN 0x06 /* Write enable */ -#define SPINOR_OP_RDSR 0x05 /* Read status register */ -#define SPINOR_OP_WRSR 0x01 /* Write status register 1 byte */ +#define SPINOR_OP_RDSR 0x05 /* Read status register 1 */ +#define SPINOR_OP_WRSR 0x01 /* Write status register 1 */ #define SPINOR_OP_RDSR2 0x3f /* Read status register 2 */ #define SPINOR_OP_WRSR2 0x3e /* Write status register 2 */ #define SPINOR_OP_READ 0x03 /* Read data bytes (low frequency) */ @@ -125,6 +125,7 @@ #define SR2_LB1 BIT(3) /* Security Register Lock Bit 1 */ #define SR2_LB2 BIT(4) /* Security Register Lock Bit 2 */ #define SR2_LB3 BIT(5) /* Security Register Lock Bit 3 */ +#define SR2_CMP_BIT6 BIT(6) #define SR2_QUAD_EN_BIT7 BIT(7) /* Supported SPI protocols */ @@ -371,6 +372,7 @@ struct spi_nor_flash_parameter; * @reg_proto: the SPI protocol for read_reg/write_reg/erase operations * @sfdp: the SFDP data of the flash * @debugfs_root: pointer to the debugfs directory + * @dfs_sr_cache: Status Register cached value for debugfs use only * @controller_ops: SPI NOR controller driver specific operations. * @params: [FLASH-SPECIFIC] SPI NOR flash parameters and settings. * The structure includes legacy flash parameters and @@ -409,6 +411,7 @@ struct spi_nor { enum spi_nor_cmd_ext cmd_ext_type; struct sfdp *sfdp; struct dentry *debugfs_root; + u8 dfs_sr_cache[2]; const struct spi_nor_controller_ops *controller_ops; diff --git a/include/linux/mtd/spinand.h b/include/linux/mtd/spinand.h index 6a024cf1c53a..5f4c00ae72a7 100644 --- a/include/linux/mtd/spinand.h +++ b/include/linux/mtd/spinand.h @@ -290,6 +290,12 @@ SPI_MEM_OP_NO_DUMMY, \ SPI_MEM_OP_NO_DATA) +#define SPINAND_PAGE_READ_PACKED_8D_8D_0_OP(addr) \ + SPI_MEM_OP(SPI_MEM_DTR_OP_PACKED_CMD(0x13, addr >> 16, 8), \ + SPI_MEM_DTR_OP_ADDR(2, addr & 0xffff, 8), \ + SPI_MEM_OP_NO_DUMMY, \ + SPI_MEM_OP_NO_DATA) + #define SPINAND_PAGE_READ_FROM_CACHE_8D_8D_8D_OP(addr, ndummy, buf, len, freq) \ SPI_MEM_OP(SPI_MEM_DTR_OP_RPT_CMD(0x9d, 8), \ SPI_MEM_DTR_OP_ADDR(2, addr, 8), \ @@ -431,6 +437,7 @@ extern const struct spinand_manufacturer esmt_c8_spinand_manufacturer; extern const struct spinand_manufacturer fmsh_spinand_manufacturer; extern const struct spinand_manufacturer foresee_spinand_manufacturer; extern const struct spinand_manufacturer gigadevice_spinand_manufacturer; +extern const struct spinand_manufacturer heyangtek_spinand_manufacturer; extern const struct spinand_manufacturer macronix_spinand_manufacturer; extern const struct spinand_manufacturer micron_spinand_manufacturer; extern const struct spinand_manufacturer paragon_spinand_manufacturer; @@ -477,11 +484,13 @@ struct spinand_ecc_info { const struct mtd_ooblayout_ops *ooblayout; }; -#define SPINAND_HAS_QE_BIT BIT(0) -#define SPINAND_HAS_CR_FEAT_BIT BIT(1) +/* SPI NAND flags */ +#define SPINAND_HAS_QE_BIT BIT(0) +#define SPINAND_HAS_CR_FEAT_BIT BIT(1) #define SPINAND_HAS_PROG_PLANE_SELECT_BIT BIT(2) #define SPINAND_HAS_READ_PLANE_SELECT_BIT BIT(3) #define SPINAND_NO_RAW_ACCESS BIT(4) +#define SPINAND_ODTR_PACKED_PAGE_READ BIT(5) /** * struct spinand_ondie_ecc_conf - private SPI-NAND on-die ECC engine structure @@ -575,6 +584,7 @@ enum spinand_bus_interface { * @op_variants.read_cache: variants of the read-cache operation * @op_variants.write_cache: variants of the write-cache operation * @op_variants.update_cache: variants of the update-cache operation + * @op_variants.cont_read_cache: variants of the continuous read-cache operation * @vendor_ops: vendor specific operations * @select_target: function used to select a target/die. Required only for * multi-die chips @@ -584,6 +594,7 @@ enum spinand_bus_interface { * @user_otp: SPI NAND user OTP info. * @read_retries: the number of read retry modes supported * @set_read_retry: enable/disable read retry for data recovery + * @set_randomizer: enable/disable randomizer support * * Each SPI NAND manufacturer driver should have a spinand_info table * describing all the chips supported by the driver. @@ -599,6 +610,7 @@ struct spinand_info { const struct spinand_op_variants *read_cache; const struct spinand_op_variants *write_cache; const struct spinand_op_variants *update_cache; + const struct spinand_op_variants *cont_read_cache; } op_variants; const struct spinand_op_variants *vendor_ops; int (*select_target)(struct spinand_device *spinand, @@ -612,6 +624,8 @@ struct spinand_info { unsigned int read_retries; int (*set_read_retry)(struct spinand_device *spinand, unsigned int read_retry); + int (*set_randomizer)(struct spinand_device *spinand, + bool enable); }; #define SPINAND_ID(__method, ...) \ @@ -628,6 +642,14 @@ struct spinand_info { .update_cache = __update, \ } +#define SPINAND_INFO_OP_VARIANTS_WITH_CONT(__read, __write, __update, __cont_read) \ + { \ + .read_cache = __read, \ + .write_cache = __write, \ + .update_cache = __update, \ + .cont_read_cache = __cont_read, \ + } + #define SPINAND_INFO_VENDOR_OPS(__ops) \ .vendor_ops = __ops @@ -668,6 +690,9 @@ struct spinand_info { .read_retries = __read_retries, \ .set_read_retry = __set_read_retry +#define SPINAND_RANDOMIZER(__set_randomizer) \ + .set_randomizer = __set_randomizer + #define SPINAND_INFO(__model, __id, __memorg, __eccreq, __op_variants, \ __flags, ...) \ { \ @@ -683,8 +708,6 @@ struct spinand_info { struct spinand_dirmap { struct spi_mem_dirmap_desc *wdesc; struct spi_mem_dirmap_desc *rdesc; - struct spi_mem_dirmap_desc *wdesc_ecc; - struct spi_mem_dirmap_desc *rdesc_ecc; }; /** @@ -701,6 +724,7 @@ struct spinand_dirmap { * @read_cache: read cache op template * @write_cache: write cache op template * @update_cache: update cache op template + * @cont_read_cache: continuous read cache op template (optional) */ struct spinand_mem_ops { struct spi_mem_op reset; @@ -715,6 +739,7 @@ struct spinand_mem_ops { const struct spi_mem_op *read_cache; const struct spi_mem_op *write_cache; const struct spi_mem_op *update_cache; + const struct spi_mem_op *cont_read_cache; }; /** @@ -753,6 +778,7 @@ struct spinand_mem_ops { * @user_otp: SPI NAND user OTP info. * @read_retries: the number of read retry modes supported * @set_read_retry: Enable/disable the read retry feature + * @set_randomizer: Enable/disable the randomizer feature */ struct spinand_device { struct nand_device base; @@ -786,6 +812,8 @@ struct spinand_device { bool cont_read_possible; int (*set_cont_read)(struct spinand_device *spinand, bool enable); + int (*set_randomizer)(struct spinand_device *spinand, + bool enable); const struct spinand_fact_otp *fact_otp; const struct spinand_user_otp *user_otp; @@ -861,6 +889,8 @@ static inline void spinand_set_of_node(struct spinand_device *spinand, nanddev_set_of_node(&spinand->base, np); } +bool spinand_op_is_odtr(const struct spi_mem_op *op); + int spinand_match_and_init(struct spinand_device *spinand, const struct spinand_info *table, unsigned int table_size, diff --git a/include/linux/mutex.h b/include/linux/mutex.h index ecaa0440f6ec..734048c02f4f 100644 --- a/include/linux/mutex.h +++ b/include/linux/mutex.h @@ -79,7 +79,7 @@ do { \ #define __MUTEX_INITIALIZER(lockname) \ { .owner = ATOMIC_LONG_INIT(0) \ , .wait_lock = __RAW_SPIN_LOCK_UNLOCKED(lockname.wait_lock) \ - , .wait_list = LIST_HEAD_INIT(lockname.wait_list) \ + , .first_waiter = NULL \ __DEBUG_MUTEX_INITIALIZER(lockname) \ __DEP_MAP_MUTEX_INITIALIZER(lockname) } @@ -87,12 +87,12 @@ do { \ struct mutex mutexname = __MUTEX_INITIALIZER(mutexname) #ifdef CONFIG_DEBUG_LOCK_ALLOC -void mutex_init_lockep(struct mutex *lock, const char *name, struct lock_class_key *key); +void mutex_init_lockdep(struct mutex *lock, const char *name, struct lock_class_key *key); static inline void __mutex_init(struct mutex *lock, const char *name, struct lock_class_key *key) { - mutex_init_lockep(lock, name, key); + mutex_init_lockdep(lock, name, key); } #else extern void mutex_init_generic(struct mutex *lock); @@ -146,7 +146,7 @@ static inline void __mutex_init(struct mutex *lock, const char *name, { mutex_rt_init_generic(lock); } -#endif /* !CONFIG_LOCKDEP */ +#endif /* !CONFIG_DEBUG_LOCK_ALLOC */ #endif /* CONFIG_PREEMPT_RT */ #ifdef CONFIG_DEBUG_MUTEXES @@ -183,7 +183,7 @@ static inline int __must_check __devm_mutex_init(struct device *dev, struct mute */ #ifdef CONFIG_DEBUG_LOCK_ALLOC extern void mutex_lock_nested(struct mutex *lock, unsigned int subclass) __acquires(lock); -extern void _mutex_lock_nest_lock(struct mutex *lock, struct lockdep_map *nest_lock); +extern void _mutex_lock_nest_lock(struct mutex *lock, struct lockdep_map *nest_lock) __acquires(lock); extern int __must_check mutex_lock_interruptible_nested(struct mutex *lock, unsigned int subclass) __cond_acquires(0, lock); extern int __must_check _mutex_lock_killable(struct mutex *lock, @@ -253,6 +253,7 @@ extern int atomic_dec_and_mutex_lock(atomic_t *cnt, struct mutex *lock) __cond_a DEFINE_LOCK_GUARD_1(mutex, struct mutex, mutex_lock(_T->lock), mutex_unlock(_T->lock)) DEFINE_LOCK_GUARD_1_COND(mutex, _try, mutex_trylock(_T->lock)) DEFINE_LOCK_GUARD_1_COND(mutex, _intr, mutex_lock_interruptible(_T->lock), _RET == 0) +DEFINE_LOCK_GUARD_1_COND(mutex, _kill, mutex_lock_killable(_T->lock), _RET == 0) DEFINE_LOCK_GUARD_1(mutex_init, struct mutex, mutex_init(_T->lock), /* */) DECLARE_LOCK_GUARD_1_ATTRS(mutex, __acquires(_T), __releases(*(struct mutex **)_T)) @@ -261,6 +262,8 @@ DECLARE_LOCK_GUARD_1_ATTRS(mutex_try, __acquires(_T), __releases(*(struct mutex #define class_mutex_try_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(mutex_try, _T) DECLARE_LOCK_GUARD_1_ATTRS(mutex_intr, __acquires(_T), __releases(*(struct mutex **)_T)) #define class_mutex_intr_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(mutex_intr, _T) +DECLARE_LOCK_GUARD_1_ATTRS(mutex_kill, __acquires(_T), __releases(*(struct mutex **)_T)) +#define class_mutex_kill_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(mutex_kill, _T) DECLARE_LOCK_GUARD_1_ATTRS(mutex_init, __acquires(_T), __releases(*(struct mutex **)_T)) #define class_mutex_init_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(mutex_init, _T) diff --git a/include/linux/mutex_types.h b/include/linux/mutex_types.h index 80975935ec48..24ed599fdda8 100644 --- a/include/linux/mutex_types.h +++ b/include/linux/mutex_types.h @@ -44,7 +44,7 @@ context_lock_struct(mutex) { #ifdef CONFIG_MUTEX_SPIN_ON_OWNER struct optimistic_spin_queue osq; /* Spinner MCS lock */ #endif - struct list_head wait_list; + struct mutex_waiter *first_waiter __guarded_by(&wait_lock); #ifdef CONFIG_DEBUG_MUTEXES void *magic; #endif diff --git a/include/linux/mux/consumer.h b/include/linux/mux/consumer.h index 2e25c838f831..449e38e6e2c5 100644 --- a/include/linux/mux/consumer.h +++ b/include/linux/mux/consumer.h @@ -16,6 +16,8 @@ struct device; struct mux_control; struct mux_state; +#if IS_ENABLED(CONFIG_MULTIPLEXER) + unsigned int mux_control_states(struct mux_control *mux); int __must_check mux_control_select_delay(struct mux_control *mux, unsigned int state, @@ -54,11 +56,115 @@ int mux_control_deselect(struct mux_control *mux); int mux_state_deselect(struct mux_state *mstate); struct mux_control *mux_control_get(struct device *dev, const char *mux_name); +struct mux_control *mux_control_get_optional(struct device *dev, const char *mux_name); void mux_control_put(struct mux_control *mux); -struct mux_control *devm_mux_control_get(struct device *dev, - const char *mux_name); -struct mux_state *devm_mux_state_get(struct device *dev, - const char *mux_name); +struct mux_control *devm_mux_control_get(struct device *dev, const char *mux_name); + +struct mux_state * +devm_mux_state_get_from_np(struct device *dev, const char *mux_name, struct device_node *np); + +struct mux_state *devm_mux_state_get_optional(struct device *dev, const char *mux_name); +struct mux_state *devm_mux_state_get_selected(struct device *dev, const char *mux_name); +struct mux_state *devm_mux_state_get_optional_selected(struct device *dev, const char *mux_name); + +#else + +static inline unsigned int mux_control_states(struct mux_control *mux) +{ + return 0; +} +static inline int __must_check mux_control_select_delay(struct mux_control *mux, + unsigned int state, unsigned int delay_us) +{ + return -EOPNOTSUPP; +} +static inline int __must_check mux_state_select_delay(struct mux_state *mstate, + unsigned int delay_us) +{ + return -EOPNOTSUPP; +} +static inline int __must_check mux_control_try_select_delay(struct mux_control *mux, + unsigned int state, + unsigned int delay_us) +{ + return -EOPNOTSUPP; +} +static inline int __must_check mux_state_try_select_delay(struct mux_state *mstate, + unsigned int delay_us) +{ + return -EOPNOTSUPP; +} + +static inline int __must_check mux_control_select(struct mux_control *mux, + unsigned int state) +{ + return -EOPNOTSUPP; +} + +static inline int __must_check mux_state_select(struct mux_state *mstate) +{ + return -EOPNOTSUPP; +} + +static inline int __must_check mux_control_try_select(struct mux_control *mux, + unsigned int state) +{ + return -EOPNOTSUPP; +} + +static inline int __must_check mux_state_try_select(struct mux_state *mstate) +{ + return -EOPNOTSUPP; +} + +static inline int mux_control_deselect(struct mux_control *mux) +{ + return -EOPNOTSUPP; +} +static inline int mux_state_deselect(struct mux_state *mstate) +{ + return -EOPNOTSUPP; +} + +static inline struct mux_control *mux_control_get(struct device *dev, const char *mux_name) +{ + return ERR_PTR(-EOPNOTSUPP); +} +static inline struct mux_control *mux_control_get_optional(struct device *dev, + const char *mux_name) +{ + return NULL; +} +static inline void mux_control_put(struct mux_control *mux) {} + +static inline struct mux_control *devm_mux_control_get(struct device *dev, const char *mux_name) +{ + return ERR_PTR(-EOPNOTSUPP); +} +static inline struct mux_state *devm_mux_state_get(struct device *dev, const char *mux_name) +{ + return ERR_PTR(-EOPNOTSUPP); +} +static inline struct mux_state *devm_mux_state_get_optional(struct device *dev, + const char *mux_name) +{ + return NULL; +} +static inline struct mux_state *devm_mux_state_get_selected(struct device *dev, + const char *mux_name) +{ + return ERR_PTR(-EOPNOTSUPP); +} +static inline struct mux_state *devm_mux_state_get_optional_selected(struct device *dev, + const char *mux_name) +{ + return NULL; +} + +#endif /* CONFIG_MULTIPLEXER */ + +#define devm_mux_state_get(dev, mux_name) \ + devm_mux_state_get_from_np(dev, mux_name, NULL) #endif /* _LINUX_MUX_CONSUMER_H */ diff --git a/include/linux/namei.h b/include/linux/namei.h index 58600cf234bc..86d657b24fc6 100644 --- a/include/linux/namei.h +++ b/include/linux/namei.h @@ -13,11 +13,6 @@ enum { MAX_NESTED_LINKS = 8 }; #define MAXSYMLINKS 40 -/* - * Type of the last component on LOOKUP_PARENT - */ -enum {LAST_NORM, LAST_ROOT, LAST_DOT, LAST_DOTDOT}; - /* pathwalk mode */ #define LOOKUP_FOLLOW BIT(0) /* follow links at the end */ #define LOOKUP_DIRECTORY BIT(1) /* require a directory */ @@ -54,9 +49,6 @@ extern int path_pts(struct path *path); extern int user_path_at(int, const char __user *, unsigned, struct path *); -struct dentry *lookup_one_qstr_excl(const struct qstr *name, - struct dentry *base, - unsigned int flags); extern int kern_path(const char *, unsigned, struct path *); struct dentry *kern_path_parent(const char *name, struct path *parent); @@ -64,13 +56,12 @@ extern struct dentry *start_creating_path(int, const char *, struct path *, unsi extern struct dentry *start_creating_user_path(int, const char __user *, struct path *, unsigned int); extern void end_creating_path(const struct path *, struct dentry *); extern struct dentry *start_removing_path(const char *, struct path *); -extern struct dentry *start_removing_user_path_at(int , const char __user *, struct path *); static inline void end_removing_path(const struct path *path , struct dentry *dentry) { end_creating_path(path, dentry); } int vfs_path_parent_lookup(struct filename *filename, unsigned int flags, - struct path *parent, struct qstr *last, int *type, + struct path *parent, struct qstr *last, const struct path *root); int vfs_path_lookup(struct dentry *, struct vfsmount *, const char *, unsigned int, struct path *); @@ -106,6 +97,9 @@ struct dentry *start_creating_dentry(struct dentry *parent, struct dentry *start_removing_dentry(struct dentry *parent, struct dentry *child); +struct file *vfs_lookup_open(struct path *parent, struct qstr *last, + int open_flag, umode_t mode); + /* end_creating - finish action started with start_creating * @child: dentry returned by start_creating() or vfs_mkdir() * @@ -168,9 +162,6 @@ extern int follow_down_one(struct path *); extern int follow_down(struct path *path, unsigned int flags); extern int follow_up(struct path *); -extern struct dentry *lock_rename(struct dentry *, struct dentry *); -extern struct dentry *lock_rename_child(struct dentry *, struct dentry *); -extern void unlock_rename(struct dentry *, struct dentry *); int start_renaming(struct renamedata *rd, int lookup_flags, struct qstr *old_last, struct qstr *new_last); int start_renaming_dentry(struct renamedata *rd, int lookup_flags, diff --git a/include/linux/nd.h b/include/linux/nd.h index fa099e295f78..62988000e7a7 100644 --- a/include/linux/nd.h +++ b/include/linux/nd.h @@ -110,7 +110,7 @@ static inline struct nd_namespace_common *to_ndns(struct device *dev) /** * struct nd_namespace_io - device representation of a persistent memory range - * @dev: namespace device created by the nd region driver + * @common: namespace device core infrastructure created by the nd region driver * @res: struct resource conversion of a NFIT SPA table * @size: cached resource_size(@res) for fast path size checks * @addr: virtual address to access the namespace range @@ -158,8 +158,11 @@ static inline struct nd_namespace_pmem *to_nd_namespace_pmem(const struct device * @offset: namespace-relative starting offset * @buf: buffer to fill * @size: transfer length + * @flags: process (0) or atomic (1) context * * @buf is up-to-date upon return from this routine. + * + * Returns: %0 on success or a negative error code on failure */ static inline int nvdimm_read_bytes(struct nd_namespace_common *ndns, resource_size_t offset, void *buf, size_t size, @@ -174,11 +177,14 @@ static inline int nvdimm_read_bytes(struct nd_namespace_common *ndns, * @offset: namespace-relative starting offset * @buf: buffer to drain * @size: transfer length + * @flags: process (0) or atomic (1) context * * NVDIMM Namepaces disks do not implement sectors internally. Depending on * the @ndns, the contents of @buf may be in cpu cache, platform buffers, * or on backing memory media upon return from this routine. Flushing * to media is handled internal to the @ndns driver, if at all. + * + * Returns: %0 on success or a negative error code on failure */ static inline int nvdimm_write_bytes(struct nd_namespace_common *ndns, resource_size_t offset, void *buf, size_t size, diff --git a/include/linux/net.h b/include/linux/net.h index f58b38ab37f8..3d82966e2243 100644 --- a/include/linux/net.h +++ b/include/linux/net.h @@ -23,9 +23,53 @@ #include <linux/fs.h> #include <linux/mm.h> #include <linux/sockptr.h> +#include <linux/uio.h> #include <uapi/linux/net.h> +/** + * struct sockopt - socket option value container + * @iter_in: iov_iter for reading optval with the content from the caller. + * Use copy_from_iter() given this iov direction is ITER_SOURCE + * @iter_out: iov_iter for protocols to update optval data to userspace + * Use _copy_to_iter() given iov direction is ITER_DEST + * @optlen: serves as both input (buffer size) and output (returned data size). + * + * Type-safe wrapper for socket option data that works with both + * user and kernel buffers. + * + * The optlen field allows callbacks to return a specific length value + * independent of the bytes written via copy_to_iter(). + */ +typedef struct sockopt { + struct iov_iter iter_in; + struct iov_iter iter_out; + int optlen; +} sockopt_t; + +/* + * Initialize a user-backed sockopt_t from the (optval, optlen) __user pair of + * a getsockopt() callback. Used by transitional __user getsockopt wrappers + * while the proto-layer callbacks are converted to take a sockopt_t; the + * caller writes opt->optlen back to the user optlen after the callback. + */ +static inline int sockopt_init_user(sockopt_t *opt, char __user *optval, + int __user *optlen) +{ + int len; + + if (get_user(len, optlen)) + return -EFAULT; + if (len < 0) + return -EINVAL; + + iov_iter_ubuf(&opt->iter_out, ITER_DEST, optval, len); + iov_iter_ubuf(&opt->iter_in, ITER_SOURCE, optval, len); + opt->optlen = len; + + return 0; +} + struct poll_table_struct; struct pipe_inode_info; struct inode; @@ -192,6 +236,8 @@ struct proto_ops { unsigned int optlen); int (*getsockopt)(struct socket *sock, int level, int optname, char __user *optval, int __user *optlen); + int (*getsockopt_iter)(struct socket *sock, int level, + int optname, sockopt_t *opt); void (*show_fdinfo)(struct seq_file *m, struct socket *sock); int (*sendmsg) (struct socket *sock, struct msghdr *m, size_t total_len); @@ -223,6 +269,7 @@ struct proto_ops { int (*sendmsg_locked)(struct sock *sk, struct msghdr *msg, size_t size); int (*set_rcvlowat)(struct sock *sk, int val); + void (*set_rcvbuf)(struct sock *sk, int val); }; #define DECLARE_SOCKADDR(type, dst, src) \ @@ -261,6 +308,7 @@ int sock_recvmsg(struct socket *sock, struct msghdr *msg, int flags); struct file *sock_alloc_file(struct socket *sock, int flags, const char *dname); struct socket *sockfd_lookup(int fd, int *err); struct socket *sock_from_file(struct file *file); +int sock_read_xattr(struct socket *sock, const char *name, void *value, size_t size); #define sockfd_put(sock) fput(sock->file) int net_ratelimit(void); @@ -304,6 +352,8 @@ do { \ #define net_get_random_once(buf, nbytes) \ get_random_once((buf), (nbytes)) +#define net_get_random_sleepable_once(buf, nbytes) \ + get_random_sleepable_once((buf), (nbytes)) /* * E.g. XFS meta- & log-data is in slab pages, or bcache meta diff --git a/include/linux/net/intel/libie/adminq.h b/include/linux/net/intel/libie/adminq.h index ab13bd777a28..839114d8975a 100644 --- a/include/linux/net/intel/libie/adminq.h +++ b/include/linux/net/intel/libie/adminq.h @@ -196,6 +196,7 @@ LIBIE_CHECK_STRUCT_LEN(16, libie_aqc_list_caps); #define LIBIE_AQC_BIT_ROCEV2_LAG BIT(0) #define LIBIE_AQC_BIT_SRIOV_LAG BIT(1) #define LIBIE_AQC_BIT_SRIOV_AA_LAG BIT(2) +#define LIBIE_AQC_CAPS_EEE 0x009B #define LIBIE_AQC_CAPS_FLEX10 0x00F1 #define LIBIE_AQC_CAPS_CEM 0x00F2 diff --git a/include/linux/net/intel/libie/controlq.h b/include/linux/net/intel/libie/controlq.h new file mode 100644 index 000000000000..175227d4eef1 --- /dev/null +++ b/include/linux/net/intel/libie/controlq.h @@ -0,0 +1,436 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright (C) 2025 Intel Corporation */ + +#ifndef __LIBIE_CONTROLQ_H +#define __LIBIE_CONTROLQ_H + +#include <linux/dmapool.h> +#include <net/libeth/rx.h> + +#include <linux/net/intel/libie/pci.h> +#include <linux/net/intel/virtchnl2.h> + +/* Default mailbox control queue */ +#define LIBIE_CTLQ_MBX_ID -1 +#define LIBIE_CTLQ_MAX_BUF_LEN SZ_4K + +/** + * enum libie_ctlq_type - control queue type + * @LIBIE_CTLQ_TYPE_TX: basic Tx control queue + * @LIBIE_CTLQ_TYPE_RX: basic Rx control queue + */ +enum libie_ctlq_type { + LIBIE_CTLQ_TYPE_TX = 0, + LIBIE_CTLQ_TYPE_RX = 1, +}; + +/* Opcode used to send controlq message to the control plane */ +#define LIBIE_CTLQ_SEND_MSG_TO_CP 0x801 +#define LIBIE_CTLQ_SEND_MSG_TO_PEER 0x804 + +#define LIBIE_CP_TX_COPYBREAK 128 + +/** + * struct libie_ctlq_ctx - contains controlq info and MMIO region info + * @mmio_info: MMIO region info structure + * @ctlqs: list that stores all the control queues + * @ctlqs_lock: lock for control queue list + */ +struct libie_ctlq_ctx { + struct libie_mmio_info mmio_info; + struct list_head ctlqs; + spinlock_t ctlqs_lock; /* protects the ctlqs list */ +}; + +/** + * struct libie_ctlq_reg - structure representing virtual addresses of the + * controlq registers and masks + * @head: controlq head register address + * @tail: controlq tail register address + * @len: register address to write controlq length and enable bit + * @addr_high: register address to write the upper 32b of ring physical address + * @addr_low: register address to write the lower 32b of ring physical address + * @len_mask: mask to read the controlq length + * @len_ena_mask: mask to write the controlq enable bit + * @head_mask: mask to read the head value + */ +struct libie_ctlq_reg { + void __iomem *head; + void __iomem *tail; + void __iomem *len; + void __iomem *addr_high; + void __iomem *addr_low; + u32 len_mask; + u32 len_ena_mask; + u32 head_mask; +}; + +/** + * struct libie_cp_dma_mem - structure for DMA memory + * @va: virtual address + * @pa: physical address + * @size: memory size + * @direction: memory to device or device to memory + */ +struct libie_cp_dma_mem { + void *va; + dma_addr_t pa; + size_t size; + int direction; +}; + +/** + * struct libie_ctlq_msg - control queue message data + * @flags: refer to 'Flags sub-structure' definitions + * @opcode: infrastructure message opcode + * @data_len: size of the payload + * @func_id: queue id for mailbox selection, 0 for default mailbox (Tx) + * @hw_retval: execution status from the HW (Rx) + * @chnl_opcode: virtchnl message opcode + * @chnl_retval: virtchnl return value + * @param0: indirect message raw parameter0 + * @sw_cookie: used to verify the response of the sent virtchnl message + * @virt_flags: virtchnl capability flags + * @addr_param: additional parameters in place of the address, given no buffer + * @recv_mem: virtual address and size of the buffer that contains + * the indirect response + * @send_mem: physical and virtual address of the DMA buffer, + * used for sending + */ +struct libie_ctlq_msg { + u16 flags; + u16 opcode; + u16 data_len; + union { + u16 func_id; + u16 hw_retval; + }; + u32 chnl_opcode; + u32 chnl_retval; + u32 param0; + u16 sw_cookie; + u16 virt_flags; + u64 addr_param; + union { + struct kvec recv_mem; + struct libie_cp_dma_mem send_mem; + }; +}; + +/** + * struct libie_ctlq_create_info - control queue create information + * @type: control queue type (Rx or Tx) + * @id: queue offset passed as input, -1 for default mailbox + * @reg: registers accessed by control queue + * @len: controlq length + */ +struct libie_ctlq_create_info { + enum libie_ctlq_type type; + int id; + struct libie_ctlq_reg reg; + u16 len; +}; + +/** + * struct libie_ctlq_info - control queue information + * @list: used to add a controlq to the list of queues in libie_ctlq_ctx + * @type: control queue type + * @qid: queue identifier + * @lock: control queue lock + * @ring_mem: descriptor ring DMA memory + * @descs: array of descriptors + * @rx_fqes: array of controlq Rx buffers + * @tx_msg: Tx messages sent to hardware + * @reg: registers used by control queue + * @dev: device that owns this control queue + * @pp: page pool for controlq Rx buffers + * @truesize: size to allocate per buffer + * @next_to_clean: next descriptor to be cleaned + * @next_to_use: next available slot to send buffer (Tx queue) + * @next_to_post: next available slot to post buffers to (Rx queue) + * @ring_len: length of the descriptor ring + */ +struct libie_ctlq_info { + struct list_head list; + enum libie_ctlq_type type; + int qid; + spinlock_t lock; /* for concurrent processing */ + struct libie_cp_dma_mem ring_mem; + struct libie_ctlq_desc *descs; + union { + struct libeth_fqe *rx_fqes; + struct libie_ctlq_msg **tx_msg; + }; + struct libie_ctlq_reg reg; + struct device *dev; + struct page_pool *pp; + u32 truesize; + u32 next_to_clean; + union { + u32 next_to_use; + u32 next_to_post; + }; + u32 ring_len; +}; + +#define LIBIE_CTLQ_MBX_ATQ_LEN GENMASK(9, 0) + +/* libie controlq descriptor qword0 details */ + +/* Flags sub-structure + * |0 |1 |2 |3 |4 |5 |6 |7 |8 |9 |10 |11 |12 |13 |14 |15 | + * |DD |CMP|ERR| * RSV * |FTYPE | *RSV* |RD |VFC|BUF| HOST_ID | + */ +#define LIBIE_CTLQ_DESC_FLAG_DD BIT(0) +#define LIBIE_CTLQ_DESC_FLAG_CMP BIT(1) +#define LIBIE_CTLQ_DESC_FLAG_ERR BIT(2) +#define LIBIE_CTLQ_DESC_FLAG_FTYPE_VM BIT(6) +#define LIBIE_CTLQ_DESC_FLAG_FTYPE_PF BIT(7) +#define LIBIE_CTLQ_DESC_FLAG_FTYPE GENMASK(7, 6) +#define LIBIE_CTLQ_DESC_FLAG_RD BIT(10) +#define LIBIE_CTLQ_DESC_FLAG_VFC BIT(11) +#define LIBIE_CTLQ_DESC_FLAG_BUF BIT(12) +#define LIBIE_CTLQ_DESC_FLAG_HOST_ID GENMASK(15, 13) + +#define LIBIE_CTLQ_DESC_FLAGS GENMASK(15, 0) +#define LIBIE_CTLQ_DESC_INFRA_OPCODE GENMASK_ULL(31, 16) +#define LIBIE_CTLQ_DESC_DATA_LEN GENMASK_ULL(47, 32) +#define LIBIE_CTLQ_DESC_HW_RETVAL GENMASK_ULL(63, 48) + +#define LIBIE_CTLQ_DESC_PFID_VFID GENMASK_ULL(63, 48) + +/* libie controlq descriptor qword1 details */ +#define LIBIE_CTLQ_DESC_VIRTCHNL_OPCODE GENMASK(27, 0) +#define LIBIE_CTLQ_DESC_VIRTCHNL_DESC_TYPE GENMASK_ULL(31, 28) +#define LIBIE_CTLQ_DESC_VIRTCHNL_MSG_RET_VAL GENMASK_ULL(63, 32) + +/* libie controlq descriptor qword2 details */ +#define LIBIE_CTLQ_DESC_MSG_PARAM0 GENMASK_ULL(31, 0) +#define LIBIE_CTLQ_DESC_SW_COOKIE GENMASK_ULL(47, 32) +#define LIBIE_CTLQ_DESC_VIRTCHNL_FLAGS GENMASK_ULL(63, 48) + +/* libie controlq descriptor qword3 details */ +#define LIBIE_CTLQ_DESC_DATA_ADDR_HIGH GENMASK_ULL(31, 0) +#define LIBIE_CTLQ_DESC_DATA_ADDR_LOW GENMASK_ULL(63, 32) + +/** + * struct libie_ctlq_desc - control queue descriptor format + * @qword0: flags, message opcode, data length etc + * @qword1: virtchnl opcode, descriptor type and return value + * @qword2: indirect message parameters + * @qword3: indirect message buffer address + */ +struct libie_ctlq_desc { + __le64 qword0; + __le64 qword1; + __le64 qword2; + __le64 qword3; +}; + +/** + * struct libie_ctlq_clean_params - cleaning parameters for Tx messages + * @rel_dma_mem: non-sleeping callback to put the DMA buffer after send + * @rel_ctx: additional context for release callback + * @ctlq: control queue information + * @num_msgs: number of messages to be cleaned + * @force: clean even if DD is not yet set, use only for final cleanup + */ +struct libie_ctlq_clean_params { + void (*rel_dma_mem)(const void *ctx, struct libie_cp_dma_mem *dma_mem); + const void *rel_ctx; + struct libie_ctlq_info *ctlq; + u16 num_msgs; + bool force; +}; + +/** + * libie_ctlq_release_rx_buf - Release Rx buffer for a specific control queue + * @rx_buf: Rx buffer to be freed + * + * Driver uses this function to post back the Rx buffer after the usage. + */ +static inline void libie_ctlq_release_rx_buf(struct kvec *rx_buf) +{ + netmem_ref netmem; + + if (!rx_buf->iov_base) + return; + + netmem = virt_to_netmem(rx_buf->iov_base); + page_pool_put_full_netmem(netmem_get_pp(netmem), netmem, false); +} + +int libie_ctlq_init(struct libie_ctlq_ctx *ctx, + const struct libie_ctlq_create_info *qinfo, u32 numq); +void libie_ctlq_deinit(struct libie_ctlq_ctx *ctx); + +struct libie_ctlq_info *libie_find_ctlq(struct libie_ctlq_ctx *ctx, + enum libie_ctlq_type type, + int id); + +u32 libie_ctlq_send_desc_avail(const struct libie_ctlq_info *ctlq); +void libie_ctlq_send(struct libie_ctlq_info *ctlq, u32 num_q_msg); +u32 libie_ctlq_send_clean(const struct libie_ctlq_clean_params *params); +u32 libie_ctlq_recv(struct libie_ctlq_info *ctlq, struct libie_ctlq_msg *msg, + u32 num_q_msg); + +int libie_ctlq_post_rx_buffs(struct libie_ctlq_info *ctlq); + +/* Only 8 bits are available in descriptor for Xn index */ +#define LIBIE_CTLQ_MAX_XN_ENTRIES 256 +#define LIBIE_CTLQ_XN_COOKIE_M GENMASK(15, 8) +#define LIBIE_CTLQ_XN_INDEX_M GENMASK(7, 0) + +/** + * enum libie_ctlq_xn_state - Transaction state of a virtchnl message + * @LIBIE_CTLQ_XN_IDLE: transaction is available to use + * @LIBIE_CTLQ_XN_WAITING: waiting for transaction to complete + * @LIBIE_CTLQ_XN_COMPLETED_SUCCESS: transaction completed with success + * @LIBIE_CTLQ_XN_COMPLETED_FAILED: transaction completed with failure + * @LIBIE_CTLQ_XN_ASYNC: asynchronous virtchnl message transaction type + * @LIBIE_CTLQ_XN_SHUTDOWN: transaction cannot be used anymore + */ +enum libie_ctlq_xn_state { + LIBIE_CTLQ_XN_IDLE = 0, + LIBIE_CTLQ_XN_WAITING, + LIBIE_CTLQ_XN_COMPLETED_SUCCESS, + LIBIE_CTLQ_XN_COMPLETED_FAILED, + LIBIE_CTLQ_XN_ASYNC, + LIBIE_CTLQ_XN_SHUTDOWN, +}; + +/** + * struct libie_ctlq_xn - structure representing a virtchnl transaction entry + * @resp_cb: non-sleeping callback to handle the response to an async message + * @xn_lock: lock to protect the transaction entry state + * @cmd_completion_event: wait until reply is received or xn is terminated + * @small_dma_mem: DMA memory for copying small send buffers from stack, + * is recycled when response is received or on timeout + * @send_dma_mem: DMA memory of send buffer + * @recv_mem: receive buffer + * @send_ctx: context for callback function + * @timeout_ms: Xn transaction timeout in msecs + * @timestamp: timestamp to record the Xn send + * @tx_msg: control queue Tx message slot to track small DMA usage + * @virtchnl_opcode: virtchnl command opcode used for Xn transaction + * @state: transaction state of a virtchnl message + * @cookie: unique message identifier, incremented every time the slot is used + * @index: index of the transaction entry + */ +struct libie_ctlq_xn { + void (*resp_cb)(void *ctx, struct kvec *mem, int status); + spinlock_t xn_lock; /* protects state */ + struct completion cmd_completion_event; + struct libie_cp_dma_mem small_dma_mem; + struct libie_cp_dma_mem send_dma_mem; + struct kvec recv_mem; + void *send_ctx; + u64 timeout_ms; + ktime_t timestamp; + struct libie_ctlq_msg *tx_msg; + u32 virtchnl_opcode; + enum libie_ctlq_xn_state state; + u8 cookie; + u8 index; +}; + +/** + * struct libie_ctlq_xn_manager - structure representing the array of virtchnl + * transaction entries + * @ctx: pointer to controlq context structure + * @free_xns_bm_lock: lock to protect the free Xn entries bit map + * @free_xns_bm: bitmap that represents the free Xn entries + * @ring: array of Xn entries + * @small_buff_pool: DMA pool for small send buffers + * @can_destroy: completion, triggered by the last released transaction + * @shutdown: shutdown process has been started, no new transactions allowed + */ +struct libie_ctlq_xn_manager { + struct libie_ctlq_ctx *ctx; + spinlock_t free_xns_bm_lock; /* get/check entries */ + DECLARE_BITMAP(free_xns_bm, LIBIE_CTLQ_MAX_XN_ENTRIES); + struct libie_ctlq_xn ring[LIBIE_CTLQ_MAX_XN_ENTRIES]; + struct dma_pool *small_buff_pool; + struct completion can_destroy; + bool shutdown; +}; + +/** + * struct libie_ctlq_xn_send_params - structure representing send Xn entry + * @resp_cb: non-sleeping callback to handle the response to an async message + * @rel_tx_buf: non-sleeping callback for freeing the send buffer + * @xnm: Xn manager to process Xn entries + * @ctlq: send control queue information + * @ctlq_msg: control queue message information + * @send_buf: buffer that carries outgoing message data, buffers larger than + * LIBIE_CP_TX_COPYBREAK bytes will always be consumed + * @recv_mem: receive buffer + * @send_ctx: context for callback function + * @timeout_ms: virtchnl transaction timeout in msecs + * @chnl_opcode: virtchnl message opcode + */ +struct libie_ctlq_xn_send_params { + void (*resp_cb)(void *ctx, struct kvec *mem, int status); + void (*rel_tx_buf)(const void *buf_va); + struct libie_ctlq_xn_manager *xnm; + struct libie_ctlq_info *ctlq; + struct libie_ctlq_msg *ctlq_msg; + struct kvec send_buf; + struct kvec recv_mem; + void *send_ctx; + u64 timeout_ms; + u32 chnl_opcode; +}; + +/** + * libie_cp_can_send_onstack - can a message be sent using a stack variable + * @size: ctlq data buffer size + * + * Return: %true if the message size is small enough for caller to pass + * an on-stack buffer, %false if kmalloc is needed + */ +static inline bool libie_cp_can_send_onstack(u32 size) +{ + return size <= LIBIE_CP_TX_COPYBREAK; +} + +/** + * struct libie_ctlq_xn_recv_params - request to receive xn responses + * @ctlq_msg_handler: handler for Rx messages with no matching xn (mandatory) + * @xnm: Xn manager to process Xn entries + * @ctlq: control queue information + * @budget: maximum number of messages to process + */ +struct libie_ctlq_xn_recv_params { + void (*ctlq_msg_handler)(struct libie_ctlq_ctx *ctx, + struct libie_ctlq_msg *msg); + struct libie_ctlq_xn_manager *xnm; + struct libie_ctlq_info *ctlq; + u32 budget; +}; + +/** + * struct libie_ctlq_xn_init_params - xn transaction manager parameters + * @cctlq_info: control queue information + * @ctx: pointer to controlq context structure + * @xnm: Xn manager to process Xn entries + * @num_qs: number of control queues to be initialized + */ +struct libie_ctlq_xn_init_params { + struct libie_ctlq_create_info *cctlq_info; + struct libie_ctlq_ctx *ctx; + struct libie_ctlq_xn_manager *xnm; + u32 num_qs; +}; + +int libie_ctlq_xn_init(struct libie_ctlq_xn_init_params *params); +void libie_ctlq_xn_deinit(struct libie_ctlq_xn_manager *xnm, + struct libie_ctlq_ctx *ctx); +void libie_ctlq_xn_shutdown(struct libie_ctlq_xn_manager *xnm); +int libie_ctlq_xn_send(struct libie_ctlq_xn_send_params *params); +u32 libie_ctlq_xn_recv(struct libie_ctlq_xn_recv_params *params); +u32 libie_ctlq_xn_send_clean(struct libie_ctlq_info *ctlq, + void (*rel_tx_buf)(const void *buf_va), + bool force); + +#endif /* __LIBIE_CONTROLQ_H */ diff --git a/include/linux/net/intel/libie/pci.h b/include/linux/net/intel/libie/pci.h new file mode 100644 index 000000000000..effd072c55c8 --- /dev/null +++ b/include/linux/net/intel/libie/pci.h @@ -0,0 +1,56 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright (C) 2025 Intel Corporation */ + +#ifndef __LIBIE_PCI_H +#define __LIBIE_PCI_H + +#include <linux/pci.h> + +/** + * struct libie_pci_mmio_region - structure for MMIO region info + * @list: used to add a MMIO region to the list of MMIO regions in + * libie_mmio_info + * @addr: virtual address of MMIO region start + * @offset: start offset of the MMIO region + * @size: size of the MMIO region + * @bar_idx: BAR index to which the MMIO region belongs to + */ +struct libie_pci_mmio_region { + struct list_head list; + void __iomem *addr; + resource_size_t offset; + resource_size_t size; + u16 bar_idx; +}; + +/** + * struct libie_mmio_info - contains list of MMIO regions + * @pdev: PCI device pointer + * @mmio_list: list of MMIO regions + */ +struct libie_mmio_info { + struct pci_dev *pdev; + struct list_head mmio_list; +}; + +#define libie_pci_map_mmio_region(mmio_info, offset, size, ...) \ + __libie_pci_map_mmio_region(mmio_info, offset, size, \ + COUNT_ARGS(__VA_ARGS__), ##__VA_ARGS__) + +#define libie_pci_get_mmio_addr(mmio_info, offset, ...) \ + __libie_pci_get_mmio_addr(mmio_info, offset, \ + COUNT_ARGS(__VA_ARGS__), ##__VA_ARGS__) + +bool __libie_pci_map_mmio_region(struct libie_mmio_info *mmio_info, + resource_size_t offset, resource_size_t size, + int num_args, ...); +void __iomem *__libie_pci_get_mmio_addr(struct libie_mmio_info *mmio_info, + resource_size_t offset, + int num_args, ...); +void libie_pci_unmap_all_mmio_regions(struct libie_mmio_info *mmio_info); +void libie_pci_unmap_fltr_regs(struct libie_mmio_info *mmio_info, + bool (*fltr)(struct libie_mmio_info *mmio_info, + struct libie_pci_mmio_region *reg)); +int libie_pci_init_dev(struct pci_dev *pdev); + +#endif /* __LIBIE_PCI_H */ diff --git a/include/linux/avf/virtchnl.h b/include/linux/net/intel/virtchnl.h index 11bdab5522fd..11bdab5522fd 100644 --- a/include/linux/avf/virtchnl.h +++ b/include/linux/net/intel/virtchnl.h diff --git a/include/linux/net/intel/virtchnl2.h b/include/linux/net/intel/virtchnl2.h new file mode 100644 index 000000000000..39fea65c075c --- /dev/null +++ b/include/linux/net/intel/virtchnl2.h @@ -0,0 +1,1819 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright (C) 2023 Intel Corporation */ + +#ifndef _VIRTCHNL2_H_ +#define _VIRTCHNL2_H_ + +#include <linux/if_ether.h> + +/* All opcodes associated with virtchnl2 are prefixed with virtchnl2 or + * VIRTCHNL2. Any future opcodes, offloads/capabilities, structures, + * and defines must be prefixed with virtchnl2 or VIRTCHNL2 to avoid confusion. + * + * PF/VF uses the virtchnl2 interface defined in this header file to communicate + * with device Control Plane (CP). Driver and the CP may run on different + * platforms with different endianness. To avoid byte order discrepancies, + * all the structures in this header follow little-endian format. + * + * This is an interface definition file where existing enums and their values + * must remain unchanged over time, so we specify explicit values for all enums. + */ + +/* This macro is used to generate compilation errors if a structure + * is not exactly the correct length. + */ +#define VIRTCHNL2_CHECK_STRUCT_LEN(n, X) \ + static_assert((n) == sizeof(struct X)) + +/* New major set of opcodes introduced and so leaving room for + * old misc opcodes to be added in future. Also these opcodes may only + * be used if both the PF and VF have successfully negotiated the + * VIRTCHNL version as 2.0 during VIRTCHNL2_OP_VERSION exchange. + */ +enum virtchnl2_op { + VIRTCHNL2_OP_UNKNOWN = 0, + VIRTCHNL2_OP_VERSION = 1, + VIRTCHNL2_OP_GET_CAPS = 500, + VIRTCHNL2_OP_CREATE_VPORT = 501, + VIRTCHNL2_OP_DESTROY_VPORT = 502, + VIRTCHNL2_OP_ENABLE_VPORT = 503, + VIRTCHNL2_OP_DISABLE_VPORT = 504, + VIRTCHNL2_OP_CONFIG_TX_QUEUES = 505, + VIRTCHNL2_OP_CONFIG_RX_QUEUES = 506, + VIRTCHNL2_OP_ENABLE_QUEUES = 507, + VIRTCHNL2_OP_DISABLE_QUEUES = 508, + VIRTCHNL2_OP_ADD_QUEUES = 509, + VIRTCHNL2_OP_DEL_QUEUES = 510, + VIRTCHNL2_OP_MAP_QUEUE_VECTOR = 511, + VIRTCHNL2_OP_UNMAP_QUEUE_VECTOR = 512, + VIRTCHNL2_OP_GET_RSS_KEY = 513, + VIRTCHNL2_OP_SET_RSS_KEY = 514, + VIRTCHNL2_OP_GET_RSS_LUT = 515, + VIRTCHNL2_OP_SET_RSS_LUT = 516, + VIRTCHNL2_OP_GET_RSS_HASH = 517, + VIRTCHNL2_OP_SET_RSS_HASH = 518, + VIRTCHNL2_OP_SET_SRIOV_VFS = 519, + VIRTCHNL2_OP_ALLOC_VECTORS = 520, + VIRTCHNL2_OP_DEALLOC_VECTORS = 521, + VIRTCHNL2_OP_EVENT = 522, + VIRTCHNL2_OP_GET_STATS = 523, + VIRTCHNL2_OP_RESET_VF = 524, + VIRTCHNL2_OP_GET_EDT_CAPS = 525, + VIRTCHNL2_OP_GET_PTYPE_INFO = 526, + /* Opcode 527 and 528 are reserved for VIRTCHNL2_OP_GET_PTYPE_ID and + * VIRTCHNL2_OP_GET_PTYPE_INFO_RAW. + */ + VIRTCHNL2_OP_RDMA = 529, + /* Opcodes 530 through 533 are reserved. */ + VIRTCHNL2_OP_LOOPBACK = 534, + VIRTCHNL2_OP_ADD_MAC_ADDR = 535, + VIRTCHNL2_OP_DEL_MAC_ADDR = 536, + VIRTCHNL2_OP_CONFIG_PROMISCUOUS_MODE = 537, + + /* TimeSync opcodes */ + VIRTCHNL2_OP_PTP_GET_CAPS = 541, + VIRTCHNL2_OP_PTP_GET_VPORT_TX_TSTAMP = 542, + VIRTCHNL2_OP_PTP_GET_DEV_CLK_TIME = 543, + VIRTCHNL2_OP_PTP_GET_CROSS_TIME = 544, + VIRTCHNL2_OP_PTP_SET_DEV_CLK_TIME = 545, + VIRTCHNL2_OP_PTP_ADJ_DEV_CLK_FINE = 546, + VIRTCHNL2_OP_PTP_ADJ_DEV_CLK_TIME = 547, + VIRTCHNL2_OP_PTP_GET_VPORT_TX_TSTAMP_CAPS = 548, + VIRTCHNL2_OP_GET_LAN_MEMORY_REGIONS = 549, + /* Opcode 550 is reserved */ + VIRTCHNL2_OP_ADD_FLOW_RULE = 551, + VIRTCHNL2_OP_GET_FLOW_RULE = 552, + VIRTCHNL2_OP_DEL_FLOW_RULE = 553, +}; + +/** + * enum virtchnl2_vport_type - Type of virtual port. + * @VIRTCHNL2_VPORT_TYPE_DEFAULT: Default virtual port type. + */ +enum virtchnl2_vport_type { + VIRTCHNL2_VPORT_TYPE_DEFAULT = 0, +}; + +/** + * enum virtchnl2_queue_model - Type of queue model. + * @VIRTCHNL2_QUEUE_MODEL_SINGLE: Single queue model. + * @VIRTCHNL2_QUEUE_MODEL_SPLIT: Split queue model. + * + * In the single queue model, the same transmit descriptor queue is used by + * software to post descriptors to hardware and by hardware to post completed + * descriptors to software. + * Likewise, the same receive descriptor queue is used by hardware to post + * completions to software and by software to post buffers to hardware. + * + * In the split queue model, hardware uses transmit completion queues to post + * descriptor/buffer completions to software, while software uses transmit + * descriptor queues to post descriptors to hardware. + * Likewise, hardware posts descriptor completions to the receive descriptor + * queue, while software uses receive buffer queues to post buffers to hardware. + */ +enum virtchnl2_queue_model { + VIRTCHNL2_QUEUE_MODEL_SINGLE = 0, + VIRTCHNL2_QUEUE_MODEL_SPLIT = 1, +}; + +/* Checksum offload capability flags */ +enum virtchnl2_cap_txrx_csum { + VIRTCHNL2_CAP_TX_CSUM_L3_IPV4 = BIT(0), + VIRTCHNL2_CAP_TX_CSUM_L4_IPV4_TCP = BIT(1), + VIRTCHNL2_CAP_TX_CSUM_L4_IPV4_UDP = BIT(2), + VIRTCHNL2_CAP_TX_CSUM_L4_IPV4_SCTP = BIT(3), + VIRTCHNL2_CAP_TX_CSUM_L4_IPV6_TCP = BIT(4), + VIRTCHNL2_CAP_TX_CSUM_L4_IPV6_UDP = BIT(5), + VIRTCHNL2_CAP_TX_CSUM_L4_IPV6_SCTP = BIT(6), + VIRTCHNL2_CAP_TX_CSUM_GENERIC = BIT(7), + VIRTCHNL2_CAP_RX_CSUM_L3_IPV4 = BIT(8), + VIRTCHNL2_CAP_RX_CSUM_L4_IPV4_TCP = BIT(9), + VIRTCHNL2_CAP_RX_CSUM_L4_IPV4_UDP = BIT(10), + VIRTCHNL2_CAP_RX_CSUM_L4_IPV4_SCTP = BIT(11), + VIRTCHNL2_CAP_RX_CSUM_L4_IPV6_TCP = BIT(12), + VIRTCHNL2_CAP_RX_CSUM_L4_IPV6_UDP = BIT(13), + VIRTCHNL2_CAP_RX_CSUM_L4_IPV6_SCTP = BIT(14), + VIRTCHNL2_CAP_RX_CSUM_GENERIC = BIT(15), + VIRTCHNL2_CAP_TX_CSUM_L3_SINGLE_TUNNEL = BIT(16), + VIRTCHNL2_CAP_TX_CSUM_L3_DOUBLE_TUNNEL = BIT(17), + VIRTCHNL2_CAP_RX_CSUM_L3_SINGLE_TUNNEL = BIT(18), + VIRTCHNL2_CAP_RX_CSUM_L3_DOUBLE_TUNNEL = BIT(19), + VIRTCHNL2_CAP_TX_CSUM_L4_SINGLE_TUNNEL = BIT(20), + VIRTCHNL2_CAP_TX_CSUM_L4_DOUBLE_TUNNEL = BIT(21), + VIRTCHNL2_CAP_RX_CSUM_L4_SINGLE_TUNNEL = BIT(22), + VIRTCHNL2_CAP_RX_CSUM_L4_DOUBLE_TUNNEL = BIT(23), +}; + +/* Segmentation offload capability flags */ +enum virtchnl2_cap_seg { + VIRTCHNL2_CAP_SEG_IPV4_TCP = BIT(0), + VIRTCHNL2_CAP_SEG_IPV4_UDP = BIT(1), + VIRTCHNL2_CAP_SEG_IPV4_SCTP = BIT(2), + VIRTCHNL2_CAP_SEG_IPV6_TCP = BIT(3), + VIRTCHNL2_CAP_SEG_IPV6_UDP = BIT(4), + VIRTCHNL2_CAP_SEG_IPV6_SCTP = BIT(5), + VIRTCHNL2_CAP_SEG_GENERIC = BIT(6), + VIRTCHNL2_CAP_SEG_TX_SINGLE_TUNNEL = BIT(7), + VIRTCHNL2_CAP_SEG_TX_DOUBLE_TUNNEL = BIT(8), +}; + +/* Receive Side Scaling and Flow Steering Flow type capability flags */ +enum virtchnl2_flow_types { + VIRTCHNL2_FLOW_IPV4_TCP = BIT(0), + VIRTCHNL2_FLOW_IPV4_UDP = BIT(1), + VIRTCHNL2_FLOW_IPV4_SCTP = BIT(2), + VIRTCHNL2_FLOW_IPV4_OTHER = BIT(3), + VIRTCHNL2_FLOW_IPV6_TCP = BIT(4), + VIRTCHNL2_FLOW_IPV6_UDP = BIT(5), + VIRTCHNL2_FLOW_IPV6_SCTP = BIT(6), + VIRTCHNL2_FLOW_IPV6_OTHER = BIT(7), + VIRTCHNL2_FLOW_IPV4_AH = BIT(8), + VIRTCHNL2_FLOW_IPV4_ESP = BIT(9), + VIRTCHNL2_FLOW_IPV4_AH_ESP = BIT(10), + VIRTCHNL2_FLOW_IPV6_AH = BIT(11), + VIRTCHNL2_FLOW_IPV6_ESP = BIT(12), + VIRTCHNL2_FLOW_IPV6_AH_ESP = BIT(13), +}; + +/* Header split capability flags */ +enum virtchnl2_cap_rx_hsplit_at { + /* for prepended metadata */ + VIRTCHNL2_CAP_RX_HSPLIT_AT_L2 = BIT(0), + /* all VLANs go into header buffer */ + VIRTCHNL2_CAP_RX_HSPLIT_AT_L3 = BIT(1), + VIRTCHNL2_CAP_RX_HSPLIT_AT_L4V4 = BIT(2), + VIRTCHNL2_CAP_RX_HSPLIT_AT_L4V6 = BIT(3), +}; + +/* Receive Side Coalescing offload capability flags */ +enum virtchnl2_cap_rsc { + VIRTCHNL2_CAP_RSC_IPV4_TCP = BIT(0), + VIRTCHNL2_CAP_RSC_IPV4_SCTP = BIT(1), + VIRTCHNL2_CAP_RSC_IPV6_TCP = BIT(2), + VIRTCHNL2_CAP_RSC_IPV6_SCTP = BIT(3), +}; + +/* Other capability flags */ +enum virtchnl2_cap_other { + VIRTCHNL2_CAP_RDMA = BIT_ULL(0), + VIRTCHNL2_CAP_SRIOV = BIT_ULL(1), + VIRTCHNL2_CAP_MACFILTER = BIT_ULL(2), + /* Other capability 3 is available + * Queue based scheduling using split queue model + */ + VIRTCHNL2_CAP_SPLITQ_QSCHED = BIT_ULL(4), + VIRTCHNL2_CAP_CRC = BIT_ULL(5), + VIRTCHNL2_CAP_ADQ = BIT_ULL(6), + VIRTCHNL2_CAP_WB_ON_ITR = BIT_ULL(7), + VIRTCHNL2_CAP_PROMISC = BIT_ULL(8), + VIRTCHNL2_CAP_LINK_SPEED = BIT_ULL(9), + VIRTCHNL2_CAP_INLINE_IPSEC = BIT_ULL(10), + VIRTCHNL2_CAP_LARGE_NUM_QUEUES = BIT_ULL(11), + VIRTCHNL2_CAP_VLAN = BIT_ULL(12), + VIRTCHNL2_CAP_PTP = BIT_ULL(13), + /* EDT: Earliest Departure Time capability used for Timing Wheel */ + VIRTCHNL2_CAP_EDT = BIT_ULL(14), + VIRTCHNL2_CAP_ADV_RSS = BIT_ULL(15), + /* Other capability 16 is available */ + VIRTCHNL2_CAP_RX_FLEX_DESC = BIT_ULL(17), + VIRTCHNL2_CAP_PTYPE = BIT_ULL(18), + VIRTCHNL2_CAP_LOOPBACK = BIT_ULL(19), + /* Other capability 20 is reserved */ + VIRTCHNL2_CAP_FLOW_STEER = BIT_ULL(21), + VIRTCHNL2_CAP_LAN_MEMORY_REGIONS = BIT_ULL(22), + + /* this must be the last capability */ + VIRTCHNL2_CAP_OEM = BIT_ULL(63), +}; + +/** + * enum virtchnl2_action_types - Available actions for sideband flow steering + * @VIRTCHNL2_ACTION_DROP: Drop the packet + * @VIRTCHNL2_ACTION_PASSTHRU: Forward the packet to the next classifier/stage + * @VIRTCHNL2_ACTION_QUEUE: Forward the packet to a receive queue + * @VIRTCHNL2_ACTION_Q_GROUP: Forward the packet to a receive queue group + * @VIRTCHNL2_ACTION_MARK: Mark the packet with specific marker value + * @VIRTCHNL2_ACTION_COUNT: Increment the corresponding counter + */ + +enum virtchnl2_action_types { + VIRTCHNL2_ACTION_DROP = BIT(0), + VIRTCHNL2_ACTION_PASSTHRU = BIT(1), + VIRTCHNL2_ACTION_QUEUE = BIT(2), + VIRTCHNL2_ACTION_Q_GROUP = BIT(3), + VIRTCHNL2_ACTION_MARK = BIT(4), + VIRTCHNL2_ACTION_COUNT = BIT(5), +}; + +/* underlying device type */ +enum virtchl2_device_type { + VIRTCHNL2_MEV_DEVICE = 0, +}; + +/** + * enum virtchnl2_txq_sched_mode - Transmit Queue Scheduling Modes. + * @VIRTCHNL2_TXQ_SCHED_MODE_QUEUE: Queue mode is the legacy mode i.e. inorder + * completions where descriptors and buffers + * are completed at the same time. + * @VIRTCHNL2_TXQ_SCHED_MODE_FLOW: Flow scheduling mode allows for out of order + * packet processing where descriptors are + * cleaned in order, but buffers can be + * completed out of order. + */ +enum virtchnl2_txq_sched_mode { + VIRTCHNL2_TXQ_SCHED_MODE_QUEUE = 0, + VIRTCHNL2_TXQ_SCHED_MODE_FLOW = 1, +}; + +/** + * enum virtchnl2_rxq_flags - Receive Queue Feature flags. + * @VIRTCHNL2_RXQ_RSC: Rx queue RSC flag. + * @VIRTCHNL2_RXQ_HDR_SPLIT: Rx queue header split flag. + * @VIRTCHNL2_RXQ_IMMEDIATE_WRITE_BACK: When set, packet descriptors are flushed + * by hardware immediately after processing + * each packet. + * @VIRTCHNL2_RX_DESC_SIZE_16BYTE: Rx queue 16 byte descriptor size. + * @VIRTCHNL2_RX_DESC_SIZE_32BYTE: Rx queue 32 byte descriptor size. + */ +enum virtchnl2_rxq_flags { + VIRTCHNL2_RXQ_RSC = BIT(0), + VIRTCHNL2_RXQ_HDR_SPLIT = BIT(1), + VIRTCHNL2_RXQ_IMMEDIATE_WRITE_BACK = BIT(2), + VIRTCHNL2_RX_DESC_SIZE_16BYTE = BIT(3), + VIRTCHNL2_RX_DESC_SIZE_32BYTE = BIT(4), +}; + +/* Type of RSS algorithm */ +enum virtchnl2_rss_alg { + VIRTCHNL2_RSS_ALG_TOEPLITZ_ASYMMETRIC = 0, + VIRTCHNL2_RSS_ALG_R_ASYMMETRIC = 1, + VIRTCHNL2_RSS_ALG_TOEPLITZ_SYMMETRIC = 2, + VIRTCHNL2_RSS_ALG_XOR_SYMMETRIC = 3, +}; + +/* Type of event */ +enum virtchnl2_event_codes { + VIRTCHNL2_EVENT_UNKNOWN = 0, + VIRTCHNL2_EVENT_LINK_CHANGE = 1, + /* Event type 2, 3 are reserved */ +}; + +/* Transmit and Receive queue types are valid in legacy as well as split queue + * models. With Split Queue model, 2 additional types are introduced - + * TX_COMPLETION and RX_BUFFER. In split queue model, receive corresponds to + * the queue where hardware posts completions. + */ +enum virtchnl2_queue_type { + VIRTCHNL2_QUEUE_TYPE_TX = 0, + VIRTCHNL2_QUEUE_TYPE_RX = 1, + VIRTCHNL2_QUEUE_TYPE_TX_COMPLETION = 2, + VIRTCHNL2_QUEUE_TYPE_RX_BUFFER = 3, + VIRTCHNL2_QUEUE_TYPE_CONFIG_TX = 4, + VIRTCHNL2_QUEUE_TYPE_CONFIG_RX = 5, + /* Queue types 6, 7, 8, 9 are reserved */ + VIRTCHNL2_QUEUE_TYPE_MBX_TX = 10, + VIRTCHNL2_QUEUE_TYPE_MBX_RX = 11, +}; + +/* Interrupt throttling rate index */ +enum virtchnl2_itr_idx { + VIRTCHNL2_ITR_IDX_0 = 0, + VIRTCHNL2_ITR_IDX_1 = 1, +}; + +/** + * enum virtchnl2_mac_addr_type - MAC address types. + * @VIRTCHNL2_MAC_ADDR_PRIMARY: PF/VF driver should set this type for the + * primary/device unicast MAC address filter for + * VIRTCHNL2_OP_ADD_MAC_ADDR and + * VIRTCHNL2_OP_DEL_MAC_ADDR. This allows for the + * underlying control plane function to accurately + * track the MAC address and for VM/function reset. + * + * @VIRTCHNL2_MAC_ADDR_EXTRA: PF/VF driver should set this type for any extra + * unicast and/or multicast filters that are being + * added/deleted via VIRTCHNL2_OP_ADD_MAC_ADDR or + * VIRTCHNL2_OP_DEL_MAC_ADDR. + */ +enum virtchnl2_mac_addr_type { + VIRTCHNL2_MAC_ADDR_PRIMARY = 1, + VIRTCHNL2_MAC_ADDR_EXTRA = 2, +}; + +/* Flags used for promiscuous mode */ +enum virtchnl2_promisc_flags { + VIRTCHNL2_UNICAST_PROMISC = BIT(0), + VIRTCHNL2_MULTICAST_PROMISC = BIT(1), +}; + +/* Protocol header type within a packet segment. A segment consists of one or + * more protocol headers that make up a logical group of protocol headers. Each + * logical group of protocol headers encapsulates or is encapsulated using/by + * tunneling or encapsulation protocols for network virtualization. + */ +enum virtchnl2_proto_hdr_type { + /* VIRTCHNL2_PROTO_HDR_ANY is a mandatory protocol id */ + VIRTCHNL2_PROTO_HDR_ANY = 0, + VIRTCHNL2_PROTO_HDR_PRE_MAC = 1, + /* VIRTCHNL2_PROTO_HDR_MAC is a mandatory protocol id */ + VIRTCHNL2_PROTO_HDR_MAC = 2, + VIRTCHNL2_PROTO_HDR_POST_MAC = 3, + VIRTCHNL2_PROTO_HDR_ETHERTYPE = 4, + VIRTCHNL2_PROTO_HDR_VLAN = 5, + VIRTCHNL2_PROTO_HDR_SVLAN = 6, + VIRTCHNL2_PROTO_HDR_CVLAN = 7, + VIRTCHNL2_PROTO_HDR_MPLS = 8, + VIRTCHNL2_PROTO_HDR_UMPLS = 9, + VIRTCHNL2_PROTO_HDR_MMPLS = 10, + VIRTCHNL2_PROTO_HDR_PTP = 11, + VIRTCHNL2_PROTO_HDR_CTRL = 12, + VIRTCHNL2_PROTO_HDR_LLDP = 13, + VIRTCHNL2_PROTO_HDR_ARP = 14, + VIRTCHNL2_PROTO_HDR_ECP = 15, + VIRTCHNL2_PROTO_HDR_EAPOL = 16, + VIRTCHNL2_PROTO_HDR_PPPOD = 17, + VIRTCHNL2_PROTO_HDR_PPPOE = 18, + /* VIRTCHNL2_PROTO_HDR_IPV4 is a mandatory protocol id */ + VIRTCHNL2_PROTO_HDR_IPV4 = 19, + /* IPv4 and IPv6 Fragment header types are only associated to + * VIRTCHNL2_PROTO_HDR_IPV4 and VIRTCHNL2_PROTO_HDR_IPV6 respectively, + * cannot be used independently. + */ + /* VIRTCHNL2_PROTO_HDR_IPV4_FRAG is a mandatory protocol id */ + VIRTCHNL2_PROTO_HDR_IPV4_FRAG = 20, + /* VIRTCHNL2_PROTO_HDR_IPV6 is a mandatory protocol id */ + VIRTCHNL2_PROTO_HDR_IPV6 = 21, + /* VIRTCHNL2_PROTO_HDR_IPV6_FRAG is a mandatory protocol id */ + VIRTCHNL2_PROTO_HDR_IPV6_FRAG = 22, + VIRTCHNL2_PROTO_HDR_IPV6_EH = 23, + /* VIRTCHNL2_PROTO_HDR_UDP is a mandatory protocol id */ + VIRTCHNL2_PROTO_HDR_UDP = 24, + /* VIRTCHNL2_PROTO_HDR_TCP is a mandatory protocol id */ + VIRTCHNL2_PROTO_HDR_TCP = 25, + /* VIRTCHNL2_PROTO_HDR_SCTP is a mandatory protocol id */ + VIRTCHNL2_PROTO_HDR_SCTP = 26, + /* VIRTCHNL2_PROTO_HDR_ICMP is a mandatory protocol id */ + VIRTCHNL2_PROTO_HDR_ICMP = 27, + /* VIRTCHNL2_PROTO_HDR_ICMPV6 is a mandatory protocol id */ + VIRTCHNL2_PROTO_HDR_ICMPV6 = 28, + VIRTCHNL2_PROTO_HDR_IGMP = 29, + VIRTCHNL2_PROTO_HDR_AH = 30, + VIRTCHNL2_PROTO_HDR_ESP = 31, + VIRTCHNL2_PROTO_HDR_IKE = 32, + VIRTCHNL2_PROTO_HDR_NATT_KEEP = 33, + /* VIRTCHNL2_PROTO_HDR_PAY is a mandatory protocol id */ + VIRTCHNL2_PROTO_HDR_PAY = 34, + VIRTCHNL2_PROTO_HDR_L2TPV2 = 35, + VIRTCHNL2_PROTO_HDR_L2TPV2_CONTROL = 36, + VIRTCHNL2_PROTO_HDR_L2TPV3 = 37, + VIRTCHNL2_PROTO_HDR_GTP = 38, + VIRTCHNL2_PROTO_HDR_GTP_EH = 39, + VIRTCHNL2_PROTO_HDR_GTPCV2 = 40, + VIRTCHNL2_PROTO_HDR_GTPC_TEID = 41, + VIRTCHNL2_PROTO_HDR_GTPU = 42, + VIRTCHNL2_PROTO_HDR_GTPU_UL = 43, + VIRTCHNL2_PROTO_HDR_GTPU_DL = 44, + VIRTCHNL2_PROTO_HDR_ECPRI = 45, + VIRTCHNL2_PROTO_HDR_VRRP = 46, + VIRTCHNL2_PROTO_HDR_OSPF = 47, + /* VIRTCHNL2_PROTO_HDR_TUN is a mandatory protocol id */ + VIRTCHNL2_PROTO_HDR_TUN = 48, + VIRTCHNL2_PROTO_HDR_GRE = 49, + VIRTCHNL2_PROTO_HDR_NVGRE = 50, + VIRTCHNL2_PROTO_HDR_VXLAN = 51, + VIRTCHNL2_PROTO_HDR_VXLAN_GPE = 52, + VIRTCHNL2_PROTO_HDR_GENEVE = 53, + VIRTCHNL2_PROTO_HDR_NSH = 54, + VIRTCHNL2_PROTO_HDR_QUIC = 55, + VIRTCHNL2_PROTO_HDR_PFCP = 56, + VIRTCHNL2_PROTO_HDR_PFCP_NODE = 57, + VIRTCHNL2_PROTO_HDR_PFCP_SESSION = 58, + VIRTCHNL2_PROTO_HDR_RTP = 59, + VIRTCHNL2_PROTO_HDR_ROCE = 60, + VIRTCHNL2_PROTO_HDR_ROCEV1 = 61, + VIRTCHNL2_PROTO_HDR_ROCEV2 = 62, + /* Protocol ids up to 32767 are reserved. + * 32768 - 65534 are used for user defined protocol ids. + * VIRTCHNL2_PROTO_HDR_NO_PROTO is a mandatory protocol id. + */ + VIRTCHNL2_PROTO_HDR_NO_PROTO = 65535, +}; + +enum virtchl2_version { + VIRTCHNL2_VERSION_MINOR_0 = 0, + VIRTCHNL2_VERSION_MAJOR_2 = 2, +}; + +/** + * struct virtchnl2_edt_caps - Get EDT granularity and time horizon. + * @tstamp_granularity_ns: Timestamp granularity in nanoseconds. + * @time_horizon_ns: Total time window in nanoseconds. + * + * Associated with VIRTCHNL2_OP_GET_EDT_CAPS. + */ +struct virtchnl2_edt_caps { + __le64 tstamp_granularity_ns; + __le64 time_horizon_ns; +}; +VIRTCHNL2_CHECK_STRUCT_LEN(16, virtchnl2_edt_caps); + +/** + * struct virtchnl2_version_info - Version information. + * @major: Major version. + * @minor: Minor version. + * + * PF/VF posts its version number to the CP. CP responds with its version number + * in the same format, along with a return code. + * If there is a major version mismatch, then the PF/VF cannot operate. + * If there is a minor version mismatch, then the PF/VF can operate but should + * add a warning to the system log. + * + * This version opcode MUST always be specified as == 1, regardless of other + * changes in the API. The CP must always respond to this message without + * error regardless of version mismatch. + * + * Associated with VIRTCHNL2_OP_VERSION. + */ +struct virtchnl2_version_info { + __le32 major; + __le32 minor; +}; +VIRTCHNL2_CHECK_STRUCT_LEN(8, virtchnl2_version_info); + +/** + * struct virtchnl2_get_capabilities - Capabilities info. + * @csum_caps: See enum virtchnl2_cap_txrx_csum. + * @seg_caps: See enum virtchnl2_cap_seg. + * @hsplit_caps: See enum virtchnl2_cap_rx_hsplit_at. + * @rsc_caps: See enum virtchnl2_cap_rsc. + * @rss_caps: See enum virtchnl2_flow_types. + * @other_caps: See enum virtchnl2_cap_other. + * @mailbox_dyn_ctl: DYN_CTL register offset and vector id for mailbox + * provided by CP. + * @mailbox_vector_id: Mailbox vector id. + * @num_allocated_vectors: Maximum number of allocated vectors for the device. + * @max_rx_q: Maximum number of supported Rx queues. + * @max_tx_q: Maximum number of supported Tx queues. + * @max_rx_bufq: Maximum number of supported buffer queues. + * @max_tx_complq: Maximum number of supported completion queues. + * @max_sriov_vfs: The PF sends the maximum VFs it is requesting. The CP + * responds with the maximum VFs granted. + * @max_vports: Maximum number of vports that can be supported. + * @default_num_vports: Default number of vports driver should allocate on load. + * @max_tx_hdr_size: Max header length hardware can parse/checksum, in bytes. + * @max_sg_bufs_per_tx_pkt: Max number of scatter gather buffers that can be + * sent per transmit packet without needing to be + * linearized. + * @pad: Padding. + * @reserved: Reserved. + * @device_type: See enum virtchl2_device_type. + * @min_sso_packet_len: Min packet length supported by device for single + * segment offload. + * @max_hdr_buf_per_lso: Max number of header buffers that can be used for + * an LSO. + * @num_rdma_allocated_vectors: Maximum number of allocated RDMA vectors for + * the device. + * @pad1: Padding for future extensions. + * + * Dataplane driver sends this message to CP to negotiate capabilities and + * provides a virtchnl2_get_capabilities structure with its desired + * capabilities, max_sriov_vfs and num_allocated_vectors. + * CP responds with a virtchnl2_get_capabilities structure updated + * with allowed capabilities and the other fields as below. + * If PF sets max_sriov_vfs as 0, CP will respond with max number of VFs + * that can be created by this PF. For any other value 'n', CP responds + * with max_sriov_vfs set to min(n, x) where x is the max number of VFs + * allowed by CP's policy. max_sriov_vfs is not applicable for VFs. + * If dataplane driver sets num_allocated_vectors as 0, CP will respond with 1 + * which is default vector associated with the default mailbox. For any other + * value 'n', CP responds with a value <= n based on the CP's policy of + * max number of vectors for a PF. + * CP will respond with the vector ID of mailbox allocated to the PF in + * mailbox_vector_id and the number of itr index registers in itr_idx_map. + * It also responds with default number of vports that the dataplane driver + * should comeup with in default_num_vports and maximum number of vports that + * can be supported in max_vports. + * + * Associated with VIRTCHNL2_OP_GET_CAPS. + */ +struct virtchnl2_get_capabilities { + __le32 csum_caps; + __le32 seg_caps; + __le32 hsplit_caps; + __le32 rsc_caps; + __le64 rss_caps; + __le64 other_caps; + __le32 mailbox_dyn_ctl; + __le16 mailbox_vector_id; + __le16 num_allocated_vectors; + __le16 max_rx_q; + __le16 max_tx_q; + __le16 max_rx_bufq; + __le16 max_tx_complq; + __le16 max_sriov_vfs; + __le16 max_vports; + __le16 default_num_vports; + __le16 max_tx_hdr_size; + u8 max_sg_bufs_per_tx_pkt; + u8 pad[3]; + u8 reserved[4]; + __le32 device_type; + u8 min_sso_packet_len; + u8 max_hdr_buf_per_lso; + __le16 num_rdma_allocated_vectors; + u8 pad1[8]; +}; +VIRTCHNL2_CHECK_STRUCT_LEN(80, virtchnl2_get_capabilities); + +/** + * struct virtchnl2_queue_reg_chunk - Single queue chunk. + * @type: See enum virtchnl2_queue_type. + * @start_queue_id: Start Queue ID. + * @num_queues: Number of queues in the chunk. + * @pad: Padding. + * @qtail_reg_start: Queue tail register offset. + * @qtail_reg_spacing: Queue tail register spacing. + * @pad1: Padding for future extensions. + */ +struct virtchnl2_queue_reg_chunk { + __le32 type; + __le32 start_queue_id; + __le32 num_queues; + __le32 pad; + __le64 qtail_reg_start; + __le32 qtail_reg_spacing; + u8 pad1[4]; +}; +VIRTCHNL2_CHECK_STRUCT_LEN(32, virtchnl2_queue_reg_chunk); + +/** + * struct virtchnl2_queue_reg_chunks - Specify several chunks of contiguous + * queues. + * @num_chunks: Number of chunks. + * @pad: Padding. + * @chunks: Chunks of queue info. + */ +struct virtchnl2_queue_reg_chunks { + __le16 num_chunks; + u8 pad[6]; + struct virtchnl2_queue_reg_chunk chunks[] __counted_by_le(num_chunks); +}; +VIRTCHNL2_CHECK_STRUCT_LEN(8, virtchnl2_queue_reg_chunks); + +/** + * enum virtchnl2_vport_flags - Vport flags that indicate vport capabilities. + * @VIRTCHNL2_VPORT_UPLINK_PORT: Representatives of underlying physical ports + * @VIRTCHNL2_VPORT_INLINE_FLOW_STEER: Inline flow steering enabled + * @VIRTCHNL2_VPORT_INLINE_FLOW_STEER_RXQ: Inline flow steering enabled + * with explicit Rx queue action + * @VIRTCHNL2_VPORT_SIDEBAND_FLOW_STEER: Sideband flow steering enabled + * @VIRTCHNL2_VPORT_ENABLE_RDMA: RDMA is enabled for this vport + */ +enum virtchnl2_vport_flags { + VIRTCHNL2_VPORT_UPLINK_PORT = BIT(0), + VIRTCHNL2_VPORT_INLINE_FLOW_STEER = BIT(1), + VIRTCHNL2_VPORT_INLINE_FLOW_STEER_RXQ = BIT(2), + VIRTCHNL2_VPORT_SIDEBAND_FLOW_STEER = BIT(3), + VIRTCHNL2_VPORT_ENABLE_RDMA = BIT(4), +}; + +/** + * struct virtchnl2_create_vport - Create vport config info. + * @vport_type: See enum virtchnl2_vport_type. + * @txq_model: See virtchnl2_queue_model. + * @rxq_model: See virtchnl2_queue_model. + * @num_tx_q: Number of Tx queues. + * @num_tx_complq: Valid only if txq_model is split queue. + * @num_rx_q: Number of Rx queues. + * @num_rx_bufq: Valid only if rxq_model is split queue. + * @default_rx_q: Relative receive queue index to be used as default. + * @vport_index: Used to align PF and CP in case of default multiple vports, + * it is filled by the PF and CP returns the same value, to + * enable the driver to support multiple asynchronous parallel + * CREATE_VPORT requests and associate a response to a specific + * request. + * @max_mtu: Max MTU. CP populates this field on response. + * @vport_id: Vport id. CP populates this field on response. + * @default_mac_addr: Default MAC address. + * @vport_flags: See enum virtchnl2_vport_flags. + * @rx_desc_ids: See VIRTCHNL2_RX_DESC_IDS definitions. + * @tx_desc_ids: See VIRTCHNL2_TX_DESC_IDS definitions. + * @pad1: Padding. + * @inline_flow_caps: Bit mask of supported inline-flow-steering + * flow types (See enum virtchnl2_flow_types) + * @sideband_flow_caps: Bit mask of supported sideband-flow-steering + * flow types (See enum virtchnl2_flow_types) + * @sideband_flow_actions: Bit mask of supported action types + * for sideband flow steering (See enum virtchnl2_action_types) + * @flow_steer_max_rules: Max rules allowed for inline and sideband + * flow steering combined + * @rss_algorithm: RSS algorithm. + * @rss_key_size: RSS key size. + * @rss_lut_size: RSS LUT size. + * @rx_split_pos: See enum virtchnl2_cap_rx_hsplit_at. + * @pad2: Padding. + * @chunks: Chunks of contiguous queues. + * + * PF sends this message to CP to create a vport by filling in required + * fields of virtchnl2_create_vport structure. + * CP responds with the updated virtchnl2_create_vport structure containing the + * necessary fields followed by chunks which in turn will have an array of + * num_chunks entries of virtchnl2_queue_chunk structures. + * + * Associated with VIRTCHNL2_OP_CREATE_VPORT. + */ +struct virtchnl2_create_vport { + __le16 vport_type; + __le16 txq_model; + __le16 rxq_model; + __le16 num_tx_q; + __le16 num_tx_complq; + __le16 num_rx_q; + __le16 num_rx_bufq; + __le16 default_rx_q; + __le16 vport_index; + /* CP populates the following fields on response */ + __le16 max_mtu; + __le32 vport_id; + u8 default_mac_addr[ETH_ALEN]; + __le16 vport_flags; + __le64 rx_desc_ids; + __le64 tx_desc_ids; + u8 pad1[48]; + __le64 inline_flow_caps; + __le64 sideband_flow_caps; + __le32 sideband_flow_actions; + __le32 flow_steer_max_rules; + __le32 rss_algorithm; + __le16 rss_key_size; + __le16 rss_lut_size; + __le32 rx_split_pos; + u8 pad2[20]; + struct virtchnl2_queue_reg_chunks chunks; +}; +VIRTCHNL2_CHECK_STRUCT_LEN(160, virtchnl2_create_vport); + +/** + * struct virtchnl2_vport - Vport ID info. + * @vport_id: Vport id. + * @pad: Padding for future extensions. + * + * PF sends this message to CP to destroy, enable or disable a vport by filling + * in the vport_id in virtchnl2_vport structure. + * CP responds with the status of the requested operation. + * + * Associated with VIRTCHNL2_OP_DESTROY_VPORT, VIRTCHNL2_OP_ENABLE_VPORT, + * VIRTCHNL2_OP_DISABLE_VPORT. + */ +struct virtchnl2_vport { + __le32 vport_id; + u8 pad[4]; +}; +VIRTCHNL2_CHECK_STRUCT_LEN(8, virtchnl2_vport); + +/** + * struct virtchnl2_txq_info - Transmit queue config info + * @dma_ring_addr: DMA address. + * @type: See enum virtchnl2_queue_type. + * @queue_id: Queue ID. + * @relative_queue_id: Valid only if queue model is split and type is transmit + * queue. Used in many to one mapping of transmit queues to + * completion queue. + * @model: See enum virtchnl2_queue_model. + * @sched_mode: See enum virtchnl2_txq_sched_mode. + * @qflags: TX queue feature flags. + * @ring_len: Ring length. + * @tx_compl_queue_id: Valid only if queue model is split and type is transmit + * queue. + * @peer_type: Valid only if queue type is VIRTCHNL2_QUEUE_TYPE_MAILBOX_TX + * @peer_rx_queue_id: Valid only if queue type is CONFIG_TX and used to deliver + * messages for the respective CONFIG_TX queue. + * @pad: Padding. + * @egress_pasid: Egress PASID info. + * @egress_hdr_pasid: Egress HDR passid. + * @egress_buf_pasid: Egress buf passid. + * @pad1: Padding for future extensions. + */ +struct virtchnl2_txq_info { + __le64 dma_ring_addr; + __le32 type; + __le32 queue_id; + __le16 relative_queue_id; + __le16 model; + __le16 sched_mode; + __le16 qflags; + __le16 ring_len; + __le16 tx_compl_queue_id; + __le16 peer_type; + __le16 peer_rx_queue_id; + u8 pad[4]; + __le32 egress_pasid; + __le32 egress_hdr_pasid; + __le32 egress_buf_pasid; + u8 pad1[8]; +}; +VIRTCHNL2_CHECK_STRUCT_LEN(56, virtchnl2_txq_info); + +/** + * struct virtchnl2_config_tx_queues - TX queue config. + * @vport_id: Vport id. + * @num_qinfo: Number of virtchnl2_txq_info structs. + * @pad: Padding. + * @qinfo: Tx queues config info. + * + * PF sends this message to set up parameters for one or more transmit queues. + * This message contains an array of num_qinfo instances of virtchnl2_txq_info + * structures. CP configures requested queues and returns a status code. If + * num_qinfo specified is greater than the number of queues associated with the + * vport, an error is returned and no queues are configured. + * + * Associated with VIRTCHNL2_OP_CONFIG_TX_QUEUES. + */ +struct virtchnl2_config_tx_queues { + __le32 vport_id; + __le16 num_qinfo; + u8 pad[10]; + struct virtchnl2_txq_info qinfo[] __counted_by_le(num_qinfo); +}; +VIRTCHNL2_CHECK_STRUCT_LEN(16, virtchnl2_config_tx_queues); + +/** + * struct virtchnl2_rxq_info - Receive queue config info. + * @desc_ids: See VIRTCHNL2_RX_DESC_IDS definitions. + * @dma_ring_addr: See VIRTCHNL2_RX_DESC_IDS definitions. + * @type: See enum virtchnl2_queue_type. + * @queue_id: Queue id. + * @model: See enum virtchnl2_queue_model. + * @hdr_buffer_size: Header buffer size. + * @data_buffer_size: Data buffer size. + * @max_pkt_size: Max packet size. + * @ring_len: Ring length. + * @buffer_notif_stride: Buffer notification stride in units of 32-descriptors. + * This field must be a power of 2. + * @pad: Padding. + * @dma_head_wb_addr: Applicable only for receive buffer queues. + * @qflags: Applicable only for receive completion queues. + * See enum virtchnl2_rxq_flags. + * @rx_buffer_low_watermark: Rx buffer low watermark. + * @rx_bufq1_id: Buffer queue index of the first buffer queue associated with + * the Rx queue. Valid only in split queue model. + * @rx_bufq2_id: Buffer queue index of the second buffer queue associated with + * the Rx queue. Valid only in split queue model. + * @bufq2_ena: It indicates if there is a second buffer, rx_bufq2_id is valid + * only if this field is set. + * @pad1: Padding. + * @ingress_pasid: Ingress PASID. + * @ingress_hdr_pasid: Ingress PASID header. + * @ingress_buf_pasid: Ingress PASID buffer. + * @pad2: Padding for future extensions. + */ +struct virtchnl2_rxq_info { + __le64 desc_ids; + __le64 dma_ring_addr; + __le32 type; + __le32 queue_id; + __le16 model; + __le16 hdr_buffer_size; + __le32 data_buffer_size; + __le32 max_pkt_size; + __le16 ring_len; + u8 buffer_notif_stride; + u8 pad; + __le64 dma_head_wb_addr; + __le16 qflags; + __le16 rx_buffer_low_watermark; + __le16 rx_bufq1_id; + __le16 rx_bufq2_id; + u8 bufq2_ena; + u8 pad1[3]; + __le32 ingress_pasid; + __le32 ingress_hdr_pasid; + __le32 ingress_buf_pasid; + u8 pad2[16]; +}; +VIRTCHNL2_CHECK_STRUCT_LEN(88, virtchnl2_rxq_info); + +/** + * struct virtchnl2_config_rx_queues - Rx queues config. + * @vport_id: Vport id. + * @num_qinfo: Number of instances. + * @pad: Padding. + * @qinfo: Rx queues config info. + * + * PF sends this message to set up parameters for one or more receive queues. + * This message contains an array of num_qinfo instances of virtchnl2_rxq_info + * structures. CP configures requested queues and returns a status code. + * If the number of queues specified is greater than the number of queues + * associated with the vport, an error is returned and no queues are configured. + * + * Associated with VIRTCHNL2_OP_CONFIG_RX_QUEUES. + */ +struct virtchnl2_config_rx_queues { + __le32 vport_id; + __le16 num_qinfo; + u8 pad[18]; + struct virtchnl2_rxq_info qinfo[] __counted_by_le(num_qinfo); +}; +VIRTCHNL2_CHECK_STRUCT_LEN(24, virtchnl2_config_rx_queues); + +/** + * struct virtchnl2_add_queues - data for VIRTCHNL2_OP_ADD_QUEUES. + * @vport_id: Vport id. + * @num_tx_q: Number of Tx qieues. + * @num_tx_complq: Number of Tx completion queues. + * @num_rx_q: Number of Rx queues. + * @num_rx_bufq: Number of Rx buffer queues. + * @pad: Padding. + * @chunks: Chunks of contiguous queues. + * + * PF sends this message to request additional transmit/receive queues beyond + * the ones that were assigned via CREATE_VPORT request. virtchnl2_add_queues + * structure is used to specify the number of each type of queues. + * CP responds with the same structure with the actual number of queues assigned + * followed by num_chunks of virtchnl2_queue_chunk structures. + * + * Associated with VIRTCHNL2_OP_ADD_QUEUES. + */ +struct virtchnl2_add_queues { + __le32 vport_id; + __le16 num_tx_q; + __le16 num_tx_complq; + __le16 num_rx_q; + __le16 num_rx_bufq; + u8 pad[4]; + struct virtchnl2_queue_reg_chunks chunks; +}; +VIRTCHNL2_CHECK_STRUCT_LEN(24, virtchnl2_add_queues); + +/** + * struct virtchnl2_vector_chunk - Structure to specify a chunk of contiguous + * interrupt vectors. + * @start_vector_id: Start vector id. + * @start_evv_id: Start EVV id. + * @num_vectors: Number of vectors. + * @pad: Padding. + * @dynctl_reg_start: DYN_CTL register offset. + * @dynctl_reg_spacing: register spacing between DYN_CTL registers of 2 + * consecutive vectors. + * @itrn_reg_start: ITRN register offset. + * @itrn_reg_spacing: Register spacing between dynctl registers of 2 + * consecutive vectors. + * @itrn_index_spacing: Register spacing between itrn registers of the same + * vector where n=0..2. + * @pad1: Padding for future extensions. + * + * Register offsets and spacing provided by CP. + * Dynamic control registers are used for enabling/disabling/re-enabling + * interrupts and updating interrupt rates in the hotpath. Any changes + * to interrupt rates in the dynamic control registers will be reflected + * in the interrupt throttling rate registers. + * itrn registers are used to update interrupt rates for specific + * interrupt indices without modifying the state of the interrupt. + */ +struct virtchnl2_vector_chunk { + __le16 start_vector_id; + __le16 start_evv_id; + __le16 num_vectors; + __le16 pad; + __le32 dynctl_reg_start; + __le32 dynctl_reg_spacing; + __le32 itrn_reg_start; + __le32 itrn_reg_spacing; + __le32 itrn_index_spacing; + u8 pad1[4]; +}; +VIRTCHNL2_CHECK_STRUCT_LEN(32, virtchnl2_vector_chunk); + +/** + * struct virtchnl2_vector_chunks - chunks of contiguous interrupt vectors. + * @num_vchunks: number of vector chunks. + * @pad: Padding. + * @vchunks: Chunks of contiguous vector info. + * + * PF sends virtchnl2_vector_chunks struct to specify the vectors it is giving + * away. CP performs requested action and returns status. + * + * Associated with VIRTCHNL2_OP_DEALLOC_VECTORS. + */ +struct virtchnl2_vector_chunks { + __le16 num_vchunks; + u8 pad[14]; + struct virtchnl2_vector_chunk vchunks[] __counted_by_le(num_vchunks); +}; +VIRTCHNL2_CHECK_STRUCT_LEN(16, virtchnl2_vector_chunks); + +/** + * struct virtchnl2_alloc_vectors - vector allocation info. + * @num_vectors: Number of vectors. + * @pad: Padding. + * @vchunks: Chunks of contiguous vector info. + * + * PF sends this message to request additional interrupt vectors beyond the + * ones that were assigned via GET_CAPS request. virtchnl2_alloc_vectors + * structure is used to specify the number of vectors requested. CP responds + * with the same structure with the actual number of vectors assigned followed + * by virtchnl2_vector_chunks structure identifying the vector ids. + * + * Associated with VIRTCHNL2_OP_ALLOC_VECTORS. + */ +struct virtchnl2_alloc_vectors { + __le16 num_vectors; + u8 pad[14]; + struct virtchnl2_vector_chunks vchunks; +}; +VIRTCHNL2_CHECK_STRUCT_LEN(32, virtchnl2_alloc_vectors); + +/** + * struct virtchnl2_rss_lut - RSS LUT info. + * @vport_id: Vport id. + * @lut_entries_start: Start of LUT entries. + * @lut_entries: Number of LUT entrties. + * @pad: Padding. + * @lut: RSS lookup table. + * + * PF sends this message to get or set RSS lookup table. Only supported if + * both PF and CP drivers set the VIRTCHNL2_CAP_RSS bit during configuration + * negotiation. + * + * Associated with VIRTCHNL2_OP_GET_RSS_LUT and VIRTCHNL2_OP_SET_RSS_LUT. + */ +struct virtchnl2_rss_lut { + __le32 vport_id; + __le16 lut_entries_start; + __le16 lut_entries; + u8 pad[4]; + __le32 lut[] __counted_by_le(lut_entries); +}; +VIRTCHNL2_CHECK_STRUCT_LEN(12, virtchnl2_rss_lut); + +/** + * struct virtchnl2_rss_hash - RSS hash info. + * @ptype_groups: Packet type groups bitmap. + * @vport_id: Vport id. + * @pad: Padding for future extensions. + * + * PF sends these messages to get and set the hash filter enable bits for RSS. + * By default, the CP sets these to all possible traffic types that the + * hardware supports. The PF can query this value if it wants to change the + * traffic types that are hashed by the hardware. + * Only supported if both PF and CP drivers set the VIRTCHNL2_CAP_RSS bit + * during configuration negotiation. + * + * Associated with VIRTCHNL2_OP_GET_RSS_HASH and VIRTCHNL2_OP_SET_RSS_HASH + */ +struct virtchnl2_rss_hash { + __le64 ptype_groups; + __le32 vport_id; + u8 pad[4]; +}; +VIRTCHNL2_CHECK_STRUCT_LEN(16, virtchnl2_rss_hash); + +/** + * struct virtchnl2_sriov_vfs_info - VFs info. + * @num_vfs: Number of VFs. + * @pad: Padding for future extensions. + * + * This message is used to set number of SRIOV VFs to be created. The actual + * allocation of resources for the VFs in terms of vport, queues and interrupts + * is done by CP. When this call completes, the IDPF driver calls + * pci_enable_sriov to let the OS instantiate the SRIOV PCIE devices. + * The number of VFs set to 0 will destroy all the VFs of this function. + * + * Associated with VIRTCHNL2_OP_SET_SRIOV_VFS. + */ +struct virtchnl2_sriov_vfs_info { + __le16 num_vfs; + __le16 pad; +}; +VIRTCHNL2_CHECK_STRUCT_LEN(4, virtchnl2_sriov_vfs_info); + +/** + * struct virtchnl2_ptype - Packet type info. + * @ptype_id_10: 10-bit packet type. + * @ptype_id_8: 8-bit packet type. + * @proto_id_count: Number of protocol ids the packet supports, maximum of 32 + * protocol ids are supported. + * @pad: Padding. + * @proto_id: proto_id_count decides the allocation of protocol id array. + * See enum virtchnl2_proto_hdr_type. + * + * Based on the descriptor type the PF supports, CP fills ptype_id_10 or + * ptype_id_8 for flex and base descriptor respectively. If ptype_id_10 value + * is set to 0xFFFF, PF should consider this ptype as dummy one and it is the + * last ptype. + */ +struct virtchnl2_ptype { + __le16 ptype_id_10; + u8 ptype_id_8; + u8 proto_id_count; + __le16 pad; + __le16 proto_id[] __counted_by(proto_id_count); +} __packed __aligned(2); +VIRTCHNL2_CHECK_STRUCT_LEN(6, virtchnl2_ptype); + +/** + * struct virtchnl2_get_ptype_info - Packet type info. + * @start_ptype_id: Starting ptype ID. + * @num_ptypes: Number of packet types from start_ptype_id. + * @pad: Padding for future extensions. + * + * The total number of supported packet types is based on the descriptor type. + * For the flex descriptor, it is 1024 (10-bit ptype), and for the base + * descriptor, it is 256 (8-bit ptype). Send this message to the CP by + * populating the 'start_ptype_id' and the 'num_ptypes'. CP responds with the + * 'start_ptype_id', 'num_ptypes', and the array of ptype (virtchnl2_ptype) that + * are added at the end of the 'virtchnl2_get_ptype_info' message (Note: There + * is no specific field for the ptypes but are added at the end of the + * ptype info message. PF/VF is expected to extract the ptypes accordingly. + * Reason for doing this is because compiler doesn't allow nested flexible + * array fields). + * + * If all the ptypes don't fit into one mailbox buffer, CP splits the + * ptype info into multiple messages, where each message will have its own + * 'start_ptype_id', 'num_ptypes', and the ptype array itself. When CP is done + * updating all the ptype information extracted from the package (the number of + * ptypes extracted might be less than what PF/VF expects), it will append a + * dummy ptype (which has 'ptype_id_10' of 'struct virtchnl2_ptype' as 0xFFFF) + * to the ptype array. + * + * PF/VF is expected to receive multiple VIRTCHNL2_OP_GET_PTYPE_INFO messages. + * + * Associated with VIRTCHNL2_OP_GET_PTYPE_INFO. + */ +struct virtchnl2_get_ptype_info { + __le16 start_ptype_id; + __le16 num_ptypes; + __le32 pad; +}; +VIRTCHNL2_CHECK_STRUCT_LEN(8, virtchnl2_get_ptype_info); + +/** + * struct virtchnl2_vport_stats - Vport statistics. + * @vport_id: Vport id. + * @pad: Padding. + * @rx_bytes: Received bytes. + * @rx_unicast: Received unicast packets. + * @rx_multicast: Received multicast packets. + * @rx_broadcast: Received broadcast packets. + * @rx_discards: Discarded packets on receive. + * @rx_errors: Receive errors. + * @rx_unknown_protocol: Unlnown protocol. + * @tx_bytes: Transmitted bytes. + * @tx_unicast: Transmitted unicast packets. + * @tx_multicast: Transmitted multicast packets. + * @tx_broadcast: Transmitted broadcast packets. + * @tx_discards: Discarded packets on transmit. + * @tx_errors: Transmit errors. + * @rx_invalid_frame_length: Packets with invalid frame length. + * @rx_overflow_drop: Packets dropped on buffer overflow. + * + * PF/VF sends this message to CP to get the update stats by specifying the + * vport_id. CP responds with stats in struct virtchnl2_vport_stats. + * + * Associated with VIRTCHNL2_OP_GET_STATS. + */ +struct virtchnl2_vport_stats { + __le32 vport_id; + u8 pad[4]; + __le64 rx_bytes; + __le64 rx_unicast; + __le64 rx_multicast; + __le64 rx_broadcast; + __le64 rx_discards; + __le64 rx_errors; + __le64 rx_unknown_protocol; + __le64 tx_bytes; + __le64 tx_unicast; + __le64 tx_multicast; + __le64 tx_broadcast; + __le64 tx_discards; + __le64 tx_errors; + __le64 rx_invalid_frame_length; + __le64 rx_overflow_drop; +}; +VIRTCHNL2_CHECK_STRUCT_LEN(128, virtchnl2_vport_stats); + +/** + * struct virtchnl2_event - Event info. + * @event: Event opcode. See enum virtchnl2_event_codes. + * @link_speed: Link_speed provided in Mbps. + * @vport_id: Vport ID. + * @link_status: Link status. + * @pad: Padding. + * @reserved: Reserved. + * + * CP sends this message to inform the PF/VF driver of events that may affect + * it. No direct response is expected from the driver, though it may generate + * other messages in response to this one. + * + * Associated with VIRTCHNL2_OP_EVENT. + */ +struct virtchnl2_event { + __le32 event; + __le32 link_speed; + __le32 vport_id; + u8 link_status; + u8 pad; + __le16 reserved; +}; +VIRTCHNL2_CHECK_STRUCT_LEN(16, virtchnl2_event); + +/** + * struct virtchnl2_rss_key - RSS key info. + * @vport_id: Vport id. + * @key_len: Length of RSS key. + * @pad: Padding. + * @key_flex: RSS hash key, packed bytes. + * PF/VF sends this message to get or set RSS key. Only supported if both + * PF/VF and CP drivers set the VIRTCHNL2_CAP_RSS bit during configuration + * negotiation. + * + * Associated with VIRTCHNL2_OP_GET_RSS_KEY and VIRTCHNL2_OP_SET_RSS_KEY. + */ +struct virtchnl2_rss_key { + __le32 vport_id; + __le16 key_len; + u8 pad; + u8 key_flex[] __counted_by_le(key_len); +} __packed; +VIRTCHNL2_CHECK_STRUCT_LEN(7, virtchnl2_rss_key); + +/** + * struct virtchnl2_queue_chunk - chunk of contiguous queues + * @type: See enum virtchnl2_queue_type. + * @start_queue_id: Starting queue id. + * @num_queues: Number of queues. + * @pad: Padding for future extensions. + */ +struct virtchnl2_queue_chunk { + __le32 type; + __le32 start_queue_id; + __le32 num_queues; + u8 pad[4]; +}; +VIRTCHNL2_CHECK_STRUCT_LEN(16, virtchnl2_queue_chunk); + +/* struct virtchnl2_queue_chunks - chunks of contiguous queues + * @num_chunks: Number of chunks. + * @pad: Padding. + * @chunks: Chunks of contiguous queues info. + */ +struct virtchnl2_queue_chunks { + __le16 num_chunks; + u8 pad[6]; + struct virtchnl2_queue_chunk chunks[] __counted_by_le(num_chunks); +}; +VIRTCHNL2_CHECK_STRUCT_LEN(8, virtchnl2_queue_chunks); + +/** + * struct virtchnl2_del_ena_dis_queues - Enable/disable queues info. + * @vport_id: Vport id. + * @pad: Padding. + * @chunks: Chunks of contiguous queues info. + * + * PF sends these messages to enable, disable or delete queues specified in + * chunks. PF sends virtchnl2_del_ena_dis_queues struct to specify the queues + * to be enabled/disabled/deleted. Also applicable to single queue receive or + * transmit. CP performs requested action and returns status. + * + * Associated with VIRTCHNL2_OP_ENABLE_QUEUES, VIRTCHNL2_OP_DISABLE_QUEUES and + * VIRTCHNL2_OP_DISABLE_QUEUES. + */ +struct virtchnl2_del_ena_dis_queues { + __le32 vport_id; + u8 pad[4]; + struct virtchnl2_queue_chunks chunks; +}; +VIRTCHNL2_CHECK_STRUCT_LEN(16, virtchnl2_del_ena_dis_queues); + +/** + * struct virtchnl2_queue_vector - Queue to vector mapping. + * @queue_id: Queue id. + * @vector_id: Vector id. + * @pad: Padding. + * @itr_idx: See enum virtchnl2_itr_idx. + * @queue_type: See enum virtchnl2_queue_type. + * @pad1: Padding for future extensions. + */ +struct virtchnl2_queue_vector { + __le32 queue_id; + __le16 vector_id; + u8 pad[2]; + __le32 itr_idx; + __le32 queue_type; + u8 pad1[8]; +}; +VIRTCHNL2_CHECK_STRUCT_LEN(24, virtchnl2_queue_vector); + +/** + * struct virtchnl2_queue_vector_maps - Map/unmap queues info. + * @vport_id: Vport id. + * @num_qv_maps: Number of queue vector maps. + * @pad: Padding. + * @qv_maps: Queue to vector maps. + * + * PF sends this message to map or unmap queues to vectors and interrupt + * throttling rate index registers. External data buffer contains + * virtchnl2_queue_vector_maps structure that contains num_qv_maps of + * virtchnl2_queue_vector structures. CP maps the requested queue vector maps + * after validating the queue and vector ids and returns a status code. + * + * Associated with VIRTCHNL2_OP_MAP_QUEUE_VECTOR and + * VIRTCHNL2_OP_UNMAP_QUEUE_VECTOR. + */ +struct virtchnl2_queue_vector_maps { + __le32 vport_id; + __le16 num_qv_maps; + u8 pad[10]; + struct virtchnl2_queue_vector qv_maps[] __counted_by_le(num_qv_maps); +}; +VIRTCHNL2_CHECK_STRUCT_LEN(16, virtchnl2_queue_vector_maps); + +/** + * struct virtchnl2_loopback - Loopback info. + * @vport_id: Vport id. + * @enable: Enable/disable. + * @pad: Padding for future extensions. + * + * PF/VF sends this message to transition to/from the loopback state. Setting + * the 'enable' to 1 enables the loopback state and setting 'enable' to 0 + * disables it. CP configures the state to loopback and returns status. + * + * Associated with VIRTCHNL2_OP_LOOPBACK. + */ +struct virtchnl2_loopback { + __le32 vport_id; + u8 enable; + u8 pad[3]; +}; +VIRTCHNL2_CHECK_STRUCT_LEN(8, virtchnl2_loopback); + +/* struct virtchnl2_mac_addr - MAC address info. + * @addr: MAC address. + * @type: MAC type. See enum virtchnl2_mac_addr_type. + * @pad: Padding for future extensions. + */ +struct virtchnl2_mac_addr { + u8 addr[ETH_ALEN]; + u8 type; + u8 pad; +}; +VIRTCHNL2_CHECK_STRUCT_LEN(8, virtchnl2_mac_addr); + +/** + * struct virtchnl2_mac_addr_list - List of MAC addresses. + * @vport_id: Vport id. + * @num_mac_addr: Number of MAC addresses. + * @pad: Padding. + * @mac_addr_list: List with MAC address info. + * + * PF/VF driver uses this structure to send list of MAC addresses to be + * added/deleted to the CP where as CP performs the action and returns the + * status. + * + * Associated with VIRTCHNL2_OP_ADD_MAC_ADDR and VIRTCHNL2_OP_DEL_MAC_ADDR. + */ +struct virtchnl2_mac_addr_list { + __le32 vport_id; + __le16 num_mac_addr; + u8 pad[2]; + struct virtchnl2_mac_addr mac_addr_list[] __counted_by_le(num_mac_addr); +}; +VIRTCHNL2_CHECK_STRUCT_LEN(8, virtchnl2_mac_addr_list); + +/** + * struct virtchnl2_promisc_info - Promisc type info. + * @vport_id: Vport id. + * @flags: See enum virtchnl2_promisc_flags. + * @pad: Padding for future extensions. + * + * PF/VF sends vport id and flags to the CP where as CP performs the action + * and returns the status. + * + * Associated with VIRTCHNL2_OP_CONFIG_PROMISCUOUS_MODE. + */ +struct virtchnl2_promisc_info { + __le32 vport_id; + /* See VIRTCHNL2_PROMISC_FLAGS definitions */ + __le16 flags; + u8 pad[2]; +}; +VIRTCHNL2_CHECK_STRUCT_LEN(8, virtchnl2_promisc_info); + +/** + * enum virtchnl2_ptp_caps - PTP capabilities + * @VIRTCHNL2_CAP_PTP_GET_DEVICE_CLK_TIME: direct access to get the time of + * device clock + * @VIRTCHNL2_CAP_PTP_GET_DEVICE_CLK_TIME_MB: mailbox access to get the time of + * device clock + * @VIRTCHNL2_CAP_PTP_GET_CROSS_TIME: direct access to cross timestamp + * @VIRTCHNL2_CAP_PTP_GET_CROSS_TIME_MB: mailbox access to cross timestamp + * @VIRTCHNL2_CAP_PTP_SET_DEVICE_CLK_TIME: direct access to set the time of + * device clock + * @VIRTCHNL2_CAP_PTP_SET_DEVICE_CLK_TIME_MB: mailbox access to set the time of + * device clock + * @VIRTCHNL2_CAP_PTP_ADJ_DEVICE_CLK: direct access to adjust the time of device + * clock + * @VIRTCHNL2_CAP_PTP_ADJ_DEVICE_CLK_MB: mailbox access to adjust the time of + * device clock + * @VIRTCHNL2_CAP_PTP_TX_TSTAMPS: direct access to the Tx timestamping + * @VIRTCHNL2_CAP_PTP_TX_TSTAMPS_MB: mailbox access to the Tx timestamping + * + * PF/VF negotiates a set of supported PTP capabilities with the Control Plane. + * There are two access methods - mailbox (_MB) and direct. + * PTP capabilities enables Main Timer operations: get/set/adjust Main Timer, + * cross timestamping and the Tx timestamping. + */ +enum virtchnl2_ptp_caps { + VIRTCHNL2_CAP_PTP_GET_DEVICE_CLK_TIME = BIT(0), + VIRTCHNL2_CAP_PTP_GET_DEVICE_CLK_TIME_MB = BIT(1), + VIRTCHNL2_CAP_PTP_GET_CROSS_TIME = BIT(2), + VIRTCHNL2_CAP_PTP_GET_CROSS_TIME_MB = BIT(3), + VIRTCHNL2_CAP_PTP_SET_DEVICE_CLK_TIME = BIT(4), + VIRTCHNL2_CAP_PTP_SET_DEVICE_CLK_TIME_MB = BIT(5), + VIRTCHNL2_CAP_PTP_ADJ_DEVICE_CLK = BIT(6), + VIRTCHNL2_CAP_PTP_ADJ_DEVICE_CLK_MB = BIT(7), + VIRTCHNL2_CAP_PTP_TX_TSTAMPS = BIT(8), + VIRTCHNL2_CAP_PTP_TX_TSTAMPS_MB = BIT(9), +}; + +/** + * struct virtchnl2_ptp_clk_reg_offsets - Offsets of device and PHY clocks + * registers. + * @dev_clk_ns_l: Device clock low register offset + * @dev_clk_ns_h: Device clock high register offset + * @phy_clk_ns_l: PHY clock low register offset + * @phy_clk_ns_h: PHY clock high register offset + * @cmd_sync_trigger: The command sync trigger register offset + * @pad: Padding for future extensions + */ +struct virtchnl2_ptp_clk_reg_offsets { + __le32 dev_clk_ns_l; + __le32 dev_clk_ns_h; + __le32 phy_clk_ns_l; + __le32 phy_clk_ns_h; + __le32 cmd_sync_trigger; + u8 pad[4]; +}; +VIRTCHNL2_CHECK_STRUCT_LEN(24, virtchnl2_ptp_clk_reg_offsets); + +/** + * struct virtchnl2_ptp_cross_time_reg_offsets - Offsets of the device cross + * time registers. + * @sys_time_ns_l: System time low register offset + * @sys_time_ns_h: System time high register offset + * @cmd_sync_trigger: The command sync trigger register offset + * @pad: Padding for future extensions + */ +struct virtchnl2_ptp_cross_time_reg_offsets { + __le32 sys_time_ns_l; + __le32 sys_time_ns_h; + __le32 cmd_sync_trigger; + u8 pad[4]; +}; +VIRTCHNL2_CHECK_STRUCT_LEN(16, virtchnl2_ptp_cross_time_reg_offsets); + +/** + * struct virtchnl2_ptp_clk_adj_reg_offsets - Offsets of device and PHY clocks + * adjustments registers. + * @dev_clk_cmd_type: Device clock command type register offset + * @dev_clk_incval_l: Device clock increment value low register offset + * @dev_clk_incval_h: Device clock increment value high registers offset + * @dev_clk_shadj_l: Device clock shadow adjust low register offset + * @dev_clk_shadj_h: Device clock shadow adjust high register offset + * @phy_clk_cmd_type: PHY timer command type register offset + * @phy_clk_incval_l: PHY timer increment value low register offset + * @phy_clk_incval_h: PHY timer increment value high register offset + * @phy_clk_shadj_l: PHY timer shadow adjust low register offset + * @phy_clk_shadj_h: PHY timer shadow adjust high register offset + */ +struct virtchnl2_ptp_clk_adj_reg_offsets { + __le32 dev_clk_cmd_type; + __le32 dev_clk_incval_l; + __le32 dev_clk_incval_h; + __le32 dev_clk_shadj_l; + __le32 dev_clk_shadj_h; + __le32 phy_clk_cmd_type; + __le32 phy_clk_incval_l; + __le32 phy_clk_incval_h; + __le32 phy_clk_shadj_l; + __le32 phy_clk_shadj_h; +}; +VIRTCHNL2_CHECK_STRUCT_LEN(40, virtchnl2_ptp_clk_adj_reg_offsets); + +/** + * struct virtchnl2_ptp_tx_tstamp_latch_caps - PTP Tx timestamp latch + * capabilities. + * @tx_latch_reg_offset_l: Tx timestamp latch low register offset + * @tx_latch_reg_offset_h: Tx timestamp latch high register offset + * @index: Latch index provided to the Tx descriptor + * @pad: Padding for future extensions + */ +struct virtchnl2_ptp_tx_tstamp_latch_caps { + __le32 tx_latch_reg_offset_l; + __le32 tx_latch_reg_offset_h; + u8 index; + u8 pad[7]; +}; +VIRTCHNL2_CHECK_STRUCT_LEN(16, virtchnl2_ptp_tx_tstamp_latch_caps); + +/** + * struct virtchnl2_ptp_get_vport_tx_tstamp_caps - Structure that defines Tx + * tstamp entries. + * @vport_id: Vport number + * @num_latches: Total number of latches + * @tstamp_ns_lo_bit: First bit for nanosecond part of the timestamp + * @tstamp_ns_hi_bit: Last bit for nanosecond part of the timestamp + * @pad: Padding for future tstamp granularity extensions + * @tstamp_latches: Capabilities of Tx timestamp entries + * + * PF/VF sends this message to negotiate the Tx timestamp latches for each + * Vport. + * + * Associated with VIRTCHNL2_OP_PTP_GET_VPORT_TX_TSTAMP_CAPS. + */ +struct virtchnl2_ptp_get_vport_tx_tstamp_caps { + __le32 vport_id; + __le16 num_latches; + u8 tstamp_ns_lo_bit; + u8 tstamp_ns_hi_bit; + u8 pad[8]; + + struct virtchnl2_ptp_tx_tstamp_latch_caps tstamp_latches[] + __counted_by_le(num_latches); +}; +VIRTCHNL2_CHECK_STRUCT_LEN(16, virtchnl2_ptp_get_vport_tx_tstamp_caps); + +/** + * struct virtchnl2_ptp_get_caps - Get PTP capabilities + * @caps: PTP capability bitmap. See enum virtchnl2_ptp_caps + * @max_adj: The maximum possible frequency adjustment + * @base_incval: The default timer increment value + * @peer_mbx_q_id: ID of the PTP Device Control daemon queue + * @peer_id: Peer ID for PTP Device Control daemon + * @secondary_mbx: Indicates to the driver that it should create a secondary + * mailbox to inetract with control plane for PTP + * @pad: Padding for future extensions + * @clk_offsets: Main timer and PHY registers offsets + * @cross_time_offsets: Cross time registers offsets + * @clk_adj_offsets: Offsets needed to adjust the PHY and the main timer + * + * PF/VF sends this message to negotiate PTP capabilities. CP updates bitmap + * with supported features and fulfills appropriate structures. + * If HW uses primary MBX for PTP: secondary_mbx is set to false. + * If HW uses secondary MBX for PTP: secondary_mbx is set to true. + * Control plane has 2 MBX and the driver has 1 MBX, send to peer + * driver may be used to send a message using valid ptp_peer_mb_q_id and + * ptp_peer_id. + * If HW does not use send to peer driver: secondary_mbx is no care field and + * peer_mbx_q_id holds invalid value (0xFFFF). + * + * Associated with VIRTCHNL2_OP_PTP_GET_CAPS. + */ +struct virtchnl2_ptp_get_caps { + __le32 caps; + __le32 max_adj; + __le64 base_incval; + __le16 peer_mbx_q_id; + u8 peer_id; + u8 secondary_mbx; + u8 pad[4]; + + struct virtchnl2_ptp_clk_reg_offsets clk_offsets; + struct virtchnl2_ptp_cross_time_reg_offsets cross_time_offsets; + struct virtchnl2_ptp_clk_adj_reg_offsets clk_adj_offsets; +}; +VIRTCHNL2_CHECK_STRUCT_LEN(104, virtchnl2_ptp_get_caps); + +/** + * struct virtchnl2_ptp_tx_tstamp_latch - Structure that describes tx tstamp + * values, index and validity. + * @tstamp: Timestamp value + * @index: Timestamp index from which the value is read + * @valid: Timestamp validity + * @pad: Padding for future extensions + */ +struct virtchnl2_ptp_tx_tstamp_latch { + __le64 tstamp; + u8 index; + u8 valid; + u8 pad[6]; +}; +VIRTCHNL2_CHECK_STRUCT_LEN(16, virtchnl2_ptp_tx_tstamp_latch); + +/** + * struct virtchnl2_ptp_get_vport_tx_tstamp_latches - Tx timestamp latches + * associated with the vport. + * @vport_id: Number of vport that requests the timestamp + * @num_latches: Number of latches + * @get_devtime_with_txtstmp: Flag to request device time along with Tx timestamp + * @pad: Padding for future extensions + * @device_time: device time if get_devtime_with_txtstmp was set in request + * @tstamp_latches: PTP TX timestamp latch + * + * PF/VF sends this message to receive a specified number of timestamps + * entries. + * + * Associated with VIRTCHNL2_OP_PTP_GET_VPORT_TX_TSTAMP. + */ +struct virtchnl2_ptp_get_vport_tx_tstamp_latches { + __le32 vport_id; + __le16 num_latches; + u8 get_devtime_with_txtstmp; + u8 pad[1]; + __le64 device_time; + + struct virtchnl2_ptp_tx_tstamp_latch tstamp_latches[] + __counted_by_le(num_latches); +}; +VIRTCHNL2_CHECK_STRUCT_LEN(16, virtchnl2_ptp_get_vport_tx_tstamp_latches); + +/** + * struct virtchnl2_ptp_get_dev_clk_time - Associated with message + * VIRTCHNL2_OP_PTP_GET_DEV_CLK_TIME. + * @dev_time_ns: Device clock time value in nanoseconds + * @pad: Padding for future extensions + * + * PF/VF sends this message to receive the time from the main timer. + */ +struct virtchnl2_ptp_get_dev_clk_time { + __le64 dev_time_ns; + u8 pad[8]; +}; +VIRTCHNL2_CHECK_STRUCT_LEN(16, virtchnl2_ptp_get_dev_clk_time); + +/** + * struct virtchnl2_ptp_get_cross_time: Associated with message + * VIRTCHNL2_OP_PTP_GET_CROSS_TIME. + * @sys_time_ns: System counter value expressed in nanoseconds, read + * synchronously with device time + * @dev_time_ns: Device clock time value expressed in nanoseconds + * @pad: Padding for future extensions + * + * PF/VF sends this message to receive the cross time. + */ +struct virtchnl2_ptp_get_cross_time { + __le64 sys_time_ns; + __le64 dev_time_ns; + u8 pad[8]; +}; +VIRTCHNL2_CHECK_STRUCT_LEN(24, virtchnl2_ptp_get_cross_time); + +/** + * struct virtchnl2_ptp_set_dev_clk_time: Associated with message + * VIRTCHNL2_OP_PTP_SET_DEV_CLK_TIME. + * @dev_time_ns: Device time value expressed in nanoseconds to set + * @pad: Padding for future extensions + * + * PF/VF sends this message to set the time of the main timer. + */ +struct virtchnl2_ptp_set_dev_clk_time { + __le64 dev_time_ns; + u8 pad[8]; +}; +VIRTCHNL2_CHECK_STRUCT_LEN(16, virtchnl2_ptp_set_dev_clk_time); + +/** + * struct virtchnl2_ptp_adj_dev_clk_fine: Associated with message + * VIRTCHNL2_OP_PTP_ADJ_DEV_CLK_FINE. + * @incval: Source timer increment value per clock cycle + * + * PF/VF sends this message to adjust the frequency of the main timer by the + * indicated increment value. + */ +struct virtchnl2_ptp_adj_dev_clk_fine { + __le64 incval; +}; +VIRTCHNL2_CHECK_STRUCT_LEN(8, virtchnl2_ptp_adj_dev_clk_fine); + +/** + * struct virtchnl2_ptp_adj_dev_clk_time: Associated with message + * VIRTCHNL2_OP_PTP_ADJ_DEV_CLK_TIME. + * @delta: Offset in nanoseconds to adjust the time by + * + * PF/VF sends this message to adjust the time of the main timer by the delta. + */ +struct virtchnl2_ptp_adj_dev_clk_time { + __le64 delta; +}; +VIRTCHNL2_CHECK_STRUCT_LEN(8, virtchnl2_ptp_adj_dev_clk_time); + +/** + * struct virtchnl2_mem_region - MMIO memory region + * @start_offset: starting offset of the MMIO memory region + * @size: size of the MMIO memory region + */ +struct virtchnl2_mem_region { + __le64 start_offset; + __le64 size; +}; +VIRTCHNL2_CHECK_STRUCT_LEN(16, virtchnl2_mem_region); + +/** + * struct virtchnl2_get_lan_memory_regions - List of LAN MMIO memory regions + * @num_memory_regions: number of memory regions + * @pad: Padding + * @mem_reg: List with memory region info + * + * PF/VF sends this message to learn what LAN MMIO memory regions it should map. + */ +struct virtchnl2_get_lan_memory_regions { + __le16 num_memory_regions; + u8 pad[6]; + struct virtchnl2_mem_region mem_reg[]; +}; +VIRTCHNL2_CHECK_STRUCT_LEN(8, virtchnl2_get_lan_memory_regions); + +#define VIRTCHNL2_MAX_NUM_PROTO_HDRS 4 +#define VIRTCHNL2_MAX_SIZE_RAW_PACKET 256 +#define VIRTCHNL2_MAX_NUM_ACTIONS 8 + +/** + * struct virtchnl2_proto_hdr - represent one protocol header + * @hdr_type: See enum virtchnl2_proto_hdr_type + * @pad: padding + * @buffer_spec: binary buffer based on header type. + * @buffer_mask: mask applied on buffer_spec. + * + * Structure to hold protocol headers based on hdr_type + */ +struct virtchnl2_proto_hdr { + __le32 hdr_type; + u8 pad[4]; + u8 buffer_spec[64]; + u8 buffer_mask[64]; +}; +VIRTCHNL2_CHECK_STRUCT_LEN(136, virtchnl2_proto_hdr); + +/** + * struct virtchnl2_proto_hdrs - struct to represent match criteria + * @tunnel_level: specify where protocol header(s) start from. + * must be 0 when sending a raw packet request. + * 0 - from the outer layer + * 1 - from the first inner layer + * 2 - from the second inner layer + * @pad: Padding bytes + * @count: total number of protocol headers in proto_hdr. 0 for raw packet. + * @proto_hdr: Array of protocol headers + * @raw: struct holding raw packet buffer when count is 0 + */ +struct virtchnl2_proto_hdrs { + u8 tunnel_level; + u8 pad[3]; + __le32 count; + union { + struct virtchnl2_proto_hdr + proto_hdr[VIRTCHNL2_MAX_NUM_PROTO_HDRS]; + struct { + __le16 pkt_len; + u8 spec[VIRTCHNL2_MAX_SIZE_RAW_PACKET]; + u8 mask[VIRTCHNL2_MAX_SIZE_RAW_PACKET]; + } raw; + }; +}; +VIRTCHNL2_CHECK_STRUCT_LEN(552, virtchnl2_proto_hdrs); + +/** + * struct virtchnl2_rule_action - struct representing single action for a flow + * @action_type: see enum virtchnl2_action_types + * @act_conf: union representing action depending on action_type. + * @act_conf.q_id: queue id to redirect the packets to. + * @act_conf.q_grp_id: queue group id to redirect the packets to. + * @act_conf.ctr_id: used for count action. If input value 0xFFFFFFFF control + * plane assigns a new counter and returns the counter ID to + * the driver. If input value is not 0xFFFFFFFF then it must + * be an existing counter given to the driver for an earlier + * flow. Then this flow will share the counter. + * @act_conf.mark_id: Value used to mark the packets. Used for mark action. + * @act_conf.reserved: Reserved for future use. + */ +struct virtchnl2_rule_action { + __le32 action_type; + union { + __le32 q_id; + __le32 q_grp_id; + __le32 ctr_id; + __le32 mark_id; + u8 reserved[8]; + } act_conf; +}; +VIRTCHNL2_CHECK_STRUCT_LEN(12, virtchnl2_rule_action); + +/** + * struct virtchnl2_rule_action_set - struct representing multiple actions + * @count: number of valid actions in the action set of a rule + * @actions: array of struct virtchnl2_rule_action + */ +struct virtchnl2_rule_action_set { + /* action count must be less than VIRTCHNL2_MAX_NUM_ACTIONS */ + __le32 count; + struct virtchnl2_rule_action actions[VIRTCHNL2_MAX_NUM_ACTIONS]; +}; +VIRTCHNL2_CHECK_STRUCT_LEN(100, virtchnl2_rule_action_set); + +/** + * struct virtchnl2_flow_rule - represent one flow steering rule + * @proto_hdrs: array of protocol header buffers representing match criteria + * @action_set: series of actions to be applied for given rule + * @priority: rule priority. + * @pad: padding for future extensions. + */ +struct virtchnl2_flow_rule { + struct virtchnl2_proto_hdrs proto_hdrs; + struct virtchnl2_rule_action_set action_set; + __le32 priority; + u8 pad[8]; +}; +VIRTCHNL2_CHECK_STRUCT_LEN(664, virtchnl2_flow_rule); + +enum virtchnl2_flow_rule_status { + VIRTCHNL2_FLOW_RULE_SUCCESS = 1, + VIRTCHNL2_FLOW_RULE_NORESOURCE = 2, + VIRTCHNL2_FLOW_RULE_EXIST = 3, + VIRTCHNL2_FLOW_RULE_TIMEOUT = 4, + VIRTCHNL2_FLOW_RULE_FLOW_TYPE_NOT_SUPPORTED = 5, + VIRTCHNL2_FLOW_RULE_MATCH_KEY_NOT_SUPPORTED = 6, + VIRTCHNL2_FLOW_RULE_ACTION_NOT_SUPPORTED = 7, + VIRTCHNL2_FLOW_RULE_ACTION_COMBINATION_INVALID = 8, + VIRTCHNL2_FLOW_RULE_ACTION_DATA_INVALID = 9, + VIRTCHNL2_FLOW_RULE_NOT_ADDED = 10, +}; + +/** + * struct virtchnl2_flow_rule_info: structure representing single flow rule + * @rule_id: rule_id associated with the flow_rule. + * @rule_cfg: structure representing rule. + * @status: status of rule programming. See enum virtchnl2_flow_rule_status. + */ +struct virtchnl2_flow_rule_info { + __le32 rule_id; + struct virtchnl2_flow_rule rule_cfg; + __le32 status; +}; +VIRTCHNL2_CHECK_STRUCT_LEN(672, virtchnl2_flow_rule_info); + +/** + * struct virtchnl2_flow_rule_add_del - add/delete a flow steering rule + * @vport_id: vport id for which the rule is to be added or deleted. + * @count: Indicates number of rules to be added or deleted. + * @rule_info: Array of flow rules to be added or deleted. + * + * For VIRTCHNL2_OP_FLOW_RULE_ADD, rule_info contains list of rules to be + * added. If rule_id is 0xFFFFFFFF, then the rule is programmed and not cached. + * + * For VIRTCHNL2_OP_FLOW_RULE_DEL, there are two possibilities. The structure + * can contain either array of rule_ids or array of match keys to be deleted. + * When match keys are used the corresponding rule_ids must be 0xFFFFFFFF. + * + * status member of each rule indicates the result. Maximum of 6 rules can be + * added or deleted using this method. Driver has to retry in case of any + * failure of ADD or DEL opcode. CP doesn't retry in case of failure. + */ +struct virtchnl2_flow_rule_add_del { + __le32 vport_id; + __le32 count; + struct virtchnl2_flow_rule_info rule_info[] __counted_by_le(count); +}; +VIRTCHNL2_CHECK_STRUCT_LEN(8, virtchnl2_flow_rule_add_del); + +#endif /* _VIRTCHNL_2_H_ */ diff --git a/include/linux/net/intel/virtchnl2_lan_desc.h b/include/linux/net/intel/virtchnl2_lan_desc.h new file mode 100644 index 000000000000..f1b577f1c452 --- /dev/null +++ b/include/linux/net/intel/virtchnl2_lan_desc.h @@ -0,0 +1,451 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright (C) 2023 Intel Corporation */ + +#ifndef _VIRTCHNL2_LAN_DESC_H_ +#define _VIRTCHNL2_LAN_DESC_H_ + +#include <linux/bits.h> + +/* This is an interface definition file where existing enums and their values + * must remain unchanged over time, so we specify explicit values for all enums. + */ + +/* Transmit descriptor ID flags + */ +enum virtchnl2_tx_desc_ids { + VIRTCHNL2_TXDID_DATA = BIT(0), + VIRTCHNL2_TXDID_CTX = BIT(1), + /* TXDID bit 2 is reserved + * TXDID bit 3 is free for future use + * TXDID bit 4 is reserved + */ + VIRTCHNL2_TXDID_FLEX_TSO_CTX = BIT(5), + /* TXDID bit 6 is reserved */ + VIRTCHNL2_TXDID_FLEX_L2TAG1_L2TAG2 = BIT(7), + /* TXDID bits 8 and 9 are free for future use + * TXDID bit 10 is reserved + * TXDID bit 11 is free for future use + */ + VIRTCHNL2_TXDID_FLEX_FLOW_SCHED = BIT(12), + /* TXDID bits 13 and 14 are free for future use */ + VIRTCHNL2_TXDID_DESC_DONE = BIT(15), +}; + +/* Receive descriptor IDs */ +enum virtchnl2_rx_desc_ids { + VIRTCHNL2_RXDID_1_32B_BASE = 1, + /* FLEX_SQ_NIC and FLEX_SPLITQ share desc ids because they can be + * differentiated based on queue model; e.g. single queue model can + * only use FLEX_SQ_NIC and split queue model can only use FLEX_SPLITQ + * for DID 2. + */ + VIRTCHNL2_RXDID_2_FLEX_SPLITQ = 2, + VIRTCHNL2_RXDID_2_FLEX_SQ_NIC = VIRTCHNL2_RXDID_2_FLEX_SPLITQ, + /* 3 through 6 are reserved */ + VIRTCHNL2_RXDID_7_HW_RSVD = 7, + /* 8 through 15 are free */ +}; + +/* Receive descriptor ID bitmasks */ +#define VIRTCHNL2_RXDID_M(bit) BIT_ULL(VIRTCHNL2_RXDID_##bit) + +enum virtchnl2_rx_desc_id_bitmasks { + VIRTCHNL2_RXDID_1_32B_BASE_M = VIRTCHNL2_RXDID_M(1_32B_BASE), + VIRTCHNL2_RXDID_2_FLEX_SPLITQ_M = VIRTCHNL2_RXDID_M(2_FLEX_SPLITQ), + VIRTCHNL2_RXDID_2_FLEX_SQ_NIC_M = VIRTCHNL2_RXDID_M(2_FLEX_SQ_NIC), + VIRTCHNL2_RXDID_7_HW_RSVD_M = VIRTCHNL2_RXDID_M(7_HW_RSVD), +}; + +/* For splitq virtchnl2_rx_flex_desc_adv_nic_3 desc members */ +#define VIRTCHNL2_RX_FLEX_DESC_ADV_RXDID_M GENMASK(3, 0) +#define VIRTCHNL2_RX_FLEX_DESC_ADV_UMBCAST_M GENMASK(7, 6) +#define VIRTCHNL2_RX_FLEX_DESC_ADV_PTYPE_M GENMASK(9, 0) +#define VIRTCHNL2_RX_FLEX_DESC_ADV_RAW_CSUM_INV_S 12 +#define VIRTCHNL2_RX_FLEX_DESC_ADV_RAW_CSUM_INV_M \ + BIT_ULL(VIRTCHNL2_RX_FLEX_DESC_ADV_RAW_CSUM_INV_S) +#define VIRTCHNL2_RX_FLEX_DESC_ADV_FF0_M GENMASK(15, 13) +#define VIRTCHNL2_RX_FLEX_DESC_ADV_LEN_PBUF_M GENMASK(13, 0) +#define VIRTCHNL2_RX_FLEX_DESC_ADV_GEN_S 14 +#define VIRTCHNL2_RX_FLEX_DESC_ADV_GEN_M \ + BIT_ULL(VIRTCHNL2_RX_FLEX_DESC_ADV_GEN_S) +#define VIRTCHNL2_RX_FLEX_DESC_ADV_BUFQ_ID_S 15 +#define VIRTCHNL2_RX_FLEX_DESC_ADV_BUFQ_ID_M \ + BIT_ULL(VIRTCHNL2_RX_FLEX_DESC_ADV_BUFQ_ID_S) +#define VIRTCHNL2_RX_FLEX_DESC_ADV_LEN_HDR_M GENMASK(9, 0) +#define VIRTCHNL2_RX_FLEX_DESC_ADV_RSC_S 10 +#define VIRTCHNL2_RX_FLEX_DESC_ADV_RSC_M \ + BIT_ULL(VIRTCHNL2_RX_FLEX_DESC_ADV_RSC_S) +#define VIRTCHNL2_RX_FLEX_DESC_ADV_SPH_S 11 +#define VIRTCHNL2_RX_FLEX_DESC_ADV_SPH_M \ + BIT_ULL(VIRTCHNL2_RX_FLEX_DESC_ADV_SPH_S) +#define VIRTCHNL2_RX_FLEX_DESC_ADV_FF1_S 12 +#define VIRTCHNL2_RX_FLEX_DESC_ADV_FF1_M GENMASK(14, 12) +#define VIRTCHNL2_RX_FLEX_DESC_ADV_MISS_S 15 +#define VIRTCHNL2_RX_FLEX_DESC_ADV_MISS_M \ + BIT_ULL(VIRTCHNL2_RX_FLEX_DESC_ADV_MISS_S) + +/* Bitmasks for splitq virtchnl2_rx_flex_desc_adv_nic_3 */ +enum virtchl2_rx_flex_desc_adv_status_error_0_qw1_bits { + VIRTCHNL2_RX_FLEX_DESC_ADV_STATUS0_DD_M = BIT(0), + VIRTCHNL2_RX_FLEX_DESC_ADV_STATUS0_EOF_M = BIT(1), + VIRTCHNL2_RX_FLEX_DESC_ADV_STATUS0_HBO_M = BIT(2), + VIRTCHNL2_RX_FLEX_DESC_ADV_STATUS0_L3L4P_M = BIT(3), + VIRTCHNL2_RX_FLEX_DESC_ADV_STATUS0_XSUM_IPE_M = BIT(4), + VIRTCHNL2_RX_FLEX_DESC_ADV_STATUS0_XSUM_L4E_M = BIT(5), + VIRTCHNL2_RX_FLEX_DESC_ADV_STATUS0_XSUM_EIPE_M = BIT(6), + VIRTCHNL2_RX_FLEX_DESC_ADV_STATUS0_XSUM_EUDPE_M = BIT(7), +}; + +/* Bitmasks for splitq virtchnl2_rx_flex_desc_adv_nic_3 */ +enum virtchnl2_rx_flex_desc_adv_status_error_0_qw0_bits { + VIRTCHNL2_RX_FLEX_DESC_ADV_STATUS0_LPBK_M = BIT(0), + VIRTCHNL2_RX_FLEX_DESC_ADV_STATUS0_IPV6EXADD_M = BIT(1), + VIRTCHNL2_RX_FLEX_DESC_ADV_STATUS0_RXE_M = BIT(2), + VIRTCHNL2_RX_FLEX_DESC_ADV_STATUS0_CRCP_M = BIT(3), + VIRTCHNL2_RX_FLEX_DESC_ADV_STATUS0_RSS_VALID_M = BIT(4), + VIRTCHNL2_RX_FLEX_DESC_ADV_STATUS0_L2TAG1P_M = BIT(5), + VIRTCHNL2_RX_FLEX_DESC_ADV_STATUS0_XTRMD0_VALID_M = BIT(6), + VIRTCHNL2_RX_FLEX_DESC_ADV_STATUS0_XTRMD1_VALID_M = BIT(7), +}; + +/* Bitmasks for splitq virtchnl2_rx_flex_desc_adv_nic_3 */ +enum virtchnl2_rx_flex_desc_adv_status_error_1_bits { + VIRTCHNL2_RX_FLEX_DESC_ADV_STATUS1_RSVD_M = GENMASK(1, 0), + VIRTCHNL2_RX_FLEX_DESC_ADV_STATUS1_ATRAEFAIL_M = BIT(2), + VIRTCHNL2_RX_FLEX_DESC_ADV_STATUS1_L2TAG2P_M = BIT(3), + VIRTCHNL2_RX_FLEX_DESC_ADV_STATUS1_XTRMD2_VALID_M = BIT(4), + VIRTCHNL2_RX_FLEX_DESC_ADV_STATUS1_XTRMD3_VALID_M = BIT(5), + VIRTCHNL2_RX_FLEX_DESC_ADV_STATUS1_XTRMD4_VALID_M = BIT(6), + VIRTCHNL2_RX_FLEX_DESC_ADV_STATUS1_XTRMD5_VALID_M = BIT(7), +}; + +/* For singleq (flex) virtchnl2_rx_flex_desc fields + * For virtchnl2_rx_flex_desc.ptype_flex_flags0 member + */ +#define VIRTCHNL2_RX_FLEX_DESC_PTYPE_M GENMASK(9, 0) + +/* For virtchnl2_rx_flex_desc.pkt_len member */ +#define VIRTCHNL2_RX_FLEX_DESC_PKT_LEN_M GENMASK(13, 0) + +/* Bitmasks for singleq (flex) virtchnl2_rx_flex_desc */ +enum virtchnl2_rx_flex_desc_status_error_0_bits { + VIRTCHNL2_RX_FLEX_DESC_STATUS0_DD_M = BIT(0), + VIRTCHNL2_RX_FLEX_DESC_STATUS0_EOF_M = BIT(1), + VIRTCHNL2_RX_FLEX_DESC_STATUS0_HBO_M = BIT(2), + VIRTCHNL2_RX_FLEX_DESC_STATUS0_L3L4P_M = BIT(3), + VIRTCHNL2_RX_FLEX_DESC_STATUS0_XSUM_IPE_M = BIT(4), + VIRTCHNL2_RX_FLEX_DESC_STATUS0_XSUM_L4E_M = BIT(5), + VIRTCHNL2_RX_FLEX_DESC_STATUS0_XSUM_EIPE_M = BIT(6), + VIRTCHNL2_RX_FLEX_DESC_STATUS0_XSUM_EUDPE_M = BIT(7), + VIRTCHNL2_RX_FLEX_DESC_STATUS0_LPBK_M = BIT(8), + VIRTCHNL2_RX_FLEX_DESC_STATUS0_IPV6EXADD_M = BIT(9), + VIRTCHNL2_RX_FLEX_DESC_STATUS0_RXE_M = BIT(10), + VIRTCHNL2_RX_FLEX_DESC_STATUS0_CRCP_M = BIT(11), + VIRTCHNL2_RX_FLEX_DESC_STATUS0_RSS_VALID_M = BIT(12), + VIRTCHNL2_RX_FLEX_DESC_STATUS0_L2TAG1P_M = BIT(13), + VIRTCHNL2_RX_FLEX_DESC_STATUS0_XTRMD0_VALID_M = BIT(14), + VIRTCHNL2_RX_FLEX_DESC_STATUS0_XTRMD1_VALID_M = BIT(15), +}; + +/* Bitmasks for singleq (flex) virtchnl2_rx_flex_desc */ +enum virtchnl2_rx_flex_desc_status_error_1_bits { + VIRTCHNL2_RX_FLEX_DESC_STATUS1_CPM_M = GENMASK(3, 0), + VIRTCHNL2_RX_FLEX_DESC_STATUS1_NAT_M = BIT(4), + VIRTCHNL2_RX_FLEX_DESC_STATUS1_CRYPTO_M = BIT(5), + /* [10:6] reserved */ + VIRTCHNL2_RX_FLEX_DESC_STATUS1_L2TAG2P_M = BIT(11), + VIRTCHNL2_RX_FLEX_DESC_STATUS1_XTRMD2_VALID_M = BIT(12), + VIRTCHNL2_RX_FLEX_DESC_STATUS1_XTRMD3_VALID_M = BIT(13), + VIRTCHNL2_RX_FLEX_DESC_STATUS1_XTRMD4_VALID_M = BIT(14), + VIRTCHNL2_RX_FLEX_DESC_STATUS1_XTRMD5_VALID_M = BIT(15), +}; + +/* For virtchnl2_rx_flex_desc.ts_low member */ +#define VIRTCHNL2_RX_FLEX_TSTAMP_VALID BIT(0) + +/* For singleq (non flex) virtchnl2_singleq_base_rx_desc legacy desc members */ +#define VIRTCHNL2_RX_BASE_DESC_QW1_LEN_PBUF_M GENMASK_ULL(51, 38) +#define VIRTCHNL2_RX_BASE_DESC_QW1_PTYPE_M GENMASK_ULL(37, 30) +#define VIRTCHNL2_RX_BASE_DESC_QW1_ERROR_M GENMASK_ULL(26, 19) +#define VIRTCHNL2_RX_BASE_DESC_QW1_STATUS_M GENMASK_ULL(18, 0) + +/* Bitmasks for singleq (base) virtchnl2_rx_base_desc */ +enum virtchnl2_rx_base_desc_status_bits { + VIRTCHNL2_RX_BASE_DESC_STATUS_DD_M = BIT(0), + VIRTCHNL2_RX_BASE_DESC_STATUS_EOF_M = BIT(1), + VIRTCHNL2_RX_BASE_DESC_STATUS_L2TAG1P_M = BIT(2), + VIRTCHNL2_RX_BASE_DESC_STATUS_L3L4P_M = BIT(3), + VIRTCHNL2_RX_BASE_DESC_STATUS_CRCP_M = BIT(4), + VIRTCHNL2_RX_BASE_DESC_STATUS_RSVD_M = GENMASK(7, 5), + VIRTCHNL2_RX_BASE_DESC_STATUS_EXT_UDP_0_M = BIT(8), + VIRTCHNL2_RX_BASE_DESC_STATUS_UMBCAST_M = GENMASK(10, 9), + VIRTCHNL2_RX_BASE_DESC_STATUS_FLM_M = BIT(11), + VIRTCHNL2_RX_BASE_DESC_STATUS_FLTSTAT_M = GENMASK(13, 12), + VIRTCHNL2_RX_BASE_DESC_STATUS_LPBK_M = BIT(14), + VIRTCHNL2_RX_BASE_DESC_STATUS_IPV6EXADD_M = BIT(15), + VIRTCHNL2_RX_BASE_DESC_STATUS_RSVD1_M = GENMASK(17, 16), + VIRTCHNL2_RX_BASE_DESC_STATUS_INT_UDP_0_M = BIT(18), +}; + +/* Bitmasks for singleq (base) virtchnl2_rx_base_desc */ +enum virtchnl2_rx_base_desc_error_bits { + VIRTCHNL2_RX_BASE_DESC_ERROR_RXE_M = BIT(0), + VIRTCHNL2_RX_BASE_DESC_ERROR_ATRAEFAIL_M = BIT(1), + VIRTCHNL2_RX_BASE_DESC_ERROR_HBO_M = BIT(2), + VIRTCHNL2_RX_BASE_DESC_ERROR_L3L4E_M = GENMASK(5, 3), + VIRTCHNL2_RX_BASE_DESC_ERROR_IPE_M = BIT(3), + VIRTCHNL2_RX_BASE_DESC_ERROR_L4E_M = BIT(4), + VIRTCHNL2_RX_BASE_DESC_ERROR_EIPE_M = BIT(5), + VIRTCHNL2_RX_BASE_DESC_ERROR_OVERSIZE_M = BIT(6), + VIRTCHNL2_RX_BASE_DESC_ERROR_PPRS_M = BIT(7), +}; + +/* Bitmasks for singleq (base) virtchnl2_rx_base_desc */ +#define VIRTCHNL2_RX_BASE_DESC_FLTSTAT_RSS_HASH_M GENMASK(13, 12) + +/** + * struct virtchnl2_splitq_rx_buf_desc - SplitQ RX buffer descriptor format + * @qword0: RX buffer struct. + * @qword0.buf_id: Buffer identifier. + * @qword0.rsvd0: Reserved. + * @qword0.rsvd1: Reserved. + * @pkt_addr: Packet buffer address. + * @hdr_addr: Header buffer address. + * @rsvd2: Rerserved. + * + * Receive Descriptors + * SplitQ buffer + * | 16| 0| + * ---------------------------------------------------------------- + * | RSV | Buffer ID | + * ---------------------------------------------------------------- + * | Rx packet buffer address | + * ---------------------------------------------------------------- + * | Rx header buffer address | + * ---------------------------------------------------------------- + * | RSV | + * ---------------------------------------------------------------- + * | 0| + */ +struct virtchnl2_splitq_rx_buf_desc { + struct { + __le16 buf_id; + __le16 rsvd0; + __le32 rsvd1; + } qword0; + __le64 pkt_addr; + __le64 hdr_addr; + __le64 rsvd2; +}; + +/** + * struct virtchnl2_singleq_rx_buf_desc - SingleQ RX buffer descriptor format. + * @pkt_addr: Packet buffer address. + * @hdr_addr: Header buffer address. + * @rsvd1: Reserved. + * @rsvd2: Reserved. + * + * SingleQ buffer + * | 0| + * ---------------------------------------------------------------- + * | Rx packet buffer address | + * ---------------------------------------------------------------- + * | Rx header buffer address | + * ---------------------------------------------------------------- + * | RSV | + * ---------------------------------------------------------------- + * | RSV | + * ---------------------------------------------------------------- + * | 0| + */ +struct virtchnl2_singleq_rx_buf_desc { + __le64 pkt_addr; + __le64 hdr_addr; + __le64 rsvd1; + __le64 rsvd2; +}; + +/** + * struct virtchnl2_singleq_base_rx_desc - RX descriptor writeback format. + * @qword0: First quad word struct. + * @qword0.lo_dword: Lower dual word struct. + * @qword0.lo_dword.mirroring_status: Mirrored packet status. + * @qword0.lo_dword.l2tag1: Stripped L2 tag from the received packet. + * @qword0.hi_dword: High dual word union. + * @qword0.hi_dword.rss: RSS hash. + * @qword0.hi_dword.fd_id: Flow director filter id. + * @qword1: Second quad word struct. + * @qword1.status_error_ptype_len: Status/error/PTYPE/length. + * @qword2: Third quad word struct. + * @qword2.ext_status: Extended status. + * @qword2.rsvd: Reserved. + * @qword2.l2tag2_1: Extracted L2 tag 2 from the packet. + * @qword2.l2tag2_2: Reserved. + * @qword3: Fourth quad word struct. + * @qword3.reserved: Reserved. + * @qword3.fd_id: Flow director filter id. + * + * Profile ID 0x1, SingleQ, base writeback format + */ +struct virtchnl2_singleq_base_rx_desc { + struct { + struct { + __le16 mirroring_status; + __le16 l2tag1; + } lo_dword; + union { + __le32 rss; + __le32 fd_id; + } hi_dword; + } qword0; + struct { + __le64 status_error_ptype_len; + } qword1; + struct { + __le16 ext_status; + __le16 rsvd; + __le16 l2tag2_1; + __le16 l2tag2_2; + } qword2; + struct { + __le32 reserved; + __le32 fd_id; + } qword3; +}; + +/** + * struct virtchnl2_rx_flex_desc_nic - RX descriptor writeback format. + * + * @rxdid: Descriptor builder profile id. + * @mir_id_umb_cast: umb_cast=[7:6], mirror=[5:0] + * @ptype_flex_flags0: ff0=[15:10], ptype=[9:0] + * @pkt_len: Packet length, [15:14] are reserved. + * @hdr_len_sph_flex_flags1: ff1/ext=[15:12], sph=[11], header=[10:0]. + * @status_error0: Status/Error section 0. + * @l2tag1: Stripped L2 tag from the received packet + * @rss_hash: RSS hash. + * @status_error1: Status/Error section 1. + * @flexi_flags2: Flexible flags section 2. + * @ts_low: Lower word of timestamp value. + * @l2tag2_1st: First L2TAG2. + * @l2tag2_2nd: Second L2TAG2. + * @flow_id: Flow id. + * @flex_ts: Timestamp and flexible flow id union. + * @flex_ts.ts_high: Timestamp higher word of the timestamp value. + * @flex_ts.flex.rsvd: Reserved. + * @flex_ts.flex.flow_id_ipv6: IPv6 flow id. + * + * Profile ID 0x2, SingleQ, flex writeback format + */ +struct virtchnl2_rx_flex_desc_nic { + /* Qword 0 */ + u8 rxdid; + u8 mir_id_umb_cast; + __le16 ptype_flex_flags0; + __le16 pkt_len; + __le16 hdr_len_sph_flex_flags1; + /* Qword 1 */ + __le16 status_error0; + __le16 l2tag1; + __le32 rss_hash; + /* Qword 2 */ + __le16 status_error1; + u8 flexi_flags2; + u8 ts_low; + __le16 l2tag2_1st; + __le16 l2tag2_2nd; + /* Qword 3 */ + __le32 flow_id; + union { + struct { + __le16 rsvd; + __le16 flow_id_ipv6; + } flex; + __le32 ts_high; + } flex_ts; +}; + +/** + * struct virtchnl2_rx_flex_desc_adv_nic_3 - RX descriptor writeback format. + * @rxdid_ucast: ucast=[7:6], rsvd=[5:4], profile_id=[3:0]. + * @status_err0_qw0: Status/Error section 0 in quad word 0. + * @ptype_err_fflags0: ff0=[15:12], udp_len_err=[11], ip_hdr_err=[10], + * ptype=[9:0]. + * @pktlen_gen_bufq_id: bufq_id=[15] only in splitq, gen=[14] only in splitq, + * plen=[13:0]. + * @hdrlen_flags: miss_prepend=[15], trunc_mirr=[14], int_udp_0=[13], + * ext_udp0=[12], sph=[11] only in splitq, rsc=[10] + * only in splitq, header=[9:0]. + * @status_err0_qw1: Status/Error section 0 in quad word 1. + * @status_err1: Status/Error section 1. + * @fflags1: Flexible flags section 1. + * @ts_low: Lower word of timestamp value. + * @buf_id: Buffer identifier. Only in splitq mode. + * @misc: Union. + * @misc.raw_cs: Raw checksum. + * @misc.l2tag1: Stripped L2 tag from the received packet + * @misc.rscseglen: + * @hash1: Lower bits of Rx hash value. + * @ff2_mirrid_hash2: Union. + * @ff2_mirrid_hash2.fflags2: Flexible flags section 2. + * @ff2_mirrid_hash2.mirrorid: Mirror id. + * @ff2_mirrid_hash2.rscseglen: RSC segment length. + * @hash3: Upper bits of Rx hash value. + * @l2tag2: Extracted L2 tag 2 from the packet. + * @fmd4: Flexible metadata container 4. + * @l2tag1: Stripped L2 tag from the received packet + * @fmd6: Flexible metadata container 6. + * @ts_high: Timestamp higher word of the timestamp value. + * + * Profile ID 0x2, SplitQ, flex writeback format + * + * Flex-field 0: BufferID + * Flex-field 1: Raw checksum/L2TAG1/RSC Seg Len (determined by HW) + * Flex-field 2: Hash[15:0] + * Flex-flags 2: Hash[23:16] + * Flex-field 3: L2TAG2 + * Flex-field 5: L2TAG1 + * Flex-field 7: Timestamp (upper 32 bits) + */ +struct virtchnl2_rx_flex_desc_adv_nic_3 { + /* Qword 0 */ + u8 rxdid_ucast; + u8 status_err0_qw0; + __le16 ptype_err_fflags0; + __le16 pktlen_gen_bufq_id; + __le16 hdrlen_flags; + /* Qword 1 */ + u8 status_err0_qw1; + u8 status_err1; + u8 fflags1; + u8 ts_low; + __le16 buf_id; + union { + __le16 raw_cs; + __le16 l2tag1; + __le16 rscseglen; + } misc; + /* Qword 2 */ + __le16 hash1; + union { + u8 fflags2; + u8 mirrorid; + u8 hash2; + } ff2_mirrid_hash2; + u8 hash3; + __le16 l2tag2; + __le16 fmd4; + /* Qword 3 */ + __le16 l2tag1; + __le16 fmd6; + __le32 ts_high; +}; + +/* Common union for accessing descriptor format structs */ +union virtchnl2_rx_desc { + struct virtchnl2_singleq_base_rx_desc base_wb; + struct virtchnl2_rx_flex_desc_nic flex_nic_wb; + struct virtchnl2_rx_flex_desc_adv_nic_3 flex_adv_nic_3_wb; +}; + +#endif /* _VIRTCHNL_LAN_DESC_H_ */ diff --git a/include/linux/netdev_features.h b/include/linux/netdev_features.h index 93e4da7046a1..8eb6b8033606 100644 --- a/include/linux/netdev_features.h +++ b/include/linux/netdev_features.h @@ -79,7 +79,7 @@ enum { NETIF_F_HW_TLS_RX_BIT, /* Hardware TLS RX offload */ NETIF_F_GRO_HW_BIT, /* Hardware Generic receive offload */ - NETIF_F_HW_TLS_RECORD_BIT, /* Offload TLS record */ + __UNUSED_NETIF_F_56, NETIF_F_GRO_FRAGLIST_BIT, /* Fraglist GRO */ NETIF_F_HW_MACSEC_BIT, /* Offload MACsec operations */ @@ -153,7 +153,6 @@ enum { #define NETIF_F_HW_ESP __NETIF_F(HW_ESP) #define NETIF_F_HW_ESP_TX_CSUM __NETIF_F(HW_ESP_TX_CSUM) #define NETIF_F_RX_UDP_TUNNEL_PORT __NETIF_F(RX_UDP_TUNNEL_PORT) -#define NETIF_F_HW_TLS_RECORD __NETIF_F(HW_TLS_RECORD) #define NETIF_F_GSO_UDP_L4 __NETIF_F(GSO_UDP_L4) #define NETIF_F_HW_TLS_TX __NETIF_F(HW_TLS_TX) #define NETIF_F_HW_TLS_RX __NETIF_F(HW_TLS_RX) diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index 7ca01eb3f7d2..87cafc932e9e 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -162,7 +162,7 @@ static inline bool dev_xmit_complete(int rc) #if defined(CONFIG_HYPERV_NET) # define LL_MAX_HEADER 128 -#elif defined(CONFIG_WLAN) || IS_ENABLED(CONFIG_AX25) +#elif defined(CONFIG_WLAN) # if defined(CONFIG_MAC80211_MESH) # define LL_MAX_HEADER 128 # else @@ -300,9 +300,11 @@ struct hh_cache { * We could use other alignment values, but we must maintain the * relationship HH alignment <= LL alignment. */ -#define LL_RESERVED_SPACE(dev) \ - ((((dev)->hard_header_len + READ_ONCE((dev)->needed_headroom)) \ +#define LL_RESERVED_SPACE_EX(dev, hlen) \ + ((((hlen) + READ_ONCE((dev)->needed_headroom)) \ & ~(HH_DATA_MOD - 1)) + HH_DATA_MOD) +#define LL_RESERVED_SPACE(dev) \ + LL_RESERVED_SPACE_EX(dev, (dev)->hard_header_len) #define LL_RESERVED_SPACE_EXTRA(dev,extra) \ ((((dev)->hard_header_len + READ_ONCE((dev)->needed_headroom) + (extra)) \ & ~(HH_DATA_MOD - 1)) + HH_DATA_MOD) @@ -830,8 +832,13 @@ struct xps_dev_maps { #define TC_BITMASK 15 /* HW offloaded queuing disciplines txq count and offset maps */ struct netdev_tc_txq { - u16 count; - u16 offset; + union { + struct { + u16 count; + u16 offset; + }; + u32 combined; + }; }; #if defined(CONFIG_FCOE) || defined(CONFIG_FCOE_MODULE) @@ -892,6 +899,7 @@ struct net_device_path { u8 h_dest[ETH_ALEN]; } encap; struct { + struct dst_entry *dst; union { struct in_addr src_v4; struct in6_addr src_v6; @@ -901,7 +909,7 @@ struct net_device_path { struct in6_addr dst_v6; }; - u8 l3_proto; + u8 inner_proto; } tun; struct { enum { @@ -938,6 +946,7 @@ struct net_device_path_stack { struct net_device_path_ctx { const struct net_device *dev; u8 daddr[ETH_ALEN]; + __be16 ether_type; int num_vlans; struct { @@ -1119,6 +1128,20 @@ struct netdev_net_notifier { * This function is called device changes address list filtering. * If driver handles unicast address filtering, it should set * IFF_UNICAST_FLT in its priv_flags. + * Cannot sleep, called with netif_addr_lock_bh held. + * Deprecated in favor of ndo_set_rx_mode_async. + * + * int (*ndo_set_rx_mode_async)(struct net_device *dev, + * struct netdev_hw_addr_list *uc, + * struct netdev_hw_addr_list *mc); + * Async version of ndo_set_rx_mode which runs in process context + * with rtnl_lock and netdev_lock_ops(dev) held. The uc/mc parameters + * are snapshots of the address lists - iterate with + * netdev_hw_addr_list_for_each(ha, uc). Return 0 on success or a + * negative errno to request a retry via the core backoff. + * + * void (*ndo_work)(struct net_device *dev, unsigned long events); + * Run deferred work scheduled with netdev_work_sched(@events). * * int (*ndo_set_mac_address)(struct net_device *dev, void *addr); * This function is called when the Media Access Control address @@ -1139,8 +1162,8 @@ struct netdev_net_notifier { * SIOCBONDSLAVEINFOQUERY, and SIOCBONDINFOQUERY * * * int (*ndo_eth_ioctl)(struct net_device *dev, struct ifreq *ifr, int cmd); - * Called for ethernet specific ioctls: SIOCGMIIPHY, SIOCGMIIREG, - * SIOCSMIIREG, SIOCSHWTSTAMP and SIOCGHWTSTAMP. + * Called for ethernet specific ioctls: SIOCGMIIPHY, SIOCGMIIREG and + * SIOCSMIIREG. * * int (*ndo_set_config)(struct net_device *dev, struct ifmap *map); * Used to set network devices bus interface parameters. This interface @@ -1213,6 +1236,12 @@ struct netdev_net_notifier { * tx queues stopped. This allows the netdevice to perform queue * management safely. * + * NB: Returning -EOPNOTSUPP for whatever commands means "this qdisc + * is not offloaded (anymore, offloading may have silently stopped)", + * and the offloading flag is cleared. Notably, this is also true for + * dump queries (e.g. TC_*_STATS commands). If the underlying device does + * not report any statistics but is still offloading, return 0 instead. + * * Fiber Channel over Ethernet (FCoE) offload functions. * int (*ndo_fcoe_enable)(struct net_device *dev); * Called when the FCoE protocol stack wants to start using LLD for FCoE @@ -1439,6 +1468,12 @@ struct net_device_ops { void (*ndo_change_rx_flags)(struct net_device *dev, int flags); void (*ndo_set_rx_mode)(struct net_device *dev); + int (*ndo_set_rx_mode_async)( + struct net_device *dev, + struct netdev_hw_addr_list *uc, + struct netdev_hw_addr_list *mc); + void (*ndo_work)(struct net_device *dev, + unsigned long events); int (*ndo_set_mac_address)(struct net_device *dev, void *addr); int (*ndo_validate_addr)(struct net_device *dev); @@ -1716,7 +1751,6 @@ struct net_device_ops { * @IFF_OPENVSWITCH: device is a Open vSwitch master * @IFF_L3MDEV_SLAVE: device is enslaved to an L3 master device * @IFF_TEAM: device is a team device - * @IFF_RXFH_CONFIGURED: device has had Rx Flow indirection table configured * @IFF_PHONY_HEADROOM: the headroom value is controlled by an external * entity (i.e. the master device for bridged veth) * @IFF_MACSEC: device is a MACsec device @@ -1752,7 +1786,6 @@ enum netdev_priv_flags { IFF_OPENVSWITCH = 1<<20, IFF_L3MDEV_SLAVE = 1<<21, IFF_TEAM = 1<<22, - IFF_RXFH_CONFIGURED = 1<<23, IFF_PHONY_HEADROOM = 1<<24, IFF_MACSEC = 1<<25, IFF_NO_RX_HANDLER = 1<<26, @@ -1776,6 +1809,12 @@ enum netdev_stat_type { NETDEV_PCPU_STAT_DSTATS, /* struct pcpu_dstats */ }; +enum netmem_tx_mode { + NETMEM_TX_NONE, /* no netmem TX support */ + NETMEM_TX_DMA, /* DMA-capable netmem TX (real HW) */ + NETMEM_TX_NO_DMA, /* no DMA, e.g. passthrough for virtual devs */ +}; + enum netdev_reg_state { NETREG_UNINITIALIZED = 0, NETREG_REGISTERED, /* completed register_netdevice */ @@ -1797,7 +1836,7 @@ enum netdev_reg_state { * @lltx: device supports lockless Tx. Deprecated for real HW * drivers. Mainly used by logical interfaces, such as * bonding and tunnels - * @netmem_tx: device support netmem_tx. + * @netmem_tx: device netmem TX mode * * @name: This is the first field of the "visible" part of this structure * (i.e. as seen by users in the "Space.c" file). It is the name @@ -1815,6 +1854,8 @@ enum netdev_reg_state { * @napi_list: List entry used for polling NAPI devices * @unreg_list: List entry when we are unregistering the * device; see the function unregister_netdev + * @unreg_list_net:List entry when we are unregistering the cross-netns + * device; see the function unregister_netdevice_queue_net() * @close_list: List entry used when we are closing the device * @ptype_all: Device-specific packet handlers for all protocols * @ptype_specific: Device-specific, protocol-specific packet handlers @@ -1905,6 +1946,14 @@ enum netdev_reg_state { * has been enabled due to the need to listen to * additional unicast addresses in a device that * does not implement ndo_set_rx_mode() + * @work_node: List entry for async netdev_work processing + * @work_tracker: Refcount tracker for async netdev_work + * @work_pending: Driver-defined pending netdev_work, passed to + * ndo_work() (see netdev_work_sched()) + * @work_core_pending: Core-defined pending netdev_work (NETDEV_WORK_*) + * @rx_mode_addr_cache: Recycled snapshot entries for rx_mode work + * @rx_mode_retry_timer: Timer that re-queues rx_mode work after failure + * @rx_mode_retry_count: Number of consecutive retries already scheduled * @uc: unicast mac addresses * @mc: multicast mac addresses * @dev_addrs: list of device hw addresses @@ -1917,10 +1966,8 @@ enum netdev_reg_state { * @vlan_info: VLAN info * @dsa_ptr: dsa specific data * @tipc_ptr: TIPC specific data - * @atalk_ptr: AppleTalk link * @ip_ptr: IPv4 specific data * @ip6_ptr: IPv6 specific data - * @ax25_ptr: AX.25 specific data * @ieee80211_ptr: IEEE 802.11 specific data, assign before registering * @ieee802154_ptr: IEEE 802.15.4 low-rate Wireless Personal Area Network * device struct @@ -1965,6 +2012,8 @@ enum netdev_reg_state { * @qdisc_hash: qdisc hash table * @watchdog_timeo: Represents the timeout that is used by * the watchdog (see dev_watchdog()) + * @watchdog_lock: protect watchdog_ref_held + * @watchdog_ref_held: True if the watchdog device ref is taken. * @watchdog_timer: List of timers * * @proto_down_reason: reason a netdev interface is held down @@ -2117,7 +2166,7 @@ struct net_device { struct_group(priv_flags_fast, unsigned long priv_flags:32; unsigned long lltx:1; - unsigned long netmem_tx:1; + unsigned long netmem_tx:2; ); const struct net_device_ops *netdev_ops; const struct header_ops *header_ops; @@ -2203,6 +2252,9 @@ struct net_device { struct list_head dev_list; struct list_head napi_list; struct list_head unreg_list; +#ifdef CONFIG_DEBUG_NET_SMALL_RTNL + struct list_head unreg_list_net; +#endif struct list_head close_list; struct list_head ptype_all; @@ -2296,6 +2348,13 @@ struct net_device { unsigned int promiscuity; unsigned int allmulti; bool uc_promisc; + struct list_head work_node; + netdevice_tracker work_tracker; + unsigned long work_pending; + unsigned long work_core_pending; + struct netdev_hw_addr_list rx_mode_addr_cache; + struct timer_list rx_mode_retry_timer; + unsigned int rx_mode_retry_count; #ifdef CONFIG_LOCKDEP unsigned char nested_level; #endif @@ -2315,12 +2374,6 @@ struct net_device { #if IS_ENABLED(CONFIG_TIPC) struct tipc_bearer __rcu *tipc_ptr; #endif -#if IS_ENABLED(CONFIG_ATALK) - void *atalk_ptr; -#endif -#if IS_ENABLED(CONFIG_AX25) - struct ax25_dev __rcu *ax25_ptr; -#endif #if IS_ENABLED(CONFIG_CFG80211) struct wireless_dev *ieee80211_ptr; #endif @@ -2377,6 +2430,8 @@ struct net_device { /* These may be needed for future network-power-down code. */ struct timer_list watchdog_timer; int watchdog_timeo; + spinlock_t watchdog_lock; + bool watchdog_ref_held; u32 proto_down_reason; @@ -2557,13 +2612,23 @@ struct net_device { * Double protects: * @up, @moving_ns, @nd_net, @xdp_features * + * Ops protects: + * @cfg, @cfg_pending, @ethtool, @hwprov + * * Double ops protects: * @real_num_rx_queues, @real_num_tx_queues * * Also protects some fields in: * struct napi_struct, struct netdev_queue, struct netdev_rx_queue * - * Ordering: take after rtnl_lock. + * Ordering: + * + * - take after rtnl_lock + * + * - for the case of netdev queue leasing, the netdev-scope lock is + * taken for both the virtual and the physical device; to prevent + * deadlocks, the virtual device's lock must always be acquired + * before the physical device's (see netdev_nl_queue_create_doit) */ struct mutex lock; @@ -2607,16 +2672,16 @@ static inline bool netif_elide_gro(const struct net_device *dev) static inline int netdev_get_prio_tc_map(const struct net_device *dev, u32 prio) { - return dev->prio_tc_map[prio & TC_BITMASK]; + return READ_ONCE(dev->prio_tc_map[prio & TC_BITMASK]); } static inline int netdev_set_prio_tc_map(struct net_device *dev, u8 prio, u8 tc) { - if (tc >= dev->num_tc) + if (tc >= READ_ONCE(dev->num_tc)) return -EINVAL; - dev->prio_tc_map[prio & TC_BITMASK] = tc & TC_BITMASK; + WRITE_ONCE(dev->prio_tc_map[prio & TC_BITMASK], tc & TC_BITMASK); return 0; } @@ -2626,9 +2691,9 @@ int netdev_set_tc_queue(struct net_device *dev, u8 tc, u16 count, u16 offset); int netdev_set_num_tc(struct net_device *dev, u8 num_tc); static inline -int netdev_get_num_tc(struct net_device *dev) +int netdev_get_num_tc(const struct net_device *dev) { - return dev->num_tc; + return READ_ONCE(dev->num_tc); } static inline void net_prefetch(void *p) @@ -2655,7 +2720,7 @@ int netdev_bind_sb_channel_queue(struct net_device *dev, int netdev_set_sb_channel(struct net_device *dev, u16 channel); static inline int netdev_get_sb_channel(struct net_device *dev) { - return max_t(int, -dev->num_tc, 0); + return max_t(int, -READ_ONCE(dev->num_tc), 0); } static inline @@ -2765,6 +2830,17 @@ static inline void *netdev_priv(const struct net_device *dev) return (void *)dev->priv; } +/** + * netdev_from_priv() - get network device from priv + * @priv: network device private data + * + * Returns: net_device to which @priv belongs + */ +static inline struct net_device *netdev_from_priv(const void *priv) +{ + return container_of(priv, struct net_device, priv); +} + /* Set the sysfs physical device reference for the network logical device * if set prior to registration will cause a symlink during initialization. */ @@ -3345,7 +3421,6 @@ static inline struct net_device *first_net_device(struct net *net) net_device_entry(net->dev_base_head.next); } -int netdev_boot_setup_check(struct net_device *dev); struct net_device *dev_getbyhwaddr(struct net *net, unsigned short type, const char *hwaddr); struct net_device *dev_getbyhwaddr_rcu(struct net *net, unsigned short type, @@ -3359,8 +3434,9 @@ void dev_remove_offload(struct packet_offload *po); int dev_get_iflink(const struct net_device *dev); int dev_fill_metadata_dst(struct net_device *dev, struct sk_buff *skb); -int dev_fill_forward_path(const struct net_device *dev, const u8 *daddr, +int dev_fill_forward_path(struct net_device_path_ctx *ctx, struct net_device_path_stack *stack); +void dev_fill_forward_path_release(struct net_device_path_stack *stack); struct net_device *dev_get_by_name(struct net *net, const char *name); struct net_device *dev_get_by_name_rcu(struct net *net, const char *name); struct net_device *__dev_get_by_name(struct net *net, const char *name); @@ -3404,11 +3480,32 @@ static inline int dev_direct_xmit(struct sk_buff *skb, u16 queue_id) int register_netdevice(struct net_device *dev); void unregister_netdevice_queue(struct net_device *dev, struct list_head *head); void unregister_netdevice_many(struct list_head *head); +bool unregister_netdevice_queued(const struct net_device *dev); + static inline void unregister_netdevice(struct net_device *dev) { unregister_netdevice_queue(dev, NULL); } +#ifdef CONFIG_DEBUG_NET_SMALL_RTNL +void unregister_netdevice_queue_net(struct net *net, struct net_device *dev, + struct list_head *head); +void unregister_netdevice_many_net(struct net *net); +void unregister_netdevice_queue_many_net(struct net *net, struct list_head *head); +#else +static inline void unregister_netdevice_queue_net(struct net *net, + struct net_device *dev, + struct list_head *head) +{ + unregister_netdevice_queue(dev, head); +} + +static inline void unregister_netdevice_queue_many_net(struct net *net, + struct list_head *head) +{ +} +#endif + int netdev_refcnt_read(const struct net_device *dev); void free_netdev(struct net_device *dev); @@ -3468,11 +3565,6 @@ static inline bool dev_validate_header(const struct net_device *dev, if (len < dev->min_header_len) return false; - if (capable(CAP_SYS_RAWIO)) { - memset(ll_header + len, 0, dev->hard_header_len - len); - return true; - } - if (dev->header_ops && dev->header_ops->validate) return dev->header_ops->validate(ll_header, len); @@ -4986,6 +5078,14 @@ void __hw_addr_unsync_dev(struct netdev_hw_addr_list *list, int (*unsync)(struct net_device *, const unsigned char *)); void __hw_addr_init(struct netdev_hw_addr_list *list); +void __hw_addr_flush(struct netdev_hw_addr_list *list); +int __hw_addr_list_snapshot(struct netdev_hw_addr_list *snap, + const struct netdev_hw_addr_list *list, + int addr_len, struct netdev_hw_addr_list *cache); +void __hw_addr_list_reconcile(struct netdev_hw_addr_list *real_list, + struct netdev_hw_addr_list *work, + struct netdev_hw_addr_list *ref, int addr_len, + struct netdev_hw_addr_list *cache); /* Functions used for device addresses handling */ void dev_addr_mod(struct net_device *dev, unsigned int offset, @@ -5095,6 +5195,7 @@ static inline void __dev_mc_unsync(struct net_device *dev, /* Functions used for secondary unicast and multicast support */ void dev_set_rx_mode(struct net_device *dev); +void netif_rx_mode_schedule_retry(struct net_device *dev); int netif_set_promiscuity(struct net_device *dev, int inc); int dev_set_promiscuity(struct net_device *dev, int inc); int netif_set_allmulti(struct net_device *dev, int inc, bool notify); @@ -5114,6 +5215,9 @@ void dev_fetch_sw_netstats(struct rtnl_link_stats64 *s, const struct pcpu_sw_netstats __percpu *netstats); void dev_get_tstats64(struct net_device *dev, struct rtnl_link_stats64 *s); +void netdev_work_sched(struct net_device *dev, unsigned long events); +unsigned long netdev_work_cancel(struct net_device *dev, unsigned long mask); + enum { NESTED_SYNC_IMM_BIT, NESTED_SYNC_TODO_BIT, @@ -5339,9 +5443,9 @@ static inline netdev_tx_t netdev_start_xmit(struct sk_buff *skb, struct net_devi } int netdev_class_create_file_ns(const struct class_attribute *class_attr, - const void *ns); + const struct ns_common *ns); void netdev_class_remove_file_ns(const struct class_attribute *class_attr, - const void *ns); + const struct ns_common *ns); extern const struct kobj_ns_type_operations net_ns_type_operations; @@ -5569,10 +5673,7 @@ static inline bool netif_is_lag_port(const struct net_device *dev) return netif_is_bond_slave(dev) || netif_is_team_port(dev); } -static inline bool netif_is_rxfh_configured(const struct net_device *dev) -{ - return dev->priv_flags & IFF_RXFH_CONFIGURED; -} +bool netif_is_rxfh_configured(const struct net_device *dev); static inline bool netif_is_failover(const struct net_device *dev) { diff --git a/include/linux/netfilter.h b/include/linux/netfilter.h index efbbfa770d66..e99afc1414cd 100644 --- a/include/linux/netfilter.h +++ b/include/linux/netfilter.h @@ -93,6 +93,7 @@ enum nf_hook_ops_type { NF_HOOK_OP_NF_TABLES, NF_HOOK_OP_BPF, NF_HOOK_OP_NFT_FT, + NF_HOOK_OP_NAT, }; struct nf_hook_ops { @@ -140,6 +141,12 @@ struct nf_hook_entries { */ }; +struct nf_nat_lookup_hook_priv { + struct nf_hook_entries __rcu *entries; + + struct rcu_head rcu_head; +}; + #ifdef CONFIG_NETFILTER static inline struct nf_hook_ops **nf_hook_entries_get_hook_ops(const struct nf_hook_entries *e) { diff --git a/include/linux/netfilter/ipset/ip_set.h b/include/linux/netfilter/ipset/ip_set.h index e9f4f845d760..c46864cc6623 100644 --- a/include/linux/netfilter/ipset/ip_set.h +++ b/include/linux/netfilter/ipset/ip_set.h @@ -244,8 +244,8 @@ extern void ip_set_type_unregister(struct ip_set_type *set_type); /* A generic IP set */ struct ip_set { - /* For call_cru in destroy */ - struct rcu_head rcu; + /* for set destruction */ + struct rcu_work rwork; /* The name of the set */ char name[IPSET_MAXNAMELEN]; /* Lock protecting the set data */ @@ -273,7 +273,7 @@ struct ip_set { /* Number of elements (vs timeout) */ u32 elements; /* Size of the dynamic extensions (vs timeout) */ - size_t ext_size; + atomic64_t ext_size; /* Element data size */ size_t dsize; /* Offsets to extensions in elements */ @@ -309,7 +309,7 @@ enum { /* register and unregister set references */ extern ip_set_id_t ip_set_get_byname(struct net *net, - const char *name, struct ip_set **set); + const struct nlattr *name, struct ip_set **set); extern void ip_set_put_byindex(struct net *net, ip_set_id_t index); extern void ip_set_name_byindex(struct net *net, ip_set_id_t index, char *name); extern ip_set_id_t ip_set_nfnl_get_byindex(struct net *net, ip_set_id_t index); diff --git a/include/linux/netfilter/nf_conntrack_amanda.h b/include/linux/netfilter/nf_conntrack_amanda.h index dfe89f38d1f7..deb560bb79c4 100644 --- a/include/linux/netfilter/nf_conntrack_amanda.h +++ b/include/linux/netfilter/nf_conntrack_amanda.h @@ -7,10 +7,14 @@ #include <linux/skbuff.h> #include <net/netfilter/nf_conntrack_expect.h> -extern unsigned int (__rcu *nf_nat_amanda_hook)(struct sk_buff *skb, - enum ip_conntrack_info ctinfo, - unsigned int protoff, - unsigned int matchoff, - unsigned int matchlen, - struct nf_conntrack_expect *exp); +typedef unsigned int +nf_nat_amanda_hook_fn(struct sk_buff *skb, + struct nf_conn *ct, + enum ip_conntrack_info ctinfo, + unsigned int protoff, + unsigned int matchoff, + unsigned int matchlen, + struct nf_conntrack_expect *exp); + +extern nf_nat_amanda_hook_fn __rcu *nf_nat_amanda_hook; #endif /* _NF_CONNTRACK_AMANDA_H */ diff --git a/include/linux/netfilter/nf_conntrack_ftp.h b/include/linux/netfilter/nf_conntrack_ftp.h index f31292642035..712702183b94 100644 --- a/include/linux/netfilter/nf_conntrack_ftp.h +++ b/include/linux/netfilter/nf_conntrack_ftp.h @@ -26,11 +26,15 @@ struct nf_ct_ftp_master { /* For NAT to hook in when we find a packet which describes what other * connection we should expect. */ -extern unsigned int (__rcu *nf_nat_ftp_hook)(struct sk_buff *skb, - enum ip_conntrack_info ctinfo, - enum nf_ct_ftp_type type, - unsigned int protoff, - unsigned int matchoff, - unsigned int matchlen, - struct nf_conntrack_expect *exp); +typedef unsigned int +nf_nat_ftp_hook_fn(struct sk_buff *skb, + struct nf_conn *ct, + enum ip_conntrack_info ctinfo, + enum nf_ct_ftp_type type, + unsigned int protoff, + unsigned int matchoff, + unsigned int matchlen, + struct nf_conntrack_expect *exp); + +extern nf_nat_ftp_hook_fn __rcu *nf_nat_ftp_hook; #endif /* _NF_CONNTRACK_FTP_H */ diff --git a/include/linux/netfilter/nf_conntrack_h323.h b/include/linux/netfilter/nf_conntrack_h323.h index 81286c499325..b15f37604cde 100644 --- a/include/linux/netfilter/nf_conntrack_h323.h +++ b/include/linux/netfilter/nf_conntrack_h323.h @@ -9,8 +9,6 @@ #include <net/netfilter/nf_conntrack_expect.h> #include <uapi/linux/netfilter/nf_conntrack_tuple_common.h> -#define RAS_PORT 1719 -#define Q931_PORT 1720 #define H323_RTP_CHANNEL_MAX 4 /* Audio, video, FAX and other */ /* This structure exists only once per master */ diff --git a/include/linux/netfilter/nf_conntrack_irc.h b/include/linux/netfilter/nf_conntrack_irc.h index 4f3ca5621998..c73b3b44a0b7 100644 --- a/include/linux/netfilter/nf_conntrack_irc.h +++ b/include/linux/netfilter/nf_conntrack_irc.h @@ -8,11 +8,15 @@ #define IRC_PORT 6667 -extern unsigned int (__rcu *nf_nat_irc_hook)(struct sk_buff *skb, - enum ip_conntrack_info ctinfo, - unsigned int protoff, - unsigned int matchoff, - unsigned int matchlen, - struct nf_conntrack_expect *exp); +typedef unsigned int +nf_nat_irc_hook_fn(struct sk_buff *skb, + struct nf_conn *ct, + enum ip_conntrack_info ctinfo, + unsigned int protoff, + unsigned int matchoff, + unsigned int matchlen, + struct nf_conntrack_expect *exp); + +extern nf_nat_irc_hook_fn __rcu *nf_nat_irc_hook; #endif /* _NF_CONNTRACK_IRC_H */ diff --git a/include/linux/netfilter/nf_conntrack_pptp.h b/include/linux/netfilter/nf_conntrack_pptp.h index c3bdb4370938..c0b305ce7c3c 100644 --- a/include/linux/netfilter/nf_conntrack_pptp.h +++ b/include/linux/netfilter/nf_conntrack_pptp.h @@ -50,8 +50,6 @@ struct nf_nat_pptp { __be16 pac_call_id; /* NAT'ed PAC call id */ }; -#define PPTP_CONTROL_PORT 1723 - #define PPTP_PACKET_CONTROL 1 #define PPTP_PACKET_MGMT 2 diff --git a/include/linux/netfilter/nf_conntrack_proto_gre.h b/include/linux/netfilter/nf_conntrack_proto_gre.h index 9ee7014400e8..ad5563f0f864 100644 --- a/include/linux/netfilter/nf_conntrack_proto_gre.h +++ b/include/linux/netfilter/nf_conntrack_proto_gre.h @@ -18,9 +18,10 @@ struct nf_ct_gre_keymap { struct rcu_head rcu; }; -/* add new tuple->key_reply pair to keymap */ -int nf_ct_gre_keymap_add(struct nf_conn *ct, enum ip_conntrack_dir dir, - struct nf_conntrack_tuple *t); +/* add tuple->key_reply pairs to keymap */ +bool nf_ct_gre_keymap_add(struct nf_conn *ct, + const struct nf_conntrack_tuple *orig, + const struct nf_conntrack_tuple *repl); /* delete keymap entries */ void nf_ct_gre_keymap_destroy(struct nf_conn *ct); diff --git a/include/linux/netfilter/nf_conntrack_sane.h b/include/linux/netfilter/nf_conntrack_sane.h index 46c7acd1b4a7..8501035d7335 100644 --- a/include/linux/netfilter/nf_conntrack_sane.h +++ b/include/linux/netfilter/nf_conntrack_sane.h @@ -3,8 +3,6 @@ #define _NF_CONNTRACK_SANE_H /* SANE tracking. */ -#define SANE_PORT 6566 - enum sane_state { SANE_STATE_NORMAL, SANE_STATE_START_REQUESTED, diff --git a/include/linux/netfilter/nf_conntrack_sip.h b/include/linux/netfilter/nf_conntrack_sip.h index dbc614dfe0d5..aafa0c04f917 100644 --- a/include/linux/netfilter/nf_conntrack_sip.h +++ b/include/linux/netfilter/nf_conntrack_sip.h @@ -115,7 +115,7 @@ struct nf_nat_sip_hooks { unsigned int *datalen); void (*seq_adjust)(struct sk_buff *skb, - unsigned int protoff, s16 off); + unsigned int protoff, s32 off); unsigned int (*expect)(struct sk_buff *skb, unsigned int protoff, diff --git a/include/linux/netfilter/nf_conntrack_snmp.h b/include/linux/netfilter/nf_conntrack_snmp.h index 99107e4f5234..bb39f04a9977 100644 --- a/include/linux/netfilter/nf_conntrack_snmp.h +++ b/include/linux/netfilter/nf_conntrack_snmp.h @@ -5,9 +5,12 @@ #include <linux/netfilter.h> #include <linux/skbuff.h> -extern int (__rcu *nf_nat_snmp_hook)(struct sk_buff *skb, - unsigned int protoff, - struct nf_conn *ct, - enum ip_conntrack_info ctinfo); +typedef int +nf_nat_snmp_hook_fn(struct sk_buff *skb, + unsigned int protoff, + struct nf_conn *ct, + enum ip_conntrack_info ctinfo); + +extern nf_nat_snmp_hook_fn __rcu *nf_nat_snmp_hook; #endif /* _NF_CONNTRACK_SNMP_H */ diff --git a/include/linux/netfilter/nf_conntrack_tftp.h b/include/linux/netfilter/nf_conntrack_tftp.h index 1490b68dd7d1..802cb7fc19cd 100644 --- a/include/linux/netfilter/nf_conntrack_tftp.h +++ b/include/linux/netfilter/nf_conntrack_tftp.h @@ -2,8 +2,6 @@ #ifndef _NF_CONNTRACK_TFTP_H #define _NF_CONNTRACK_TFTP_H -#define TFTP_PORT 69 - #include <linux/netfilter.h> #include <linux/skbuff.h> #include <linux/types.h> @@ -19,8 +17,12 @@ struct tftphdr { #define TFTP_OPCODE_ACK 4 #define TFTP_OPCODE_ERROR 5 -extern unsigned int (__rcu *nf_nat_tftp_hook)(struct sk_buff *skb, - enum ip_conntrack_info ctinfo, - struct nf_conntrack_expect *exp); +typedef unsigned int +nf_nat_tftp_hook_fn(struct sk_buff *skb, + struct nf_conn *ct, + enum ip_conntrack_info ctinfo, + struct nf_conntrack_expect *exp); + +extern nf_nat_tftp_hook_fn __rcu *nf_nat_tftp_hook; #endif /* _NF_CONNTRACK_TFTP_H */ diff --git a/include/linux/netfilter/x_tables.h b/include/linux/netfilter/x_tables.h index 77c778d84d4c..25062f4a0dd5 100644 --- a/include/linux/netfilter/x_tables.h +++ b/include/linux/netfilter/x_tables.h @@ -18,7 +18,7 @@ * @match: the match extension * @target: the target extension * @matchinfo: per-match data - * @targetinfo: per-target data + * @targinfo: per-target data * @state: pointer to hook state this packet came from * @fragoff: packet is a fragment, this is the data offset * @thoff: position of transport header relative to skb->data @@ -77,7 +77,9 @@ static inline u_int8_t xt_family(const struct xt_action_param *par) * @match: struct xt_match through which this function was invoked * @matchinfo: per-match data * @hook_mask: via which hooks the new rule is reachable - * Other fields as above. + * @family: actual NFPROTO_* through which the function is invoked + * (helpful when match->family == NFPROTO_UNSPEC) + * @nft_compat: running from the nft compat layer if true */ struct xt_mtchk_param { struct net *net; @@ -91,8 +93,13 @@ struct xt_mtchk_param { }; /** - * struct xt_mdtor_param - match destructor parameters - * Fields as above. + * struct xt_mtdtor_param - match destructor parameters + * + * @net: network namespace through which the check was invoked + * @match: struct xt_match through which this function was invoked + * @matchinfo: per-match data + * @family: actual NFPROTO_* through which the function is invoked + * (helpful when match->family == NFPROTO_UNSPEC) */ struct xt_mtdtor_param { struct net *net; @@ -105,10 +112,16 @@ struct xt_mtdtor_param { * struct xt_tgchk_param - parameters for target extensions' * checkentry functions * + * @net: network namespace through which the check was invoked + * @table: table the rule is tried to be inserted into * @entryinfo: the family-specific rule data * (struct ipt_entry, ip6t_entry, arpt_entry, ebt_entry) - * - * Other fields see above. + * @target: the target extension + * @targinfo: per-target data + * @hook_mask: via which hooks the new rule is reachable + * @family: actual NFPROTO_* through which the function is invoked + * (helpful when match->family == NFPROTO_UNSPEC) + * @nft_compat: running from the nft compat layer if true */ struct xt_tgchk_param { struct net *net; @@ -146,6 +159,9 @@ struct xt_match { /* Called when user tries to insert an entry of this type. */ int (*checkentry)(const struct xt_mtchk_param *); + /* Called to validate hooks based on the match configuration. */ + int (*check_hooks)(const struct xt_mtchk_param *); + /* Called when entry of this type deleted. */ void (*destroy)(const struct xt_mtdtor_param *); #ifdef CONFIG_NETFILTER_XTABLES_COMPAT @@ -187,6 +203,9 @@ struct xt_target { /* Should return 0 on success or an error code otherwise (-Exxxx). */ int (*checkentry)(const struct xt_tgchk_param *); + /* Called to validate hooks based on the target configuration. */ + int (*check_hooks)(const struct xt_tgchk_param *); + /* Called when entry of this type deleted. */ void (*destroy)(const struct xt_tgdtor_param *); #ifdef CONFIG_NETFILTER_XTABLES_COMPAT @@ -279,8 +298,10 @@ bool xt_find_jump_offset(const unsigned int *offsets, int xt_check_proc_name(const char *name, unsigned int size); +int xt_check_hooks_match(struct xt_mtchk_param *par); int xt_check_match(struct xt_mtchk_param *, unsigned int size, u16 proto, bool inv_proto); +int xt_check_hooks_target(struct xt_tgchk_param *par); int xt_check_target(struct xt_tgchk_param *, unsigned int size, u16 proto, bool inv_proto); @@ -297,9 +318,11 @@ struct xt_counters *xt_counters_alloc(unsigned int counters); struct xt_table *xt_register_table(struct net *net, const struct xt_table *table, + const struct nf_hook_ops *template_ops, struct xt_table_info *bootstrap, struct xt_table_info *newinfo); -void *xt_unregister_table(struct xt_table *table); +void xt_unregister_table_pre_exit(struct net *net, u8 af, const char *name); +struct xt_table *xt_unregister_table_exit(struct net *net, u8 af, const char *name); struct xt_table_info *xt_replace_table(struct xt_table *table, unsigned int num_counters, @@ -326,9 +349,9 @@ struct xt_table_info *xt_alloc_table_info(unsigned int size); void xt_free_table_info(struct xt_table_info *info); /** - * xt_recseq - recursive seqcount for netfilter use + * var xt_recseq - recursive seqcount for netfilter use * - * Packet processing changes the seqcount only if no recursion happened + * Packet processing changes the seqcount only if no recursion happened. * get_counters() can use read_seqcount_begin()/read_seqcount_retry(), * because we use the normal seqcount convention : * Low order bit set to 1 if a writer is active. @@ -524,4 +547,21 @@ int xt_compat_check_entry_offsets(const void *base, const char *elems, unsigned int next_offset); #endif /* CONFIG_NETFILTER_XTABLES_COMPAT */ + +static inline bool xt_compat_check(void) +{ +#ifdef CONFIG_NETFILTER_XTABLES_COMPAT + if (!in_compat_syscall()) + return true; + + pr_warn_once("%s %s\n", + "xtables 32bit compat interface no longer supported", + "in namespaces and will be removed soon."); + + if (!capable(CAP_NET_ADMIN)) + return false; +#endif + return true; +} + #endif /* _X_TABLES_H */ diff --git a/include/linux/netfilter_arp/arp_tables.h b/include/linux/netfilter_arp/arp_tables.h index a40aaf645fa4..8b8d472eff34 100644 --- a/include/linux/netfilter_arp/arp_tables.h +++ b/include/linux/netfilter_arp/arp_tables.h @@ -53,27 +53,7 @@ int arpt_register_table(struct net *net, const struct xt_table *table, const struct arpt_replace *repl, const struct nf_hook_ops *ops); void arpt_unregister_table(struct net *net, const char *name); -void arpt_unregister_table_pre_exit(struct net *net, const char *name); extern unsigned int arpt_do_table(void *priv, struct sk_buff *skb, const struct nf_hook_state *state); -#ifdef CONFIG_NETFILTER_XTABLES_COMPAT -#include <net/compat.h> - -struct compat_arpt_entry { - struct arpt_arp arp; - __u16 target_offset; - __u16 next_offset; - compat_uint_t comefrom; - struct compat_xt_counters counters; - unsigned char elems[]; -}; - -static inline struct xt_entry_target * -compat_arpt_get_target(struct compat_arpt_entry *e) -{ - return (void *)e + e->target_offset; -} - -#endif /* CONFIG_COMPAT */ #endif /* _ARPTABLES_H */ diff --git a/include/linux/netfilter_ipv4/ip_tables.h b/include/linux/netfilter_ipv4/ip_tables.h index 132b0e4a6d4d..13593391d605 100644 --- a/include/linux/netfilter_ipv4/ip_tables.h +++ b/include/linux/netfilter_ipv4/ip_tables.h @@ -26,7 +26,6 @@ int ipt_register_table(struct net *net, const struct xt_table *table, const struct ipt_replace *repl, const struct nf_hook_ops *ops); -void ipt_unregister_table_pre_exit(struct net *net, const char *name); void ipt_unregister_table_exit(struct net *net, const char *name); /* Standard entry. */ diff --git a/include/linux/netfilter_ipv6.h b/include/linux/netfilter_ipv6.h index 61aa48f46dd7..5ce45b6d890f 100644 --- a/include/linux/netfilter_ipv6.h +++ b/include/linux/netfilter_ipv6.h @@ -34,59 +34,13 @@ struct ip6_rt_info { struct nf_queue_entry; struct nf_bridge_frag_data; -/* - * Hook functions for ipv6 to allow xt_* modules to be built-in even - * if IPv6 is a module. - */ -struct nf_ipv6_ops { -#if IS_MODULE(CONFIG_IPV6) - int (*chk_addr)(struct net *net, const struct in6_addr *addr, - const struct net_device *dev, int strict); - int (*route_me_harder)(struct net *net, struct sock *sk, struct sk_buff *skb); - int (*dev_get_saddr)(struct net *net, const struct net_device *dev, - const struct in6_addr *daddr, unsigned int srcprefs, - struct in6_addr *saddr); - int (*route)(struct net *net, struct dst_entry **dst, struct flowi *fl, - bool strict); - u32 (*cookie_init_sequence)(const struct ipv6hdr *iph, - const struct tcphdr *th, u16 *mssp); - int (*cookie_v6_check)(const struct ipv6hdr *iph, - const struct tcphdr *th); -#endif - void (*route_input)(struct sk_buff *skb); - int (*fragment)(struct net *net, struct sock *sk, struct sk_buff *skb, - int (*output)(struct net *, struct sock *, struct sk_buff *)); - int (*reroute)(struct sk_buff *skb, const struct nf_queue_entry *entry); -#if IS_MODULE(CONFIG_IPV6) - int (*br_fragment)(struct net *net, struct sock *sk, - struct sk_buff *skb, - struct nf_bridge_frag_data *data, - int (*output)(struct net *, struct sock *sk, - const struct nf_bridge_frag_data *data, - struct sk_buff *)); -#endif -}; - #ifdef CONFIG_NETFILTER #include <net/addrconf.h> -extern const struct nf_ipv6_ops __rcu *nf_ipv6_ops; -static inline const struct nf_ipv6_ops *nf_get_ipv6_ops(void) -{ - return rcu_dereference(nf_ipv6_ops); -} - static inline int nf_ipv6_chk_addr(struct net *net, const struct in6_addr *addr, const struct net_device *dev, int strict) { -#if IS_MODULE(CONFIG_IPV6) - const struct nf_ipv6_ops *v6_ops = nf_get_ipv6_ops(); - - if (!v6_ops) - return 1; - - return v6_ops->chk_addr(net, addr, dev, strict); -#elif IS_BUILTIN(CONFIG_IPV6) +#if IS_ENABLED(CONFIG_IPV6) return ipv6_chk_addr(net, addr, dev, strict); #else return 1; @@ -99,15 +53,7 @@ int __nf_ip6_route(struct net *net, struct dst_entry **dst, static inline int nf_ip6_route(struct net *net, struct dst_entry **dst, struct flowi *fl, bool strict) { -#if IS_MODULE(CONFIG_IPV6) - const struct nf_ipv6_ops *v6ops = nf_get_ipv6_ops(); - - if (v6ops) - return v6ops->route(net, dst, fl, strict); - - return -EHOSTUNREACH; -#endif -#if IS_BUILTIN(CONFIG_IPV6) +#if IS_ENABLED(CONFIG_IPV6) return __nf_ip6_route(net, dst, fl, strict); #else return -EHOSTUNREACH; @@ -129,14 +75,7 @@ static inline int nf_br_ip6_fragment(struct net *net, struct sock *sk, const struct nf_bridge_frag_data *data, struct sk_buff *)) { -#if IS_MODULE(CONFIG_IPV6) - const struct nf_ipv6_ops *v6_ops = nf_get_ipv6_ops(); - - if (!v6_ops) - return 1; - - return v6_ops->br_fragment(net, sk, skb, data, output); -#elif IS_BUILTIN(CONFIG_IPV6) +#if IS_ENABLED(CONFIG_IPV6) return br_ip6_fragment(net, sk, skb, data, output); #else return 1; @@ -147,14 +86,7 @@ int ip6_route_me_harder(struct net *net, struct sock *sk, struct sk_buff *skb); static inline int nf_ip6_route_me_harder(struct net *net, struct sock *sk, struct sk_buff *skb) { -#if IS_MODULE(CONFIG_IPV6) - const struct nf_ipv6_ops *v6_ops = nf_get_ipv6_ops(); - - if (!v6_ops) - return -EHOSTUNREACH; - - return v6_ops->route_me_harder(net, sk, skb); -#elif IS_BUILTIN(CONFIG_IPV6) +#if IS_ENABLED(CONFIG_IPV6) return ip6_route_me_harder(net, sk, skb); #else return -EHOSTUNREACH; @@ -165,32 +97,18 @@ static inline u32 nf_ipv6_cookie_init_sequence(const struct ipv6hdr *iph, const struct tcphdr *th, u16 *mssp) { -#if IS_ENABLED(CONFIG_SYN_COOKIES) -#if IS_MODULE(CONFIG_IPV6) - const struct nf_ipv6_ops *v6_ops = nf_get_ipv6_ops(); - - if (v6_ops) - return v6_ops->cookie_init_sequence(iph, th, mssp); -#elif IS_BUILTIN(CONFIG_IPV6) +#if IS_ENABLED(CONFIG_IPV6) && IS_ENABLED(CONFIG_SYN_COOKIES) return __cookie_v6_init_sequence(iph, th, mssp); #endif -#endif return 0; } static inline int nf_cookie_v6_check(const struct ipv6hdr *iph, const struct tcphdr *th) { -#if IS_ENABLED(CONFIG_SYN_COOKIES) -#if IS_MODULE(CONFIG_IPV6) - const struct nf_ipv6_ops *v6_ops = nf_get_ipv6_ops(); - - if (v6_ops) - return v6_ops->cookie_v6_check(iph, th); -#elif IS_BUILTIN(CONFIG_IPV6) +#if IS_ENABLED(CONFIG_IPV6) && IS_ENABLED(CONFIG_SYN_COOKIES) return __cookie_v6_check(iph, th); #endif -#endif return 0; } @@ -198,14 +116,6 @@ __sum16 nf_ip6_checksum(struct sk_buff *skb, unsigned int hook, unsigned int dataoff, u_int8_t protocol); int nf_ip6_check_hbh_len(struct sk_buff *skb, u32 *plen); - -int ipv6_netfilter_init(void); -void ipv6_netfilter_fini(void); - -#else /* CONFIG_NETFILTER */ -static inline int ipv6_netfilter_init(void) { return 0; } -static inline void ipv6_netfilter_fini(void) { return; } -static inline const struct nf_ipv6_ops *nf_get_ipv6_ops(void) { return NULL; } #endif /* CONFIG_NETFILTER */ #endif /*__LINUX_IP6_NETFILTER_H*/ diff --git a/include/linux/netfilter_ipv6/ip6_tables.h b/include/linux/netfilter_ipv6/ip6_tables.h index 8b8885a73c76..c6d5b927830d 100644 --- a/include/linux/netfilter_ipv6/ip6_tables.h +++ b/include/linux/netfilter_ipv6/ip6_tables.h @@ -27,7 +27,6 @@ extern void *ip6t_alloc_initial_table(const struct xt_table *); int ip6t_register_table(struct net *net, const struct xt_table *table, const struct ip6t_replace *repl, const struct nf_hook_ops *ops); -void ip6t_unregister_table_pre_exit(struct net *net, const char *name); void ip6t_unregister_table_exit(struct net *net, const char *name); extern unsigned int ip6t_do_table(void *priv, struct sk_buff *skb, const struct nf_hook_state *state); diff --git a/include/linux/netfs.h b/include/linux/netfs.h index 72ee7d210a74..b4dd32863dd4 100644 --- a/include/linux/netfs.h +++ b/include/linux/netfs.h @@ -61,14 +61,16 @@ struct netfs_inode { #if IS_ENABLED(CONFIG_FSCACHE) struct fscache_cookie *cache; #endif - struct mutex wb_lock; /* Writeback serialisation */ - loff_t remote_i_size; /* Size of the remote file */ - loff_t zero_point; /* Size after which we assume there's no data + struct list_head wb_queue; /* Queue of processes wanting to do writeback */ + loff_t _remote_i_size; /* Size of the remote file */ + loff_t _zero_point; /* Size after which we assume there's no data * on the server */ + spinlock_t lock; /* Lock covering wb_queue */ atomic_t io_count; /* Number of outstanding reqs */ unsigned long flags; #define NETFS_ICTX_ODIRECT 0 /* The file has DIO in progress */ #define NETFS_ICTX_UNBUFFERED 1 /* I/O should not use the pagecache */ +#define NETFS_ICTX_WB_LOCK 2 /* Writeback serialisation lock */ #define NETFS_ICTX_MODIFIED_ATTR 3 /* Indicate change in mtime/ctime */ #define NETFS_ICTX_SINGLE_NO_UPLOAD 4 /* Monolithic payload, cache but no upload */ }; @@ -140,7 +142,6 @@ struct netfs_io_stream { void (*issue_write)(struct netfs_io_subrequest *subreq); /* Collection tracking */ struct list_head subrequests; /* Contributory I/O operations */ - struct netfs_io_subrequest *front; /* Op being collected */ unsigned long long collected_to; /* Position we've collected results to */ size_t transferred; /* The amount transferred from this stream */ unsigned short error; /* Aggregate error for the stream */ @@ -190,7 +191,6 @@ struct netfs_io_subrequest { #define NETFS_SREQ_COPY_TO_CACHE 0 /* Set if should copy the data to the cache */ #define NETFS_SREQ_CLEAR_TAIL 1 /* Set if the rest of the read should be cleared */ #define NETFS_SREQ_MADE_PROGRESS 4 /* Set if we transferred at least some data */ -#define NETFS_SREQ_ONDEMAND 5 /* Set if it's from on-demand read mode */ #define NETFS_SREQ_BOUNDARY 6 /* Set if ends on hard boundary (eg. ceph object) */ #define NETFS_SREQ_HIT_EOF 7 /* Set if short due to EOF */ #define NETFS_SREQ_IN_PROGRESS 8 /* Unlocked when the subrequest completes */ @@ -246,6 +246,7 @@ struct netfs_io_request { unsigned long long submitted; /* Amount submitted for I/O so far */ unsigned long long len; /* Length of the request */ size_t transferred; /* Amount to be indicated as transferred */ + size_t progress_at; /* Report read progress when hit this much read */ long error; /* 0 or error that occurred */ unsigned long long i_size; /* Size of the file */ unsigned long long start; /* Start position */ @@ -253,7 +254,8 @@ struct netfs_io_request { unsigned long long collected_to; /* Point we've collected to */ unsigned long long cleaned_to; /* Position we've cleaned folios to */ unsigned long long abandon_to; /* Position to abandon folios to */ - pgoff_t no_unlock_folio; /* Don't unlock this folio after read */ + const struct folio *no_unlock_folio; /* Don't unlock this folio after read */ + gfp_t gfp; /* GFP flags to use */ unsigned int direct_bv_count; /* Number of elements in direct_bv[] */ unsigned int debug_id; unsigned int rsize; /* Maximum read size (0 for none) */ @@ -261,7 +263,6 @@ struct netfs_io_request { atomic_t subreq_counter; /* Next subreq->debug_index */ unsigned int nr_group_rel; /* Number of refs to release on ->group */ spinlock_t lock; /* Lock for queuing subreqs */ - unsigned char front_folio_order; /* Order (size) of front folio */ enum netfs_io_origin origin; /* Origin of the request */ bool direct_bv_unpin; /* T if direct_bv[] must be unpinned */ refcount_t ref; @@ -274,9 +275,10 @@ struct netfs_io_request { #define NETFS_RREQ_SHORT_TRANSFER 5 /* Set if we have a short transfer */ #define NETFS_RREQ_OFFLOAD_COLLECTION 8 /* Offload collection to workqueue */ #define NETFS_RREQ_NO_UNLOCK_FOLIO 9 /* Don't unlock no_unlock_folio on completion */ -#define NETFS_RREQ_FOLIO_COPY_TO_CACHE 10 /* Copy current folio to cache from read */ +#define NETFS_RREQ_CANCEL_CACHING 10 /* Set to cancel caching */ #define NETFS_RREQ_UPLOAD_TO_SERVER 11 /* Need to write to the server */ #define NETFS_RREQ_USE_IO_ITER 12 /* Use ->io_iter rather than ->i_pages */ +#define NETFS_RREQ_NEED_PUT_RA_REFS 17 /* Need to put the folio refs RA gave us */ #define NETFS_RREQ_USE_PGPRIV2 31 /* [DEPRECATED] Use PG_private_2 to mark * write to cache on read */ const struct netfs_request_ops *netfs_ops; @@ -372,14 +374,6 @@ struct netfs_cache_ops { loff_t *_start, size_t *_len, size_t upper_len, loff_t i_size, bool no_space_allocated_yet); - /* Prepare an on-demand read operation, shortening it to a cached/uncached - * boundary as appropriate. - */ - enum netfs_io_source (*prepare_ondemand_read)(struct netfs_cache_resources *cres, - loff_t start, size_t *_len, - loff_t i_size, - unsigned long *_flags, ino_t ino); - /* Query the occupancy of the cache in a region, returning where the * next chunk of data starts and how long it is. */ @@ -463,6 +457,10 @@ int netfs_alloc_folioq_buffer(struct address_space *mapping, size_t *_cur_size, ssize_t size, gfp_t gfp); void netfs_free_folioq_buffer(struct folio_queue *fq); +/* Writeback exclusion API. */ +bool netfs_wb_begin(struct netfs_inode *ictx, bool nowait); +void netfs_wb_end(struct netfs_inode *ictx); + /** * netfs_inode - Get the netfs inode context from the inode * @inode: The inode to query @@ -476,6 +474,254 @@ static inline struct netfs_inode *netfs_inode(struct inode *inode) } /** + * netfs_read_remote_i_size - Read remote_i_size safely + * @inode: The inode to access + * + * Read remote_i_size safely without the potential for tearing on 32-bit + * arches. + * + * NOTE: in a 32bit arch with a preemptable kernel and an UP compile the + * i_size_read/write must be atomic with respect to the local cpu (unlike with + * preempt disabled), but they don't need to be atomic with respect to other + * cpus like in true SMP (so they need either to either locally disable irq + * around the read or for example on x86 they can be still implemented as a + * cmpxchg8b without the need of the lock prefix). For SMP compiles and 64bit + * archs it makes no difference if preempt is enabled or not. + */ +static inline unsigned long long netfs_read_remote_i_size(const struct inode *inode) +{ + const struct netfs_inode *ictx = container_of(inode, struct netfs_inode, inode); + unsigned long long remote_i_size; + +#if BITS_PER_LONG==32 && defined(CONFIG_SMP) + unsigned int seq; + + do { + seq = read_seqcount_begin(&inode->i_size_seqcount); + remote_i_size = ictx->_remote_i_size; + } while (read_seqcount_retry(&inode->i_size_seqcount, seq)); +#elif BITS_PER_LONG==32 && defined(CONFIG_PREEMPTION) + preempt_disable(); + remote_i_size = ictx->_remote_i_size; + preempt_enable(); +#else + /* Pairs with smp_store_release() in netfs_write_remote_i_size() */ + remote_i_size = smp_load_acquire(&ictx->_remote_i_size); +#endif + return remote_i_size; +} + +/* + * netfs_write_remote_i_size - Set remote_i_size safely + * @inode: The inode to access + * @remote_i_size: The new value for the size of the file on the server + * + * Set remote_i_size safely without the potential for tearing on 32-bit arches. + * + * Context: The caller must hold inode->i_lock. + * + * NOTE: unlike netfs_read_remote_i_size(), netfs_write_remote_i_size() does + * need locking around it (normally i_rwsem), otherwise on 32bit/SMP an update + * of i_size_seqcount can be lost, resulting in subsequent i_size_read() calls + * spinning forever. + */ +static inline void netfs_write_remote_i_size(struct inode *inode, + unsigned long long remote_i_size) +{ + struct netfs_inode *ictx = netfs_inode(inode); + +#if BITS_PER_LONG==32 && defined(CONFIG_SMP) + write_seqcount_begin(&inode->i_size_seqcount); + ictx->_remote_i_size = remote_i_size; + write_seqcount_end(&inode->i_size_seqcount); +#elif BITS_PER_LONG==32 && defined(CONFIG_PREEMPTION) + preempt_disable(); + ictx->_remote_i_size = remote_i_size; + preempt_enable(); +#else + /* + * Pairs with smp_load_acquire() in netfs_read_remote_i_size() to + * ensure changes related to inode size (such as page contents) are + * visible before we see the changed inode size. + */ + smp_store_release(&ictx->_remote_i_size, remote_i_size); +#endif +} + +/** + * netfs_read_zero_point - Read zero_point safely + * @inode: The inode to access + * + * Read zero_point safely without the potential for tearing on 32-bit + * arches. + * + * NOTE: in a 32bit arch with a preemptable kernel and an UP compile the + * i_size_read/write must be atomic with respect to the local cpu (unlike with + * preempt disabled), but they don't need to be atomic with respect to other + * cpus like in true SMP (so they need either to either locally disable irq + * around the read or for example on x86 they can be still implemented as a + * cmpxchg8b without the need of the lock prefix). For SMP compiles and 64bit + * archs it makes no difference if preempt is enabled or not. + */ +static inline unsigned long long netfs_read_zero_point(const struct inode *inode) +{ + struct netfs_inode *ictx = container_of(inode, struct netfs_inode, inode); + unsigned long long zero_point; + +#if BITS_PER_LONG==32 && defined(CONFIG_SMP) + unsigned int seq; + + do { + seq = read_seqcount_begin(&inode->i_size_seqcount); + zero_point = ictx->_zero_point; + } while (read_seqcount_retry(&inode->i_size_seqcount, seq)); +#elif BITS_PER_LONG==32 && defined(CONFIG_PREEMPTION) + preempt_disable(); + zero_point = ictx->_zero_point; + preempt_enable(); +#else + /* Pairs with smp_store_release() in netfs_write_zero_point() */ + zero_point = smp_load_acquire(&ictx->_zero_point); +#endif + return zero_point; +} + +/* + * netfs_write_zero_point - Set zero_point safely + * @inode: The inode to access + * @zero_point: The new value for the point beyond which the server has no data + * + * Set zero_point safely without the potential for tearing on 32-bit arches. + * + * Context: The caller must hold inode->i_lock. + * + * NOTE: unlike netfs_read_zero_point(), netfs_write_zero_point() does need + * locking around it (normally i_rwsem), otherwise on 32bit/SMP an update of + * i_size_seqcount can be lost, resulting in subsequent read calls spinning + * forever. + */ +static inline void netfs_write_zero_point(struct inode *inode, + unsigned long long zero_point) +{ + struct netfs_inode *ictx = netfs_inode(inode); + +#if BITS_PER_LONG==32 && defined(CONFIG_SMP) + write_seqcount_begin(&inode->i_size_seqcount); + ictx->_zero_point = zero_point; + write_seqcount_end(&inode->i_size_seqcount); +#elif BITS_PER_LONG==32 && defined(CONFIG_PREEMPTION) + preempt_disable(); + ictx->_zero_point = zero_point; + preempt_enable(); +#else + /* + * Pairs with smp_load_acquire() in netfs_read_zero_point() to + * ensure changes related to inode size (such as page contents) are + * visible before we see the changed inode size. + */ + smp_store_release(&ictx->_zero_point, zero_point); +#endif +} + +/** + * netfs_read_sizes - Read remote_i_size and zero_point safely + * @inode: The inode to access + * @i_size: Where to return the local file size. + * @remote_i_size: Where to return the size of the file on the server + * @zero_point: Where to return the the point beyond which the server has no data + * + * Read remote_i_size and zero_point safely without the potential for tearing + * on 32-bit arches. + * + * NOTE: in a 32bit arch with a preemptable kernel and an UP compile the + * i_size_read/write must be atomic with respect to the local cpu (unlike with + * preempt disabled), but they don't need to be atomic with respect to other + * cpus like in true SMP (so they need either to either locally disable irq + * around the read or for example on x86 they can be still implemented as a + * cmpxchg8b without the need of the lock prefix). For SMP compiles and 64bit + * archs it makes no difference if preempt is enabled or not. + */ +static inline void netfs_read_sizes(const struct inode *inode, + unsigned long long *i_size, + unsigned long long *remote_i_size, + unsigned long long *zero_point) +{ + const struct netfs_inode *ictx = container_of(inode, struct netfs_inode, inode); +#if BITS_PER_LONG==32 && defined(CONFIG_SMP) + unsigned int seq; + + do { + seq = read_seqcount_begin(&inode->i_size_seqcount); + *i_size = inode->i_size; + *remote_i_size = ictx->_remote_i_size; + *zero_point = ictx->_zero_point; + } while (read_seqcount_retry(&inode->i_size_seqcount, seq)); +#elif BITS_PER_LONG==32 && defined(CONFIG_PREEMPTION) + preempt_disable(); + *i_size = inode->i_size; + *remote_i_size = ictx->_remote_i_size; + *zero_point = ictx->_zero_point; + preempt_enable(); +#else + /* Pairs with smp_store_release() in i_size_write() */ + *i_size = smp_load_acquire(&inode->i_size); + /* Pairs with smp_store_release() in netfs_write_remote_i_size() */ + *remote_i_size = smp_load_acquire(&ictx->_remote_i_size); + /* Pairs with smp_store_release() in netfs_write_zero_point() */ + *zero_point = smp_load_acquire(&ictx->_zero_point); +#endif +} + +/* + * netfs_write_sizes - Set i_size, remote_i_size and zero_point safely + * @inode: The inode to access + * @i_size: The new value for the local size of the file + * @remote_i_size: The new value for the size of the file on the server + * @zero_point: The new value for the point beyond which the server has no data + * + * Set both remote_i_size and zero_point safely without the potential for + * tearing on 32-bit arches. + * + * Context: The caller must hold inode->i_lock. + * + * NOTE: unlike netfs_read_zero_point(), netfs_write_zero_point() does need + * locking around it (normally i_rwsem), otherwise on 32bit/SMP an update of + * i_size_seqcount can be lost, resulting in subsequent read calls spinning + * forever. + */ +static inline void netfs_write_sizes(struct inode *inode, + unsigned long long i_size, + unsigned long long remote_i_size, + unsigned long long zero_point) +{ + struct netfs_inode *ictx = netfs_inode(inode); + +#if BITS_PER_LONG==32 && defined(CONFIG_SMP) + write_seqcount_begin(&inode->i_size_seqcount); + inode->i_size = i_size; + ictx->_remote_i_size = remote_i_size; + ictx->_zero_point = zero_point; + write_seqcount_end(&inode->i_size_seqcount); +#elif BITS_PER_LONG==32 && defined(CONFIG_PREEMPTION) + preempt_disable(); + inode->i_size = i_size; + ictx->_remote_i_size = remote_i_size; + ictx->_zero_point = zero_point; + preempt_enable(); +#else + /* + * Pairs with smp_load_acquire() in i_size_read(), + * netfs_read_remote_i_size() and netfs_read_zero_point() to ensure + * changes related to inode size (such as page contents) are visible + * before we see the changed inode size. + */ + smp_store_release(&inode->i_size, i_size); + smp_store_release(&ictx->_remote_i_size, remote_i_size); + smp_store_release(&ictx->_zero_point, zero_point); +#endif +} + +/** * netfs_inode_init - Initialise a netfslib inode context * @ctx: The netfs inode to initialise * @ops: The netfs's operations list @@ -489,36 +735,64 @@ static inline void netfs_inode_init(struct netfs_inode *ctx, bool use_zero_point) { ctx->ops = ops; - ctx->remote_i_size = i_size_read(&ctx->inode); - ctx->zero_point = LLONG_MAX; + ctx->_remote_i_size = i_size_read(&ctx->inode); + ctx->_zero_point = LLONG_MAX; ctx->flags = 0; atomic_set(&ctx->io_count, 0); #if IS_ENABLED(CONFIG_FSCACHE) ctx->cache = NULL; #endif - mutex_init(&ctx->wb_lock); + INIT_LIST_HEAD(&ctx->wb_queue); + spin_lock_init(&ctx->lock); /* ->releasepage() drives zero_point */ if (use_zero_point) { - ctx->zero_point = ctx->remote_i_size; + ctx->_zero_point = ctx->_remote_i_size; mapping_set_release_always(ctx->inode.i_mapping); } } /** * netfs_resize_file - Note that a file got resized - * @ctx: The netfs inode being resized + * @ictx: The netfs inode being resized * @new_i_size: The new file size * @changed_on_server: The change was applied to the server * * Inform the netfs lib that a file got resized so that it can adjust its state. */ -static inline void netfs_resize_file(struct netfs_inode *ctx, loff_t new_i_size, +static inline void netfs_resize_file(struct netfs_inode *ictx, + unsigned long long new_i_size, bool changed_on_server) { +#if BITS_PER_LONG==32 && defined(CONFIG_SMP) + struct inode *inode = &ictx->inode; + + preempt_disable(); + write_seqcount_begin(&inode->i_size_seqcount); + if (changed_on_server) + ictx->_remote_i_size = new_i_size; + if (new_i_size < ictx->_zero_point) + ictx->_zero_point = new_i_size; + write_seqcount_end(&inode->i_size_seqcount); + preempt_enable(); +#elif BITS_PER_LONG==32 && defined(CONFIG_PREEMPTION) + preempt_disable(); if (changed_on_server) - ctx->remote_i_size = new_i_size; - if (new_i_size < ctx->zero_point) - ctx->zero_point = new_i_size; + ictx->_remote_i_size = new_i_size; + if (new_i_size < ictx->_zero_point) + ictx->_zero_point = new_i_size; + preempt_enable(); +#else + /* + * Pairs with smp_load_acquire() in netfs_read_remote_i_size and + * netfs_read_zero_point() to ensure changes related to inode size + * (such as page contents) are visible before we see the changed inode + * size. + */ + if (changed_on_server) + smp_store_release(&ictx->_remote_i_size, new_i_size); + if (new_i_size < ictx->_zero_point) + smp_store_release(&ictx->_zero_point, new_i_size); +#endif } /** diff --git a/include/linux/netpoll.h b/include/linux/netpoll.h index f22eec466040..1c6b1eec5efd 100644 --- a/include/linux/netpoll.h +++ b/include/linux/netpoll.h @@ -13,6 +13,8 @@ #include <linux/rcupdate.h> #include <linux/list.h> #include <linux/refcount.h> +#include <linux/ip.h> +#include <linux/udp.h> union inet_addr { __be32 ip; @@ -30,13 +32,6 @@ struct netpoll { char dev_name[IFNAMSIZ]; u8 dev_mac[ETH_ALEN]; const char *name; - - union inet_addr local_ip, remote_ip; - bool ipv6; - u16 local_port, remote_port; - u8 remote_mac[ETH_ALEN]; - struct sk_buff_head skb_pool; - struct work_struct refill_wq; }; #define np_info(np, fmt, ...) \ @@ -67,13 +62,13 @@ static inline void netpoll_poll_disable(struct net_device *dev) { return; } static inline void netpoll_poll_enable(struct net_device *dev) { return; } #endif -int netpoll_send_udp(struct netpoll *np, const char *msg, int len); int __netpoll_setup(struct netpoll *np, struct net_device *ndev); -int netpoll_setup(struct netpoll *np); void __netpoll_free(struct netpoll *np); void netpoll_cleanup(struct netpoll *np); void do_netpoll_cleanup(struct netpoll *np); netdev_tx_t netpoll_send_skb(struct netpoll *np, struct sk_buff *skb); +void netpoll_zap_completion_queue(void); +unsigned int netpoll_get_carrier_timeout(void); #ifdef CONFIG_NETPOLL static inline void *netpoll_poll_lock(struct napi_struct *napi) diff --git a/include/linux/nfs4.h b/include/linux/nfs4.h index d87be1f25273..1a3981c26b23 100644 --- a/include/linux/nfs4.h +++ b/include/linux/nfs4.h @@ -171,133 +171,6 @@ Needs to be updated if more operations are defined in future.*/ #define LAST_NFS42_OP OP_REMOVEXATTR #define LAST_NFS4_OP LAST_NFS42_OP -enum nfsstat4 { - NFS4_OK = 0, - NFS4ERR_PERM = 1, - NFS4ERR_NOENT = 2, - NFS4ERR_IO = 5, - NFS4ERR_NXIO = 6, - NFS4ERR_ACCESS = 13, - NFS4ERR_EXIST = 17, - NFS4ERR_XDEV = 18, - /* Unused/reserved 19 */ - NFS4ERR_NOTDIR = 20, - NFS4ERR_ISDIR = 21, - NFS4ERR_INVAL = 22, - NFS4ERR_FBIG = 27, - NFS4ERR_NOSPC = 28, - NFS4ERR_ROFS = 30, - NFS4ERR_MLINK = 31, - NFS4ERR_NAMETOOLONG = 63, - NFS4ERR_NOTEMPTY = 66, - NFS4ERR_DQUOT = 69, - NFS4ERR_STALE = 70, - NFS4ERR_BADHANDLE = 10001, - NFS4ERR_BAD_COOKIE = 10003, - NFS4ERR_NOTSUPP = 10004, - NFS4ERR_TOOSMALL = 10005, - NFS4ERR_SERVERFAULT = 10006, - NFS4ERR_BADTYPE = 10007, - NFS4ERR_DELAY = 10008, - NFS4ERR_SAME = 10009, - NFS4ERR_DENIED = 10010, - NFS4ERR_EXPIRED = 10011, - NFS4ERR_LOCKED = 10012, - NFS4ERR_GRACE = 10013, - NFS4ERR_FHEXPIRED = 10014, - NFS4ERR_SHARE_DENIED = 10015, - NFS4ERR_WRONGSEC = 10016, - NFS4ERR_CLID_INUSE = 10017, - NFS4ERR_RESOURCE = 10018, - NFS4ERR_MOVED = 10019, - NFS4ERR_NOFILEHANDLE = 10020, - NFS4ERR_MINOR_VERS_MISMATCH = 10021, - NFS4ERR_STALE_CLIENTID = 10022, - NFS4ERR_STALE_STATEID = 10023, - NFS4ERR_OLD_STATEID = 10024, - NFS4ERR_BAD_STATEID = 10025, - NFS4ERR_BAD_SEQID = 10026, - NFS4ERR_NOT_SAME = 10027, - NFS4ERR_LOCK_RANGE = 10028, - NFS4ERR_SYMLINK = 10029, - NFS4ERR_RESTOREFH = 10030, - NFS4ERR_LEASE_MOVED = 10031, - NFS4ERR_ATTRNOTSUPP = 10032, - NFS4ERR_NO_GRACE = 10033, - NFS4ERR_RECLAIM_BAD = 10034, - NFS4ERR_RECLAIM_CONFLICT = 10035, - NFS4ERR_BADXDR = 10036, - NFS4ERR_LOCKS_HELD = 10037, - NFS4ERR_OPENMODE = 10038, - NFS4ERR_BADOWNER = 10039, - NFS4ERR_BADCHAR = 10040, - NFS4ERR_BADNAME = 10041, - NFS4ERR_BAD_RANGE = 10042, - NFS4ERR_LOCK_NOTSUPP = 10043, - NFS4ERR_OP_ILLEGAL = 10044, - NFS4ERR_DEADLOCK = 10045, - NFS4ERR_FILE_OPEN = 10046, - NFS4ERR_ADMIN_REVOKED = 10047, - NFS4ERR_CB_PATH_DOWN = 10048, - - /* nfs41 */ - NFS4ERR_BADIOMODE = 10049, - NFS4ERR_BADLAYOUT = 10050, - NFS4ERR_BAD_SESSION_DIGEST = 10051, - NFS4ERR_BADSESSION = 10052, - NFS4ERR_BADSLOT = 10053, - NFS4ERR_COMPLETE_ALREADY = 10054, - NFS4ERR_CONN_NOT_BOUND_TO_SESSION = 10055, - NFS4ERR_DELEG_ALREADY_WANTED = 10056, - NFS4ERR_BACK_CHAN_BUSY = 10057, /* backchan reqs outstanding */ - NFS4ERR_LAYOUTTRYLATER = 10058, - NFS4ERR_LAYOUTUNAVAILABLE = 10059, - NFS4ERR_NOMATCHING_LAYOUT = 10060, - NFS4ERR_RECALLCONFLICT = 10061, - NFS4ERR_UNKNOWN_LAYOUTTYPE = 10062, - NFS4ERR_SEQ_MISORDERED = 10063, /* unexpected seq.id in req */ - NFS4ERR_SEQUENCE_POS = 10064, /* [CB_]SEQ. op not 1st op */ - NFS4ERR_REQ_TOO_BIG = 10065, /* request too big */ - NFS4ERR_REP_TOO_BIG = 10066, /* reply too big */ - NFS4ERR_REP_TOO_BIG_TO_CACHE = 10067, /* rep. not all cached */ - NFS4ERR_RETRY_UNCACHED_REP = 10068, /* retry & rep. uncached */ - NFS4ERR_UNSAFE_COMPOUND = 10069, /* retry/recovery too hard */ - NFS4ERR_TOO_MANY_OPS = 10070, /* too many ops in [CB_]COMP */ - NFS4ERR_OP_NOT_IN_SESSION = 10071, /* op needs [CB_]SEQ. op */ - NFS4ERR_HASH_ALG_UNSUPP = 10072, /* hash alg. not supp. */ - /* Error 10073 is unused. */ - NFS4ERR_CLIENTID_BUSY = 10074, /* clientid has state */ - NFS4ERR_PNFS_IO_HOLE = 10075, /* IO to _SPARSE file hole */ - NFS4ERR_SEQ_FALSE_RETRY = 10076, /* retry not original */ - NFS4ERR_BAD_HIGH_SLOT = 10077, /* sequence arg bad */ - NFS4ERR_DEADSESSION = 10078, /* persistent session dead */ - NFS4ERR_ENCR_ALG_UNSUPP = 10079, /* SSV alg mismatch */ - NFS4ERR_PNFS_NO_LAYOUT = 10080, /* direct I/O with no layout */ - NFS4ERR_NOT_ONLY_OP = 10081, /* bad compound */ - NFS4ERR_WRONG_CRED = 10082, /* permissions:state change */ - NFS4ERR_WRONG_TYPE = 10083, /* current operation mismatch */ - NFS4ERR_DIRDELEG_UNAVAIL = 10084, /* no directory delegation */ - NFS4ERR_REJECT_DELEG = 10085, /* on callback */ - NFS4ERR_RETURNCONFLICT = 10086, /* outstanding layoutreturn */ - NFS4ERR_DELEG_REVOKED = 10087, /* deleg./layout revoked */ - - /* nfs42 */ - NFS4ERR_PARTNER_NOTSUPP = 10088, - NFS4ERR_PARTNER_NO_AUTH = 10089, - NFS4ERR_UNION_NOTSUPP = 10090, - NFS4ERR_OFFLOAD_DENIED = 10091, - NFS4ERR_WRONG_LFS = 10092, - NFS4ERR_BADLABEL = 10093, - NFS4ERR_OFFLOAD_NO_REQS = 10094, - - /* xattr (RFC8276) */ - NFS4ERR_NOXATTR = 10095, - NFS4ERR_XATTR2BIG = 10096, - - /* can be used for internal errors */ - NFS4ERR_FIRST_FREE -}; - /* error codes for internal client use */ #define NFS4ERR_RESET_TO_MDS 12001 #define NFS4ERR_RESET_TO_PNFS 12002 @@ -517,6 +390,14 @@ enum { }; /* + * Symbol name and value are from draft-ietf-nfsv4-uncacheable-files + * Section 7. "XDR for Uncacheable Attribute" + */ +enum { + FATTR4_UNCACHEABLE_FILE_DATA = 87, +}; + +/* * The following internal definitions enable processing the above * attribute bits within 32-bit word boundaries. */ @@ -602,6 +483,7 @@ enum { #define FATTR4_WORD2_ACL_TRUEFORM_SCOPE BIT(FATTR4_ACL_TRUEFORM_SCOPE - 64) #define FATTR4_WORD2_POSIX_DEFAULT_ACL BIT(FATTR4_POSIX_DEFAULT_ACL - 64) #define FATTR4_WORD2_POSIX_ACCESS_ACL BIT(FATTR4_POSIX_ACCESS_ACL - 64) +#define FATTR4_WORD2_UNCACHEABLE_FILE_DATA BIT(FATTR4_UNCACHEABLE_FILE_DATA - 64) /* MDS threshold bitmap bits */ #define THRESHOLD_RD (1UL << 0) diff --git a/include/linux/nfs_fs.h b/include/linux/nfs_fs.h index 8dd79a3f3d66..b85a73ae7919 100644 --- a/include/linux/nfs_fs.h +++ b/include/linux/nfs_fs.h @@ -109,6 +109,8 @@ struct nfs_open_context { #define NFS_CONTEXT_BAD (2) #define NFS_CONTEXT_UNLOCK (3) #define NFS_CONTEXT_FILE_OPEN (4) +#define NFS_CONTEXT_WRITE_SYNC (5) +#define NFS_CONTEXT_O_DIRECT (6) struct nfs4_threshold *mdsthreshold; struct list_head list; @@ -145,11 +147,6 @@ struct nfs4_xattr_cache; */ struct nfs_inode { /* - * The 64bit 'inode number' - */ - __u64 fileid; - - /* * NFS file handle */ struct nfs_fh fh; @@ -166,6 +163,8 @@ struct nfs_inode { struct timespec64 btime; + bool uncacheable_file_data : 1; + /* * read_cache_jiffies is when we started read-caching this inode. * attrtimeo is for how long the cached information is assumed @@ -323,6 +322,7 @@ struct nfs4_copy_state { #define NFS_INO_INVALID_NLINK BIT(16) /* cached nlinks is invalid */ #define NFS_INO_INVALID_MODE BIT(17) /* cached mode is invalid */ #define NFS_INO_INVALID_BTIME BIT(18) /* cached btime is invalid */ +#define NFS_INO_INVALID_UNCACHEABLE_FILE_DATA BIT(19) /* cached uncacheable_file_data is invalid */ #define NFS_INO_INVALID_ATTR (NFS_INO_INVALID_CHANGE \ | NFS_INO_INVALID_CTIME \ @@ -393,16 +393,6 @@ static inline int NFS_STALE(const struct inode *inode) return test_bit(NFS_INO_STALE, &NFS_I(inode)->flags); } -static inline __u64 NFS_FILEID(const struct inode *inode) -{ - return NFS_I(inode)->fileid; -} - -static inline void set_nfs_fileid(struct inode *inode, __u64 fileid) -{ - NFS_I(inode)->fileid = fileid; -} - static inline void nfs_mark_for_revalidate(struct inode *inode) { struct nfs_inode *nfsi = NFS_I(inode); @@ -472,7 +462,6 @@ extern void nfs_file_set_open_context(struct file *filp, struct nfs_open_context extern void nfs_file_clear_open_context(struct file *flip); extern struct nfs_lock_context *nfs_get_lock_context(struct nfs_open_context *ctx); extern void nfs_put_lock_context(struct nfs_lock_context *l_ctx); -extern u64 nfs_compat_user_ino64(u64 fileid); extern void nfs_fattr_init(struct nfs_fattr *fattr); extern void nfs_fattr_set_barrier(struct nfs_fattr *fattr); extern unsigned long nfs_inc_attr_generation_counter(void); @@ -563,7 +552,6 @@ static inline const struct cred *nfs_file_cred(struct file *file) /* * linux/fs/nfs/direct.c */ -int nfs_swap_rw(struct kiocb *iocb, struct iov_iter *iter); ssize_t nfs_file_direct_read(struct kiocb *iocb, struct iov_iter *iter, bool swap); ssize_t nfs_file_direct_write(struct kiocb *iocb, @@ -667,15 +655,6 @@ static inline loff_t nfs_size_to_loff_t(__u64 size) return min_t(u64, size, OFFSET_MAX); } -static inline ino_t -nfs_fileid_to_ino_t(u64 fileid) -{ - ino_t ino = (ino_t) fileid; - if (sizeof(ino_t) < sizeof(u64)) - ino ^= fileid >> (sizeof(u64)-sizeof(ino_t)) * 8; - return ino; -} - static inline void nfs_ooo_clear(struct nfs_inode *nfsi) { nfsi->cache_validity &= ~NFS_INO_DATA_INVAL_DEFER; diff --git a/include/linux/nfs_fs_sb.h b/include/linux/nfs_fs_sb.h index 4daee27fa5eb..34d294774f8c 100644 --- a/include/linux/nfs_fs_sb.h +++ b/include/linux/nfs_fs_sb.h @@ -306,7 +306,7 @@ struct nfs_server { #define NFS_CAP_ATOMIC_OPEN (1U << 4) #define NFS_CAP_LGOPEN (1U << 5) #define NFS_CAP_CASE_INSENSITIVE (1U << 6) -#define NFS_CAP_CASE_PRESERVING (1U << 7) +#define NFS_CAP_CASE_NONPRESERVING (1U << 7) #define NFS_CAP_REBOOT_LAYOUTRETURN (1U << 8) #define NFS_CAP_OFFLOAD_STATUS (1U << 9) #define NFS_CAP_ZERO_RANGE (1U << 10) diff --git a/include/linux/nfs_page.h b/include/linux/nfs_page.h index afe1d8f09d89..4b9a35dbc062 100644 --- a/include/linux/nfs_page.h +++ b/include/linux/nfs_page.h @@ -33,6 +33,7 @@ enum { PG_TEARDOWN, /* page group sync for destroy */ PG_UNLOCKPAGE, /* page group sync bit in read path */ PG_UPTODATE, /* page group sync bit in read path */ + PG_READ_FAILED, /* page group saw a read error */ PG_WB_END, /* page group sync bit in write path */ PG_REMOVE, /* page group sync bit in write path */ PG_CONTENDED1, /* Is someone waiting for a lock? */ diff --git a/include/linux/nfs_xdr.h b/include/linux/nfs_xdr.h index ff1f12aa73d2..7ed8fdb930d6 100644 --- a/include/linux/nfs_xdr.h +++ b/include/linux/nfs_xdr.h @@ -17,6 +17,9 @@ #define NFS_BITMASK_SZ 3 +/* aux_flags in nfs_fattr */ +#define NFS_AUX_UNCACHEABLE_FILE_DATA BIT(0) + struct nfs4_string { unsigned int len; char *data; @@ -68,6 +71,7 @@ struct nfs_fattr { struct timespec64 mtime; struct timespec64 ctime; struct timespec64 btime; + __u32 aux_flags; /* NFSv4 auxiliary flags bitfield */ __u64 change_attr; /* NFSv4 change attribute */ __u64 pre_change_attr;/* pre-op NFSv4 change attribute */ __u64 pre_size; /* pre_op_attr.size */ @@ -108,6 +112,7 @@ struct nfs_fattr { #define NFS_ATTR_FATTR_GROUP_NAME BIT_ULL(24) #define NFS_ATTR_FATTR_V4_SECURITY_LABEL BIT_ULL(25) #define NFS_ATTR_FATTR_BTIME BIT_ULL(26) +#define NFS_ATTR_FATTR_UNCACHEABLE_FILE_DATA BIT_ULL(27) #define NFS_ATTR_FATTR (NFS_ATTR_FATTR_TYPE \ | NFS_ATTR_FATTR_MODE \ @@ -129,7 +134,8 @@ struct nfs_fattr { #define NFS_ATTR_FATTR_V4 (NFS_ATTR_FATTR \ | NFS_ATTR_FATTR_SPACE_USED \ | NFS_ATTR_FATTR_BTIME \ - | NFS_ATTR_FATTR_V4_SECURITY_LABEL) + | NFS_ATTR_FATTR_V4_SECURITY_LABEL \ + | NFS_ATTR_FATTR_UNCACHEABLE_FILE_DATA) /* * Maximal number of supported layout drivers. @@ -182,6 +188,8 @@ struct nfs_pathconf { struct nfs_fattr *fattr; /* Post-op attributes */ __u32 max_link; /* max # of hard links */ __u32 max_namelen; /* max name length */ + bool case_insensitive; + bool case_preserving; }; struct nfs4_change_info { @@ -352,7 +360,7 @@ struct nfs4_layoutreturn { struct nfs4_xdr_opaque_data ld_private; }; -#define PNFS_LAYOUTSTATS_MAXSIZE 256 +#define PNFS_LAYOUTSTATS_MAXSIZE 384 struct nfs42_layoutstat_args; struct nfs42_layoutstat_devinfo; @@ -580,7 +588,6 @@ struct nfs_lock_args { struct nfs_lowner lock_owner; unsigned char block : 1; unsigned char reclaim : 1; - unsigned char new_lock : 1; unsigned char new_lock_owner : 1; }; @@ -1611,12 +1618,15 @@ struct nfs42_listxattrsres { struct nfs42_removexattrargs { struct nfs4_sequence_args seq_args; struct nfs_fh *fh; + const u32 *bitmask; const char *xattr_name; }; struct nfs42_removexattrres { struct nfs4_sequence_res seq_res; struct nfs4_change_info cinfo; + struct nfs_fattr *fattr; + const struct nfs_server *server; }; #endif /* CONFIG_NFS_V4_2 */ @@ -1740,7 +1750,6 @@ struct nfs_unlinkdata { struct nfs_removeargs args; struct nfs_removeres res; struct dentry *dentry; - wait_queue_head_t wq; const struct cred *cred; struct nfs_fattr dir_attr; long timeout; diff --git a/include/linux/nmi.h b/include/linux/nmi.h index 207156f2143c..af69712df5f4 100644 --- a/include/linux/nmi.h +++ b/include/linux/nmi.h @@ -21,6 +21,7 @@ void lockup_detector_soft_poweroff(void); extern int watchdog_user_enabled; extern int watchdog_thresh; extern unsigned long watchdog_enabled; +extern int watchdog_hardlockup_miss_thresh; extern struct cpumask watchdog_cpumask; extern unsigned long *watchdog_cpumask_bits; @@ -76,9 +77,9 @@ static inline void reset_hung_task_detector(void) { } * detectors are 'suspended' while 'watchdog_thresh' is equal zero. */ #define WATCHDOG_HARDLOCKUP_ENABLED_BIT 0 -#define WATCHDOG_SOFTOCKUP_ENABLED_BIT 1 +#define WATCHDOG_SOFTLOCKUP_ENABLED_BIT 1 #define WATCHDOG_HARDLOCKUP_ENABLED (1 << WATCHDOG_HARDLOCKUP_ENABLED_BIT) -#define WATCHDOG_SOFTOCKUP_ENABLED (1 << WATCHDOG_SOFTOCKUP_ENABLED_BIT) +#define WATCHDOG_SOFTLOCKUP_ENABLED (1 << WATCHDOG_SOFTLOCKUP_ENABLED_BIT) #if defined(CONFIG_HARDLOCKUP_DETECTOR) extern void hardlockup_detector_disable(void); @@ -156,27 +157,29 @@ static inline void touch_nmi_watchdog(void) * to allow calling code to fall back to some other mechanism: */ #ifdef arch_trigger_cpumask_backtrace +void cpumask_backtrace(const cpumask_t *mask, int exclude_cpu); + static inline bool trigger_all_cpu_backtrace(void) { - arch_trigger_cpumask_backtrace(cpu_online_mask, -1); + cpumask_backtrace(cpu_online_mask, -1); return true; } static inline bool trigger_allbutcpu_cpu_backtrace(int exclude_cpu) { - arch_trigger_cpumask_backtrace(cpu_online_mask, exclude_cpu); + cpumask_backtrace(cpu_online_mask, exclude_cpu); return true; } static inline bool trigger_cpumask_backtrace(struct cpumask *mask) { - arch_trigger_cpumask_backtrace(mask, -1); + cpumask_backtrace(mask, -1); return true; } static inline bool trigger_single_cpu_backtrace(int cpu) { - arch_trigger_cpumask_backtrace(cpumask_of(cpu), -1); + cpumask_backtrace(cpumask_of(cpu), -1); return true; } diff --git a/include/linux/nodemask.h b/include/linux/nodemask.h index 204c92462f3c..607373bd83ee 100644 --- a/include/linux/nodemask.h +++ b/include/linux/nodemask.h @@ -24,23 +24,23 @@ * void nodes_setall(mask) set all bits * void nodes_clear(mask) clear all bits * int node_isset(node, mask) true iff bit 'node' set in mask - * int node_test_and_set(node, mask) test and set bit 'node' in mask + * bool node_test_and_set(node, mask) test and set bit 'node' in mask * - * void nodes_and(dst, src1, src2) dst = src1 & src2 [intersection] + * bool nodes_and(dst, src1, src2) dst = src1 & src2 [intersection] * void nodes_or(dst, src1, src2) dst = src1 | src2 [union] * void nodes_xor(dst, src1, src2) dst = src1 ^ src2 - * void nodes_andnot(dst, src1, src2) dst = src1 & ~src2 + * bool nodes_andnot(dst, src1, src2) dst = src1 & ~src2 * void nodes_complement(dst, src) dst = ~src * - * int nodes_equal(mask1, mask2) Does mask1 == mask2? - * int nodes_intersects(mask1, mask2) Do mask1 and mask2 intersect? - * int nodes_subset(mask1, mask2) Is mask1 a subset of mask2? - * int nodes_empty(mask) Is mask empty (no bits sets)? - * int nodes_full(mask) Is mask full (all bits sets)? + * bool nodes_equal(mask1, mask2) Does mask1 == mask2? + * bool nodes_intersects(mask1, mask2) Do mask1 and mask2 intersect? + * bool nodes_subset(mask1, mask2) Is mask1 a subset of mask2? + * bool nodes_empty(mask) Is mask empty (no bits sets)? + * bool nodes_full(mask) Is mask full (all bits sets)? * int nodes_weight(mask) Hamming weight - number of set bits * * unsigned int first_node(mask) Number lowest set bit, or MAX_NUMNODES - * unsigend int next_node(node, mask) Next node past 'node', or MAX_NUMNODES + * unsigned int next_node(node, mask) Next node past 'node', or MAX_NUMNODES * unsigned int next_node_in(node, mask) Next node past 'node', or wrap to first, * or MAX_NUMNODES * unsigned int first_unset_node(mask) First node not set in mask, or @@ -95,6 +95,14 @@ extern nodemask_t _unused_nodemask_arg_; +#if MAX_NUMNODES > 1 +extern unsigned int nr_node_ids; +extern unsigned int nr_online_nodes; +#else +#define nr_node_ids 1U +#define nr_online_nodes 1U +#endif + /** * nodemask_pr_args - printf args to output a nodemask * @maskp: nodemask to be printed @@ -105,7 +113,7 @@ extern nodemask_t _unused_nodemask_arg_; __nodemask_pr_bits(maskp) static __always_inline unsigned int __nodemask_pr_numnodes(const nodemask_t *m) { - return m ? MAX_NUMNODES : 0; + return m ? nr_node_ids : 0; } static __always_inline const unsigned long *__nodemask_pr_bits(const nodemask_t *m) { @@ -438,9 +446,6 @@ static __always_inline unsigned int next_memory_node(int nid) return next_node(nid, node_states[N_MEMORY]); } -extern unsigned int nr_node_ids; -extern unsigned int nr_online_nodes; - static __always_inline void node_set_online(int nid) { node_set_state(nid, N_ONLINE); @@ -480,8 +485,6 @@ static __always_inline int num_node_state(enum node_states state) #define first_memory_node 0 #define next_online_node(nid) (MAX_NUMNODES) #define next_memory_node(nid) (MAX_NUMNODES) -#define nr_node_ids 1U -#define nr_online_nodes 1U #define node_set_online(node) node_set_state((node), N_ONLINE) #define node_set_offline(node) node_clear_state((node), N_ONLINE) diff --git a/include/linux/ns/ns_common_types.h b/include/linux/ns/ns_common_types.h index 0014fbc1c626..6ed6b497831c 100644 --- a/include/linux/ns/ns_common_types.h +++ b/include/linux/ns/ns_common_types.h @@ -7,6 +7,7 @@ #include <linux/rbtree.h> #include <linux/refcount.h> #include <linux/types.h> +#include <uapi/linux/sched.h> struct cgroup_namespace; struct dentry; @@ -115,10 +116,8 @@ struct ns_common { struct dentry *stashed; const struct proc_ns_operations *ops; unsigned int inum; - union { - struct ns_tree; - struct rcu_head ns_rcu; - }; + struct ns_tree; + struct rcu_head ns_rcu; }; #define to_ns_common(__ns) \ @@ -184,15 +183,38 @@ struct ns_common { struct user_namespace *: (IS_ENABLED(CONFIG_USER_NS) ? &userns_operations : NULL), \ struct uts_namespace *: (IS_ENABLED(CONFIG_UTS_NS) ? &utsns_operations : NULL)) -#define ns_common_type(__ns) \ - _Generic((__ns), \ - struct cgroup_namespace *: CLONE_NEWCGROUP, \ - struct ipc_namespace *: CLONE_NEWIPC, \ - struct mnt_namespace *: CLONE_NEWNS, \ - struct net *: CLONE_NEWNET, \ - struct pid_namespace *: CLONE_NEWPID, \ - struct time_namespace *: CLONE_NEWTIME, \ - struct user_namespace *: CLONE_NEWUSER, \ - struct uts_namespace *: CLONE_NEWUTS) +/* + * FOR_EACH_NS_TYPE - Canonical list of namespace types + * + * Enumerates all (struct type, CLONE_NEW* flag) pairs. This is the + * single source of truth used to derive ns_common_type() and + * CLONE_NS_ALL. When adding a new namespace type, add a single entry + * here; all consumers update automatically. + * + * @X: Callback macro taking (struct_name, clone_flag) as arguments. + */ +#define FOR_EACH_NS_TYPE(X) \ + X(cgroup_namespace, CLONE_NEWCGROUP) \ + X(ipc_namespace, CLONE_NEWIPC) \ + X(mnt_namespace, CLONE_NEWNS) \ + X(net, CLONE_NEWNET) \ + X(pid_namespace, CLONE_NEWPID) \ + X(time_namespace, CLONE_NEWTIME) \ + X(user_namespace, CLONE_NEWUSER) \ + X(uts_namespace, CLONE_NEWUTS) + +/* Bitmask of all known CLONE_NEW* flags. */ +#define _NS_TYPE_FLAG_OR(struct_name, flag) | (flag) +#define CLONE_NS_ALL (0 FOR_EACH_NS_TYPE(_NS_TYPE_FLAG_OR)) + +/* + * ns_common_type - Map a namespace struct pointer to its CLONE_NEW* flag + * + * Uses a leading-comma pattern so the FOR_EACH_NS_TYPE expansion + * produces ", struct foo *: FLAG" entries without a trailing comma. + */ +#define _NS_TYPE_ASSOC(struct_name, flag) , struct struct_name *: (flag) + +#define ns_common_type(__ns) _Generic((__ns)FOR_EACH_NS_TYPE(_NS_TYPE_ASSOC)) #endif /* _LINUX_NS_COMMON_TYPES_H */ diff --git a/include/linux/nstree.h b/include/linux/nstree.h index 175e4625bfa6..5b64d4572881 100644 --- a/include/linux/nstree.h +++ b/include/linux/nstree.h @@ -61,7 +61,7 @@ static inline void __ns_tree_add(struct ns_common *ns, struct ns_tree_root *ns_t /** * ns_tree_add_raw - Add a namespace to a namespace - * @ns: Namespace to add + * @__ns: Namespace to add * * This function adds a namespace to the appropriate namespace tree * without assigning a id. @@ -70,7 +70,7 @@ static inline void __ns_tree_add(struct ns_common *ns, struct ns_tree_root *ns_t /** * ns_tree_add - Add a namespace to a namespace tree - * @ns: Namespace to add + * @__ns: Namespace to add * * This function assigns a new id to the namespace and adds it to the * appropriate namespace tree and list. @@ -81,7 +81,7 @@ static inline void __ns_tree_add(struct ns_common *ns, struct ns_tree_root *ns_t /** * ns_tree_remove - Remove a namespace from a namespace tree - * @ns: Namespace to remove + * @__ns: Namespace to remove * * This function removes a namespace from the appropriate namespace * tree and list. diff --git a/include/linux/ntb.h b/include/linux/ntb.h index 8ff9d663096b..ae72caa03859 100644 --- a/include/linux/ntb.h +++ b/include/linux/ntb.h @@ -256,6 +256,7 @@ static inline int ntb_ctx_ops_is_valid(const struct ntb_ctx_ops *ops) * @msg_clear_mask: See ntb_msg_clear_mask(). * @msg_read: See ntb_msg_read(). * @peer_msg_write: See ntb_peer_msg_write(). + * @get_dma_dev: See ntb_get_dma_dev(). */ struct ntb_dev_ops { int (*port_number)(struct ntb_dev *ntb); @@ -329,6 +330,7 @@ struct ntb_dev_ops { int (*msg_clear_mask)(struct ntb_dev *ntb, u64 mask_bits); u32 (*msg_read)(struct ntb_dev *ntb, int *pidx, int midx); int (*peer_msg_write)(struct ntb_dev *ntb, int pidx, int midx, u32 msg); + struct device *(*get_dma_dev)(struct ntb_dev *ntb); }; static inline int ntb_dev_ops_is_valid(const struct ntb_dev_ops *ops) @@ -391,6 +393,8 @@ static inline int ntb_dev_ops_is_valid(const struct ntb_dev_ops *ops) /* !ops->msg_clear_mask == !ops->msg_count && */ !ops->msg_read == !ops->msg_count && !ops->peer_msg_write == !ops->msg_count && + + /* ops->get_dma_dev is optional */ 1; } @@ -563,7 +567,7 @@ void ntb_msg_event(struct ntb_dev *ntb); int ntb_default_port_number(struct ntb_dev *ntb); /** - * ntb_default_port_count() - get the default number of peer device ports + * ntb_default_peer_port_count() - get the default number of peer device ports * @ntb: NTB device context. * * By default hardware driver supports just one peer device. @@ -578,7 +582,7 @@ int ntb_default_peer_port_count(struct ntb_dev *ntb); /** * ntb_default_peer_port_number() - get the default peer port by given index * @ntb: NTB device context. - * @idx: Peer port index (should not differ from zero). + * @pidx: Peer port index (should not differ from zero). * * By default hardware driver supports just one peer device, so this method * shall return the corresponding value from enum ntb_default_port. @@ -1564,6 +1568,26 @@ static inline int ntb_peer_msg_write(struct ntb_dev *ntb, int pidx, int midx, } /** + * ntb_get_dma_dev() - get the device to use for DMA allocations/mappings + * @ntb: NTB device context. + * + * Return a struct device suitable for DMA API allocations and mappings. + * This is typically the parent of the NTB device, but may be overridden by a + * driver by implementing .get_dma_dev(). + * + * Drivers that implement .get_dma_dev() must return a non-NULL pointer. + * + * Return: device pointer to use for DMA operations. + */ +static inline struct device *ntb_get_dma_dev(struct ntb_dev *ntb) +{ + if (!ntb->ops->get_dma_dev) + return ntb->dev.parent; + + return ntb->ops->get_dma_dev(ntb); +} + +/** * ntb_peer_resource_idx() - get a resource index for a given peer idx * @ntb: NTB device context. * @pidx: Peer port index. diff --git a/include/linux/nvme-auth.h b/include/linux/nvme-auth.h index e75c29c51464..d674d8ab26e6 100644 --- a/include/linux/nvme-auth.h +++ b/include/linux/nvme-auth.h @@ -7,6 +7,7 @@ #define _NVME_AUTH_H #include <crypto/kpp.h> +#include <crypto/sha2.h> struct nvme_dhchap_key { size_t len; @@ -20,32 +21,44 @@ const char *nvme_auth_dhgroup_kpp(u8 dhgroup_id); u8 nvme_auth_dhgroup_id(const char *dhgroup_name); const char *nvme_auth_hmac_name(u8 hmac_id); -const char *nvme_auth_digest_name(u8 hmac_id); size_t nvme_auth_hmac_hash_len(u8 hmac_id); u8 nvme_auth_hmac_id(const char *hmac_name); +struct nvme_auth_hmac_ctx { + u8 hmac_id; + union { + struct hmac_sha256_ctx sha256; + struct hmac_sha384_ctx sha384; + struct hmac_sha512_ctx sha512; + }; +}; +int nvme_auth_hmac_init(struct nvme_auth_hmac_ctx *hmac, u8 hmac_id, + const u8 *key, size_t key_len); +void nvme_auth_hmac_update(struct nvme_auth_hmac_ctx *hmac, const u8 *data, + size_t data_len); +void nvme_auth_hmac_final(struct nvme_auth_hmac_ctx *hmac, u8 *out); u32 nvme_auth_key_struct_size(u32 key_len); -struct nvme_dhchap_key *nvme_auth_extract_key(unsigned char *secret, - u8 key_hash); +struct nvme_dhchap_key *nvme_auth_extract_key(const char *secret, u8 key_hash); void nvme_auth_free_key(struct nvme_dhchap_key *key); struct nvme_dhchap_key *nvme_auth_alloc_key(u32 len, u8 hash); struct nvme_dhchap_key *nvme_auth_transform_key( - struct nvme_dhchap_key *key, char *nqn); -int nvme_auth_generate_key(u8 *secret, struct nvme_dhchap_key **ret_key); -int nvme_auth_augmented_challenge(u8 hmac_id, u8 *skey, size_t skey_len, - u8 *challenge, u8 *aug, size_t hlen); + const struct nvme_dhchap_key *key, const char *nqn); +int nvme_auth_parse_key(const char *secret, struct nvme_dhchap_key **ret_key); +int nvme_auth_augmented_challenge(u8 hmac_id, const u8 *skey, size_t skey_len, + const u8 *challenge, u8 *aug, size_t hlen); int nvme_auth_gen_privkey(struct crypto_kpp *dh_tfm, u8 dh_gid); int nvme_auth_gen_pubkey(struct crypto_kpp *dh_tfm, u8 *host_key, size_t host_key_len); -int nvme_auth_gen_shared_secret(struct crypto_kpp *dh_tfm, - u8 *ctrl_key, size_t ctrl_key_len, - u8 *sess_key, size_t sess_key_len); -int nvme_auth_generate_psk(u8 hmac_id, u8 *skey, size_t skey_len, - u8 *c1, u8 *c2, size_t hash_len, +int nvme_auth_gen_session_key(struct crypto_kpp *dh_tfm, + const u8 *public_key, size_t public_key_len, + u8 *sess_key, size_t sess_key_len, u8 hash_id); +int nvme_auth_generate_psk(u8 hmac_id, const u8 *skey, size_t skey_len, + const u8 *c1, const u8 *c2, size_t hash_len, u8 **ret_psk, size_t *ret_len); -int nvme_auth_generate_digest(u8 hmac_id, u8 *psk, size_t psk_len, - char *subsysnqn, char *hostnqn, u8 **ret_digest); -int nvme_auth_derive_tls_psk(int hmac_id, u8 *psk, size_t psk_len, - u8 *psk_digest, u8 **ret_psk); +int nvme_auth_generate_digest(u8 hmac_id, const u8 *psk, size_t psk_len, + const char *subsysnqn, const char *hostnqn, + char **ret_digest); +int nvme_auth_derive_tls_psk(int hmac_id, const u8 *psk, size_t psk_len, + const char *psk_digest, u8 **ret_psk); #endif /* _NVME_AUTH_H */ diff --git a/include/linux/nvme-tcp.h b/include/linux/nvme-tcp.h index e435250fcb4d..859338da8573 100644 --- a/include/linux/nvme-tcp.h +++ b/include/linux/nvme-tcp.h @@ -77,7 +77,7 @@ struct nvme_tcp_hdr { __le32 plen; }; -/** +/* * struct nvme_tcp_icreq_pdu - nvme tcp initialize connection request pdu * * @hdr: pdu generic header @@ -95,7 +95,7 @@ struct nvme_tcp_icreq_pdu { __u8 rsvd2[112]; }; -/** +/* * struct nvme_tcp_icresp_pdu - nvme tcp initialize connection response pdu * * @hdr: pdu common header @@ -113,12 +113,13 @@ struct nvme_tcp_icresp_pdu { __u8 rsvd[112]; }; -/** +/* * struct nvme_tcp_term_pdu - nvme tcp terminate connection pdu * * @hdr: pdu common header * @fes: fatal error status - * @fei: fatal error information + * @feil: fatal error information (low 16 bits) + * @feih: fatal error information (high 16 bits) */ struct nvme_tcp_term_pdu { struct nvme_tcp_hdr hdr; @@ -128,7 +129,7 @@ struct nvme_tcp_term_pdu { __u8 rsvd[10]; }; -/** +/* * struct nvme_tcp_cmd_pdu - nvme tcp command capsule pdu * * @hdr: pdu common header @@ -139,10 +140,9 @@ struct nvme_tcp_cmd_pdu { struct nvme_command cmd; }; -/** +/* * struct nvme_tcp_rsp_pdu - nvme tcp response capsule pdu * - * @hdr: pdu common header * @hdr: nvme-tcp generic header * @cqe: nvme completion queue entry */ @@ -151,7 +151,7 @@ struct nvme_tcp_rsp_pdu { struct nvme_completion cqe; }; -/** +/* * struct nvme_tcp_r2t_pdu - nvme tcp ready-to-transfer pdu * * @hdr: pdu common header @@ -169,7 +169,7 @@ struct nvme_tcp_r2t_pdu { __u8 rsvd[4]; }; -/** +/* * struct nvme_tcp_data_pdu - nvme tcp data pdu * * @hdr: pdu common header diff --git a/include/linux/nvme.h b/include/linux/nvme.h index 655d194f8e72..91ce434a7e8d 100644 --- a/include/linux/nvme.h +++ b/include/linux/nvme.h @@ -513,9 +513,16 @@ struct nvme_id_ns_nvm { __u8 pic; __u8 rsvd9[3]; __le32 elbaf[64]; - __u8 rsvd268[3828]; + __le32 npdgl; + __le32 nprg; + __le32 npra; + __le32 nors; + __le32 npdal; + __u8 rsvd288[3808]; }; +static_assert(sizeof(struct nvme_id_ns_nvm) == 4096); + enum { NVME_ID_NS_NVM_STS_MASK = 0x7f, NVME_ID_NS_NVM_GUARD_SHIFT = 7, @@ -590,7 +597,11 @@ enum { enum { NVME_NS_FEAT_THIN = 1 << 0, NVME_NS_FEAT_ATOMICS = 1 << 1, - NVME_NS_FEAT_IO_OPT = 1 << 4, + NVME_NS_FEAT_OPTPERF_SHIFT = 4, + /* In NVMe version 2.0 and below, OPTPERF is only bit 4 of NSFEAT */ + NVME_NS_FEAT_OPTPERF_MASK = 0x1, + /* Since version 2.1, OPTPERF is bits 4 and 5 of NSFEAT */ + NVME_NS_FEAT_OPTPERF_MASK_2_1 = 0x3, NVME_NS_ATTR_RO = 1 << 0, NVME_NS_FLBAS_LBA_MASK = 0xf, NVME_NS_FLBAS_LBA_UMASK = 0x60, @@ -1837,6 +1848,11 @@ enum { NVME_AUTH_HASH_INVALID = 0xff, }; +/* Maximum digest size for any NVME_AUTH_HASH_* value */ +enum { + NVME_AUTH_MAX_DIGEST_SIZE = 64, +}; + /* Defined Diffie-Hellman group identifiers for DH-HMAC-CHAP authentication */ enum { NVME_AUTH_DHGROUP_NULL = 0x00, @@ -2256,14 +2272,14 @@ struct nvme_completion { #define NVME_TERTIARY(ver) ((ver) & 0xff) enum { - NVME_AEN_RESV_LOG_PAGE_AVALIABLE = 0x00, + NVME_AEN_RESV_LOG_PAGE_AVAILABLE = 0x00, }; enum { NVME_PR_LOG_EMPTY_LOG_PAGE = 0x00, NVME_PR_LOG_REGISTRATION_PREEMPTED = 0x01, NVME_PR_LOG_RESERVATION_RELEASED = 0x02, - NVME_PR_LOG_RESERVATOIN_PREEMPTED = 0x03, + NVME_PR_LOG_RESERVATION_PREEMPTED = 0x03, }; enum { @@ -2332,4 +2348,8 @@ enum nvme_pr_change_ptpl { #define NVME_PR_IGNORE_KEY (1 << 3) +/* Section 8.3.4.5.2 of the NVMe 2.1 */ +#define NVME_AUTH_DHCHAP_MAX_HASH_IDS 30 +#define NVME_AUTH_DHCHAP_MAX_DH_IDS 30 + #endif /* _LINUX_NVME_H */ diff --git a/include/linux/nvmem-provider.h b/include/linux/nvmem-provider.h index f3b13da78aac..6063fe5b7784 100644 --- a/include/linux/nvmem-provider.h +++ b/include/linux/nvmem-provider.h @@ -214,6 +214,12 @@ static inline int nvmem_layout_register(struct nvmem_layout *layout) static inline void nvmem_layout_unregister(struct nvmem_layout *layout) {} +static inline int nvmem_add_cells_from_dt(struct nvmem_device *nvmem, + struct device_node *np) +{ + return -EOPNOTSUPP; +} + #endif /* CONFIG_NVMEM */ #if IS_ENABLED(CONFIG_NVMEM) && IS_ENABLED(CONFIG_OF) diff --git a/include/linux/oa_tc6.h b/include/linux/oa_tc6.h index 15f58e3c56c7..27f652d4920b 100644 --- a/include/linux/oa_tc6.h +++ b/include/linux/oa_tc6.h @@ -10,15 +10,86 @@ #include <linux/etherdevice.h> #include <linux/spi/spi.h> +/* OPEN Alliance TC6 registers */ +/* Standard Capabilities Register */ +#define OA_TC6_REG_STDCAP 0x0002 +#define OA_TC6_STDCAP_DIRECT_PHY_REG_ACCESS BIT(8) + +/* Reset Control and Status Register */ +#define OA_TC6_REG_RESET 0x0003 +#define OA_TC6_RESET_SWRESET BIT(0) /* Software Reset */ + +/* Configuration Register #0 */ +#define OA_TC6_REG_CONFIG0 0x0004 +#define OA_TC6_CONFIG0_SYNC BIT(15) +#define OA_TC6_CONFIG0_ZARFE_ENABLE BIT(12) +#define OA_TC6_CONFIG0_PROTE BIT(5) + +/* Configuration Register #2 */ +#define OA_TC6_REG_CONFIG2 0x0006 + +/* Status Register #0 */ +#define OA_TC6_REG_STATUS0 0x0008 +#define OA_TC6_STATUS0_RESETC BIT(6) /* Reset Complete */ +#define OA_TC6_STATUS0_HEADER_ERROR BIT(5) +#define OA_TC6_STATUS0_LOSS_OF_FRAME_ERROR BIT(4) +#define OA_TC6_STATUS0_RX_BUFFER_OVERFLOW_ERROR BIT(3) +#define OA_TC6_STATUS0_TX_PROTOCOL_ERROR BIT(0) + +/* Buffer Status Register */ +#define OA_TC6_REG_BUFFER_STATUS 0x000B +#define OA_TC6_BUFFER_STATUS_TX_CREDITS_AVAILABLE GENMASK(15, 8) +#define OA_TC6_BUFFER_STATUS_RX_CHUNKS_AVAILABLE GENMASK(7, 0) + +/* Interrupt Mask Register #0 */ +#define OA_TC6_REG_INT_MASK0 0x000C +#define OA_TC6_INT_MASK0_HEADER_ERR_MASK BIT(5) +#define OA_TC6_INT_MASK0_LOSS_OF_FRAME_ERR_MASK BIT(4) +#define OA_TC6_INT_MASK0_RX_BUFFER_OVERFLOW_ERR_MASK BIT(3) +#define OA_TC6_INT_MASK0_TX_PROTOCOL_ERR_MASK BIT(0) +#define OA_TC6_INT_MASK0_ALL_INTERRUPTS (GENMASK(5, 0) | \ + GENMASK(12, 7)) + +/* PHY Clause 22 registers base address and mask */ +#define OA_TC6_PHY_STD_REG_ADDR_BASE 0xFF00 +#define OA_TC6_PHY_STD_REG_ADDR_MASK 0x1F + +/* Memory map selector (MMS) values as per table 6 in the + * OPEN Alliance specification. + */ +#define OA_TC6_MAC_MMS1 1 +#define OA_TC6_PHY_C45_PCS_MMS2 2 /* MMD 3 */ +#define OA_TC6_PHY_C45_PMA_PMD_MMS3 3 /* MMD 1 */ +#define OA_TC6_PHY_C45_VS_PLCA_MMS4 4 /* MMD 31 */ +#define OA_TC6_PHY_C45_AUTO_NEG_MMS5 5 /* MMD 7 */ +#define OA_TC6_PHY_C45_POWER_UNIT_MMS6 6 /* MMD 13 */ + struct oa_tc6; -struct oa_tc6 *oa_tc6_init(struct spi_device *spi, struct net_device *netdev); +enum oa_tc6_quirk_flag { + OA_TC6_BROKEN_PHY = BIT(0), +}; + +struct oa_tc6_quirks { + enum oa_tc6_quirk_flag quirk_flags; +}; + +struct oa_tc6 *oa_tc6_init(struct spi_device *spi, struct net_device *netdev, + struct oa_tc6_quirks *quirks); void oa_tc6_exit(struct oa_tc6 *tc6); int oa_tc6_write_register(struct oa_tc6 *tc6, u32 address, u32 value); +int oa_tc6_write_register_mms(struct oa_tc6 *tc6, u8 mms, u16 address, + u32 value); int oa_tc6_write_registers(struct oa_tc6 *tc6, u32 address, u32 value[], u8 length); int oa_tc6_read_register(struct oa_tc6 *tc6, u32 address, u32 *value); +int oa_tc6_read_register_mms(struct oa_tc6 *tc6, u8 mms, u16 address, + u32 *value); int oa_tc6_read_registers(struct oa_tc6 *tc6, u32 address, u32 value[], u8 length); netdev_tx_t oa_tc6_start_xmit(struct oa_tc6 *tc6, struct sk_buff *skb); int oa_tc6_zero_align_receive_frame_enable(struct oa_tc6 *tc6); +int oa_tc6_mdiobus_read_c45(struct mii_bus *bus, int addr, int devnum, + int regnum); +int oa_tc6_mdiobus_write_c45(struct mii_bus *bus, int addr, int devnum, + int regnum, u16 val); diff --git a/include/linux/objtool.h b/include/linux/objtool.h index 9a00e701454c..af2e68e496e5 100644 --- a/include/linux/objtool.h +++ b/include/linux/objtool.h @@ -7,7 +7,7 @@ #ifdef CONFIG_OBJTOOL -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #define UNWIND_HINT(type, sp_reg, sp_offset, signal) \ "987: \n\t" \ @@ -53,7 +53,7 @@ #define __ASM_BREF(label) label ## b -#else /* __ASSEMBLY__ */ +#else /* __ASSEMBLER__ */ /* * In asm, there are two kinds of code: normal C-type callable functions and @@ -102,11 +102,11 @@ #endif .endm -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ #else /* !CONFIG_OBJTOOL */ -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #define UNWIND_HINT(type, sp_reg, sp_offset, signal) "\n\t" #define STACK_FRAME_NON_STANDARD(func) diff --git a/include/linux/objtool_types.h b/include/linux/objtool_types.h index c6def4049b1a..c24e9ea39269 100644 --- a/include/linux/objtool_types.h +++ b/include/linux/objtool_types.h @@ -2,7 +2,7 @@ #ifndef _LINUX_OBJTOOL_TYPES_H #define _LINUX_OBJTOOL_TYPES_H -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #include <linux/types.h> @@ -18,7 +18,7 @@ struct unwind_hint { u8 signal; }; -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ /* * UNWIND_HINT_TYPE_UNDEFINED: A blind spot in ORC coverage which can result in diff --git a/include/linux/of.h b/include/linux/of.h index be6ec4916adf..b920aac6b975 100644 --- a/include/linux/of.h +++ b/include/linux/of.h @@ -16,7 +16,7 @@ #include <linux/cleanup.h> #include <linux/errno.h> #include <linux/kobject.h> -#include <linux/mod_devicetable.h> +#include <linux/device-id/of.h> #include <linux/property.h> #include <linux/list.h> @@ -410,7 +410,7 @@ extern int of_alias_get_id(const struct device_node *np, const char *stem); extern int of_alias_get_highest_id(const char *stem); bool of_machine_compatible_match(const char *const *compats); -bool of_machine_device_match(const struct of_device_id *matches); +const struct of_device_id *of_machine_get_match(const struct of_device_id *matches); const void *of_machine_get_match_data(const struct of_device_id *matches); /** @@ -426,6 +426,9 @@ static inline bool of_machine_is_compatible(const char *compat) return of_machine_compatible_match(compats); } +int of_machine_read_compatible(const char **compatible, unsigned int index); +int of_machine_read_model(const char **model); + extern int of_add_property(struct device_node *np, struct property *prop); extern int of_remove_property(struct device_node *np, struct property *prop); extern int of_update_property(struct device_node *np, struct property *newprop); @@ -462,8 +465,17 @@ const char *of_prop_next_string(const struct property *prop, const char *cur); bool of_console_check(const struct device_node *dn, char *name, int index); int of_map_id(const struct device_node *np, u32 id, - const char *map_name, const char *map_mask_name, - struct device_node **target, u32 *id_out); + const char *map_name, const char *cells_name, + const char *map_mask_name, + struct device_node * const *filter_np, + struct of_phandle_args *arg); + +int of_map_iommu_id(const struct device_node *np, u32 id, + struct of_phandle_args *arg); + +int of_map_msi_id(const struct device_node *np, u32 id, + struct device_node * const *filter_np, + struct of_phandle_args *arg); phys_addr_t of_dma_get_max_cpu_address(struct device_node *np); @@ -851,6 +863,17 @@ static inline int of_machine_is_compatible(const char *compat) return 0; } +static inline int of_machine_read_compatible(const char **compatible, + unsigned int index) +{ + return -ENOSYS; +} + +static inline int of_machine_read_model(const char **model) +{ + return -ENOSYS; +} + static inline int of_add_property(struct device_node *np, struct property *prop) { return 0; @@ -866,9 +889,9 @@ static inline bool of_machine_compatible_match(const char *const *compats) return false; } -static inline bool of_machine_device_match(const struct of_device_id *matches) +static inline const struct of_device_id *of_machine_get_match(const struct of_device_id *matches) { - return false; + return NULL; } static inline const void * @@ -928,8 +951,23 @@ static inline void of_property_clear_flag(struct property *p, unsigned long flag } static inline int of_map_id(const struct device_node *np, u32 id, - const char *map_name, const char *map_mask_name, - struct device_node **target, u32 *id_out) + const char *map_name, const char *cells_name, + const char *map_mask_name, + struct device_node * const *filter_np, + struct of_phandle_args *arg) +{ + return -EINVAL; +} + +static inline int of_map_iommu_id(const struct device_node *np, u32 id, + struct of_phandle_args *arg) +{ + return -EINVAL; +} + +static inline int of_map_msi_id(const struct device_node *np, u32 id, + struct device_node * const *filter_np, + struct of_phandle_args *arg) { return -EINVAL; } @@ -976,6 +1014,11 @@ static inline int of_numa_init(void) } #endif +static inline bool of_machine_device_match(const struct of_device_id *matches) +{ + return of_machine_get_match(matches) != NULL; +} + static inline struct device_node *of_find_matching_node( struct device_node *from, const struct of_device_id *matches) @@ -1457,6 +1500,13 @@ static inline int of_property_read_s32(const struct device_node *np, return of_property_read_u32(np, propname, (u32*) out_value); } +static inline int of_property_read_s32_index(const struct device_node *np, + const char *propname, u32 index, + s32 *out_value) +{ + return of_property_read_u32_index(np, propname, index, (u32 *)out_value); +} + #define of_for_each_phandle(it, err, np, ln, cn, cc) \ for (of_phandle_iterator_init((it), (np), (ln), (cn), (cc)), \ err = of_phandle_iterator_next(it); \ diff --git a/include/linux/of_dma.h b/include/linux/of_dma.h index fd706cdf255c..16b08234d03b 100644 --- a/include/linux/of_dma.h +++ b/include/linux/of_dma.h @@ -38,6 +38,26 @@ extern int of_dma_controller_register(struct device_node *np, void *data); extern void of_dma_controller_free(struct device_node *np); +static void __of_dma_controller_free(void *np) +{ + of_dma_controller_free(np); +} + +static inline int +devm_of_dma_controller_register(struct device *dev, struct device_node *np, + struct dma_chan *(*of_dma_xlate) + (struct of_phandle_args *, struct of_dma *), + void *data) +{ + int ret; + + ret = of_dma_controller_register(np, of_dma_xlate, data); + if (ret) + return ret; + + return devm_add_action_or_reset(dev, __of_dma_controller_free, np); +} + extern int of_dma_router_register(struct device_node *np, void *(*of_dma_route_allocate) (struct of_phandle_args *, struct of_dma *), @@ -64,6 +84,15 @@ static inline void of_dma_controller_free(struct device_node *np) { } +static inline int +devm_of_dma_controller_register(struct device *dev, struct device_node *np, + struct dma_chan *(*of_dma_xlate) + (struct of_phandle_args *, struct of_dma *), + void *data) +{ + return -ENODEV; +} + static inline int of_dma_router_register(struct device_node *np, void *(*of_dma_route_allocate) (struct of_phandle_args *, struct of_dma *), diff --git a/include/linux/of_gpio.h b/include/linux/of_gpio.h deleted file mode 100644 index d0f66a5e1b2a..000000000000 --- a/include/linux/of_gpio.h +++ /dev/null @@ -1,38 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0+ */ -/* - * OF helpers for the GPIO API - * - * Copyright (c) 2007-2008 MontaVista Software, Inc. - * - * Author: Anton Vorontsov <avorontsov@ru.mvista.com> - */ - -#ifndef __LINUX_OF_GPIO_H -#define __LINUX_OF_GPIO_H - -#include <linux/compiler.h> -#include <linux/gpio/driver.h> -#include <linux/gpio.h> /* FIXME: Shouldn't be here */ -#include <linux/of.h> - -struct device_node; - -#ifdef CONFIG_OF_GPIO - -extern int of_get_named_gpio(const struct device_node *np, - const char *list_name, int index); - -#else /* CONFIG_OF_GPIO */ - -#include <linux/errno.h> - -/* Drivers may not strictly depend on the GPIO support, so let them link. */ -static inline int of_get_named_gpio(const struct device_node *np, - const char *propname, int index) -{ - return -ENOSYS; -} - -#endif /* CONFIG_OF_GPIO */ - -#endif /* __LINUX_OF_GPIO_H */ diff --git a/include/linux/of_pci.h b/include/linux/of_pci.h index 29658c0ee71f..649fe8eafcfa 100644 --- a/include/linux/of_pci.h +++ b/include/linux/of_pci.h @@ -30,12 +30,18 @@ static inline void of_pci_check_probe_only(void) { } #if IS_ENABLED(CONFIG_OF_IRQ) int of_irq_parse_and_map_pci(const struct pci_dev *dev, u8 slot, u8 pin); +void pci_configure_of_wake_gpio(struct pci_dev *dev); +void pci_remove_of_wake_gpio(struct pci_dev *dev); #else static inline int of_irq_parse_and_map_pci(const struct pci_dev *dev, u8 slot, u8 pin) { return 0; } + +static inline void pci_configure_of_wake_gpio(struct pci_dev *dev) { } + +static inline void pci_remove_of_wake_gpio(struct pci_dev *dev) { } #endif #endif diff --git a/include/linux/of_platform.h b/include/linux/of_platform.h index 17471ef8e092..181c438a9b0a 100644 --- a/include/linux/of_platform.h +++ b/include/linux/of_platform.h @@ -6,7 +6,9 @@ * <benh@kernel.crashing.org> */ -#include <linux/mod_devicetable.h> +#include <linux/types.h> +#include <linux/err.h> +#include <linux/device-id/of.h> struct device; struct device_node; diff --git a/include/linux/of_reserved_mem.h b/include/linux/of_reserved_mem.h index f573423359f4..49da515859f2 100644 --- a/include/linux/of_reserved_mem.h +++ b/include/linux/of_reserved_mem.h @@ -11,7 +11,6 @@ struct resource; struct reserved_mem { const char *name; - unsigned long fdt_node; const struct reserved_mem_ops *ops; phys_addr_t base; phys_addr_t size; @@ -19,18 +18,20 @@ struct reserved_mem { }; struct reserved_mem_ops { + int (*node_validate)(unsigned long fdt_node, phys_addr_t *align); + int (*node_fixup)(unsigned long fdt_node, phys_addr_t base, + phys_addr_t size); + int (*node_init)(unsigned long fdt_node, struct reserved_mem *rmem); int (*device_init)(struct reserved_mem *rmem, struct device *dev); void (*device_release)(struct reserved_mem *rmem, struct device *dev); }; -typedef int (*reservedmem_of_init_fn)(struct reserved_mem *rmem); - #ifdef CONFIG_OF_RESERVED_MEM -#define RESERVEDMEM_OF_DECLARE(name, compat, init) \ - _OF_DECLARE(reservedmem, name, compat, init, reservedmem_of_init_fn) +#define RESERVEDMEM_OF_DECLARE(name, compat, ops) \ + _OF_DECLARE(reservedmem, name, compat, ops, struct reserved_mem_ops *) int of_reserved_mem_device_init_by_idx(struct device *dev, struct device_node *np, int idx); @@ -39,6 +40,8 @@ int of_reserved_mem_device_init_by_name(struct device *dev, const char *name); void of_reserved_mem_device_release(struct device *dev); +int devm_of_reserved_mem_device_init(struct device *dev); + struct reserved_mem *of_reserved_mem_lookup(struct device_node *np); int of_reserved_mem_region_to_resource(const struct device_node *np, unsigned int idx, struct resource *res); @@ -48,8 +51,9 @@ int of_reserved_mem_region_count(const struct device_node *np); #else -#define RESERVEDMEM_OF_DECLARE(name, compat, init) \ - _OF_DECLARE_STUB(reservedmem, name, compat, init, reservedmem_of_init_fn) +#define RESERVEDMEM_OF_DECLARE(name, compat, ops) \ + _OF_DECLARE_STUB(reservedmem, name, compat, ops, \ + struct reserved_mem_ops *) static inline int of_reserved_mem_device_init_by_idx(struct device *dev, struct device_node *np, int idx) @@ -66,6 +70,11 @@ static inline int of_reserved_mem_device_init_by_name(struct device *dev, static inline void of_reserved_mem_device_release(struct device *pdev) { } +static inline int devm_of_reserved_mem_device_init(struct device *dev) +{ + return -EOPNOTSUPP; +} + static inline struct reserved_mem *of_reserved_mem_lookup(struct device_node *np) { return NULL; diff --git a/include/linux/oom.h b/include/linux/oom.h index 7b02bc1d0a7e..00da05d227e6 100644 --- a/include/linux/oom.h +++ b/include/linux/oom.h @@ -30,7 +30,7 @@ struct oom_control { struct zonelist *zonelist; /* Used to determine mempolicy */ - nodemask_t *nodemask; + const nodemask_t *nodemask; /* Memory cgroup in which oom is invoked, or NULL for global oom */ struct mem_cgroup *memcg; diff --git a/include/linux/padata.h b/include/linux/padata.h index 765f2778e264..b6232bea6edf 100644 --- a/include/linux/padata.h +++ b/include/linux/padata.h @@ -149,23 +149,23 @@ struct padata_mt_job { /** * struct padata_instance - The overall control structure. * - * @cpu_online_node: Linkage for CPU online callback. - * @cpu_dead_node: Linkage for CPU offline callback. + * @cpuhp_node: Linkage for CPU hotplug callbacks. * @parallel_wq: The workqueue used for parallel work. * @serial_wq: The workqueue used for serial work. * @pslist: List of padata_shell objects attached to this instance. * @cpumask: User supplied cpumasks for parallel and serial works. + * @validate_cpumask: Internal cpumask used to validate @cpumask during hotplug. * @kobj: padata instance kernel object. * @lock: padata instance lock. * @flags: padata flags. */ struct padata_instance { - struct hlist_node cpu_online_node; - struct hlist_node cpu_dead_node; + struct hlist_node cpuhp_node; struct workqueue_struct *parallel_wq; struct workqueue_struct *serial_wq; struct list_head pslist; struct padata_cpumask cpumask; + cpumask_var_t validate_cpumask; struct kobject kobj; struct mutex lock; u8 flags; diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h index f7a0e4af0c73..7a863572adce 100644 --- a/include/linux/page-flags.h +++ b/include/linux/page-flags.h @@ -198,97 +198,91 @@ enum pageflags { #ifndef __GENERATING_BOUNDS_H -#ifdef CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP -DECLARE_STATIC_KEY_FALSE(hugetlb_optimize_vmemmap_key); - /* - * Return the real head page struct iff the @page is a fake head page, otherwise - * return the @page itself. See Documentation/mm/vmemmap_dedup.rst. + * For tail pages, if the size of struct page is power-of-2 ->compound_info + * encodes the mask that converts the address of the tail page address to + * the head page address. + * + * Otherwise, ->compound_info has direct pointer to head pages. */ -static __always_inline const struct page *page_fixed_fake_head(const struct page *page) +static __always_inline bool compound_info_has_mask(void) { - if (!static_branch_unlikely(&hugetlb_optimize_vmemmap_key)) - return page; - /* - * Only addresses aligned with PAGE_SIZE of struct page may be fake head - * struct page. The alignment check aims to avoid access the fields ( - * e.g. compound_head) of the @page[1]. It can avoid touch a (possibly) - * cold cacheline in some cases. + * Limit mask usage to HugeTLB vmemmap optimization (HVO) where it + * makes a difference. + * + * The approach with mask would work in the wider set of conditions, + * but it requires validating that struct pages are naturally aligned + * for all orders up to the MAX_FOLIO_ORDER, which can be tricky. */ - if (IS_ALIGNED((unsigned long)page, PAGE_SIZE) && - test_bit(PG_head, &page->flags.f)) { - /* - * We can safely access the field of the @page[1] with PG_head - * because the @page is a compound page composed with at least - * two contiguous pages. - */ - unsigned long head = READ_ONCE(page[1].compound_head); - - if (likely(head & 1)) - return (const struct page *)(head - 1); - } - return page; + if (!IS_ENABLED(CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP)) + return false; + + return is_power_of_2(sizeof(struct page)); } -static __always_inline bool page_count_writable(const struct page *page, int u) +static __always_inline unsigned long _compound_head(const struct page *page) { - if (!static_branch_unlikely(&hugetlb_optimize_vmemmap_key)) - return true; + unsigned long info = READ_ONCE(page->compound_info); + unsigned long mask; + + if (!compound_info_has_mask()) { + /* Bit 0 encodes PageTail() */ + if (info & 1) + return info - 1; + + return (unsigned long)page; + } /* - * The refcount check is ordered before the fake-head check to prevent - * the following race: - * CPU 1 (HVO) CPU 2 (speculative PFN walker) - * - * page_ref_freeze() - * synchronize_rcu() - * rcu_read_lock() - * page_is_fake_head() is false - * vmemmap_remap_pte() - * XXX: struct page[] becomes r/o - * - * page_ref_unfreeze() - * page_ref_count() is not zero + * If compound_info_has_mask() is true the rest of the info encodes + * the mask that converts the address of the tail page to the head page. * - * atomic_add_unless(&page->_refcount) - * XXX: try to modify r/o struct page[] + * No need to clear bit 0 in the mask as 'page' always has it clear. * - * The refcount check also prevents modification attempts to other (r/o) - * tail pages that are not fake heads. + * Let's do it in a branchless manner. */ - if (atomic_read_acquire(&page->_refcount) == u) - return false; - return page_fixed_fake_head(page) == page; -} -#else -static inline const struct page *page_fixed_fake_head(const struct page *page) -{ - return page; -} + /* Non-tail: -1UL, Tail: 0 */ + mask = (info & 1) - 1; -static inline bool page_count_writable(const struct page *page, int u) -{ - return true; -} -#endif + /* Non-tail: -1UL, Tail: info */ + mask |= info; -static __always_inline int page_is_fake_head(const struct page *page) -{ - return page_fixed_fake_head(page) != page; + return (unsigned long)page & mask; } -static __always_inline unsigned long _compound_head(const struct page *page) +#define compound_head(page) ((typeof(page))_compound_head(page)) + +static __always_inline void set_compound_head(struct page *tail, + const struct page *head, unsigned int order) { - unsigned long head = READ_ONCE(page->compound_head); + unsigned int shift; + unsigned long mask; + + if (!compound_info_has_mask()) { + WRITE_ONCE(tail->compound_info, (unsigned long)head | 1); + return; + } - if (unlikely(head & 1)) - return head - 1; - return (unsigned long)page_fixed_fake_head(page); + /* + * If the size of struct page is power-of-2, bits [shift:0] of the + * virtual address of compound head are zero. + * + * Calculate mask that can be applied to the virtual address of + * the tail page to get address of the head page. + */ + shift = order + order_base_2(sizeof(struct page)); + mask = GENMASK(BITS_PER_LONG - 1, shift); + + /* Bit 0 encodes PageTail() */ + WRITE_ONCE(tail->compound_info, mask | 1); } -#define compound_head(page) ((typeof(page))_compound_head(page)) +static __always_inline void clear_compound_head(struct page *page) +{ + WRITE_ONCE(page->compound_info, 0); +} /** * page_folio - Converts from page to folio. @@ -320,13 +314,13 @@ static __always_inline unsigned long _compound_head(const struct page *page) static __always_inline int PageTail(const struct page *page) { - return READ_ONCE(page->compound_head) & 1 || page_is_fake_head(page); + return READ_ONCE(page->compound_info) & 1; } static __always_inline int PageCompound(const struct page *page) { return test_bit(PG_head, &page->flags.f) || - READ_ONCE(page->compound_head) & 1; + READ_ONCE(page->compound_info) & 1; } #define PAGE_POISON_PATTERN -1l @@ -348,7 +342,7 @@ static const unsigned long *const_folio_flags(const struct folio *folio, { const struct page *page = &folio->page; - VM_BUG_ON_PGFLAGS(page->compound_head & 1, page); + VM_BUG_ON_PGFLAGS(page->compound_info & 1, page); VM_BUG_ON_PGFLAGS(n > 0 && !test_bit(PG_head, &page->flags.f), page); return &page[n].flags.f; } @@ -357,7 +351,7 @@ static unsigned long *folio_flags(struct folio *folio, unsigned n) { struct page *page = &folio->page; - VM_BUG_ON_PGFLAGS(page->compound_head & 1, page); + VM_BUG_ON_PGFLAGS(page->compound_info & 1, page); VM_BUG_ON_PGFLAGS(n > 0 && !test_bit(PG_head, &page->flags.f), page); return &page[n].flags.f; } @@ -724,6 +718,11 @@ static __always_inline bool folio_test_anon(const struct folio *folio) return ((unsigned long)folio->mapping & FOLIO_MAPPING_ANON) != 0; } +static __always_inline bool folio_test_lazyfree(const struct folio *folio) +{ + return folio_test_anon(folio) && !folio_test_swapbacked(folio); +} + static __always_inline bool PageAnonNotKsm(const struct page *page) { unsigned long flags = (unsigned long)page_folio(page)->mapping; @@ -847,7 +846,7 @@ static __always_inline bool folio_test_head(const struct folio *folio) static __always_inline int PageHead(const struct page *page) { PF_POISONED_CHECK(page); - return test_bit(PG_head, &page->flags.f) && !page_is_fake_head(page); + return test_bit(PG_head, &page->flags.f); } __SETPAGEFLAG(Head, head, PF_ANY) @@ -865,16 +864,6 @@ static inline bool folio_test_large(const struct folio *folio) return folio_test_head(folio); } -static __always_inline void set_compound_head(struct page *page, struct page *head) -{ - WRITE_ONCE(page->compound_head, (unsigned long)head + 1); -} - -static __always_inline void clear_compound_head(struct page *page) -{ - WRITE_ONCE(page->compound_head, 0); -} - #ifdef CONFIG_TRANSPARENT_HUGEPAGE static inline void ClearPageCompound(struct page *page) { @@ -890,20 +879,6 @@ FOLIO_FLAG_FALSE(partially_mapped) #define PG_head_mask ((1UL << PG_head)) -#ifdef CONFIG_TRANSPARENT_HUGEPAGE -/* - * PageTransCompound returns true for both transparent huge pages - * and hugetlbfs pages, so it should only be called when it's known - * that hugetlbfs pages aren't involved. - */ -static inline int PageTransCompound(const struct page *page) -{ - return PageCompound(page); -} -#else -TESTPAGEFLAG_FALSE(TransCompound, transcompound) -#endif - #if defined(CONFIG_MEMORY_FAILURE) && defined(CONFIG_TRANSPARENT_HUGEPAGE) /* * PageHasHWPoisoned indicates that at least one subpage is hwpoisoned in the diff --git a/include/linux/page_ext.h b/include/linux/page_ext.h index 61e876e255e8..79c53ec45dfa 100644 --- a/include/linux/page_ext.h +++ b/include/linux/page_ext.h @@ -55,7 +55,6 @@ struct page_ext { extern bool early_page_ext; extern unsigned long page_ext_size; -extern void pgdat_page_ext_init(struct pglist_data *pgdat); static inline bool early_page_ext_enabled(void) { @@ -120,14 +119,18 @@ struct page_ext_iter { * page_ext_iter_begin() - Prepare for iterating through page extensions. * @iter: page extension iterator. * @pfn: PFN of the page we're interested in. + * @count: maximum number of page extensions to return. * * Must be called with RCU read lock taken. * * Return: NULL if no page_ext exists for this page. */ static inline struct page_ext *page_ext_iter_begin(struct page_ext_iter *iter, - unsigned long pfn) + unsigned long pfn, unsigned long count) { + if (!count) + return NULL; + iter->index = 0; iter->start_pfn = pfn; iter->page_ext = page_ext_lookup(pfn); @@ -138,19 +141,22 @@ static inline struct page_ext *page_ext_iter_begin(struct page_ext_iter *iter, /** * page_ext_iter_next() - Get next page extension * @iter: page extension iterator. + * @count: maximum number of page extensions to return. * * Must be called with RCU read lock taken. * * Return: NULL if no next page_ext exists. */ -static inline struct page_ext *page_ext_iter_next(struct page_ext_iter *iter) +static inline struct page_ext *page_ext_iter_next(struct page_ext_iter *iter, + unsigned long count) { unsigned long pfn; if (WARN_ON_ONCE(!iter->page_ext)) return NULL; - iter->index++; + if (++iter->index >= count) + return NULL; pfn = iter->start_pfn + iter->index; if (page_ext_iter_next_fast_possible(pfn)) @@ -183,9 +189,9 @@ static inline struct page_ext *page_ext_iter_get(const struct page_ext_iter *ite * IMPORTANT: must be called with RCU read lock taken. */ #define for_each_page_ext(__page, __pgcount, __page_ext, __iter) \ - for (__page_ext = page_ext_iter_begin(&__iter, page_to_pfn(__page));\ - __page_ext && __iter.index < __pgcount; \ - __page_ext = page_ext_iter_next(&__iter)) + for (__page_ext = page_ext_iter_begin(&__iter, page_to_pfn(__page), __pgcount); \ + __page_ext; \ + __page_ext = page_ext_iter_next(&__iter, __pgcount)) #else /* !CONFIG_PAGE_EXTENSION */ struct page_ext; @@ -195,10 +201,6 @@ static inline bool early_page_ext_enabled(void) return false; } -static inline void pgdat_page_ext_init(struct pglist_data *pgdat) -{ -} - static inline void page_ext_init(void) { } diff --git a/include/linux/page_owner.h b/include/linux/page_owner.h index 3328357f6dba..8188ddc5c412 100644 --- a/include/linux/page_owner.h +++ b/include/linux/page_owner.h @@ -3,6 +3,7 @@ #define __LINUX_PAGE_OWNER_H #include <linux/jump_label.h> +#include <linux/migrate_mode.h> #ifdef CONFIG_PAGE_OWNER extern struct static_key_false page_owner_inited; @@ -14,7 +15,7 @@ extern void __set_page_owner(struct page *page, extern void __split_page_owner(struct page *page, int old_order, int new_order); extern void __folio_copy_owner(struct folio *newfolio, struct folio *old); -extern void __folio_set_owner_migrate_reason(struct folio *folio, int reason); +void __folio_set_owner_migrate_reason(struct folio *folio, enum migrate_reason reason); extern void __dump_page_owner(const struct page *page); extern void pagetypeinfo_showmixedcount_print(struct seq_file *m, pg_data_t *pgdat, struct zone *zone); @@ -43,7 +44,7 @@ static inline void folio_copy_owner(struct folio *newfolio, struct folio *old) if (static_branch_unlikely(&page_owner_inited)) __folio_copy_owner(newfolio, old); } -static inline void folio_set_owner_migrate_reason(struct folio *folio, int reason) +static inline void folio_set_owner_migrate_reason(struct folio *folio, enum migrate_reason reason) { if (static_branch_unlikely(&page_owner_inited)) __folio_set_owner_migrate_reason(folio, reason); @@ -68,7 +69,7 @@ static inline void split_page_owner(struct page *page, int old_order, static inline void folio_copy_owner(struct folio *newfolio, struct folio *folio) { } -static inline void folio_set_owner_migrate_reason(struct folio *folio, int reason) +static inline void folio_set_owner_migrate_reason(struct folio *folio, enum migrate_reason reason) { } static inline void dump_page_owner(const struct page *page) diff --git a/include/linux/page_ref.h b/include/linux/page_ref.h index 544150d1d5fd..9f5c75d06f76 100644 --- a/include/linux/page_ref.h +++ b/include/linux/page_ref.h @@ -71,6 +71,12 @@ static inline int page_ref_count(const struct page *page) * folio_ref_count - The reference count on this folio. * @folio: The folio. * + * Folios contain a reference count. When that reference count reaches + * zero, the folio is referred to as frozen. At this point, it will + * usually be returned to the memory allocator, but some parts of the + * kernel freeze folios in order to perform unusual operations on them + * such as splitting or migration. + * * The refcount is usually incremented by calls to folio_get() and * decremented by calls to folio_put(). Some typical users of the * folio refcount: @@ -82,6 +88,18 @@ static inline int page_ref_count(const struct page *page) * - Pipes * - Direct IO which references this page in the process address space * + * The reference count has three components: expected, temporary and + * spurious. The expected reference count of a folio is that which + * we would logically expect it to be from just reading the code. + * Temporary refcounts are gained by threads which need a temporary + * reference to make sure the folio isn't reallocated while they use it. + * Spurious refcounts are gained by threads which, thanks to RCU walks + * of the page tables or file cache, find a stale pointer to a folio. + * These threads will drop the refcount after discoveering the pointer + * is stale, but it can surprise other users to see the spurious refcount + * on a freshly allocated folio (eg they may see a refcount of 2 instead + * of 1). + * * Return: The number of references to this folio. */ static inline int folio_ref_count(const struct folio *folio) @@ -228,24 +246,18 @@ static inline int folio_ref_dec_return(struct folio *folio) return page_ref_dec_return(&folio->page); } -static inline bool page_ref_add_unless(struct page *page, int nr, int u) +static inline bool page_ref_add_unless_zero(struct page *page, int nr) { - bool ret = false; - - rcu_read_lock(); - /* avoid writing to the vmemmap area being remapped */ - if (page_count_writable(page, u)) - ret = atomic_add_unless(&page->_refcount, nr, u); - rcu_read_unlock(); + bool ret = atomic_add_unless(&page->_refcount, nr, 0); if (page_ref_tracepoint_active(page_ref_mod_unless)) __page_ref_mod_unless(page, nr, ret); return ret; } -static inline bool folio_ref_add_unless(struct folio *folio, int nr, int u) +static inline bool folio_ref_add_unless_zero(struct folio *folio, int nr) { - return page_ref_add_unless(&folio->page, nr, u); + return page_ref_add_unless_zero(&folio->page, nr); } /** @@ -261,12 +273,12 @@ static inline bool folio_ref_add_unless(struct folio *folio, int nr, int u) */ static inline bool folio_try_get(struct folio *folio) { - return folio_ref_add_unless(folio, 1, 0); + return folio_ref_add_unless_zero(folio, 1); } static inline bool folio_ref_try_add(struct folio *folio, int count) { - return folio_ref_add_unless(folio, count, 0); + return folio_ref_add_unless_zero(folio, count); } static inline int page_ref_freeze(struct page *page, int count) diff --git a/include/linux/page_reporting.h b/include/linux/page_reporting.h index fe648dfa3a7c..272b1274efdc 100644 --- a/include/linux/page_reporting.h +++ b/include/linux/page_reporting.h @@ -5,8 +5,8 @@ #include <linux/mmzone.h> #include <linux/scatterlist.h> -/* This value should always be a power of 2, see page_reporting_cycle() */ #define PAGE_REPORTING_CAPACITY 32 +#define PAGE_REPORTING_ORDER_UNSPECIFIED -1 struct page_reporting_dev_info { /* function that alters pages to make them "reported" */ @@ -21,6 +21,9 @@ struct page_reporting_dev_info { /* Minimal order of page reporting */ unsigned int order; + + /* Max pages per report batch; 0 (default) means PAGE_REPORTING_CAPACITY */ + unsigned int capacity; }; /* Tear-down and bring-up for page reporting devices */ diff --git a/include/linux/pageblock-flags.h b/include/linux/pageblock-flags.h index e046278a01fa..9a6c3ea17684 100644 --- a/include/linux/pageblock-flags.h +++ b/include/linux/pageblock-flags.h @@ -36,12 +36,12 @@ enum pageblock_bits { #define NR_PAGEBLOCK_BITS (roundup_pow_of_two(__NR_PAGEBLOCK_BITS)) -#define MIGRATETYPE_MASK (BIT(PB_migrate_0)|BIT(PB_migrate_1)|BIT(PB_migrate_2)) +#define PAGEBLOCK_MIGRATETYPE_MASK (BIT(PB_migrate_0)|BIT(PB_migrate_1)|BIT(PB_migrate_2)) #ifdef CONFIG_MEMORY_ISOLATION -#define MIGRATETYPE_AND_ISO_MASK (MIGRATETYPE_MASK | BIT(PB_migrate_isolate)) +#define PAGEBLOCK_ISO_MASK BIT(PB_migrate_isolate) #else -#define MIGRATETYPE_AND_ISO_MASK MIGRATETYPE_MASK +#define PAGEBLOCK_ISO_MASK 0 #endif #if defined(CONFIG_HUGETLB_PAGE) diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h index ec442af3f886..0adfa6605653 100644 --- a/include/linux/pagemap.h +++ b/include/linux/pagemap.h @@ -210,7 +210,6 @@ enum mapping_flags { AS_WRITEBACK_MAY_DEADLOCK_ON_RECLAIM = 9, AS_KERNEL_FILE = 10, /* mapping for a fake kernel file that shouldn't account usage to user cgroups */ - AS_NO_DATA_INTEGRITY = 11, /* no data integrity guarantees */ /* Bits 16-25 are used for FOLIO_ORDER */ AS_FOLIO_ORDER_BITS = 5, AS_FOLIO_ORDER_MIN = 16, @@ -346,16 +345,6 @@ static inline bool mapping_writeback_may_deadlock_on_reclaim(const struct addres return test_bit(AS_WRITEBACK_MAY_DEADLOCK_ON_RECLAIM, &mapping->flags); } -static inline void mapping_set_no_data_integrity(struct address_space *mapping) -{ - set_bit(AS_NO_DATA_INTEGRITY, &mapping->flags); -} - -static inline bool mapping_no_data_integrity(const struct address_space *mapping) -{ - return test_bit(AS_NO_DATA_INTEGRITY, &mapping->flags); -} - static inline gfp_t mapping_gfp_mask(const struct address_space *mapping) { return mapping->gfp_mask; @@ -524,39 +513,37 @@ static inline bool mapping_large_folio_support(const struct address_space *mappi return mapping_max_folio_order(mapping) > 0; } -/* Return the maximum folio size for this pagecache mapping, in bytes. */ -static inline size_t mapping_max_folio_size(const struct address_space *mapping) +/** + * mapping_pmd_folio_support() - Check if a mapping supports PMD-sized folio + * @mapping: The address_space + * + * While some mappings support large folios, they might not support PMD-sized + * folios. This function checks whether a mapping supports PMD-sized folios. + * For example, khugepaged needs this information before attempting to + * collapsing THPs. + * + * Return: True if PMD-sized folios are supported, otherwise false. + */ +#ifdef CONFIG_TRANSPARENT_HUGEPAGE +static inline bool mapping_pmd_folio_support(const struct address_space *mapping) { - return PAGE_SIZE << mapping_max_folio_order(mapping); -} + /* AS_FOLIO_ORDER is only reasonable for pagecache folios */ + VM_WARN_ON_ONCE((unsigned long)mapping & FOLIO_MAPPING_ANON); -static inline int filemap_nr_thps(const struct address_space *mapping) -{ -#ifdef CONFIG_READ_ONLY_THP_FOR_FS - return atomic_read(&mapping->nr_thps); -#else - return 0; -#endif + return mapping_min_folio_order(mapping) <= PMD_ORDER && + mapping_max_folio_order(mapping) >= PMD_ORDER; } - -static inline void filemap_nr_thps_inc(struct address_space *mapping) -{ -#ifdef CONFIG_READ_ONLY_THP_FOR_FS - if (!mapping_large_folio_support(mapping)) - atomic_inc(&mapping->nr_thps); #else - WARN_ON_ONCE(mapping_large_folio_support(mapping) == 0); -#endif +static inline bool mapping_pmd_folio_support(const struct address_space *mapping) +{ + return false; } +#endif -static inline void filemap_nr_thps_dec(struct address_space *mapping) +/* Return the maximum folio size for this pagecache mapping, in bytes. */ +static inline size_t mapping_max_folio_size(const struct address_space *mapping) { -#ifdef CONFIG_READ_ONLY_THP_FOR_FS - if (!mapping_large_folio_support(mapping)) - atomic_dec(&mapping->nr_thps); -#else - WARN_ON_ONCE(mapping_large_folio_support(mapping) == 0); -#endif + return PAGE_SIZE << mapping_max_folio_order(mapping); } struct address_space *folio_mapping(const struct folio *folio); @@ -1076,12 +1063,75 @@ static inline pgoff_t folio_pgoff(const struct folio *folio) return folio->index; } +/** + * linear_page_delta() - Determine the relative page offset of @address within + * @vma. + * @vma: The VMA in which @address resides. + * @address: The address whose relative page offset is required. + * + * The result is identical for both file-backed and anonymous mappings and + * simply determines how many pages @address lies from @vma->vm_start. + * + * Returns: The number of pages @address is offset by within @vma. + */ +static inline pgoff_t linear_page_delta(const struct vm_area_struct *vma, + const unsigned long address) +{ + return (address - vma->vm_start) >> PAGE_SHIFT; +} + +/** + * linear_page_index() - Determine the absolute page offset of @address within + * @vma. + * @vma: The VMA in which @address resides. + * @address: The address whose absolute page offset is required. + * + * See the comment for vma_start_pgoff() for a description of what the page + * offset signifies. + * + * Returns: The absolute page offset of @address within @vma. + */ static inline pgoff_t linear_page_index(const struct vm_area_struct *vma, const unsigned long address) { - pgoff_t pgoff; - pgoff = (address - vma->vm_start) >> PAGE_SHIFT; - pgoff += vma->vm_pgoff; + return linear_page_delta(vma, address) + vma_start_pgoff(vma); +} + +static inline pgoff_t __linear_anon_page_index(const struct vm_area_struct *vma, + const unsigned long address) +{ + return linear_page_delta(vma, address) + vma_start_anon_pgoff(vma); +} + +/** + * linear_anon_page_index() - Determine the absolute anonymous page offset of + * @address within @vma. + * @vma: An anonymous or MAP_PRIVATE file-backed VMA in which @address resides. + * @address: The address whose absolute page offset is required. + * + * This returns the anonymous page offset of @address, which is the page offset + * the address possessed at the time the VMA was first faulted. + * + * For anonymous mappings, this returns the same value as linear_page_index(). + * + * For MAP_PRIVATE file-backed mappings, this returns the anonymous page offset + * of @address, which is the page offset the address possessed at the time the + * VMA was first faulted. + * + * It is not valid to call this function for shared file-backed mappings. + * + * Returns: The absolute anonymous page offset of @address within @vma. + */ +static inline pgoff_t linear_anon_page_index(const struct vm_area_struct *vma, + const unsigned long address) +{ + const pgoff_t pgoff = __linear_anon_page_index(vma, address); + + VM_WARN_ON_ONCE(!vma_is_cow_mapping(vma)); + /* Account for MAP_PRIVATE-/dev/zero which is only semi-anonymous. */ + if (vma_is_anonymous(vma) && !vma->vm_file) + VM_WARN_ON_ONCE(pgoff != linear_page_index(vma, address)); + return pgoff; } @@ -1232,7 +1282,7 @@ static inline vm_fault_t folio_lock_or_retry(struct folio *folio, void folio_wait_bit(struct folio *folio, int bit_nr); int folio_wait_bit_killable(struct folio *folio, int bit_nr); -/* +/* * Wait for a folio to be unlocked. * * This must be called with the caller "holding" the folio, diff --git a/include/linux/pagewalk.h b/include/linux/pagewalk.h index 88e18615dd72..b41d7265c01b 100644 --- a/include/linux/pagewalk.h +++ b/include/linux/pagewalk.h @@ -148,14 +148,8 @@ int walk_page_mapping(struct address_space *mapping, pgoff_t first_index, typedef int __bitwise folio_walk_flags_t; -/* - * Walk migration entries as well. Careful: a large folio might get split - * concurrently. - */ -#define FW_MIGRATION ((__force folio_walk_flags_t)BIT(0)) - /* Walk shared zeropages (small + huge) as well. */ -#define FW_ZEROPAGE ((__force folio_walk_flags_t)BIT(1)) +#define FW_ZEROPAGE ((__force folio_walk_flags_t)BIT(0)) enum folio_walk_level { FW_LEVEL_PTE, diff --git a/include/linux/parport.h b/include/linux/parport.h index 464c2ad28039..f64cb0676e3b 100644 --- a/include/linux/parport.h +++ b/include/linux/parport.h @@ -240,6 +240,7 @@ struct parport { unsigned long devflags; #define PARPORT_DEVPROC_REGISTERED 0 +#define PARPORT_ANNOUNCED 1 struct pardevice *proc_device; /* Currently register proc device */ struct list_head full_list; diff --git a/include/linux/pci-ats.h b/include/linux/pci-ats.h index 75c6c86cf09d..f3723b686129 100644 --- a/include/linux/pci-ats.h +++ b/include/linux/pci-ats.h @@ -12,6 +12,7 @@ int pci_prepare_ats(struct pci_dev *dev, int ps); void pci_disable_ats(struct pci_dev *dev); int pci_ats_queue_depth(struct pci_dev *dev); int pci_ats_page_aligned(struct pci_dev *dev); +bool pci_ats_required(struct pci_dev *dev); #else /* CONFIG_PCI_ATS */ static inline bool pci_ats_supported(struct pci_dev *d) { return false; } @@ -24,6 +25,8 @@ static inline int pci_ats_queue_depth(struct pci_dev *d) { return -ENODEV; } static inline int pci_ats_page_aligned(struct pci_dev *dev) { return 0; } +static inline bool pci_ats_required(struct pci_dev *dev) +{ return false; } #endif /* CONFIG_PCI_ATS */ #ifdef CONFIG_PCI_PRI diff --git a/include/linux/pci-ecam.h b/include/linux/pci-ecam.h index d930651473b4..044f67ced6ff 100644 --- a/include/linux/pci-ecam.h +++ b/include/linux/pci-ecam.h @@ -81,6 +81,9 @@ void __iomem *pci_ecam_map_bus(struct pci_bus *bus, unsigned int devfn, /* default ECAM ops */ extern const struct pci_ecam_ops pci_generic_ecam_ops; +/* default CAM ops */ +extern const struct pci_ecam_ops pci_generic_cam_ops; + #if defined(CONFIG_ACPI) && defined(CONFIG_PCI_QUIRKS) extern const struct pci_ecam_ops pci_32b_ops; /* 32-bit accesses only */ extern const struct pci_ecam_ops pci_32b_read_ops; /* 32-bit read only */ diff --git a/include/linux/pci-epc.h b/include/linux/pci-epc.h index c021c7af175f..f247cf9bcf1a 100644 --- a/include/linux/pci-epc.h +++ b/include/linux/pci-epc.h @@ -62,6 +62,47 @@ struct pci_epc_map { }; /** + * enum pci_epc_aux_resource_type - auxiliary resource type identifiers + * @PCI_EPC_AUX_DOORBELL_MMIO: Doorbell MMIO, that might be outside the DMA + * controller register window + * + * EPC backends may expose auxiliary blocks (e.g. DMA engines) by mapping their + * register windows and descriptor memories into BAR space. This enum + * identifies the type of each exposable resource. + */ +enum pci_epc_aux_resource_type { + PCI_EPC_AUX_DOORBELL_MMIO, +}; + +/** + * struct pci_epc_aux_resource - a physical auxiliary resource that may be + * exposed for peer use + * @type: resource type, see enum pci_epc_aux_resource_type + * @phys_addr: physical base address of the resource + * @size: size of the resource in bytes + * @bar: BAR number where this resource is already exposed to the RC + * (NO_BAR if not) + * @bar_offset: offset within @bar where the resource starts (valid iff + * @bar != NO_BAR) + * @u: type-specific metadata + */ +struct pci_epc_aux_resource { + enum pci_epc_aux_resource_type type; + phys_addr_t phys_addr; + resource_size_t size; + enum pci_barno bar; + resource_size_t bar_offset; + + union { + /* PCI_EPC_AUX_DOORBELL_MMIO */ + struct { + int irq; /* IRQ number for the doorbell handler */ + u32 data; /* write value to ring the doorbell */ + } db_mmio; + } u; +}; + +/** * struct pci_epc_ops - set of function pointers for performing EPC operations * @write_header: ops to populate configuration space header * @set_bar: ops to configure the BAR @@ -84,6 +125,9 @@ struct pci_epc_map { * @start: ops to start the PCI link * @stop: ops to stop the PCI link * @get_features: ops to get the features supported by the EPC + * @get_aux_resources_count: ops to get the number of controller-owned + * auxiliary resources + * @get_aux_resources: ops to retrieve controller-owned auxiliary resources * @owner: the module owner containing the ops */ struct pci_epc_ops { @@ -115,6 +159,11 @@ struct pci_epc_ops { void (*stop)(struct pci_epc *epc); const struct pci_epc_features* (*get_features)(struct pci_epc *epc, u8 func_no, u8 vfunc_no); + int (*get_aux_resources_count)(struct pci_epc *epc, u8 func_no, + u8 vfunc_no); + int (*get_aux_resources)(struct pci_epc *epc, u8 func_no, u8 vfunc_no, + struct pci_epc_aux_resource *resources, + int num_resources); struct module *owner; }; @@ -191,13 +240,49 @@ struct pci_epc { * @BAR_RESIZABLE: The BAR implements the PCI-SIG Resizable BAR Capability. * NOTE: An EPC driver can currently only set a single supported * size. - * @BAR_RESERVED: The BAR should not be touched by an EPF driver. + * @BAR_RESERVED: Used for HW-backed BARs (e.g. MSI-X table, DMA regs). The BAR + * should not be disabled by an EPC driver. The BAR should not be + * reprogrammed by an EPF driver. An EPF driver is allowed to + * disable the BAR if absolutely necessary. (However, right now + * there is no EPC operation to disable a BAR that has not been + * programmed using pci_epc_set_bar().) + * @BAR_DISABLED: The BAR should be disabled by an EPC driver. The BAR will be + * unavailable to an EPF driver. */ enum pci_epc_bar_type { BAR_PROGRAMMABLE = 0, BAR_FIXED, BAR_RESIZABLE, BAR_RESERVED, + BAR_DISABLED, +}; + +/** + * enum pci_epc_bar_rsvd_region_type - type of a fixed subregion behind a BAR + * @PCI_EPC_BAR_RSVD_DMA_CTRL_MMIO: Integrated DMA controller MMIO window + * @PCI_EPC_BAR_RSVD_MSIX_TBL_RAM: MSI-X table structure + * @PCI_EPC_BAR_RSVD_MSIX_PBA_RAM: MSI-X PBA structure + * + * BARs marked BAR_RESERVED are owned by the SoC/EPC hardware and must not be + * reprogrammed by EPF drivers. Some of them still expose fixed subregions that + * EPFs may want to reference (e.g. embedded doorbell fallback). + */ +enum pci_epc_bar_rsvd_region_type { + PCI_EPC_BAR_RSVD_DMA_CTRL_MMIO = 0, + PCI_EPC_BAR_RSVD_MSIX_TBL_RAM, + PCI_EPC_BAR_RSVD_MSIX_PBA_RAM, +}; + +/** + * struct pci_epc_bar_rsvd_region - fixed subregion behind a BAR + * @type: reserved region type + * @offset: offset within the BAR aperture + * @size: size of the reserved region + */ +struct pci_epc_bar_rsvd_region { + enum pci_epc_bar_rsvd_region_type type; + resource_size_t offset; + resource_size_t size; }; /** @@ -206,18 +291,16 @@ enum pci_epc_bar_type { * @fixed_size: the fixed size, only applicable if type is BAR_FIXED_MASK. * @only_64bit: if true, an EPF driver is not allowed to choose if this BAR * should be configured as 32-bit or 64-bit, the EPF driver must - * configure this BAR as 64-bit. Additionally, the BAR succeeding - * this BAR must be set to type BAR_RESERVED. - * - * only_64bit should not be set on a BAR of type BAR_RESERVED. - * (If BARx is a 64-bit BAR that an EPF driver is not allowed to - * touch, then both BARx and BARx+1 must be set to type - * BAR_RESERVED.) + * configure this BAR as 64-bit. + * @nr_rsvd_regions: number of fixed subregions described for BAR_RESERVED + * @rsvd_regions: fixed subregions behind BAR_RESERVED */ struct pci_epc_bar_desc { enum pci_epc_bar_type type; u64 fixed_size; bool only_64bit; + u8 nr_rsvd_regions; + const struct pci_epc_bar_rsvd_region *rsvd_regions; }; /** @@ -309,6 +392,11 @@ int pci_epc_start(struct pci_epc *epc); void pci_epc_stop(struct pci_epc *epc); const struct pci_epc_features *pci_epc_get_features(struct pci_epc *epc, u8 func_no, u8 vfunc_no); +int pci_epc_get_aux_resources_count(struct pci_epc *epc, u8 func_no, + u8 vfunc_no); +int pci_epc_get_aux_resources(struct pci_epc *epc, u8 func_no, u8 vfunc_no, + struct pci_epc_aux_resource *resources, + int num_resources); enum pci_barno pci_epc_get_first_free_bar(const struct pci_epc_features *epc_features); enum pci_barno pci_epc_get_next_free_bar(const struct pci_epc_features diff --git a/include/linux/pci-epf.h b/include/linux/pci-epf.h index 7737a7c03260..704e1dc8b30a 100644 --- a/include/linux/pci-epf.h +++ b/include/linux/pci-epf.h @@ -11,7 +11,7 @@ #include <linux/configfs.h> #include <linux/device.h> -#include <linux/mod_devicetable.h> +#include <linux/device-id/pci.h> #include <linux/msi.h> #include <linux/pci.h> @@ -152,14 +152,41 @@ struct pci_epf_bar { struct pci_epf_bar_submap *submap; }; +enum pci_epf_doorbell_type { + PCI_EPF_DOORBELL_MSI = 0, + PCI_EPF_DOORBELL_EMBEDDED, +}; + /** * struct pci_epf_doorbell_msg - represents doorbell message - * @msg: MSI message - * @virq: IRQ number of this doorbell MSI message + * @msg: Doorbell address/data pair to be mapped into BAR space. + * For MSI-backed doorbells this is the MSI message, while for + * "embedded" doorbells this represents an MMIO write that asserts + * an interrupt on the EP side. + * @virq: IRQ number of this doorbell message + * @irq_flags: Required flags for request_irq()/request_threaded_irq(). + * Callers may OR-in additional flags (e.g. IRQF_ONESHOT). + * @type: Doorbell type. + * @bar: BAR number where the doorbell target is already exposed to the RC + * (NO_BAR if not) + * @offset: offset within @bar for the doorbell target (valid iff + * @bar != NO_BAR) + * @iova_base: Internal: base DMA address returned by dma_map_resource() for the + * embedded doorbell MMIO window (used only for unmapping). Valid + * when @type is PCI_EPF_DOORBELL_EMBEDDED and @iova_size is + * non-zero. + * @iova_size: Internal: size of the dma_map_resource() mapping at @iova_base. + * Zero when no mapping was created (e.g. pre-exposed fixed BAR). */ struct pci_epf_doorbell_msg { struct msi_msg msg; int virq; + unsigned long irq_flags; + enum pci_epf_doorbell_type type; + enum pci_barno bar; + resource_size_t offset; + dma_addr_t iova_base; + size_t iova_size; }; /** diff --git a/include/linux/pci-tph.h b/include/linux/pci-tph.h index ba28140ce670..be68cd17f2f8 100644 --- a/include/linux/pci-tph.h +++ b/include/linux/pci-tph.h @@ -25,7 +25,7 @@ int pcie_tph_set_st_entry(struct pci_dev *pdev, unsigned int index, u16 tag); int pcie_tph_get_cpu_st(struct pci_dev *dev, enum tph_mem_type mem_type, - unsigned int cpu_uid, u16 *tag); + unsigned int cpu, u16 *tag); void pcie_disable_tph(struct pci_dev *pdev); int pcie_enable_tph(struct pci_dev *pdev, int mode); u16 pcie_tph_get_st_table_size(struct pci_dev *pdev); @@ -36,7 +36,7 @@ static inline int pcie_tph_set_st_entry(struct pci_dev *pdev, { return -EINVAL; } static inline int pcie_tph_get_cpu_st(struct pci_dev *dev, enum tph_mem_type mem_type, - unsigned int cpu_uid, u16 *tag) + unsigned int cpu, u16 *tag) { return -EINVAL; } static inline void pcie_disable_tph(struct pci_dev *pdev) { } static inline int pcie_enable_tph(struct pci_dev *pdev, int mode) diff --git a/include/linux/pci.h b/include/linux/pci.h index 1c270f1d5123..d31a8d107b1e 100644 --- a/include/linux/pci.h +++ b/include/linux/pci.h @@ -24,9 +24,10 @@ #define LINUX_PCI_H #include <linux/args.h> -#include <linux/mod_devicetable.h> +#include <linux/device-id/pci.h> #include <linux/types.h> +#include <linux/sizes.h> #include <linux/init.h> #include <linux/ioport.h> #include <linux/list.h> @@ -72,12 +73,24 @@ /* return bus from PCI devid = ((u16)bus_number) << 8) | devfn */ #define PCI_BUS_NUM(x) (((x) >> 8) & 0xff) +/* + * PCI_SLOT_ALL_DEVICES indicates a slot that covers all devices on the bus. + * Used for PCIe hotplug where the physical slot is the entire secondary bus, + * and, if ARI Forwarding is enabled, functions may appear to be on multiple + * devices. + */ +#define PCI_SLOT_ALL_DEVICES 0xfeff + +/* Used to identify a slot as a placeholder */ +#define PCI_SLOT_PLACEHOLDER 0xffff + /* pci_slot represents a physical slot */ struct pci_slot { struct pci_bus *bus; /* Bus this slot is on */ struct list_head list; /* Node in list of slots */ struct hotplug_slot *hotplug; /* Hotplug info (move here) */ - unsigned char number; /* PCI_SLOT(pci_dev->devfn) */ + u16 number; /* Device nr, or PCI_SLOT_ALL_DEVICES */ + unsigned int per_func_slot:1; /* Allow per function slot */ struct kobject kobj; }; @@ -500,15 +513,13 @@ struct pci_dev { unsigned int no_command_memory:1; /* No PCI_COMMAND_MEMORY */ unsigned int rom_bar_overlap:1; /* ROM BAR disable broken */ unsigned int rom_attr_enabled:1; /* Display of ROM attribute enabled? */ - unsigned int non_mappable_bars:1; /* BARs can't be mapped to user-space */ + unsigned int non_mappable_bars:1; /* BARs can't be mapped by CPU or peers */ pci_dev_flags_t dev_flags; atomic_t enable_cnt; /* pci_enable_device has been called */ spinlock_t pcie_cap_lock; /* Protects RMW ops in capability accessors */ u32 saved_config_space[16]; /* Config space saved at suspend time */ struct hlist_head saved_cap_space; - struct bin_attribute *res_attr[DEVICE_COUNT_RESOURCE]; /* sysfs file for resources */ - struct bin_attribute *res_attr_wc[DEVICE_COUNT_RESOURCE]; /* sysfs file for WC mapping of resources */ #ifdef CONFIG_HOTPLUG_PCI_PCIE unsigned int broken_cmd_compl:1; /* No compl for some cmds */ @@ -518,7 +529,7 @@ struct pci_dev { unsigned int ptm_root:1; unsigned int ptm_responder:1; unsigned int ptm_requester:1; - unsigned int ptm_enabled:1; + atomic_t ptm_enable_cnt; u8 ptm_granularity; #endif #ifdef CONFIG_PCI_MSI @@ -575,17 +586,13 @@ struct pci_dev { u8 supported_speeds; /* Supported Link Speeds Vector */ phys_addr_t rom; /* Physical address if not from BAR */ size_t romlen; /* Length if not from BAR */ - /* - * Driver name to force a match. Do not set directly, because core - * frees it. Use driver_set_override() to set or clear it. - */ - const char *driver_override; - unsigned long priv_flags; /* Private flags for the PCI driver */ /* These methods index pci_reset_fn_methods[] */ u8 reset_methods[PCI_NUM_RESET_METHODS]; /* In priority order */ + struct gpio_desc *wake; /* WAKE# GPIO */ + #ifdef CONFIG_PCIE_TPH u16 tph_cap; /* TPH capability offset */ u8 tph_mode; /* TPH mode */ @@ -634,6 +641,7 @@ struct pci_host_bridge { int domain_nr; struct list_head windows; /* resource_entry */ struct list_head dma_ranges; /* dma ranges resource list */ + struct list_head ports; /* Root Port list (pci_host_port) */ #ifdef CONFIG_PCI_IDE u16 nr_ide_streams; /* Max streams possibly active in @ide_stream_ida */ struct ida ide_stream_ida; @@ -644,6 +652,7 @@ struct pci_host_bridge { void (*release_fn)(struct pci_host_bridge *); int (*enable_device)(struct pci_host_bridge *bridge, struct pci_dev *dev); void (*disable_device)(struct pci_host_bridge *bridge, struct pci_dev *dev); + int (*reset_root_port)(struct pci_host_bridge *bridge, struct pci_dev *dev); void *release_data; unsigned int ignore_reset_delay:1; /* For entire hierarchy */ unsigned int no_ext_tags:1; /* No Extended Tags */ @@ -658,6 +667,8 @@ struct pci_host_bridge { unsigned int preserve_config:1; /* Preserve FW resource setup */ unsigned int size_windows:1; /* Enable root bus sizing */ unsigned int msi_domain:1; /* Bridge wants MSI domain */ + unsigned int broken_l1ss_resume:1; /* Resuming from L1SS during + system suspend is broken */ /* Resource alignment requirements */ resource_size_t (*align_resource)(struct pci_dev *dev, @@ -725,8 +736,6 @@ struct pci_bus { pci_bus_flags_t bus_flags; /* Inherited by child buses */ struct device *bridge; struct device dev; - struct bin_attribute *legacy_io; /* Legacy I/O for this bus */ - struct bin_attribute *legacy_mem; /* Legacy mem */ unsigned int is_added:1; unsigned int unsafe_warn:1; /* warned about RW1C config write */ unsigned int flit_mode:1; /* Link in Flit mode */ @@ -977,6 +986,7 @@ struct module; * function returns zero when the driver chooses to * take "ownership" of the device or an error code * (negative number) otherwise. + * The pci_device_id parameter is only valid during probe. * The probe function always gets called from process * context, so it can sleep. * @remove: The remove() function gets called whenever a device @@ -1169,6 +1179,10 @@ enum { /* These external functions are only available when PCI support is enabled */ #ifdef CONFIG_PCI +/* PCI legacy I/O port and memory address space sizes. */ +#define PCI_LEGACY_IO_SIZE (SZ_64K - 1) +#define PCI_LEGACY_MEM_SIZE SZ_1M + extern unsigned int pci_flags; static inline void pci_set_flags(int flags) { pci_flags = flags; } @@ -1193,8 +1207,6 @@ extern const struct bus_type pci_bus_type; /* Do NOT directly access these two variables, unless you are arch-specific PCI * code, or PCI core code. */ extern struct list_head pci_root_buses; /* List of all known PCI buses */ -/* Some device drivers need know if PCI is initiated */ -int no_pci_devices(void); void pcibios_resource_survey_bus(struct pci_bus *bus); void pcibios_bus_add_device(struct pci_dev *pdev); @@ -1206,9 +1218,15 @@ int __must_check pcibios_enable_device(struct pci_dev *, int mask); char *pcibios_setup(char *str); /* Used only when drivers/pci/setup.c is used */ -resource_size_t pcibios_align_resource(void *, const struct resource *, - resource_size_t, - resource_size_t); +resource_size_t pcibios_align_resource(void *data, const struct resource *res, + const struct resource *empty_res, + resource_size_t size, + resource_size_t align); +resource_size_t pci_align_resource(struct pci_dev *dev, + const struct resource *res, + const struct resource *empty_res, + resource_size_t size, + resource_size_t align); /* Generic PCI functions used internally */ @@ -1773,6 +1791,26 @@ void pci_free_irq_vectors(struct pci_dev *dev); int pci_irq_vector(struct pci_dev *dev, unsigned int nr); const struct cpumask *pci_irq_get_affinity(struct pci_dev *pdev, int vec); +/** + * pci_irq_type - Get the interrupt type of a PCI device + * @pdev: the PCI device to operate on + * + * Discriminate the interrupt type the PCI core selected for this device + * after a successful pci_alloc_irq_vectors() call. + * + * Return: %PCI_IRQ_MSIX, %PCI_IRQ_MSI, or %PCI_IRQ_INTX. + */ +static inline unsigned int pci_irq_type(struct pci_dev *pdev) +{ + if (pdev->msix_enabled) + return PCI_IRQ_MSIX; + + if (pdev->msi_enabled) + return PCI_IRQ_MSI; + + return PCI_IRQ_INTX; +} + #else static inline int pci_msi_vec_count(struct pci_dev *dev) { return -ENOSYS; } static inline void pci_disable_msi(struct pci_dev *dev) { } @@ -1835,6 +1873,11 @@ static inline const struct cpumask *pci_irq_get_affinity(struct pci_dev *pdev, { return cpu_possible_mask; } + +static inline unsigned int pci_irq_type(struct pci_dev *pdev) +{ + return PCI_IRQ_INTX; +} #endif /** @@ -1975,11 +2018,11 @@ struct pci_ptm_debugfs { }; #ifdef CONFIG_PCIE_PTM -int pci_enable_ptm(struct pci_dev *dev, u8 *granularity); +int pci_enable_ptm(struct pci_dev *dev); void pci_disable_ptm(struct pci_dev *dev); bool pcie_ptm_enabled(struct pci_dev *dev); #else -static inline int pci_enable_ptm(struct pci_dev *dev, u8 *granularity) +static inline int pci_enable_ptm(struct pci_dev *dev) { return -EINVAL; } static inline void pci_disable_ptm(struct pci_dev *dev) { } static inline bool pcie_ptm_enabled(struct pci_dev *dev) @@ -2080,6 +2123,8 @@ pci_release_mem_regions(struct pci_dev *pdev) pci_select_bars(pdev, IORESOURCE_MEM)); } +bool pci_suspend_retains_context(struct pci_dev *pdev); + #else /* CONFIG_PCI is not enabled */ static inline void pci_set_flags(int flags) { } @@ -2132,7 +2177,6 @@ static inline struct pci_dev *pci_get_base_class(unsigned int class, static inline int pci_dev_present(const struct pci_device_id *ids) { return 0; } -#define no_pci_devices() (1) #define pci_dev_put(dev) do { } while (0) static inline void pci_set_master(struct pci_dev *dev) { } @@ -2239,6 +2283,16 @@ pci_alloc_irq_vectors(struct pci_dev *dev, unsigned int min_vecs, static inline void pci_free_irq_vectors(struct pci_dev *dev) { } + +static inline bool pci_suspend_retains_context(struct pci_dev *pdev) +{ + return true; +} + +static inline unsigned int pci_irq_type(struct pci_dev *pdev) +{ + return 0; +} #endif /* CONFIG_PCI */ /* Include architecture-dependent settings and functions */ @@ -2295,6 +2349,31 @@ int pci_iobar_pfn(struct pci_dev *pdev, int bar, struct vm_area_struct *vma); CONCATENATE(__pci_dev_for_each_res, COUNT_ARGS(__VA_ARGS__)) \ (dev, res, __VA_ARGS__) +/** + * pci_resource_is_io - check if a PCI resource is of I/O port type. + * @dev: PCI device to check. + * @resno: The resource number (BAR index) to check. + * + * Returns true if the resource type is I/O port. + */ +static inline bool pci_resource_is_io(const struct pci_dev *dev, int resno) +{ + return resource_type(pci_resource_n(dev, resno)) == IORESOURCE_IO; +} + +/** + * pci_resource_is_mem - check if a PCI resource is of memory type. + * @dev: PCI device to check. + * @resno: The resource number (BAR index) to check. + * + * Returns true if the resource type is memory, including + * prefetchable memory. + */ +static inline bool pci_resource_is_mem(const struct pci_dev *dev, int resno) +{ + return resource_type(pci_resource_n(dev, resno)) == IORESOURCE_MEM; +} + /* * Similar to the helpers above, these manipulate per-pci_dev * driver-specific data. They are really just a wrapper around @@ -2501,11 +2580,6 @@ int pcibios_alloc_irq(struct pci_dev *dev); void pcibios_free_irq(struct pci_dev *dev); resource_size_t pcibios_default_alignment(void); -#if !defined(HAVE_PCI_MMAP) && !defined(ARCH_GENERIC_PCI_MMAP_RESOURCE) -extern int pci_create_resource_files(struct pci_dev *dev); -extern void pci_remove_resource_files(struct pci_dev *dev); -#endif - #if defined(CONFIG_PCI_MMCONFIG) || defined(CONFIG_ACPI_MCFG) void __init pci_mmcfg_early_init(void); void __init pci_mmcfg_late_init(void); @@ -2533,9 +2607,9 @@ void pci_iov_remove_virtfn(struct pci_dev *dev, int id); int pci_num_vf(struct pci_dev *dev); int pci_vfs_assigned(struct pci_dev *dev); int pci_sriov_set_totalvfs(struct pci_dev *dev, u16 numvfs); -int pci_sriov_get_totalvfs(struct pci_dev *dev); +unsigned int pci_sriov_get_totalvfs(struct pci_dev *dev); int pci_sriov_configure_simple(struct pci_dev *dev, int nr_virtfn); -resource_size_t pci_iov_resource_size(struct pci_dev *dev, int resno); +resource_size_t pci_iov_resource_size(const struct pci_dev *dev, int resno); int pci_iov_vf_bar_set_size(struct pci_dev *dev, int resno, int size); u32 pci_iov_vf_bar_get_sizes(struct pci_dev *dev, int resno, int num_vfs); void pci_vf_drivers_autoprobe(struct pci_dev *dev, bool probe); @@ -2543,7 +2617,8 @@ void pci_vf_drivers_autoprobe(struct pci_dev *dev, bool probe); /* Arch may override these (weak) */ int pcibios_sriov_enable(struct pci_dev *pdev, u16 num_vfs); int pcibios_sriov_disable(struct pci_dev *pdev); -resource_size_t pcibios_iov_resource_alignment(struct pci_dev *dev, int resno); +resource_size_t pcibios_iov_resource_alignment(const struct pci_dev *dev, + int resno); #else static inline int pci_iov_virtfn_bus(struct pci_dev *dev, int id) { @@ -2585,10 +2660,11 @@ static inline int pci_vfs_assigned(struct pci_dev *dev) { return 0; } static inline int pci_sriov_set_totalvfs(struct pci_dev *dev, u16 numvfs) { return 0; } -static inline int pci_sriov_get_totalvfs(struct pci_dev *dev) +static inline unsigned int pci_sriov_get_totalvfs(struct pci_dev *dev) { return 0; } #define pci_sriov_configure_simple NULL -static inline resource_size_t pci_iov_resource_size(struct pci_dev *dev, int resno) +static inline resource_size_t pci_iov_resource_size(const struct pci_dev *dev, + int resno) { return 0; } static inline int pci_iov_vf_bar_set_size(struct pci_dev *dev, int resno, int size) { return -ENODEV; } diff --git a/include/linux/pci_ids.h b/include/linux/pci_ids.h index 406abf629be2..63774ae2bf71 100644 --- a/include/linux/pci_ids.h +++ b/include/linux/pci_ids.h @@ -2586,6 +2586,8 @@ #define PCI_VENDOR_ID_AZWAVE 0x1a3b +#define PCI_VENDOR_ID_GOOGLE 0x1ae0 + #define PCI_VENDOR_ID_REDHAT_QUMRANET 0x1af4 #define PCI_SUBVENDOR_ID_REDHAT_QUMRANET 0x1af4 #define PCI_SUBDEVICE_ID_QEMU 0x1100 @@ -2638,6 +2640,12 @@ #define PCI_VENDOR_ID_SUNIX 0x1fd4 #define PCI_DEVICE_ID_SUNIX_1999 0x1999 +#define PCI_VENDOR_ID_SPACEMIT 0x201f +#define PCI_DEVICE_ID_SPACEMIT_K1 0x0001 +#define PCI_DEVICE_ID_SPACEMIT_K3 0x0002 + +#define PCI_VENDOR_ID_PICOHEART 0x20fa + #define PCI_VENDOR_ID_HINT 0x3388 #define PCI_DEVICE_ID_HINT_VXPROII_IDE 0x8013 @@ -2730,6 +2738,9 @@ #define PCI_DEVICE_ID_INTEL_82815_MC 0x1130 #define PCI_DEVICE_ID_INTEL_82815_CGC 0x1132 #define PCI_DEVICE_ID_INTEL_SST_TNG 0x119a +#define PCI_DEVICE_ID_INTEL_DSA_GNRD 0x11fb +#define PCI_DEVICE_ID_INTEL_DSA_DMR 0x1212 +#define PCI_DEVICE_ID_INTEL_IAA_DMR 0x1216 #define PCI_DEVICE_ID_INTEL_82092AA_0 0x1221 #define PCI_DEVICE_ID_INTEL_82437 0x122d #define PCI_DEVICE_ID_INTEL_82371FB_0 0x122e @@ -2887,6 +2898,7 @@ #define PCI_DEVICE_ID_INTEL_HDA_ICH8 0x284b #define PCI_DEVICE_ID_INTEL_ICH8_6 0x2850 #define PCI_DEVICE_ID_INTEL_VMD_28C0 0x28c0 +#define PCI_DEVICE_ID_INTEL_VMD_28C1 0x28c1 #define PCI_DEVICE_ID_INTEL_ICH9_0 0x2910 #define PCI_DEVICE_ID_INTEL_ICH9_2 0x2912 #define PCI_DEVICE_ID_INTEL_ICH9_3 0x2913 @@ -3050,6 +3062,11 @@ #define PCI_DEVICE_ID_INTEL_5400_FBD1 0x4036 #define PCI_DEVICE_ID_INTEL_HDA_TGL_H 0x43c8 #define PCI_DEVICE_ID_INTEL_HDA_DG1 0x490d +#define PCI_DEVICE_ID_INTEL_QAT_4XXX 0x4940 +#define PCI_DEVICE_ID_INTEL_QAT_401XX 0x4942 +#define PCI_DEVICE_ID_INTEL_QAT_402XX 0x4944 +#define PCI_DEVICE_ID_INTEL_QAT_420XX 0x4946 +#define PCI_DEVICE_ID_INTEL_QAT_6XXX 0x4948 #define PCI_DEVICE_ID_INTEL_HDA_EHL_0 0x4b55 #define PCI_DEVICE_ID_INTEL_HDA_EHL_3 0x4b58 #define PCI_DEVICE_ID_INTEL_HDA_WCL 0x4d28 diff --git a/include/linux/pcie-dwc.h b/include/linux/pcie-dwc.h index 8ff778e7aec0..097e806c714c 100644 --- a/include/linux/pcie-dwc.h +++ b/include/linux/pcie-dwc.h @@ -11,6 +11,8 @@ #include <linux/pci_ids.h> +#define PCI_VENDOR_ID_LECARC 0x0720 + struct dwc_pcie_vsec_id { u16 vendor_id; u16 vsec_id; @@ -26,12 +28,16 @@ static const struct dwc_pcie_vsec_id dwc_pcie_rasdes_vsec_ids[] = { .vsec_id = 0x02, .vsec_rev = 0x4 }, { .vendor_id = PCI_VENDOR_ID_AMPERE, .vsec_id = 0x02, .vsec_rev = 0x4 }, + { .vendor_id = PCI_VENDOR_ID_PICOHEART, + .vsec_id = 0x02, .vsec_rev = 0x4 }, { .vendor_id = PCI_VENDOR_ID_QCOM, .vsec_id = 0x02, .vsec_rev = 0x4 }, { .vendor_id = PCI_VENDOR_ID_ROCKCHIP, .vsec_id = 0x02, .vsec_rev = 0x4 }, { .vendor_id = PCI_VENDOR_ID_SAMSUNG, .vsec_id = 0x02, .vsec_rev = 0x4 }, + { .vendor_id = PCI_VENDOR_ID_LECARC, + .vsec_id = 0x02, .vsec_rev = 0x4 }, {} }; diff --git a/include/linux/pds/pds_core_if.h b/include/linux/pds/pds_core_if.h index 17a87c1a55d7..901e1e628f89 100644 --- a/include/linux/pds/pds_core_if.h +++ b/include/linux/pds/pds_core_if.h @@ -40,6 +40,14 @@ enum pds_core_cmd_opcode { PDS_CORE_CMD_FW_DOWNLOAD = 4, PDS_CORE_CMD_FW_CONTROL = 5, + PDS_CORE_CMD_GET_COMPONENT_INFO = 6, + PDS_CORE_CMD_SEND_PKG_DATA = 7, + PDS_CORE_CMD_SEND_COMPONENT_TBL = 8, + PDS_CORE_CMD_SEND_COMPONENT = 9, + PDS_CORE_CMD_FINALIZE_UPDATE = 10, + PDS_CORE_CMD_MATCH_RECORD_DESC = 11, + PDS_CORE_CMD_HOST_MEM = 12, + /* SR/IOV commands */ PDS_CORE_CMD_VF_GETATTR = 60, PDS_CORE_CMD_VF_SETATTR = 61, @@ -100,6 +108,16 @@ struct pds_core_drv_identity { char driver_ver_str[32]; }; +/** + * enum pds_core_dev_capability - Device capabilities + * @PDS_CORE_DEV_CAP_PLDM_FW_UPDATE: Device only supports FW update via PLDM + * @PDS_CORE_DEV_CAP_HOST_MEM: Device supports host memory for fw use + */ +enum pds_core_dev_capability { + PDS_CORE_DEV_CAP_PLDM_FW_UPDATE = BIT(0), + PDS_CORE_DEV_CAP_HOST_MEM = BIT(1), +}; + #define PDS_DEV_TYPE_MAX 16 /** * struct pds_core_dev_identity - Device identity information @@ -119,6 +137,11 @@ struct pds_core_drv_identity { * value in usecs to device units using: * device units = usecs * mult / div * @vif_types: How many of each VIF device type is supported + * @max_fw_slots: Number of firmware components reported by device + * only supported on version >= PDS_CORE_IDENTITY_VERSION_2 + * @rsvd2: Word boundary padding + * @capabilities: Device capabilities + * only supported on version >= PDS_CORE_IDENTITY_VERSION_2 */ struct pds_core_dev_identity { u8 version; @@ -131,9 +154,13 @@ struct pds_core_dev_identity { __le32 intr_coal_mult; __le32 intr_coal_div; __le16 vif_types[PDS_DEV_TYPE_MAX]; + __le16 max_fw_slots; + u8 rsvd2[6]; + __le64 capabilities; }; #define PDS_CORE_IDENTITY_VERSION_1 1 +#define PDS_CORE_IDENTITY_VERSION_2 2 /** * struct pds_core_dev_identify_cmd - Driver/device identify command @@ -275,11 +302,20 @@ enum pds_core_fw_control_oper { PDS_CORE_FW_GET_LIST = 7, }; +/** + * enum pds_core_fw_slot - Firmware slot identifiers + * @PDS_CORE_FW_SLOT_INVALID: Let firmware select slot based on package metadata + * @PDS_CORE_FW_SLOT_A: Primary firmware slot A + * @PDS_CORE_FW_SLOT_B: Primary firmware slot B + * @PDS_CORE_FW_SLOT_GOLD: Gold/recovery firmware slot + * @PDS_CORE_FW_SLOT_MAX: Sentinel value indicating no slot resolved + */ enum pds_core_fw_slot { PDS_CORE_FW_SLOT_INVALID = 0, PDS_CORE_FW_SLOT_A = 1, PDS_CORE_FW_SLOT_B = 2, PDS_CORE_FW_SLOT_GOLD = 3, + PDS_CORE_FW_SLOT_MAX = 0xff, }; /** @@ -446,6 +482,423 @@ struct pds_core_vf_ctrl_comp { u8 status; }; +/** + * struct pds_core_send_pkg_data_cmd - Send package data command + * @opcode: Opcode PDS_CORE_CMD_SEND_PKG_DATA + * @ver: Driver's max support version of this command + * @total_len: Total length of the package data + * @offset: Offset in the package data, non-zero if multiple commands are + * needed for sending the package data + * @data_len: Length of data stored at data_pa + * @data_pa: Data physical address for DMA to device + * + * The package data may be too large to store in a single buffer, so multiple + * PDS_CORE_CMD_SEND_PKG_DATA devcmds may be needed. + */ +struct pds_core_send_pkg_data_cmd { + u8 opcode; + u8 ver; + __le16 total_len; + __le16 offset; + __le16 data_len; + __le64 data_pa; +}; + +/** + * struct pds_core_send_pkg_data_comp - Send package data completion + * @status: Status of the command (enum pds_core_status_code) + * @ver: Device's max supported version of this command + * @rsvd: Word boundary padding + */ +struct pds_core_send_pkg_data_comp { + u8 status; + u8 ver; + u8 rsvd[2]; +}; + +/** + * struct pds_core_component_tbl - Component table details + * @comparison_stamp: Comparison stamp used for component version checks + * @classification: Vendor specific classification info + * @identifier: Component's ID + * @transfer_flag: Part of the component table this request represents + * @version_str_type: The types of strings used + * @version_str_len: Length of @version_str + * @version_str: Component version information + */ +struct pds_core_component_tbl { + __le32 comparison_stamp; + __le16 classification; + __le16 identifier; + u8 transfer_flag; + u8 version_str_type; + u8 version_str_len; + u8 version_str[]; +}; + +/** + * struct pds_core_send_component_tbl_cmd - Send component table command + * @opcode: Opcode PDS_CORE_CMD_SEND_COMPONENT_TBL + * @ver: Driver's max support version of this command + * @slot_id: enum pds_core_fw_slot + * @rsvd: Word boundary padding + * + * Expects to find component table info (struct pds_core_component_tbl) + * in cmd_regs->data. Driver should keep the devcmd interface locked + * while preparing the component table info. + */ +struct pds_core_send_component_tbl_cmd { + u8 opcode; + u8 ver; + u8 slot_id; + u8 rsvd; +}; + +enum pds_core_component_resp_code { + PDS_CORE_COMPONENT_VALID = 0x0, + PDS_CORE_COMPONENT_STAMP_IDENTICAL = 0x1, + PDS_CORE_COMPONENT_STAMP_LOWER = 0x2, + PDS_CORE_COMPONENT_STAMP_OR_VERSION_INVALID = 0x3, + PDS_CORE_COMPONENT_CONFLICT = 0x4, + PDS_CORE_COMPONENT_PREREQS_NOT_MET = 0x5, + PDS_CORE_COMPONENT_NOT_SUPPORTED = 0x6, + PDS_CORE_COMPONENT_FW_TYPE_INVALID = 0xd0, +}; + +/** + * struct pds_core_send_component_tbl_comp - Send component table completion + * @status: Status of the command (enum pds_core_status_code) + * @ver: Device's max supported version of this command + * @completion_code: Component completion code + * @response: Component response + * @response_code: Component response code + * @slot_id: Actual slot_id of the component (enum pds_core_fw_slot) + * @rsvd: Word boundary padding + */ +struct pds_core_send_component_tbl_comp { + u8 status; + u8 ver; + u8 completion_code; + u8 response; + u8 response_code; + u8 slot_id; + u8 rsvd[2]; +}; + +/** + * enum pds_core_send_component_op - PDS_CORE_CMD_SEND_COMPONENT operation + * @PDS_CORE_SEND_COMPONENT_START: Initial operation to start transfer + * @PDS_CORE_SEND_COMPONENT_STATUS: Subsequent calls to check on status + */ +enum pds_core_send_component_op { + PDS_CORE_SEND_COMPONENT_START = 0, + PDS_CORE_SEND_COMPONENT_STATUS = 1, +}; + +#define PDS_CORE_FW_COMPONENT_ID_INVALID 0xFFFF +/** + * struct pds_core_flash_component - Component details + * @comparison_stamp: Comparison stamp used for component version checks + * @image_size: Component image size + * @classification: Vendor specific classification info + * @identifier: Component's ID + * @options: Component options + * @rsvd: Word boundary padding + * @version_str_type: The types of strings used + * @version_str_len: Length of @version_str + * @version_str: Component version information + */ +struct pds_core_flash_component { + __le32 comparison_stamp; + __le32 image_size; + __le16 classification; + __le16 identifier; + __le16 options; + u8 rsvd[3]; + u8 version_str_type; + u8 version_str_len; + u8 version_str[]; +}; + +/** + * struct pds_core_send_component_cmd - Send component command + * @opcode: Opcode PDS_CORE_CMD_SEND_COMPONENT + * @ver: Driver's max supported version of this command + * @slot_id: enum pds_core_fw_slot + * @operation: enum pds_core_send_component_op + * @offset: Offset into the component, non-zero if multiple commands + * are needed for a single component + * @data_len: Length of this part of the component stored at @data_pa + * @rsvd: Word boundary padding + * @data_pa: DMA address of the component + * + * A component may be too large to store in a single buffer, so multiple + * PDS_CORE_CMD_SEND_COMPONENT devcmds may be needed. + * + * Expects to find flash component info (struct pds_core_flash_component) + * in cmd_regs->data. Driver should keep the devcmd interface locked + * while preparing and sending the flash component info. + */ +struct pds_core_send_component_cmd { + u8 opcode; + u8 ver; + u8 slot_id; + u8 operation; + __le32 offset; + __le32 data_len; + u8 rsvd[4]; + __le64 data_pa; +}; + +/** + * struct pds_core_send_component_comp - Send component completion + * @status: Status of the command (enum pds_core_status_code) + * @ver: Device's max supported version of this command + * @completion_code: Completion code + * @compat_response: Compatibility response (0 = Component can be updated) + * @compat_response_code: Compatibility response code + * @rsvd: Word boundary padding + */ +struct pds_core_send_component_comp { + u8 status; + u8 ver; + u8 completion_code; + u8 compat_response; + u8 compat_response_code; + u8 rsvd[3]; +}; + +/** + * enum pds_core_fw_component_type - Firmware component type + * @PDS_CORE_FW_TYPE_UNKNOWN: Unknown component type + * @PDS_CORE_FW_TYPE_MAIN: Main firmware + * @PDS_CORE_FW_TYPE_BOOT: Boot loader + * @PDS_CORE_FW_TYPE_CPLD: CPLD firmware + * @PDS_CORE_FW_TYPE_SECURE: Secure firmware + * @PDS_CORE_FW_TYPE_FPGA: FPGA configuration + * @PDS_CORE_FW_TYPE_SUC_MAIN: System Unit Controller firmware + * @PDS_CORE_FW_TYPE_SUC_BOOT: System Unit Controller bootloader + * @PDS_CORE_FW_TYPE_UBOOT: U-Boot bootloader + * + * Gold/recovery variants are identified by slot_id == PDS_CORE_FW_SLOT_GOLD + * and reported with a ".gold" suffix (e.g., fw.gold). + */ +enum pds_core_fw_component_type { + PDS_CORE_FW_TYPE_UNKNOWN = 0, + PDS_CORE_FW_TYPE_MAIN = 1, + PDS_CORE_FW_TYPE_BOOT = 2, + PDS_CORE_FW_TYPE_CPLD = 3, + PDS_CORE_FW_TYPE_SECURE = 4, + PDS_CORE_FW_TYPE_FPGA = 5, + PDS_CORE_FW_TYPE_SUC_MAIN = 6, + PDS_CORE_FW_TYPE_SUC_BOOT = 7, + PDS_CORE_FW_TYPE_UBOOT = 8, +}; + +/** + * enum pds_core_component_info_flags - Component info flags + * @PDS_CORE_FW_COMPONENT_INFO_F_RUNNING: Component is currently running + * @PDS_CORE_FW_COMPONENT_INFO_F_STARTUP: Component version on next FW boot + * @PDS_CORE_FW_COMPONENT_INFO_F_FIXED: Component is fixed and cannot be updated + * @PDS_CORE_FW_COMPONENT_INFO_F_UPDATE_BY_NAME: Component can be updated + * by name + */ +enum pds_core_component_info_flags { + PDS_CORE_FW_COMPONENT_INFO_F_RUNNING = BIT(0), + PDS_CORE_FW_COMPONENT_INFO_F_STARTUP = BIT(1), + PDS_CORE_FW_COMPONENT_INFO_F_FIXED = BIT(2), + PDS_CORE_FW_COMPONENT_INFO_F_UPDATE_BY_NAME = BIT(3), +}; + +/** + * struct pds_core_fw_component_info - GET_COMPONENT_INFO entry + * @name: Component's name + * @component_type: enum pds_core_fw_component_type + * @rsvd: Word boundary padding + * @flags: enum pds_core_component_info_flags + * @identifier: Component's identifier + * @slot_id: Component's slot identifier + * @version: Component's version + */ +struct pds_core_fw_component_info { +#define PDS_CORE_FW_COMPONENT_NAME_BUFLEN 24 + char name[PDS_CORE_FW_COMPONENT_NAME_BUFLEN]; + u8 component_type; + u8 rsvd[3]; + __le16 flags; + u8 identifier; + u8 slot_id; +#define PDS_CORE_FW_COMPONENT_VER_BUFLEN 32 + char version[PDS_CORE_FW_COMPONENT_VER_BUFLEN]; +}; + +#define PDS_CORE_FW_COMPONENT_LIST_LEN ((PDS_PAGE_SIZE - 8) / \ + sizeof(struct pds_core_fw_component_info)) + +/** + * struct pds_core_component_list_info - GET_COMPONENT_INFO completion data + * @num_components: Number of valid components + * @rsvd: Word boundary padding + * @info: List of valid components + */ +struct pds_core_component_list_info { + u8 num_components; + u8 rsvd[7]; + struct pds_core_fw_component_info info[PDS_CORE_FW_COMPONENT_LIST_LEN]; +}; + +/** + * struct pds_core_get_component_info_cmd - GET_COMPONENT_INFO command + * @opcode: PDS_CORE_CMD_GET_COMPONENT_INFO + * @ver: Driver's max supported version of this command + * @data_len: Length of data at data_pa + * @rsvd: Word boundary padding + * @data_pa: DMA address of data + * + * FW populates struct pds_core_component_list_info pointed to by @data_pa + */ +struct pds_core_get_component_info_cmd { + u8 opcode; + u8 ver; + __le16 data_len; + u8 rsvd[4]; + __le64 data_pa; +}; + +/** + * struct pds_core_get_component_info_comp - GET_COMPONENT_INFO completion + * @status: enum pds_core_status_code + * @ver: Device's max supported version of this command + * @rsvd: Word boundary padding + */ +struct pds_core_get_component_info_comp { + u8 status; + u8 ver; + u8 rsvd[2]; +}; + +/** + * struct pds_core_finalize_update_cmd - FINALIZE_UPDATE command + * @opcode: PDS_CORE_CMD_FINALIZE_UPDATE + * @ver: Driver's max support version of this command + * @rsvd: Word boundary padding + * + * Driver sends at the end of updating all components to finalize the update + */ +struct pds_core_finalize_update_cmd { + u8 opcode; + u8 ver; + u8 rsvd[2]; +}; + +/** + * struct pds_core_finalize_update_comp - FINALIZE_UPDATE completion + * @status: enum pds_core_status_code + * @ver: Device's max supported version of this command + * @rsvd: Word boundary padding + */ +struct pds_core_finalize_update_comp { + u8 status; + u8 ver; + u8 rsvd[2]; +}; + +/** + * struct pds_core_match_record_desc_cmd - MATCH_RECORD_DESC command + * @opcode: PDS_CORE_CMD_MATCH_RECORD_DESC + * @ver: Driver's max supported version of this command + * @type: PLDM Descriptor Identifier Type + * @size: Length of the Descriptor Identifier Value + * @rsvd: Word boundary padding + * + * Expects to find the Descriptor Identifier Data in cmd_regs->data. Driver + * should keep the devcmd interface locked while preparing and sending this + * command. + */ +struct pds_core_match_record_desc_cmd { + u8 opcode; + u8 ver; + __le16 type; + __le16 size; + u8 rsvd[2]; +}; + +/** + * struct pds_core_match_record_desc_comp - MATCH_RECORD_DESC completion + * @status: enum pds_core_status_code + * @ver: Device's max supported version of this command + * @match: Whether or not the Record Descriptor matches the device + * @rsvd: Word boundary padding + * + * When status is PDS_RC_SUCCESS, then @match is valid, otherwise it's + * undefined. + */ +struct pds_core_match_record_desc_comp { + u8 status; + u8 ver; + u8 match; + u8 rsvd; +}; + +/** + * enum pds_core_host_mem_oper - HOST_MEM sub-operations + * @PDS_CORE_HOST_MEM_GET_COUNT: Query number of memory requests + * @PDS_CORE_HOST_MEM_QUERY: Query details of a memory request + * @PDS_CORE_HOST_MEM_ADD: Provide allocated memory to firmware + * @PDS_CORE_HOST_MEM_DEL: Notify firmware of memory deallocation + */ +enum pds_core_host_mem_oper { + PDS_CORE_HOST_MEM_GET_COUNT = 0, + PDS_CORE_HOST_MEM_QUERY = 1, + PDS_CORE_HOST_MEM_ADD = 2, + PDS_CORE_HOST_MEM_DEL = 3, +}; + +/** + * struct pds_core_host_mem_cmd - HOST_MEM command + * @opcode: Opcode PDS_CORE_CMD_HOST_MEM + * @oper: Operation (enum pds_core_host_mem_oper) + * @index: Memory request index (GET_COUNT: max_count, QUERY: index) + * @tag: Tag for this memory request (ADD/DEL) + * @reason: Reason for deletion (DEL only) + * @rsvd: Reserved + * @max_contig: Maximum contiguous memory size (GET_COUNT only) + * @size: Size of memory in bytes (ADD only) + * @buf_pa: DMA address of memory (ADD only) + * + * Unified command for all host memory operations. Fields are reused + * across operations to minimize opcode space usage. + */ +struct pds_core_host_mem_cmd { + u8 opcode; + u8 oper; + __le16 index; + __le16 tag; + u8 reason; + u8 rsvd; + __le32 max_contig; + __le32 size; + __le64 buf_pa; +}; + +/** + * struct pds_core_host_mem_comp - HOST_MEM completion + * @status: Status of the command (enum pds_core_status_code) + * @oper: Operation that was performed + * @count: Number of memory requests (GET_COUNT) + * @size: Size of memory request in bytes (QUERY) + * @tag: Tag for this memory request (QUERY/DEL) + * @rsvd: Reserved + */ +struct pds_core_host_mem_comp { + u8 status; + u8 oper; + __le16 count; + __le32 size; + __le16 tag; + u8 rsvd[6]; +}; + /* * union pds_core_dev_cmd - Overlay of core device command structures */ @@ -462,6 +915,14 @@ union pds_core_dev_cmd { struct pds_core_vf_setattr_cmd vf_setattr; struct pds_core_vf_getattr_cmd vf_getattr; struct pds_core_vf_ctrl_cmd vf_ctrl; + + struct pds_core_get_component_info_cmd get_component_info; + struct pds_core_send_pkg_data_cmd send_pkg_data; + struct pds_core_send_component_tbl_cmd send_component_tbl; + struct pds_core_send_component_cmd send_component; + struct pds_core_finalize_update_cmd finalize_update; + struct pds_core_match_record_desc_cmd match_record_desc; + struct pds_core_host_mem_cmd host_mem; }; /* @@ -480,6 +941,14 @@ union pds_core_dev_comp { struct pds_core_vf_setattr_comp vf_setattr; struct pds_core_vf_getattr_comp vf_getattr; struct pds_core_vf_ctrl_comp vf_ctrl; + + struct pds_core_get_component_info_comp get_component_info; + struct pds_core_send_pkg_data_comp send_pkg_data; + struct pds_core_send_component_tbl_comp send_component_tbl; + struct pds_core_send_component_comp send_component; + struct pds_core_finalize_update_comp finalize_update; + struct pds_core_match_record_desc_comp match_record_desc; + struct pds_core_host_mem_comp host_mem; }; /** diff --git a/include/linux/percpu-defs.h b/include/linux/percpu-defs.h index 43c854a273c3..dbe3267a0a13 100644 --- a/include/linux/percpu-defs.h +++ b/include/linux/percpu-defs.h @@ -65,13 +65,8 @@ * * Archs which need weak percpu definitions should set * CONFIG_ARCH_MODULE_NEEDS_WEAK_PER_CPU when necessary. - * - * To ensure that the generic code observes the above two - * restrictions, if CONFIG_DEBUG_FORCE_WEAK_PER_CPU is set weak - * definition is used for all cases. */ -#if (defined(CONFIG_ARCH_MODULE_NEEDS_WEAK_PER_CPU) && defined(MODULE)) || \ - defined(CONFIG_DEBUG_FORCE_WEAK_PER_CPU) +#if defined(CONFIG_ARCH_MODULE_NEEDS_WEAK_PER_CPU) && defined(MODULE) /* * __pcpu_scope_* dummy variable is used to enforce scope. It * receives the static modifier when it's used in front of @@ -203,7 +198,7 @@ /* * Accessors and operations. */ -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ /* * __verify_pcpu_ptr() verifies @ptr is a percpu pointer without evaluating @@ -514,5 +509,5 @@ do { \ #define this_cpu_inc_return(pcp) this_cpu_add_return(pcp, 1) #define this_cpu_dec_return(pcp) this_cpu_add_return(pcp, -1) -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ #endif /* _LINUX_PERCPU_DEFS_H */ diff --git a/include/linux/percpu-refcount.h b/include/linux/percpu-refcount.h index d73a1c08c3e3..1e3212e2c827 100644 --- a/include/linux/percpu-refcount.h +++ b/include/linux/percpu-refcount.h @@ -12,8 +12,8 @@ * start shutting down you call percpu_ref_kill() _before_ dropping the initial * refcount. * - * The refcount will have a range of 0 to ((1U << 31) - 1), i.e. one bit less - * than an atomic_t - this is because of the way shutdown works, see + * The refcount will have a range of 0 to LONG_MAX, i.e. one bit less + * than an atomic_long_t - this is because of the way shutdown works, see * percpu_ref_kill()/PERCPU_COUNT_BIAS. * * Before you call percpu_ref_kill(), percpu_ref_put() does not check for the @@ -269,6 +269,7 @@ static inline bool percpu_ref_tryget(struct percpu_ref *ref) /** * percpu_ref_tryget_live_rcu - same as percpu_ref_tryget_live() but the * caller is responsible for taking RCU. + * @ref: percpu_ref to try-get * * This function is safe to call as long as @ref is between init and exit. */ diff --git a/include/linux/percpu-rwsem.h b/include/linux/percpu-rwsem.h index c8cb010d655e..39d5bf8e6562 100644 --- a/include/linux/percpu-rwsem.h +++ b/include/linux/percpu-rwsem.h @@ -107,6 +107,8 @@ static inline bool percpu_down_read_trylock(struct percpu_rw_semaphore *sem) return ret; } +extern void __percpu_up_read(struct percpu_rw_semaphore *sem); + static inline void percpu_up_read(struct percpu_rw_semaphore *sem) { rwsem_release(&sem->dep_map, _RET_IP_); @@ -118,18 +120,7 @@ static inline void percpu_up_read(struct percpu_rw_semaphore *sem) if (likely(rcu_sync_is_idle(&sem->rss))) { this_cpu_dec(*sem->read_count); } else { - /* - * slowpath; reader will only ever wake a single blocked - * writer. - */ - smp_mb(); /* B matches C */ - /* - * In other words, if they see our decrement (presumably to - * aggregate zero, as that is the only time it matters) they - * will also see our critical section. - */ - this_cpu_dec(*sem->read_count); - rcuwait_wake_up(&sem->writer); + __percpu_up_read(sem); } preempt_enable(); } diff --git a/include/linux/percpu.h b/include/linux/percpu.h index 85bf8dd9f087..4f36b8585995 100644 --- a/include/linux/percpu.h +++ b/include/linux/percpu.h @@ -3,13 +3,14 @@ #define __LINUX_PERCPU_H #include <linux/alloc_tag.h> +#include <linux/cleanup.h> +#include <linux/compiler_types.h> +#include <linux/init.h> #include <linux/mmdebug.h> -#include <linux/preempt.h> -#include <linux/smp.h> #include <linux/pfn.h> -#include <linux/init.h> -#include <linux/cleanup.h> +#include <linux/preempt.h> #include <linux/sched.h> +#include <linux/smp.h> #include <asm/percpu.h> @@ -36,7 +37,7 @@ #define PCPU_BITMAP_BLOCK_BITS (PCPU_BITMAP_BLOCK_SIZE >> \ PCPU_MIN_ALLOC_SHIFT) -#ifdef CONFIG_RANDOM_KMALLOC_CACHES +#ifdef CONFIG_KMALLOC_PARTITION_CACHES # if defined(CONFIG_LOCKDEP) && !defined(CONFIG_PAGE_SIZE_4KB) # define PERCPU_DYNAMIC_SIZE_SHIFT 13 # else diff --git a/include/linux/perf/arm_pmu.h b/include/linux/perf/arm_pmu.h index 52b37f7bdbf9..02d2c7f45b52 100644 --- a/include/linux/perf/arm_pmu.h +++ b/include/linux/perf/arm_pmu.h @@ -119,7 +119,7 @@ struct arm_pmu { /* PMUv3 only */ int pmuver; - bool has_smt; + bool avoid_pmccntr; u64 reg_pmmir; u64 reg_brbidr; #define ARMV8_PMUV3_MAX_COMMON_EVENTS 0x40 diff --git a/include/linux/perf/riscv_pmu.h b/include/linux/perf/riscv_pmu.h index f82a28040594..ecaa40370830 100644 --- a/include/linux/perf/riscv_pmu.h +++ b/include/linux/perf/riscv_pmu.h @@ -55,7 +55,7 @@ struct riscv_pmu { irqreturn_t (*handle_irq)(int irq_num, void *dev); - unsigned long cmask; + DECLARE_BITMAP(cmask, RISCV_MAX_COUNTERS); u64 (*ctr_read)(struct perf_event *event); int (*ctr_get_idx)(struct perf_event *event); int (*ctr_get_width)(int idx); diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h index 48d851fbd8ea..5842552294c1 100644 --- a/include/linux/perf_event.h +++ b/include/linux/perf_event.h @@ -1791,22 +1791,8 @@ static inline int perf_is_paranoid(void) } extern int perf_allow_kernel(void); - -static inline int perf_allow_cpu(void) -{ - if (sysctl_perf_event_paranoid > 0 && !perfmon_capable()) - return -EACCES; - - return security_perf_event_open(PERF_SECURITY_CPU); -} - -static inline int perf_allow_tracepoint(void) -{ - if (sysctl_perf_event_paranoid > -1 && !perfmon_capable()) - return -EPERM; - - return security_perf_event_open(PERF_SECURITY_TRACEPOINT); -} +extern int perf_allow_cpu(void); +extern int perf_allow_tracepoint(void); extern int perf_exclude_event(struct perf_event *event, struct pt_regs *regs); @@ -2023,6 +2009,19 @@ perf_event_pause(struct perf_event *event, bool reset) { return 0; } static inline int perf_exclude_event(struct perf_event *event, struct pt_regs *regs) { return 0; } +static inline int perf_allow_kernel(void) +{ + return perfmon_capable() ? 0 : -EACCES; +} +static inline int perf_allow_cpu(void) +{ + return perfmon_capable() ? 0 : -EACCES; +} +static inline int perf_allow_tracepoint(void) +{ + return perfmon_capable() ? 0 : -EPERM; +} + #endif /* !CONFIG_PERF_EVENTS */ #if defined(CONFIG_PERF_EVENTS) && defined(CONFIG_CPU_SUP_INTEL) diff --git a/include/linux/pfn.h b/include/linux/pfn.h index b90ca0b6c331..cfedf0f61bb3 100644 --- a/include/linux/pfn.h +++ b/include/linux/pfn.h @@ -2,7 +2,7 @@ #ifndef _LINUX_PFN_H_ #define _LINUX_PFN_H_ -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #include <linux/types.h> #endif diff --git a/include/linux/pgalloc_tag.h b/include/linux/pgalloc_tag.h index 38a82d65e58e..951d33362268 100644 --- a/include/linux/pgalloc_tag.h +++ b/include/linux/pgalloc_tag.h @@ -181,7 +181,7 @@ static inline struct alloc_tag *__pgalloc_tag_get(struct page *page) if (get_page_tag_ref(page, &ref, &handle)) { alloc_tag_sub_check(&ref); - if (ref.ct) + if (ref.ct && !is_codetag_empty(&ref)) tag = ct_to_alloc_tag(ref.ct); put_page_tag_ref(handle); } diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h index a50df42a893f..8c093c119e5a 100644 --- a/include/linux/pgtable.h +++ b/include/linux/pgtable.h @@ -8,7 +8,7 @@ #define PMD_ORDER (PMD_SHIFT - PAGE_SHIFT) #define PUD_ORDER (PUD_SHIFT - PAGE_SHIFT) -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #ifdef CONFIG_MMU #include <linux/mm_types.h> @@ -301,6 +301,28 @@ static inline void lazy_mmu_mode_disable(void) } /** + * __task_lazy_mmu_mode_pause() - Pause the lazy MMU mode for a task. + * @tsk: The task to check. + * + * Pauses the lazy MMU mode of @tsk. + * + * This function only operates on the state saved in task_struct; to pause + * current lazy_mmu_mode_pause() should be used instead. + * + * This function is intended for architectures that implement the lazy MMU + * mode; it must not be called from generic code. + */ +static inline void __task_lazy_mmu_mode_pause(struct task_struct *tsk) +{ + struct lazy_mmu_state *state = &tsk->lazy_mmu_state; + + VM_WARN_ON_ONCE(state->pause_count == U8_MAX); + + if (state->pause_count++ == 0 && state->enable_count > 0) + arch_leave_lazy_mmu_mode(); +} + +/** * lazy_mmu_mode_pause() - Pause the lazy MMU mode. * * Pauses the lazy MMU mode; if it is currently active, disables it and calls @@ -315,15 +337,32 @@ static inline void lazy_mmu_mode_disable(void) */ static inline void lazy_mmu_mode_pause(void) { - struct lazy_mmu_state *state = ¤t->lazy_mmu_state; - if (in_interrupt()) return; - VM_WARN_ON_ONCE(state->pause_count == U8_MAX); + __task_lazy_mmu_mode_pause(current); +} - if (state->pause_count++ == 0 && state->enable_count > 0) - arch_leave_lazy_mmu_mode(); +/** + * __task_lazy_mmu_mode_resume() - Resume the lazy MMU mode for a task. + * @tsk: The task to check. + * + * Resumes the lazy MMU mode of @tsk. + * + * This function only operates on the state saved in task_struct; to resume + * current lazy_mmu_mode_resume() should be used instead. + * + * This function is intended for architectures that implement the lazy MMU + * mode; it must not be called from generic code. + */ +static inline void __task_lazy_mmu_mode_resume(struct task_struct *tsk) +{ + struct lazy_mmu_state *state = &tsk->lazy_mmu_state; + + VM_WARN_ON_ONCE(state->pause_count == 0); + + if (--state->pause_count == 0 && state->enable_count > 0) + arch_enter_lazy_mmu_mode(); } /** @@ -341,15 +380,10 @@ static inline void lazy_mmu_mode_pause(void) */ static inline void lazy_mmu_mode_resume(void) { - struct lazy_mmu_state *state = ¤t->lazy_mmu_state; - if (in_interrupt()) return; - VM_WARN_ON_ONCE(state->pause_count == 0); - - if (--state->pause_count == 0 && state->enable_count > 0) - arch_enter_lazy_mmu_mode(); + __task_lazy_mmu_mode_resume(current); } #else static inline void lazy_mmu_mode_enable(void) {} @@ -491,64 +525,63 @@ static inline pgd_t pgdp_get(pgd_t *pgdp) #endif #ifndef __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG -static inline int ptep_test_and_clear_young(struct vm_area_struct *vma, - unsigned long address, - pte_t *ptep) +static inline bool ptep_test_and_clear_young(struct vm_area_struct *vma, + unsigned long address, pte_t *ptep) { pte_t pte = ptep_get(ptep); - int r = 1; + bool young = true; + if (!pte_young(pte)) - r = 0; + young = false; else set_pte_at(vma->vm_mm, address, ptep, pte_mkold(pte)); - return r; + return young; } #endif #ifndef __HAVE_ARCH_PMDP_TEST_AND_CLEAR_YOUNG #if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG) -static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma, - unsigned long address, - pmd_t *pmdp) +static inline bool pmdp_test_and_clear_young(struct vm_area_struct *vma, + unsigned long address, pmd_t *pmdp) { pmd_t pmd = *pmdp; - int r = 1; + bool young = true; + if (!pmd_young(pmd)) - r = 0; + young = false; else set_pmd_at(vma->vm_mm, address, pmdp, pmd_mkold(pmd)); - return r; + return young; } #else -static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma, - unsigned long address, - pmd_t *pmdp) +static inline bool pmdp_test_and_clear_young(struct vm_area_struct *vma, + unsigned long address, pmd_t *pmdp) { BUILD_BUG(); - return 0; + return false; } #endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG */ #endif #ifndef __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH -int ptep_clear_flush_young(struct vm_area_struct *vma, - unsigned long address, pte_t *ptep); +bool ptep_clear_flush_young(struct vm_area_struct *vma, + unsigned long address, pte_t *ptep); #endif #ifndef __HAVE_ARCH_PMDP_CLEAR_YOUNG_FLUSH #ifdef CONFIG_TRANSPARENT_HUGEPAGE -extern int pmdp_clear_flush_young(struct vm_area_struct *vma, - unsigned long address, pmd_t *pmdp); +bool pmdp_clear_flush_young(struct vm_area_struct *vma, + unsigned long address, pmd_t *pmdp); #else /* * Despite relevant to THP only, this API is called from generic rmap code * under PageTransHuge(), hence needs a dummy implementation for !THP */ -static inline int pmdp_clear_flush_young(struct vm_area_struct *vma, - unsigned long address, pmd_t *pmdp) +static inline bool pmdp_clear_flush_young(struct vm_area_struct *vma, + unsigned long address, pmd_t *pmdp) { BUILD_BUG(); - return 0; + return false; } #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ #endif @@ -955,49 +988,30 @@ static inline void update_mmu_tlb(struct vm_area_struct *vma, update_mmu_tlb_range(vma, address, ptep, 1); } -/* - * Some architectures may be able to avoid expensive synchronization - * primitives when modifications are made to PTE's which are already - * not present, or in the process of an address space destruction. - */ -#ifndef __HAVE_ARCH_PTE_CLEAR_NOT_PRESENT_FULL -static inline void pte_clear_not_present_full(struct mm_struct *mm, - unsigned long address, - pte_t *ptep, - int full) -{ - pte_clear(mm, address, ptep); -} -#endif - -#ifndef clear_not_present_full_ptes /** - * clear_not_present_full_ptes - Clear multiple not present PTEs which are - * consecutive in the pgtable. + * clear_nonpresent_ptes - Clear multiple non-present PTEs which are + * consecutive in the pgtable. * @mm: Address space the ptes represent. * @addr: Address of the first pte. * @ptep: Page table pointer for the first entry. * @nr: Number of entries to clear. - * @full: Whether we are clearing a full mm. - * - * May be overridden by the architecture; otherwise, implemented as a simple - * loop over pte_clear_not_present_full(). * * Context: The caller holds the page table lock. The PTEs are all not present. * The PTEs are all in the same PMD. */ -static inline void clear_not_present_full_ptes(struct mm_struct *mm, - unsigned long addr, pte_t *ptep, unsigned int nr, int full) +static inline void clear_nonpresent_ptes(struct mm_struct *mm, + unsigned long addr, pte_t *ptep, unsigned int nr) { + (void)addr; + for (;;) { - pte_clear_not_present_full(mm, addr, ptep, full); + pte_clear(mm, addr, ptep); if (--nr == 0) break; ptep++; addr += PAGE_SIZE; } } -#endif #ifndef __HAVE_ARCH_PTEP_CLEAR_FLUSH extern pte_t ptep_clear_flush(struct vm_area_struct *vma, @@ -1037,6 +1051,49 @@ static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long addres } #endif +#ifndef ptep_try_set +/** + * ptep_try_set - atomically set an empty kernel PTE + * @ptep: page table entry + * @new_pte: value to install + * + * Atomically set *@ptep to @new_pte iff *@ptep is pte_none(). Return true on + * success, false if the slot was already populated or the arch has no + * implementation. + * + * For special kernel page tables only - never user page tables. The caller must + * prevent concurrent teardown of @ptep and must accept that other writers may + * race. Concurrent clearers must use ptep_get_and_clear() so racing accesses + * agree on the outcome. + * + * Architectures opt in by providing a cmpxchg-based override and defining + * ptep_try_set as an identity macro. The generic stub returns false, which is + * correct for callers that fall through to oops on failure. + */ +static inline bool ptep_try_set(pte_t *ptep, pte_t new_pte) +{ + return false; +} +#endif + +#ifndef flush_tlb_before_set +/** + * flush_tlb_before_set - invalidate a kernel PTE's TLB before re-setting it + * @addr: kernel virtual address whose PTE was just cleared + * + * Some architectures (e.g. arm64) do not allow a live page-table entry to be + * repointed at a different page in one step. The old entry must first be made + * invalid and its translation flushed from every TLB, and only then may the new + * entry be written. + * + * This is only for the lockless atomic kernel-PTE installers (ptep_try_set()). + * It must be callable with interrupts disabled. + */ +static inline void flush_tlb_before_set(unsigned long addr) +{ +} +#endif + #ifndef wrprotect_ptes /** * wrprotect_ptes - Write-protect PTEs that map consecutive pages of the same @@ -1086,10 +1143,10 @@ static inline void wrprotect_ptes(struct mm_struct *mm, unsigned long addr, * Context: The caller holds the page table lock. The PTEs map consecutive * pages that belong to the same folio. The PTEs are all in the same PMD. */ -static inline int clear_flush_young_ptes(struct vm_area_struct *vma, +static inline bool clear_flush_young_ptes(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep, unsigned int nr) { - int young = 0; + bool young = false; for (;;) { young |= ptep_clear_flush_young(vma, addr, ptep); @@ -1103,6 +1160,43 @@ static inline int clear_flush_young_ptes(struct vm_area_struct *vma, } #endif +#ifndef test_and_clear_young_ptes +/** + * test_and_clear_young_ptes - Mark PTEs that map consecutive pages of the same + * folio as old + * @vma: The virtual memory area the pages are mapped into. + * @addr: Address the first page is mapped at. + * @ptep: Page table pointer for the first entry. + * @nr: Number of entries to clear access bit. + * + * May be overridden by the architecture; otherwise, implemented as a simple + * loop over ptep_test_and_clear_young(). + * + * Note that PTE bits in the PTE range besides the PFN can differ. For example, + * some PTEs might be write-protected. + * + * Context: The caller holds the page table lock. The PTEs map consecutive + * pages that belong to the same folio. The PTEs are all in the same PMD. + * + * Returns: whether any PTE was young. + */ +static inline bool test_and_clear_young_ptes(struct vm_area_struct *vma, + unsigned long addr, pte_t *ptep, unsigned int nr) +{ + bool young = false; + + for (;;) { + young |= ptep_test_and_clear_young(vma, addr, ptep); + if (--nr == 0) + break; + ptep++; + addr += PAGE_SIZE; + } + + return young; +} +#endif + /* * On some architectures hardware does not set page access bit when accessing * memory page, it is responsibility of software setting this bit. It brings @@ -1745,7 +1839,7 @@ static inline pgprot_t pgprot_modify(pgprot_t oldprot, pgprot_t newprot) #endif #ifdef CONFIG_HAVE_ARCH_SOFT_DIRTY -#ifndef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifndef CONFIG_ARCH_HAS_PMD_SOFTLEAVES static inline pmd_t pmd_swp_mksoft_dirty(pmd_t pmd) { return pmd; @@ -1917,41 +2011,56 @@ static inline void pfnmap_setup_cachemode_pfn(unsigned long pfn, pgprot_t *prot) pfnmap_setup_cachemode(pfn, PAGE_SIZE, prot); } -#ifdef CONFIG_MMU +/* + * ZERO_PAGE() is global shared page(s) that is always zero. It is used for + * zero-mapped memory areas, CoW etc. + * + * On architectures that __HAVE_COLOR_ZERO_PAGE there are several such pages + * for different ranges in the virtual address space. + * + * zero_page_pfn identifies the first (or the only) pfn for these pages. + * + * For architectures that don't __HAVE_COLOR_ZERO_PAGE the zero page lives in + * empty_zero_page in BSS. + */ +void arch_setup_zero_pages(void); + #ifdef __HAVE_COLOR_ZERO_PAGE static inline int is_zero_pfn(unsigned long pfn) { - extern unsigned long zero_pfn; - unsigned long offset_from_zero_pfn = pfn - zero_pfn; + extern unsigned long zero_page_pfn; + unsigned long offset_from_zero_pfn = pfn - zero_page_pfn; + return offset_from_zero_pfn <= (zero_page_mask >> PAGE_SHIFT); } -#define my_zero_pfn(addr) page_to_pfn(ZERO_PAGE(addr)) +#define zero_pfn(addr) page_to_pfn(ZERO_PAGE(addr)) #else static inline int is_zero_pfn(unsigned long pfn) { - extern unsigned long zero_pfn; - return pfn == zero_pfn; -} + extern unsigned long zero_page_pfn; -static inline unsigned long my_zero_pfn(unsigned long addr) -{ - extern unsigned long zero_pfn; - return zero_pfn; + return pfn == zero_page_pfn; } -#endif -#else -static inline int is_zero_pfn(unsigned long pfn) + +static inline unsigned long zero_pfn(unsigned long addr) { - return 0; + extern unsigned long zero_page_pfn; + + return zero_page_pfn; } -static inline unsigned long my_zero_pfn(unsigned long addr) +extern const uint8_t empty_zero_page[PAGE_SIZE]; +extern struct page *__zero_page; + +static inline struct page *_zero_page(unsigned long addr) { - return 0; + return __zero_page; } -#endif /* CONFIG_MMU */ +#define ZERO_PAGE(vaddr) _zero_page(vaddr) + +#endif /* __HAVE_COLOR_ZERO_PAGE */ #ifdef CONFIG_MMU @@ -1989,7 +2098,7 @@ static inline int pud_trans_unstable(pud_t *pud) { #if defined(CONFIG_TRANSPARENT_HUGEPAGE) && \ defined(CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD) - pud_t pudval = READ_ONCE(*pud); + pud_t pudval = pudp_get(pud); if (pud_none(pudval) || pud_trans_huge(pudval)) return 1; @@ -2001,18 +2110,26 @@ static inline int pud_trans_unstable(pud_t *pud) return 0; } -#ifndef CONFIG_NUMA_BALANCING +#ifndef CONFIG_ARCH_HAS_PTE_PROTNONE /* - * In an inaccessible (PROT_NONE) VMA, pte_protnone() may indicate "yes". It is - * perfectly valid to indicate "no" in that case, which is why our default - * implementation defaults to "always no". - * - * In an accessible VMA, however, pte_protnone() reliably indicates PROT_NONE - * page protection due to NUMA hinting. NUMA hinting faults only apply in - * accessible VMAs. - * - * So, to reliably identify PROT_NONE PTEs that require a NUMA hinting fault, - * looking at the VMA accessibility is sufficient. + * In an inaccessible (PROT_NONE) VMA, pte_protnone() may indicate "yes". It + * is perfectly valid to indicate "no" in that case, which is why our + * default implementation defaults to "always no". + * + * In an accessible VMA, pte_protnone() reliably indicates a present + * PROT_NONE page protection. Today the kernel uses such PTEs for two + * purposes: NUMA hinting faults, and userfaultfd RWP tracking on + * VM_UFFD_RWP VMAs. The two are distinguished by the uffd PTE bit and + * the VMA flag; see include/linux/userfaultfd_k.h. + * + * So, to reliably identify PROT_NONE PTEs that require kernel handling, + * looking at the VMA accessibility (and the uffd bit on RWP VMAs) is + * sufficient. + * + * Architectures without CONFIG_ARCH_HAS_PTE_PROTNONE get the always-zero + * stubs below; PAGE_NONE references that survive to runtime fire the + * BUILD_BUG() fallback, since callers should have folded such paths to + * dead code via IS_ENABLED(CONFIG_ARCH_HAS_PTE_PROTNONE). */ static inline int pte_protnone(pte_t pte) { @@ -2023,7 +2140,11 @@ static inline int pmd_protnone(pmd_t pmd) { return 0; } -#endif /* CONFIG_NUMA_BALANCING */ + +#ifndef PAGE_NONE +#define PAGE_NONE ({ BUILD_BUG(); (pgprot_t){0}; }) +#endif +#endif /* CONFIG_ARCH_HAS_PTE_PROTNONE */ #endif /* CONFIG_MMU */ @@ -2192,7 +2313,7 @@ static inline const char *pgtable_level_to_str(enum pgtable_level level) } } -#endif /* !__ASSEMBLY__ */ +#endif /* !__ASSEMBLER__ */ #if !defined(MAX_POSSIBLE_PHYSMEM_BITS) && !defined(CONFIG_64BIT) #ifdef CONFIG_PHYS_ADDR_T_64BIT diff --git a/include/linux/phy.h b/include/linux/phy.h index 6f9979a26892..5f8d65868e0f 100644 --- a/include/linux/phy.h +++ b/include/linux/phy.h @@ -23,7 +23,7 @@ #include <linux/module.h> #include <linux/timer.h> #include <linux/workqueue.h> -#include <linux/mod_devicetable.h> +#include <linux/device-id/mdio.h> #include <linux/u64_stats_sync.h> #include <linux/irqreturn.h> #include <linux/iopoll.h> @@ -612,6 +612,8 @@ struct phy_oatc14_sqi_capability { * @advertising_eee: Currently advertised EEE linkmodes * @enable_tx_lpi: When True, MAC should transmit LPI to PHY * @eee_active: phylib private state, indicating that EEE has been negotiated + * @autonomous_eee_disabled: Set when autonomous EEE has been disabled, + * used to re-apply after PHY soft reset * @eee_cfg: User configuration of EEE * @lp_advertising: Current link partner advertised linkmodes * @host_interfaces: PHY interface modes supported by host @@ -739,6 +741,7 @@ struct phy_device { __ETHTOOL_DECLARE_LINK_MODE_MASK(eee_disabled_modes); bool enable_tx_lpi; bool eee_active; + bool autonomous_eee_disabled; struct eee_config eee_cfg; /* Host supported PHY interface types. Should be ignored if empty. */ @@ -1359,6 +1362,17 @@ struct phy_driver { void (*get_stats)(struct phy_device *dev, struct ethtool_stats *stats, u64 *data); + /** + * @disable_autonomous_eee: Disable PHY-autonomous EEE + * + * Some PHYs manage EEE autonomously, preventing the MAC from + * controlling LPI signaling. This callback disables autonomous + * EEE at the PHY. + * + * Return: 0 on success, negative errno on failure. + */ + int (*disable_autonomous_eee)(struct phy_device *dev); + /* Get and Set PHY tunables */ /** @get_tunable: Return the value of a tunable */ int (*get_tunable)(struct phy_device *dev, @@ -2152,8 +2166,6 @@ int phy_suspend(struct phy_device *phydev); int phy_resume(struct phy_device *phydev); int __phy_resume(struct phy_device *phydev); int phy_loopback(struct phy_device *phydev, bool enable, int speed); -struct phy_device *phy_attach(struct net_device *dev, const char *bus_id, - phy_interface_t interface); struct phy_device *phy_find_next(struct mii_bus *bus, struct phy_device *pos); int phy_attach_direct(struct net_device *dev, struct phy_device *phydev, u32 flags, phy_interface_t interface); @@ -2219,6 +2231,11 @@ static inline void phy_device_reset(struct phy_device *phydev, int value) mdio_device_reset(&phydev->mdio, value); } +static inline bool phy_device_has_reset(struct phy_device *phydev) +{ + return mdiodev_has_reset(&phydev->mdio); +} + #define phydev_err(_phydev, format, args...) \ dev_err(&_phydev->mdio.dev, format, ##args) @@ -2285,6 +2302,9 @@ static inline int genphy_no_config_intr(struct phy_device *phydev) { return 0; } +int genphy_read_mmd_c45(struct phy_device *phydev, int devnum, u16 regnum); +int genphy_write_mmd_c45(struct phy_device *phydev, int devnum, u16 regnum, + u16 val); int genphy_read_mmd_unsupported(struct phy_device *phdev, int devad, u16 regnum); int genphy_write_mmd_unsupported(struct phy_device *phdev, int devnum, @@ -2297,6 +2317,7 @@ int genphy_c37_read_status(struct phy_device *phydev, bool *changed); /* Clause 45 PHY */ int genphy_c45_restart_aneg(struct phy_device *phydev); int genphy_c45_check_and_restart_aneg(struct phy_device *phydev, bool restart); +int genphy_c45_pma_soft_reset(struct phy_device *phydev); int genphy_c45_aneg_done(struct phy_device *phydev); int genphy_c45_read_link(struct phy_device *phydev); int genphy_c45_read_lpa(struct phy_device *phydev); @@ -2448,9 +2469,6 @@ int __phy_hwtstamp_set(struct phy_device *phydev, struct phy_port *phy_get_sfp_port(struct phy_device *phydev); -extern const struct bus_type mdio_bus_type; -extern const struct class mdio_bus_class; - /** * phy_module_driver() - Helper macro for registering PHY drivers * @__phy_drivers: array of PHY drivers to register diff --git a/include/linux/phy_link_topology.h b/include/linux/phy_link_topology.h index 68a59e25821c..95575f68d5bc 100644 --- a/include/linux/phy_link_topology.h +++ b/include/linux/phy_link_topology.h @@ -36,6 +36,11 @@ struct phy_device_node { struct phy_device *phy; }; +static inline bool phy_link_topo_empty(struct net_device *dev) +{ + return !dev->link_topo; +} + #if IS_ENABLED(CONFIG_PHYLIB) int phy_link_topo_add_phy(struct net_device *dev, struct phy_device *phy, diff --git a/include/linux/phylink.h b/include/linux/phylink.h index 2bc0db3d52ac..1dda5c7ed5f1 100644 --- a/include/linux/phylink.h +++ b/include/linux/phylink.h @@ -791,6 +791,7 @@ static inline int phylink_get_link_timer_ns(phy_interface_t interface) { switch (interface) { case PHY_INTERFACE_MODE_SGMII: + case PHY_INTERFACE_MODE_PSGMII: case PHY_INTERFACE_MODE_QSGMII: case PHY_INTERFACE_MODE_USXGMII: case PHY_INTERFACE_MODE_10G_QXGMII: diff --git a/include/linux/pidfs.h b/include/linux/pidfs.h index 416bdff4d6ce..0abf7da9ab23 100644 --- a/include/linux/pidfs.h +++ b/include/linux/pidfs.h @@ -2,6 +2,8 @@ #ifndef _LINUX_PID_FS_H #define _LINUX_PID_FS_H +#include <linux/gfp_types.h> + struct coredump_params; struct file *pidfs_alloc_file(struct pid *pid, unsigned int flags); @@ -14,7 +16,21 @@ void pidfs_exit(struct task_struct *tsk); void pidfs_coredump(const struct coredump_params *cprm); #endif extern const struct dentry_operations pidfs_dentry_operations; -int pidfs_register_pid(struct pid *pid); +int pidfs_register_pid_gfp(struct pid *pid, gfp_t gfp); + +/** + * pidfs_register_pid - register a struct pid in pidfs + * @pid: pid to pin + * + * Register a struct pid in pidfs. + * + * Return: On success zero, on error a negative error code is returned. + */ +static inline int pidfs_register_pid(struct pid *pid) +{ + return pidfs_register_pid_gfp(pid, GFP_KERNEL); +} + void pidfs_free_pid(struct pid *pid); #endif /* _LINUX_PID_FS_H */ diff --git a/include/linux/pinctrl/consumer.h b/include/linux/pinctrl/consumer.h index 63ce16191eb9..11b8f0b8da0c 100644 --- a/include/linux/pinctrl/consumer.h +++ b/include/linux/pinctrl/consumer.h @@ -35,6 +35,8 @@ int pinctrl_gpio_direction_output(struct gpio_chip *gc, unsigned int offset); int pinctrl_gpio_set_config(struct gpio_chip *gc, unsigned int offset, unsigned long config); +int pinctrl_gpio_get_config(struct gpio_chip *gc, unsigned int offset, + unsigned long *config); struct pinctrl * __must_check pinctrl_get(struct device *dev); void pinctrl_put(struct pinctrl *p); @@ -102,6 +104,13 @@ pinctrl_gpio_direction_output(struct gpio_chip *gc, unsigned int offset) } static inline int +pinctrl_gpio_get_config(struct gpio_chip *gc, unsigned int offset, + unsigned long *config) +{ + return 0; +} + +static inline int pinctrl_gpio_set_config(struct gpio_chip *gc, unsigned int offset, unsigned long config) { diff --git a/include/linux/pinctrl/pinconf-generic.h b/include/linux/pinctrl/pinconf-generic.h index 89277808ea61..a5d4b2d8633a 100644 --- a/include/linux/pinctrl/pinconf-generic.h +++ b/include/linux/pinctrl/pinconf-generic.h @@ -83,6 +83,8 @@ struct pinctrl_map; * schmitt-trigger mode is disabled. * @PIN_CONFIG_INPUT_SCHMITT_UV: this will configure an input pin to run in * schmitt-trigger mode. The argument is in uV. + * @PIN_CONFIG_INPUT_VOLTAGE_UV: this will configure the input voltage level of + * the pin. The argument is specified in microvolts. * @PIN_CONFIG_MODE_LOW_POWER: this will configure the pin for low power * operation, if several modes of operation are supported these can be * passed in the argument on a custom form, else just use argument 1 @@ -115,7 +117,7 @@ struct pinctrl_map; * @PIN_CONFIG_SKEW_DELAY_INPUT_PS: if the pin has independent values for the * programmable skew rate (on inputs) and latch delay (on outputs), then * this parameter specifies the clock skew only. The argument is in ps. - * @PIN_CONFIG_SKEW_DELAY_OUPUT_PS: if the pin has independent values for the + * @PIN_CONFIG_SKEW_DELAY_OUTPUT_PS: if the pin has independent values for the * programmable skew rate (on inputs) and latch delay (on outputs), then * this parameter specifies the latch delay only. The argument is in ps. * @PIN_CONFIG_SLEEP_HARDWARE_STATE: indicate this is sleep related state. @@ -145,6 +147,7 @@ enum pin_config_param { PIN_CONFIG_INPUT_SCHMITT, PIN_CONFIG_INPUT_SCHMITT_ENABLE, PIN_CONFIG_INPUT_SCHMITT_UV, + PIN_CONFIG_INPUT_VOLTAGE_UV, PIN_CONFIG_MODE_LOW_POWER, PIN_CONFIG_MODE_PWM, PIN_CONFIG_LEVEL, diff --git a/include/linux/pinctrl/pinmux.h b/include/linux/pinctrl/pinmux.h index 094bbe2fd6fd..77664937eeb2 100644 --- a/include/linux/pinctrl/pinmux.h +++ b/include/linux/pinctrl/pinmux.h @@ -51,6 +51,8 @@ struct pinctrl_gpio_range; * are handled by the pinmux subsystem. The @func_selector selects a * certain function whereas @group_selector selects a certain set of pins * to be used. On simple controllers the latter argument may be ignored + * @release_mux: Release software resources acquired by @set_mux. This callback + * must not change hardware state to avoid glitches when switching mux. * @gpio_request_enable: requests and enables GPIO on a certain pin. * Implement this only if you can mux every pin individually as GPIO. The * affected GPIO range is passed along with an offset(pin number) into that @@ -80,6 +82,9 @@ struct pinmux_ops { unsigned int selector); int (*set_mux) (struct pinctrl_dev *pctldev, unsigned int func_selector, unsigned int group_selector); + void (*release_mux) (struct pinctrl_dev *pctldev, + unsigned int func_selector, + unsigned int group_selector); int (*gpio_request_enable) (struct pinctrl_dev *pctldev, struct pinctrl_gpio_range *range, unsigned int offset); diff --git a/include/linux/pipe_fs_i.h b/include/linux/pipe_fs_i.h index 7f6a92ac9704..6402930282e5 100644 --- a/include/linux/pipe_fs_i.h +++ b/include/linux/pipe_fs_i.h @@ -14,6 +14,9 @@ #define PIPE_BUF_FLAG_LOSS 0x40 /* Message loss happened after this buffer */ #endif +#define PIPE_PREALLOC_MAX 8 /* max pages in prealloc pool */ +#define PIPE_PREALLOC_KEEP 2 /* keep at least this many after trim */ + /** * struct pipe_buffer - a linux kernel pipe buffer * @page: the page containing the data for the pipe buffer @@ -59,6 +62,21 @@ union pipe_index { }; /** + * struct anon_pipe_prealloc - per-pipe page preallocation pool + * @pages: array of cached pages (pool) + * @count: number of pages currently in the pool + * + * Each pipe keeps a small bounded pool of preallocated pages to reduce + * allocation overhead during writes. The pool is bounded at PIPE_PREALLOC_MAX + * and trimmed down to PIPE_PREALLOC_KEEP after a write completes. + */ +struct anon_pipe_prealloc { + struct page *pages[PIPE_PREALLOC_MAX]; + + unsigned int __data_racy count; +}; + +/** * struct pipe_inode_info - a linux kernel pipe * @mutex: mutex protecting the whole thing * @rd_wait: reader wait point in case of empty pipe @@ -68,13 +86,13 @@ union pipe_index { * @max_usage: The maximum number of slots that may be used in the ring * @ring_size: total number of buffers (should be a power of 2) * @nr_accounted: The amount this pipe accounts for in user->pipe_bufs - * @tmp_page: cached released page + * @prealloc: per-pipe page preallocation pool * @readers: number of current readers of this pipe * @writers: number of current writers of this pipe * @files: number of struct file referring this pipe (protected by ->i_lock) * @r_counter: reader counter * @w_counter: writer counter - * @poll_usage: is this pipe used for epoll, which has crazy wakeups? + * @pseudo_edgetrigger: has an EPOLLET consumer, enable per-write wakeups * @fasync_readers: reader side fasync * @fasync_writers: writer side fasync * @bufs: the circular array of pipe buffers @@ -95,11 +113,11 @@ struct pipe_inode_info { unsigned int files; unsigned int r_counter; unsigned int w_counter; - bool poll_usage; + bool pseudo_edgetrigger; #ifdef CONFIG_WATCH_QUEUE bool note_loss; #endif - struct page *tmp_page[2]; + struct anon_pipe_prealloc prealloc; struct fasync_struct *fasync_readers; struct fasync_struct *fasync_writers; struct pipe_buffer *bufs; diff --git a/include/linux/platform_data/apds990x.h b/include/linux/platform_data/apds990x.h deleted file mode 100644 index 94dfbaa365e1..000000000000 --- a/include/linux/platform_data/apds990x.h +++ /dev/null @@ -1,65 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * This file is part of the APDS990x sensor driver. - * Chip is combined proximity and ambient light sensor. - * - * Copyright (C) 2010 Nokia Corporation and/or its subsidiary(-ies). - * - * Contact: Samu Onkalo <samu.p.onkalo@nokia.com> - */ - -#ifndef __APDS990X_H__ -#define __APDS990X_H__ - - -#define APDS_IRLED_CURR_12mA 0x3 -#define APDS_IRLED_CURR_25mA 0x2 -#define APDS_IRLED_CURR_50mA 0x1 -#define APDS_IRLED_CURR_100mA 0x0 - -/** - * struct apds990x_chip_factors - defines effect of the cover window - * @ga: Total glass attenuation - * @cf1: clear channel factor 1 for raw to lux conversion - * @irf1: IR channel factor 1 for raw to lux conversion - * @cf2: clear channel factor 2 for raw to lux conversion - * @irf2: IR channel factor 2 for raw to lux conversion - * @df: device factor for conversion formulas - * - * Structure for tuning ALS calculation to match with environment. - * Values depend on the material above the sensor and the sensor - * itself. If the GA is zero, driver will use uncovered sensor default values - * format: decimal value * APDS_PARAM_SCALE except df which is plain integer. - */ -#define APDS_PARAM_SCALE 4096 -struct apds990x_chip_factors { - int ga; - int cf1; - int irf1; - int cf2; - int irf2; - int df; -}; - -/** - * struct apds990x_platform_data - platform data for apsd990x.c driver - * @cf: chip factor data - * @pddrive: IR-led driving current - * @ppcount: number of IR pulses used for proximity estimation - * @setup_resources: interrupt line setup call back function - * @release_resources: interrupt line release call back function - * - * Proximity detection result depends heavily on correct ppcount, pdrive - * and cover window. - * - */ - -struct apds990x_platform_data { - struct apds990x_chip_factors cf; - u8 pdrive; - u8 ppcount; - int (*setup_resources)(void); - int (*release_resources)(void); -}; - -#endif diff --git a/include/linux/platform_data/asoc-pxa.h b/include/linux/platform_data/asoc-pxa.h index 7b5b9e20fbf5..0d5eaf4b100c 100644 --- a/include/linux/platform_data/asoc-pxa.h +++ b/include/linux/platform_data/asoc-pxa.h @@ -6,27 +6,6 @@ #include <sound/pcm.h> #include <sound/ac97_codec.h> -/* - * @reset_gpio: AC97 reset gpio (normally gpio113 or gpio95) - * a -1 value means no gpio will be used for reset - * @codec_pdata: AC97 codec platform_data - - * reset_gpio should only be specified for pxa27x CPUs where a silicon - * bug prevents correct operation of the reset line. If not specified, - * the default behaviour on these CPUs is to consider gpio 113 as the - * AC97 reset line, which is the default on most boards. - */ -typedef struct { - int (*startup)(struct snd_pcm_substream *, void *); - void (*shutdown)(struct snd_pcm_substream *, void *); - void (*suspend)(void *); - void (*resume)(void *); - void *priv; - int reset_gpio; - void *codec_pdata[AC97_BUS_MAX_DEVICES]; -} pxa2xx_audio_ops_t; - -extern void pxa_set_ac97_info(pxa2xx_audio_ops_t *ops); -extern void pxa27x_configure_ac97reset(int reset_gpio, bool to_gpio); +extern void pxa27x_configure_ac97reset(struct gpio_desc *reset_gpio, bool to_gpio); #endif diff --git a/include/linux/platform_data/cros_ec_proto.h b/include/linux/platform_data/cros_ec_proto.h index de14923720a5..a1ccecf5e1f8 100644 --- a/include/linux/platform_data/cros_ec_proto.h +++ b/include/linux/platform_data/cros_ec_proto.h @@ -228,6 +228,7 @@ struct cros_ec_platform { /** * struct cros_ec_dev - ChromeOS EC device entry point. * @class_dev: Device structure used in sysfs. + * @group: sysfs attributes groups for this EC. * @ec_dev: cros_ec_device structure to talk to the physical device. * @dev: Pointer to the platform device. * @debug_info: cros_ec_debugfs structure for debugging information. @@ -237,6 +238,7 @@ struct cros_ec_platform { */ struct cros_ec_dev { struct device class_dev; + const struct attribute_group *group; struct cros_ec_device *ec_dev; struct device *dev; struct cros_ec_debugfs *debug_info; @@ -269,6 +271,8 @@ int cros_ec_get_next_event(struct cros_ec_device *ec_dev, u32 cros_ec_get_host_event(struct cros_ec_device *ec_dev); +int cros_ec_read_features(struct cros_ec_dev *ec); + bool cros_ec_check_features(struct cros_ec_dev *ec, int feature); int cros_ec_get_sensor_count(struct cros_ec_dev *ec); diff --git a/include/linux/platform_data/crypto-ux500.h b/include/linux/platform_data/crypto-ux500.h deleted file mode 100644 index 5d43350e32cc..000000000000 --- a/include/linux/platform_data/crypto-ux500.h +++ /dev/null @@ -1,22 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright (C) ST-Ericsson SA 2011 - * - * Author: Joakim Bech <joakim.xx.bech@stericsson.com> for ST-Ericsson - */ -#ifndef _CRYPTO_UX500_H -#define _CRYPTO_UX500_H -#include <linux/dmaengine.h> -#include <linux/platform_data/dma-ste-dma40.h> - -struct hash_platform_data { - void *mem_to_engine; - bool (*dma_filter)(struct dma_chan *chan, void *filter_param); -}; - -struct cryp_platform_data { - struct stedma40_chan_cfg mem_to_engine; - struct stedma40_chan_cfg engine_to_mem; -}; - -#endif diff --git a/include/linux/platform_data/dma-iop32x.h b/include/linux/platform_data/dma-iop32x.h deleted file mode 100644 index ac83cff89549..000000000000 --- a/include/linux/platform_data/dma-iop32x.h +++ /dev/null @@ -1,110 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright © 2006, Intel Corporation. - */ -#ifndef IOP_ADMA_H -#define IOP_ADMA_H -#include <linux/types.h> -#include <linux/dmaengine.h> -#include <linux/interrupt.h> - -#define IOP_ADMA_SLOT_SIZE 32 -#define IOP_ADMA_THRESHOLD 4 -#ifdef DEBUG -#define IOP_PARANOIA 1 -#else -#define IOP_PARANOIA 0 -#endif -#define iop_paranoia(x) BUG_ON(IOP_PARANOIA && (x)) - -#define DMA0_ID 0 -#define DMA1_ID 1 -#define AAU_ID 2 - -/** - * struct iop_adma_device - internal representation of an ADMA device - * @pdev: Platform device - * @id: HW ADMA Device selector - * @dma_desc_pool: base of DMA descriptor region (DMA address) - * @dma_desc_pool_virt: base of DMA descriptor region (CPU address) - * @common: embedded struct dma_device - */ -struct iop_adma_device { - struct platform_device *pdev; - int id; - dma_addr_t dma_desc_pool; - void *dma_desc_pool_virt; - struct dma_device common; -}; - -/** - * struct iop_adma_chan - internal representation of an ADMA device - * @pending: allows batching of hardware operations - * @lock: serializes enqueue/dequeue operations to the slot pool - * @mmr_base: memory mapped register base - * @chain: device chain view of the descriptors - * @device: parent device - * @common: common dmaengine channel object members - * @last_used: place holder for allocation to continue from where it left off - * @all_slots: complete domain of slots usable by the channel - * @slots_allocated: records the actual size of the descriptor slot pool - * @irq_tasklet: bottom half where iop_adma_slot_cleanup runs - */ -struct iop_adma_chan { - int pending; - spinlock_t lock; /* protects the descriptor slot pool */ - void __iomem *mmr_base; - struct list_head chain; - struct iop_adma_device *device; - struct dma_chan common; - struct iop_adma_desc_slot *last_used; - struct list_head all_slots; - int slots_allocated; - struct tasklet_struct irq_tasklet; -}; - -/** - * struct iop_adma_desc_slot - IOP-ADMA software descriptor - * @slot_node: node on the iop_adma_chan.all_slots list - * @chain_node: node on the op_adma_chan.chain list - * @hw_desc: virtual address of the hardware descriptor chain - * @phys: hardware address of the hardware descriptor chain - * @group_head: first operation in a transaction - * @slot_cnt: total slots used in an transaction (group of operations) - * @slots_per_op: number of slots per operation - * @idx: pool index - * @tx_list: list of descriptors that are associated with one operation - * @async_tx: support for the async_tx api - * @group_list: list of slots that make up a multi-descriptor transaction - * for example transfer lengths larger than the supported hw max - * @xor_check_result: result of zero sum - * @crc32_result: result crc calculation - */ -struct iop_adma_desc_slot { - struct list_head slot_node; - struct list_head chain_node; - void *hw_desc; - struct iop_adma_desc_slot *group_head; - u16 slot_cnt; - u16 slots_per_op; - u16 idx; - struct list_head tx_list; - struct dma_async_tx_descriptor async_tx; - union { - u32 *xor_check_result; - u32 *crc32_result; - u32 *pq_check_result; - }; -}; - -struct iop_adma_platform_data { - int hw_id; - dma_cap_mask_t cap_mask; - size_t pool_size; -}; - -#define to_iop_sw_desc(addr_hw_desc) \ - container_of(addr_hw_desc, struct iop_adma_desc_slot, hw_desc) -#define iop_hw_desc_slot_idx(hw_desc, idx) \ - ( (void *) (((unsigned long) hw_desc) + ((idx) << 5)) ) -#endif diff --git a/include/linux/platform_data/dma-mcf-edma.h b/include/linux/platform_data/dma-mcf-edma.h index d718ccfa3421..0b31af66a1ac 100644 --- a/include/linux/platform_data/dma-mcf-edma.h +++ b/include/linux/platform_data/dma-mcf-edma.h @@ -26,8 +26,9 @@ bool mcf_edma_filter_fn(struct dma_chan *chan, void *param); /** * struct mcf_edma_platform_data - platform specific data for eDMA engine * - * @ver The eDMA module version. - * @dma_channels The number of eDMA channels. + * @dma_channels: The number of eDMA channels. + * @slave_map: Slave device map + * @slavecnt: Number of entries in @slave_map */ struct mcf_edma_platform_data { int dma_channels; diff --git a/include/linux/platform_data/dsa.h b/include/linux/platform_data/dsa.h index d4d9bf2060a6..77424bb24723 100644 --- a/include/linux/platform_data/dsa.h +++ b/include/linux/platform_data/dsa.h @@ -3,20 +3,11 @@ #define __DSA_PDATA_H struct device; -struct net_device; -#define DSA_MAX_SWITCHES 4 #define DSA_MAX_PORTS 12 -#define DSA_RTABLE_NONE -1 struct dsa_chip_data { /* - * How to access the switch configuration registers. - */ - struct device *host_dev; - int sw_addr; - - /* * Reference to network devices */ struct device *netdev[DSA_MAX_PORTS]; @@ -24,12 +15,6 @@ struct dsa_chip_data { /* set to size of eeprom if supported by the switch */ int eeprom_len; - /* Device tree node pointer for this specific switch chip - * used during switch setup in case additional properties - * and resources needs to be used - */ - struct device_node *of_node; - /* * The names of the switch's ports. Use "cpu" to * designate the switch port that the cpu is connected to, @@ -38,31 +23,6 @@ struct dsa_chip_data { * or any other string to indicate this is a physical port. */ char *port_names[DSA_MAX_PORTS]; - struct device_node *port_dn[DSA_MAX_PORTS]; - - /* - * An array of which element [a] indicates which port on this - * switch should be used to send packets to that are destined - * for switch a. Can be NULL if there is only one switch chip. - */ - s8 rtable[DSA_MAX_SWITCHES]; }; -struct dsa_platform_data { - /* - * Reference to a Linux network interface that connects - * to the root switch chip of the tree. - */ - struct device *netdev; - struct net_device *of_netdev; - - /* - * Info structs describing each of the switch chips - * connected via this network interface. - */ - int nr_chips; - struct dsa_chip_data *chip; -}; - - #endif /* __DSA_PDATA_H */ diff --git a/include/linux/platform_data/ina2xx.h b/include/linux/platform_data/ina2xx.h deleted file mode 100644 index 2aa5ee9a9050..000000000000 --- a/include/linux/platform_data/ina2xx.h +++ /dev/null @@ -1,16 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Driver for Texas Instruments INA219, INA226 power monitor chips - * - * Copyright (C) 2012 Lothar Felten <lothar.felten@gmail.com> - * - * For further information, see the Documentation/hwmon/ina2xx.rst file. - */ - -/** - * struct ina2xx_platform_data - ina2xx info - * @shunt_uohms shunt resistance in microohms - */ -struct ina2xx_platform_data { - long shunt_uohms; -}; diff --git a/include/linux/platform_data/mdio-gpio.h b/include/linux/platform_data/mdio-gpio.h deleted file mode 100644 index 13874fa6e767..000000000000 --- a/include/linux/platform_data/mdio-gpio.h +++ /dev/null @@ -1,14 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * MDIO-GPIO bus platform data structure - */ - -#ifndef __LINUX_MDIO_GPIO_PDATA_H -#define __LINUX_MDIO_GPIO_PDATA_H - -struct mdio_gpio_platform_data { - u32 phy_mask; - u32 phy_ignore_ta_mask; -}; - -#endif /* __LINUX_MDIO_GPIO_PDATA_H */ diff --git a/include/linux/platform_data/mtd-nand-omap2.h b/include/linux/platform_data/mtd-nand-omap2.h index 8c2f1f185353..2ddb624b97fd 100644 --- a/include/linux/platform_data/mtd-nand-omap2.h +++ b/include/linux/platform_data/mtd-nand-omap2.h @@ -7,7 +7,6 @@ #define _MTD_NAND_OMAP2_H #include <linux/mtd/partitions.h> -#include <linux/mod_devicetable.h> #define GPMC_BCH_NUM_REMAINDER 8 @@ -63,10 +62,4 @@ struct gpmc_nand_regs { void __iomem *gpmc_bch_result6[GPMC_BCH_NUM_REMAINDER]; }; -static const struct of_device_id omap_nand_ids[] = { - { .compatible = "ti,omap2-nand", }, - { .compatible = "ti,am64-nand", }, - {}, -}; - #endif /* _MTD_NAND_OMAP2_H */ diff --git a/include/linux/platform_data/omap-twl4030.h b/include/linux/platform_data/omap-twl4030.h deleted file mode 100644 index 7fcb55fe21c9..000000000000 --- a/include/linux/platform_data/omap-twl4030.h +++ /dev/null @@ -1,42 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/** - * omap-twl4030.h - ASoC machine driver for TI SoC based boards with twl4030 - * codec, header. - * - * Copyright (C) 2012 Texas Instruments Incorporated - https://www.ti.com - * All rights reserved. - * - * Author: Peter Ujfalusi <peter.ujfalusi@ti.com> - */ - -#ifndef _OMAP_TWL4030_H_ -#define _OMAP_TWL4030_H_ - -/* To select if only one channel is connected in a stereo port */ -#define OMAP_TWL4030_LEFT (1 << 0) -#define OMAP_TWL4030_RIGHT (1 << 1) - -struct omap_tw4030_pdata { - const char *card_name; - /* Voice port is connected to McBSP3 */ - bool voice_connected; - - /* The driver will parse the connection flags if this flag is set */ - bool custom_routing; - /* Flags to indicate connected audio ports. */ - u8 has_hs; - u8 has_hf; - u8 has_predriv; - u8 has_carkit; - bool has_ear; - - bool has_mainmic; - bool has_submic; - bool has_hsmic; - bool has_carkitmic; - bool has_digimic0; - bool has_digimic1; - u8 has_linein; -}; - -#endif /* _OMAP_TWL4030_H_ */ diff --git a/include/linux/platform_data/pxa2xx_udc.h b/include/linux/platform_data/pxa2xx_udc.h index bc99cc6a3c5f..c1e4d03bae2c 100644 --- a/include/linux/platform_data/pxa2xx_udc.h +++ b/include/linux/platform_data/pxa2xx_udc.h @@ -10,21 +10,6 @@ #ifndef PXA2XX_UDC_H #define PXA2XX_UDC_H -struct pxa2xx_udc_mach_info { - int (*udc_is_connected)(void); /* do we see host? */ - void (*udc_command)(int cmd); -#define PXA2XX_UDC_CMD_CONNECT 0 /* let host see us */ -#define PXA2XX_UDC_CMD_DISCONNECT 1 /* so host won't see us */ - - /* Boards following the design guidelines in the developer's manual, - * with on-chip GPIOs not Lubbock's weird hardware, can have a sane - * VBUS IRQ and omit the methods above. Store the GPIO number - * here. Note that sometimes the signals go through inverters... - */ - bool gpio_pullup_inverted; - int gpio_pullup; /* high == pullup activated */ -}; - #ifdef CONFIG_PXA27x extern void pxa27x_clear_otgph(void); #else diff --git a/include/linux/platform_data/spi-s3c64xx.h b/include/linux/platform_data/spi-s3c64xx.h index 1d6e6c424fc6..f92bb4a1213b 100644 --- a/include/linux/platform_data/spi-s3c64xx.h +++ b/include/linux/platform_data/spi-s3c64xx.h @@ -30,6 +30,7 @@ struct s3c64xx_spi_csinfo { * @src_clk_nr: Clock source index for the CLK_CFG[SPI_CLKSEL] field. * @num_cs: Number of CS this controller emulates. * @no_cs: Used when CS line is not connected. + * @polling: Using polling mode when %true (no 'dmas' property in devicetree) * @cfg_gpio: Configure pins for this SPI controller. */ struct s3c64xx_spi_info { @@ -41,7 +42,7 @@ struct s3c64xx_spi_info { }; /** - * s3c64xx_spi_set_platdata - SPI Controller configure callback by the board + * s3c64xx_spi0_set_platdata - SPI Controller configure callback by the board * initialization code. * @src_clk_nr: Clock the SPI controller is to use to generate SPI clocks. * @num_cs: Number of elements in the 'cs' array. diff --git a/include/linux/platform_data/tsl2772.h b/include/linux/platform_data/tsl2772.h index f8ade15a35e2..f042e82b39c3 100644 --- a/include/linux/platform_data/tsl2772.h +++ b/include/linux/platform_data/tsl2772.h @@ -61,9 +61,9 @@ struct tsl2772_lux { * @prox_pulse_count: Number if proximity emitter pulses. * @prox_max_samples_cal: The number of samples that are taken when performing * a proximity calibration. - * @prox_diode Which diode(s) to use for driving the external + * @prox_diode: Which diode(s) to use for driving the external * LED(s) for proximity sensing. - * @prox_power The amount of power to use for the external LED(s). + * @prox_power: The amount of power to use for the external LED(s). */ struct tsl2772_settings { int als_time; diff --git a/include/linux/platform_data/usb-xhci-prom21.h b/include/linux/platform_data/usb-xhci-prom21.h new file mode 100644 index 000000000000..ee672ad452a8 --- /dev/null +++ b/include/linux/platform_data/usb-xhci-prom21.h @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * AMD Promontory 21 xHCI auxiliary device platform data. + * + * Copyright (C) 2026 Jihong Min <hurryman2212@gmail.com> + */ + +#ifndef _LINUX_PLATFORM_DATA_USB_XHCI_PROM21_H +#define _LINUX_PLATFORM_DATA_USB_XHCI_PROM21_H + +#include <linux/compiler_types.h> +#include <linux/types.h> + +struct pci_dev; + +struct prom21_xhci_pdata { + struct pci_dev *pdev; + void __iomem *regs; + resource_size_t rsrc_len; +}; + +#endif diff --git a/include/linux/platform_data/voltage-omap.h b/include/linux/platform_data/voltage-omap.h index 6d74e507dbd2..2b48f2b0135d 100644 --- a/include/linux/platform_data/voltage-omap.h +++ b/include/linux/platform_data/voltage-omap.h @@ -10,14 +10,14 @@ /** * struct omap_volt_data - Omap voltage specific data. - * @voltage_nominal: The possible voltage value in uV + * @volt_nominal: The possible voltage value in uV * @sr_efuse_offs: The offset of the efuse register(from system * control module base address) from where to read * the n-target value for the smartreflex module. * @sr_errminlimit: Error min limit value for smartreflex. This value * differs at differnet opp and thus is linked * with voltage. - * @vp_errorgain: Error gain value for the voltage processor. This + * @vp_errgain: Error gain value for the voltage processor. This * field also differs according to the voltage/opp. */ struct omap_volt_data { diff --git a/include/linux/platform_data/wiznet.h b/include/linux/platform_data/wiznet.h deleted file mode 100644 index 1154c4db8a13..000000000000 --- a/include/linux/platform_data/wiznet.h +++ /dev/null @@ -1,23 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* - * Ethernet driver for the WIZnet W5x00 chip. - */ - -#ifndef PLATFORM_DATA_WIZNET_H -#define PLATFORM_DATA_WIZNET_H - -#include <linux/if_ether.h> - -struct wiznet_platform_data { - int link_gpio; - u8 mac_addr[ETH_ALEN]; -}; - -#ifndef CONFIG_WIZNET_BUS_SHIFT -#define CONFIG_WIZNET_BUS_SHIFT 0 -#endif - -#define W5100_BUS_DIRECT_SIZE (0x8000 << CONFIG_WIZNET_BUS_SHIFT) -#define W5300_BUS_DIRECT_SIZE (0x0400 << CONFIG_WIZNET_BUS_SHIFT) - -#endif /* PLATFORM_DATA_WIZNET_H */ diff --git a/include/linux/platform_data/x86/asus-wmi.h b/include/linux/platform_data/x86/asus-wmi.h index 554f41b827e1..b5ed8c83ace1 100644 --- a/include/linux/platform_data/x86/asus-wmi.h +++ b/include/linux/platform_data/x86/asus-wmi.h @@ -147,6 +147,13 @@ #define ASUS_WMI_DEVID_GPU_MUX 0x00090016 #define ASUS_WMI_DEVID_GPU_MUX_VIVO 0x00090026 +/* Keystone dongle insert/remove state. + * PRESENCE_BIT (0x00010000) encodes insert state: + * 0x00010000 = inserted, 0x00000000 = absent. STATUS_BIT is never set. + * 0xFFFFFFFE means no keystone slot on this machine. + */ +#define ASUS_WMI_DEVID_KEYSTONE 0x00120091 + /* TUF laptop RGB modes/colours */ #define ASUS_WMI_DEVID_TUF_RGB_MODE 0x00100056 #define ASUS_WMI_DEVID_TUF_RGB_MODE2 0x0010005A @@ -196,6 +203,7 @@ int asus_wmi_evaluate_method(u32 method_id, u32 arg0, u32 arg1, u32 *retval); int asus_hid_register_listener(struct asus_hid_listener *cdev); void asus_hid_unregister_listener(struct asus_hid_listener *cdev); int asus_hid_event(enum asus_hid_event event); +bool asus_wmi_custom_fan_curve_is_enabled(void); #else static inline void set_ally_mcu_hack(enum asus_ally_mcu_hack status) { @@ -227,6 +235,10 @@ static inline int asus_hid_event(enum asus_hid_event event) { return -ENODEV; } +static inline bool asus_wmi_custom_fan_curve_is_enabled(void) +{ + return false; +} #endif #endif /* __PLATFORM_DATA_X86_ASUS_WMI_H */ diff --git a/include/linux/platform_data/x86/int3472.h b/include/linux/platform_data/x86/int3472.h index dbe745dc88d5..a73841dfae27 100644 --- a/include/linux/platform_data/x86/int3472.h +++ b/include/linux/platform_data/x86/int3472.h @@ -13,6 +13,7 @@ #include <linux/leds.h> #include <linux/regulator/driver.h> #include <linux/regulator/machine.h> +#include <linux/device-id/dmi.h> #include <linux/types.h> /* FIXME drop this once the I2C_DEV_NAME_FORMAT macro has been added to include/linux/i2c.h */ @@ -23,6 +24,7 @@ /* PMIC GPIO Types */ #define INT3472_GPIO_TYPE_RESET 0x00 #define INT3472_GPIO_TYPE_POWERDOWN 0x01 +#define INT3472_GPIO_TYPE_STROBE 0x02 #define INT3472_GPIO_TYPE_POWER_ENABLE 0x0b #define INT3472_GPIO_TYPE_CLK_ENABLE 0x0c #define INT3472_GPIO_TYPE_PRIVACY_LED 0x0d @@ -32,6 +34,7 @@ #define INT3472_PDEV_MAX_NAME_LEN 23 #define INT3472_MAX_SENSOR_GPIOS 3 +#define INT3472_MAX_LEDS 2 #define INT3472_MAX_REGULATORS 3 /* E.g. "dovdd\0" */ @@ -70,7 +73,6 @@ container_of(clk, struct int3472_discrete_device, clock) struct acpi_device; -struct dmi_system_id; struct i2c_client; struct platform_device; @@ -122,16 +124,17 @@ struct int3472_discrete_device { u8 imgclk_index; } clock; - struct int3472_pled { + struct int3472_led { struct led_classdev classdev; struct led_lookup_data lookup; char name[INT3472_LED_MAX_NAME_LEN]; struct gpio_desc *gpio; - } pled; + } leds[INT3472_MAX_LEDS]; struct int3472_discrete_quirks quirks; unsigned int ngpios; /* how many GPIOs have we seen */ + unsigned int n_leds; /* how many LEDs have we registered */ unsigned int n_sensor_gpios; /* how many have we mapped to sensor */ unsigned int n_regulator_gpios; /* how many have we mapped to a regulator */ struct gpiod_lookup_table gpios; @@ -161,7 +164,8 @@ int skl_int3472_register_regulator(struct int3472_discrete_device *int3472, const char *second_sensor); void skl_int3472_unregister_regulator(struct int3472_discrete_device *int3472); -int skl_int3472_register_pled(struct int3472_discrete_device *int3472, struct gpio_desc *gpio); -void skl_int3472_unregister_pled(struct int3472_discrete_device *int3472); +int skl_int3472_register_led(struct int3472_discrete_device *int3472, struct gpio_desc *gpio, + const char *con_id); +void skl_int3472_unregister_leds(struct int3472_discrete_device *int3472); #endif diff --git a/include/linux/platform_data/x86/soc.h b/include/linux/platform_data/x86/soc.h index f981907a5cb0..a6a6b313dfa7 100644 --- a/include/linux/platform_data/x86/soc.h +++ b/include/linux/platform_data/x86/soc.h @@ -12,7 +12,7 @@ #if IS_ENABLED(CONFIG_X86) -#include <linux/mod_devicetable.h> +#include <linux/device-id/x86_cpu.h> #include <asm/cpu_device_id.h> diff --git a/include/linux/platform_device.h b/include/linux/platform_device.h index ed1d50d1c3c1..3d5bbcbae730 100644 --- a/include/linux/platform_device.h +++ b/include/linux/platform_device.h @@ -11,6 +11,7 @@ #define _PLATFORM_DEVICE_H_ #include <linux/device.h> +#include <linux/device-id/platform.h> #define PLATFORM_DEVID_NONE (-1) #define PLATFORM_DEVID_AUTO (-2) @@ -18,7 +19,8 @@ struct irq_affinity; struct mfd_cell; struct property_entry; -struct platform_device_id; +struct device_node; +struct fwnode_handle; struct platform_device { const char *name; @@ -113,22 +115,58 @@ extern int platform_get_irq_byname_optional(struct platform_device *dev, const char *name); extern int platform_add_devices(struct platform_device **, int); +/** + * struct platform_device_info - set of parameters for creating a platform device + * @parent: parent device for the new platform device. + * @fwnode: firmware node associated with the device. + * @of_node_reused: indicates that device tree node associated with the device + * is shared with another device, typically its ancestor. Setting this to + * %true prevents the device from being matched via the OF match table, + * and stops the device core from automatically binding pinctrl + * configuration to avoid disrupting the other device. + * @name: name of the device. + * @id: instance ID of the device. Use %PLATFORM_DEVID_NONE if there is only + * one instance of the device, or %PLATFORM_DEVID_AUTO to let the + * kernel automatically assign a unique instance ID. + * @res: set of resources to attach to the device. + * @num_res: number of entries in @res. + * @data: device-specific data for this platform device. + * @size_data: size of device-specific data. + * @dma_mask: DMA mask for the device. + * @swnode: a secondary software node to be attached to the device. The node + * will be automatically registered and its lifetime tied to the platform + * device if it is not registered yet. + * @properties: a set of software properties for the device. If provided, + * a managed software node will be automatically created and + * assigned to the device. The properties array must be terminated + * with a sentinel entry. Specifying both @properties and @swnode is not + * allowed. + * + * This structure is used to hold information needed to create and register + * a platform device using platform_device_register_full(). + * + * platform_device_register_full() makes deep copies of @name, @res, @data and + * @properties, so the caller does not need to keep them after registration. + * If the registration is performed during initialization, these can be marked + * as __initconst. + */ struct platform_device_info { - struct device *parent; - struct fwnode_handle *fwnode; - bool of_node_reused; + struct device *parent; + struct fwnode_handle *fwnode; + bool of_node_reused; - const char *name; - int id; + const char *name; + int id; - const struct resource *res; - unsigned int num_res; + const struct resource *res; + unsigned int num_res; - const void *data; - size_t size_data; - u64 dma_mask; + const void *data; + size_t size_data; + u64 dma_mask; - const struct property_entry *properties; + const struct software_node *swnode; + const struct property_entry *properties; }; extern struct platform_device *platform_device_register_full( const struct platform_device_info *pdevinfo); @@ -226,6 +264,12 @@ extern int platform_device_add_resources(struct platform_device *pdev, unsigned int num); extern int platform_device_add_data(struct platform_device *pdev, const void *data, size_t size); +void platform_device_set_of_node(struct platform_device *pdev, + struct device_node *np); +void platform_device_set_fwnode(struct platform_device *pdev, + struct fwnode_handle *fwnode); +void platform_device_set_of_node_from_dev(struct platform_device *pdev, + const struct device *dev2); extern int platform_device_add(struct platform_device *pdev); extern void platform_device_del(struct platform_device *pdev); extern void platform_device_put(struct platform_device *pdev); @@ -257,18 +301,19 @@ struct platform_driver { * use a macro to avoid include chaining to get THIS_MODULE */ #define platform_driver_register(drv) \ - __platform_driver_register(drv, THIS_MODULE) + __platform_driver_register(drv, THIS_MODULE, KBUILD_MODNAME) extern int __platform_driver_register(struct platform_driver *, - struct module *); + struct module *, const char *mod_name); extern void platform_driver_unregister(struct platform_driver *); /* non-hotpluggable platform devices may use this so that probe() and * its support may live in __init sections, conserving runtime memory. */ #define platform_driver_probe(drv, probe) \ - __platform_driver_probe(drv, probe, THIS_MODULE) + __platform_driver_probe(drv, probe, THIS_MODULE, KBUILD_MODNAME) extern int __platform_driver_probe(struct platform_driver *driver, - int (*probe)(struct platform_device *), struct module *module); + int (*probe)(struct platform_device *), struct module *module, + const char *mod_name); static inline void *platform_get_drvdata(const struct platform_device *pdev) { @@ -332,19 +377,19 @@ static int __init __platform_driver##_init(void) \ device_initcall(__platform_driver##_init); \ #define platform_create_bundle(driver, probe, res, n_res, data, size) \ - __platform_create_bundle(driver, probe, res, n_res, data, size, THIS_MODULE) + __platform_create_bundle(driver, probe, res, n_res, data, size, THIS_MODULE, KBUILD_MODNAME) extern struct platform_device *__platform_create_bundle( struct platform_driver *driver, int (*probe)(struct platform_device *), struct resource *res, unsigned int n_res, - const void *data, size_t size, struct module *module); + const void *data, size_t size, struct module *module, const char *mod_name); int __platform_register_drivers(struct platform_driver * const *drivers, - unsigned int count, struct module *owner); + unsigned int count, struct module *owner, const char *mod_name); void platform_unregister_drivers(struct platform_driver * const *drivers, unsigned int count); #define platform_register_drivers(drivers, count) \ - __platform_register_drivers(drivers, count, THIS_MODULE) + __platform_register_drivers(drivers, count, THIS_MODULE, KBUILD_MODNAME) #ifdef CONFIG_SUSPEND extern int platform_pm_suspend(struct device *dev); diff --git a/include/linux/pm_domain.h b/include/linux/pm_domain.h index 93ba0143ca47..f925614aebdb 100644 --- a/include/linux/pm_domain.h +++ b/include/linux/pm_domain.h @@ -49,8 +49,8 @@ struct dev_pm_domain_attach_data { const char * const *pd_names; - const u32 num_pd_names; - const u32 pd_flags; + u32 num_pd_names; + u32 pd_flags; }; struct dev_pm_domain_list { @@ -183,6 +183,7 @@ struct genpd_power_state { u64 rejected; u64 above; u64 below; + u64 usage_s2idle; struct fwnode_handle *fwnode; u64 idle_time; void *data; @@ -466,6 +467,10 @@ struct generic_pm_domain *of_genpd_remove_last(struct device_node *np); int of_genpd_parse_idle_states(struct device_node *dn, struct genpd_power_state **states, int *n); void of_genpd_sync_state(struct device_node *np); +int of_genpd_add_child_ids(struct device_node *np, + struct genpd_onecell_data *data); +int of_genpd_remove_child_ids(struct device_node *np, + struct genpd_onecell_data *data); int genpd_dev_pm_attach(struct device *dev); struct device *genpd_dev_pm_attach_by_id(struct device *dev, @@ -535,6 +540,18 @@ struct generic_pm_domain *of_genpd_remove_last(struct device_node *np) { return ERR_PTR(-EOPNOTSUPP); } + +static inline int of_genpd_add_child_ids(struct device_node *np, + struct genpd_onecell_data *data) +{ + return -EOPNOTSUPP; +} + +static inline int of_genpd_remove_child_ids(struct device_node *np, + struct genpd_onecell_data *data) +{ + return -EOPNOTSUPP; +} #endif /* CONFIG_PM_GENERIC_DOMAINS_OF */ #ifdef CONFIG_PM diff --git a/include/linux/pnp.h b/include/linux/pnp.h index 23fe3eaf242d..e0c0d17eb7d8 100644 --- a/include/linux/pnp.h +++ b/include/linux/pnp.h @@ -12,7 +12,7 @@ #include <linux/device.h> #include <linux/list.h> #include <linux/errno.h> -#include <linux/mod_devicetable.h> +#include <linux/device-id/pnp.h> #include <linux/console.h> #define PNP_NAME_LEN 50 diff --git a/include/linux/posix-timers.h b/include/linux/posix-timers.h index 4d3dbcef379e..9a1a0c61361c 100644 --- a/include/linux/posix-timers.h +++ b/include/linux/posix-timers.h @@ -37,7 +37,7 @@ static inline int clockid_to_fd(const clockid_t clk) return ~(clk >> 3); } -static inline bool clockid_aux_valid(clockid_t id) +static inline bool clockid_is_aux_clock(clockid_t id) { return IS_ENABLED(CONFIG_POSIX_AUX_CLOCKS) && id >= CLOCK_AUX && id <= CLOCK_AUX_LAST; } @@ -47,7 +47,7 @@ static inline bool clockid_aux_valid(clockid_t id) #include <linux/signal_types.h> /** - * cpu_timer - Posix CPU timer representation for k_itimer + * struct cpu_timer - Posix CPU timer representation for k_itimer * @node: timerqueue node to queue in the task/sig * @head: timerqueue head on which this timer is queued * @pid: Pointer to target task PID @@ -174,6 +174,7 @@ static inline void posix_cputimers_init_work(void) { } * @it_sigqueue_seq: The sequence count at the point where the signal was queued * @it_sigev_notify: The notify word of sigevent struct for signal delivery * @it_interval: The interval for periodic timers + * @it_pid_type: The type of the PID * @it_signal: Pointer to the creators signal struct * @it_pid: The pid of the process/task targeted by the signal * @it_process: The task to wakeup on clock_nanosleep (CPU timers) diff --git a/include/linux/posix-timers_types.h b/include/linux/posix-timers_types.h index a4712c1008c9..b40cf352e01d 100644 --- a/include/linux/posix-timers_types.h +++ b/include/linux/posix-timers_types.h @@ -34,7 +34,7 @@ #ifdef CONFIG_POSIX_TIMERS /** - * posix_cputimer_base - Container per posix CPU clock + * struct posix_cputimer_base - Container per posix CPU clock * @nextevt: Earliest-expiration cache * @tqhead: timerqueue head for cpu_timers */ @@ -44,7 +44,7 @@ struct posix_cputimer_base { }; /** - * posix_cputimers - Container for posix CPU timer related data + * struct posix_cputimers - Container for posix CPU timer related data * @bases: Base container for posix CPU clocks * @timers_active: Timers are queued. * @expiry_active: Timer expiry is active. Used for @@ -60,7 +60,7 @@ struct posix_cputimers { }; /** - * posix_cputimers_work - Container for task work based posix CPU timer expiry + * struct posix_cputimers_work - Container for task work based posix CPU timer expiry * @work: The task work to be scheduled * @mutex: Mutex held around expiry in context of this task work * @scheduled: @work has been scheduled already, no further processing diff --git a/include/linux/power/jz4740-battery.h b/include/linux/power/jz4740-battery.h deleted file mode 100644 index 10da211678c8..000000000000 --- a/include/linux/power/jz4740-battery.h +++ /dev/null @@ -1,15 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* - * Copyright (C) 2009, Jiejing Zhang <kzjeef@gmail.com> - */ - -#ifndef __JZ4740_BATTERY_H -#define __JZ4740_BATTERY_H - -struct jz_battery_platform_data { - struct power_supply_info info; - int gpio_charge; /* GPIO port of Charger state */ - int gpio_charge_active_low; -}; - -#endif diff --git a/include/linux/power/max17042_battery.h b/include/linux/power/max17042_battery.h index c417abd2ab70..810e068eafd2 100644 --- a/include/linux/power/max17042_battery.h +++ b/include/linux/power/max17042_battery.h @@ -17,12 +17,16 @@ #define MAX17042_DEFAULT_VMAX (4500) /* LiHV cell max */ #define MAX17042_DEFAULT_TEMP_MIN (0) /* For sys without temp sensor */ #define MAX17042_DEFAULT_TEMP_MAX (700) /* 70 degrees Celcius */ +#define MAX17042_DEFAULT_TASK_PERIOD (5760) /* Consider RepCap which is less then 10 units below FullCAP full */ #define MAX17042_FULL_THRESHOLD 10 #define MAX17042_CHARACTERIZATION_DATA_SIZE 48 +#define MAX17055_MODELCFG_REFRESH_BIT BIT(15) +#define MAX17055_MODELCFG_VCHG_BIT BIT(10) + enum max17042_register { MAX17042_STATUS = 0x00, MAX17042_VALRT_Th = 0x01, @@ -105,7 +109,7 @@ enum max17042_register { MAX17042_OCV = 0xEE, - MAX17042_OCVInternal = 0xFB, /* MAX17055 VFOCV */ + MAX17042_OCVInternal = 0xFB, /* MAX17055/77759 VFOCV */ MAX17042_VFSOC = 0xFF, }; @@ -121,6 +125,7 @@ enum max17055_register { MAX17055_ConvgCfg = 0x49, MAX17055_VFRemCap = 0x4A, + MAX17055_SoftWakeup = 0x60, MAX17055_STATUS2 = 0xB0, MAX17055_POWER = 0xB1, @@ -156,7 +161,7 @@ enum max17055_register { MAX17055_AtAvCap = 0xDF, }; -/* Registers specific to max17047/50/55 */ +/* Registers specific to max17047/50/55/77759 */ enum max17047_register { MAX17047_QRTbl00 = 0x12, MAX17047_FullSOCThr = 0x13, @@ -167,26 +172,36 @@ enum max17047_register { MAX17047_QRTbl30 = 0x42, }; +enum max77759_register { + MAX77759_AvgTA0 = 0x26, + MAX77759_AtTTF = 0x33, + MAX77759_Tconvert = 0x34, + MAX77759_AvgCurrent0 = 0x3B, + MAX77759_THMHOT = 0x40, + MAX77759_CTESample = 0x41, + MAX77759_ISys = 0x43, + MAX77759_AvgVCell0 = 0x44, + MAX77759_RlxSOC = 0x47, + MAX77759_AvgISys = 0x4B, + MAX77759_QH0 = 0x4C, + MAX77759_MixAtFull = 0x4F, + MAX77759_VSys = 0xB1, + MAX77759_TAlrtTh2 = 0xB2, + MAX77759_VByp = 0xB3, + MAX77759_IIn = 0xD0, +}; + enum max170xx_chip_type { MAXIM_DEVICE_TYPE_UNKNOWN = 0, MAXIM_DEVICE_TYPE_MAX17042, MAXIM_DEVICE_TYPE_MAX17047, MAXIM_DEVICE_TYPE_MAX17050, MAXIM_DEVICE_TYPE_MAX17055, + MAXIM_DEVICE_TYPE_MAX77759, MAXIM_DEVICE_TYPE_NUM }; -/* - * used for setting a register to a desired value - * addr : address for a register - * data : setting value for the register - */ -struct max17042_reg_data { - u8 addr; - u16 data; -}; - struct max17042_config_data { /* External current sense resistor value in milli-ohms */ u32 cur_sense_val; @@ -208,6 +223,7 @@ struct max17042_config_data { u16 full_soc_thresh; /* 0x13 */ u16 design_cap; /* 0x18 */ u16 ichgt_term; /* 0x1E */ + u16 model_cfg; /* 0xDB */ /* MG3 config */ u16 at_rate; /* 0x04 */ @@ -244,23 +260,4 @@ struct max17042_config_data { u16 cell_char_tbl[MAX17042_CHARACTERIZATION_DATA_SIZE]; } __packed; -struct max17042_platform_data { - struct max17042_reg_data *init_data; - struct max17042_config_data *config_data; - int num_init_data; /* Number of enties in init_data array */ - bool enable_current_sense; - bool enable_por_init; /* Use POR init from Maxim appnote */ - - /* - * R_sns in micro-ohms. - * default 10000 (if r_sns = 0) as it is the recommended value by - * the datasheet although it can be changed by board designers. - */ - unsigned int r_sns; - int vmin; /* in millivolts */ - int vmax; /* in millivolts */ - int temp_min; /* in tenths of degree Celsius */ - int temp_max; /* in tenths of degree Celsius */ -}; - #endif /* __MAX17042_BATTERY_H_ */ diff --git a/include/linux/power_supply.h b/include/linux/power_supply.h index 360ffdf272da..e749d2189335 100644 --- a/include/linux/power_supply.h +++ b/include/linux/power_supply.h @@ -83,6 +83,10 @@ enum { POWER_SUPPLY_TECHNOLOGY_LiFe, POWER_SUPPLY_TECHNOLOGY_NiCd, POWER_SUPPLY_TECHNOLOGY_LiMn, + POWER_SUPPLY_TECHNOLOGY_PbAc, + POWER_SUPPLY_TECHNOLOGY_NiZn, + POWER_SUPPLY_TECHNOLOGY_RAM, + POWER_SUPPLY_TECHNOLOGY_ZnAr, }; enum { @@ -210,6 +214,9 @@ enum power_supply_usb_type { POWER_SUPPLY_USB_TYPE_PD, /* Power Delivery Port */ POWER_SUPPLY_USB_TYPE_PD_DRP, /* PD Dual Role Port */ POWER_SUPPLY_USB_TYPE_PD_PPS, /* PD Programmable Power Supply */ + /* PD Standard Power Range Adjustable Voltage Supply */ + POWER_SUPPLY_USB_TYPE_PD_SPR_AVS, + POWER_SUPPLY_USB_TYPE_PD_PPS_SPR_AVS, /* Supports both PD PPS + SPR AVS */ POWER_SUPPLY_USB_TYPE_APPLE_BRICK_ID, /* Apple Charging Method */ }; @@ -278,6 +285,15 @@ struct power_supply_desc { int (*property_is_writeable)(struct power_supply *psy, enum power_supply_property psp); void (*external_power_changed)(struct power_supply *psy); + /* + * Optional registration-time initialization. This runs in sleepable + * process context after driver data and any battery info are available, + * but before the device is added. The callback must not publish changes or + * start asynchronous activity that can access the power supply before + * registration completes. Return 0 on success or a negative errno on + * failure. + */ + int (*init)(struct power_supply *psy); /* * Set if thermal zone should not be created for this power supply. @@ -343,7 +359,8 @@ struct power_supply { #endif #ifdef CONFIG_LEDS_TRIGGERS - struct led_trigger *trig; + struct led_trigger *charging_or_full_trig; + struct led_trigger *online_trig; struct led_trigger *charging_trig; struct led_trigger *full_trig; struct led_trigger *charging_blink_full_solid_trig; @@ -799,15 +816,33 @@ struct power_supply_battery_info { int bti_resistance_tolerance; }; +#if IS_ENABLED(CONFIG_POWER_SUPPLY) extern int power_supply_reg_notifier(struct notifier_block *nb); extern void power_supply_unreg_notifier(struct notifier_block *nb); -#if IS_ENABLED(CONFIG_POWER_SUPPLY) extern struct power_supply *power_supply_get_by_name(const char *name); +extern int __must_check power_supply_get_system_batteries(struct device *dev, + struct power_supply ***psys); +extern void power_supply_put_system_batteries(struct power_supply **psys, int count); extern void power_supply_put(struct power_supply *psy); #else +static inline int power_supply_reg_notifier(struct notifier_block *nb) +{ return -EOPNOTSUPP; } +static inline void power_supply_unreg_notifier(struct notifier_block *nb) {} static inline void power_supply_put(struct power_supply *psy) {} static inline struct power_supply *power_supply_get_by_name(const char *name) { return NULL; } +static inline int __must_check power_supply_get_system_batteries(struct device *dev, + struct power_supply ***psys) +{ + if (psys) + *psys = NULL; + return 0; +} + +static inline void power_supply_put_system_batteries(struct power_supply **psys, + int count) +{ +} #endif extern struct power_supply *power_supply_get_by_reference(struct fwnode_handle *fwnode, const char *property); diff --git a/include/linux/powercap.h b/include/linux/powercap.h index 3d557bbcd2c7..603419db924c 100644 --- a/include/linux/powercap.h +++ b/include/linux/powercap.h @@ -238,7 +238,7 @@ static inline void *powercap_get_zone_data(struct powercap_zone *power_zone) * Advantage of this parameter is that client can embed * this data in its data structures and allocate in a * single call, preventing multiple allocations. -* @control_type_name: The Name of this control_type, which will be shown +* @name: The Name of this control_type, which will be shown * in the sysfs Interface. * @ops: Callbacks for control type. This parameter is optional. * @@ -277,7 +277,7 @@ int powercap_unregister_control_type(struct powercap_control_type *instance); * @name: A name for this zone. * @parent: A pointer to the parent power zone instance if any or NULL * @ops: Pointer to zone operation callback structure. -* @no_constraints: Number of constraints for this zone +* @nr_constraints: Number of constraints for this zone * @const_ops: Pointer to constraint callback structure * * Register a power zone under a given control type. A power zone must register diff --git a/include/linux/ppp_channel.h b/include/linux/ppp_channel.h index f73fbea0dbc2..2f63e9a6cc88 100644 --- a/include/linux/ppp_channel.h +++ b/include/linux/ppp_channel.h @@ -55,7 +55,7 @@ extern void ppp_input(struct ppp_channel *, struct sk_buff *); /* Called by the channel when an input error occurs, indicating that we may have missed a packet. */ -extern void ppp_input_error(struct ppp_channel *, int code); +extern void ppp_input_error(struct ppp_channel *); /* Attach a channel to a given PPP unit in specified net. */ extern int ppp_register_net_channel(struct net *, struct ppp_channel *); @@ -72,7 +72,9 @@ extern int ppp_channel_index(struct ppp_channel *); /* Get the unit number associated with a channel, or -1 if none */ extern int ppp_unit_number(struct ppp_channel *); -/* Get the device name associated with a channel, or NULL if none */ +/* Get the device name associated with a channel, or NULL if none. + * Caller must hold RCU read lock. + */ extern char *ppp_dev_name(struct ppp_channel *); /* diff --git a/include/linux/ppp_defs.h b/include/linux/ppp_defs.h index b7e57fdbd413..b1d1f46d7d3b 100644 --- a/include/linux/ppp_defs.h +++ b/include/linux/ppp_defs.h @@ -8,6 +8,7 @@ #define _PPP_DEFS_H_ #include <linux/crc-ccitt.h> +#include <linux/skbuff.h> #include <uapi/linux/ppp_defs.h> #define PPP_FCS(fcs, c) crc_ccitt_byte(fcs, c) @@ -25,4 +26,19 @@ static inline bool ppp_proto_is_valid(u16 proto) return !!((proto & 0x0101) == 0x0001); } +/** + * ppp_skb_is_compressed_proto - checks if PPP protocol in a skb is compressed + * @skb: skb to check + * + * Check if the PPP protocol field is compressed (the least significant + * bit of the most significant octet is 1). skb->data must point to the PPP + * protocol header. + * + * Return: Whether the PPP protocol field is compressed. + */ +static inline bool ppp_skb_is_compressed_proto(const struct sk_buff *skb) +{ + return unlikely(skb->data[0] & 0x01); +} + #endif /* _PPP_DEFS_H_ */ diff --git a/include/linux/pps_kernel.h b/include/linux/pps_kernel.h index aab0aebb529e..00b840970d56 100644 --- a/include/linux/pps_kernel.h +++ b/include/linux/pps_kernel.h @@ -60,6 +60,7 @@ struct pps_device { struct device dev; struct fasync_struct *async_queue; /* fasync method */ spinlock_t lock; + bool kc_removed; }; /* @@ -99,12 +100,14 @@ static inline void timespec_to_pps_ktime(struct pps_ktime *kt, static inline void pps_get_ts(struct pps_event_time *ts) { +#ifdef CONFIG_NTP_PPS struct system_time_snapshot snap; - ktime_get_snapshot(&snap); - ts->ts_real = ktime_to_timespec64(snap.real); -#ifdef CONFIG_NTP_PPS - ts->ts_raw = ktime_to_timespec64(snap.raw); + ktime_get_snapshot_id(CLOCK_REALTIME, &snap); + ts->ts_real = ktime_to_timespec64(snap.systime); + ts->ts_raw = ktime_to_timespec64(snap.monoraw); +#else + ktime_get_real_ts64(&ts->ts_real); #endif } diff --git a/include/linux/preempt.h b/include/linux/preempt.h index d964f965c8ff..2e689de7b29a 100644 --- a/include/linux/preempt.h +++ b/include/linux/preempt.h @@ -17,6 +17,9 @@ * * - bits 0-7 are the preemption count (max preemption depth: 256) * - bits 8-15 are the softirq count (max # of softirqs: 256) + * - bits 16-23 are the hardirq disable count (max # of hardirq disable: 256) + * - bits 24-27 are the hardirq count (max # of hardirqs: 16) + * - bit 28 is the NMI flag (no nesting count, tracked separately) * * The hardirq count could in theory be the same as the number of * interrupts in the system, but we run all interrupt handlers with @@ -24,31 +27,56 @@ * there are a few palaeontologic drivers which reenable interrupts in * the handler, so we need more than one bit here. * + * NMI nesting depth is tracked in a separate per-CPU variable + * (nmi_nesting) to save bits in preempt_count. + * * PREEMPT_MASK: 0x000000ff * SOFTIRQ_MASK: 0x0000ff00 - * HARDIRQ_MASK: 0x000f0000 - * NMI_MASK: 0x00f00000 + * HARDIRQ_DISABLE_MASK: 0x00ff0000 + * HARDIRQ_MASK: 0x0f000000 + * + * When HAS_SEPARATE_PREEMPT_RESCHED_BITS=y, PREEMPT_NEED_RESCHED is put in a + * separate word and that allows 64bit load-store architectures to 'set' + * PREEMPT_NEED_RESCHED without messing up the otherwise symmetric + * modifications used on preempt_count and still load the whole thing + * (single-copy) atomically, without having to resort to full atomic + * operations. + * + * Because of the above, NMI_MASK bits are different depending on + * HAS_SEPARATE_PREEMPT_RESCHED_BITS: + * + * - HAS_SEPARATE_PREEMPT_RESCHED_BITS=n: + * + * NMI_MASK: 0x10000000 * PREEMPT_NEED_RESCHED: 0x80000000 + * + * - HAS_SEPARATE_PREEMPT_RESCHED_BITS=y: + * NMI_MASK: 0xf0000000 + * (PREEMPT_NEED_RESCHED is in a different word) */ #define PREEMPT_BITS 8 #define SOFTIRQ_BITS 8 +#define HARDIRQ_DISABLE_BITS 8 #define HARDIRQ_BITS 4 -#define NMI_BITS 4 +#define NMI_BITS (1 + 3*IS_ENABLED(CONFIG_HAS_SEPARATE_PREEMPT_RESCHED_BITS)) #define PREEMPT_SHIFT 0 #define SOFTIRQ_SHIFT (PREEMPT_SHIFT + PREEMPT_BITS) -#define HARDIRQ_SHIFT (SOFTIRQ_SHIFT + SOFTIRQ_BITS) +#define HARDIRQ_DISABLE_SHIFT (SOFTIRQ_SHIFT + SOFTIRQ_BITS) +#define HARDIRQ_SHIFT (HARDIRQ_DISABLE_SHIFT + HARDIRQ_DISABLE_BITS) #define NMI_SHIFT (HARDIRQ_SHIFT + HARDIRQ_BITS) #define __IRQ_MASK(x) ((1UL << (x))-1) #define PREEMPT_MASK (__IRQ_MASK(PREEMPT_BITS) << PREEMPT_SHIFT) #define SOFTIRQ_MASK (__IRQ_MASK(SOFTIRQ_BITS) << SOFTIRQ_SHIFT) +#define HARDIRQ_DISABLE_MASK (__IRQ_MASK(HARDIRQ_DISABLE_BITS) << HARDIRQ_DISABLE_SHIFT) #define HARDIRQ_MASK (__IRQ_MASK(HARDIRQ_BITS) << HARDIRQ_SHIFT) #define NMI_MASK (__IRQ_MASK(NMI_BITS) << NMI_SHIFT) #define PREEMPT_OFFSET (1UL << PREEMPT_SHIFT) #define SOFTIRQ_OFFSET (1UL << SOFTIRQ_SHIFT) +#define HARDIRQ_DISABLE_OFFSET (1UL << HARDIRQ_DISABLE_SHIFT) #define HARDIRQ_OFFSET (1UL << HARDIRQ_SHIFT) #define NMI_OFFSET (1UL << NMI_SHIFT) @@ -105,8 +133,8 @@ static __always_inline unsigned char interrupt_context_level(void) * preempt_count() is commonly implemented with READ_ONCE(). */ -#define nmi_count() (preempt_count() & NMI_MASK) -#define hardirq_count() (preempt_count() & HARDIRQ_MASK) +#define nmi_count() (preempt_count() & NMI_MASK) +#define hardirq_count() (preempt_count() & HARDIRQ_MASK) #ifdef CONFIG_PREEMPT_RT # define softirq_count() (current->softirq_disable_cnt & SOFTIRQ_MASK) # define irq_count() ((preempt_count() & (NMI_MASK | HARDIRQ_MASK)) | softirq_count()) diff --git a/include/linux/printk.h b/include/linux/printk.h index 63d516c873b4..f594c1266bfd 100644 --- a/include/linux/printk.h +++ b/include/linux/printk.h @@ -801,8 +801,22 @@ static inline void print_hex_dump_debug(const char *prefix_str, int prefix_type, } #endif +#if defined(DEBUG) +#define print_hex_dump_devel(prefix_str, prefix_type, rowsize, \ + groupsize, buf, len, ascii) \ + print_hex_dump(KERN_DEBUG, prefix_str, prefix_type, rowsize, \ + groupsize, buf, len, ascii) +#else +static inline void print_hex_dump_devel(const char *prefix_str, int prefix_type, + int rowsize, int groupsize, + const void *buf, size_t len, bool ascii) +{ +} +#endif + /** - * print_hex_dump_bytes - shorthand form of print_hex_dump() with default params + * print_hex_dump_bytes - shorthand form of print_hex_dump_debug() with default + * params * @prefix_str: string to prefix each line with; * caller supplies trailing spaces for alignment if desired * @prefix_type: controls whether prefix of an offset, address, or none @@ -810,7 +824,7 @@ static inline void print_hex_dump_debug(const char *prefix_str, int prefix_type, * @buf: data blob to dump * @len: number of bytes in the @buf * - * Calls print_hex_dump(), with log level of KERN_DEBUG, + * Calls print_hex_dump_debug(), with log level of KERN_DEBUG, * rowsize of 16, groupsize of 1, and ASCII output included. */ #define print_hex_dump_bytes(prefix_str, prefix_type, buf, len) \ diff --git a/include/linux/proc_fs.h b/include/linux/proc_fs.h index 19d1c5e5f335..47d7deaeed8f 100644 --- a/include/linux/proc_fs.h +++ b/include/linux/proc_fs.h @@ -67,6 +67,7 @@ enum proc_pidonly { struct proc_fs_info { struct pid_namespace *pid_ns; kgid_t pid_gid; + const struct cred *mounter_cred; enum proc_hidepid hide_pid; enum proc_pidonly pidonly; struct rcu_head rcu; @@ -248,4 +249,16 @@ static inline struct pid_namespace *proc_pid_ns(struct super_block *sb) bool proc_ns_file(const struct file *file); +#if defined CONFIG_PROC_FS && !defined MODULE +void impl_proc_make_permanent(struct proc_dir_entry *pde); +#endif + +static inline void proc_make_permanent(struct proc_dir_entry *pde) +{ + /* Don't give matches to modules. */ +#if defined CONFIG_PROC_FS && !defined MODULE + impl_proc_make_permanent(pde); +#endif +} + #endif /* _LINUX_PROC_FS_H */ diff --git a/include/linux/property.h b/include/linux/property.h index e30ef23a9af3..907c790a3f01 100644 --- a/include/linux/property.h +++ b/include/linux/property.h @@ -397,11 +397,13 @@ struct property_entry { union { const void *pointer; union { + /* private: internal representation of @value */ u8 u8_data[sizeof(u64) / sizeof(u8)]; u16 u16_data[sizeof(u64) / sizeof(u16)]; u32 u32_data[sizeof(u64) / sizeof(u32)]; u64 u64_data[sizeof(u64) / sizeof(u64)]; const char *str[sizeof(u64) / sizeof(char *)]; + /* public: */ } value; }; }; @@ -471,12 +473,18 @@ struct property_entry { #define PROPERTY_ENTRY_STRING(_name_, _val_) \ __PROPERTY_ENTRY_ELEMENT(_name_, str, STRING, _val_) +#define __PROPERTY_ENTRY_REF_ARGS(_ref_, ...) \ + _Generic(_ref_, \ + const struct software_node_ref_args *: _ref_, \ + struct software_node_ref_args *: _ref_, \ + default: &SOFTWARE_NODE_REFERENCE(_ref_, ##__VA_ARGS__)) + #define PROPERTY_ENTRY_REF(_name_, _ref_, ...) \ (struct property_entry) { \ .name = _name_, \ .length = sizeof(struct software_node_ref_args), \ .type = DEV_PROP_REF, \ - { .pointer = &SOFTWARE_NODE_REFERENCE(_ref_, ##__VA_ARGS__), }, \ + { .pointer = __PROPERTY_ENTRY_REF_ARGS(_ref_, ##__VA_ARGS__) }, \ } #define PROPERTY_ENTRY_BOOL(_name_) \ diff --git a/include/linux/psi.h b/include/linux/psi.h index e0745873e3f2..7966e3ac03b9 100644 --- a/include/linux/psi.h +++ b/include/linux/psi.h @@ -25,7 +25,9 @@ void psi_memstall_leave(unsigned long *flags); int psi_show(struct seq_file *s, struct psi_group *group, enum psi_res res); struct psi_trigger *psi_trigger_create(struct psi_group *group, char *buf, enum psi_res res, struct file *file, - struct kernfs_open_file *of); + struct kernfs_open_file *of, + bool *need_rtpoll_worker); +int psi_trigger_create_rtpoll_worker(struct psi_group *group); void psi_trigger_destroy(struct psi_trigger *t); __poll_t psi_trigger_poll(void **trigger_ptr, struct file *file, diff --git a/include/linux/psp-sev.h b/include/linux/psp-sev.h index 69ffa4b4d1fa..03a79786df1d 100644 --- a/include/linux/psp-sev.h +++ b/include/linux/psp-sev.h @@ -129,6 +129,7 @@ enum sev_cmd { SEV_CMD_SNP_LAUNCH_FINISH = 0x0A2, SEV_CMD_SNP_DBG_DECRYPT = 0x0B0, SEV_CMD_SNP_DBG_ENCRYPT = 0x0B1, + SEV_CMD_SNP_VERIFY_MITIGATION = 0x0B2, SEV_CMD_SNP_PAGE_SWAP_OUT = 0x0C0, SEV_CMD_SNP_PAGE_SWAP_IN = 0x0C1, SEV_CMD_SNP_PAGE_MOVE = 0x0C2, @@ -829,12 +830,14 @@ struct sev_data_range_list { * * @len: length of the command buffer read by the PSP * @iommu_snp_shutdown: Disable enforcement of SNP in the IOMMU + * @x86_snp_shutdown: Disable SNP on all cores * @rsvd1: reserved */ struct sev_data_snp_shutdown_ex { u32 len; u32 iommu_snp_shutdown:1; - u32 rsvd1:31; + u32 x86_snp_shutdown:1; + u32 rsvd1:30; } __packed; /** @@ -891,14 +894,101 @@ struct snp_feature_info { } __packed; /* Feature bits in ECX */ +#define SNP_X86_SHUTDOWN_SUPPORTED BIT(1) #define SNP_RAPL_DISABLE_SUPPORTED BIT(2) #define SNP_CIPHER_TEXT_HIDING_SUPPORTED BIT(3) #define SNP_AES_256_XTS_POLICY_SUPPORTED BIT(4) #define SNP_CXL_ALLOW_POLICY_SUPPORTED BIT(5) +#define SNP_VERIFY_MITIGATION_SUPPORTED BIT(13) /* Feature bits in EBX */ #define SNP_SEV_TIO_SUPPORTED BIT(1) +#define SNP_MIT_SUBCMD_REQ_STATUS 0x0 +#define SNP_MIT_SUBCMD_REQ_VERIFY 0x1 + +/** + * struct sev_data_snp_verify_mitigation - SNP_VERIFY_MITIGATION command params + * + * @length: Length of the command buffer read by the PSP + * @subcommand: Mitigation sub-command for the firmware to execute. + * REQ_STATUS: 0x0 - Request status about currently supported and + * verified mitigations + * REQ_VERIFY: 0x1 - Request to initiate verification mitigation + * operation on a specific mitigation + * @rsvd: Reserved + * @vector: Bit specifying the vulnerability mitigation to process + * @dst_paddr_en: Destination paddr enabled + * @src_paddr_en: Source paddr enabled + * @rsvd1: Reserved + * @rsvd2: Reserved + * @src_paddr: Source address for optional input data + * @dst_paddr: Destination address to write the result + * @rsvd3: Reserved + */ +struct sev_data_snp_verify_mitigation { + u32 length; + u16 subcommand; + u16 rsvd; + u64 vector; + u32 dst_paddr_en : 1, + src_paddr_en : 1, + rsvd1 : 30; + u8 rsvd2[4]; + u64 src_paddr; + u64 dst_paddr; + u8 rsvd3[24]; +} __packed; + +/** + * struct sev_data_snp_verify_mitigation_dst - mitigation result vectors + * + * @mit_verified_vector: Bit vector of vulnerability mitigations verified + * @mit_supported_vector: Bit vector of vulnerability mitigations supported + * @mit_failure_status: Status of the verification operation + */ +struct sev_data_snp_verify_mitigation_dst { + u64 mit_verified_vector; /* OUT */ + u64 mit_supported_vector; /* OUT */ + u32 mit_failure_status; /* OUT */ +} __packed; + +/** + * struct sev_snp_tcb_version_genoa_milan + * + * @boot_loader: SVN of PSP bootloader + * @tee: SVN of PSP operating system + * @reserved: reserved + * @snp: SVN of SNP firmware + * @microcode: Lowest current patch level of all cores + */ +struct sev_snp_tcb_version_genoa_milan { + u8 boot_loader; + u8 tee; + u8 reserved[4]; + u8 snp; + u8 microcode; +}; + +/** + * struct sev_snp_tcb_version_turin + * + * @fmc: SVN of FMC firmware + * @boot_loader: SVN of PSP bootloader + * @tee: SVN of PSP operating system + * @snp: SVN of SNP firmware + * @reserved: reserved + * @microcode: Lowest current patch level of all cores + */ +struct sev_snp_tcb_version_turin { + u8 fmc; + u8 boot_loader; + u8 tee; + u8 snp; + u8 reserved[3]; + u8 microcode; +}; + #ifdef CONFIG_CRYPTO_DEV_SP_PSP /** @@ -1045,6 +1135,7 @@ void snp_free_firmware_page(void *addr); void sev_platform_shutdown(void); bool sev_is_snp_ciphertext_hiding_supported(void); u64 sev_get_snp_policy_bits(void); +int sev_firmware_supported_vm_types(void); #else /* !CONFIG_CRYPTO_DEV_SP_PSP */ diff --git a/include/linux/ptp_clock_kernel.h b/include/linux/ptp_clock_kernel.h index 884364596dd3..36a27a910595 100644 --- a/include/linux/ptp_clock_kernel.h +++ b/include/linux/ptp_clock_kernel.h @@ -12,6 +12,7 @@ #include <linux/pps_kernel.h> #include <linux/ptp_clock.h> #include <linux/timecounter.h> +#include <linux/timekeeping.h> #include <linux/skbuff.h> #define PTP_CLOCK_NAME_LEN 32 @@ -45,13 +46,13 @@ struct system_device_crosststamp; /** * struct ptp_system_timestamp - system time corresponding to a PHC timestamp - * @pre_ts: system timestamp before capturing PHC - * @post_ts: system timestamp after capturing PHC - * @clockid: clock-base used for capturing the system timestamps + * @pre_sts: system time snapshot before capturing PHC + * @post_sts: system time snapshot after capturing PHC + * @clockid: clock-base used for capturing the system timestamps */ struct ptp_system_timestamp { - struct timespec64 pre_ts; - struct timespec64 post_ts; + struct system_time_snapshot pre_sts; + struct system_time_snapshot post_sts; clockid_t clockid; }; @@ -510,13 +511,13 @@ static inline ktime_t ptp_convert_timestamp(const ktime_t *hwtstamp, static inline void ptp_read_system_prets(struct ptp_system_timestamp *sts) { if (sts) - ktime_get_clock_ts64(sts->clockid, &sts->pre_ts); + ktime_get_snapshot_id(sts->clockid, &sts->pre_sts); } static inline void ptp_read_system_postts(struct ptp_system_timestamp *sts) { if (sts) - ktime_get_clock_ts64(sts->clockid, &sts->post_ts); + ktime_get_snapshot_id(sts->clockid, &sts->post_sts); } #endif diff --git a/include/linux/ptr_ring.h b/include/linux/ptr_ring.h index 534531807d95..631c43fde440 100644 --- a/include/linux/ptr_ring.h +++ b/include/linux/ptr_ring.h @@ -48,7 +48,7 @@ struct ptr_ring { */ static inline bool __ptr_ring_full(struct ptr_ring *r) { - return r->queue[r->producer]; + return data_race(r->queue[r->producer]); } static inline bool ptr_ring_full(struct ptr_ring *r) @@ -96,6 +96,26 @@ static inline bool ptr_ring_full_bh(struct ptr_ring *r) return ret; } +/* Report whether the next __ptr_ring_produce() has room for one entry: + * 0 means the single slot at r->queue[r->producer] is free, -ENOSPC means + * the ring is full, which is transient, and -EINVAL means r->size is 0, + * which is permanent. A caller that stops producing and waits for space + * must therefore do so only for -ENOSPC. + * + * Note: callers invoking this in a loop must use a compiler barrier, + * for example cpu_relax(). Callers must hold producer_lock. + */ +static inline int __ptr_ring_check_produce(struct ptr_ring *r) +{ + if (unlikely(!r->size)) + return -EINVAL; + + if (data_race(r->queue[r->producer])) + return -ENOSPC; + + return 0; +} + /* Note: callers invoking this in a loop must use a compiler barrier, * for example cpu_relax(). Callers must hold producer_lock. * Callers are responsible for making sure pointer that is being queued @@ -103,8 +123,10 @@ static inline bool ptr_ring_full_bh(struct ptr_ring *r) */ static inline int __ptr_ring_produce(struct ptr_ring *r, void *ptr) { - if (unlikely(!r->size) || r->queue[r->producer]) - return -ENOSPC; + int ret = __ptr_ring_check_produce(r); + + if (ret) + return ret; /* Make sure the pointer we are storing points to a valid data. */ /* Pairs with the dependency ordering in __ptr_ring_consume. */ @@ -194,7 +216,7 @@ static inline void *__ptr_ring_peek(struct ptr_ring *r) static inline bool __ptr_ring_empty(struct ptr_ring *r) { if (likely(r->size)) - return !r->queue[READ_ONCE(r->consumer_head)]; + return !data_race(r->queue[READ_ONCE(r->consumer_head)]); return true; } @@ -256,7 +278,7 @@ static inline void __ptr_ring_zero_tail(struct ptr_ring *r, int consumer_head) * besides the first one until we write out all entries. */ while (likely(head > r->consumer_tail)) - r->queue[--head] = NULL; + data_race(r->queue[--head] = NULL); r->consumer_tail = consumer_head; } diff --git a/include/linux/ptrace.h b/include/linux/ptrace.h index 90507d4afcd6..2839d5c8eaa0 100644 --- a/include/linux/ptrace.h +++ b/include/linux/ptrace.h @@ -17,6 +17,7 @@ struct syscall_info { struct seccomp_data data; }; +bool ptracer_access_allowed(struct task_struct *tsk); extern int ptrace_access_vm(struct task_struct *tsk, unsigned long addr, void *buf, int len, unsigned int gup_flags); @@ -404,13 +405,13 @@ extern void sigaction_compat_abi(struct k_sigaction *act, struct k_sigaction *oa /* * ptrace report for syscall entry and exit looks identical. */ -static inline int ptrace_report_syscall(unsigned long message) +static inline bool ptrace_report_syscall(unsigned long message) { int ptrace = current->ptrace; int signr; if (!(ptrace & PT_PTRACED)) - return 0; + return true; signr = ptrace_notify(SIGTRAP | ((ptrace & PT_TRACESYSGOOD) ? 0x80 : 0), message); @@ -423,11 +424,11 @@ static inline int ptrace_report_syscall(unsigned long message) if (signr) send_sig(signr, current, 1); - return fatal_signal_pending(current); + return !fatal_signal_pending(current); } /** - * ptrace_report_syscall_entry - task is about to attempt a system call + * ptrace_report_syscall_permit_entry - task is about to attempt a system call * @regs: user register state of current task * * This will be called if %SYSCALL_WORK_SYSCALL_TRACE or @@ -437,7 +438,7 @@ static inline int ptrace_report_syscall(unsigned long message) * call number and arguments to be tried. It is safe to block here, * preventing the system call from beginning. * - * Returns zero normally, or nonzero if the calling arch code should abort + * Returns True normally, or False if the calling architecture code should abort * the system call. That must prevent normal entry so no system call is * made. If @task ever returns to user mode after this, its register state * is unspecified, but should be something harmless like an %ENOSYS error @@ -446,8 +447,7 @@ static inline int ptrace_report_syscall(unsigned long message) * * Called without locks, just after entering kernel mode. */ -static inline __must_check int ptrace_report_syscall_entry( - struct pt_regs *regs) +static inline __must_check bool ptrace_report_syscall_permit_entry(struct pt_regs *regs) { return ptrace_report_syscall(PTRACE_EVENTMSG_SYSCALL_ENTRY); } diff --git a/include/linux/pwrseq/consumer.h b/include/linux/pwrseq/consumer.h index 7d583b4f266e..16fad5f3e3ab 100644 --- a/include/linux/pwrseq/consumer.h +++ b/include/linux/pwrseq/consumer.h @@ -20,8 +20,10 @@ void pwrseq_put(struct pwrseq_desc *desc); struct pwrseq_desc * __must_check devm_pwrseq_get(struct device *dev, const char *target); -int pwrseq_power_on(struct pwrseq_desc *desc); -int pwrseq_power_off(struct pwrseq_desc *desc); +int pwrseq_enable(struct pwrseq_desc *desc); +int pwrseq_disable(struct pwrseq_desc *desc); + +struct device *pwrseq_to_device(struct pwrseq_desc *desc); #else /* CONFIG_POWER_SEQUENCING */ @@ -41,16 +43,21 @@ devm_pwrseq_get(struct device *dev, const char *target) return ERR_PTR(-ENOSYS); } -static inline int pwrseq_power_on(struct pwrseq_desc *desc) +static inline int pwrseq_enable(struct pwrseq_desc *desc) { return -ENOSYS; } -static inline int pwrseq_power_off(struct pwrseq_desc *desc) +static inline int pwrseq_disable(struct pwrseq_desc *desc) { return -ENOSYS; } +static inline struct device *pwrseq_to_device(struct pwrseq_desc *desc) +{ + return NULL; +} + #endif /* CONFIG_POWER_SEQUENCING */ #endif /* __POWER_SEQUENCING_CONSUMER_H__ */ diff --git a/include/linux/quotaops.h b/include/linux/quotaops.h index c334f82ed385..f9c0f9d7c9d9 100644 --- a/include/linux/quotaops.h +++ b/include/linux/quotaops.h @@ -44,14 +44,7 @@ int dquot_initialize(struct inode *inode); bool dquot_initialize_needed(struct inode *inode); void dquot_drop(struct inode *inode); struct dquot *dqget(struct super_block *sb, struct kqid qid); -static inline struct dquot *dqgrab(struct dquot *dquot) -{ - /* Make sure someone else has active reference to dquot */ - WARN_ON_ONCE(!atomic_read(&dquot->dq_count)); - WARN_ON_ONCE(!test_bit(DQ_ACTIVE_B, &dquot->dq_flags)); - atomic_inc(&dquot->dq_count); - return dquot; -} +struct dquot *dqgrab(struct dquot *dquot); static inline bool dquot_is_busy(struct dquot *dquot) { diff --git a/include/linux/radix-tree.h b/include/linux/radix-tree.h index eae67015ce51..057edc4cbb6e 100644 --- a/include/linux/radix-tree.h +++ b/include/linux/radix-tree.h @@ -111,7 +111,7 @@ struct radix_tree_iter { }; /** - * Radix-tree synchronization + * DOC: Radix-tree synchronization * * The radix-tree API requires that users provide all synchronisation (with * specific exceptions, noted below). @@ -182,6 +182,7 @@ static inline void *radix_tree_deref_slot(void __rcu **slot) /** * radix_tree_deref_slot_protected - dereference a slot with tree lock held * @slot: slot pointer, returned by radix_tree_lookup_slot + * @treelock: caller must hold this spinlock * * Similar to radix_tree_deref_slot. The caller does not hold the RCU read * lock but it must hold the tree lock to prevent parallel updates. @@ -306,7 +307,7 @@ radix_tree_iter_init(struct radix_tree_iter *iter, unsigned long start) * Also it fills @iter with data about chunk: position in the tree (index), * its end (next_index), and constructs a bit mask for tagged iterating (tags). */ -void __rcu **radix_tree_next_chunk(const struct radix_tree_root *, +void __rcu **radix_tree_next_chunk(const struct radix_tree_root *root, struct radix_tree_iter *iter, unsigned flags); /** diff --git a/include/linux/raid/pq.h b/include/linux/raid/pq.h index 2467b3be15c9..b3bf9339cd86 100644 --- a/include/linux/raid/pq.h +++ b/include/linux/raid/pq.h @@ -1,206 +1,24 @@ /* SPDX-License-Identifier: GPL-2.0-or-later */ -/* -*- linux-c -*- ------------------------------------------------------- * +/* + * Copyright 2003 H. Peter Anvin - All Rights Reserved * - * Copyright 2003 H. Peter Anvin - All Rights Reserved - * - * ----------------------------------------------------------------------- */ - -#ifndef LINUX_RAID_RAID6_H -#define LINUX_RAID_RAID6_H - -#ifdef __KERNEL__ - -#include <linux/blkdev.h> -#include <linux/mm.h> - -/* This should be const but the raid6 code is too convoluted for that. */ -static inline void *raid6_get_zero_page(void) -{ - return page_address(ZERO_PAGE(0)); -} - -#else /* ! __KERNEL__ */ -/* Used for testing in user space */ - -#include <errno.h> -#include <inttypes.h> -#include <stddef.h> -#include <string.h> -#include <sys/mman.h> -#include <sys/time.h> -#include <sys/types.h> - -/* Not standard, but glibc defines it */ -#define BITS_PER_LONG __WORDSIZE - -typedef uint8_t u8; -typedef uint16_t u16; -typedef uint32_t u32; -typedef uint64_t u64; - -#ifndef PAGE_SIZE -# define PAGE_SIZE 4096 -#endif -#ifndef PAGE_SHIFT -# define PAGE_SHIFT 12 -#endif -extern const char raid6_empty_zero_page[PAGE_SIZE]; - -#define __init -#define __exit -#ifndef __attribute_const__ -# define __attribute_const__ __attribute__((const)) -#endif -#define noinline __attribute__((noinline)) - -#define preempt_enable() -#define preempt_disable() -#define cpu_has_feature(x) 1 -#define enable_kernel_altivec() -#define disable_kernel_altivec() - -#undef EXPORT_SYMBOL -#define EXPORT_SYMBOL(sym) -#undef EXPORT_SYMBOL_GPL -#define EXPORT_SYMBOL_GPL(sym) -#define MODULE_LICENSE(licence) -#define MODULE_DESCRIPTION(desc) -#define subsys_initcall(x) -#define module_exit(x) - -#define IS_ENABLED(x) (x) -#define CONFIG_RAID6_PQ_BENCHMARK 1 -#endif /* __KERNEL__ */ - -/* Routine choices */ -struct raid6_calls { - void (*gen_syndrome)(int, size_t, void **); - void (*xor_syndrome)(int, int, int, size_t, void **); - int (*valid)(void); /* Returns 1 if this routine set is usable */ - const char *name; /* Name of this routine set */ - int priority; /* Relative priority ranking if non-zero */ -}; - -/* Selected algorithm */ -extern struct raid6_calls raid6_call; - -/* Various routine sets */ -extern const struct raid6_calls raid6_intx1; -extern const struct raid6_calls raid6_intx2; -extern const struct raid6_calls raid6_intx4; -extern const struct raid6_calls raid6_intx8; -extern const struct raid6_calls raid6_mmxx1; -extern const struct raid6_calls raid6_mmxx2; -extern const struct raid6_calls raid6_sse1x1; -extern const struct raid6_calls raid6_sse1x2; -extern const struct raid6_calls raid6_sse2x1; -extern const struct raid6_calls raid6_sse2x2; -extern const struct raid6_calls raid6_sse2x4; -extern const struct raid6_calls raid6_altivec1; -extern const struct raid6_calls raid6_altivec2; -extern const struct raid6_calls raid6_altivec4; -extern const struct raid6_calls raid6_altivec8; -extern const struct raid6_calls raid6_avx2x1; -extern const struct raid6_calls raid6_avx2x2; -extern const struct raid6_calls raid6_avx2x4; -extern const struct raid6_calls raid6_avx512x1; -extern const struct raid6_calls raid6_avx512x2; -extern const struct raid6_calls raid6_avx512x4; -extern const struct raid6_calls raid6_s390vx8; -extern const struct raid6_calls raid6_vpermxor1; -extern const struct raid6_calls raid6_vpermxor2; -extern const struct raid6_calls raid6_vpermxor4; -extern const struct raid6_calls raid6_vpermxor8; -extern const struct raid6_calls raid6_lsx; -extern const struct raid6_calls raid6_lasx; -extern const struct raid6_calls raid6_rvvx1; -extern const struct raid6_calls raid6_rvvx2; -extern const struct raid6_calls raid6_rvvx4; -extern const struct raid6_calls raid6_rvvx8; - -struct raid6_recov_calls { - void (*data2)(int, size_t, int, int, void **); - void (*datap)(int, size_t, int, void **); - int (*valid)(void); - const char *name; - int priority; -}; - -extern const struct raid6_recov_calls raid6_recov_intx1; -extern const struct raid6_recov_calls raid6_recov_ssse3; -extern const struct raid6_recov_calls raid6_recov_avx2; -extern const struct raid6_recov_calls raid6_recov_avx512; -extern const struct raid6_recov_calls raid6_recov_s390xc; -extern const struct raid6_recov_calls raid6_recov_neon; -extern const struct raid6_recov_calls raid6_recov_lsx; -extern const struct raid6_recov_calls raid6_recov_lasx; -extern const struct raid6_recov_calls raid6_recov_rvv; - -extern const struct raid6_calls raid6_neonx1; -extern const struct raid6_calls raid6_neonx2; -extern const struct raid6_calls raid6_neonx4; -extern const struct raid6_calls raid6_neonx8; - -/* Algorithm list */ -extern const struct raid6_calls * const raid6_algos[]; -extern const struct raid6_recov_calls *const raid6_recov_algos[]; -int raid6_select_algo(void); - -/* Return values from chk_syndrome */ -#define RAID6_OK 0 -#define RAID6_P_BAD 1 -#define RAID6_Q_BAD 2 -#define RAID6_PQ_BAD 3 - -/* Galois field tables */ -extern const u8 raid6_gfmul[256][256] __attribute__((aligned(256))); -extern const u8 raid6_vgfmul[256][32] __attribute__((aligned(256))); -extern const u8 raid6_gfexp[256] __attribute__((aligned(256))); -extern const u8 raid6_gflog[256] __attribute__((aligned(256))); -extern const u8 raid6_gfinv[256] __attribute__((aligned(256))); -extern const u8 raid6_gfexi[256] __attribute__((aligned(256))); - -/* Recovery routines */ -extern void (*raid6_2data_recov)(int disks, size_t bytes, int faila, int failb, - void **ptrs); -extern void (*raid6_datap_recov)(int disks, size_t bytes, int faila, - void **ptrs); -void raid6_dual_recov(int disks, size_t bytes, int faila, int failb, - void **ptrs); - -/* Some definitions to allow code to be compiled for testing in userspace */ -#ifndef __KERNEL__ - -# define jiffies raid6_jiffies() -# define printk printf -# define pr_err(format, ...) fprintf(stderr, format, ## __VA_ARGS__) -# define pr_info(format, ...) fprintf(stdout, format, ## __VA_ARGS__) -# define GFP_KERNEL 0 -# define __get_free_pages(x, y) ((unsigned long)mmap(NULL, PAGE_SIZE << (y), \ - PROT_READ|PROT_WRITE, \ - MAP_PRIVATE|MAP_ANONYMOUS,\ - 0, 0)) -# define free_pages(x, y) munmap((void *)(x), PAGE_SIZE << (y)) + * Public interface to the RAID6 P/Q calculation and recovery library. + */ +#ifndef LINUX_RAID_PQ_H +#define LINUX_RAID_PQ_H -static inline void cpu_relax(void) -{ - /* Nothing */ -} +#include <linux/types.h> -#undef HZ -#define HZ 1000 -static inline uint32_t raid6_jiffies(void) -{ - struct timeval tv; - gettimeofday(&tv, NULL); - return tv.tv_sec*1000 + tv.tv_usec/1000; -} +#define RAID6_MIN_DISKS 4 -static inline void *raid6_get_zero_page(void) -{ - return raid6_empty_zero_page; -} +void raid6_gen_syndrome(int disks, size_t bytes, void **ptrs); +void raid6_xor_syndrome(int disks, int start, int stop, size_t bytes, + void **ptrs); +bool raid6_can_xor_syndrome(void); -#endif /* ! __KERNEL__ */ +void raid6_recov_2data(int disks, size_t bytes, int faila, int failb, + void **ptrs); +void raid6_recov_datap(int disks, size_t bytes, int faila, + void **ptrs); -#endif /* LINUX_RAID_RAID6_H */ +#endif /* LINUX_RAID_PQ_H */ diff --git a/include/linux/raid/pq_tables.h b/include/linux/raid/pq_tables.h new file mode 100644 index 000000000000..7b1ebe675677 --- /dev/null +++ b/include/linux/raid/pq_tables.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright 2003 H. Peter Anvin - All Rights Reserved + * + * Galois field tables for the Linux RAID6 P/Q parity algorithm. + */ +#ifndef _LINUX_RAID_PQ_TABLES_H +#define _LINUX_RAID_PQ_TABLES_H + +#include <linux/types.h> + +extern const u8 raid6_gfmul[256][256] __attribute__((aligned(256))); +extern const u8 raid6_vgfmul[256][32] __attribute__((aligned(256))); +extern const u8 raid6_gfexp[256] __attribute__((aligned(256))); +extern const u8 raid6_gflog[256] __attribute__((aligned(256))); +extern const u8 raid6_gfinv[256] __attribute__((aligned(256))); +extern const u8 raid6_gfexi[256] __attribute__((aligned(256))); + +#endif /* _LINUX_RAID_PQ_TABLES_H */ diff --git a/include/linux/raid/xor.h b/include/linux/raid/xor.h index 51b811b62322..870558c9d36e 100644 --- a/include/linux/raid/xor.h +++ b/include/linux/raid/xor.h @@ -2,29 +2,6 @@ #ifndef _XOR_H #define _XOR_H -#define MAX_XOR_BLOCKS 4 +void xor_gen(void *dest, void **srcs, unsigned int src_cnt, unsigned int bytes); -extern void xor_blocks(unsigned int count, unsigned int bytes, - void *dest, void **srcs); - -struct xor_block_template { - struct xor_block_template *next; - const char *name; - int speed; - void (*do_2)(unsigned long, unsigned long * __restrict, - const unsigned long * __restrict); - void (*do_3)(unsigned long, unsigned long * __restrict, - const unsigned long * __restrict, - const unsigned long * __restrict); - void (*do_4)(unsigned long, unsigned long * __restrict, - const unsigned long * __restrict, - const unsigned long * __restrict, - const unsigned long * __restrict); - void (*do_5)(unsigned long, unsigned long * __restrict, - const unsigned long * __restrict, - const unsigned long * __restrict, - const unsigned long * __restrict, - const unsigned long * __restrict); -}; - -#endif +#endif /* _XOR_H */ diff --git a/include/linux/randomize_kstack.h b/include/linux/randomize_kstack.h index 1d982dbdd0d0..d8b939f50bba 100644 --- a/include/linux/randomize_kstack.h +++ b/include/linux/randomize_kstack.h @@ -6,10 +6,10 @@ #include <linux/kernel.h> #include <linux/jump_label.h> #include <linux/percpu-defs.h> +#include <linux/prandom.h> DECLARE_STATIC_KEY_MAYBE(CONFIG_RANDOMIZE_KSTACK_OFFSET_DEFAULT, randomize_kstack_offset); -DECLARE_PER_CPU(u32, kstack_offset); /* * Do not use this anywhere else in the kernel. This is used here because @@ -46,19 +46,31 @@ DECLARE_PER_CPU(u32, kstack_offset); #define KSTACK_OFFSET_MAX(x) ((x) & 0b1111111100) #endif +DECLARE_PER_CPU(struct rnd_state, kstack_rnd_state); + +static __always_inline u32 get_kstack_offset(void) +{ + struct rnd_state *state; + u32 rnd; + + state = &get_cpu_var(kstack_rnd_state); + rnd = prandom_u32_state(state); + put_cpu_var(kstack_rnd_state); + + return rnd; +} + /** - * add_random_kstack_offset - Increase stack utilization by previously - * chosen random offset + * add_random_kstack_offset - Increase stack utilization by a random offset. * - * This should be used in the syscall entry path when interrupts and - * preempt are disabled, and after user registers have been stored to - * the stack. For testing the resulting entropy, please see: - * tools/testing/selftests/lkdtm/stack-entropy.sh + * This should be used in the syscall entry path after user registers have been + * stored to the stack. Preemption may be enabled. For testing the resulting + * entropy, please see: tools/testing/selftests/lkdtm/stack-entropy.sh */ #define add_random_kstack_offset() do { \ if (static_branch_maybe(CONFIG_RANDOMIZE_KSTACK_OFFSET_DEFAULT, \ &randomize_kstack_offset)) { \ - u32 offset = raw_cpu_read(kstack_offset); \ + u32 offset = get_kstack_offset(); \ u8 *ptr = __kstack_alloca(KSTACK_OFFSET_MAX(offset)); \ /* Keep allocation even after "ptr" loses scope. */ \ asm volatile("" :: "r"(ptr) : "memory"); \ @@ -66,33 +78,26 @@ DECLARE_PER_CPU(u32, kstack_offset); } while (0) /** - * choose_random_kstack_offset - Choose the random offset for the next - * add_random_kstack_offset() + * add_random_kstack_offset_irqsoff - Increase stack utilization by a random offset. * - * This should only be used during syscall exit when interrupts and - * preempt are disabled. This position in the syscall flow is done to - * frustrate attacks from userspace attempting to learn the next offset: - * - Maximize the timing uncertainty visible from userspace: if the - * offset is chosen at syscall entry, userspace has much more control - * over the timing between choosing offsets. "How long will we be in - * kernel mode?" tends to be more difficult to predict than "how long - * will we be in user mode?" - * - Reduce the lifetime of the new offset sitting in memory during - * kernel mode execution. Exposure of "thread-local" memory content - * (e.g. current, percpu, etc) tends to be easier than arbitrary - * location memory exposure. + * This should be used in the syscall entry path after user registers have been + * stored to the stack. Interrupts must be still disabled. */ -#define choose_random_kstack_offset(rand) do { \ - if (static_branch_maybe(CONFIG_RANDOMIZE_KSTACK_OFFSET_DEFAULT, \ - &randomize_kstack_offset)) { \ - u32 offset = raw_cpu_read(kstack_offset); \ - offset = ror32(offset, 5) ^ (rand); \ - raw_cpu_write(kstack_offset, offset); \ - } \ +#define add_random_kstack_offset_irqsoff() \ +do { \ + lockdep_assert_irqs_disabled(); \ + if (static_branch_maybe(CONFIG_RANDOMIZE_KSTACK_OFFSET_DEFAULT, \ + &randomize_kstack_offset)) { \ + u32 offset = prandom_u32_state(raw_cpu_ptr(&kstack_rnd_state)); \ + u8 *ptr = __kstack_alloca(KSTACK_OFFSET_MAX(offset)); \ + /* Keep allocation even after "ptr" loses scope. */ \ + asm volatile("" :: "r"(ptr) : "memory"); \ + } \ } while (0) + #else /* CONFIG_RANDOMIZE_KSTACK_OFFSET */ #define add_random_kstack_offset() do { } while (0) -#define choose_random_kstack_offset(rand) do { } while (0) +#define add_random_kstack_offset_irqsoff() do { } while (0) #endif /* CONFIG_RANDOMIZE_KSTACK_OFFSET */ #endif diff --git a/include/linux/raspberrypi/vchiq_bus.h b/include/linux/raspberrypi/vchiq_bus.h index 9de179b39f85..e52291a3b247 100644 --- a/include/linux/raspberrypi/vchiq_bus.h +++ b/include/linux/raspberrypi/vchiq_bus.h @@ -7,7 +7,7 @@ #define _VCHIQ_DEVICE_H #include <linux/device.h> -#include <linux/mod_devicetable.h> +#include <linux/device-id/vchiq.h> struct vchiq_drv_mgmt; diff --git a/include/linux/rbtree.h b/include/linux/rbtree.h index 4091e978aef2..b7078b0e1eb2 100644 --- a/include/linux/rbtree.h +++ b/include/linux/rbtree.h @@ -35,10 +35,15 @@ #define RB_CLEAR_NODE(node) \ ((node)->__rb_parent_color = (unsigned long)(node)) +#define RB_EMPTY_LINKED_NODE(lnode) RB_EMPTY_NODE(&(lnode)->node) +#define RB_CLEAR_LINKED_NODE(lnode) ({ \ + RB_CLEAR_NODE(&(lnode)->node); \ + (lnode)->prev = (lnode)->next = NULL; \ +}) extern void rb_insert_color(struct rb_node *, struct rb_root *); extern void rb_erase(struct rb_node *, struct rb_root *); - +extern bool rb_erase_linked(struct rb_node_linked *, struct rb_root_linked *); /* Find logical next and previous nodes in a tree */ extern struct rb_node *rb_next(const struct rb_node *); @@ -213,15 +218,10 @@ rb_add_cached(struct rb_node *node, struct rb_root_cached *tree, return leftmost ? node : NULL; } -/** - * rb_add() - insert @node into @tree - * @node: node to insert - * @tree: tree to insert @node into - * @less: operator defining the (partial) node order - */ static __always_inline void -rb_add(struct rb_node *node, struct rb_root *tree, - bool (*less)(struct rb_node *, const struct rb_node *)) +__rb_add(struct rb_node *node, struct rb_root *tree, + bool (*less)(struct rb_node *, const struct rb_node *), + void (*linkop)(struct rb_node *, struct rb_node *, struct rb_node **)) { struct rb_node **link = &tree->rb_node; struct rb_node *parent = NULL; @@ -234,10 +234,73 @@ rb_add(struct rb_node *node, struct rb_root *tree, link = &parent->rb_right; } + linkop(node, parent, link); rb_link_node(node, parent, link); rb_insert_color(node, tree); } +#define __node_2_linked_node(_n) \ + rb_entry((_n), struct rb_node_linked, node) + +static inline void +rb_link_linked_node(struct rb_node *node, struct rb_node *parent, struct rb_node **link) +{ + if (!parent) + return; + + struct rb_node_linked *nnew = __node_2_linked_node(node); + struct rb_node_linked *npar = __node_2_linked_node(parent); + + if (link == &parent->rb_left) { + nnew->prev = npar->prev; + nnew->next = npar; + npar->prev = nnew; + if (nnew->prev) + nnew->prev->next = nnew; + } else { + nnew->next = npar->next; + nnew->prev = npar; + npar->next = nnew; + if (nnew->next) + nnew->next->prev = nnew; + } +} + +/** + * rb_add_linked() - insert @node into the leftmost linked tree @tree + * @node: node to insert + * @tree: linked tree to insert @node into + * @less: operator defining the (partial) node order + * + * Returns @true when @node is the new leftmost, @false otherwise. + */ +static __always_inline bool +rb_add_linked(struct rb_node_linked *node, struct rb_root_linked *tree, + bool (*less)(struct rb_node *, const struct rb_node *)) +{ + __rb_add(&node->node, &tree->rb_root, less, rb_link_linked_node); + if (!node->prev) + tree->rb_leftmost = node; + return !node->prev; +} + +/* Empty linkop function which is optimized away by the compiler */ +static __always_inline void +rb_link_noop(struct rb_node *n, struct rb_node *p, struct rb_node **l) { } + +/** + * rb_add() - insert @node into @tree + * @node: node to insert + * @tree: tree to insert @node into + * @less: operator defining the (partial) node order + */ +static __always_inline void +rb_add(struct rb_node *node, struct rb_root *tree, + bool (*less)(struct rb_node *, const struct rb_node *)) +{ + __rb_add(node, tree, less, rb_link_noop); +} + /** * rb_find_add_cached() - find equivalent @node in @tree, or add @node * @node: node to look-for / insert @@ -437,7 +500,7 @@ rb_find_first(const void *key, const struct rb_root *tree, /** * rb_next_match() - find the next @key in @tree * @key: key to match - * @tree: tree to search + * @node: tree to search * @cmp: operator defining node order * * Returns the next node matching @key, or NULL. diff --git a/include/linux/rbtree_augmented.h b/include/linux/rbtree_augmented.h index 6dbc5a1bf6a8..d2fa1c41bfd2 100644 --- a/include/linux/rbtree_augmented.h +++ b/include/linux/rbtree_augmented.h @@ -87,18 +87,18 @@ rb_add_augmented_cached(struct rb_node *node, struct rb_root_cached *tree, } /* - * Template for declaring augmented rbtree callbacks (generic case) + * Template for declaring augmented rbtree callbacks (generic multi fields) * * RBSTATIC: 'static' or empty * RBNAME: name of the rb_augment_callbacks structure * RBSTRUCT: struct type of the tree nodes * RBFIELD: name of struct rb_node field within RBSTRUCT - * RBAUGMENTED: name of field within RBSTRUCT holding data for subtree - * RBCOMPUTE: name of function that recomputes the RBAUGMENTED data + * RBCOPY: name of function that copies the RBAUGMENTED datas + * RBCOMPUTE: name of function that recomputes the RBAUGMENTED datas */ -#define RB_DECLARE_CALLBACKS(RBSTATIC, RBNAME, \ - RBSTRUCT, RBFIELD, RBAUGMENTED, RBCOMPUTE) \ +#define RB_DECLARE_CALLBACKS_MULTI(RBSTATIC, RBNAME, \ + RBSTRUCT, RBFIELD, RBCOPY, RBCOMPUTE) \ static inline void \ RBNAME ## _propagate(struct rb_node *rb, struct rb_node *stop) \ { \ @@ -114,14 +114,14 @@ RBNAME ## _copy(struct rb_node *rb_old, struct rb_node *rb_new) \ { \ RBSTRUCT *old = rb_entry(rb_old, RBSTRUCT, RBFIELD); \ RBSTRUCT *new = rb_entry(rb_new, RBSTRUCT, RBFIELD); \ - new->RBAUGMENTED = old->RBAUGMENTED; \ + RBCOPY(new, old); \ } \ static void \ RBNAME ## _rotate(struct rb_node *rb_old, struct rb_node *rb_new) \ { \ RBSTRUCT *old = rb_entry(rb_old, RBSTRUCT, RBFIELD); \ RBSTRUCT *new = rb_entry(rb_new, RBSTRUCT, RBFIELD); \ - new->RBAUGMENTED = old->RBAUGMENTED; \ + RBCOPY(new, old); \ RBCOMPUTE(old, false); \ } \ RBSTATIC const struct rb_augment_callbacks RBNAME = { \ @@ -131,6 +131,27 @@ RBSTATIC const struct rb_augment_callbacks RBNAME = { \ }; /* + * Template for declaring augmented rbtree callbacks (generic single field) + * + * RBSTATIC: 'static' or empty + * RBNAME: name of the rb_augment_callbacks structure + * RBSTRUCT: struct type of the tree nodes + * RBFIELD: name of struct rb_node field within RBSTRUCT + * RBAUGMENTED: name of field within RBSTRUCT holding data for subtree + * RBCOMPUTE: name of function that recomputes the RBAUGMENTED data + */ + +#define RB_DECLARE_CALLBACKS(RBSTATIC, RBNAME, \ + RBSTRUCT, RBFIELD, RBAUGMENTED, RBCOMPUTE) \ +static inline void \ +RBNAME ## _copy_single(RBSTRUCT *new, RBSTRUCT *old) \ +{ \ + new->RBAUGMENTED = old->RBAUGMENTED; \ +} \ +RB_DECLARE_CALLBACKS_MULTI(RBSTATIC, RBNAME, \ + RBSTRUCT, RBFIELD, RBNAME ## _copy_single, RBCOMPUTE) + +/* * Template for declaring augmented rbtree callbacks, * computing RBAUGMENTED scalar as max(RBCOMPUTE(node)) for all subtree nodes. * diff --git a/include/linux/rbtree_latch.h b/include/linux/rbtree_latch.h index 2f630eb8307e..b55a3a01bc8a 100644 --- a/include/linux/rbtree_latch.h +++ b/include/linux/rbtree_latch.h @@ -47,7 +47,7 @@ struct latch_tree_root { }; /** - * latch_tree_ops - operators to define the tree order + * struct latch_tree_ops - operators to define the tree order * @less: used for insertion; provides the (partial) order between two elements. * @comp: used for lookups; provides the order between the search key and an element. * diff --git a/include/linux/rbtree_types.h b/include/linux/rbtree_types.h index 45b6ecde3665..3c7ae53e8139 100644 --- a/include/linux/rbtree_types.h +++ b/include/linux/rbtree_types.h @@ -9,6 +9,12 @@ struct rb_node { } __attribute__((aligned(sizeof(long)))); /* The alignment might seem pointless, but allegedly CRIS needs it */ +struct rb_node_linked { + struct rb_node node; + struct rb_node_linked *prev; + struct rb_node_linked *next; +}; + struct rb_root { struct rb_node *rb_node; }; @@ -28,7 +34,17 @@ struct rb_root_cached { struct rb_node *rb_leftmost; }; +/* + * Leftmost tree with links. This would allow a trivial rb_rightmost update, + * but that has been omitted due to the lack of users. + */ +struct rb_root_linked { + struct rb_root rb_root; + struct rb_node_linked *rb_leftmost; +}; + #define RB_ROOT (struct rb_root) { NULL, } #define RB_ROOT_CACHED (struct rb_root_cached) { {NULL, }, NULL } +#define RB_ROOT_LINKED (struct rb_root_linked) { {NULL, }, NULL } #endif diff --git a/include/linux/rcu_segcblist.h b/include/linux/rcu_segcblist.h index 2fdc2208f1ca..08b63ecf719b 100644 --- a/include/linux/rcu_segcblist.h +++ b/include/linux/rcu_segcblist.h @@ -50,12 +50,14 @@ struct rcu_cblist { * Note that RCU_WAIT_TAIL cannot be empty unless RCU_NEXT_READY_TAIL is also * empty. * - * The ->gp_seq[] array contains the grace-period number at which the - * corresponding segment of callbacks will be ready to invoke. A given - * element of this array is meaningful only when the corresponding segment - * is non-empty, and it is never valid for RCU_DONE_TAIL (whose callbacks - * are already ready to invoke) or for RCU_NEXT_TAIL (whose callbacks have - * not yet been assigned a grace-period number). + * The ->gp_seq[] array contains the grace-period state at which the + * corresponding segment of callbacks will be ready to invoke. This tracks + * both normal and expedited grace periods, allowing callbacks to complete + * when either type of GP finishes. A given element of this array is + * meaningful only when the corresponding segment is non-empty, and it is + * never valid for RCU_DONE_TAIL (whose callbacks are already ready to + * invoke) or for RCU_NEXT_TAIL (whose callbacks have not yet been assigned + * a grace-period state). */ #define RCU_DONE_TAIL 0 /* Also RCU_WAIT head. */ #define RCU_WAIT_TAIL 1 /* Also RCU_NEXT_READY head. */ @@ -190,7 +192,7 @@ struct rcu_cblist { struct rcu_segcblist { struct rcu_head *head; struct rcu_head **tails[RCU_CBLIST_NSEGS]; - unsigned long gp_seq[RCU_CBLIST_NSEGS]; + struct rcu_gp_seq gp_seq[RCU_CBLIST_NSEGS]; #ifdef CONFIG_RCU_NOCB_CPU atomic_long_t len; #else diff --git a/include/linux/rculist.h b/include/linux/rculist.h index 2abba7552605..e3bc44225692 100644 --- a/include/linux/rculist.h +++ b/include/linux/rculist.h @@ -261,6 +261,35 @@ static inline void list_replace_rcu(struct list_head *old, old->prev = LIST_POISON2; } +static inline void __list_splice_rcu(struct list_head *list, + struct list_head *prev, + struct list_head *next) +{ + struct list_head *first = list->next; + struct list_head *last = list->prev; + + last->next = next; + first->prev = prev; + next->prev = last; + rcu_assign_pointer(list_next_rcu(prev), first); +} + +/** + * list_splice_rcu - splice a non-RCU list into an RCU-protected list, + * designed for stacks. + * @list: the non RCU-protected list to splice + * @head: the place in the existing RCU-protected list to splice + * + * The list pointed to by @head can be RCU-read traversed concurrently with + * this function. + */ +static inline void list_splice_rcu(struct list_head *list, + struct list_head *head) +{ + if (!list_empty(list)) + __list_splice_rcu(list, head, head->next); +} + /** * __list_splice_init_rcu - join an RCU-protected list into an existing list. * @list: the RCU-protected list to splice diff --git a/include/linux/rculist_bl.h b/include/linux/rculist_bl.h index 0b952d06eb0b..36363b876e53 100644 --- a/include/linux/rculist_bl.h +++ b/include/linux/rculist_bl.h @@ -8,21 +8,31 @@ #include <linux/list_bl.h> #include <linux/rcupdate.h> +/* return the first ptr or next element in an RCU protected list */ +#define hlist_bl_first_rcu(head) \ + (*((struct hlist_bl_node __rcu **)(&(head)->first))) +#define hlist_bl_next_rcu(node) \ + (*((struct hlist_bl_node __rcu **)(&(node)->next))) + static inline void hlist_bl_set_first_rcu(struct hlist_bl_head *h, struct hlist_bl_node *n) { LIST_BL_BUG_ON((unsigned long)n & LIST_BL_LOCKMASK); LIST_BL_BUG_ON(((unsigned long)h->first & LIST_BL_LOCKMASK) != LIST_BL_LOCKMASK); - rcu_assign_pointer(h->first, + rcu_assign_pointer(hlist_bl_first_rcu(h), (struct hlist_bl_node *)((unsigned long)n | LIST_BL_LOCKMASK)); } -static inline struct hlist_bl_node *hlist_bl_first_rcu(struct hlist_bl_head *h) -{ - return (struct hlist_bl_node *) - ((unsigned long)rcu_dereference_check(h->first, hlist_bl_is_locked(h)) & ~LIST_BL_LOCKMASK); -} +#define hlist_bl_first_rcu_dereference(head) \ +({ \ + struct hlist_bl_head *__head = (head); \ + \ + (struct hlist_bl_node *) \ + ((unsigned long)rcu_dereference_check(hlist_bl_first_rcu(__head), \ + hlist_bl_is_locked(__head)) & \ + ~LIST_BL_LOCKMASK); \ +}) /** * hlist_bl_del_rcu - deletes entry from hash list without re-initialization @@ -73,7 +83,7 @@ static inline void hlist_bl_add_head_rcu(struct hlist_bl_node *n, { struct hlist_bl_node *first; - /* don't need hlist_bl_first_rcu because we're under lock */ + /* don't need hlist_bl_first_rcu* because we're under lock */ first = hlist_bl_first(h); n->next = first; @@ -93,9 +103,30 @@ static inline void hlist_bl_add_head_rcu(struct hlist_bl_node *n, * */ #define hlist_bl_for_each_entry_rcu(tpos, pos, head, member) \ - for (pos = hlist_bl_first_rcu(head); \ + for (pos = hlist_bl_first_rcu_dereference(head); \ pos && \ ({ tpos = hlist_bl_entry(pos, typeof(*tpos), member); 1; }); \ - pos = rcu_dereference_raw(pos->next)) + pos = rcu_dereference_raw(hlist_bl_next_rcu(pos))) + +/** + * hlist_bl_for_each_entry_continue_rcu - continue iteration over list of given + * type + * @tpos: the type * to use as a loop cursor. + * @pos: the &struct hlist_bl_node to use as a loop cursor. + * @member: the name of the hlist_bl_node within the struct. + * + * Continue to iterate over list of given type, continuing after + * the current position which must have been in the list when the RCU read + * lock was taken. + * This would typically require either that you obtained the node from a + * previous walk of the list in the same RCU read-side critical section, or + * that you held some sort of non-RCU reference (such as a reference count) + * to keep the node alive *and* in the list. + */ +#define hlist_bl_for_each_entry_continue_rcu(tpos, pos, member) \ + for (pos = rcu_dereference_raw(hlist_bl_next_rcu(&(tpos)->member)); \ + pos && \ + ({ tpos = hlist_bl_entry(pos, typeof(*tpos), member); 1; }); \ + pos = rcu_dereference_raw(hlist_bl_next_rcu(pos))) #endif diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h index 04f3f86a4145..44c07a66edff 100644 --- a/include/linux/rcupdate.h +++ b/include/linux/rcupdate.h @@ -52,9 +52,18 @@ void call_rcu(struct rcu_head *head, rcu_callback_t func); void rcu_barrier_tasks(void); void synchronize_rcu(void); -struct rcu_gp_oldstate; +/* + * Grace-period sequence snapshot for the polled RCU APIs: ->norm for the + * normal grace period and ->exp for the expedited one. ->exp is unused by + * Tiny RCU, but is present unconditionally so that a single definition + * serves both Tiny RCU and Tree RCU. + */ +struct rcu_gp_seq { + unsigned long norm; + unsigned long exp; +}; unsigned long get_completed_synchronize_rcu(void); -void get_completed_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp); +void get_completed_synchronize_rcu_full(struct rcu_gp_seq *gsp); // Maximum number of unsigned long values corresponding to // not-yet-completed RCU grace periods. @@ -206,29 +215,17 @@ static inline void exit_tasks_rcu_finish(void) { } #endif /* #else #ifdef CONFIG_TASKS_RCU_GENERIC */ /** - * rcu_trace_implies_rcu_gp - does an RCU Tasks Trace grace period imply an RCU grace period? - * - * As an accident of implementation, an RCU Tasks Trace grace period also - * acts as an RCU grace period. However, this could change at any time. - * Code relying on this accident must call this function to verify that - * this accident is still happening. - * - * You have been warned! - */ -static inline bool rcu_trace_implies_rcu_gp(void) { return true; } - -/** * cond_resched_tasks_rcu_qs - Report potential quiescent states to RCU * - * This macro resembles cond_resched(), except that it is defined to + * This function resembles cond_resched(), except that it is defined to * report potential quiescent states to RCU-tasks even if the cond_resched() * machinery were to be shut off, as some advocate for PREEMPTION kernels. */ -#define cond_resched_tasks_rcu_qs() \ -do { \ - rcu_tasks_qs(current, false); \ - cond_resched(); \ -} while (0) +static inline void cond_resched_tasks_rcu_qs(void) +{ + rcu_tasks_qs(current, false); + cond_resched(); +} /** * rcu_softirq_qs_periodic - Report RCU and RCU-Tasks quiescent states @@ -502,12 +499,12 @@ context_unsafe( \ */ #define unrcu_pointer(p) __unrcu_pointer(p, __UNIQUE_ID(rcu)) -#define __rcu_access_pointer(p, local, space) \ +#define __rcu_access_pointer(p, local, space) context_unsafe( \ ({ \ typeof(*p) *local = (typeof(*p) *__force)READ_ONCE(p); \ rcu_check_sparse(p, space); \ ((typeof(*p) __force __kernel *)(local)); \ -}) +}) ) #define __rcu_dereference_check(p, local, c, space) \ ({ \ /* Dependency order vs. p above. */ \ @@ -604,11 +601,13 @@ context_unsafe( \ * lockdep checks for being in an RCU read-side critical section. This is * useful when the value of this pointer is accessed, but the pointer is * not dereferenced, for example, when testing an RCU-protected pointer - * against NULL. Although rcu_access_pointer() may also be used in cases - * where update-side locks prevent the value of the pointer from changing, - * you should instead use rcu_dereference_protected() for this use case. - * Within an RCU read-side critical section, there is little reason to - * use rcu_access_pointer(). + * against NULL. Within an RCU read-side critical section, there is little + * reason to use rcu_access_pointer(). Although rcu_access_pointer() may + * also be used in cases where update-side locks prevent the value of the + * pointer from changing, you should instead use rcu_dereference_protected() + * for this use case. It is also permissible to use rcu_access_pointer() + * within lockless updaters to obtain the old value for an atomic operation, + * for example, for cmpxchg(). * * It is usually best to test the rcu_access_pointer() return value * directly in order to avoid accidental dereferences being introduced @@ -1108,19 +1107,22 @@ static inline void rcu_read_unlock_migrate(void) /* * In mm/slab_common.c, no suitable header to include here. */ -void kvfree_call_rcu(struct rcu_head *head, void *ptr); +void kvfree_call_rcu(struct kvfree_rcu_head *head, void *ptr); +void kfree_call_rcu_nolock(struct kvfree_rcu_head *head, void *ptr); /* * The BUILD_BUG_ON() makes sure the rcu_head offset can be handled. See the * comment of kfree_rcu() for details. */ -#define kvfree_rcu_arg_2(ptr, rhf) \ +#define kvfree_rcu_arg_2(ptr, kvrhf) \ do { \ typeof (ptr) ___p = (ptr); \ + struct kvfree_rcu_head *___head; \ \ if (___p) { \ - BUILD_BUG_ON(offsetof(typeof(*(ptr)), rhf) >= 4096); \ - kvfree_call_rcu(&((___p)->rhf), (void *) (___p)); \ + BUILD_BUG_ON(offsetof(typeof(*(ptr)), kvrhf) >= 4096); \ + ___head = (struct kvfree_rcu_head *) &(___p)->kvrhf; \ + kvfree_call_rcu(___head, (void *) (___p)); \ } \ } while (0) @@ -1132,6 +1134,27 @@ do { \ kvfree_call_rcu(NULL, (void *) (___p)); \ } while (0) +/** + * kfree_rcu_nolock() - a version of kfree_rcu() that can be called in any context. + * @ptr: pointer to kfree for double-argument invocations. + * @kvrhf: the name of the struct kvfree_rcu_head within the type of @ptr. + * + * With KVFREE_RCU_BATCHED, kfree_rcu_nolock() tries hard to free objects + * without any deferred processing, but may still defer freeing. + * Large kmalloc and vmalloc objects are always deferred. + * + * kfree_rcu_nolock() supports 2-arg variant only. + */ +#define kfree_rcu_nolock(ptr, kvrhf) \ +do { \ + typeof (ptr) ___p = (ptr); \ + \ + if (___p) { \ + BUILD_BUG_ON(offsetof(typeof(*(ptr)), kvrhf) >= 4096); \ + kfree_call_rcu_nolock(&((___p)->kvrhf), (void *) (___p)); \ + } \ +} while (0) + /* * Place this after a lock-acquisition primitive to guarantee that * an UNLOCK+LOCK pair acts as a full barrier. This guarantee applies diff --git a/include/linux/rcupdate_trace.h b/include/linux/rcupdate_trace.h index cee89e51e45c..273c59a03251 100644 --- a/include/linux/rcupdate_trace.h +++ b/include/linux/rcupdate_trace.h @@ -95,10 +95,13 @@ static inline void rcu_read_unlock_tasks_trace(struct srcu_ctr __percpu *scp) */ static inline void rcu_read_lock_trace(void) { + int n; struct task_struct *t = current; rcu_try_lock_acquire(&rcu_tasks_trace_srcu_struct.dep_map); - if (t->trc_reader_nesting++) { + n = READ_ONCE(t->trc_reader_nesting); + WRITE_ONCE(t->trc_reader_nesting, n + 1); + if (n) { // In case we interrupted a Tasks Trace RCU reader. return; } @@ -119,12 +122,17 @@ static inline void rcu_read_lock_trace(void) */ static inline void rcu_read_unlock_trace(void) { + int n; struct srcu_ctr __percpu *scp; struct task_struct *t = current; - scp = t->trc_reader_scp; - barrier(); // scp before nesting to protect against interrupt handler. - if (!--t->trc_reader_nesting) { + n = READ_ONCE(t->trc_reader_nesting) - 1; + if (n) { + WRITE_ONCE(t->trc_reader_nesting, n); + } else { + scp = t->trc_reader_scp; // Compiler cannot hoist load due to data raciness. + barrier(); // scp before nesting to protect against interrupt handler. + WRITE_ONCE(t->trc_reader_nesting, n); if (!IS_ENABLED(CONFIG_TASKS_TRACE_RCU_NO_MB)) smp_mb(); // Placeholder for more selective ordering __srcu_read_unlock_fast(&rcu_tasks_trace_srcu_struct, scp); @@ -198,10 +206,13 @@ static inline void rcu_tasks_trace_expedite_current(void) srcu_expedite_current(&rcu_tasks_trace_srcu_struct); } +unsigned long rcu_tasks_trace_batches_completed(void); + // Placeholders to enable stepwise transition. void __init rcu_tasks_trace_suppress_unused(void); #else +static inline unsigned long rcu_tasks_trace_batches_completed(void) { return 0; } /* * The BPF JIT forms these addresses even when it doesn't call these * functions, so provide definitions that result in runtime errors. diff --git a/include/linux/rcupdate_wait.h b/include/linux/rcupdate_wait.h index 4c92d4291cce..fa884704a3b7 100644 --- a/include/linux/rcupdate_wait.h +++ b/include/linux/rcupdate_wait.h @@ -18,7 +18,7 @@ struct rcu_synchronize { struct completion completion; /* This is for debugging. */ - struct rcu_gp_oldstate oldstate; + struct rcu_gp_seq oldstate; }; void wakeme_after_rcu(struct rcu_head *head); diff --git a/include/linux/rcutiny.h b/include/linux/rcutiny.h index f519cd680228..e56ded733b1b 100644 --- a/include/linux/rcutiny.h +++ b/include/linux/rcutiny.h @@ -14,11 +14,7 @@ #include <asm/param.h> /* for HZ */ -struct rcu_gp_oldstate { - unsigned long rgos_norm; -}; - -// Maximum number of rcu_gp_oldstate values corresponding to +// Maximum number of rcu_gp_seq values corresponding to // not-yet-completed RCU grace periods. #define NUM_ACTIVE_RCU_POLL_FULL_OLDSTATE 2 @@ -26,31 +22,31 @@ struct rcu_gp_oldstate { * Are the two oldstate values the same? See the Tree RCU version for * docbook header. */ -static inline bool same_state_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp1, - struct rcu_gp_oldstate *rgosp2) +static inline bool same_state_synchronize_rcu_full(struct rcu_gp_seq *rgosp1, + struct rcu_gp_seq *rgosp2) { - return rgosp1->rgos_norm == rgosp2->rgos_norm; + return rgosp1->norm == rgosp2->norm; } unsigned long get_state_synchronize_rcu(void); -static inline void get_state_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp) +static inline void get_state_synchronize_rcu_full(struct rcu_gp_seq *gsp) { - rgosp->rgos_norm = get_state_synchronize_rcu(); + gsp->norm = get_state_synchronize_rcu(); } unsigned long start_poll_synchronize_rcu(void); -static inline void start_poll_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp) +static inline void start_poll_synchronize_rcu_full(struct rcu_gp_seq *gsp) { - rgosp->rgos_norm = start_poll_synchronize_rcu(); + gsp->norm = start_poll_synchronize_rcu(); } bool poll_state_synchronize_rcu(unsigned long oldstate); -static inline bool poll_state_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp) +static inline bool poll_state_synchronize_rcu_full(struct rcu_gp_seq *gsp) { - return poll_state_synchronize_rcu(rgosp->rgos_norm); + return poll_state_synchronize_rcu(gsp->norm); } static inline void cond_synchronize_rcu(unsigned long oldstate) @@ -58,9 +54,9 @@ static inline void cond_synchronize_rcu(unsigned long oldstate) might_sleep(); } -static inline void cond_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp) +static inline void cond_synchronize_rcu_full(struct rcu_gp_seq *gsp) { - cond_synchronize_rcu(rgosp->rgos_norm); + cond_synchronize_rcu(gsp->norm); } static inline unsigned long start_poll_synchronize_rcu_expedited(void) @@ -68,9 +64,9 @@ static inline unsigned long start_poll_synchronize_rcu_expedited(void) return start_poll_synchronize_rcu(); } -static inline void start_poll_synchronize_rcu_expedited_full(struct rcu_gp_oldstate *rgosp) +static inline void start_poll_synchronize_rcu_expedited_full(struct rcu_gp_seq *gsp) { - rgosp->rgos_norm = start_poll_synchronize_rcu_expedited(); + gsp->norm = start_poll_synchronize_rcu_expedited(); } static inline void cond_synchronize_rcu_expedited(unsigned long oldstate) @@ -78,9 +74,9 @@ static inline void cond_synchronize_rcu_expedited(unsigned long oldstate) cond_synchronize_rcu(oldstate); } -static inline void cond_synchronize_rcu_expedited_full(struct rcu_gp_oldstate *rgosp) +static inline void cond_synchronize_rcu_expedited_full(struct rcu_gp_seq *gsp) { - cond_synchronize_rcu_expedited(rgosp->rgos_norm); + cond_synchronize_rcu_expedited(gsp->norm); } extern void rcu_barrier(void); diff --git a/include/linux/rcutree.h b/include/linux/rcutree.h index 9d2d7bd251d4..16a04202888b 100644 --- a/include/linux/rcutree.h +++ b/include/linux/rcutree.h @@ -38,12 +38,7 @@ void synchronize_rcu_expedited(void); void rcu_barrier(void); void rcu_momentary_eqs(void); -struct rcu_gp_oldstate { - unsigned long rgos_norm; - unsigned long rgos_exp; -}; - -// Maximum number of rcu_gp_oldstate values corresponding to +// Maximum number of rcu_gp_seq values corresponding to // not-yet-completed RCU grace periods. #define NUM_ACTIVE_RCU_POLL_FULL_OLDSTATE 4 @@ -60,29 +55,29 @@ struct rcu_gp_oldstate { * to a list header, allowing those structures to be slightly smaller. * * Note that equality is judged on a bitwise basis, so that an - * @rcu_gp_oldstate structure with an already-completed state in one field + * @rcu_gp_seq structure with an already-completed state in one field * will compare not-equal to a structure with an already-completed state - * in the other field. After all, the @rcu_gp_oldstate structure is opaque + * in the other field. After all, the @rcu_gp_seq structure is opaque * so how did such a situation come to pass in the first place? */ -static inline bool same_state_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp1, - struct rcu_gp_oldstate *rgosp2) +static inline bool same_state_synchronize_rcu_full(struct rcu_gp_seq *rgosp1, + struct rcu_gp_seq *rgosp2) { - return rgosp1->rgos_norm == rgosp2->rgos_norm && rgosp1->rgos_exp == rgosp2->rgos_exp; + return rgosp1->norm == rgosp2->norm && rgosp1->exp == rgosp2->exp; } unsigned long start_poll_synchronize_rcu_expedited(void); -void start_poll_synchronize_rcu_expedited_full(struct rcu_gp_oldstate *rgosp); +void start_poll_synchronize_rcu_expedited_full(struct rcu_gp_seq *gsp); void cond_synchronize_rcu_expedited(unsigned long oldstate); -void cond_synchronize_rcu_expedited_full(struct rcu_gp_oldstate *rgosp); +void cond_synchronize_rcu_expedited_full(struct rcu_gp_seq *gsp); unsigned long get_state_synchronize_rcu(void); -void get_state_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp); +void get_state_synchronize_rcu_full(struct rcu_gp_seq *gsp); unsigned long start_poll_synchronize_rcu(void); -void start_poll_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp); +void start_poll_synchronize_rcu_full(struct rcu_gp_seq *gsp); bool poll_state_synchronize_rcu(unsigned long oldstate); -bool poll_state_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp); +bool poll_state_synchronize_rcu_full(struct rcu_gp_seq *gsp); void cond_synchronize_rcu(unsigned long oldstate); -void cond_synchronize_rcu_full(struct rcu_gp_oldstate *rgosp); +void cond_synchronize_rcu_full(struct rcu_gp_seq *gsp); #ifdef CONFIG_PROVE_RCU void rcu_irq_exit_check_preempt(void); diff --git a/include/linux/refcount.h b/include/linux/refcount.h index 3da377ffb0c2..ba7657ced281 100644 --- a/include/linux/refcount.h +++ b/include/linux/refcount.h @@ -170,7 +170,7 @@ static inline unsigned int refcount_read(const refcount_t *r) return atomic_read(&r->refs); } -static inline __must_check __signed_wrap +static inline __must_check bool __refcount_add_not_zero(int i, refcount_t *r, int *oldp) { int old = refcount_read(r); @@ -212,7 +212,7 @@ static inline __must_check bool refcount_add_not_zero(int i, refcount_t *r) return __refcount_add_not_zero(i, r, NULL); } -static inline __must_check __signed_wrap +static inline __must_check bool __refcount_add_not_zero_limited_acquire(int i, refcount_t *r, int *oldp, int limit) { @@ -244,7 +244,7 @@ __refcount_inc_not_zero_limited_acquire(refcount_t *r, int *oldp, int limit) return __refcount_add_not_zero_limited_acquire(1, r, oldp, limit); } -static inline __must_check __signed_wrap +static inline __must_check bool __refcount_add_not_zero_acquire(int i, refcount_t *r, int *oldp) { return __refcount_add_not_zero_limited_acquire(i, r, oldp, INT_MAX); @@ -277,7 +277,7 @@ static inline __must_check bool refcount_add_not_zero_acquire(int i, refcount_t return __refcount_add_not_zero_acquire(i, r, NULL); } -static inline __signed_wrap +static inline void __refcount_add(int i, refcount_t *r, int *oldp) { int old = atomic_fetch_add_relaxed(i, &r->refs); @@ -383,7 +383,7 @@ static inline void refcount_inc(refcount_t *r) __refcount_inc(r, NULL); } -static inline __must_check __signed_wrap +static inline __must_check bool __refcount_sub_and_test(int i, refcount_t *r, int *oldp) { int old = atomic_fetch_sub_release(i, &r->refs); diff --git a/include/linux/regmap.h b/include/linux/regmap.h index caff2240bdab..c8aebd148e08 100644 --- a/include/linux/regmap.h +++ b/include/linux/regmap.h @@ -10,15 +10,16 @@ * Author: Mark Brown <broonie@opensource.wolfsonmicro.com> */ -#include <linux/list.h> -#include <linux/rbtree.h> -#include <linux/ktime.h> +#include <linux/bug.h> +#include <linux/cleanup.h> #include <linux/delay.h> #include <linux/err.h> -#include <linux/bug.h> -#include <linux/lockdep.h> -#include <linux/iopoll.h> #include <linux/fwnode.h> +#include <linux/iopoll.h> +#include <linux/ktime.h> +#include <linux/list.h> +#include <linux/lockdep.h> +#include <linux/rbtree.h> struct module; struct clk; @@ -586,7 +587,7 @@ typedef void (*regmap_hw_free_context)(void *context); * must serialise with respect to non-async I/O. * @reg_write: Write a single register value to the given register address. This * write operation has to complete when returning from the function. - * @reg_write_noinc: Write multiple register value to the same register. This + * @reg_noinc_write: Write multiple register values to the same register. This * write operation has to complete when returning from the function. * @reg_update_bits: Update bits operation to be used against volatile * registers, intended for devices supporting some mechanism @@ -595,6 +596,8 @@ typedef void (*regmap_hw_free_context)(void *context); * @read: Read operation. Data is returned in the buffer used to transmit * data. * @reg_read: Read a single register value from a given register address. + * @reg_noinc_read: Read multiple register values from the same register. This + * read operation has to complete when returning from the function. * @free_context: Free context. * @async_alloc: Allocate a regmap_async() structure. * @read_flag_mask: Mask to be set in the top byte of the register when doing @@ -692,6 +695,10 @@ struct regmap *__regmap_init_sdw_mbq(struct device *dev, struct sdw_slave *sdw, const struct regmap_sdw_mbq_cfg *mbq_config, struct lock_class_key *lock_key, const char *lock_name); +struct regmap *__regmap_init_i3c(struct i3c_device *i3c, + const struct regmap_config *config, + struct lock_class_key *lock_key, + const char *lock_name); struct regmap *__regmap_init_spi_avmm(struct spi_device *spi, const struct regmap_config *config, struct lock_class_key *lock_key, @@ -986,7 +993,8 @@ bool regmap_ac97_default_volatile(struct device *dev, unsigned int reg); /** * regmap_init_sdw_mbq_cfg() - Initialise MBQ SDW register map with config * - * @sdw: Device that will be interacted with + * @dev: &struct device that will be interacted with + * @sdw: Soundwire device that will be interacted with * @config: Configuration for register map * @mbq_config: Properties for the MBQ registers * @@ -999,6 +1007,19 @@ bool regmap_ac97_default_volatile(struct device *dev, unsigned int reg); dev, sdw, config, mbq_config) /** + * regmap_init_i3c() - Initialise register map + * + * @i3c: Device that will be interacted with + * @config: Configuration for register map + * + * The return value will be an ERR_PTR() on error or a valid pointer to + * a struct regmap. + */ +#define regmap_init_i3c(i3c, config) \ + __regmap_lockdep_wrapper(__regmap_init_i3c, #config, \ + i3c, config) + +/** * regmap_init_spi_avmm() - Initialize register map for Intel SPI Slave * to AVMM Bus Bridge * @@ -1460,6 +1481,8 @@ struct regmap_field *regmap_field_alloc(struct regmap *regmap, struct reg_field reg_field); void regmap_field_free(struct regmap_field *field); +DEFINE_FREE(regmap_field, struct regmap_field *, if (_T) regmap_field_free(_T)) + struct regmap_field *devm_regmap_field_alloc(struct device *dev, struct regmap *regmap, struct reg_field reg_field); void devm_regmap_field_free(struct device *dev, struct regmap_field *field); @@ -1564,6 +1587,7 @@ regmap_fields_force_update_bits(struct regmap_field *field, unsigned int id, * struct regmap_irq_type - IRQ type definitions. * * @type_reg_offset: Offset register for the irq type setting. + * @type_reg_mask: Device interrupt mask * @type_rising_val: Register value to configure RISING type irq. * @type_falling_val: Register value to configure FALLING type irq. * @type_level_low_val: Register value to configure LEVEL_LOW type irq. @@ -1697,6 +1721,8 @@ struct regmap_irq_chip_data; * main status base, [0, num_config_regs[ for any config * register base, and [0, num_regs[ for any other base. * If unspecified then regmap_irq_get_irq_reg_linear() is used. + * @irq_reqres: Callback function to request IRQ resources (may be %NULL) + * @irq_relres: Callback function to release IRQ resources (may be %NULL) * @irq_drv_data: Driver specific IRQ data which is passed as parameter when * driver specific pre/post interrupt handler is called. * @@ -1750,6 +1776,8 @@ struct regmap_irq_chip { void *irq_drv_data); unsigned int (*get_irq_reg)(struct regmap_irq_chip_data *data, unsigned int base, int index); + int (*irq_reqres)(void *irq_drv_data, irq_hw_number_t hwirq); + void (*irq_relres)(void *irq_drv_data, irq_hw_number_t hwirq); void *irq_drv_data; }; diff --git a/include/linux/regulator/mt6359-regulator.h b/include/linux/regulator/mt6359-regulator.h index 6d6e5a58f482..ce2cd0fc9d95 100644 --- a/include/linux/regulator/mt6359-regulator.h +++ b/include/linux/regulator/mt6359-regulator.h @@ -29,8 +29,7 @@ enum { MT6359_ID_VCN18, MT6359_ID_VFE28, MT6359_ID_VCN13, - MT6359_ID_VCN33_1_BT, - MT6359_ID_VCN33_1_WIFI, + MT6359_ID_VCN33_1, MT6359_ID_VAUX18, MT6359_ID_VSRAM_OTHERS, MT6359_ID_VEFUSE, @@ -39,8 +38,7 @@ enum { MT6359_ID_VBIF28, MT6359_ID_VIO28, MT6359_ID_VEMC, - MT6359_ID_VCN33_2_BT, - MT6359_ID_VCN33_2_WIFI, + MT6359_ID_VCN33_2, MT6359_ID_VA12, MT6359_ID_VA09, MT6359_ID_VRF18, @@ -51,6 +49,10 @@ enum { MT6359_ID_VSRAM_PROC1, MT6359_ID_VSIM2, MT6359_ID_VSRAM_OTHERS_SSHUB, + MT6359_ID_VCN33_1_BT, + MT6359_ID_VCN33_1_WIFI, + MT6359_ID_VCN33_2_BT, + MT6359_ID_VCN33_2_WIFI, MT6359_ID_RG_MAX, }; diff --git a/include/linux/remoteproc.h b/include/linux/remoteproc.h index b4795698d8c2..a44368737b39 100644 --- a/include/linux/remoteproc.h +++ b/include/linux/remoteproc.h @@ -1,35 +1,8 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ /* - * Remote Processor Framework - * * Copyright(c) 2011 Texas Instruments, Inc. * Copyright(c) 2011 Google, Inc. * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name Texas Instruments nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #ifndef REMOTEPROC_H @@ -37,279 +10,13 @@ #include <linux/types.h> #include <linux/mutex.h> +#include <linux/spinlock.h> #include <linux/virtio.h> #include <linux/cdev.h> #include <linux/completion.h> #include <linux/idr.h> #include <linux/of.h> - -/** - * struct resource_table - firmware resource table header - * @ver: version number - * @num: number of resource entries - * @reserved: reserved (must be zero) - * @offset: array of offsets pointing at the various resource entries - * - * A resource table is essentially a list of system resources required - * by the remote processor. It may also include configuration entries. - * If needed, the remote processor firmware should contain this table - * as a dedicated ".resource_table" ELF section. - * - * Some resources entries are mere announcements, where the host is informed - * of specific remoteproc configuration. Other entries require the host to - * do something (e.g. allocate a system resource). Sometimes a negotiation - * is expected, where the firmware requests a resource, and once allocated, - * the host should provide back its details (e.g. address of an allocated - * memory region). - * - * The header of the resource table, as expressed by this structure, - * contains a version number (should we need to change this format in the - * future), the number of available resource entries, and their offsets - * in the table. - * - * Immediately following this header are the resource entries themselves, - * each of which begins with a resource entry header (as described below). - */ -struct resource_table { - u32 ver; - u32 num; - u32 reserved[2]; - u32 offset[]; -} __packed; - -/** - * struct fw_rsc_hdr - firmware resource entry header - * @type: resource type - * @data: resource data - * - * Every resource entry begins with a 'struct fw_rsc_hdr' header providing - * its @type. The content of the entry itself will immediately follow - * this header, and it should be parsed according to the resource type. - */ -struct fw_rsc_hdr { - u32 type; - u8 data[]; -} __packed; - -/** - * enum fw_resource_type - types of resource entries - * - * @RSC_CARVEOUT: request for allocation of a physically contiguous - * memory region. - * @RSC_DEVMEM: request to iommu_map a memory-based peripheral. - * @RSC_TRACE: announces the availability of a trace buffer into which - * the remote processor will be writing logs. - * @RSC_VDEV: declare support for a virtio device, and serve as its - * virtio header. - * @RSC_LAST: just keep this one at the end of standard resources - * @RSC_VENDOR_START: start of the vendor specific resource types range - * @RSC_VENDOR_END: end of the vendor specific resource types range - * - * For more details regarding a specific resource type, please see its - * dedicated structure below. - * - * Please note that these values are used as indices to the rproc_handle_rsc - * lookup table, so please keep them sane. Moreover, @RSC_LAST is used to - * check the validity of an index before the lookup table is accessed, so - * please update it as needed. - */ -enum fw_resource_type { - RSC_CARVEOUT = 0, - RSC_DEVMEM = 1, - RSC_TRACE = 2, - RSC_VDEV = 3, - RSC_LAST = 4, - RSC_VENDOR_START = 128, - RSC_VENDOR_END = 512, -}; - -#define FW_RSC_ADDR_ANY (-1) - -/** - * struct fw_rsc_carveout - physically contiguous memory request - * @da: device address - * @pa: physical address - * @len: length (in bytes) - * @flags: iommu protection flags - * @reserved: reserved (must be zero) - * @name: human-readable name of the requested memory region - * - * This resource entry requests the host to allocate a physically contiguous - * memory region. - * - * These request entries should precede other firmware resource entries, - * as other entries might request placing other data objects inside - * these memory regions (e.g. data/code segments, trace resource entries, ...). - * - * Allocating memory this way helps utilizing the reserved physical memory - * (e.g. CMA) more efficiently, and also minimizes the number of TLB entries - * needed to map it (in case @rproc is using an IOMMU). Reducing the TLB - * pressure is important; it may have a substantial impact on performance. - * - * If the firmware is compiled with static addresses, then @da should specify - * the expected device address of this memory region. If @da is set to - * FW_RSC_ADDR_ANY, then the host will dynamically allocate it, and then - * overwrite @da with the dynamically allocated address. - * - * We will always use @da to negotiate the device addresses, even if it - * isn't using an iommu. In that case, though, it will obviously contain - * physical addresses. - * - * Some remote processors needs to know the allocated physical address - * even if they do use an iommu. This is needed, e.g., if they control - * hardware accelerators which access the physical memory directly (this - * is the case with OMAP4 for instance). In that case, the host will - * overwrite @pa with the dynamically allocated physical address. - * Generally we don't want to expose physical addresses if we don't have to - * (remote processors are generally _not_ trusted), so we might want to - * change this to happen _only_ when explicitly required by the hardware. - * - * @flags is used to provide IOMMU protection flags, and @name should - * (optionally) contain a human readable name of this carveout region - * (mainly for debugging purposes). - */ -struct fw_rsc_carveout { - u32 da; - u32 pa; - u32 len; - u32 flags; - u32 reserved; - u8 name[32]; -} __packed; - -/** - * struct fw_rsc_devmem - iommu mapping request - * @da: device address - * @pa: physical address - * @len: length (in bytes) - * @flags: iommu protection flags - * @reserved: reserved (must be zero) - * @name: human-readable name of the requested region to be mapped - * - * This resource entry requests the host to iommu map a physically contiguous - * memory region. This is needed in case the remote processor requires - * access to certain memory-based peripherals; _never_ use it to access - * regular memory. - * - * This is obviously only needed if the remote processor is accessing memory - * via an iommu. - * - * @da should specify the required device address, @pa should specify - * the physical address we want to map, @len should specify the size of - * the mapping and @flags is the IOMMU protection flags. As always, @name may - * (optionally) contain a human readable name of this mapping (mainly for - * debugging purposes). - * - * Note: at this point we just "trust" those devmem entries to contain valid - * physical addresses, but this isn't safe and will be changed: eventually we - * want remoteproc implementations to provide us ranges of physical addresses - * the firmware is allowed to request, and not allow firmwares to request - * access to physical addresses that are outside those ranges. - */ -struct fw_rsc_devmem { - u32 da; - u32 pa; - u32 len; - u32 flags; - u32 reserved; - u8 name[32]; -} __packed; - -/** - * struct fw_rsc_trace - trace buffer declaration - * @da: device address - * @len: length (in bytes) - * @reserved: reserved (must be zero) - * @name: human-readable name of the trace buffer - * - * This resource entry provides the host information about a trace buffer - * into which the remote processor will write log messages. - * - * @da specifies the device address of the buffer, @len specifies - * its size, and @name may contain a human readable name of the trace buffer. - * - * After booting the remote processor, the trace buffers are exposed to the - * user via debugfs entries (called trace0, trace1, etc..). - */ -struct fw_rsc_trace { - u32 da; - u32 len; - u32 reserved; - u8 name[32]; -} __packed; - -/** - * struct fw_rsc_vdev_vring - vring descriptor entry - * @da: device address - * @align: the alignment between the consumer and producer parts of the vring - * @num: num of buffers supported by this vring (must be power of two) - * @notifyid: a unique rproc-wide notify index for this vring. This notify - * index is used when kicking a remote processor, to let it know that this - * vring is triggered. - * @pa: physical address - * - * This descriptor is not a resource entry by itself; it is part of the - * vdev resource type (see below). - * - * Note that @da should either contain the device address where - * the remote processor is expecting the vring, or indicate that - * dynamically allocation of the vring's device address is supported. - */ -struct fw_rsc_vdev_vring { - u32 da; - u32 align; - u32 num; - u32 notifyid; - u32 pa; -} __packed; - -/** - * struct fw_rsc_vdev - virtio device header - * @id: virtio device id (as in virtio_ids.h) - * @notifyid: a unique rproc-wide notify index for this vdev. This notify - * index is used when kicking a remote processor, to let it know that the - * status/features of this vdev have changes. - * @dfeatures: specifies the virtio device features supported by the firmware - * @gfeatures: a place holder used by the host to write back the - * negotiated features that are supported by both sides. - * @config_len: the size of the virtio config space of this vdev. The config - * space lies in the resource table immediate after this vdev header. - * @status: a place holder where the host will indicate its virtio progress. - * @num_of_vrings: indicates how many vrings are described in this vdev header - * @reserved: reserved (must be zero) - * @vring: an array of @num_of_vrings entries of 'struct fw_rsc_vdev_vring'. - * - * This resource is a virtio device header: it provides information about - * the vdev, and is then used by the host and its peer remote processors - * to negotiate and share certain virtio properties. - * - * By providing this resource entry, the firmware essentially asks remoteproc - * to statically allocate a vdev upon registration of the rproc (dynamic vdev - * allocation is not yet supported). - * - * Note: - * 1. unlike virtualization systems, the term 'host' here means - * the Linux side which is running remoteproc to control the remote - * processors. We use the name 'gfeatures' to comply with virtio's terms, - * though there isn't really any virtualized guest OS here: it's the host - * which is responsible for negotiating the final features. - * Yeah, it's a bit confusing. - * - * 2. immediately following this structure is the virtio config space for - * this vdev (which is specific to the vdev; for more info, read the virtio - * spec). The size of the config space is specified by @config_len. - */ -struct fw_rsc_vdev { - u32 id; - u32 notifyid; - u32 dfeatures; - u32 gfeatures; - u32 config_len; - u8 status; - u8 num_of_vrings; - u8 reserved[2]; - struct fw_rsc_vdev_vring vring[]; -} __packed; +#include <linux/rsc_table.h> struct rproc; @@ -394,10 +101,10 @@ struct rproc_ops { int (*parse_fw)(struct rproc *rproc, const struct firmware *fw); int (*handle_rsc)(struct rproc *rproc, u32 rsc_type, void *rsc, int offset, int avail); - struct resource_table *(*find_loaded_rsc_table)( - struct rproc *rproc, const struct firmware *fw); - struct resource_table *(*get_loaded_rsc_table)( - struct rproc *rproc, size_t *size); + struct resource_table *(*find_loaded_rsc_table)(struct rproc *rproc, + const struct firmware *fw); + struct resource_table *(*get_loaded_rsc_table)(struct rproc *rproc, + size_t *size); int (*load)(struct rproc *rproc, const struct firmware *fw); int (*sanity_check)(struct rproc *rproc, const struct firmware *fw); u64 (*get_boot_addr)(struct rproc *rproc, const struct firmware *fw); @@ -412,7 +119,6 @@ struct rproc_ops { * a message. * @RPROC_RUNNING: device is up and running * @RPROC_CRASHED: device has crashed; need to start recovery - * @RPROC_DELETED: device is deleted * @RPROC_ATTACHED: device has been booted by another entity and the core * has attached to it * @RPROC_DETACHED: device has been booted by another entity and waiting @@ -430,10 +136,9 @@ enum rproc_state { RPROC_SUSPENDED = 1, RPROC_RUNNING = 2, RPROC_CRASHED = 3, - RPROC_DELETED = 4, - RPROC_ATTACHED = 5, - RPROC_DETACHED = 6, - RPROC_LAST = 7, + RPROC_ATTACHED = 4, + RPROC_DETACHED = 5, + RPROC_LAST = 6, }; /** @@ -526,7 +231,10 @@ enum rproc_features { * @subdevs: list of subdevices, to following the running state * @notifyids: idr for dynamically assigning rproc-wide unique notify ids * @index: index of this rproc device + * @attach_work: workqueue for attaching rproc * @crash_handler: workqueue for handling a crash + * @crash_handler_lock: serializes crash handler queueing and deletion + * @deleting: remoteproc deletion has begun * @crash_cnt: crash counter * @recovery_disabled: flag that state if recovery was disabled * @max_notifyid: largest allocated notify id. @@ -538,6 +246,7 @@ enum rproc_features { * @has_iommu: flag to indicate if remote processor is behind an MMU * @auto_boot: flag to indicate if remote processor should be auto-started * @sysfs_read_only: flag to make remoteproc sysfs files read only + * @subdevs_started: flag to indicate if subdevs have started * @dump_segments: list of segments in the firmware * @nb_vdev: number of vdev currently handled by rproc * @elf_class: firmware ELF class @@ -568,7 +277,10 @@ struct rproc { struct list_head subdevs; struct idr notifyids; int index; + struct work_struct attach_work; struct work_struct crash_handler; + spinlock_t crash_handler_lock; + bool deleting; unsigned int crash_cnt; bool recovery_disabled; int max_notifyid; @@ -579,6 +291,7 @@ struct rproc { bool has_iommu; bool auto_boot; bool sysfs_read_only; + bool subdevs_started; struct list_head dump_segments; int nb_vdev; u8 elf_class; @@ -642,7 +355,6 @@ struct rproc_vring { * @index: vdev position versus other vdev declared in resource table */ struct rproc_vdev { - struct rproc_subdev subdev; struct platform_device *pdev; diff --git a/include/linux/remoteproc/mtk_scp.h b/include/linux/remoteproc/mtk_scp.h index 344ff41c22c7..4070537d6542 100644 --- a/include/linux/remoteproc/mtk_scp.h +++ b/include/linux/remoteproc/mtk_scp.h @@ -58,7 +58,7 @@ int scp_ipi_register(struct mtk_scp *scp, u32 id, scp_ipi_handler_t handler, void *priv); void scp_ipi_unregister(struct mtk_scp *scp, u32 id); -int scp_ipi_send(struct mtk_scp *scp, u32 id, void *buf, unsigned int len, +int scp_ipi_send(struct mtk_scp *scp, u32 id, const void *buf, unsigned int len, unsigned int wait); unsigned int scp_get_vdec_hw_capa(struct mtk_scp *scp); diff --git a/include/linux/resctrl.h b/include/linux/resctrl.h index 006e57fd7ca5..dd09c2ce9a0f 100644 --- a/include/linux/resctrl.h +++ b/include/linux/resctrl.h @@ -54,11 +54,11 @@ enum resctrl_res_level { RDT_RESOURCE_MBA, RDT_RESOURCE_SMBA, RDT_RESOURCE_PERF_PKG, - - /* Must be the last */ - RDT_NUM_RESOURCES, + RDT_RESOURCE_LAST = RDT_RESOURCE_PERF_PKG }; +#define RDT_NUM_RESOURCES (RDT_RESOURCE_LAST + 1) + /** * enum resctrl_conf_type - The type of configuration. * @CDP_NONE: No prioritisation, both code and data are controlled or monitored. @@ -69,9 +69,10 @@ enum resctrl_conf_type { CDP_NONE, CDP_CODE, CDP_DATA, + CDP_LAST = CDP_DATA }; -#define CDP_NUM_TYPES (CDP_DATA + 1) +#define CDP_NUM_TYPES (CDP_LAST + 1) /* * struct pseudo_lock_region - pseudo-lock region information @@ -286,13 +287,15 @@ enum resctrl_schema_fmt { /** * struct resctrl_mon - Monitoring related data of a resctrl resource. - * @num_rmid: Number of RMIDs available. - * @mbm_cfg_mask: Memory transactions that can be tracked when bandwidth - * monitoring events can be configured. - * @num_mbm_cntrs: Number of assignable counters. - * @mbm_cntr_assignable:Is system capable of supporting counter assignment? - * @mbm_assign_on_mkdir:True if counters should automatically be assigned to MBM - * events of monitor groups created via mkdir. + * @num_rmid: Number of RMIDs available. + * @mbm_cfg_mask: Memory transactions that can be tracked when + * bandwidth monitoring events can be configured. + * @num_mbm_cntrs: Number of assignable counters. + * @mbm_cntr_assignable: Is system capable of supporting counter assignment? + * @mbm_assign_on_mkdir: True if counters should automatically be assigned to MBM + * events of monitor groups created via mkdir. + * @mbm_cntr_configurable: True if assignable counters are configurable. + * @mbm_cntr_assign_fixed: True if the counter assignment mode is fixed. */ struct resctrl_mon { u32 num_rmid; @@ -300,6 +303,8 @@ struct resctrl_mon { int num_mbm_cntrs; bool mbm_cntr_assignable; bool mbm_assign_on_mkdir; + bool mbm_cntr_configurable; + bool mbm_cntr_assign_fixed; }; /** @@ -319,7 +324,7 @@ struct resctrl_mon { * @cdp_capable: Is the CDP feature available on this resource */ struct rdt_resource { - int rid; + enum resctrl_res_level rid; bool alloc_capable; bool mon_capable; enum resctrl_scope ctrl_scope; diff --git a/include/linux/reset-controller.h b/include/linux/reset-controller.h index 46514cb1b9e0..52a5a4e81f18 100644 --- a/include/linux/reset-controller.h +++ b/include/linux/reset-controller.h @@ -3,7 +3,10 @@ #define _LINUX_RESET_CONTROLLER_H_ #include <linux/list.h> +#include <linux/mutex.h> +struct fwnode_handle; +struct fwnode_reference_args; struct reset_controller_dev; /** @@ -35,14 +38,16 @@ struct of_phandle_args; * @reset_control_head: head of internal list of requested reset controls * @dev: corresponding driver model device struct * @of_node: corresponding device tree node as phandle target - * @of_args: for reset-gpios controllers: corresponding phandle args with - * of_node and GPIO number complementing of_node; either this or - * of_node should be present * @of_reset_n_cells: number of cells in reset line specifiers * @of_xlate: translation function to translate from specifier as found in the - * device tree to id as given to the reset control ops, defaults - * to :c:func:`of_reset_simple_xlate`. + * device tree to id as given to the reset control ops + * @fwnode: firmware node associated with this device + * @fwnode_reset_n_cells: number of cells in reset line specifiers + * @fwnode_xlate: translation function to translate from firmware specifier to + * id as given to the reset control ops, defaults to + * :c:func:`fwnode_reset_simple_xlate` * @nr_resets: number of reset controls in this reset controller device + * @lock: protects the reset control list from concurrent access */ struct reset_controller_dev { const struct reset_control_ops *ops; @@ -51,11 +56,15 @@ struct reset_controller_dev { struct list_head reset_control_head; struct device *dev; struct device_node *of_node; - const struct of_phandle_args *of_args; int of_reset_n_cells; int (*of_xlate)(struct reset_controller_dev *rcdev, const struct of_phandle_args *reset_spec); + struct fwnode_handle *fwnode; + int fwnode_reset_n_cells; + int (*fwnode_xlate)(struct reset_controller_dev *rcdev, + const struct fwnode_reference_args *reset_spec); unsigned int nr_resets; + struct mutex lock; }; #if IS_ENABLED(CONFIG_RESET_CONTROLLER) diff --git a/include/linux/reset.h b/include/linux/reset.h index 44f9e3415f92..9c391cf0c822 100644 --- a/include/linux/reset.h +++ b/include/linux/reset.h @@ -5,10 +5,12 @@ #include <linux/bits.h> #include <linux/err.h> #include <linux/errno.h> +#include <linux/of.h> #include <linux/types.h> struct device; struct device_node; +struct fwnode_handle; struct reset_control; /** @@ -84,7 +86,7 @@ int reset_control_bulk_deassert(int num_rstcs, struct reset_control_bulk_data *r int reset_control_bulk_acquire(int num_rstcs, struct reset_control_bulk_data *rstcs); void reset_control_bulk_release(int num_rstcs, struct reset_control_bulk_data *rstcs); -struct reset_control *__of_reset_control_get(struct device_node *node, +struct reset_control *__fwnode_reset_control_get(struct fwnode_handle *fwnode, const char *id, int index, enum reset_control_flags flags); struct reset_control *__reset_control_get(struct device *dev, const char *id, int index, enum reset_control_flags flags); @@ -103,7 +105,8 @@ int __devm_reset_control_bulk_get(struct device *dev, int num_rstcs, struct reset_control *devm_reset_control_array_get(struct device *dev, enum reset_control_flags flags); -struct reset_control *of_reset_control_array_get(struct device_node *np, enum reset_control_flags); +struct reset_control *fwnode_reset_control_array_get(struct fwnode_handle *fwnode, + enum reset_control_flags); int reset_control_get_count(struct device *dev); @@ -152,8 +155,8 @@ static inline int __device_reset(struct device *dev, bool optional) return optional ? 0 : -ENOTSUPP; } -static inline struct reset_control *__of_reset_control_get( - struct device_node *node, +static inline struct reset_control *__fwnode_reset_control_get( + struct fwnode_handle *fwnode, const char *id, int index, enum reset_control_flags flags) { bool optional = flags & RESET_CONTROL_FLAGS_BIT_OPTIONAL; @@ -242,7 +245,7 @@ devm_reset_control_array_get(struct device *dev, enum reset_control_flags flags) } static inline struct reset_control * -of_reset_control_array_get(struct device_node *np, enum reset_control_flags flags) +fwnode_reset_control_array_get(struct fwnode_handle *fwnode, enum reset_control_flags flags) { bool optional = flags & RESET_CONTROL_FLAGS_BIT_OPTIONAL; @@ -500,7 +503,8 @@ reset_control_bulk_get_optional_shared(struct device *dev, int num_rstcs, static inline struct reset_control *of_reset_control_get_exclusive( struct device_node *node, const char *id) { - return __of_reset_control_get(node, id, 0, RESET_CONTROL_EXCLUSIVE); + return __fwnode_reset_control_get(of_fwnode_handle(node), id, 0, + RESET_CONTROL_EXCLUSIVE); } /** @@ -520,7 +524,8 @@ static inline struct reset_control *of_reset_control_get_exclusive( static inline struct reset_control *of_reset_control_get_optional_exclusive( struct device_node *node, const char *id) { - return __of_reset_control_get(node, id, 0, RESET_CONTROL_OPTIONAL_EXCLUSIVE); + return __fwnode_reset_control_get(of_fwnode_handle(node), id, 0, + RESET_CONTROL_OPTIONAL_EXCLUSIVE); } /** @@ -545,7 +550,8 @@ static inline struct reset_control *of_reset_control_get_optional_exclusive( static inline struct reset_control *of_reset_control_get_shared( struct device_node *node, const char *id) { - return __of_reset_control_get(node, id, 0, RESET_CONTROL_SHARED); + return __fwnode_reset_control_get(of_fwnode_handle(node), id, 0, + RESET_CONTROL_SHARED); } /** @@ -562,7 +568,8 @@ static inline struct reset_control *of_reset_control_get_shared( static inline struct reset_control *of_reset_control_get_exclusive_by_index( struct device_node *node, int index) { - return __of_reset_control_get(node, NULL, index, RESET_CONTROL_EXCLUSIVE); + return __fwnode_reset_control_get(of_fwnode_handle(node), NULL, index, + RESET_CONTROL_EXCLUSIVE); } /** @@ -590,7 +597,8 @@ static inline struct reset_control *of_reset_control_get_exclusive_by_index( static inline struct reset_control *of_reset_control_get_shared_by_index( struct device_node *node, int index) { - return __of_reset_control_get(node, NULL, index, RESET_CONTROL_SHARED); + return __fwnode_reset_control_get(of_fwnode_handle(node), NULL, index, + RESET_CONTROL_SHARED); } /** @@ -1032,30 +1040,35 @@ devm_reset_control_array_get_optional_shared(struct device *dev) static inline struct reset_control * of_reset_control_array_get_exclusive(struct device_node *node) { - return of_reset_control_array_get(node, RESET_CONTROL_EXCLUSIVE); + return fwnode_reset_control_array_get(of_fwnode_handle(node), + RESET_CONTROL_EXCLUSIVE); } static inline struct reset_control * of_reset_control_array_get_exclusive_released(struct device_node *node) { - return of_reset_control_array_get(node, RESET_CONTROL_EXCLUSIVE_RELEASED); + return fwnode_reset_control_array_get(of_fwnode_handle(node), + RESET_CONTROL_EXCLUSIVE_RELEASED); } static inline struct reset_control * of_reset_control_array_get_shared(struct device_node *node) { - return of_reset_control_array_get(node, RESET_CONTROL_SHARED); + return fwnode_reset_control_array_get(of_fwnode_handle(node), + RESET_CONTROL_SHARED); } static inline struct reset_control * of_reset_control_array_get_optional_exclusive(struct device_node *node) { - return of_reset_control_array_get(node, RESET_CONTROL_OPTIONAL_EXCLUSIVE); + return fwnode_reset_control_array_get(of_fwnode_handle(node), + RESET_CONTROL_OPTIONAL_EXCLUSIVE); } static inline struct reset_control * of_reset_control_array_get_optional_shared(struct device_node *node) { - return of_reset_control_array_get(node, RESET_CONTROL_OPTIONAL_SHARED); + return fwnode_reset_control_array_get(of_fwnode_handle(node), + RESET_CONTROL_OPTIONAL_SHARED); } #endif diff --git a/include/linux/rfkill.h b/include/linux/rfkill.h index 6816e4c5f3f0..deea02a034c3 100644 --- a/include/linux/rfkill.h +++ b/include/linux/rfkill.h @@ -100,7 +100,8 @@ struct rfkill * __must_check rfkill_alloc(const char *name, int __must_check rfkill_register(struct rfkill *rfkill); /** - * rfkill_pause_polling(struct rfkill *rfkill) + * rfkill_pause_polling - Pause polling + * @rfkill: rfkill struct * * Pause polling -- say transmitter is off for other reasons. * NOTE: not necessary for suspend/resume -- in that case the @@ -110,9 +111,9 @@ int __must_check rfkill_register(struct rfkill *rfkill); void rfkill_pause_polling(struct rfkill *rfkill); /** - * rfkill_resume_polling(struct rfkill *rfkill) + * rfkill_resume_polling - Resume polling + * @rfkill: rfkill struct * - * Resume polling * NOTE: not necessary for suspend/resume -- in that case the * core stops polling anyway */ @@ -325,6 +326,8 @@ static inline enum rfkill_type rfkill_find_type(const char *name) #ifdef CONFIG_RFKILL_LEDS /** * rfkill_get_led_trigger_name - Get the LED trigger name for the button's LED. + * @rfkill: rfkill struct + * * This function might return a NULL pointer if registering of the * LED trigger failed. Use this as "default_trigger" for the LED. */ diff --git a/include/linux/rhashtable-types.h b/include/linux/rhashtable-types.h index 015c8298bebc..57c11ec9dc64 100644 --- a/include/linux/rhashtable-types.h +++ b/include/linux/rhashtable-types.h @@ -12,6 +12,7 @@ #include <linux/alloc_tag.h> #include <linux/atomic.h> #include <linux/compiler.h> +#include <linux/irq_work_types.h> #include <linux/mutex.h> #include <linux/workqueue_types.h> @@ -49,6 +50,7 @@ typedef int (*rht_obj_cmpfn_t)(struct rhashtable_compare_arg *arg, * @head_offset: Offset of rhash_head in struct to be hashed * @max_size: Maximum size while expanding * @min_size: Minimum size while shrinking + * @insecure_elasticity: Set to true to disable chain length checks * @automatic_shrinking: Enable automatic shrinking of tables * @hashfn: Hash function (default: jhash2 if !(key_len % 4), or jhash) * @obj_hashfn: Function to hash object @@ -61,6 +63,7 @@ struct rhashtable_params { u16 head_offset; unsigned int max_size; u16 min_size; + bool insecure_elasticity; bool automatic_shrinking; rht_hashfn_t hashfn; rht_obj_hashfn_t obj_hashfn; @@ -75,6 +78,7 @@ struct rhashtable_params { * @p: Configuration parameters * @rhlist: True if this is an rhltable * @run_work: Deferred worker to expand/shrink asynchronously + * @run_irq_work: Bounces the @run_work kick through hard IRQ context. * @mutex: Mutex to protect current/future table swapping * @lock: Spin lock to protect walker list * @nelems: Number of elements in table @@ -86,6 +90,7 @@ struct rhashtable { struct rhashtable_params p; bool rhlist; struct work_struct run_work; + struct irq_work run_irq_work; struct mutex mutex; spinlock_t lock; atomic_t nelems; @@ -131,12 +136,26 @@ struct rhashtable_iter { bool end_of_table; }; -int rhashtable_init_noprof(struct rhashtable *ht, - const struct rhashtable_params *params); +int __rhashtable_init_noprof(struct rhashtable *ht, + const struct rhashtable_params *params, + struct lock_class_key *key); +#define rhashtable_init_noprof(ht, params) \ +({ \ + static struct lock_class_key __key; \ + \ + __rhashtable_init_noprof(ht, params, &__key); \ +}) #define rhashtable_init(...) alloc_hooks(rhashtable_init_noprof(__VA_ARGS__)) -int rhltable_init_noprof(struct rhltable *hlt, - const struct rhashtable_params *params); +int __rhltable_init_noprof(struct rhltable *hlt, + const struct rhashtable_params *params, + struct lock_class_key *key); +#define rhltable_init_noprof(hlt, params) \ +({ \ + static struct lock_class_key __key; \ + \ + __rhltable_init_noprof(hlt, params, &__key); \ +}) #define rhltable_init(...) alloc_hooks(rhltable_init_noprof(__VA_ARGS__)) #endif /* _LINUX_RHASHTABLE_TYPES_H */ diff --git a/include/linux/rhashtable.h b/include/linux/rhashtable.h index 133ccb39137a..57a2a29bef0e 100644 --- a/include/linux/rhashtable.h +++ b/include/linux/rhashtable.h @@ -20,9 +20,9 @@ #include <linux/err.h> #include <linux/errno.h> +#include <linux/irq_work.h> #include <linux/jhash.h> #include <linux/list_nulls.h> -#include <linux/workqueue.h> #include <linux/rculist.h> #include <linux/bit_spinlock.h> @@ -129,10 +129,10 @@ static __always_inline unsigned int rht_key_get_hash(struct rhashtable *ht, unsigned int hash; /* params must be equal to ht->p if it isn't constant. */ - if (!__builtin_constant_p(params.key_len)) + if (!__builtin_constant_p(params.key_len)) { hash = ht->p.hashfn(key, ht->key_len, hash_rnd); - else if (params.key_len) { - unsigned int key_len = params.key_len; + } else { + unsigned int key_len = params.key_len ? : ht->p.key_len; if (params.hashfn) hash = params.hashfn(key, key_len, hash_rnd); @@ -140,13 +140,6 @@ static __always_inline unsigned int rht_key_get_hash(struct rhashtable *ht, hash = jhash(key, key_len, hash_rnd); else hash = jhash2(key, key_len / sizeof(u32), hash_rnd); - } else { - unsigned int key_len = ht->p.key_len; - - if (params.hashfn) - hash = params.hashfn(key, key_len, hash_rnd); - else - hash = jhash(key, key_len, hash_rnd); } return hash; @@ -269,6 +262,8 @@ struct rhash_lock_head __rcu **__rht_bucket_nested( struct rhash_lock_head __rcu **rht_bucket_nested_insert( struct rhashtable *ht, struct bucket_table *tbl, unsigned int hash); +void *rhashtable_next_key(struct rhashtable *ht, const void *prev_key); + #define rht_dereference(p, ht) \ rcu_dereference_protected(p, lockdep_rht_mutex_is_held(ht)) @@ -828,14 +823,15 @@ slow_path: goto out; } - if (elasticity <= 0) + if (elasticity <= 0 && !params.insecure_elasticity) goto slow_path; data = ERR_PTR(-E2BIG); if (unlikely(rht_grow_above_max(ht, tbl))) goto out_unlock; - if (unlikely(rht_grow_above_100(ht, tbl))) + if (unlikely(rht_grow_above_100(ht, tbl)) && + !params.insecure_elasticity) goto slow_path; /* Inserting at head of list makes unlocking free. */ @@ -853,7 +849,7 @@ slow_path: rht_assign_unlock(tbl, bkt, obj, flags); if (rht_grow_above_75(ht, tbl)) - schedule_work(&ht->run_work); + irq_work_queue(&ht->run_irq_work); data = NULL; out: @@ -1122,7 +1118,7 @@ unlocked: atomic_dec(&ht->nelems); if (unlikely(ht->p.automatic_shrinking && rht_shrink_below_30(ht, tbl))) - schedule_work(&ht->run_work); + irq_work_queue(&ht->run_irq_work); err = 0; } diff --git a/include/linux/ring_buffer.h b/include/linux/ring_buffer.h index d862fa610270..eac3e9080c3c 100644 --- a/include/linux/ring_buffer.h +++ b/include/linux/ring_buffer.h @@ -3,8 +3,9 @@ #define _LINUX_RING_BUFFER_H #include <linux/mm.h> -#include <linux/seq_file.h> #include <linux/poll.h> +#include <linux/ring_buffer_types.h> +#include <linux/seq_file.h> #include <uapi/linux/trace_mmap.h> @@ -218,14 +219,15 @@ bool ring_buffer_time_stamp_abs(struct trace_buffer *buffer); size_t ring_buffer_nr_dirty_pages(struct trace_buffer *buffer, int cpu); struct buffer_data_read_page; -struct buffer_data_read_page * -ring_buffer_alloc_read_page(struct trace_buffer *buffer, int cpu); +int ring_buffer_alloc_read_page(struct trace_buffer *buffer, int cpu, + struct buffer_data_read_page **rpage); void ring_buffer_free_read_page(struct trace_buffer *buffer, int cpu, struct buffer_data_read_page *page); int ring_buffer_read_page(struct trace_buffer *buffer, struct buffer_data_read_page *data_page, size_t len, int cpu, int full); void *ring_buffer_read_page_data(struct buffer_data_read_page *page); +unsigned int ring_buffer_read_page_size(struct buffer_data_read_page *rpage); struct trace_seq; @@ -238,6 +240,7 @@ int ring_buffer_subbuf_size_get(struct trace_buffer *buffer); enum ring_buffer_flags { RB_FL_OVERWRITE = 1 << 0, + RB_FL_TESTING = 1 << 1, }; #ifdef CONFIG_RING_BUFFER @@ -251,4 +254,76 @@ int ring_buffer_map(struct trace_buffer *buffer, int cpu, void ring_buffer_map_dup(struct trace_buffer *buffer, int cpu); int ring_buffer_unmap(struct trace_buffer *buffer, int cpu); int ring_buffer_map_get_reader(struct trace_buffer *buffer, int cpu); + +struct ring_buffer_desc { + int cpu; + unsigned int nr_page_va; /* excludes the meta page */ + unsigned long meta_va; + unsigned long page_va[] __counted_by(nr_page_va); +}; + +struct trace_buffer_desc { + int nr_cpus; + size_t struct_len; + char __data[]; /* list of ring_buffer_desc */ +}; + +static inline struct ring_buffer_desc *__next_ring_buffer_desc(struct ring_buffer_desc *desc) +{ + size_t len = struct_size(desc, page_va, desc->nr_page_va); + + return (struct ring_buffer_desc *)((void *)desc + len); +} + +static inline struct ring_buffer_desc *__first_ring_buffer_desc(struct trace_buffer_desc *desc) +{ + return (struct ring_buffer_desc *)(&desc->__data[0]); +} + +/* + * Returns the number of pages for a ring_buffer_desc. The caller must ensure it + * does not overflow ring_buffer_desc::nr_page_va. + */ +static inline unsigned long __calc_nr_pages_ring_buffer_desc(size_t size) +{ + /* Takes into account the reader page */ + return max(DIV_ROUND_UP(size, PAGE_SIZE - BUF_PAGE_HDR_SIZE), 2UL) + 1; +} + +static inline size_t trace_buffer_desc_size(size_t buffer_size, unsigned int nr_cpus) +{ + unsigned long nr_pages = __calc_nr_pages_ring_buffer_desc(buffer_size); + struct ring_buffer_desc *rbdesc; + + /* Capped by ring_buffer_desc::nr_page_va */ + if (nr_pages > UINT_MAX) + return SIZE_MAX; + + return size_add(offsetof(struct trace_buffer_desc, __data), + size_mul(nr_cpus, struct_size(rbdesc, page_va, nr_pages))); +} + +#define for_each_ring_buffer_desc(__pdesc, __cpu, __trace_pdesc) \ + for (__pdesc = __first_ring_buffer_desc(__trace_pdesc), __cpu = 0; \ + (__cpu) < (__trace_pdesc)->nr_cpus; \ + (__cpu)++, __pdesc = __next_ring_buffer_desc(__pdesc)) + +struct ring_buffer_remote { + struct trace_buffer_desc *desc; + int (*swap_reader_page)(unsigned int cpu, void *priv); + int (*reset)(unsigned int cpu, void *priv); + void *priv; +}; + +int ring_buffer_poll_remote(struct trace_buffer *buffer, int cpu); + +struct trace_buffer * +__ring_buffer_alloc_remote(struct ring_buffer_remote *remote, + struct lock_class_key *key); + +#define ring_buffer_alloc_remote(remote) \ +({ \ + static struct lock_class_key __key; \ + __ring_buffer_alloc_remote(remote, &__key); \ +}) #endif /* _LINUX_RING_BUFFER_H */ diff --git a/include/linux/ring_buffer_types.h b/include/linux/ring_buffer_types.h new file mode 100644 index 000000000000..54577021a49d --- /dev/null +++ b/include/linux/ring_buffer_types.h @@ -0,0 +1,41 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LINUX_RING_BUFFER_TYPES_H +#define _LINUX_RING_BUFFER_TYPES_H + +#include <asm/local.h> + +#define TS_SHIFT 27 +#define TS_MASK ((1ULL << TS_SHIFT) - 1) +#define TS_DELTA_TEST (~TS_MASK) + +/* + * We need to fit the time_stamp delta into 27 bits. + */ +static inline bool test_time_stamp(u64 delta) +{ + return !!(delta & TS_DELTA_TEST); +} + +#define BUF_PAGE_HDR_SIZE offsetof(struct buffer_data_page, data) + +#define RB_EVNT_HDR_SIZE (offsetof(struct ring_buffer_event, array)) +#define RB_ALIGNMENT 4U +#define RB_MAX_SMALL_DATA (RB_ALIGNMENT * RINGBUF_TYPE_DATA_TYPE_LEN_MAX) +#define RB_EVNT_MIN_SIZE 8U /* two 32bit words */ + +#ifndef CONFIG_HAVE_64BIT_ALIGNED_ACCESS +# define RB_FORCE_8BYTE_ALIGNMENT 0 +# define RB_ARCH_ALIGNMENT RB_ALIGNMENT +#else +# define RB_FORCE_8BYTE_ALIGNMENT 1 +# define RB_ARCH_ALIGNMENT 8U +#endif + +#define RB_ALIGN_DATA __aligned(RB_ARCH_ALIGNMENT) + +struct buffer_data_page { + u64 time_stamp; /* page time stamp */ + local_t commit; /* write committed index */ + unsigned char data[] RB_ALIGN_DATA; /* data of buffer page */ +}; +#endif diff --git a/include/linux/rio.h b/include/linux/rio.h index 2c29f21ba9e5..f42379775ce8 100644 --- a/include/linux/rio.h +++ b/include/linux/rio.h @@ -16,7 +16,7 @@ #include <linux/errno.h> #include <linux/device.h> #include <linux/rio_regs.h> -#include <linux/mod_devicetable.h> +#include <linux/device-id/rio.h> #ifdef CONFIG_RAPIDIO_DMA_ENGINE #include <linux/dmaengine.h> #endif diff --git a/include/linux/rmap.h b/include/linux/rmap.h index 8dc0871e5f00..0b332770abee 100644 --- a/include/linux/rmap.h +++ b/include/linux/rmap.h @@ -843,7 +843,7 @@ static inline int folio_try_share_anon_rmap_pmd(struct folio *folio, * Called from mm/vmscan.c to handle paging out */ int folio_referenced(struct folio *, int is_locked, - struct mem_cgroup *memcg, vm_flags_t *vm_flags); + struct mem_cgroup *memcg, vma_flags_t *vma_flags); void try_to_migrate(struct folio *folio, enum ttu_flags flags); void try_to_unmap(struct folio *, enum ttu_flags flags); @@ -864,13 +864,14 @@ struct page *make_device_exclusive(struct mm_struct *mm, unsigned long addr, struct page_vma_mapped_walk { unsigned long pfn; unsigned long nr_pages; - pgoff_t pgoff; + pgoff_t pgoff; /* Only meaningful if nr_pages > 1 and not a KSM walk */ struct vm_area_struct *vma; unsigned long address; pmd_t *pmd; pte_t *pte; spinlock_t *ptl; unsigned int flags; + bool pgoff_is_anon : 1; }; #define DEFINE_FOLIO_VMA_WALK(name, _folio, _vma, _address, _flags) \ @@ -881,6 +882,7 @@ struct page_vma_mapped_walk { .vma = _vma, \ .address = _address, \ .flags = _flags, \ + .pgoff_is_anon = folio_test_anon(_folio), \ } static inline void page_vma_mapped_walk_done(struct page_vma_mapped_walk *pvmw) @@ -975,10 +977,9 @@ struct anon_vma *folio_lock_anon_vma_read(const struct folio *folio, #define anon_vma_prepare(vma) (0) static inline int folio_referenced(struct folio *folio, int is_locked, - struct mem_cgroup *memcg, - vm_flags_t *vm_flags) + struct mem_cgroup *memcg, vma_flags_t *vma_flags) { - *vm_flags = 0; + vma_flags_clear_all(vma_flags); return 0; } diff --git a/include/linux/rolling_buffer.h b/include/linux/rolling_buffer.h index ac15b1ffdd83..a97f7cfaacaa 100644 --- a/include/linux/rolling_buffer.h +++ b/include/linux/rolling_buffer.h @@ -43,13 +43,13 @@ struct rolling_buffer_snapshot { #define ROLLBUF_MARK_2 BIT(1) int rolling_buffer_init(struct rolling_buffer *roll, unsigned int rreq_id, - unsigned int direction); -int rolling_buffer_make_space(struct rolling_buffer *roll); -ssize_t rolling_buffer_load_from_ra(struct rolling_buffer *roll, - struct readahead_control *ractl, - struct folio_batch *put_batch); + unsigned int direction, gfp_t gfp); +int rolling_buffer_make_space(struct rolling_buffer *roll, gfp_t gfp); +ssize_t rolling_buffer_bulk_load_from_ra(struct rolling_buffer *roll, + struct readahead_control *ractl, + unsigned int rreq_id, gfp_t gfp); ssize_t rolling_buffer_append(struct rolling_buffer *roll, struct folio *folio, - unsigned int flags); + unsigned int flags, gfp_t gfp); struct folio_queue *rolling_buffer_delete_spent(struct rolling_buffer *roll); void rolling_buffer_clear(struct rolling_buffer *roll); diff --git a/include/linux/rpmsg.h b/include/linux/rpmsg.h index fb7ab9165645..0171c490339c 100644 --- a/include/linux/rpmsg.h +++ b/include/linux/rpmsg.h @@ -13,7 +13,7 @@ #include <linux/types.h> #include <linux/device.h> #include <linux/err.h> -#include <linux/mod_devicetable.h> +#include <linux/device-id/rpmsg.h> #include <linux/kref.h> #include <linux/mutex.h> #include <linux/poll.h> @@ -41,9 +41,6 @@ struct rpmsg_channel_info { * rpmsg_device - device that belong to the rpmsg bus * @dev: the device struct * @id: device id (used to match between rpmsg drivers and devices) - * @driver_override: driver name to force a match; do not set directly, - * because core frees it; use driver_set_override() to - * set or clear it. * @src: local address * @dst: destination address * @ept: the rpmsg endpoint of this channel @@ -53,7 +50,6 @@ struct rpmsg_channel_info { struct rpmsg_device { struct device dev; struct rpmsg_device_id id; - const char *driver_override; u32 src; u32 dst; struct rpmsg_endpoint *ept; @@ -182,11 +178,11 @@ struct rpmsg_endpoint *rpmsg_create_ept(struct rpmsg_device *, rpmsg_rx_cb_t cb, void *priv, struct rpmsg_channel_info chinfo); -int rpmsg_send(struct rpmsg_endpoint *ept, void *data, int len); -int rpmsg_sendto(struct rpmsg_endpoint *ept, void *data, int len, u32 dst); +int rpmsg_send(struct rpmsg_endpoint *ept, const void *data, int len); +int rpmsg_sendto(struct rpmsg_endpoint *ept, const void *data, int len, u32 dst); -int rpmsg_trysend(struct rpmsg_endpoint *ept, void *data, int len); -int rpmsg_trysendto(struct rpmsg_endpoint *ept, void *data, int len, u32 dst); +int rpmsg_trysend(struct rpmsg_endpoint *ept, const void *data, int len); +int rpmsg_trysendto(struct rpmsg_endpoint *ept, const void *data, int len, u32 dst); __poll_t rpmsg_poll(struct rpmsg_endpoint *ept, struct file *filp, poll_table *wait); @@ -249,7 +245,7 @@ static inline struct rpmsg_endpoint *rpmsg_create_ept(struct rpmsg_device *rpdev return NULL; } -static inline int rpmsg_send(struct rpmsg_endpoint *ept, void *data, int len) +static inline int rpmsg_send(struct rpmsg_endpoint *ept, const void *data, int len) { /* This shouldn't be possible */ WARN_ON(1); @@ -257,7 +253,7 @@ static inline int rpmsg_send(struct rpmsg_endpoint *ept, void *data, int len) return -ENXIO; } -static inline int rpmsg_sendto(struct rpmsg_endpoint *ept, void *data, int len, +static inline int rpmsg_sendto(struct rpmsg_endpoint *ept, const void *data, int len, u32 dst) { /* This shouldn't be possible */ @@ -267,7 +263,8 @@ static inline int rpmsg_sendto(struct rpmsg_endpoint *ept, void *data, int len, } -static inline int rpmsg_trysend(struct rpmsg_endpoint *ept, void *data, int len) +static inline int rpmsg_trysend(struct rpmsg_endpoint *ept, const void *data, + int len) { /* This shouldn't be possible */ WARN_ON(1); @@ -275,7 +272,7 @@ static inline int rpmsg_trysend(struct rpmsg_endpoint *ept, void *data, int len) return -ENXIO; } -static inline int rpmsg_trysendto(struct rpmsg_endpoint *ept, void *data, +static inline int rpmsg_trysendto(struct rpmsg_endpoint *ept, const void *data, int len, u32 dst) { /* This shouldn't be possible */ diff --git a/include/linux/rpmsg/mtk_rpmsg.h b/include/linux/rpmsg/mtk_rpmsg.h index 363b60178040..badcbc89917f 100644 --- a/include/linux/rpmsg/mtk_rpmsg.h +++ b/include/linux/rpmsg/mtk_rpmsg.h @@ -25,7 +25,7 @@ struct mtk_rpmsg_info { ipi_handler_t handler, void *priv); void (*unregister_ipi)(struct platform_device *pdev, u32 id); int (*send_ipi)(struct platform_device *pdev, u32 id, - void *buf, unsigned int len, unsigned int wait); + const void *buf, unsigned int len, unsigned int wait); int ns_ipi_id; }; diff --git a/include/linux/rsc_table.h b/include/linux/rsc_table.h new file mode 100644 index 000000000000..71b60125310e --- /dev/null +++ b/include/linux/rsc_table.h @@ -0,0 +1,337 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ +/* + * Copyright(c) 2011 Texas Instruments, Inc. + * Copyright(c) 2011 Google, Inc. + * All rights reserved. + */ + +#ifndef RSC_TABLE_H +#define RSC_TABLE_H + +/** + * struct resource_table - firmware resource table header + * @ver: version number + * @num: number of resource entries + * @reserved: reserved (must be zero) + * @offset: array of offsets pointing at the various resource entries + * + * A resource table is essentially a list of system resources required + * by the remote processor. It may also include configuration entries. + * If needed, the remote processor firmware should contain this table + * as a dedicated ".resource_table" ELF section. + * + * Some resources entries are mere announcements, where the host is informed + * of specific remoteproc configuration. Other entries require the host to + * do something (e.g. allocate a system resource). Sometimes a negotiation + * is expected, where the firmware requests a resource, and once allocated, + * the host should provide back its details (e.g. address of an allocated + * memory region). + * + * The header of the resource table, as expressed by this structure, + * contains a version number (should we need to change this format in the + * future), the number of available resource entries, and their offsets + * in the table. + * + * Immediately following this header are the resource entries themselves, + * each of which begins with a resource entry header (as described below). + */ +struct resource_table { + u32 ver; + u32 num; + u32 reserved[2]; + u32 offset[]; +} __packed; + +/** + * struct fw_rsc_hdr - firmware resource entry header + * @type: resource type + * @data: resource data + * + * Every resource entry begins with a 'struct fw_rsc_hdr' header providing + * its @type. The content of the entry itself will immediately follow + * this header, and it should be parsed according to the resource type. + */ +struct fw_rsc_hdr { + u32 type; + u8 data[]; +} __packed; + +/** + * enum fw_resource_type - types of resource entries + * + * @RSC_CARVEOUT: request for allocation of a physically contiguous + * memory region. + * @RSC_DEVMEM: request to iommu_map a memory-based peripheral. + * @RSC_TRACE: announces the availability of a trace buffer into which + * the remote processor will be writing logs. + * @RSC_VDEV: declare support for a virtio device, and serve as its + * virtio header. + * @RSC_LAST: just keep this one at the end of standard resources + * @RSC_VENDOR_START: start of the vendor specific resource types range + * @RSC_VENDOR_END: end of the vendor specific resource types range + * + * For more details regarding a specific resource type, please see its + * dedicated structure below. + * + * Please note that these values are used as indices to the rproc_handle_rsc + * lookup table, so please keep them sane. Moreover, @RSC_LAST is used to + * check the validity of an index before the lookup table is accessed, so + * please update it as needed. + */ +enum fw_resource_type { + RSC_CARVEOUT = 0, + RSC_DEVMEM = 1, + RSC_TRACE = 2, + RSC_VDEV = 3, + RSC_LAST = 4, + RSC_VENDOR_START = 128, + RSC_VENDOR_END = 512, +}; + +#define FW_RSC_ADDR_ANY (-1) + +/** + * struct fw_rsc_carveout - physically contiguous memory request + * @da: device address + * @pa: physical address + * @len: length (in bytes) + * @flags: iommu protection flags + * @reserved: reserved (must be zero) + * @name: human-readable name of the requested memory region + * + * This resource entry requests the host to allocate a physically contiguous + * memory region. + * + * These request entries should precede other firmware resource entries, + * as other entries might request placing other data objects inside + * these memory regions (e.g. data/code segments, trace resource entries, ...). + * + * Allocating memory this way helps utilizing the reserved physical memory + * (e.g. CMA) more efficiently, and also minimizes the number of TLB entries + * needed to map it (in case @rproc is using an IOMMU). Reducing the TLB + * pressure is important; it may have a substantial impact on performance. + * + * If the firmware is compiled with static addresses, then @da should specify + * the expected device address of this memory region. If @da is set to + * FW_RSC_ADDR_ANY, then the host will dynamically allocate it, and then + * overwrite @da with the dynamically allocated address. + * + * We will always use @da to negotiate the device addresses, even if it + * isn't using an iommu. In that case, though, it will obviously contain + * physical addresses. + * + * Some remote processors needs to know the allocated physical address + * even if they do use an iommu. This is needed, e.g., if they control + * hardware accelerators which access the physical memory directly (this + * is the case with OMAP4 for instance). In that case, the host will + * overwrite @pa with the dynamically allocated physical address. + * Generally we don't want to expose physical addresses if we don't have to + * (remote processors are generally _not_ trusted), so we might want to + * change this to happen _only_ when explicitly required by the hardware. + * + * @flags is used to provide IOMMU protection flags, and @name should + * (optionally) contain a human readable name of this carveout region + * (mainly for debugging purposes). + */ +struct fw_rsc_carveout { + u32 da; + u32 pa; + u32 len; + u32 flags; + u32 reserved; + u8 name[32]; +} __packed; + +/** + * struct fw_rsc_devmem - iommu mapping request + * @da: device address + * @pa: physical address + * @len: length (in bytes) + * @flags: iommu protection flags + * @reserved: reserved (must be zero) + * @name: human-readable name of the requested region to be mapped + * + * This resource entry requests the host to iommu map a physically contiguous + * memory region. This is needed in case the remote processor requires + * access to certain memory-based peripherals; _never_ use it to access + * regular memory. + * + * This is obviously only needed if the remote processor is accessing memory + * via an iommu. + * + * @da should specify the required device address, @pa should specify + * the physical address we want to map, @len should specify the size of + * the mapping and @flags is the IOMMU protection flags. As always, @name may + * (optionally) contain a human readable name of this mapping (mainly for + * debugging purposes). + * + * Note: at this point we just "trust" those devmem entries to contain valid + * physical addresses, but this isn't safe and will be changed: eventually we + * want remoteproc implementations to provide us ranges of physical addresses + * the firmware is allowed to request, and not allow firmwares to request + * access to physical addresses that are outside those ranges. + */ +struct fw_rsc_devmem { + u32 da; + u32 pa; + u32 len; + u32 flags; + u32 reserved; + u8 name[32]; +} __packed; + +/** + * struct fw_rsc_trace - trace buffer declaration + * @da: device address + * @len: length (in bytes) + * @reserved: reserved (must be zero) + * @name: human-readable name of the trace buffer + * + * This resource entry provides the host information about a trace buffer + * into which the remote processor will write log messages. + * + * @da specifies the device address of the buffer, @len specifies + * its size, and @name may contain a human readable name of the trace buffer. + * + * After booting the remote processor, the trace buffers are exposed to the + * user via debugfs entries (called trace0, trace1, etc..). + */ +struct fw_rsc_trace { + u32 da; + u32 len; + u32 reserved; + u8 name[32]; +} __packed; + +/** + * struct fw_rsc_vdev_vring - vring descriptor entry + * @da: device address + * @align: the alignment between the consumer and producer parts of the vring + * @num: num of buffers supported by this vring (must be power of two) + * @notifyid: a unique rproc-wide notify index for this vring. This notify + * index is used when kicking a remote processor, to let it know that this + * vring is triggered. + * @pa: physical address + * + * This descriptor is not a resource entry by itself; it is part of the + * vdev resource type (see below). + * + * Note that @da should either contain the device address where + * the remote processor is expecting the vring, or indicate that + * dynamically allocation of the vring's device address is supported. + */ +struct fw_rsc_vdev_vring { + u32 da; + u32 align; + u32 num; + u32 notifyid; + u32 pa; +} __packed; + +/** + * struct fw_rsc_vdev - virtio device header + * @id: virtio device id (as in virtio_ids.h) + * @notifyid: a unique rproc-wide notify index for this vdev. This notify + * index is used when kicking a remote processor, to let it know that the + * status/features of this vdev have changes. + * @dfeatures: specifies the virtio device features supported by the firmware + * @gfeatures: a place holder used by the host to write back the + * negotiated features that are supported by both sides. + * @config_len: the size of the virtio config space of this vdev. The config + * space lies in the resource table immediate after this vdev header. + * @status: a place holder where the host will indicate its virtio progress. + * @num_of_vrings: indicates how many vrings are described in this vdev header + * @reserved: reserved (must be zero) + * @vring: an array of @num_of_vrings entries of 'struct fw_rsc_vdev_vring'. + * + * This resource is a virtio device header: it provides information about + * the vdev, and is then used by the host and its peer remote processors + * to negotiate and share certain virtio properties. + * + * By providing this resource entry, the firmware essentially asks remoteproc + * to statically allocate a vdev upon registration of the rproc (dynamic vdev + * allocation is not yet supported). + * + * Note: + * 1. unlike virtualization systems, the term 'host' here means + * the Linux side which is running remoteproc to control the remote + * processors. We use the name 'gfeatures' to comply with virtio's terms, + * though there isn't really any virtualized guest OS here: it's the host + * which is responsible for negotiating the final features. + * Yeah, it's a bit confusing. + * + * 2. immediately following this structure is the virtio config space for + * this vdev (which is specific to the vdev; for more info, read the virtio + * spec). The size of the config space is specified by @config_len. + */ +struct fw_rsc_vdev { + u32 id; + u32 notifyid; + u32 dfeatures; + u32 gfeatures; + u32 config_len; + u8 status; + u8 num_of_vrings; + u8 reserved[2]; + struct fw_rsc_vdev_vring vring[]; +} __packed; + +/** + * rsc_table_for_each_entry() - iterate over all entries in a resource table + * @table: pointer to the resource table + * @table_sz: total size of the table buffer in bytes + * @dev: device used for error logging + * @cb: callback invoked for each entry: + * @type - value from enum fw_resource_type + * @rsc - pointer to the entry payload (past struct fw_rsc_hdr) + * @offset - byte offset of the payload within the table; callers + * that write back into the table (e.g. to record a + * dynamically allocated address) use this to locate the + * entry for later update + * @avail - bytes available in the payload + * @data - caller-supplied private pointer + * Return 0 to continue iteration, non-zero to stop. + * @data: private pointer forwarded to @cb on every call + * + * Iterates over every resource entry in @table, performing the standard + * truncation check, and invokes @cb for each one. Iteration stops on the + * first non-zero return from @cb or on a malformed table. + * + * Returns 0 after a complete iteration, -EINVAL if the table is truncated, + * or the first non-zero value returned by @cb. + */ +static inline int rsc_table_for_each_entry(struct resource_table *table, + size_t table_sz, + struct device *dev, + int (*cb)(u32 type, void *rsc, + int offset, int avail, + void *data), + void *data) { + int i, ret; + + for (i = 0; i < table->num; i++) { + u32 offset = table->offset[i]; + struct fw_rsc_hdr *hdr; + int avail, rsc_offset; + void *rsc; + + if (offset < sizeof(*table) || offset >= table_sz || + table_sz - offset < sizeof(*hdr)) { + dev_err(dev, "rsc table is truncated\n"); + return -EINVAL; + } + + hdr = (void *)table + offset; + avail = table_sz - offset - sizeof(*hdr); + rsc_offset = offset + sizeof(*hdr); + rsc = (void *)hdr + sizeof(*hdr); + + ret = cb(hdr->type, rsc, rsc_offset, avail, data); + if (ret) + return ret; + } + + return 0; +} + +#endif /* RSC_TABLE_H */ diff --git a/include/linux/rseq.h b/include/linux/rseq.h index b9d62fc2140d..7ef79b25e714 100644 --- a/include/linux/rseq.h +++ b/include/linux/rseq.h @@ -9,6 +9,11 @@ void __rseq_handle_slowpath(struct pt_regs *regs); +static __always_inline bool rseq_v2(struct task_struct *t) +{ + return IS_ENABLED(CONFIG_GENERIC_IRQ_ENTRY) && likely(t->rseq.event.has_rseq > 1); +} + /* Invoked from resume_user_mode_work() */ static inline void rseq_handle_slowpath(struct pt_regs *regs) { @@ -16,8 +21,7 @@ static inline void rseq_handle_slowpath(struct pt_regs *regs) if (current->rseq.event.slowpath) __rseq_handle_slowpath(regs); } else { - /* '&' is intentional to spare one conditional branch */ - if (current->rseq.event.sched_switch & current->rseq.event.has_rseq) + if (current->rseq.event.sched_switch && current->rseq.event.has_rseq) __rseq_handle_slowpath(regs); } } @@ -30,9 +34,9 @@ void __rseq_signal_deliver(int sig, struct pt_regs *regs); */ static inline void rseq_signal_deliver(struct ksignal *ksig, struct pt_regs *regs) { - if (IS_ENABLED(CONFIG_GENERIC_IRQ_ENTRY)) { - /* '&' is intentional to spare one conditional branch */ - if (current->rseq.event.has_rseq & current->rseq.event.user_irq) + if (rseq_v2(current)) { + /* has_rseq is implied in rseq_v2() */ + if (current->rseq.event.user_irq) __rseq_signal_deliver(ksig->sig, regs); } else { if (current->rseq.event.has_rseq) @@ -50,15 +54,22 @@ static __always_inline void rseq_sched_switch_event(struct task_struct *t) { struct rseq_event *ev = &t->rseq.event; - if (IS_ENABLED(CONFIG_GENERIC_IRQ_ENTRY)) { + /* + * Only apply the user_irq optimization for RSEQ ABI V2 registrations. + * Legacy users like TCMalloc rely on the original ABI V1 behaviour + * which updates IDs on every context swtich. + */ + if (rseq_v2(t)) { /* - * Avoid a boat load of conditionals by using simple logic - * to determine whether NOTIFY_RESUME needs to be raised. + * Avoid a boat load of conditionals by using simple logic to + * determine whether TIF_NOTIFY_RESUME or TIF_RSEQ needs to be + * raised. * - * It's required when the CPU or MM CID has changed or - * the entry was from user space. + * It's required when the CPU or MM CID has changed or the entry + * was via interrupt from user space. ev->has_rseq does not have + * to be evaluated here because rseq_v2() implies has_rseq. */ - bool raise = (ev->user_irq | ev->ids_changed) & ev->has_rseq; + bool raise = ev->user_irq | ev->ids_changed; if (raise) { ev->sched_switch = true; @@ -66,6 +77,7 @@ static __always_inline void rseq_sched_switch_event(struct task_struct *t) } } else { if (ev->has_rseq) { + t->rseq.event.ids_changed = true; t->rseq.event.sched_switch = true; rseq_raise_notify_resume(t); } @@ -119,6 +131,8 @@ static inline void rseq_virt_userspace_exit(void) static inline void rseq_reset(struct task_struct *t) { + /* Protect against preemption and membarrier IPI */ + guard(irqsave)(); memset(&t->rseq, 0, sizeof(t->rseq)); t->rseq.ids.cpu_id = RSEQ_CPU_ID_UNINITIALIZED; } @@ -159,6 +173,7 @@ static inline unsigned int rseq_alloc_align(void) } #else /* CONFIG_RSEQ */ +static inline bool rseq_v2(struct task_struct *t) { return false; } static inline void rseq_handle_slowpath(struct pt_regs *regs) { } static inline void rseq_signal_deliver(struct ksignal *ksig, struct pt_regs *regs) { } static inline void rseq_sched_switch_event(struct task_struct *t) { } diff --git a/include/linux/rseq_entry.h b/include/linux/rseq_entry.h index c6831c93cd6e..31ce349ed42c 100644 --- a/include/linux/rseq_entry.h +++ b/include/linux/rseq_entry.h @@ -40,6 +40,7 @@ DECLARE_PER_CPU(struct rseq_stats, rseq_stats); #endif /* !CONFIG_RSEQ_STATS */ #ifdef CONFIG_RSEQ +#include <linux/hrtimer_rearm.h> #include <linux/jump_label.h> #include <linux/rseq.h> #include <linux/sched/signal.h> @@ -110,7 +111,21 @@ static __always_inline void rseq_slice_clear_grant(struct task_struct *t) t->rseq.slice.state.granted = false; } -static __always_inline bool rseq_grant_slice_extension(bool work_pending) +/* + * Open coded, so it can be invoked within a user access region. + * + * This clears the user space state of the time slice extensions field only when + * the task has registered the optimized RSEQ_ABI V2. Some legacy registrations, + * e.g. TCMalloc, have conflicting non-ABI fields in struct RSEQ, which would be + * overwritten by an unconditional write. + */ +#define rseq_slice_clear_user(rseq, efault) \ +do { \ + if (rseq_slice_extension_enabled()) \ + unsafe_put_user(0U, &rseq->slice_ctrl.all, efault); \ +} while (0) + +static __always_inline bool __rseq_grant_slice_extension(bool work_pending) { struct task_struct *curr = current; struct rseq_slice_ctrl usr_ctrl; @@ -215,15 +230,25 @@ efault: return false; } +static __always_inline bool rseq_grant_slice_extension(unsigned long ti_work, unsigned long mask) +{ + if (unlikely(__rseq_grant_slice_extension(ti_work & mask))) { + guard(irq)(); + hrtimer_rearm_deferred_tif(ti_work); + return true; + } + return false; +} + #else /* CONFIG_RSEQ_SLICE_EXTENSION */ static __always_inline bool rseq_slice_extension_enabled(void) { return false; } static __always_inline bool rseq_arm_slice_extension_timer(void) { return false; } static __always_inline void rseq_slice_clear_grant(struct task_struct *t) { } -static __always_inline bool rseq_grant_slice_extension(bool work_pending) { return false; } +static __always_inline bool rseq_grant_slice_extension(unsigned long ti_work, unsigned long mask) { return false; } +#define rseq_slice_clear_user(rseq, efault) do { } while (0) #endif /* !CONFIG_RSEQ_SLICE_EXTENSION */ bool rseq_debug_update_user_cs(struct task_struct *t, struct pt_regs *regs, unsigned long csaddr); -bool rseq_debug_validate_ids(struct task_struct *t); static __always_inline void rseq_note_user_irq_entry(void) { @@ -343,43 +368,6 @@ efault: return false; } -/* - * On debug kernels validate that user space did not mess with it if the - * debug branch is enabled. - */ -bool rseq_debug_validate_ids(struct task_struct *t) -{ - struct rseq __user *rseq = t->rseq.usrptr; - u32 cpu_id, uval, node_id; - - /* - * On the first exit after registering the rseq region CPU ID is - * RSEQ_CPU_ID_UNINITIALIZED and node_id in user space is 0! - */ - node_id = t->rseq.ids.cpu_id != RSEQ_CPU_ID_UNINITIALIZED ? - cpu_to_node(t->rseq.ids.cpu_id) : 0; - - scoped_user_read_access(rseq, efault) { - unsafe_get_user(cpu_id, &rseq->cpu_id_start, efault); - if (cpu_id != t->rseq.ids.cpu_id) - goto die; - unsafe_get_user(uval, &rseq->cpu_id, efault); - if (uval != cpu_id) - goto die; - unsafe_get_user(uval, &rseq->node_id, efault); - if (uval != node_id) - goto die; - unsafe_get_user(uval, &rseq->mm_cid, efault); - if (uval != t->rseq.ids.mm_cid) - goto die; - } - return true; -die: - t->rseq.event.fatal = true; -efault: - return false; -} - #endif /* RSEQ_BUILD_SLOW_PATH */ /* @@ -489,37 +477,50 @@ efault: * faults in task context are fatal too. */ static rseq_inline -bool rseq_set_ids_get_csaddr(struct task_struct *t, struct rseq_ids *ids, - u32 node_id, u64 *csaddr) +bool rseq_set_ids_get_csaddr(struct task_struct *t, struct rseq_ids *ids, u64 *csaddr) { struct rseq __user *rseq = t->rseq.usrptr; - if (static_branch_unlikely(&rseq_debug_enabled)) { - if (!rseq_debug_validate_ids(t)) - return false; - } - scoped_user_rw_access(rseq, efault) { + /* Validate the R/O fields for debug and optimized mode */ + if (static_branch_unlikely(&rseq_debug_enabled) || rseq_v2(t)) { + u32 cpu_id, uval; + + unsafe_get_user(cpu_id, &rseq->cpu_id_start, efault); + if (cpu_id != t->rseq.ids.cpu_id) + goto die; + unsafe_get_user(uval, &rseq->cpu_id, efault); + if (uval != cpu_id) + goto die; + unsafe_get_user(uval, &rseq->node_id, efault); + if (uval != t->rseq.ids.node_id) + goto die; + unsafe_get_user(uval, &rseq->mm_cid, efault); + if (uval != t->rseq.ids.mm_cid) + goto die; + } + unsafe_put_user(ids->cpu_id, &rseq->cpu_id_start, efault); unsafe_put_user(ids->cpu_id, &rseq->cpu_id, efault); - unsafe_put_user(node_id, &rseq->node_id, efault); + unsafe_put_user(ids->node_id, &rseq->node_id, efault); unsafe_put_user(ids->mm_cid, &rseq->mm_cid, efault); if (csaddr) unsafe_get_user(*csaddr, &rseq->rseq_cs, efault); - /* Open coded, so it's in the same user access region */ - if (rseq_slice_extension_enabled()) { - /* Unconditionally clear it, no point in conditionals */ - unsafe_put_user(0U, &rseq->slice_ctrl.all, efault); - } + /* RSEQ ABI V2 only operations */ + if (rseq_v2(t)) + rseq_slice_clear_user(rseq, efault); } rseq_slice_clear_grant(t); /* Cache the new values */ - t->rseq.ids.cpu_cid = ids->cpu_cid; + t->rseq.ids = *ids; rseq_stat_inc(rseq_stats.ids); rseq_trace_update(t, ids); return true; + +die: + t->rseq.event.fatal = true; efault: return false; } @@ -529,11 +530,11 @@ efault: * is in a critical section. */ static rseq_inline bool rseq_update_usr(struct task_struct *t, struct pt_regs *regs, - struct rseq_ids *ids, u32 node_id) + struct rseq_ids *ids) { u64 csaddr; - if (!rseq_set_ids_get_csaddr(t, ids, node_id, &csaddr)) + if (!rseq_set_ids_get_csaddr(t, ids, &csaddr)) return false; /* @@ -602,6 +603,14 @@ static __always_inline bool rseq_exit_user_update(struct pt_regs *regs, struct t * interrupts disabled */ guard(pagefault)(); + /* + * This optimization is only valid when the task registered for the + * optimized RSEQ_ABI_V2 variant. Some legacy users rely on the original + * RSEQ implementation behaviour which unconditionally updated the IDs. + * rseq_sched_switch_event() ensures that legacy registrations always + * have both sched_switch and ids_changed set, which is compatible with + * the historical TIF_NOTIFY_RESUME behaviour. + */ if (likely(!t->rseq.event.ids_changed)) { struct rseq __user *rseq = t->rseq.usrptr; /* @@ -613,11 +622,9 @@ static __always_inline bool rseq_exit_user_update(struct pt_regs *regs, struct t scoped_user_rw_access(rseq, efault) { unsafe_get_user(csaddr, &rseq->rseq_cs, efault); - /* Open coded, so it's in the same user access region */ - if (rseq_slice_extension_enabled()) { - /* Unconditionally clear it, no point in conditionals */ - unsafe_put_user(0U, &rseq->slice_ctrl.all, efault); - } + /* RSEQ ABI V2 only operations */ + if (rseq_v2(t)) + rseq_slice_clear_user(rseq, efault); } rseq_slice_clear_grant(t); @@ -629,13 +636,14 @@ static __always_inline bool rseq_exit_user_update(struct pt_regs *regs, struct t return true; } + int cpu = task_cpu(t); struct rseq_ids ids = { - .cpu_id = task_cpu(t), - .mm_cid = task_mm_cid(t), + .cpu_id = cpu, + .mm_cid = task_mm_cid(t), + .node_id = cpu_to_node(cpu), }; - u32 node_id = cpu_to_node(ids.cpu_id); - return rseq_update_usr(t, regs, &ids, node_id); + return rseq_update_usr(t, regs, &ids); efault: return false; } @@ -743,24 +751,6 @@ static __always_inline void rseq_irqentry_exit_to_user_mode(void) ev->events = 0; } -/* Required to keep ARM64 working */ -static __always_inline void rseq_exit_to_user_mode_legacy(void) -{ - struct rseq_event *ev = ¤t->rseq.event; - - rseq_stat_inc(rseq_stats.exit); - - if (static_branch_unlikely(&rseq_debug_enabled)) - WARN_ON_ONCE(ev->sched_switch); - - /* - * Ensure that event (especially user_irq) is cleared when the - * interrupt did not result in a schedule and therefore the - * rseq processing did not clear it. - */ - ev->events = 0; -} - void __rseq_debug_syscall_return(struct pt_regs *regs); static __always_inline void rseq_debug_syscall_return(struct pt_regs *regs) @@ -776,9 +766,8 @@ static inline bool rseq_exit_to_user_mode_restart(struct pt_regs *regs, unsigned } static inline void rseq_syscall_exit_to_user_mode(void) { } static inline void rseq_irqentry_exit_to_user_mode(void) { } -static inline void rseq_exit_to_user_mode_legacy(void) { } static inline void rseq_debug_syscall_return(struct pt_regs *regs) { } -static inline bool rseq_grant_slice_extension(bool work_pending) { return false; } +static inline bool rseq_grant_slice_extension(unsigned long ti_work, unsigned long mask) { return false; } #endif /* !CONFIG_RSEQ */ #endif /* _LINUX_RSEQ_ENTRY_H */ diff --git a/include/linux/rseq_types.h b/include/linux/rseq_types.h index 0b42045988db..85739a63e85e 100644 --- a/include/linux/rseq_types.h +++ b/include/linux/rseq_types.h @@ -9,6 +9,12 @@ #ifdef CONFIG_RSEQ struct rseq; +/* + * rseq_event::has_rseq contains the ABI version number so preserving it + * in AND operations requires a mask. + */ +#define RSEQ_HAS_RSEQ_VERSION_MASK 0xff + /** * struct rseq_event - Storage for rseq related event management * @all: Compound to initialize and clear the data efficiently @@ -17,7 +23,8 @@ struct rseq; * exit to user * @ids_changed: Indicator that IDs need to be updated * @user_irq: True on interrupt entry from user mode - * @has_rseq: True if the task has a rseq pointer installed + * @has_rseq: Greater than 0 if the task has a rseq pointer installed. + * Contains the RSEQ version number * @error: Compound error code for the slow path to analyze * @fatal: User space data corrupted or invalid * @slowpath: Indicator that slow path processing via TIF_NOTIFY_RESUME @@ -59,8 +66,9 @@ struct rseq_event { * compiler emit a single compare on 64-bit * @cpu_id: The CPU ID which was written last to user space * @mm_cid: The MM CID which was written last to user space + * @node_id: The node ID which was written last to user space * - * @cpu_id and @mm_cid are updated when the data is written to user space. + * @cpu_id, @mm_cid and @node_id are updated when the data is written to user space. */ struct rseq_ids { union { @@ -70,6 +78,7 @@ struct rseq_ids { u32 mm_cid; }; }; + u32 node_id; }; /** diff --git a/include/linux/rtc.h b/include/linux/rtc.h index 95da051fb155..c09fc22819d0 100644 --- a/include/linux/rtc.h +++ b/include/linux/rtc.h @@ -190,6 +190,8 @@ extern int rtc_set_time(struct rtc_device *rtc, struct rtc_time *tm); int __rtc_read_alarm(struct rtc_device *rtc, struct rtc_wkalrm *alarm); extern int rtc_read_alarm(struct rtc_device *rtc, struct rtc_wkalrm *alrm); +extern int rtc_read_next_alarm(struct rtc_device *rtc, + struct rtc_wkalrm *alrm); extern int rtc_set_alarm(struct rtc_device *rtc, struct rtc_wkalrm *alrm); extern int rtc_initialize_alarm(struct rtc_device *rtc, diff --git a/include/linux/rtmutex.h b/include/linux/rtmutex.h index ede4c6bf6f22..9e1f012f89db 100644 --- a/include/linux/rtmutex.h +++ b/include/linux/rtmutex.h @@ -22,8 +22,8 @@ extern int max_lock_depth; struct rt_mutex_base { raw_spinlock_t wait_lock; - struct rb_root_cached waiters; - struct task_struct *owner; + struct rb_root_cached waiters __guarded_by(&wait_lock); + struct task_struct *owner __guarded_by(&wait_lock); }; #define __RT_MUTEX_BASE_INITIALIZER(rtbasename) \ @@ -41,7 +41,7 @@ struct rt_mutex_base { */ static inline bool rt_mutex_base_is_locked(struct rt_mutex_base *lock) { - return READ_ONCE(lock->owner) != NULL; + return data_race(READ_ONCE(lock->owner) != NULL); } #ifdef CONFIG_RT_MUTEXES @@ -49,13 +49,15 @@ static inline bool rt_mutex_base_is_locked(struct rt_mutex_base *lock) static inline struct task_struct *rt_mutex_owner(struct rt_mutex_base *lock) { - unsigned long owner = (unsigned long) READ_ONCE(lock->owner); + unsigned long owner = (unsigned long) data_race(READ_ONCE(lock->owner)); return (struct task_struct *) (owner & ~RT_MUTEX_HAS_WAITERS); } #endif extern void rt_mutex_base_init(struct rt_mutex_base *rtb); +context_lock_struct(rt_mutex); + /** * The rt_mutex structure * @@ -108,8 +110,10 @@ do { \ extern void __rt_mutex_init(struct rt_mutex *lock, const char *name, struct lock_class_key *key); #ifdef CONFIG_DEBUG_LOCK_ALLOC -extern void rt_mutex_lock_nested(struct rt_mutex *lock, unsigned int subclass); -extern void _rt_mutex_lock_nest_lock(struct rt_mutex *lock, struct lockdep_map *nest_lock); +extern void rt_mutex_lock_nested(struct rt_mutex *lock, unsigned int subclass) + __acquires(lock); +extern void _rt_mutex_lock_nest_lock(struct rt_mutex *lock, struct lockdep_map *nest_lock) + __acquires(lock); #define rt_mutex_lock(lock) rt_mutex_lock_nested(lock, 0) #define rt_mutex_lock_nest_lock(lock, nest_lock) \ do { \ @@ -118,15 +122,19 @@ extern void _rt_mutex_lock_nest_lock(struct rt_mutex *lock, struct lockdep_map * } while (0) #else -extern void rt_mutex_lock(struct rt_mutex *lock); +extern void rt_mutex_lock(struct rt_mutex *lock) __acquires(lock); #define rt_mutex_lock_nested(lock, subclass) rt_mutex_lock(lock) #define rt_mutex_lock_nest_lock(lock, nest_lock) rt_mutex_lock(lock) #endif -extern int rt_mutex_lock_interruptible(struct rt_mutex *lock); -extern int rt_mutex_lock_killable(struct rt_mutex *lock); -extern int rt_mutex_trylock(struct rt_mutex *lock); +extern int rt_mutex_lock_interruptible(struct rt_mutex *lock) + __cond_acquires(0, lock); +extern int rt_mutex_lock_killable(struct rt_mutex *lock) + __cond_acquires(0, lock); +extern int rt_mutex_trylock(struct rt_mutex *lock) + __cond_acquires(true, lock); -extern void rt_mutex_unlock(struct rt_mutex *lock); +extern void rt_mutex_unlock(struct rt_mutex *lock) + __releases(lock); #endif diff --git a/include/linux/rtnetlink.h b/include/linux/rtnetlink.h index ea39dd23a197..95729339e7a5 100644 --- a/include/linux/rtnetlink.h +++ b/include/linux/rtnetlink.h @@ -115,6 +115,10 @@ bool rtnl_net_is_locked(struct net *net); bool lockdep_rtnl_net_is_held(struct net *net); +void rtnl_net_queue_work(struct net *net); +void rtnl_net_flush_workqueue(void); +void rtnl_net_work_func(struct work_struct *work); + #define rcu_dereference_rtnl_net(net, p) \ rcu_dereference_check(p, lockdep_rtnl_net_is_held(net)) #define rtnl_net_dereference(net, p) \ @@ -150,6 +154,10 @@ static inline void ASSERT_RTNL_NET(struct net *net) ASSERT_RTNL(); } +static inline void rtnl_net_flush_workqueue(void) +{ +} + #define rcu_dereference_rtnl_net(net, p) \ rcu_dereference_rtnl(p) #define rtnl_net_dereference(net, p) \ diff --git a/include/linux/rtsx_usb.h b/include/linux/rtsx_usb.h index 276b509c03e3..0fc5a74700a8 100644 --- a/include/linux/rtsx_usb.h +++ b/include/linux/rtsx_usb.h @@ -61,6 +61,9 @@ struct rtsx_ucr { struct timer_list sg_timer; struct mutex dev_mutex; + + u16 card_status_cache; + bool card_status_valid; }; /* buffer size */ diff --git a/include/linux/rv.h b/include/linux/rv.h index 58774eb3aecf..541ba404926a 100644 --- a/include/linux/rv.h +++ b/include/linux/rv.h @@ -13,6 +13,7 @@ #define RV_MON_GLOBAL 0 #define RV_MON_PER_CPU 1 #define RV_MON_PER_TASK 2 +#define RV_MON_PER_OBJ 3 #ifdef CONFIG_RV #include <linux/array_size.h> @@ -81,11 +82,49 @@ struct ltl_monitor {}; #endif /* CONFIG_RV_LTL_MONITOR */ +#ifdef CONFIG_RV_HA_MONITOR +/* + * In the future, hybrid automata may rely on multiple + * environment variables, e.g. different clocks started at + * different times or running at different speed. + * For now we support only 1 variable. + */ +#define MAX_HA_ENV_LEN 1 + +/* + * Monitors can pick the preferred timer implementation: + * No timer: if monitors don't have state invariants. + * Timer wheel: lightweight invariants check but far less precise. + * Hrtimer: accurate invariants check with higher overhead. + */ +#define HA_TIMER_NONE 0 +#define HA_TIMER_WHEEL 1 +#define HA_TIMER_HRTIMER 2 + +/* + * Hybrid automaton per-object variables. + */ +struct ha_monitor { + struct da_monitor da_mon; + u64 env_store[MAX_HA_ENV_LEN]; + union { + struct hrtimer hrtimer; + struct timer_list timer; + }; +}; + +#else + +struct ha_monitor { }; + +#endif /* CONFIG_RV_HA_MONITOR */ + #define RV_PER_TASK_MONITOR_INIT (CONFIG_RV_PER_TASK_MONITORS) union rv_task_monitor { struct da_monitor da_mon; struct ltl_monitor ltl_mon; + struct ha_monitor ha_mon; }; #ifdef CONFIG_RV_REACTORS diff --git a/include/linux/rwlock.h b/include/linux/rwlock.h index 3390d21c95dd..4e67cd934d8f 100644 --- a/include/linux/rwlock.h +++ b/include/linux/rwlock.h @@ -30,17 +30,27 @@ do { \ #ifdef CONFIG_DEBUG_SPINLOCK extern void do_raw_read_lock(rwlock_t *lock) __acquires_shared(lock); - extern int do_raw_read_trylock(rwlock_t *lock); + extern int do_raw_read_trylock(rwlock_t *lock) __cond_acquires_shared(true, lock); extern void do_raw_read_unlock(rwlock_t *lock) __releases_shared(lock); extern void do_raw_write_lock(rwlock_t *lock) __acquires(lock); - extern int do_raw_write_trylock(rwlock_t *lock); +extern int do_raw_write_trylock(rwlock_t *lock) __cond_acquires(true, lock); extern void do_raw_write_unlock(rwlock_t *lock) __releases(lock); #else # define do_raw_read_lock(rwlock) do {__acquire_shared(lock); arch_read_lock(&(rwlock)->raw_lock); } while (0) -# define do_raw_read_trylock(rwlock) arch_read_trylock(&(rwlock)->raw_lock) +static inline int do_raw_read_trylock(rwlock_t *rwlock) + __cond_acquires_shared(true, rwlock) + __no_context_analysis +{ + return arch_read_trylock(&(rwlock)->raw_lock); +} # define do_raw_read_unlock(rwlock) do {arch_read_unlock(&(rwlock)->raw_lock); __release_shared(lock); } while (0) # define do_raw_write_lock(rwlock) do {__acquire(lock); arch_write_lock(&(rwlock)->raw_lock); } while (0) -# define do_raw_write_trylock(rwlock) arch_write_trylock(&(rwlock)->raw_lock) +static inline int do_raw_write_trylock(rwlock_t *rwlock) + __cond_acquires(true, rwlock) + __no_context_analysis +{ + return arch_write_trylock(&(rwlock)->raw_lock); +} # define do_raw_write_unlock(rwlock) do {arch_write_unlock(&(rwlock)->raw_lock); __release(lock); } while (0) #endif diff --git a/include/linux/rwlock_api_smp.h b/include/linux/rwlock_api_smp.h index 61a852609eab..9e02a5f28cd1 100644 --- a/include/linux/rwlock_api_smp.h +++ b/include/linux/rwlock_api_smp.h @@ -23,7 +23,7 @@ void __lockfunc _raw_write_lock_bh(rwlock_t *lock) __acquires(lock); void __lockfunc _raw_read_lock_irq(rwlock_t *lock) __acquires_shared(lock); void __lockfunc _raw_write_lock_irq(rwlock_t *lock) __acquires(lock); unsigned long __lockfunc _raw_read_lock_irqsave(rwlock_t *lock) - __acquires(lock); + __acquires_shared(lock); unsigned long __lockfunc _raw_write_lock_irqsave(rwlock_t *lock) __acquires(lock); int __lockfunc _raw_read_trylock(rwlock_t *lock) __cond_acquires_shared(true, lock); @@ -36,7 +36,7 @@ void __lockfunc _raw_read_unlock_irq(rwlock_t *lock) __releases_shared(lock); void __lockfunc _raw_write_unlock_irq(rwlock_t *lock) __releases(lock); void __lockfunc _raw_read_unlock_irqrestore(rwlock_t *lock, unsigned long flags) - __releases(lock); + __releases_shared(lock); void __lockfunc _raw_write_unlock_irqrestore(rwlock_t *lock, unsigned long flags) __releases(lock); @@ -116,6 +116,7 @@ _raw_write_unlock_irqrestore(rwlock_t *lock, unsigned long flags) #endif static inline int __raw_read_trylock(rwlock_t *lock) + __cond_acquires_shared(true, lock) { preempt_disable(); if (do_raw_read_trylock(lock)) { @@ -127,6 +128,7 @@ static inline int __raw_read_trylock(rwlock_t *lock) } static inline int __raw_write_trylock(rwlock_t *lock) + __cond_acquires(true, lock) { preempt_disable(); if (do_raw_write_trylock(lock)) { diff --git a/include/linux/rwsem.h b/include/linux/rwsem.h index 9bf1d93d3d7b..6a1a7bae5f81 100644 --- a/include/linux/rwsem.h +++ b/include/linux/rwsem.h @@ -57,7 +57,7 @@ context_lock_struct(rw_semaphore) { struct optimistic_spin_queue osq; /* spinner MCS lock */ #endif raw_spinlock_t wait_lock; - struct list_head wait_list; + struct rwsem_waiter *first_waiter __guarded_by(&wait_lock); #ifdef CONFIG_DEBUG_RWSEMS void *magic; #endif @@ -106,7 +106,7 @@ static inline void rwsem_assert_held_write_nolockdep(const struct rw_semaphore * .owner = ATOMIC_LONG_INIT(0), \ __RWSEM_OPT_INIT(name) \ .wait_lock = __RAW_SPIN_LOCK_UNLOCKED(name.wait_lock),\ - .wait_list = LIST_HEAD_INIT((name).wait_list), \ + .first_waiter = NULL, \ __RWSEM_DEBUG_INIT(name) \ __RWSEM_DEP_MAP_INIT(name) } @@ -129,9 +129,9 @@ do { \ * rwsem to see if somebody from an incompatible type is wanting access to the * lock. */ -static inline int rwsem_is_contended(struct rw_semaphore *sem) +static inline bool rwsem_is_contended(struct rw_semaphore *sem) { - return !list_empty(&sem->wait_list); + return data_race(sem->first_waiter != NULL); } #if defined(CONFIG_DEBUG_RWSEMS) || defined(CONFIG_DETECT_HUNG_TASK_BLOCKER) diff --git a/include/linux/scc.h b/include/linux/scc.h deleted file mode 100644 index 745eabd17c10..000000000000 --- a/include/linux/scc.h +++ /dev/null @@ -1,86 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* $Id: scc.h,v 1.29 1997/04/02 14:56:45 jreuter Exp jreuter $ */ -#ifndef _SCC_H -#define _SCC_H - -#include <uapi/linux/scc.h> - - -enum {TX_OFF, TX_ON}; /* command for scc_key_trx() */ - -/* Vector masks in RR2B */ - -#define VECTOR_MASK 0x06 -#define TXINT 0x00 -#define EXINT 0x02 -#define RXINT 0x04 -#define SPINT 0x06 - -#ifdef CONFIG_SCC_DELAY -#define Inb(port) inb_p(port) -#define Outb(port, val) outb_p(val, port) -#else -#define Inb(port) inb(port) -#define Outb(port, val) outb(val, port) -#endif - -/* SCC channel control structure for KISS */ - -struct scc_kiss { - unsigned char txdelay; /* Transmit Delay 10 ms/cnt */ - unsigned char persist; /* Persistence (0-255) as a % */ - unsigned char slottime; /* Delay to wait on persistence hit */ - unsigned char tailtime; /* Delay after last byte written */ - unsigned char fulldup; /* Full Duplex mode 0=CSMA 1=DUP 2=ALWAYS KEYED */ - unsigned char waittime; /* Waittime before any transmit attempt */ - unsigned int maxkeyup; /* Maximum time to transmit (seconds) */ - unsigned int mintime; /* Minimal offtime after MAXKEYUP timeout (seconds) */ - unsigned int idletime; /* Maximum idle time in ALWAYS KEYED mode (seconds) */ - unsigned int maxdefer; /* Timer for CSMA channel busy limit */ - unsigned char tx_inhibit; /* Transmit is not allowed when set */ - unsigned char group; /* Group ID for AX.25 TX interlocking */ - unsigned char mode; /* 'normal' or 'hwctrl' mode (unused) */ - unsigned char softdcd; /* Use DPLL instead of DCD pin for carrier detect */ -}; - - -/* SCC channel structure */ - -struct scc_channel { - int init; /* channel exists? */ - - struct net_device *dev; /* link to device control structure */ - struct net_device_stats dev_stat;/* device statistics */ - - char brand; /* manufacturer of the board */ - long clock; /* used clock */ - - io_port ctrl; /* I/O address of CONTROL register */ - io_port data; /* I/O address of DATA register */ - io_port special; /* I/O address of special function port */ - int irq; /* Number of Interrupt */ - - char option; - char enhanced; /* Enhanced SCC support */ - - unsigned char wreg[16]; /* Copy of last written value in WRx */ - unsigned char status; /* Copy of R0 at last external interrupt */ - unsigned char dcd; /* DCD status */ - - struct scc_kiss kiss; /* control structure for KISS params */ - struct scc_stat stat; /* statistical information */ - struct scc_modem modem; /* modem information */ - - struct sk_buff_head tx_queue; /* next tx buffer */ - struct sk_buff *rx_buff; /* pointer to frame currently received */ - struct sk_buff *tx_buff; /* pointer to frame currently transmitted */ - - /* Timer */ - struct timer_list tx_t; /* tx timer for this channel */ - struct timer_list tx_wdog; /* tx watchdogs */ - - /* Channel lock */ - spinlock_t lock; /* Channel guard lock */ -}; - -#endif /* defined(_SCC_H) */ diff --git a/include/linux/sched.h b/include/linux/sched.h index 5a5d3dbc9cdf..705970d07614 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -16,6 +16,7 @@ #include <linux/cpumask_types.h> #include <linux/cache.h> +#include <linux/futex_types.h> #include <linux/irqflags_types.h> #include <linux/smp_types.h> #include <linux/pid_types.h> @@ -64,7 +65,6 @@ struct bpf_net_context; struct capture_control; struct cfs_rq; struct fs_struct; -struct futex_pi_state; struct io_context; struct io_uring_task; struct mempolicy; @@ -76,7 +76,6 @@ struct pid_namespace; struct pipe_inode_info; struct rcu_node; struct reclaim_state; -struct robust_list_head; struct root_domain; struct rq; struct sched_attr; @@ -85,6 +84,7 @@ struct seq_file; struct sighand_struct; struct signal_struct; struct task_delay_info; +struct task_exec_state; struct task_group; struct task_struct; struct timespec64; @@ -161,7 +161,7 @@ struct user_event_mm; */ #define is_special_task_state(state) \ ((state) & (__TASK_STOPPED | __TASK_TRACED | TASK_PARKED | \ - TASK_DEAD | TASK_FROZEN)) + TASK_DEAD | TASK_WAKING | TASK_FROZEN)) #ifdef CONFIG_DEBUG_ATOMIC_SLEEP # define debug_normal_state_change(state_value) \ @@ -550,7 +550,6 @@ struct sched_statistics { s64 exec_max; u64 slice_max; - u64 nr_migrations_cold; u64 nr_failed_migrations_affine; u64 nr_failed_migrations_running; u64 nr_failed_migrations_hot; @@ -563,8 +562,6 @@ struct sched_statistics { u64 nr_wakeups_remote; u64 nr_wakeups_affine; u64 nr_wakeups_affine_attempts; - u64 nr_wakeups_passive; - u64 nr_wakeups_idle; #ifdef CONFIG_SCHED_CORE u64 core_forceidle_sum; @@ -575,6 +572,7 @@ struct sched_statistics { struct sched_entity { /* For load-balancing: */ struct load_weight load; + struct load_weight h_load; struct rb_node run_node; u64 deadline; u64 min_vruntime; @@ -702,6 +700,11 @@ struct sched_dl_entity { * running, skipping the defer phase. * * @dl_defer_idle tracks idle state + * + * @dl_bw_attached tells if this server's bandwidth currently + * contributes to the root domain's total_bw. Only meaningful for server + * entities (@dl_server == 1). Allows toggling the reservation on/off + * without losing the configured @dl_runtime/@dl_period. */ unsigned int dl_throttled : 1; unsigned int dl_yielded : 1; @@ -713,6 +716,7 @@ struct sched_dl_entity { unsigned int dl_defer_armed : 1; unsigned int dl_defer_running : 1; unsigned int dl_defer_idle : 1; + unsigned int dl_bw_attached : 1; /* * Bandwidth enforcement timer. Each -deadline task has its @@ -817,6 +821,17 @@ struct kmap_ctrl { #endif }; +#if defined(CONFIG_SMP) && defined(CONFIG_PREEMPTION) +struct task_ipi_mask { + union { + cpumask_t *ipi_mask_ptr; + unsigned long ipi_mask_val; + }; +}; +#else +struct task_ipi_mask { }; +#endif + struct task_struct { #ifdef CONFIG_THREAD_INFO_IN_TASK /* @@ -846,7 +861,11 @@ struct task_struct { struct alloc_tag *alloc_tag; #endif - int on_cpu; + u8 on_cpu; + u8 on_rq; + u8 is_blocked; + u8 __pad; + struct __call_single_node wake_entry; unsigned int wakee_flips; unsigned long wakee_flip_decay_ts; @@ -861,7 +880,6 @@ struct task_struct { */ int recent_used_cpu; int wake_cpu; - int on_rq; int prio; int static_prio; @@ -949,6 +967,10 @@ struct task_struct { struct srcu_ctr __percpu *trc_reader_scp; #endif /* #ifdef CONFIG_TASKS_TRACE_RCU */ +#ifdef CONFIG_TRIVIAL_PREEMPT_RCU + int rcu_trivial_preempt_nesting; +#endif /* #ifdef CONFIG_TRIVIAL_PREEMPT_RCU */ + struct sched_info sched_info; struct list_head tasks; @@ -958,6 +980,8 @@ struct task_struct { struct mm_struct *mm; struct mm_struct *active_mm; + struct task_exec_state __rcu *exec_state; + int exit_state; int exit_code; int exit_signal; @@ -1159,12 +1183,9 @@ struct task_struct { /* * executable name, excluding path. * - * - normally initialized begin_new_exec() - * - set it with set_task_comm() - * - strscpy_pad() to ensure it is always NUL-terminated and - * zero-padded - * - task_lock() to ensure the operation is atomic and the name is - * fully updated. + * - normally initialized by begin_new_exec() + * - set it with set_task_comm() to ensure it is always + * NUL-terminated and zero-padded */ char comm[TASK_COMM_LEN]; @@ -1179,6 +1200,7 @@ struct task_struct { unsigned long last_switch_time; #endif /* Filesystem information: */ + struct fs_struct *real_fs; struct fs_struct *fs; /* Open file information: */ @@ -1238,6 +1260,14 @@ struct task_struct { #endif struct mutex *blocked_on; /* lock we're blocked on */ + raw_spinlock_t blocked_lock; + + /* + * The task that is boosting this task; a back link for the current + * donor stack. Set in schedule() -> find_proxy_task() and only stable + * under preempt_disable(). + */ + struct task_struct *blocked_donor; #ifdef CONFIG_DETECT_HUNG_TASK_BLOCKER /* @@ -1268,6 +1298,7 @@ struct task_struct { u64 curr_chain_key; int lockdep_depth; unsigned int lockdep_recursion; + unsigned int lockdep_seq; struct held_lock held_locks[MAX_LOCK_DEPTH]; #endif @@ -1329,16 +1360,9 @@ struct task_struct { u32 closid; u32 rmid; #endif -#ifdef CONFIG_FUTEX - struct robust_list_head __user *robust_list; -#ifdef CONFIG_COMPAT - struct compat_robust_list_head __user *compat_robust_list; -#endif - struct list_head pi_state_list; - struct futex_pi_state *pi_state_cache; - struct mutex futex_exit_mutex; - unsigned int futex_state; -#endif + + struct futex_sched_data futex; + #ifdef CONFIG_PERF_EVENTS u8 perf_recursion[PERF_NR_CONTEXTS]; struct perf_event_context *perf_event_ctxp; @@ -1346,6 +1370,7 @@ struct task_struct { struct list_head perf_event_list; struct perf_ctx_data __rcu *perf_ctx_data; #endif + struct task_ipi_mask __private ipi_mask; #ifdef CONFIG_DEBUG_PREEMPT unsigned long preempt_disable_ip; #endif @@ -1406,6 +1431,13 @@ struct task_struct { unsigned long numa_pages_migrated; #endif /* CONFIG_NUMA_BALANCING */ +#ifdef CONFIG_SCHED_CACHE + struct callback_head cache_work; + int preferred_llc; + /* 1: task was enqueued to its preferred LLC, 0 otherwise */ + int pref_llc_queued; +#endif + struct rseq_data rseq; struct sched_mm_cid mm_cid; @@ -1512,6 +1544,9 @@ struct task_struct { /* KCOV descriptor wired with this task or NULL: */ struct kcov *kcov; + /* KCOV descriptor for remote coverage collection from other tasks: */ + struct kcov *kcov_remote; + /* KCOV common handle for remote coverage collection: */ u64 kcov_handle; @@ -1520,6 +1555,14 @@ struct task_struct { /* Collect coverage from softirq context: */ unsigned int kcov_softirq; + + /* Temporary storage for preempting remote coverage collection: */ + unsigned int kcov_saved_mode; + unsigned int kcov_saved_size; + void *kcov_saved_area; + struct kcov *kcov_saved_kcov; + int kcov_saved_sequence; + #endif #ifdef CONFIG_MEMCG_V1 @@ -1533,7 +1576,7 @@ struct task_struct { /* Used by memcontrol for targeted memcg charge: */ struct mem_cgroup *active_memcg; - /* Cache for current->cgroups->memcg->objcg lookups: */ + /* Cache for current->cgroups->memcg->nodeinfo[nid]->objcg lookups: */ struct obj_cgroup *objcg; #endif @@ -1744,7 +1787,7 @@ static inline bool is_lazy_mmu_mode_active(void) } #endif -extern struct pid *cad_pid; +extern struct pid __rcu *cad_pid; /* * Per process flags @@ -1826,7 +1869,6 @@ static __always_inline bool is_user_task(struct task_struct *task) /* Per-process atomic flags. */ #define PFA_NO_NEW_PRIVS 0 /* May not gain new privileges. */ #define PFA_SPREAD_PAGE 1 /* Spread page cache over cpuset */ -#define PFA_SPREAD_SLAB 2 /* Spread some slab caches over cpuset */ #define PFA_SPEC_SSB_DISABLE 3 /* Speculative Store Bypass disabled */ #define PFA_SPEC_SSB_FORCE_DISABLE 4 /* Speculative Store Bypass force disabled*/ #define PFA_SPEC_IB_DISABLE 5 /* Indirect branch speculation restricted */ @@ -1852,10 +1894,6 @@ TASK_PFA_TEST(SPREAD_PAGE, spread_page) TASK_PFA_SET(SPREAD_PAGE, spread_page) TASK_PFA_CLEAR(SPREAD_PAGE, spread_page) -TASK_PFA_TEST(SPREAD_SLAB, spread_slab) -TASK_PFA_SET(SPREAD_SLAB, spread_slab) -TASK_PFA_CLEAR(SPREAD_SLAB, spread_slab) - TASK_PFA_TEST(SPEC_SSB_DISABLE, spec_ssb_disable) TASK_PFA_SET(SPEC_SSB_DISABLE, spec_ssb_disable) TASK_PFA_CLEAR(SPEC_SSB_DISABLE, spec_ssb_disable) @@ -2179,59 +2217,45 @@ extern int __cond_resched_rwlock_write(rwlock_t *lock) __must_hold(lock); }) #ifndef CONFIG_PREEMPT_RT + static inline struct mutex *__get_task_blocked_on(struct task_struct *p) { - struct mutex *m = p->blocked_on; - - if (m) - lockdep_assert_held_once(&m->wait_lock); - return m; + lockdep_assert_held_once(&p->blocked_lock); + return p->blocked_on; } static inline void __set_task_blocked_on(struct task_struct *p, struct mutex *m) { - struct mutex *blocked_on = READ_ONCE(p->blocked_on); - WARN_ON_ONCE(!m); /* The task should only be setting itself as blocked */ WARN_ON_ONCE(p != current); - /* Currently we serialize blocked_on under the mutex::wait_lock */ - lockdep_assert_held_once(&m->wait_lock); + /* Currently we serialize blocked_on under the task::blocked_lock */ + lockdep_assert_held_once(&p->blocked_lock); /* * Check ensure we don't overwrite existing mutex value * with a different mutex. Note, setting it to the same * lock repeatedly is ok. */ - WARN_ON_ONCE(blocked_on && blocked_on != m); - WRITE_ONCE(p->blocked_on, m); -} - -static inline void set_task_blocked_on(struct task_struct *p, struct mutex *m) -{ - guard(raw_spinlock_irqsave)(&m->wait_lock); - __set_task_blocked_on(p, m); + WARN_ON_ONCE(p->blocked_on && p->blocked_on != m); + p->blocked_on = m; } static inline void __clear_task_blocked_on(struct task_struct *p, struct mutex *m) { - if (m) { - struct mutex *blocked_on = READ_ONCE(p->blocked_on); - - /* Currently we serialize blocked_on under the mutex::wait_lock */ - lockdep_assert_held_once(&m->wait_lock); - /* - * There may be cases where we re-clear already cleared - * blocked_on relationships, but make sure we are not - * clearing the relationship with a different lock. - */ - WARN_ON_ONCE(blocked_on && blocked_on != m); - } - WRITE_ONCE(p->blocked_on, NULL); + /* Currently we serialize blocked_on under the task::blocked_lock */ + lockdep_assert_held_once(&p->blocked_lock); + /* + * There may be cases where we re-clear already cleared + * blocked_on relationships, but make sure we are not + * clearing the relationship with a different lock. + */ + WARN_ON_ONCE(m && p->blocked_on && p->blocked_on != m); + p->blocked_on = NULL; } static inline void clear_task_blocked_on(struct task_struct *p, struct mutex *m) { - guard(raw_spinlock_irqsave)(&m->wait_lock); + guard(raw_spinlock_irqsave)(&p->blocked_lock); __clear_task_blocked_on(p, m); } #else @@ -2374,6 +2398,29 @@ static __always_inline int task_mm_cid(struct task_struct *t) } #endif +#ifdef CONFIG_SCHED_CACHE + +struct sched_cache_time { + u64 runtime; + unsigned long epoch; +}; + +struct sched_cache_stat { + struct sched_cache_time __percpu *pcpu_sched; + raw_spinlock_t lock; + unsigned long epoch; + u64 nr_running_avg; + unsigned long next_scan; + unsigned long footprint; + int cpu; +} ____cacheline_aligned_in_smp; + +#else + +struct sched_cache_stat { }; + +#endif + #ifndef MODULE #ifndef COMPILE_OFFSETS diff --git a/include/linux/sched/clock.h b/include/linux/sched/clock.h index 196f0ca351a2..39f0a7f94bfc 100644 --- a/include/linux/sched/clock.h +++ b/include/linux/sched/clock.h @@ -33,6 +33,11 @@ extern u64 sched_clock_cpu(int cpu); extern void sched_clock_init(void); #ifndef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK +static inline int sched_clock_stable(void) +{ + return 1; +} + static inline void sched_clock_tick(void) { } diff --git a/include/linux/sched/coredump.h b/include/linux/sched/coredump.h index 624fda17a785..20957ccde3b5 100644 --- a/include/linux/sched/coredump.h +++ b/include/linux/sched/coredump.h @@ -2,43 +2,18 @@ #ifndef _LINUX_SCHED_COREDUMP_H #define _LINUX_SCHED_COREDUMP_H -#include <linux/mm_types.h> - -#define SUID_DUMP_DISABLE 0 /* No setuid dumping */ -#define SUID_DUMP_USER 1 /* Dump as user of process */ -#define SUID_DUMP_ROOT 2 /* Dump as root */ - -static inline unsigned long __mm_flags_get_dumpable(const struct mm_struct *mm) -{ - /* - * By convention, dumpable bits are contained in first 32 bits of the - * bitmap, so we can simply access this first unsigned long directly. - */ - return __mm_flags_get_word(mm); -} - -static inline void __mm_flags_set_mask_dumpable(struct mm_struct *mm, int value) -{ - __mm_flags_set_mask_bits_word(mm, MMF_DUMPABLE_MASK, value); -} - -extern void set_dumpable(struct mm_struct *mm, int value); /* - * This returns the actual value of the suid_dumpable flag. For things - * that are using this for checking for privilege transitions, it must - * test against SUID_DUMP_USER rather than treating it as a boolean - * value. + * Task dumpability mode. Gates core dump production and ptrace_attach() + * authorization. The numeric values are stable ABI (suid_dumpable + * sysctl, prctl(PR_SET_DUMPABLE)); do not renumber. */ -static inline int __get_dumpable(unsigned long mm_flags) -{ - return mm_flags & MMF_DUMPABLE_MASK; -} - -static inline int get_dumpable(struct mm_struct *mm) -{ - unsigned long flags = __mm_flags_get_dumpable(mm); - - return __get_dumpable(flags); -} +enum task_dumpable { + TASK_DUMPABLE_OFF = 0, /* no dump; ptrace needs CAP_SYS_PTRACE */ + TASK_DUMPABLE_OWNER = 1, /* default; dump and ptrace by uid match */ + TASK_DUMPABLE_ROOT = 2, /* dump as root; ptrace needs CAP_SYS_PTRACE */ +}; + +void task_exec_state_set_dumpable(enum task_dumpable value); +enum task_dumpable task_exec_state_get_dumpable(struct task_struct *task); #endif /* _LINUX_SCHED_COREDUMP_H */ diff --git a/include/linux/sched/deadline.h b/include/linux/sched/deadline.h index c40115d4e34d..273538200a44 100644 --- a/include/linux/sched/deadline.h +++ b/include/linux/sched/deadline.h @@ -33,8 +33,44 @@ struct root_domain; extern void dl_add_task_root_domain(struct task_struct *p); extern void dl_clear_root_domain(struct root_domain *rd); extern void dl_clear_root_domain_cpu(int cpu); +/* + * Return whether moving DL task @p to @new_mask requires moving DL + * bandwidth accounting between root domains. This helper is specific to + * DL bandwidth move accounting semantics and is shared by + * cpuset_can_attach() and set_cpus_allowed_dl() so both paths use the + * same source root-domain test. + */ +extern bool dl_task_needs_bw_move(struct task_struct *p, + const struct cpumask *new_mask); extern u64 dl_cookie; extern bool dl_bw_visited(int cpu, u64 cookie); +static inline bool dl_server(struct sched_dl_entity *dl_se) +{ + return dl_se->dl_server; +} + +static inline struct task_struct *dl_task_of(struct sched_dl_entity *dl_se) +{ + BUG_ON(dl_server(dl_se)); + return container_of(dl_se, struct task_struct, dl); +} + +/* + * Regarding the deadline, a task with implicit deadline has a relative + * deadline == relative period. A task with constrained deadline has a + * relative deadline <= relative period. + * + * We support constrained deadline tasks. However, there are some restrictions + * applied only for tasks which do not have an implicit deadline. See + * update_dl_entity() to know more about such restrictions. + * + * The dl_is_implicit() returns true if the task has an implicit deadline. + */ +static inline bool dl_is_implicit(struct sched_dl_entity *dl_se) +{ + return dl_se->dl_deadline == dl_se->dl_period; +} + #endif /* _LINUX_SCHED_DEADLINE_H */ diff --git a/include/linux/sched/exec_state.h b/include/linux/sched/exec_state.h new file mode 100644 index 000000000000..9b61782510b8 --- /dev/null +++ b/include/linux/sched/exec_state.h @@ -0,0 +1,31 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Christian Brauner <brauner@kernel.org> */ +#ifndef _LINUX_SCHED_EXEC_STATE_H +#define _LINUX_SCHED_EXEC_STATE_H + +#include <linux/init.h> +#include <linux/rcupdate.h> +#include <linux/refcount.h> +#include <linux/sched/coredump.h> +#include <linux/user_namespace.h> + +struct task_exec_state { + refcount_t count; + enum task_dumpable dumpable; + struct user_namespace *user_ns; + struct rcu_head rcu; +}; + +extern struct task_exec_state init_task_exec_state; + +struct task_exec_state *alloc_task_exec_state(struct user_namespace *user_ns); +void put_task_exec_state(struct task_exec_state *exec_state); +struct task_exec_state *task_exec_state_rcu(const struct task_struct *tsk); +struct task_exec_state *task_exec_state_replace(struct task_struct *tsk, + struct task_exec_state *exec_state); +int task_exec_state_copy(struct task_struct *tsk); +void __init exec_state_init(void); + +DEFINE_FREE(put_task_exec_state, struct task_exec_state *, put_task_exec_state(_T)) + +#endif /* _LINUX_SCHED_EXEC_STATE_H */ diff --git a/include/linux/sched/ext.h b/include/linux/sched/ext.h index bcb962d5ee7d..582d7cd4a983 100644 --- a/include/linux/sched/ext.h +++ b/include/linux/sched/ext.h @@ -58,10 +58,22 @@ enum scx_dsq_id_flags { SCX_DSQ_GLOBAL = SCX_DSQ_FLAG_BUILTIN | 1, SCX_DSQ_LOCAL = SCX_DSQ_FLAG_BUILTIN | 2, SCX_DSQ_BYPASS = SCX_DSQ_FLAG_BUILTIN | 3, + SCX_DSQ_REJECT = SCX_DSQ_FLAG_BUILTIN | 4, /* internal - see find_dsq_for_dispatch() */ + SCX_DSQ_RESCUE = SCX_DSQ_FLAG_BUILTIN | 5, /* internal - see find_dsq_for_dispatch() */ SCX_DSQ_LOCAL_ON = SCX_DSQ_FLAG_BUILTIN | SCX_DSQ_FLAG_LOCAL_ON, SCX_DSQ_LOCAL_CPU_MASK = 0xffffffffLLU, }; +struct scx_deferred_reenq_user { + struct list_head node; + u64 flags; +}; + +struct scx_dsq_pcpu { + struct scx_dispatch_q *dsq; + struct scx_deferred_reenq_user deferred_reenq_user; +}; + /* * A dispatch queue (DSQ) can be either a FIFO or p->scx.dsq_vtime ordered * queue. A built-in DSQ is always a FIFO. The built-in local DSQs are used to @@ -78,30 +90,65 @@ struct scx_dispatch_q { u64 id; struct rhash_head hash_node; struct llist_node free_node; + struct scx_sched *sched; + struct scx_dsq_pcpu __percpu *pcpu; struct rcu_head rcu; }; -/* scx_entity.flags */ +/* sched_ext_entity.flags */ enum scx_ent_flags { SCX_TASK_QUEUED = 1 << 0, /* on ext runqueue */ + SCX_TASK_IN_CUSTODY = 1 << 1, /* in custody, needs ops.dequeue() when leaving */ SCX_TASK_RESET_RUNNABLE_AT = 1 << 2, /* runnable_at should be reset */ SCX_TASK_DEQD_FOR_SLEEP = 1 << 3, /* last dequeue was for SLEEP */ + SCX_TASK_SUB_INIT = 1 << 4, /* task being initialized for a sub sched */ + SCX_TASK_IMMED = 1 << 5, /* task is on local DSQ with %SCX_ENQ_IMMED */ + SCX_TASK_PROTECTED = 1 << 6, /* slice and DSQ head position protected */ - SCX_TASK_STATE_SHIFT = 8, /* bit 8 and 9 are used to carry scx_task_state */ - SCX_TASK_STATE_BITS = 2, + /* + * Bits 8 to 10 are used to carry task state: + * + * NONE ops.init_task() not called yet + * INIT_BEGIN ops.init_task() in flight; see sched_ext_dead() + * INIT ops.init_task() succeeded, but task can be cancelled + * READY fully initialized, but not in sched_ext + * ENABLED fully initialized and in sched_ext + * DEAD terminal state set by sched_ext_dead() + */ + SCX_TASK_STATE_SHIFT = 8, + SCX_TASK_STATE_BITS = 3, SCX_TASK_STATE_MASK = ((1 << SCX_TASK_STATE_BITS) - 1) << SCX_TASK_STATE_SHIFT, - SCX_TASK_CURSOR = 1 << 31, /* iteration cursor, not a task */ -}; - -/* scx_entity.flags & SCX_TASK_STATE_MASK */ -enum scx_task_state { - SCX_TASK_NONE, /* ops.init_task() not called yet */ - SCX_TASK_INIT, /* ops.init_task() succeeded, but task can be cancelled */ - SCX_TASK_READY, /* fully initialized, but not in sched_ext */ - SCX_TASK_ENABLED, /* fully initialized and in sched_ext */ + SCX_TASK_NONE = 0 << SCX_TASK_STATE_SHIFT, + SCX_TASK_INIT_BEGIN = 1 << SCX_TASK_STATE_SHIFT, + SCX_TASK_INIT = 2 << SCX_TASK_STATE_SHIFT, + SCX_TASK_READY = 3 << SCX_TASK_STATE_SHIFT, + SCX_TASK_ENABLED = 4 << SCX_TASK_STATE_SHIFT, + SCX_TASK_DEAD = 5 << SCX_TASK_STATE_SHIFT, - SCX_TASK_NR_STATES, + /* + * Bits 12 to 14 are used to carry reenqueue reason. In addition to + * %SCX_ENQ_REENQ flag, ops.enqueue() can also test for + * %SCX_TASK_REENQ_REASON_NONE to distinguish reenqueues. + * + * NONE not being reenqueued + * KFUNC reenqueued by scx_bpf_dsq_reenq() and friends + * IMMED reenqueued due to failed ENQ_IMMED + * PREEMPTED preempted while running + * CAP sub-sched cap miss, see p->scx.reenq_reason_* + */ + SCX_TASK_REENQ_REASON_SHIFT = 12, + SCX_TASK_REENQ_REASON_BITS = 3, + SCX_TASK_REENQ_REASON_MASK = ((1 << SCX_TASK_REENQ_REASON_BITS) - 1) << SCX_TASK_REENQ_REASON_SHIFT, + + SCX_TASK_REENQ_NONE = 0 << SCX_TASK_REENQ_REASON_SHIFT, + SCX_TASK_REENQ_KFUNC = 1 << SCX_TASK_REENQ_REASON_SHIFT, + SCX_TASK_REENQ_IMMED = 2 << SCX_TASK_REENQ_REASON_SHIFT, + SCX_TASK_REENQ_PREEMPTED = 3 << SCX_TASK_REENQ_REASON_SHIFT, + SCX_TASK_REENQ_CAP = 4 << SCX_TASK_REENQ_REASON_SHIFT, + + /* iteration cursor, not a task */ + SCX_TASK_CURSOR = 1 << 31, }; /* scx_entity.dsq_flags */ @@ -109,33 +156,6 @@ enum scx_ent_dsq_flags { SCX_TASK_DSQ_ON_PRIQ = 1 << 0, /* task is queued on the priority queue of a dsq */ }; -/* - * Mask bits for scx_entity.kf_mask. Not all kfuncs can be called from - * everywhere and the following bits track which kfunc sets are currently - * allowed for %current. This simple per-task tracking works because SCX ops - * nest in a limited way. BPF will likely implement a way to allow and disallow - * kfuncs depending on the calling context which will replace this manual - * mechanism. See scx_kf_allow(). - */ -enum scx_kf_mask { - SCX_KF_UNLOCKED = 0, /* sleepable and not rq locked */ - /* ENQUEUE and DISPATCH may be nested inside CPU_RELEASE */ - SCX_KF_CPU_RELEASE = 1 << 0, /* ops.cpu_release() */ - /* - * ops.dispatch() may release rq lock temporarily and thus ENQUEUE and - * SELECT_CPU may be nested inside. ops.dequeue (in REST) may also be - * nested inside DISPATCH. - */ - SCX_KF_DISPATCH = 1 << 1, /* ops.dispatch() */ - SCX_KF_ENQUEUE = 1 << 2, /* ops.enqueue() and ops.select_cpu() */ - SCX_KF_SELECT_CPU = 1 << 3, /* ops.select_cpu() */ - SCX_KF_REST = 1 << 4, /* other rq-locked operations */ - - __SCX_KF_RQ_LOCKED = SCX_KF_CPU_RELEASE | SCX_KF_DISPATCH | - SCX_KF_ENQUEUE | SCX_KF_SELECT_CPU | SCX_KF_REST, - __SCX_KF_TERMINAL = SCX_KF_ENQUEUE | SCX_KF_SELECT_CPU | SCX_KF_REST, -}; - enum scx_dsq_lnode_flags { SCX_DSQ_LNODE_ITER_CURSOR = 1 << 0, @@ -149,48 +169,70 @@ struct scx_dsq_list_node { u32 priv; /* can be used by iter cursor */ }; -#define INIT_DSQ_LIST_CURSOR(__node, __flags, __priv) \ +#define INIT_DSQ_LIST_CURSOR(__cursor, __dsq, __flags) \ (struct scx_dsq_list_node) { \ - .node = LIST_HEAD_INIT((__node).node), \ + .node = LIST_HEAD_INIT((__cursor).node), \ .flags = SCX_DSQ_LNODE_ITER_CURSOR | (__flags), \ - .priv = (__priv), \ + .priv = READ_ONCE((__dsq)->seq), \ } +struct scx_sched; + /* * The following is embedded in task_struct and contains all fields necessary * for a task to be scheduled by SCX. */ struct sched_ext_entity { +#ifdef CONFIG_CGROUPS + /* + * Associated scx_sched. Updated either during fork or while holding + * both p->pi_lock and rq lock. + */ + struct scx_sched __rcu *sched; +#endif struct scx_dispatch_q *dsq; + atomic_long_t ops_state; + u64 ddsp_dsq_id; + u64 ddsp_enq_flags; + u64 ddsp_slice; + u64 ddsp_vtime; struct scx_dsq_list_node dsq_list; /* dispatch order */ struct rb_node dsq_priq; /* p->scx.dsq_vtime order */ u32 dsq_seq; u32 dsq_flags; /* protected by DSQ lock */ u32 flags; /* protected by rq lock */ u32 weight; + u32 reenq_cnt; /* reenqueues since last run */ s32 sticky_cpu; s32 holding_cpu; s32 selected_cpu; - u32 kf_mask; /* see scx_kf_mask above */ + s32 runnable_cpu; /* cpu @p is runnable on, -1 if not */ struct task_struct *kf_tasks[2]; /* see SCX_CALL_OP_TASK() */ - atomic_long_t ops_state; struct list_head runnable_node; /* rq->scx.runnable_list */ unsigned long runnable_at; -#ifdef CONFIG_SCHED_CORE - u64 core_sched_at; /* see scx_prio_less() */ +#ifdef CONFIG_EXT_SUB_SCHED + unsigned long rescue_at; /* queued on a rescue DSQ at, jiffies */ #endif - u64 ddsp_dsq_id; - u64 ddsp_enq_flags; + + /* + * Unique non-zero task ID assigned at fork. Persists across exec and + * is never reused. Lets BPF schedulers identify tasks without storing + * kernel pointers - arena-backed schedulers being one example. See + * scx_bpf_tid_to_task(). + */ + u64 tid; + struct rhash_head tid_hash_node; /* see SCX_OPS_TID_TO_TASK */ /* BPF scheduler modifiable fields */ /* - * Runtime budget in nsecs. This is usually set through - * scx_bpf_dsq_insert() but can also be modified directly by the BPF - * scheduler. Automatically decreased by SCX as the task executes. On - * depletion, a scheduling event is triggered. + * Runtime budget in nsecs - how long the task may hold its cpu. Owned + * by the task's scheduler. Set it when enqueuing via + * scx_bpf_dsq_insert(), or otherwise via scx_bpf_task_set_slice(). + * Automatically decreased as the task executes. On depletion a + * scheduling event is triggered. * * This value is cleared to zero if the task is preempted by * %SCX_KICK_PREEMPT and shouldn't be used to determine how long the @@ -208,15 +250,31 @@ struct sched_ext_entity { u64 dsq_vtime; /* + * Out-of-band slice request from scx_bpf_task_set_slice() when the + * caller does not hold the rq lock, applied under the rq lock at the + * next slice consideration. One atomic64 packs the pending flag, the + * issuing sch's id, and the requested slice. See scx_slice_oob_consts. + */ + atomic64_t slice_oob; + + /* + * Sub-sched cap rejected reenq context, valid only while + * %SCX_TASK_REENQ_CAP is set. @reenq_reason_caps is the SCX_CAP_* bits + * that were needed but missing. @reenq_reason_cid is the target cid. + */ + u64 reenq_reason_caps; + s32 reenq_reason_cid; + + /* * If set, reject future sched_setscheduler(2) calls updating the policy * to %SCHED_EXT with -%EACCES. * * Can be set from ops.init_task() while the BPF scheduler is being - * loaded (!scx_init_task_args->fork). If set and the task's policy is - * already %SCHED_EXT, the task's policy is rejected and forcefully - * reverted to %SCHED_NORMAL. The number of such events are reported - * through /sys/kernel/debug/sched_ext::nr_rejected. Setting this flag - * during fork is not allowed. + * loaded. If set and the task's policy is already %SCHED_EXT, the + * task's policy is rejected and forcefully reverted to %SCHED_NORMAL. + * The number of such events are reported through + * /sys/kernel/sched_ext/nr_rejected. Setting this flag from any other + * ops.init_task() invocation, such as during fork, fails the scheduler. */ bool disallow; /* reject switching into SCX */ @@ -231,7 +289,7 @@ void sched_ext_dead(struct task_struct *p); void print_scx_info(const char *log_lvl, struct task_struct *p); void scx_softlockup(u32 dur_s); bool scx_hardlockup(int cpu); -bool scx_rcu_cpu_stall(void); +bool scx_rcu_cpu_stall(const struct cpumask *stalled_mask); #else /* !CONFIG_SCHED_CLASS_EXT */ @@ -239,12 +297,27 @@ static inline void sched_ext_dead(struct task_struct *p) {} static inline void print_scx_info(const char *log_lvl, struct task_struct *p) {} static inline void scx_softlockup(u32 dur_s) {} static inline bool scx_hardlockup(int cpu) { return false; } -static inline bool scx_rcu_cpu_stall(void) { return false; } +static inline bool scx_rcu_cpu_stall(const struct cpumask *stalled_mask) { return false; } #endif /* CONFIG_SCHED_CLASS_EXT */ struct scx_task_group { #ifdef CONFIG_EXT_GROUP_SCHED + /* + * The sched this tg is on, NULL if none. SCX_TG_INITED tracks whether + * ops.cgroup_init() succeeded on it. When a child sched exits and its + * tgs move to the parent, a failed init leaves the tg on the parent + * with INITED clear (see scx_cgroup_return_subtree()). + * + * This is tracked separately from cgrp->scx_sched because the tg + * hierarchy can diverge from the cgroup2 hierarchy in both lifetime and + * shape. A tg stays online past its cgroup's removal while the + * cgrp->scx_sched rewrites visit only live cgroups, leaving a removed + * cgroup's pointer stale. The cpu controller can also be mounted on + * cgroup1. + */ + struct scx_sched *sched; + u32 flags; /* SCX_TG_* */ u32 weight; u64 bw_period_us; diff --git a/include/linux/sched/isolation.h b/include/linux/sched/isolation.h index dc3975ff1b2e..cf0fd03dd7a2 100644 --- a/include/linux/sched/isolation.h +++ b/include/linux/sched/isolation.h @@ -21,6 +21,11 @@ enum hk_type { HK_TYPE_MAX, /* + * HK_TYPE_KTHREAD is now an alias of HK_TYPE_DOMAIN + */ + HK_TYPE_KTHREAD = HK_TYPE_DOMAIN, + + /* * The following housekeeping types are only set by the nohz_full * boot commandline option. So they can share the same value. */ @@ -29,7 +34,6 @@ enum hk_type { HK_TYPE_RCU = HK_TYPE_KERNEL_NOISE, HK_TYPE_MISC = HK_TYPE_KERNEL_NOISE, HK_TYPE_WQ = HK_TYPE_KERNEL_NOISE, - HK_TYPE_KTHREAD = HK_TYPE_KERNEL_NOISE }; #ifdef CONFIG_CPU_ISOLATION diff --git a/include/linux/sched/mm.h b/include/linux/sched/mm.h index 95d0040df584..d7c6a942aa7e 100644 --- a/include/linux/sched/mm.h +++ b/include/linux/sched/mm.h @@ -155,10 +155,12 @@ extern struct mm_struct *get_task_mm(struct task_struct *task); * succeeds. */ extern struct mm_struct *mm_access(struct task_struct *task, unsigned int mode); -/* Remove the current tasks stale references to the old mm_struct on exit() */ -extern void exit_mm_release(struct task_struct *, struct mm_struct *); -/* Remove the current tasks stale references to the old mm_struct on exec() */ -extern void exec_mm_release(struct task_struct *, struct mm_struct *); + +/* + * Remove the current tasks stale references to the old mm_struct on exit() and + * exec(). Cleans up futexes as well. + */ +extern void mm_exit_exec_release(struct task_struct *, struct mm_struct *); #ifdef CONFIG_MEMCG extern void mm_update_next_owner(struct mm_struct *mm); @@ -193,21 +195,18 @@ unsigned long mm_get_unmapped_area(struct file *filp, unsigned long addr, unsigned long len, unsigned long pgoff, unsigned long flags); -unsigned long mm_get_unmapped_area_vmflags(struct file *filp, - unsigned long addr, - unsigned long len, - unsigned long pgoff, - unsigned long flags, - vm_flags_t vm_flags); +unsigned long mm_get_unmapped_area_vmaflags(struct file *filp, + unsigned long addr, unsigned long len, unsigned long pgoff, + unsigned long flags, vma_flags_t vma_flags); unsigned long generic_get_unmapped_area(struct file *filp, unsigned long addr, unsigned long len, unsigned long pgoff, - unsigned long flags, vm_flags_t vm_flags); + unsigned long flags, vma_flags_t vma_flags); unsigned long generic_get_unmapped_area_topdown(struct file *filp, unsigned long addr, unsigned long len, unsigned long pgoff, - unsigned long flags, vm_flags_t vm_flags); + unsigned long flags, vma_flags_t vma_flags); #else static inline void arch_pick_mmap_layout(struct mm_struct *mm, const struct rlimit *rlim_stack) {} diff --git a/include/linux/sched/rt.h b/include/linux/sched/rt.h index 4e3338103654..922935cc3383 100644 --- a/include/linux/sched/rt.h +++ b/include/linux/sched/rt.h @@ -52,8 +52,10 @@ static inline bool rt_or_dl_task_policy(struct task_struct *tsk) #ifdef CONFIG_RT_MUTEXES extern void rt_mutex_pre_schedule(void); +extern void rt_mutex_futex_pre_schedule(void); extern void rt_mutex_schedule(void); extern void rt_mutex_post_schedule(void); +extern void rt_mutex_futex_post_schedule(void); /* * Must hold either p->pi_lock or task_rq(p)->lock. diff --git a/include/linux/sched/sd_flags.h b/include/linux/sched/sd_flags.h index 42839cfa2778..dc3ec2452ee1 100644 --- a/include/linux/sched/sd_flags.h +++ b/include/linux/sched/sd_flags.h @@ -146,8 +146,7 @@ SD_FLAG(SD_ASYM_PACKING, SDF_NEEDS_GROUPS) /* * Prefer to place tasks in a sibling domain * - * Set up until domains start spanning NUMA nodes. Close to being a SHARED_CHILD - * flag, but cleared below domains with SD_ASYM_CPUCAPACITY. + * Set up until domains start spanning NUMA nodes. * * NEEDS_GROUPS: Load balancing flag. */ diff --git a/include/linux/sched/signal.h b/include/linux/sched/signal.h index a22248aebcf9..d45a5476b97d 100644 --- a/include/linux/sched/signal.h +++ b/include/linux/sched/signal.h @@ -132,6 +132,7 @@ struct signal_struct { */ unsigned int is_child_subreaper:1; unsigned int has_child_subreaper:1; + unsigned int autoreap:1; #ifdef CONFIG_POSIX_TIMERS @@ -561,10 +562,7 @@ static inline sigset_t *sigmask_to_save(void) return res; } -static inline int kill_cad_pid(int sig, int priv) -{ - return kill_pid(cad_pid, sig, priv); -} +int kill_cad_pid(int sig, int priv); /* These can be the second arg to send_sig_info/send_group_sig_info. */ #define SEND_SIG_NOINFO ((struct kernel_siginfo *) 0) @@ -739,7 +737,7 @@ static inline int thread_group_empty(struct task_struct *p) extern struct sighand_struct *lock_task_sighand(struct task_struct *task, unsigned long *flags) - __acquires(&task->sighand->siglock); + __cond_acquires(nonnull, &task->sighand->siglock); static inline void unlock_task_sighand(struct task_struct *task, unsigned long *flags) diff --git a/include/linux/sched/smt.h b/include/linux/sched/smt.h index 166b19af956f..cde6679c0278 100644 --- a/include/linux/sched/smt.h +++ b/include/linux/sched/smt.h @@ -4,16 +4,12 @@ #include <linux/static_key.h> -#ifdef CONFIG_SCHED_SMT extern struct static_key_false sched_smt_present; static __always_inline bool sched_smt_active(void) { return static_branch_likely(&sched_smt_present); } -#else -static __always_inline bool sched_smt_active(void) { return false; } -#endif void arch_smt_update(void); diff --git a/include/linux/sched/task.h b/include/linux/sched/task.h index 41ed884cffc9..e0c1ca8c6a18 100644 --- a/include/linux/sched/task.h +++ b/include/linux/sched/task.h @@ -31,6 +31,7 @@ struct kernel_clone_args { u32 io_thread:1; u32 user_worker:1; u32 no_files:1; + u32 umh:1; unsigned long stack; unsigned long stack_size; unsigned long tls; diff --git a/include/linux/sched/topology.h b/include/linux/sched/topology.h index 45c0022b91ce..b5d9d7c2b8ad 100644 --- a/include/linux/sched/topology.h +++ b/include/linux/sched/topology.h @@ -67,7 +67,25 @@ struct sched_domain_shared { atomic_t ref; atomic_t nr_busy_cpus; int has_idle_cores; - int nr_idle_scan; + union { + int nr_idle_scan; + /* + * Used during allocation to claim the sched_domain_shared + * object at multiple levels. + * + * Note: between build and the first periodic LB tick, which + * rewrites the union via update_idle_cpu_scan(), readers of + * nr_idle_scan may observe the transient SD_* flag value as + * the scan bound. The flag bits are small positive integers, + * so the effect is just a slightly relaxed scan bound for one + * window and self-heals on the first tick. + */ + int alloc_flags; + }; +#ifdef CONFIG_SCHED_CACHE + unsigned long util_avg; + unsigned long capacity; +#endif }; struct sched_domain { @@ -95,9 +113,16 @@ struct sched_domain { unsigned int newidle_call; unsigned int newidle_success; unsigned int newidle_ratio; + u64 newidle_stamp; u64 max_newidle_lb_cost; unsigned long last_decay_max_lb_cost; +#ifdef CONFIG_SCHED_CACHE + unsigned int llc_max; + unsigned int *llc_counts __counted_by_ptr(llc_max); + unsigned long llc_bytes; +#endif + #ifdef CONFIG_SCHEDSTATS /* sched_balance_rq() stats */ unsigned int lb_count[CPU_MAX_IDLE_TYPES]; @@ -141,18 +166,30 @@ struct sched_domain { unsigned int span_weight; /* - * Span of all CPUs in this domain. + * See sched_domain_span(), on why flex arrays are broken. * - * NOTE: this field is variable length. (Allocated dynamically - * by attaching extra space to the end of the structure, - * depending on how many CPUs the kernel has booted up with) - */ unsigned long span[]; + */ }; static inline struct cpumask *sched_domain_span(struct sched_domain *sd) { - return to_cpumask(sd->span); + /* + * Turns out that C flexible arrays are fundamentally broken since it + * is allowed for offsetof(*sd, span) < sizeof(*sd), this means that + * structure initialzation *sd = { ... }; which writes every byte + * inside sizeof(*type), will over-write the start of the flexible + * array. + * + * Luckily, the way we allocate sched_domain is by: + * + * sizeof(*sd) + cpumask_size() + * + * this means that we have sufficient space for the whole flex array + * *outside* of sizeof(*sd). So use that, and avoid using sd->span. + */ + unsigned long *bitmap = (void *)sd + sizeof(*sd); + return to_cpumask(bitmap); } extern void partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[], @@ -171,7 +208,6 @@ typedef int (*sched_domain_flags_f)(void); struct sd_data { struct sched_domain *__percpu *sd; - struct sched_domain_shared *__percpu *sds; struct sched_group *__percpu *sg; struct sched_group_capacity *__percpu *sgc; }; @@ -244,4 +280,10 @@ static inline int task_node(const struct task_struct *p) return cpu_to_node(task_cpu(p)); } +#ifdef CONFIG_SCHED_CACHE +extern void sched_update_llc_bytes(unsigned int cpu); +#else +static inline void sched_update_llc_bytes(unsigned int cpu) { } +#endif + #endif /* _LINUX_SCHED_TOPOLOGY_H */ diff --git a/include/linux/sched/user.h b/include/linux/sched/user.h index 4cc52698e214..8d7e5521f7cd 100644 --- a/include/linux/sched/user.h +++ b/include/linux/sched/user.h @@ -25,7 +25,8 @@ struct user_struct { #if defined(CONFIG_PERF_EVENTS) || defined(CONFIG_BPF_SYSCALL) || \ defined(CONFIG_NET) || defined(CONFIG_IO_URING) || \ - defined(CONFIG_VFIO_PCI_ZDEV_KVM) || IS_ENABLED(CONFIG_IOMMUFD) + defined(CONFIG_VFIO_PCI_ZDEV_KVM) || IS_ENABLED(CONFIG_IOMMUFD) || \ + defined(CONFIG_SECRETMEM) atomic_long_t locked_vm; #endif #ifdef CONFIG_WATCH_QUEUE diff --git a/include/linux/scmi_imx_protocol.h b/include/linux/scmi_imx_protocol.h index 2407d7693b6b..ab867463c08c 100644 --- a/include/linux/scmi_imx_protocol.h +++ b/include/linux/scmi_imx_protocol.h @@ -52,6 +52,17 @@ struct scmi_imx_misc_ctrl_notify_report { unsigned int flags; }; + +#define MISC_EXT_INFO_LEN_MAX 4 +struct scmi_imx_misc_reset_reason { + bool valid:1; + bool orig_valid:1; + bool err_valid:1; + u32 reason; + u32 origin; + u32 errid; +}; + struct scmi_imx_misc_proto_ops { int (*misc_ctrl_set)(const struct scmi_protocol_handle *ph, u32 id, u32 num, u32 *val); @@ -61,6 +72,9 @@ struct scmi_imx_misc_proto_ops { u32 ctrl_id, u32 evt_id, u32 flags); int (*misc_syslog)(const struct scmi_protocol_handle *ph, u16 *size, void *array); + int (*misc_reset_reason)(const struct scmi_protocol_handle *ph, + bool system, struct scmi_imx_misc_reset_reason *boot_r, + struct scmi_imx_misc_reset_reason *shut_r, u32 *extinfo); }; /* See LMM_ATTRIBUTES in imx95.rst */ diff --git a/include/linux/scmi_protocol.h b/include/linux/scmi_protocol.h index aafaac1496b0..5ab73b1ab9aa 100644 --- a/include/linux/scmi_protocol.h +++ b/include/linux/scmi_protocol.h @@ -15,10 +15,9 @@ #define SCMI_MAX_STR_SIZE 64 #define SCMI_SHORT_NAME_MAX_SIZE 16 -#define SCMI_MAX_NUM_RATES 16 /** - * struct scmi_revision_info - version information structure + * struct scmi_base_info - version information structure * * @major_ver: Major ABI version. Change here implies risk of backward * compatibility break. @@ -31,7 +30,7 @@ * @vendor_id: A vendor identifier(Null terminated ASCII string) * @sub_vendor_id: A sub-vendor identifier(Null terminated ASCII string) */ -struct scmi_revision_info { +struct scmi_base_info { u16 major_ver; u16 minor_ver; u8 num_protocols; @@ -41,27 +40,23 @@ struct scmi_revision_info { char sub_vendor_id[SCMI_SHORT_NAME_MAX_SIZE]; }; +struct scmi_clock_rates { + bool rate_discrete; + unsigned int num_rates; + u64 *rates; +}; + struct scmi_clock_info { char name[SCMI_MAX_STR_SIZE]; unsigned int enable_latency; - bool rate_discrete; bool rate_changed_notifications; bool rate_change_requested_notifications; bool state_ctrl_forbidden; bool rate_ctrl_forbidden; bool parent_ctrl_forbidden; bool extended_config; - union { - struct { - int num_rates; - u64 rates[SCMI_MAX_NUM_RATES]; - } list; - struct { - u64 min_rate; - u64 max_rate; - u64 step_size; - } range; - }; + u64 min_rate; + u64 max_rate; int num_parents; u32 *parents; }; @@ -91,6 +86,11 @@ enum scmi_clock_oem_config { * @info_get: get the information of the specified clock * @rate_get: request the current clock rate of a clock * @rate_set: set the clock rate of a clock + * @determine_rate: determine the effective rate that can be supported by a + * clock calculating the closest allowed rate. + * Note that @rate is an input/output parameter used both to + * describe the requested rate and report the closest match + * @all_rates_get: get the list of all available rates for the specified clock. * @enable: enables the specified clock * @disable: disables the specified clock * @state_get: get the status of the specified clock @@ -108,6 +108,10 @@ struct scmi_clk_proto_ops { u64 *rate); int (*rate_set)(const struct scmi_protocol_handle *ph, u32 clk_id, u64 rate); + int (*determine_rate)(const struct scmi_protocol_handle *ph, u32 clk_id, + unsigned long *rate); + const struct scmi_clock_rates __must_check *(*all_rates_get) + (const struct scmi_protocol_handle *ph, u32 clk_id); int (*enable)(const struct scmi_protocol_handle *ph, u32 clk_id, bool atomic); int (*disable)(const struct scmi_protocol_handle *ph, u32 clk_id, @@ -901,7 +905,7 @@ struct scmi_notify_ops { */ struct scmi_handle { struct device *dev; - struct scmi_revision_info *version; + struct scmi_base_info *version; int __must_check (*devm_protocol_acquire)(struct scmi_device *sdev, u8 proto); diff --git a/include/linux/seccomp.h b/include/linux/seccomp.h index 9b959972bf4a..fcb3eb9825e5 100644 --- a/include/linux/seccomp.h +++ b/include/linux/seccomp.h @@ -22,14 +22,14 @@ #include <linux/atomic.h> #include <asm/seccomp.h> -extern int __secure_computing(void); +extern bool __seccomp_permit_syscall(void); #ifdef CONFIG_HAVE_ARCH_SECCOMP_FILTER -static inline int secure_computing(void) +static __always_inline bool seccomp_permit_syscall(void) { if (unlikely(test_syscall_work(SECCOMP))) - return __secure_computing(); - return 0; + return __seccomp_permit_syscall(); + return true; } #else extern void secure_computing_strict(int this_syscall); @@ -50,11 +50,11 @@ static inline int seccomp_mode(struct seccomp *s) struct seccomp_data; #ifdef CONFIG_HAVE_ARCH_SECCOMP_FILTER -static inline int secure_computing(void) { return 0; } +static inline bool seccomp_permit_syscall(void) { return true; } #else static inline void secure_computing_strict(int this_syscall) { return; } #endif -static inline int __secure_computing(void) { return 0; } +static inline bool __seccomp_permit_syscall(void) { return true; } static inline long prctl_get_seccomp(void) { diff --git a/include/linux/secure_boot.h b/include/linux/secure_boot.h new file mode 100644 index 000000000000..d17e92351567 --- /dev/null +++ b/include/linux/secure_boot.h @@ -0,0 +1,23 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2026 Red Hat, Inc. All Rights Reserved. + * + * Author: Coiby Xu <coxu@redhat.com> + */ + +#ifndef _LINUX_SECURE_BOOT_H +#define _LINUX_SECURE_BOOT_H + +#include <linux/types.h> + +#ifdef CONFIG_HAVE_ARCH_GET_SECUREBOOT +/* + * Returns true if the platform secure boot is enabled. + * Returns false if disabled or not supported. + */ +bool arch_get_secureboot(void); +#else +static inline bool arch_get_secureboot(void) { return false; } +#endif + +#endif /* _LINUX_SECURE_BOOT_H */ diff --git a/include/linux/security.h b/include/linux/security.h index 83a646d72f6f..153e9043058f 100644 --- a/include/linux/security.h +++ b/include/linux/security.h @@ -145,6 +145,7 @@ enum lockdown_reason { LOCKDOWN_BPF_WRITE_USER, LOCKDOWN_DBG_WRITE_KERNEL, LOCKDOWN_RTAS_ERROR_INJECTION, + LOCKDOWN_XEN_USER_ACTIONS, LOCKDOWN_INTEGRITY_MAX, LOCKDOWN_KCORE, LOCKDOWN_KPROBES, @@ -458,7 +459,7 @@ int security_inode_getsecurity(struct mnt_idmap *idmap, struct inode *inode, const char *name, void **buffer, bool alloc); int security_inode_setsecurity(struct inode *inode, const char *name, const void *value, size_t size, int flags); -int security_inode_listsecurity(struct inode *inode, char *buffer, size_t buffer_size); +int security_inode_listsecurity(struct inode *inode, char **buffer, ssize_t *remaining_size); void security_inode_getlsmprop(struct inode *inode, struct lsm_prop *prop); int security_inode_copy_up(struct dentry *src, struct cred **new); int security_inode_copy_up_xattr(struct dentry *src, const char *name); @@ -471,11 +472,17 @@ int security_file_permission(struct file *file, int mask); int security_file_alloc(struct file *file); void security_file_release(struct file *file); void security_file_free(struct file *file); +int security_backing_file_alloc(struct file *backing_file, + const struct file *user_file); +void security_backing_file_free(struct file *backing_file); int security_file_ioctl(struct file *file, unsigned int cmd, unsigned long arg); int security_file_ioctl_compat(struct file *file, unsigned int cmd, unsigned long arg); int security_mmap_file(struct file *file, unsigned long prot, unsigned long flags); +int security_mmap_backing_file(struct vm_area_struct *vma, + struct file *backing_file, + struct file *user_file); int security_mmap_addr(unsigned long addr); int security_file_mprotect(struct vm_area_struct *vma, unsigned long reqprot, unsigned long prot); @@ -1090,7 +1097,8 @@ static inline int security_inode_setsecurity(struct inode *inode, const char *na return -EOPNOTSUPP; } -static inline int security_inode_listsecurity(struct inode *inode, char *buffer, size_t buffer_size) +static inline int security_inode_listsecurity(struct inode *inode, + char **buffer, ssize_t *remaining_size) { return 0; } @@ -1140,6 +1148,15 @@ static inline void security_file_release(struct file *file) static inline void security_file_free(struct file *file) { } +static inline int security_backing_file_alloc(struct file *backing_file, + const struct file *user_file) +{ + return 0; +} + +static inline void security_backing_file_free(struct file *backing_file) +{ } + static inline int security_file_ioctl(struct file *file, unsigned int cmd, unsigned long arg) { @@ -1159,6 +1176,13 @@ static inline int security_mmap_file(struct file *file, unsigned long prot, return 0; } +static inline int security_mmap_backing_file(struct vm_area_struct *vma, + struct file *backing_file, + struct file *user_file) +{ + return 0; +} + static inline int security_mmap_addr(unsigned long addr) { return cap_mmap_addr(addr); @@ -1931,6 +1955,17 @@ static inline int security_mptcp_add_subflow(struct sock *sk, struct sock *ssk) } #endif /* CONFIG_SECURITY_NETWORK */ +#if defined(CONFIG_SECURITY_NETWORK) && defined(CONFIG_SECURITY_PATH) + +int security_unix_find(const struct path *path, struct sock *other, int flags); + +#else /* CONFIG_SECURITY_NETWORK && CONFIG_SECURITY_PATH */ +static inline int security_unix_find(const struct path *path, struct sock *other, int flags) +{ + return 0; +} +#endif /* CONFIG_SECURITY_NETWORK && CONFIG_SECURITY_PATH */ + #ifdef CONFIG_SECURITY_INFINIBAND int security_ib_pkey_access(void *sec, u64 subnet_prefix, u16 pkey); int security_ib_endport_manage_subnet(void *sec, const char *name, u8 port_num); diff --git a/include/linux/sed-opal.h b/include/linux/sed-opal.h index 80f33a93f944..0630430cc01a 100644 --- a/include/linux/sed-opal.h +++ b/include/linux/sed-opal.h @@ -53,6 +53,11 @@ static inline bool is_sed_ioctl(unsigned int cmd) case IOC_OPAL_DISCOVERY: case IOC_OPAL_REVERT_LSP: case IOC_OPAL_SET_SID_PW: + case IOC_OPAL_REACTIVATE_LSP: + case IOC_OPAL_LR_SET_START_LEN: + case IOC_OPAL_ENABLE_DISABLE_LR: + case IOC_OPAL_GET_SUM_STATUS: + case IOC_OPAL_STACK_RESET: return true; } return false; diff --git a/include/linux/semaphore.h b/include/linux/semaphore.h index 89706157e622..a4c8651ef021 100644 --- a/include/linux/semaphore.h +++ b/include/linux/semaphore.h @@ -15,7 +15,7 @@ struct semaphore { raw_spinlock_t lock; unsigned int count; - struct list_head wait_list; + struct semaphore_waiter *first_waiter; #ifdef CONFIG_DETECT_HUNG_TASK_BLOCKER unsigned long last_holder; @@ -33,7 +33,7 @@ struct semaphore { { \ .lock = __RAW_SPIN_LOCK_UNLOCKED((name).lock), \ .count = n, \ - .wait_list = LIST_HEAD_INIT((name).wait_list) \ + .first_waiter = NULL \ __LAST_HOLDER_SEMAPHORE_INITIALIZER \ } diff --git a/include/linux/seq_file.h b/include/linux/seq_file.h index 2fb266ea69fa..dc0e8c62d9e0 100644 --- a/include/linux/seq_file.h +++ b/include/linux/seq_file.h @@ -104,7 +104,7 @@ static inline void seq_setwidth(struct seq_file *m, size_t size) } void seq_pad(struct seq_file *m, char c); -char *mangle_path(char *s, const char *p, const char *esc); +char *seq_mangle_path(char *s, const char *p, const char *esc); int seq_open(struct file *, const struct seq_operations *); ssize_t seq_read(struct file *, char __user *, size_t, loff_t *); ssize_t seq_read_iter(struct kiocb *iocb, struct iov_iter *iter); diff --git a/include/linux/seqlock.h b/include/linux/seqlock.h index 5a40252b8334..f865491c4f2c 100644 --- a/include/linux/seqlock.h +++ b/include/linux/seqlock.h @@ -1259,14 +1259,15 @@ static __always_inline void __scoped_seqlock_cleanup(struct ss_tmp *sst) extern void __scoped_seqlock_invalid_target(void); -#if (defined(CONFIG_CC_IS_GCC) && CONFIG_GCC_VERSION < 90000) || defined(CONFIG_KASAN) +#if (defined(CONFIG_CC_IS_GCC) && CONFIG_GCC_VERSION < 90000) || \ + defined(CONFIG_KASAN) || defined(CONFIG_UBSAN_ALIGNMENT) /* * For some reason some GCC-8 architectures (nios2, alpha) have trouble * determining that the ss_done state is impossible in __scoped_seqlock_next() * below. * - * Similarly KASAN is known to confuse compilers enough to break this. But we - * don't care about code quality for KASAN builds anyway. + * Similarly KASAN and UBSAN_ALIGNMENT are known to confuse compilers enough + * to break this. But we don't care about code quality for such builds anyway. */ static inline void __scoped_seqlock_bug(void) { } #else diff --git a/include/linux/serdev.h b/include/linux/serdev.h index 5654c58eb73c..b6c3d957ec15 100644 --- a/include/linux/serdev.h +++ b/include/linux/serdev.h @@ -37,8 +37,8 @@ struct serdev_device_ops { * @nr: Device number on serdev bus. * @ctrl: serdev controller managing this device. * @ops: Device operations. - * @write_comp Completion used by serdev_device_write() internally - * @write_lock Lock to serialize access when writing data + * @write_comp: Completion used by serdev_device_write() internally + * @write_lock: Lock to serialize access when writing data */ struct serdev_device { struct device dev; @@ -49,10 +49,7 @@ struct serdev_device { struct mutex write_lock; }; -static inline struct serdev_device *to_serdev_device(struct device *d) -{ - return container_of(d, struct serdev_device, dev); -} +#define to_serdev_device(d) container_of_const(d, struct serdev_device, dev) /** * struct serdev_device_driver - serdev slave device driver @@ -60,6 +57,7 @@ static inline struct serdev_device *to_serdev_device(struct device *d) * structure. * @probe: binds this driver to a serdev device. * @remove: unbinds this driver from the serdev device. + * @shutdown: shut down this serdev device. */ struct serdev_device_driver { struct device_driver driver; @@ -68,10 +66,7 @@ struct serdev_device_driver { void (*shutdown)(struct serdev_device *); }; -static inline struct serdev_device_driver *to_serdev_device_driver(struct device_driver *d) -{ - return container_of(d, struct serdev_device_driver, driver); -} +#define to_serdev_device_driver(d) container_of_const(d, struct serdev_device_driver, driver) enum serdev_parity { SERDEV_PARITY_NONE, @@ -112,10 +107,7 @@ struct serdev_controller { const struct serdev_controller_ops *ops; }; -static inline struct serdev_controller *to_serdev_controller(struct device *d) -{ - return container_of(d, struct serdev_controller, dev); -} +#define to_serdev_controller(d) container_of_const(d, struct serdev_controller, dev) static inline void *serdev_device_get_drvdata(const struct serdev_device *serdev) { @@ -129,7 +121,7 @@ static inline void serdev_device_set_drvdata(struct serdev_device *serdev, void /** * serdev_device_put() - decrement serdev device refcount - * @serdev serdev device. + * @serdev: serdev device. */ static inline void serdev_device_put(struct serdev_device *serdev) { @@ -157,7 +149,7 @@ static inline void serdev_controller_set_drvdata(struct serdev_controller *ctrl, /** * serdev_controller_put() - decrement controller refcount - * @ctrl serdev controller. + * @ctrl: serdev controller. */ static inline void serdev_controller_put(struct serdev_controller *ctrl) { @@ -343,4 +335,13 @@ static inline bool serdev_acpi_get_uart_resource(struct acpi_resource *ares, } #endif /* CONFIG_ACPI */ +#ifdef CONFIG_OF +struct serdev_controller *of_find_serdev_controller_by_node(struct device_node *node); +#else +static inline struct serdev_controller *of_find_serdev_controller_by_node(struct device_node *node) +{ + return NULL; +} +#endif /* CONFIG_OF */ + #endif /*_LINUX_SERDEV_H */ diff --git a/include/linux/serial_8250.h b/include/linux/serial_8250.h index a95b2d143d24..eeb868e93e15 100644 --- a/include/linux/serial_8250.h +++ b/include/linux/serial_8250.h @@ -150,8 +150,20 @@ struct uart_8250_port { #define LSR_SAVE_FLAGS UART_LSR_BRK_ERROR_BITS u16 lsr_saved_flags; u16 lsr_save_mask; + + /* + * Track when a console line has been fully written to the + * hardware, i.e. true when the most recent byte written to + * UART_TX by the console was '\n'. + */ + bool console_line_ended; + + /* Allow queuing irq_work for MSR handling */ + bool console_msr_work_allow; + #define MSR_SAVE_FLAGS UART_MSR_ANY_DELTA unsigned char msr_saved_flags; + struct irq_work console_msr_work; struct uart_8250_dma *dma; const struct uart_8250_ops *ops; @@ -192,6 +204,7 @@ void serial8250_do_shutdown(struct uart_port *port); void serial8250_do_pm(struct uart_port *port, unsigned int state, unsigned int oldstate); void serial8250_do_set_mctrl(struct uart_port *port, unsigned int mctrl); +void serial8250_do_break_ctl(struct uart_port *port, int break_state); void serial8250_do_set_divisor(struct uart_port *port, unsigned int baud, unsigned int quot); int fsl8250_handle_irq(struct uart_port *port); @@ -203,8 +216,8 @@ void serial8250_tx_chars(struct uart_8250_port *up); unsigned int serial8250_modem_status(struct uart_8250_port *up); void serial8250_init_port(struct uart_8250_port *up); void serial8250_set_defaults(struct uart_8250_port *up); -void serial8250_console_write(struct uart_8250_port *up, const char *s, - unsigned int count); +void serial8250_console_write(struct uart_8250_port *up, + struct nbcon_write_context *wctxt, bool in_atomic); int serial8250_console_setup(struct uart_port *port, char *options, bool probe); int serial8250_console_exit(struct uart_port *port); diff --git a/include/linux/serial_core.h b/include/linux/serial_core.h index 666430b47899..904760876ca8 100644 --- a/include/linux/serial_core.h +++ b/include/linux/serial_core.h @@ -437,6 +437,7 @@ enum uart_iotype { UPIO_TSI = SERIAL_IO_TSI, /* Tsi108/109 type IO */ UPIO_MEM32BE = SERIAL_IO_MEM32BE, /* 32b big endian */ UPIO_MEM16 = SERIAL_IO_MEM16, /* 16b little endian */ + UPIO_BUS = SERIAL_IO_BUS, /* Serial bus I/O access (ex: SPI, I2C) */ }; struct uart_port { @@ -459,10 +460,13 @@ struct uart_port { unsigned int baud, unsigned int quot, unsigned int quot_frac); + int (*get_rxtrig)(struct uart_port *port); + int (*set_rxtrig)(struct uart_port *port, unsigned char bytes); int (*startup)(struct uart_port *port); void (*shutdown)(struct uart_port *port); void (*throttle)(struct uart_port *port); void (*unthrottle)(struct uart_port *port); + void (*break_ctl)(struct uart_port *port, int break_state); int (*handle_irq)(struct uart_port *); void (*pm)(struct uart_port *, unsigned int state, unsigned int old); @@ -533,6 +537,7 @@ struct uart_port { #define UPF_HARD_FLOW ((__force upf_t) (UPF_AUTO_CTS | UPF_AUTO_RTS)) /* Port has hardware-assisted s/w flow control */ #define UPF_SOFT_FLOW ((__force upf_t) BIT_ULL(22)) +/* Deprecated: use uart_set_cons_flow_enabled()/uart_cons_flow_enabled() instead. */ #define UPF_CONS_FLOW ((__force upf_t) BIT_ULL(23)) #define UPF_SHARE_IRQ ((__force upf_t) BIT_ULL(24)) #define UPF_EXAR_EFR ((__force upf_t) BIT_ULL(25)) @@ -567,6 +572,7 @@ struct uart_port { #define UPSTAT_SYNC_FIFO ((__force upstat_t) (1 << 5)) bool hw_stopped; /* sw-assisted CTS flow state */ + bool cons_flow; /* user specified console flow control */ unsigned int mctrl; /* current modem ctrl settings */ unsigned int frame_time; /* frame timing in ns */ unsigned int type; /* port type */ @@ -1163,6 +1169,24 @@ static inline bool uart_softcts_mode(struct uart_port *uport) return ((uport->status & mask) == UPSTAT_CTS_ENABLE); } +static inline void uart_set_cons_flow_enabled(struct uart_port *uport, bool enabled) +{ + uport->cons_flow = enabled; +} + +static inline bool uart_cons_flow_enabled(const struct uart_port *uport) +{ + return uport->cons_flow; +} + +static inline bool uart_console_hwflow_active(struct uart_port *uport) +{ + return uart_console(uport) && + !(uport->rs485.flags & SER_RS485_ENABLED) && + uart_cons_flow_enabled(uport) && + uart_cts_enabled(uport); +} + /* * The following are helper functions for the low level drivers. */ @@ -1275,6 +1299,18 @@ static inline void uart_unlock_and_check_sysrq_irqrestore(struct uart_port *port #endif /* CONFIG_MAGIC_SYSRQ_SERIAL */ /* + * Variant of guard(uart_port_lock_irqsave) for IRQ handlers that may capture + * a SysRq character via uart_prepare_sysrq_char(). The destructor uses the + * sysrq-aware unlock helper so that a captured port->sysrq_ch is dispatched + * to handle_sysrq() on scope exit. The plain guard variant silently drops + * sysrq_ch and must not be used by callers that process RX. + */ +DEFINE_LOCK_GUARD_1(uart_port_lock_check_sysrq_irqsave, struct uart_port, + uart_port_lock_irqsave(_T->lock, &_T->flags), + uart_unlock_and_check_sysrq_irqrestore(_T->lock, _T->flags), + unsigned long flags); + +/* * We do the SysRQ and SAK checking like this... */ static inline int uart_handle_break(struct uart_port *port) @@ -1306,4 +1342,9 @@ static inline int uart_handle_break(struct uart_port *port) !((cflag) & CLOCAL)) int uart_get_rs485_mode(struct uart_port *port); + +void uart_get_ioinfos(struct uart_port *port, char *buf, size_t size); +bool uart_iotype_mmio(enum uart_iotype iotype); +bool uart_iotype_io(enum uart_iotype iotype); + #endif /* LINUX_SERIAL_CORE_H */ diff --git a/include/linux/serial_s3c.h b/include/linux/serial_s3c.h index 102aa33d956c..f54cb6e23f85 100644 --- a/include/linux/serial_s3c.h +++ b/include/linux/serial_s3c.h @@ -269,7 +269,7 @@ #define APPLE_S5L_UTRSTAT_RXTO BIT(9) #define APPLE_S5L_UTRSTAT_ALL_FLAGS GENMASK(9, 3) -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #include <linux/serial_core.h> @@ -294,7 +294,7 @@ struct s3c2410_uartcfg { unsigned long ufcon; /* value of ufcon for port */ }; -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ #endif /* __ASM_ARM_REGS_SERIAL_H */ diff --git a/include/linux/serial_sci.h b/include/linux/serial_sci.h index 0f2f50b8d28e..36c795d61f7e 100644 --- a/include/linux/serial_sci.h +++ b/include/linux/serial_sci.h @@ -51,7 +51,6 @@ struct plat_sci_port_ops { */ struct plat_sci_port { unsigned int type; /* SCI / SCIF / IRDA / HSCIF */ - upf_t flags; /* UPF_* flags */ unsigned int sampling_rate; unsigned int scscr; /* SCSCR initialization */ diff --git a/include/linux/serio.h b/include/linux/serio.h index 69a47674af65..98be7084412c 100644 --- a/include/linux/serio.h +++ b/include/linux/serio.h @@ -13,7 +13,7 @@ #include <linux/spinlock.h> #include <linux/mutex.h> #include <linux/device.h> -#include <linux/mod_devicetable.h> +#include <linux/device-id/serio.h> #include <uapi/linux/serio.h> extern const struct bus_type serio_bus; diff --git a/include/linux/shmem_fs.h b/include/linux/shmem_fs.h index a8273b32e041..5663dff53186 100644 --- a/include/linux/shmem_fs.h +++ b/include/linux/shmem_fs.h @@ -12,8 +12,6 @@ #include <linux/userfaultfd_k.h> #include <linux/bits.h> -struct swap_iocb; - /* inode in-kernel data */ #ifdef CONFIG_TMPFS_QUOTA @@ -48,7 +46,7 @@ struct shmem_inode_info { }; struct timespec64 i_crtime; /* file creation time */ struct shared_policy policy; /* NUMA memory alloc policy */ - struct simple_xattrs xattrs; /* list of xattrs */ + struct list_head xattrs; /* list of xattrs */ pgoff_t fallocend; /* highest fallocate endindex */ unsigned int fsflags; /* for FS_IOC_[SG]ETFLAGS */ atomic_t stop_eviction; /* hold when working on inode */ @@ -89,6 +87,7 @@ struct shmem_sb_info { struct list_head shrinklist; /* List of shinkable inodes */ unsigned long shrinklist_len; /* Length of shrinklist */ struct shmem_quota_limits qlimits; /* Default quota limits */ + struct simple_xattr_cache xa_cache; }; static inline struct shmem_inode_info *SHMEM_I(struct inode *inode) @@ -122,12 +121,11 @@ static inline bool shmem_mapping(const struct address_space *mapping) void shmem_unlock_mapping(struct address_space *mapping); struct page *shmem_read_mapping_page_gfp(struct address_space *mapping, pgoff_t index, gfp_t gfp_mask); -int shmem_writeout(struct folio *folio, struct swap_iocb **plug, - struct list_head *folio_list); +int shmem_write_folio(struct folio *folio); void shmem_truncate_range(struct inode *inode, loff_t start, uoff_t end); int shmem_unuse(unsigned int type); -#ifdef CONFIG_TRANSPARENT_HUGEPAGE +#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && defined(CONFIG_SHMEM) unsigned long shmem_allowable_huge_orders(struct inode *inode, struct vm_area_struct *vma, pgoff_t index, loff_t write_end, bool shmem_huge_force); @@ -221,20 +219,6 @@ static inline pgoff_t shmem_fallocend(struct inode *inode, pgoff_t eof) extern bool shmem_charge(struct inode *inode, long pages); -#ifdef CONFIG_USERFAULTFD -#ifdef CONFIG_SHMEM -extern int shmem_mfill_atomic_pte(pmd_t *dst_pmd, - struct vm_area_struct *dst_vma, - unsigned long dst_addr, - unsigned long src_addr, - uffd_flags_t flags, - struct folio **foliop); -#else /* !CONFIG_SHMEM */ -#define shmem_mfill_atomic_pte(dst_pmd, dst_vma, dst_addr, \ - src_addr, flags, foliop) ({ BUG(); 0; }) -#endif /* CONFIG_SHMEM */ -#endif /* CONFIG_USERFAULTFD */ - /* * Used space is stored as unsigned 64-bit value in bytes but * quota core supports only signed 64-bit values so use that diff --git a/include/linux/simple_ring_buffer.h b/include/linux/simple_ring_buffer.h new file mode 100644 index 000000000000..21aec556293e --- /dev/null +++ b/include/linux/simple_ring_buffer.h @@ -0,0 +1,65 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _LINUX_SIMPLE_RING_BUFFER_H +#define _LINUX_SIMPLE_RING_BUFFER_H + +#include <linux/list.h> +#include <linux/ring_buffer.h> +#include <linux/ring_buffer_types.h> +#include <linux/types.h> + +/* + * Ideally those struct would stay private but the caller needs to know + * the allocation size for simple_ring_buffer_init(). + */ +struct simple_buffer_page { + struct list_head link; + struct buffer_data_page *page; + u64 entries; + u32 write; + u32 id; +}; + +struct simple_rb_per_cpu { + struct simple_buffer_page *tail_page; + struct simple_buffer_page *reader_page; + struct simple_buffer_page *head_page; + struct simple_buffer_page *bpages; + struct trace_buffer_meta *meta; + u32 nr_pages; + +#define SIMPLE_RB_UNAVAILABLE 0 +#define SIMPLE_RB_READY 1 +#define SIMPLE_RB_WRITING 2 + u32 status; + + u64 last_overrun; + u64 write_stamp; + + struct simple_rb_cbs *cbs; +}; + +int simple_ring_buffer_init(struct simple_rb_per_cpu *cpu_buffer, struct simple_buffer_page *bpages, + const struct ring_buffer_desc *desc); + +void simple_ring_buffer_unload(struct simple_rb_per_cpu *cpu_buffer); + +void *simple_ring_buffer_reserve(struct simple_rb_per_cpu *cpu_buffer, unsigned long length, + u64 timestamp); + +void simple_ring_buffer_commit(struct simple_rb_per_cpu *cpu_buffer); + +int simple_ring_buffer_enable_tracing(struct simple_rb_per_cpu *cpu_buffer, bool enable); + +int simple_ring_buffer_reset(struct simple_rb_per_cpu *cpu_buffer); + +int simple_ring_buffer_swap_reader_page(struct simple_rb_per_cpu *cpu_buffer); + +int simple_ring_buffer_init_mm(struct simple_rb_per_cpu *cpu_buffer, + struct simple_buffer_page *bpages, + const struct ring_buffer_desc *desc, + void *(*load_page)(unsigned long va), + void (*unload_page)(void *va)); + +void simple_ring_buffer_unload_mm(struct simple_rb_per_cpu *cpu_buffer, + void (*unload_page)(void *)); +#endif diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index daa4e4944ce3..421f6fc45451 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -821,6 +821,7 @@ enum skb_tstamp_type { * @_sk_redir: socket redirection information for skmsg * @_nfct: Associated connection, if any (with nfctinfo bits) * @skb_iif: ifindex of device we arrived on + * @tc_depth: counter for packet duplication * @tc_index: Traffic control index * @hash: the packet hash * @queue_mapping: Queue mapping for multiqueue devices @@ -1030,6 +1031,7 @@ struct sk_buff { __u8 csum_not_inet:1; #endif __u8 unreadable:1; + __u8 tc_depth:2; #if defined(CONFIG_NET_SCHED) || defined(CONFIG_NET_XGRESS) __u16 tc_index; /* traffic control index */ #endif @@ -1082,9 +1084,7 @@ struct sk_buff { __u16 network_header; __u16 mac_header; -#ifdef CONFIG_KCOV - u64 kcov_handle; -#endif + struct kcov_common_handle_id kcov_handle; ); /* end headers group */ @@ -1313,7 +1313,8 @@ static inline bool skb_data_unref(const struct sk_buff *skb, return true; } -void __fix_address sk_skb_reason_drop(struct sock *sk, struct sk_buff *skb, +void __fix_address sk_skb_reason_drop(const struct sock *sk, + struct sk_buff *skb, enum skb_drop_reason reason); static inline void @@ -2605,8 +2606,9 @@ static inline void skb_len_add(struct sk_buff *skb, int delta) * * Does not take any additional reference on the fragment. */ -static inline void __skb_fill_netmem_desc(struct sk_buff *skb, int i, - netmem_ref netmem, int off, int size) +static __always_inline void +__skb_fill_netmem_desc(struct sk_buff *skb, int i, netmem_ref netmem, + int off, int size) { struct page *page; @@ -2628,14 +2630,16 @@ static inline void __skb_fill_netmem_desc(struct sk_buff *skb, int i, skb->pfmemalloc = true; } -static inline void __skb_fill_page_desc(struct sk_buff *skb, int i, - struct page *page, int off, int size) +static __always_inline void +__skb_fill_page_desc(struct sk_buff *skb, int i, struct page *page, + int off, int size) { __skb_fill_netmem_desc(skb, i, page_to_netmem(page), off, size); } -static inline void skb_fill_netmem_desc(struct sk_buff *skb, int i, - netmem_ref netmem, int off, int size) +static __always_inline void +skb_fill_netmem_desc(struct sk_buff *skb, int i, netmem_ref netmem, + int off, int size) { __skb_fill_netmem_desc(skb, i, netmem, off, size); skb_shinfo(skb)->nr_frags = i + 1; @@ -2655,8 +2659,9 @@ static inline void skb_fill_netmem_desc(struct sk_buff *skb, int i, * * Does not take any additional reference on the fragment. */ -static inline void skb_fill_page_desc(struct sk_buff *skb, int i, - struct page *page, int off, int size) +static __always_inline void +skb_fill_page_desc(struct sk_buff *skb, int i, struct page *page, + int off, int size) { skb_fill_netmem_desc(skb, i, page_to_netmem(page), off, size); } @@ -2682,8 +2687,17 @@ static inline void skb_fill_page_desc_noacc(struct sk_buff *skb, int i, shinfo->nr_frags = i + 1; } -void skb_add_rx_frag_netmem(struct sk_buff *skb, int i, netmem_ref netmem, - int off, int size, unsigned int truesize); +static inline void skb_add_rx_frag_netmem(struct sk_buff *skb, int i, + netmem_ref netmem, int off, + int size, unsigned int truesize) +{ + DEBUG_NET_WARN_ON_ONCE(size > truesize); + + skb_fill_netmem_desc(skb, i, netmem, off, size); + skb->len += size; + skb->data_len += size; + skb->truesize += truesize; +} static inline void skb_add_rx_frag(struct sk_buff *skb, int i, struct page *page, int off, int size, @@ -2819,7 +2833,7 @@ static inline void *__skb_push(struct sk_buff *skb, unsigned int len) } void *skb_pull(struct sk_buff *skb, unsigned int len); -static inline void *__skb_pull(struct sk_buff *skb, unsigned int len) +static __always_inline void *__skb_pull(struct sk_buff *skb, unsigned int len) { DEBUG_NET_WARN_ON_ONCE(len > INT_MAX); @@ -2844,7 +2858,7 @@ void *skb_pull_data(struct sk_buff *skb, size_t len); void *__pskb_pull_tail(struct sk_buff *skb, int delta); -static inline enum skb_drop_reason +static __always_inline enum skb_drop_reason pskb_may_pull_reason(struct sk_buff *skb, unsigned int len) { DEBUG_NET_WARN_ON_ONCE(len > INT_MAX); @@ -2862,12 +2876,13 @@ pskb_may_pull_reason(struct sk_buff *skb, unsigned int len) return SKB_NOT_DROPPED_YET; } -static inline bool pskb_may_pull(struct sk_buff *skb, unsigned int len) +static __always_inline bool +pskb_may_pull(struct sk_buff *skb, unsigned int len) { return pskb_may_pull_reason(skb, len) == SKB_NOT_DROPPED_YET; } -static inline void *pskb_pull(struct sk_buff *skb, unsigned int len) +static __always_inline void *pskb_pull(struct sk_buff *skb, unsigned int len) { if (!pskb_may_pull(skb, len)) return NULL; @@ -3067,6 +3082,11 @@ static inline bool skb_transport_header_was_set(const struct sk_buff *skb) return skb->transport_header != (typeof(skb->transport_header))~0U; } +static inline void skb_unset_transport_header(struct sk_buff *skb) +{ + skb->transport_header = (typeof(skb->transport_header))~0U; +} + static inline unsigned char *skb_transport_header(const struct sk_buff *skb) { DEBUG_NET_WARN_ON_ONCE(!skb_transport_header_was_set(skb)); @@ -3111,6 +3131,30 @@ static inline void skb_set_transport_header(struct sk_buff *skb, skb->transport_header += offset; } +/** + * skb_set_transport_header_careful - conditionally set transport header + * @skb: buffer to alter + * @offset: offset to add to skb->data + * + * Hardened version of skb_set_transport_header(). + * + * Returns: true if the operation was a success. + */ +static inline bool __must_check +skb_set_transport_header_careful(struct sk_buff *skb, const int offset) +{ + long thoff = skb->data - skb->head + offset; + + if (unlikely(thoff != (typeof(skb->transport_header))thoff)) + return false; + + if (unlikely(thoff == (typeof(skb->transport_header))~0U)) + return false; + + skb->transport_header = thoff; + return true; +} + static inline unsigned char *skb_network_header(const struct sk_buff *skb) { return skb->head + skb->network_header; @@ -3328,7 +3372,7 @@ static inline int __pskb_trim(struct sk_buff *skb, unsigned int len) return 0; } -static inline int pskb_trim(struct sk_buff *skb, unsigned int len) +static __always_inline int pskb_trim(struct sk_buff *skb, unsigned int len) { skb_might_realloc(skb); return (len < skb->len) ? __pskb_trim(skb, len) : 0; @@ -3371,7 +3415,7 @@ static inline int __skb_grow(struct sk_buff *skb, unsigned int len) * destructor function and make the @skb unowned. The buffer continues * to exist but is no longer charged to its former owner. */ -static inline void skb_orphan(struct sk_buff *skb) +static __always_inline void skb_orphan(struct sk_buff *skb) { if (skb->destructor) { skb->destructor(skb); @@ -3558,7 +3602,7 @@ static inline struct page *__dev_alloc_pages_noprof(gfp_t gfp_mask, * 3. If requesting a order 0 page it will not be compound * due to the check to see if order has a value in prep_new_page * 4. __GFP_MEMALLOC is ignored if __GFP_NOMEMALLOC is set due to - * code in gfp_to_alloc_flags that should be enforcing this. + * code in alloc_flags_slowpath() that should be enforcing this. */ gfp_mask |= __GFP_COMP | __GFP_MEMALLOC; @@ -3750,6 +3794,17 @@ static inline void *skb_frag_address_safe(const skb_frag_t *frag) } /** + * skb_frag_phys - gets the physical address of the data in a paged fragment + * @frag: the paged fragment buffer + * + * Returns: the physical address of the data within @frag. + */ +static inline phys_addr_t skb_frag_phys(const skb_frag_t *frag) +{ + return page_to_phys(skb_frag_page(frag)) + skb_frag_off(frag); +} + +/** * skb_frag_page_copy() - sets the page in a fragment from another fragment * @fragto: skb fragment where page is set * @fragfrom: skb fragment page is copied from @@ -4035,8 +4090,8 @@ __skb_postpull_rcsum(struct sk_buff *skb, const void *start, unsigned int len, * update the CHECKSUM_COMPLETE checksum, or set ip_summed to * CHECKSUM_NONE so that it can be recomputed from scratch. */ -static inline void skb_postpull_rcsum(struct sk_buff *skb, - const void *start, unsigned int len) +static __always_inline void +skb_postpull_rcsum(struct sk_buff *skb, const void *start, unsigned int len) { if (skb->ip_summed == CHECKSUM_COMPLETE) skb->csum = wsum_negate(csum_partial(start, len, @@ -4295,7 +4350,7 @@ __skb_header_pointer(const struct sk_buff *skb, int offset, int len, return buffer; } -static inline void * __must_check +static __always_inline void * __must_check skb_header_pointer(const struct sk_buff *skb, int offset, int len, void *buffer) { return __skb_header_pointer(skb, offset, len, skb->data, @@ -4467,7 +4522,7 @@ DECLARE_STATIC_KEY_FALSE(netstamp_needed_key); /* It is used in the ingress path to clear the delivery_time. * If needed, set the skb->tstamp to the (rcv) timestamp. */ -static inline void skb_clear_delivery_time(struct sk_buff *skb) +static __always_inline void skb_clear_delivery_time(struct sk_buff *skb) { if (skb->tstamp_type) { skb->tstamp_type = SKB_CLOCK_REALTIME; @@ -4494,7 +4549,8 @@ static inline ktime_t skb_tstamp(const struct sk_buff *skb) return skb->tstamp; } -static inline ktime_t skb_tstamp_cond(const struct sk_buff *skb, bool cond) +static __always_inline ktime_t +skb_tstamp_cond(const struct sk_buff *skb, bool cond) { if (skb->tstamp_type != SKB_CLOCK_MONOTONIC && skb->tstamp) return skb->tstamp; @@ -4977,6 +5033,7 @@ static inline unsigned long skb_get_nfct(const struct sk_buff *skb) static inline void skb_set_nfct(struct sk_buff *skb, unsigned long nfct) { #if IS_ENABLED(CONFIG_NF_CONNTRACK) + DEBUG_NET_WARN_ON_ONCE(skb->_nfct & NFCT_PTRMASK); skb->slow_gro |= !!nfct; skb->_nfct = nfct; #endif @@ -5097,6 +5154,7 @@ static inline bool skb_has_extensions(struct sk_buff *skb) return unlikely(skb->active_extensions); } #else +static inline void __skb_ext_put(struct skb_ext *ext) {} static inline void skb_ext_put(struct sk_buff *skb) {} static inline void skb_ext_reset(struct sk_buff *skb) {} static inline void skb_ext_del(struct sk_buff *skb, int unused) {} @@ -5283,7 +5341,7 @@ static inline void skb_decrease_gso_size(struct skb_shared_info *shinfo, void __skb_warn_lro_forwarding(const struct sk_buff *skb); -static inline bool skb_warn_if_lro(const struct sk_buff *skb) +static __always_inline bool skb_warn_if_lro(const struct sk_buff *skb) { /* LRO sets gso_size but not gso_type, whereas if GSO is really * wanted then gso_type will be set. */ @@ -5410,20 +5468,14 @@ static inline void skb_reset_csum_not_inet(struct sk_buff *skb) } static inline void skb_set_kcov_handle(struct sk_buff *skb, - const u64 kcov_handle) + struct kcov_common_handle_id kcov_handle) { -#ifdef CONFIG_KCOV skb->kcov_handle = kcov_handle; -#endif } -static inline u64 skb_get_kcov_handle(struct sk_buff *skb) +static inline struct kcov_common_handle_id skb_get_kcov_handle(struct sk_buff *skb) { -#ifdef CONFIG_KCOV return skb->kcov_handle; -#else - return 0; -#endif } static inline void skb_mark_for_recycle(struct sk_buff *skb) diff --git a/include/linux/skmsg.h b/include/linux/skmsg.h index 19f4f253b4f9..d5e35f24738d 100644 --- a/include/linux/skmsg.h +++ b/include/linux/skmsg.h @@ -4,6 +4,7 @@ #ifndef _LINUX_SKMSG_H #define _LINUX_SKMSG_H +#include <linux/bitops.h> #include <linux/bpf.h> #include <linux/filter.h> #include <linux/scatterlist.h> @@ -199,11 +200,14 @@ static inline void sk_msg_xfer(struct sk_msg *dst, struct sk_msg *src, int which, u32 size) { dst->sg.data[which] = src->sg.data[which]; + __assign_bit(which, dst->sg.copy, test_bit(which, src->sg.copy)); dst->sg.data[which].length = size; dst->sg.size += size; src->sg.size -= size; src->sg.data[which].length -= size; src->sg.data[which].offset += size; + if (!src->sg.data[which].length) + __clear_bit(which, src->sg.copy); } static inline void sk_msg_xfer_full(struct sk_msg *dst, struct sk_msg *src) @@ -273,16 +277,19 @@ static inline void sk_msg_page_add(struct sk_msg *msg, struct page *page, static inline void sk_msg_sg_copy(struct sk_msg *msg, u32 i, bool copy_state) { do { - if (copy_state) - __set_bit(i, msg->sg.copy); - else - __clear_bit(i, msg->sg.copy); + __assign_bit(i, msg->sg.copy, copy_state); sk_msg_iter_var_next(i); if (i == msg->sg.end) break; } while (1); } +static inline void sk_msg_sg_copy_assign(struct sk_msg *dst, u32 dst_i, + const struct sk_msg *src, u32 src_i) +{ + __assign_bit(dst_i, dst->sg.copy, test_bit(src_i, src->sg.copy)); +} + static inline void sk_msg_sg_copy_set(struct sk_msg *msg, u32 start) { sk_msg_sg_copy(msg, start, true); @@ -544,29 +551,6 @@ static inline void psock_progs_drop(struct sk_psock_progs *progs) psock_set_prog(&progs->skb_verdict, NULL); } -int sk_psock_tls_strp_read(struct sk_psock *psock, struct sk_buff *skb); - -static inline bool sk_psock_strp_enabled(struct sk_psock *psock) -{ - if (!psock) - return false; - return !!psock->saved_data_ready; -} - -/* for tcp only, sk is locked */ -static inline ssize_t sk_psock_msg_inq(struct sock *sk) -{ - struct sk_psock *psock; - ssize_t inq = 0; - - psock = sk_psock_get(sk); - if (likely(psock)) { - inq = sk_psock_get_msg_len_nolock(psock); - sk_psock_put(sk, psock); - } - return inq; -} - /* for udp only, sk is not locked */ static inline ssize_t sk_msg_first_len(struct sock *sk) { diff --git a/include/linux/slab.h b/include/linux/slab.h index 15a60b501b95..cda126def67a 100644 --- a/include/linux/slab.h +++ b/include/linux/slab.h @@ -45,6 +45,7 @@ enum _slab_flag_bits { #endif #ifdef CONFIG_MEMCG _SLAB_ACCOUNT, + _SLAB_MAY_ACCOUNT, #endif #ifdef CONFIG_KASAN_GENERIC _SLAB_KASAN, @@ -58,10 +59,13 @@ enum _slab_flag_bits { #endif _SLAB_OBJECT_POISON, _SLAB_CMPXCHG_DOUBLE, +#ifdef CONFIG_SLAB_OBJ_EXT _SLAB_NO_OBJ_EXT, -#if defined(CONFIG_SLAB_OBJ_EXT) && defined(CONFIG_64BIT) +#ifdef CONFIG_64BIT _SLAB_OBJ_EXT_IN_OBJ, #endif +#endif + _SLAB_NO_SHEAVES, _SLAB_FLAGS_LAST_BIT }; @@ -201,8 +205,10 @@ enum _slab_flag_bits { */ #ifdef CONFIG_MEMCG # define SLAB_ACCOUNT __SLAB_FLAG_BIT(_SLAB_ACCOUNT) +# define SLAB_MAY_ACCOUNT __SLAB_FLAG_BIT(_SLAB_MAY_ACCOUNT) #else # define SLAB_ACCOUNT __SLAB_FLAG_UNUSED +# define SLAB_MAY_ACCOUNT __SLAB_FLAG_UNUSED #endif #ifdef CONFIG_KASAN_GENERIC @@ -239,8 +245,14 @@ enum _slab_flag_bits { #endif #define SLAB_TEMPORARY SLAB_RECLAIM_ACCOUNT /* Objects are short-lived */ -/* Slab created using create_boot_cache */ +/* Slab caches without obj_exts array */ +#ifdef CONFIG_SLAB_OBJ_EXT #define SLAB_NO_OBJ_EXT __SLAB_FLAG_BIT(_SLAB_NO_OBJ_EXT) +#else +#define SLAB_NO_OBJ_EXT __SLAB_FLAG_UNUSED +#endif + +#define SLAB_NO_SHEAVES __SLAB_FLAG_BIT(_SLAB_NO_SHEAVES) #if defined(CONFIG_SLAB_OBJ_EXT) && defined(CONFIG_64BIT) #define SLAB_OBJ_EXT_IN_OBJ __SLAB_FLAG_BIT(_SLAB_OBJ_EXT_IN_OBJ) @@ -499,14 +511,84 @@ int kmem_cache_shrink(struct kmem_cache *s); .usersize = sizeof_field(struct __struct, __field), \ }, (__flags)) +#ifdef CONFIG_KMALLOC_PARTITION_CACHES +typedef struct { unsigned long v; } kmalloc_token_t; +#ifdef CONFIG_KMALLOC_PARTITION_RANDOM +extern unsigned long random_kmalloc_seed; +#define __kmalloc_token(...) ((kmalloc_token_t){ .v = _CODE_LOCATION_ }) +#elif defined(CONFIG_KMALLOC_PARTITION_TYPED) +#ifdef __CHECKER__ +#define __kmalloc_token(...) ((kmalloc_token_t){ .v = 0 }) +#else /* !__CHECKER__ */ +#define __kmalloc_token(...) ((kmalloc_token_t){ .v = __builtin_infer_alloc_token(__VA_ARGS__) }) +#endif /* __CHECKER__ */ +#endif /* CONFIG_KMALLOC_PARTITION_TYPED */ +#define DECL_TOKEN_PARAM(_token) , kmalloc_token_t (_token) +#define _PASS_TOKEN_PARAM(_token) , (_token) +#define PASS_TOKEN_PARAM(_token) (_token) +#define DECL_TOKEN_PARAMS(_size, _token) size_t (_size), kmalloc_token_t (_token) +#define PASS_TOKEN_PARAMS(_size, _token) (_size), (_token) +#else /* !CONFIG_KMALLOC_PARTITION_CACHES */ +typedef struct {} kmalloc_token_t; +#define __kmalloc_token(...) ((kmalloc_token_t){}) /* no-op */ +#define DECL_TOKEN_PARAM(_token) +#define _PASS_TOKEN_PARAM(_token) +#define PASS_TOKEN_PARAM(_token) ((kmalloc_token_t){}) +#define DECL_TOKEN_PARAMS(_size, _token) size_t (_size) +#define PASS_TOKEN_PARAMS(_size, _token) (_size) +#endif /* CONFIG_KMALLOC_PARTITION_CACHES */ + /* * Common kmalloc functions provided by all allocators */ -void * __must_check krealloc_node_align_noprof(const void *objp, size_t new_size, +void * __must_check krealloc_node_align_noprof(const void *objp, + DECL_TOKEN_PARAMS(new_size, token), unsigned long align, gfp_t flags, int nid) __realloc_size(2); -#define krealloc_noprof(_o, _s, _f) krealloc_node_align_noprof(_o, _s, 1, _f, NUMA_NO_NODE) -#define krealloc_node_align(...) alloc_hooks(krealloc_node_align_noprof(__VA_ARGS__)) +#define krealloc_noprof(_o, _s, _f) krealloc_node_align_noprof(_o, PASS_TOKEN_PARAMS(_s, __kmalloc_token(_s)), 1, _f, NUMA_NO_NODE) +#if 0 /* kernel-doc */ +/** + * krealloc_node_align - reallocate memory. The contents will remain unchanged. + * @p: object to reallocate memory for. + * @new_size: how many bytes of memory are required. + * @align: desired alignment. + * @flags: the type of memory to allocate. + * @nid: NUMA node or NUMA_NO_NODE + * + * If @p is %NULL, krealloc() behaves exactly like kmalloc(). If @new_size + * is 0 and @p is not a %NULL pointer, the object pointed to is freed. + * + * Only alignments up to those guaranteed by kmalloc() will be honored. Please see + * Documentation/core-api/memory-allocation.rst for more details. + * + * If __GFP_ZERO logic is requested, callers must ensure that, starting with the + * initial memory allocation, every subsequent call to this API for the same + * memory allocation is flagged with __GFP_ZERO. Otherwise, it is possible that + * __GFP_ZERO is not fully honored by this API. + * + * When slub_debug_orig_size() is off, krealloc() only knows about the bucket + * size of an allocation (but not the exact size it was allocated with) and + * hence implements the following semantics for shrinking and growing buffers + * with __GFP_ZERO:: + * + * new bucket + * 0 size size + * |--------|----------------| + * | keep | zero | + * + * Otherwise, the original allocation size 'orig_size' could be used to + * precisely clear the requested size, and the new size will also be stored + * as the new 'orig_size'. + * + * In any case, the contents of the object pointed to are preserved up to the + * lesser of the new and old sizes. + * + * Return: pointer to the allocated memory or %NULL in case of error + */ +void *krealloc_node_align(const void *p, size_t new_size, unsigned long align, gfp_t flags, int nid); +#endif +#define krealloc_node_align(p, new_size, align, flags, nid) \ + alloc_hooks(krealloc_node_align_noprof(p, PASS_TOKEN_PARAMS(new_size, __kmalloc_token(new_size)), align, flags, nid)) #define krealloc_node(_o, _s, _f, _n) krealloc_node_align(_o, _s, 1, _f, _n) #define krealloc(...) krealloc_node(__VA_ARGS__, NUMA_NO_NODE) @@ -612,10 +694,10 @@ static inline unsigned int arch_slab_minalign(void) #define SLAB_OBJ_MIN_SIZE (KMALLOC_MIN_SIZE < 16 ? \ (KMALLOC_MIN_SIZE) : 16) -#ifdef CONFIG_RANDOM_KMALLOC_CACHES -#define RANDOM_KMALLOC_CACHES_NR 15 // # of cache copies +#ifdef CONFIG_KMALLOC_PARTITION_CACHES +#define KMALLOC_PARTITION_CACHES_NR 15 // # of cache copies #else -#define RANDOM_KMALLOC_CACHES_NR 0 +#define KMALLOC_PARTITION_CACHES_NR 0 #endif /* @@ -634,8 +716,11 @@ enum kmalloc_cache_type { #ifndef CONFIG_MEMCG KMALLOC_CGROUP = KMALLOC_NORMAL, #endif - KMALLOC_RANDOM_START = KMALLOC_NORMAL, - KMALLOC_RANDOM_END = KMALLOC_RANDOM_START + RANDOM_KMALLOC_CACHES_NR, +#ifndef CONFIG_SLAB_OBJ_EXT + KMALLOC_NO_OBJ_EXT = KMALLOC_NORMAL, +#endif + KMALLOC_PARTITION_START = KMALLOC_NORMAL, + KMALLOC_PARTITION_END = KMALLOC_PARTITION_START + KMALLOC_PARTITION_CACHES_NR, #ifdef CONFIG_SLUB_TINY KMALLOC_RECLAIM = KMALLOC_NORMAL, #else @@ -647,6 +732,9 @@ enum kmalloc_cache_type { #ifdef CONFIG_MEMCG KMALLOC_CGROUP, #endif +#ifdef CONFIG_SLAB_OBJ_EXT + KMALLOC_NO_OBJ_EXT, +#endif NR_KMALLOC_TYPES }; @@ -662,19 +750,19 @@ extern kmem_buckets kmalloc_caches[NR_KMALLOC_TYPES]; (IS_ENABLED(CONFIG_ZONE_DMA) ? __GFP_DMA : 0) | \ (IS_ENABLED(CONFIG_MEMCG) ? __GFP_ACCOUNT : 0)) -extern unsigned long random_kmalloc_seed; - -static __always_inline enum kmalloc_cache_type kmalloc_type(gfp_t flags, unsigned long caller) +static __always_inline enum kmalloc_cache_type kmalloc_type(gfp_t flags, kmalloc_token_t token) { /* * The most common case is KMALLOC_NORMAL, so test for it * with a single branch for all the relevant flags. */ if (likely((flags & KMALLOC_NOT_NORMAL_BITS) == 0)) -#ifdef CONFIG_RANDOM_KMALLOC_CACHES - /* RANDOM_KMALLOC_CACHES_NR (=15) copies + the KMALLOC_NORMAL */ - return KMALLOC_RANDOM_START + hash_64(caller ^ random_kmalloc_seed, - ilog2(RANDOM_KMALLOC_CACHES_NR + 1)); +#ifdef CONFIG_KMALLOC_PARTITION_RANDOM + /* KMALLOC_PARTITION_CACHES_NR (=15) copies + the KMALLOC_NORMAL */ + return KMALLOC_PARTITION_START + hash_64(token.v ^ random_kmalloc_seed, + ilog2(KMALLOC_PARTITION_CACHES_NR + 1)); +#elif defined(CONFIG_KMALLOC_PARTITION_TYPED) + return KMALLOC_PARTITION_START + token.v; #else return KMALLOC_NORMAL; #endif @@ -815,8 +903,10 @@ kmem_buckets *kmem_buckets_create(const char *name, slab_flags_t flags, */ void kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p); -int kmem_cache_alloc_bulk_noprof(struct kmem_cache *s, gfp_t flags, size_t size, void **p); -#define kmem_cache_alloc_bulk(...) alloc_hooks(kmem_cache_alloc_bulk_noprof(__VA_ARGS__)) +bool kmem_cache_alloc_bulk_noprof(struct kmem_cache *s, gfp_t flags, + size_t size, void **p); +#define kmem_cache_alloc_bulk(...) \ + alloc_hooks(kmem_cache_alloc_bulk_noprof(__VA_ARGS__)) static __always_inline void kfree_bulk(size_t size, void **p) { @@ -858,16 +948,22 @@ unsigned int kmem_cache_sheaf_size(struct slab_sheaf *sheaf); #define PASS_BUCKET_PARAM(_b) NULL #endif +#define DECL_KMALLOC_PARAMS(_size, _b, _token) DECL_BUCKET_PARAMS(_size, _b) \ + DECL_TOKEN_PARAM(_token) + +#define PASS_KMALLOC_PARAMS(_size, _b, _token) PASS_BUCKET_PARAMS(_size, _b) \ + _PASS_TOKEN_PARAM(_token) + /* * The following functions are not to be used directly and are intended only * for internal use from kmalloc() and kmalloc_node() * with the exception of kunit tests */ -void *__kmalloc_noprof(size_t size, gfp_t flags) +void *__kmalloc_noprof(DECL_TOKEN_PARAMS(size, token), gfp_t flags) __assume_kmalloc_alignment __alloc_size(1); -void *__kmalloc_node_noprof(DECL_BUCKET_PARAMS(size, b), gfp_t flags, int node) +void *__kmalloc_node_noprof(DECL_KMALLOC_PARAMS(size, b, token), gfp_t flags, int node) __assume_kmalloc_alignment __alloc_size(1); void *__kmalloc_cache_noprof(struct kmem_cache *s, gfp_t flags, size_t size) @@ -883,6 +979,23 @@ void *__kmalloc_large_noprof(size_t size, gfp_t flags) void *__kmalloc_large_node_noprof(size_t size, gfp_t flags, int node) __assume_page_alignment __alloc_size(1); +static __always_inline __alloc_size(1) void *_kmalloc_noprof(size_t size, gfp_t flags, kmalloc_token_t token) +{ + if (__builtin_constant_p(size) && size) { + unsigned int index; + + if (size > KMALLOC_MAX_CACHE_SIZE) + return __kmalloc_large_noprof(size, flags); + + index = kmalloc_index(size); + return __kmalloc_cache_noprof( + kmalloc_caches[kmalloc_type(flags, token)][index], + flags, size); + } + return __kmalloc_noprof(PASS_TOKEN_PARAMS(size, token), flags); +} +#define kmalloc_noprof(...) _kmalloc_noprof(__VA_ARGS__, __kmalloc_token(__VA_ARGS__)) +#if 0 /* kernel-doc */ /** * kmalloc - allocate kernel memory * @size: how many bytes of memory are required. @@ -938,25 +1051,28 @@ void *__kmalloc_large_node_noprof(size_t size, gfp_t flags, int node) * Try really hard to succeed the allocation but fail * eventually. */ -static __always_inline __alloc_size(1) void *kmalloc_noprof(size_t size, gfp_t flags) -{ - if (__builtin_constant_p(size) && size) { - unsigned int index; - - if (size > KMALLOC_MAX_CACHE_SIZE) - return __kmalloc_large_noprof(size, flags); - - index = kmalloc_index(size); - return __kmalloc_cache_noprof( - kmalloc_caches[kmalloc_type(flags, _RET_IP_)][index], - flags, size); - } - return __kmalloc_noprof(size, flags); -} -#define kmalloc(...) alloc_hooks(kmalloc_noprof(__VA_ARGS__)) +void *kmalloc(size_t size, gfp_t flags); +#endif +#define kmalloc(size, flags) alloc_hooks(kmalloc_noprof(size, flags)) -void *kmalloc_nolock_noprof(size_t size, gfp_t gfp_flags, int node); -#define kmalloc_nolock(...) alloc_hooks(kmalloc_nolock_noprof(__VA_ARGS__)) +void *_kmalloc_nolock_noprof(DECL_TOKEN_PARAMS(size, token), gfp_t gfp_flags, int node); +#define kmalloc_nolock_noprof(_s, _f, _n) _kmalloc_nolock_noprof(PASS_TOKEN_PARAMS(_s, __kmalloc_token(_s)), _f, _n) +#if 0 /* kernel-doc */ +/** + * kmalloc_nolock - Allocate an object of given size from any context. + * @size: size to allocate + * @gfp_flags: GFP flags. Only __GFP_ACCOUNT and __GFP_ZERO allowed. Also + * __GFP_NOWARN and __GFP_NOMEMALLOC are allowed but added internally thus not + * necessary. + * @node: node number of the target node. + * + * Return: pointer to the new object or NULL in case of error. + * NULL does not mean EBUSY or EAGAIN. It means ENOMEM. + * There is no reason to call it again and expect !NULL. + */ +void *kmalloc_nolock(size_t size, gfp_t gfp_flags, int node); +#endif +#define kmalloc_nolock(size, gfp_flags, node) alloc_hooks(kmalloc_nolock_noprof(size, gfp_flags, node)) /** * __alloc_objs - Allocate objects of a given type using @@ -1000,7 +1116,7 @@ void *kmalloc_nolock_noprof(size_t size, gfp_t gfp_flags, int node); /** * kmalloc_obj - Allocate a single instance of the given type * @VAR_OR_TYPE: Variable or type to allocate. - * @GFP: GFP flags for the allocation. + * @...: optional GFP flags for the allocation (GFP_KERNEL when not specified). * * Returns: newly allocated pointer to a @VAR_OR_TYPE on success, or NULL * on failure. @@ -1012,7 +1128,7 @@ void *kmalloc_nolock_noprof(size_t size, gfp_t gfp_flags, int node); * kmalloc_objs - Allocate an array of the given type * @VAR_OR_TYPE: Variable or type to allocate an array of. * @COUNT: How many elements in the array. - * @GFP: GFP flags for the allocation. + * @...: optional GFP flags for the allocation (GFP_KERNEL when not specified). * * Returns: newly allocated pointer to array of @VAR_OR_TYPE on success, * or NULL on failure. @@ -1025,7 +1141,7 @@ void *kmalloc_nolock_noprof(size_t size, gfp_t gfp_flags, int node); * @VAR_OR_TYPE: Variable or type to allocate (with its flex array). * @FAM: The name of the flexible array member of the structure. * @COUNT: How many flexible array member elements are desired. - * @GFP: GFP flags for the allocation. + * @...: optional GFP flags for the allocation (GFP_KERNEL when not specified). * * Returns: newly allocated pointer to @VAR_OR_TYPE on success, NULL on * failure. If @FAM has been annotated with __counted_by(), the allocation @@ -1060,12 +1176,15 @@ void *kmalloc_nolock_noprof(size_t size, gfp_t gfp_flags, int node); __alloc_flex(kvzalloc, default_gfp(__VA_ARGS__), typeof(P), FAM, COUNT) #define kmem_buckets_alloc(_b, _size, _flags) \ - alloc_hooks(__kmalloc_node_noprof(PASS_BUCKET_PARAMS(_size, _b), _flags, NUMA_NO_NODE)) + alloc_hooks(__kmalloc_node_noprof(PASS_KMALLOC_PARAMS(_size, _b, __kmalloc_token(_size)), _flags, NUMA_NO_NODE)) + +#define kmem_buckets_alloc_node_track_caller(_b, _size, _flags, _node) \ + alloc_hooks(__kmalloc_node_track_caller_noprof(PASS_KMALLOC_PARAMS(_size, _b, __kmalloc_token(_size)), _flags, _node, _RET_IP_)) -#define kmem_buckets_alloc_track_caller(_b, _size, _flags) \ - alloc_hooks(__kmalloc_node_track_caller_noprof(PASS_BUCKET_PARAMS(_size, _b), _flags, NUMA_NO_NODE, _RET_IP_)) +#define kmem_buckets_alloc_track_caller(_b, _size, _flags) \ + kmem_buckets_alloc_node_track_caller(_b, _size, _flags, NUMA_NO_NODE) -static __always_inline __alloc_size(1) void *kmalloc_node_noprof(size_t size, gfp_t flags, int node) +static __always_inline __alloc_size(1) void *_kmalloc_node_noprof(size_t size, gfp_t flags, int node, kmalloc_token_t token) { if (__builtin_constant_p(size) && size) { unsigned int index; @@ -1075,29 +1194,48 @@ static __always_inline __alloc_size(1) void *kmalloc_node_noprof(size_t size, gf index = kmalloc_index(size); return __kmalloc_cache_node_noprof( - kmalloc_caches[kmalloc_type(flags, _RET_IP_)][index], + kmalloc_caches[kmalloc_type(flags, token)][index], flags, node, size); } - return __kmalloc_node_noprof(PASS_BUCKET_PARAMS(size, NULL), flags, node); + return __kmalloc_node_noprof(PASS_KMALLOC_PARAMS(size, NULL, token), flags, node); } +#define kmalloc_node_noprof(...) _kmalloc_node_noprof(__VA_ARGS__, __kmalloc_token(__VA_ARGS__)) #define kmalloc_node(...) alloc_hooks(kmalloc_node_noprof(__VA_ARGS__)) +static inline __alloc_size(1, 2) void *_kmalloc_array_noprof(size_t n, size_t size, gfp_t flags, kmalloc_token_t token) +{ + size_t bytes; + + if (unlikely(check_mul_overflow(n, size, &bytes))) + return NULL; + return _kmalloc_noprof(bytes, flags, token); +} +#define kmalloc_array_noprof(...) _kmalloc_array_noprof(__VA_ARGS__, __kmalloc_token(__VA_ARGS__)) +#if 0 /* kernel-doc */ /** * kmalloc_array - allocate memory for an array. * @n: number of elements. * @size: element size. * @flags: the type of memory to allocate (see kmalloc). */ -static inline __alloc_size(1, 2) void *kmalloc_array_noprof(size_t n, size_t size, gfp_t flags) +void *kmalloc_array(size_t n, size_t size, gfp_t flags); +#endif +#define kmalloc_array(n, size, flags) alloc_hooks(kmalloc_array_noprof(n, size, flags)) + +static inline __realloc_size(2, 3) void * __must_check _krealloc_array_noprof(void *p, + size_t new_n, + size_t new_size, + gfp_t flags, kmalloc_token_t token) { size_t bytes; - if (unlikely(check_mul_overflow(n, size, &bytes))) + if (unlikely(check_mul_overflow(new_n, new_size, &bytes))) return NULL; - return kmalloc_noprof(bytes, flags); -} -#define kmalloc_array(...) alloc_hooks(kmalloc_array_noprof(__VA_ARGS__)) + return krealloc_node_align_noprof(p, PASS_TOKEN_PARAMS(bytes, token), 1, flags, NUMA_NO_NODE); +} +#define krealloc_array_noprof(...) _krealloc_array_noprof(__VA_ARGS__, __kmalloc_token(__VA_ARGS__)) +#if 0 /* kernel-doc */ /** * krealloc_array - reallocate memory for an array. * @p: pointer to the memory chunk to reallocate @@ -1115,19 +1253,9 @@ static inline __alloc_size(1, 2) void *kmalloc_array_noprof(size_t n, size_t siz * In any case, the contents of the object pointed to are preserved up to the * lesser of the new and old sizes. */ -static inline __realloc_size(2, 3) void * __must_check krealloc_array_noprof(void *p, - size_t new_n, - size_t new_size, - gfp_t flags) -{ - size_t bytes; - - if (unlikely(check_mul_overflow(new_n, new_size, &bytes))) - return NULL; - - return krealloc_noprof(p, bytes, flags); -} -#define krealloc_array(...) alloc_hooks(krealloc_array_noprof(__VA_ARGS__)) +void *krealloc_array(void *p, size_t new_n, size_t new_size, gfp_t flags); +#endif +#define krealloc_array(p, new_n, new_size, flags) alloc_hooks(krealloc_array_noprof(p, new_n, new_size, flags)) /** * kcalloc - allocate memory for an array. The memory is set to zero. @@ -1137,10 +1265,10 @@ static inline __realloc_size(2, 3) void * __must_check krealloc_array_noprof(voi */ #define kcalloc(n, size, flags) kmalloc_array(n, size, (flags) | __GFP_ZERO) -void *__kmalloc_node_track_caller_noprof(DECL_BUCKET_PARAMS(size, b), gfp_t flags, int node, +void *__kmalloc_node_track_caller_noprof(DECL_KMALLOC_PARAMS(size, b, token), gfp_t flags, int node, unsigned long caller) __alloc_size(1); #define kmalloc_node_track_caller_noprof(size, flags, node, caller) \ - __kmalloc_node_track_caller_noprof(PASS_BUCKET_PARAMS(size, NULL), flags, node, caller) + __kmalloc_node_track_caller_noprof(PASS_KMALLOC_PARAMS(size, NULL, __kmalloc_token(size)), flags, node, caller) #define kmalloc_node_track_caller(...) \ alloc_hooks(kmalloc_node_track_caller_noprof(__VA_ARGS__, _RET_IP_)) @@ -1157,17 +1285,18 @@ void *__kmalloc_node_track_caller_noprof(DECL_BUCKET_PARAMS(size, b), gfp_t flag #define kmalloc_track_caller_noprof(...) \ kmalloc_node_track_caller_noprof(__VA_ARGS__, NUMA_NO_NODE, _RET_IP_) -static inline __alloc_size(1, 2) void *kmalloc_array_node_noprof(size_t n, size_t size, gfp_t flags, - int node) +static inline __alloc_size(1, 2) void *_kmalloc_array_node_noprof(size_t n, size_t size, gfp_t flags, + int node, kmalloc_token_t token) { size_t bytes; if (unlikely(check_mul_overflow(n, size, &bytes))) return NULL; if (__builtin_constant_p(n) && __builtin_constant_p(size)) - return kmalloc_node_noprof(bytes, flags, node); - return __kmalloc_node_noprof(PASS_BUCKET_PARAMS(bytes, NULL), flags, node); + return _kmalloc_node_noprof(bytes, flags, node, token); + return __kmalloc_node_noprof(PASS_KMALLOC_PARAMS(bytes, NULL, token), flags, node); } +#define kmalloc_array_node_noprof(...) _kmalloc_array_node_noprof(__VA_ARGS__, __kmalloc_token(__VA_ARGS__)) #define kmalloc_array_node(...) alloc_hooks(kmalloc_array_node_noprof(__VA_ARGS__)) #define kcalloc_node(_n, _size, _flags, _node) \ @@ -1178,44 +1307,73 @@ static inline __alloc_size(1, 2) void *kmalloc_array_node_noprof(size_t n, size_ */ #define kmem_cache_zalloc(_k, _flags) kmem_cache_alloc(_k, (_flags)|__GFP_ZERO) +static inline __alloc_size(1) void *_kzalloc_noprof(size_t size, gfp_t flags, kmalloc_token_t token) +{ + return _kmalloc_noprof(size, flags | __GFP_ZERO, token); +} +#define kzalloc_noprof(...) _kzalloc_noprof(__VA_ARGS__, __kmalloc_token(__VA_ARGS__)) +#if 0 /* kernel-doc */ /** * kzalloc - allocate memory. The memory is set to zero. * @size: how many bytes of memory are required. * @flags: the type of memory to allocate (see kmalloc). */ -static inline __alloc_size(1) void *kzalloc_noprof(size_t size, gfp_t flags) -{ - return kmalloc_noprof(size, flags | __GFP_ZERO); -} -#define kzalloc(...) alloc_hooks(kzalloc_noprof(__VA_ARGS__)) +void *kzalloc(size_t size, gfp_t flags); +#endif +#define kzalloc(size, flags) alloc_hooks(kzalloc_noprof(size, flags)) #define kzalloc_node(_size, _flags, _node) kmalloc_node(_size, (_flags)|__GFP_ZERO, _node) -void *__kvmalloc_node_noprof(DECL_BUCKET_PARAMS(size, b), unsigned long align, +void *__kvmalloc_node_noprof(DECL_KMALLOC_PARAMS(size, b, token), unsigned long align, gfp_t flags, int node) __alloc_size(1); #define kvmalloc_node_align_noprof(_size, _align, _flags, _node) \ - __kvmalloc_node_noprof(PASS_BUCKET_PARAMS(_size, NULL), _align, _flags, _node) + __kvmalloc_node_noprof(PASS_KMALLOC_PARAMS(_size, NULL, __kmalloc_token(_size)), _align, _flags, _node) #define kvmalloc_node_align(...) \ alloc_hooks(kvmalloc_node_align_noprof(__VA_ARGS__)) -#define kvmalloc_node(_s, _f, _n) kvmalloc_node_align(_s, 1, _f, _n) +#if 0 /* kernel-doc */ +/** + * kvmalloc_node - attempt to allocate physically contiguous memory, but upon + * failure, fall back to non-contiguous (vmalloc) allocation. + * @size: size of the request. + * @flags: gfp mask for the allocation - must be compatible (superset) with GFP_KERNEL. + * @node: numa node to allocate from + * + * Only alignments up to those guaranteed by kmalloc() will be honored. Please see + * Documentation/core-api/memory-allocation.rst for more details. + * + * Uses kmalloc to get the memory but if the allocation fails then falls back + * to the vmalloc allocator. Use kvfree for freeing the memory. + * + * GFP_NOWAIT and GFP_ATOMIC are supported, the __GFP_NORETRY modifier is not. + * __GFP_RETRY_MAYFAIL is supported, and it should be used only if kmalloc is + * preferable to the vmalloc fallback, due to visible performance drawbacks. + * + * Return: pointer to the allocated memory of %NULL in case of failure + */ +void *kvmalloc_node(size_t size, gfp_t flags, int node); +#endif +#define kvmalloc_node(size, flags, node) kvmalloc_node_align(size, 1, flags, node) +#define kvmalloc_node_noprof(size, flags, node) \ + kvmalloc_node_align_noprof(size, 1, flags, node) #define kvmalloc(...) kvmalloc_node(__VA_ARGS__, NUMA_NO_NODE) +#define kvmalloc_noprof(_size, _flags) kvmalloc_node_noprof(_size, _flags, NUMA_NO_NODE) #define kvzalloc(_size, _flags) kvmalloc(_size, (_flags)|__GFP_ZERO) #define kvzalloc_node(_size, _flags, _node) kvmalloc_node(_size, (_flags)|__GFP_ZERO, _node) #define kmem_buckets_valloc(_b, _size, _flags) \ - alloc_hooks(__kvmalloc_node_noprof(PASS_BUCKET_PARAMS(_size, _b), 1, _flags, NUMA_NO_NODE)) + alloc_hooks(__kvmalloc_node_noprof(PASS_KMALLOC_PARAMS(_size, _b, __kmalloc_token(_size)), 1, _flags, NUMA_NO_NODE)) static inline __alloc_size(1, 2) void * -kvmalloc_array_node_noprof(size_t n, size_t size, gfp_t flags, int node) +_kvmalloc_array_node_noprof(size_t n, size_t size, gfp_t flags, int node, kmalloc_token_t token) { size_t bytes; if (unlikely(check_mul_overflow(n, size, &bytes))) return NULL; - return kvmalloc_node_align_noprof(bytes, 1, flags, node); + return __kvmalloc_node_noprof(PASS_KMALLOC_PARAMS(bytes, NULL, token), 1, flags, node); } - +#define kvmalloc_array_node_noprof(...) _kvmalloc_array_node_noprof(__VA_ARGS__, __kmalloc_token(__VA_ARGS__)) #define kvmalloc_array_noprof(...) kvmalloc_array_node_noprof(__VA_ARGS__, NUMA_NO_NODE) #define kvcalloc_node_noprof(_n,_s,_f,_node) kvmalloc_array_node_noprof(_n,_s,(_f)|__GFP_ZERO,_node) #define kvcalloc_noprof(...) kvcalloc_node_noprof(__VA_ARGS__, NUMA_NO_NODE) @@ -1224,40 +1382,63 @@ kvmalloc_array_node_noprof(size_t n, size_t size, gfp_t flags, int node) #define kvcalloc_node(...) alloc_hooks(kvcalloc_node_noprof(__VA_ARGS__)) #define kvcalloc(...) alloc_hooks(kvcalloc_noprof(__VA_ARGS__)) -void *kvrealloc_node_align_noprof(const void *p, size_t size, unsigned long align, +void *kvrealloc_node_align_noprof(const void *p, DECL_TOKEN_PARAMS(size, token), unsigned long align, gfp_t flags, int nid) __realloc_size(2); -#define kvrealloc_node_align(...) \ - alloc_hooks(kvrealloc_node_align_noprof(__VA_ARGS__)) +#if 0 /* kernel-doc */ +/** + * kvrealloc_node_align - reallocate memory; contents remain unchanged + * @p: object to reallocate memory for + * @size: the size to reallocate + * @align: desired alignment + * @flags: the flags for the page level allocator + * @nid: NUMA node id + * + * If @p is %NULL, kvrealloc() behaves exactly like kvmalloc(). If @size is 0 + * and @p is not a %NULL pointer, the object pointed to is freed. + * + * Only alignments up to those guaranteed by kmalloc() will be honored. Please see + * Documentation/core-api/memory-allocation.rst for more details. + * + * If __GFP_ZERO logic is requested, callers must ensure that, starting with the + * initial memory allocation, every subsequent call to this API for the same + * memory allocation is flagged with __GFP_ZERO. Otherwise, it is possible that + * __GFP_ZERO is not fully honored by this API. + * + * In any case, the contents of the object pointed to are preserved up to the + * lesser of the new and old sizes. + * + * This function must not be called concurrently with itself or kvfree() for the + * same memory allocation. + * + * Return: pointer to the allocated memory or %NULL in case of error + */ +void *kvrealloc_node_align(const void *p, size_t size, unsigned long align, gfp_t flags, int nid); +#endif +#define kvrealloc_node_align(p, size, align, flags, nid) \ + alloc_hooks(kvrealloc_node_align_noprof(p, PASS_TOKEN_PARAMS(size, __kmalloc_token(size)), align, flags, nid)) #define kvrealloc_node(_p, _s, _f, _n) kvrealloc_node_align(_p, _s, 1, _f, _n) #define kvrealloc(...) kvrealloc_node(__VA_ARGS__, NUMA_NO_NODE) extern void kvfree(const void *addr); DEFINE_FREE(kvfree, void *, if (!IS_ERR_OR_NULL(_T)) kvfree(_T)) +extern void kvfree_atomic(const void *addr); +DEFINE_FREE(kvfree_atomic, void *, if (!IS_ERR_OR_NULL(_T)) kvfree_atomic(_T)) + extern void kvfree_sensitive(const void *addr, size_t len); unsigned int kmem_cache_size(struct kmem_cache *s); #ifndef CONFIG_KVFREE_RCU_BATCHED -static inline void kvfree_rcu_barrier(void) -{ - rcu_barrier(); -} - -static inline void kvfree_rcu_barrier_on_cache(struct kmem_cache *s) -{ - rcu_barrier(); -} - static inline void kfree_rcu_scheduler_running(void) { } #else +void kfree_rcu_scheduler_running(void); +#endif + void kvfree_rcu_barrier(void); void kvfree_rcu_barrier_on_cache(struct kmem_cache *s); -void kfree_rcu_scheduler_running(void); -#endif - /** * kmalloc_size_roundup - Report allocation bucket size for the given size * diff --git a/include/linux/slimbus.h b/include/linux/slimbus.h index a4608d9a9684..ca6f1da4bdf3 100644 --- a/include/linux/slimbus.h +++ b/include/linux/slimbus.h @@ -8,7 +8,7 @@ #include <linux/device.h> #include <linux/module.h> #include <linux/completion.h> -#include <linux/mod_devicetable.h> +#include <linux/device-id/slim.h> extern const struct bus_type slimbus_bus; diff --git a/include/linux/smbdirect.h b/include/linux/smbdirect.h new file mode 100644 index 000000000000..97f5ba730fa7 --- /dev/null +++ b/include/linux/smbdirect.h @@ -0,0 +1,186 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * Copyright (C) 2025, Stefan Metzmacher + */ + +#ifndef __LINUX_SMBDIRECT_H__ +#define __LINUX_SMBDIRECT_H__ + +#include <linux/types.h> + +/* SMB-DIRECT buffer descriptor V1 structure [MS-SMBD] 2.2.3.1 */ +struct smbdirect_buffer_descriptor_v1 { + __le64 offset; + __le32 token; + __le32 length; +} __packed; + +/* + * Connection parameters mostly from [MS-SMBD] 3.1.1.1 + * + * These are setup and negotiated at the beginning of a + * connection and remain constant unless explicitly changed. + * + * Some values are important for the upper layer. + */ +struct smbdirect_socket_parameters { + __u64 flags; +#define SMBDIRECT_FLAG_PORT_RANGE_ONLY_IB ((__u64)0x1) +#define SMBDIRECT_FLAG_PORT_RANGE_ONLY_IW ((__u64)0x2) + __u32 resolve_addr_timeout_msec; + __u32 resolve_route_timeout_msec; + __u32 rdma_connect_timeout_msec; + __u32 negotiate_timeout_msec; + __u16 initiator_depth; /* limited to U8_MAX */ + __u16 responder_resources; /* limited to U8_MAX */ + __u16 recv_credit_max; + __u16 send_credit_target; + __u32 max_send_size; + __u32 max_fragmented_send_size; + __u32 max_recv_size; + __u32 max_fragmented_recv_size; + __u32 max_read_write_size; + __u32 max_frmr_depth; + __u32 keepalive_interval_msec; + __u32 keepalive_timeout_msec; +} __packed; + +#define SMBDIRECT_FLAG_PORT_RANGE_MASK ( \ + SMBDIRECT_FLAG_PORT_RANGE_ONLY_IB | \ + SMBDIRECT_FLAG_PORT_RANGE_ONLY_IW) + +struct smbdirect_socket; +struct smbdirect_send_batch; +struct smbdirect_mr_io; + +#include <rdma/rw.h> + +u8 smbdirect_netdev_rdma_capable_node_type(struct net_device *netdev); + +bool smbdirect_frwr_is_supported(const struct ib_device_attr *attrs); + +int smbdirect_socket_create_kern(struct net *net, struct smbdirect_socket **_sc); + +int smbdirect_socket_create_accepting(struct rdma_cm_id *id, struct smbdirect_socket **_sc); + +int smbdirect_socket_set_initial_parameters(struct smbdirect_socket *sc, + const struct smbdirect_socket_parameters *sp); + +const struct smbdirect_socket_parameters * +smbdirect_socket_get_current_parameters(struct smbdirect_socket *sc); + +int smbdirect_socket_set_kernel_settings(struct smbdirect_socket *sc, + enum ib_poll_context poll_ctx, + gfp_t gfp_mask); + +#define SMBDIRECT_LOG_ERR 0x0 +#define SMBDIRECT_LOG_INFO 0x1 + +#define SMBDIRECT_LOG_OUTGOING 0x1 +#define SMBDIRECT_LOG_INCOMING 0x2 +#define SMBDIRECT_LOG_READ 0x4 +#define SMBDIRECT_LOG_WRITE 0x8 +#define SMBDIRECT_LOG_RDMA_SEND 0x10 +#define SMBDIRECT_LOG_RDMA_RECV 0x20 +#define SMBDIRECT_LOG_KEEP_ALIVE 0x40 +#define SMBDIRECT_LOG_RDMA_EVENT 0x80 +#define SMBDIRECT_LOG_RDMA_MR 0x100 +#define SMBDIRECT_LOG_RDMA_RW 0x200 +#define SMBDIRECT_LOG_NEGOTIATE 0x400 +void smbdirect_socket_set_logging(struct smbdirect_socket *sc, + void *private_ptr, + bool (*needed)(struct smbdirect_socket *sc, + void *private_ptr, + unsigned int lvl, + unsigned int cls), + void (*vaprintf)(struct smbdirect_socket *sc, + const char *func, + unsigned int line, + void *private_ptr, + unsigned int lvl, + unsigned int cls, + struct va_format *vaf)); + +bool smbdirect_connection_is_connected(struct smbdirect_socket *sc); + +int smbdirect_connection_wait_for_connected(struct smbdirect_socket *sc); + +int smbdirect_socket_bind(struct smbdirect_socket *sc, struct sockaddr *addr); + +void smbdirect_socket_shutdown(struct smbdirect_socket *sc); + +void smbdirect_socket_release(struct smbdirect_socket *sc); + +int smbdirect_connection_send_batch_flush(struct smbdirect_socket *sc, + struct smbdirect_send_batch *batch, + bool is_last); + +/* + * This is only temporary and only needed + * as long as the client still requires + * to use smbdirect_connection_send_single_iter() + */ +struct smbdirect_send_batch_storage { + union { + struct list_head __msg_list; + __aligned_u64 __space[5]; + }; +}; + +struct smbdirect_send_batch * +smbdirect_init_send_batch_storage(struct smbdirect_send_batch_storage *storage, + bool need_invalidate_rkey, + unsigned int remote_key); + +int smbdirect_connection_send_single_iter(struct smbdirect_socket *sc, + struct smbdirect_send_batch *batch, + struct iov_iter *iter, + unsigned int flags, + u32 remaining_data_length); + +int smbdirect_connection_send_wait_zero_pending(struct smbdirect_socket *sc); + +int smbdirect_connection_send_iter(struct smbdirect_socket *sc, + struct iov_iter *iter, + unsigned int flags, + bool need_invalidate, + unsigned int remote_key); + +int smbdirect_connection_recvmsg(struct smbdirect_socket *sc, + struct msghdr *msg, + unsigned int flags); + +int smbdirect_connect(struct smbdirect_socket *sc, + const struct sockaddr *dst); + +int smbdirect_connect_sync(struct smbdirect_socket *sc, + const struct sockaddr *dst); + +int smbdirect_socket_listen(struct smbdirect_socket *sc, int backlog); + +struct smbdirect_socket *smbdirect_socket_accept(struct smbdirect_socket *lsc, + long timeo, + struct proto_accept_arg *arg); + +int smbdirect_connection_rdma_xmit(struct smbdirect_socket *sc, + void *buf, size_t buf_len, + struct smbdirect_buffer_descriptor_v1 *desc, + size_t desc_len, + bool is_read); + +struct smbdirect_mr_io * +smbdirect_connection_register_mr_io(struct smbdirect_socket *sc, + struct iov_iter *iter, + bool writing, + bool need_invalidate); + +void smbdirect_mr_io_fill_buffer_descriptor(struct smbdirect_mr_io *mr, + struct smbdirect_buffer_descriptor_v1 *v1); + +void smbdirect_connection_deregister_mr_io(struct smbdirect_mr_io *mr); + +void smbdirect_connection_legacy_debug_proc_show(struct smbdirect_socket *sc, + unsigned int rdma_readwrite_threshold, + struct seq_file *m); + +#endif /* __LINUX_SMBDIRECT_H__ */ diff --git a/include/linux/smp.h b/include/linux/smp.h index 1ebd88026119..2dfa7390717a 100644 --- a/include/linux/smp.h +++ b/include/linux/smp.h @@ -47,8 +47,7 @@ extern void __smp_call_single_queue(int cpu, struct llist_node *node); /* total number of cpus in this system (may exceed NR_CPUS) */ extern unsigned int total_cpus; -int smp_call_function_single(int cpuid, smp_call_func_t func, void *info, - int wait); +int smp_call_function_single(int cpuid, smp_call_func_t func, void *info, bool wait); void on_each_cpu_cond_mask(smp_cond_func_t cond_func, smp_call_func_t func, void *info, bool wait, const struct cpumask *mask); @@ -73,7 +72,7 @@ static inline void on_each_cpu(smp_call_func_t func, void *info, int wait) } /** - * on_each_cpu_mask(): Run a function on processors specified by + * on_each_cpu_mask() - Run a function on processors specified by * cpumask, which may include the local processor. * @mask: The set of cpus to run on (only runs on online subset). * @func: The function to run. This must be fast and non-blocking. @@ -239,13 +238,42 @@ static inline int get_boot_cpu_id(void) #endif /* !SMP */ -/** +#if defined(CONFIG_PREEMPTION) && defined(CONFIG_SMP) +int smp_task_ipi_mask_alloc(struct task_struct *task); +void smp_task_ipi_mask_free(struct task_struct *task); +#else +static inline int smp_task_ipi_mask_alloc(struct task_struct *task) +{ + return 0; +} + +static inline void smp_task_ipi_mask_free(struct task_struct *task) { } +#endif + +/* * raw_smp_processor_id() - get the current (unstable) CPU id * - * For then you know what you are doing and need an unstable + * raw_smp_processor_id() is arch-specific/arch-defined and + * may be a macro or a static inline function. + * + * For when you know what you are doing and need an unstable * CPU id. */ +/* + * Allow the architecture to differentiate between a stable and unstable read. + * For example, x86 uses an IRQ-safe asm-volatile read for the unstable but a + * regular asm read for the stable. + */ +#ifndef __smp_processor_id +#define __smp_processor_id() raw_smp_processor_id() +#endif + +#ifdef CONFIG_DEBUG_PREEMPT + extern unsigned int debug_smp_processor_id(void); +# define smp_processor_id() debug_smp_processor_id() + +#else /** * smp_processor_id() - get the current (stable) CPU id * @@ -258,23 +286,10 @@ static inline int get_boot_cpu_id(void) * - preemption is disabled; * - the task is CPU affine. * - * When CONFIG_DEBUG_PREEMPT; we verify these assumption and WARN + * When CONFIG_DEBUG_PREEMPT=y, we verify these assumptions and WARN * when smp_processor_id() is used when the CPU id is not stable. */ -/* - * Allow the architecture to differentiate between a stable and unstable read. - * For example, x86 uses an IRQ-safe asm-volatile read for the unstable but a - * regular asm read for the stable. - */ -#ifndef __smp_processor_id -#define __smp_processor_id() raw_smp_processor_id() -#endif - -#ifdef CONFIG_DEBUG_PREEMPT - extern unsigned int debug_smp_processor_id(void); -# define smp_processor_id() debug_smp_processor_id() -#else # define smp_processor_id() __smp_processor_id() #endif diff --git a/include/linux/soc/airoha/airoha_offload.h b/include/linux/soc/airoha/airoha_offload.h index d01ef4a6b3d7..7589fccfeef6 100644 --- a/include/linux/soc/airoha/airoha_offload.h +++ b/include/linux/soc/airoha/airoha_offload.h @@ -71,9 +71,9 @@ static inline void airoha_ppe_dev_check_skb(struct airoha_ppe_dev *dev, #define NPU_RX1_DESC_NUM 512 /* CTRL */ -#define NPU_RX_DMA_DESC_LAST_MASK BIT(27) -#define NPU_RX_DMA_DESC_LEN_MASK GENMASK(26, 14) -#define NPU_RX_DMA_DESC_CUR_LEN_MASK GENMASK(13, 1) +#define NPU_RX_DMA_DESC_LAST_MASK BIT(29) +#define NPU_RX_DMA_DESC_LEN_MASK GENMASK(28, 15) +#define NPU_RX_DMA_DESC_CUR_LEN_MASK GENMASK(14, 1) #define NPU_RX_DMA_DESC_DONE_MASK BIT(0) /* INFO */ #define NPU_RX_DMA_PKT_COUNT_MASK GENMASK(31, 29) diff --git a/include/linux/soc/mediatek/mtk-cmdq.h b/include/linux/soc/mediatek/mtk-cmdq.h index a06b5a61f337..03bb85462566 100644 --- a/include/linux/soc/mediatek/mtk-cmdq.h +++ b/include/linux/soc/mediatek/mtk-cmdq.h @@ -446,6 +446,24 @@ static inline int cmdq_pkt_jump(struct cmdq_pkt *pkt, dma_addr_t addr, u8 shift_ int cmdq_pkt_jump_rel(struct cmdq_pkt *pkt, s32 offset, u8 shift_pa); /** + * cmdq_pkt_jump_rel_temp() - Temporary wrapper for new CMDQ helper API + * @pkt: the CMDQ packet + * @offset: relative offset of target instruction buffer from current PC. + * @shift_pa: [DEPRECATED] shift bits of physical address in CMDQ instruction. + * This value is got by cmdq_get_shift_pa(). + * + * This function is a temporary wrapper that was introduced only for ease of + * migration of the many users of the CMDQ API located in multiple kernel + * subsystems. + * + * This has to be removed after all users are migrated to the newer CMDQ API. + */ +static inline int cmdq_pkt_jump_rel_temp(struct cmdq_pkt *pkt, s32 offset, u8 shift_pa) +{ + return cmdq_pkt_jump_rel(pkt, offset, shift_pa); +} + +/** * cmdq_pkt_eoc() - Append EOC and ask GCE to generate an IRQ at end of execution * @pkt: The CMDQ packet * @@ -599,6 +617,12 @@ static inline int cmdq_pkt_jump_rel(struct cmdq_pkt *pkt, s32 offset, u8 shift_p return -EINVAL; } +/* This wrapper has to be removed after all users migrated to jump_rel */ +static inline int cmdq_pkt_jump_rel_temp(struct cmdq_pkt *pkt, s32 offset, u8 shift_pa) +{ + return -EINVAL; +} + static inline int cmdq_pkt_eoc(struct cmdq_pkt *pkt) { return -EINVAL; diff --git a/include/linux/soc/mediatek/mtk_sip_svc.h b/include/linux/soc/mediatek/mtk_sip_svc.h index abe24a73ee19..6c95a29b79fa 100644 --- a/include/linux/soc/mediatek/mtk_sip_svc.h +++ b/include/linux/soc/mediatek/mtk_sip_svc.h @@ -22,6 +22,9 @@ ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL, MTK_SIP_SMC_CONVENTION, \ ARM_SMCCC_OWNER_SIP, fn_id) +/* Modem related SMC call */ +#define MTK_SIP_KERNEL_CCCI_CONTROL MTK_SIP_SMC_CMD(0x505) + /* DVFSRC SMC calls */ #define MTK_SIP_DVFSRC_VCOREFS_CONTROL MTK_SIP_SMC_CMD(0x506) diff --git a/include/linux/soc/pxa/cpu.h b/include/linux/soc/pxa/cpu.h index 5782450ee45c..c5b275a80854 100644 --- a/include/linux/soc/pxa/cpu.h +++ b/include/linux/soc/pxa/cpu.h @@ -46,14 +46,6 @@ * PXA31x A2 0x69056892 0x2E649013 * PXA32x B1 0x69056825 0x5E642013 * PXA32x B2 0x69056826 0x6E642013 - * - * PXA930 B0 0x69056835 0x5E643013 - * PXA930 B1 0x69056837 0x7E643013 - * PXA930 B2 0x69056838 0x8E643013 - * - * PXA935 A0 0x56056931 0x1E653013 - * PXA935 B0 0x56056936 0x6E653013 - * PXA935 B1 0x56056938 0x8E653013 */ #ifdef CONFIG_PXA25x #define __cpu_is_pxa210(id) \ @@ -126,26 +118,6 @@ #define __cpu_is_pxa320(id) (0) #endif -#ifdef CONFIG_CPU_PXA930 -#define __cpu_is_pxa930(id) \ - ({ \ - unsigned int _id = (id) >> 4 & 0xfff; \ - _id == 0x683; \ - }) -#else -#define __cpu_is_pxa930(id) (0) -#endif - -#ifdef CONFIG_CPU_PXA935 -#define __cpu_is_pxa935(id) \ - ({ \ - unsigned int _id = (id) >> 4 & 0xfff; \ - _id == 0x693; \ - }) -#else -#define __cpu_is_pxa935(id) (0) -#endif - #define cpu_is_pxa210() \ ({ \ __cpu_is_pxa210(read_cpuid_id()); \ @@ -186,18 +158,6 @@ __cpu_is_pxa320(read_cpuid_id()); \ }) -#define cpu_is_pxa930() \ - ({ \ - __cpu_is_pxa930(read_cpuid_id()); \ - }) - -#define cpu_is_pxa935() \ - ({ \ - __cpu_is_pxa935(read_cpuid_id()); \ - }) - - - /* * CPUID Core Generation Bit * <= 0x2 for pxa21x/pxa25x/pxa26x/pxa27x @@ -217,23 +177,12 @@ ({ \ __cpu_is_pxa300(id) \ || __cpu_is_pxa310(id) \ - || __cpu_is_pxa320(id) \ - || __cpu_is_pxa93x(id); \ + || __cpu_is_pxa320(id); \ }) #else #define __cpu_is_pxa3xx(id) (0) #endif -#if defined(CONFIG_CPU_PXA930) || defined(CONFIG_CPU_PXA935) -#define __cpu_is_pxa93x(id) \ - ({ \ - __cpu_is_pxa930(id) \ - || __cpu_is_pxa935(id); \ - }) -#else -#define __cpu_is_pxa93x(id) (0) -#endif - #define cpu_is_pxa2xx() \ ({ \ __cpu_is_pxa2xx(read_cpuid_id()); \ @@ -244,9 +193,4 @@ __cpu_is_pxa3xx(read_cpuid_id()); \ }) -#define cpu_is_pxa93x() \ - ({ \ - __cpu_is_pxa93x(read_cpuid_id()); \ - }) - #endif diff --git a/include/linux/soc/qcom/apr.h b/include/linux/soc/qcom/apr.h index 6e1b1202e818..909e84f84e0c 100644 --- a/include/linux/soc/qcom/apr.h +++ b/include/linux/soc/qcom/apr.h @@ -5,7 +5,7 @@ #include <linux/spinlock.h> #include <linux/device.h> -#include <linux/mod_devicetable.h> +#include <linux/device-id/apr.h> #include <dt-bindings/soc/qcom,apr.h> #include <dt-bindings/soc/qcom,gpr.h> @@ -191,7 +191,7 @@ int apr_send_pkt(struct apr_device *adev, struct apr_pkt *pkt); gpr_port_t *gpr_alloc_port(gpr_device_t *gdev, struct device *dev, gpr_port_cb cb, void *priv); void gpr_free_port(gpr_port_t *port); -int gpr_send_port_pkt(gpr_port_t *port, struct gpr_pkt *pkt); -int gpr_send_pkt(gpr_device_t *gdev, struct gpr_pkt *pkt); +int gpr_send_port_pkt(gpr_port_t *port, const struct gpr_pkt *pkt); +int gpr_send_pkt(gpr_device_t *gdev, const struct gpr_pkt *pkt); #endif /* __QCOM_APR_H_ */ diff --git a/include/linux/soc/qcom/geni-se.h b/include/linux/soc/qcom/geni-se.h index 0a984e2579fe..29a53bbc0dd4 100644 --- a/include/linux/soc/qcom/geni-se.h +++ b/include/linux/soc/qcom/geni-se.h @@ -60,28 +60,37 @@ struct geni_icc_path { * @dev: Pointer to the Serial Engine device * @wrapper: Pointer to the parent QUP Wrapper core * @clk: Handle to the core serial engine clock + * @core_clk: Auxiliary clock, which may be required by a protocol * @num_clk_levels: Number of valid clock levels in clk_perf_tbl * @clk_perf_tbl: Table of clock frequency input to serial engine clock * @icc_paths: Array of ICC paths for SE + * @pd_list: Power domain list for managing power domains + * @has_opp: Indicates if OPP is supported */ struct geni_se { void __iomem *base; struct device *dev; struct geni_wrapper *wrapper; struct clk *clk; + struct clk *core_clk; unsigned int num_clk_levels; unsigned long *clk_perf_tbl; struct geni_icc_path icc_paths[3]; + struct dev_pm_domain_list *pd_list; + bool has_opp; }; /* Common SE registers */ +#define GENI_GENERAL_CFG 0x10 #define GENI_FORCE_DEFAULT_REG 0x20 #define GENI_OUTPUT_CTRL 0x24 #define SE_GENI_STATUS 0x40 #define GENI_SER_M_CLK_CFG 0x48 #define GENI_SER_S_CLK_CFG 0x4c +#define GENI_CLK_CTRL_RO 0x60 #define GENI_IF_DISABLE_RO 0x64 #define GENI_FW_REVISION_RO 0x68 +#define GENI_FW_MULTILOCK_MSA_RO 0x74 #define SE_GENI_CLK_SEL 0x7c #define SE_GENI_CFG_SEQ_START 0x84 #define SE_GENI_DMA_MODE_EN 0x258 @@ -92,6 +101,8 @@ struct geni_se { #define SE_GENI_M_IRQ_CLEAR 0x618 #define SE_GENI_M_IRQ_EN_SET 0x61c #define SE_GENI_M_IRQ_EN_CLEAR 0x620 +#define M_CMD_ERR_STATUS 0x624 +#define M_FW_ERR_STATUS 0x628 #define SE_GENI_S_CMD0 0x630 #define SE_GENI_S_CMD_CTRL_REG 0x634 #define SE_GENI_S_IRQ_STATUS 0x640 @@ -109,15 +120,43 @@ struct geni_se { #define SE_GENI_IOS 0x908 #define SE_GENI_M_GP_LENGTH 0x910 #define SE_GENI_S_GP_LENGTH 0x914 +/* TX DMA registers */ +#define SE_DMA_TX_PTR_L 0xc30 +#define SE_DMA_TX_PTR_H 0xc34 +#define SE_DMA_TX_ATTR 0xc38 +#define SE_DMA_TX_LEN 0xc3c #define SE_DMA_TX_IRQ_STAT 0xc40 #define SE_DMA_TX_IRQ_CLR 0xc44 +#define SE_DMA_TX_IRQ_EN 0xc48 +#define SE_DMA_TX_IRQ_EN_SET 0xc4c +#define SE_DMA_TX_IRQ_EN_CLR 0xc50 +#define SE_DMA_TX_LEN_IN 0xc54 #define SE_DMA_TX_FSM_RST 0xc58 +#define SE_DMA_TX_MAX_BURST 0xc5c +/* RX DMA registers */ +#define SE_DMA_RX_PTR_L 0xd30 +#define SE_DMA_RX_PTR_H 0xd34 +#define SE_DMA_RX_ATTR 0xd38 +#define SE_DMA_RX_LEN 0xd3c #define SE_DMA_RX_IRQ_STAT 0xd40 #define SE_DMA_RX_IRQ_CLR 0xd44 +#define SE_DMA_RX_IRQ_EN 0xd48 +#define SE_DMA_RX_IRQ_EN_SET 0xd4c +#define SE_DMA_RX_IRQ_EN_CLR 0xd50 #define SE_DMA_RX_LEN_IN 0xd54 #define SE_DMA_RX_FSM_RST 0xd58 +#define SE_DMA_RX_MAX_BURST 0xd5c +/* DMA general / debug registers */ +#define SE_GSI_EVENT_EN 0xe18 +#define SE_IRQ_EN 0xe1c +#define DMA_IF_EN_RO 0xe20 #define SE_HW_PARAM_0 0xe24 #define SE_HW_PARAM_1 0xe28 +#define SE_HW_PARAM_2 0xe2c +#define DMA_GENERAL_CFG 0xe30 +#define SE_DMA_QSB_TRANS_CFG 0xe38 +#define SE_DMA_DEBUG_REG0 0xe40 +#define SE_DMA_IF_EN 0x2004 /* GENI_FORCE_DEFAULT_REG fields */ #define FORCE_DEFAULT BIT(0) @@ -263,6 +302,12 @@ struct geni_se { #define RX_GENI_GP_IRQ_EXT GENMASK(13, 12) #define RX_GENI_CANCEL_IRQ BIT(14) +/* SE_DMA_DEBUG_REG0 fields */ +#define DMA_TX_ACTIVE BIT(0) +#define DMA_RX_ACTIVE BIT(1) +#define DMA_TX_STATE GENMASK(7, 4) +#define DMA_RX_STATE GENMASK(11, 8) + /* SE_HW_PARAM_0 fields */ #define TX_FIFO_WIDTH_MSK GENMASK(29, 24) #define TX_FIFO_WIDTH_SHFT 24 @@ -285,6 +330,9 @@ struct geni_se { #define RX_FIFO_DEPTH_MSK GENMASK(21, 16) #define RX_FIFO_DEPTH_SHFT 16 +/* SE_HW_PARAM_2 fields */ +#define PROG_RAM_DEPTH_MSK GENMASK(10, 0) + #define HW_VER_MAJOR_MASK GENMASK(31, 28) #define HW_VER_MAJOR_SHFT 28 #define HW_VER_MINOR_MASK GENMASK(27, 16) @@ -528,12 +576,25 @@ void geni_se_rx_dma_unprep(struct geni_se *se, dma_addr_t iova, size_t len); int geni_icc_get(struct geni_se *se, const char *icc_ddr); int geni_icc_set_bw(struct geni_se *se); +int geni_icc_set_bw_ab(struct geni_se *se, u32 core_ab, u32 cfg_ab, u32 ddr_ab); void geni_icc_set_tag(struct geni_se *se, u32 tag); int geni_icc_enable(struct geni_se *se); int geni_icc_disable(struct geni_se *se); +int geni_se_resources_init(struct geni_se *se); + +int geni_se_resources_activate(struct geni_se *se); + +int geni_se_resources_deactivate(struct geni_se *se); + int geni_load_se_firmware(struct geni_se *se, enum geni_se_protocol_type protocol); + +int geni_se_domain_attach(struct geni_se *se); + +int geni_se_set_perf_level(struct geni_se *se, unsigned long level); + +int geni_se_set_perf_opp(struct geni_se *se, unsigned long clk_freq); #endif #endif diff --git a/include/linux/soc/qcom/llcc-qcom.h b/include/linux/soc/qcom/llcc-qcom.h index 8243ab3a12a8..f3ed63e475ab 100644 --- a/include/linux/soc/qcom/llcc-qcom.h +++ b/include/linux/soc/qcom/llcc-qcom.h @@ -89,16 +89,20 @@ /** * struct llcc_slice_desc - Cache slice descriptor - * @slice_id: llcc slice id - * @slice_size: Size allocated for the llcc slice + * @slice_id: LLCC slice id + * @uid: Unique ID associated with the LLCC device + * @slice_size: Size allocated for the LLCC slice + * @refcount: Atomic counter to track activate/deactivate calls */ struct llcc_slice_desc { u32 slice_id; + u32 uid; size_t slice_size; + refcount_t refcount; }; /** - * struct llcc_edac_reg_data - llcc edac registers data for each error type + * struct llcc_edac_reg_data - LLCC EDAC registers data for each error type * @name: Name of the error * @reg_cnt: Number of registers * @count_mask: Mask value to get the error count @@ -144,22 +148,23 @@ struct llcc_edac_reg_offset { }; /** - * struct llcc_drv_data - Data associated with the llcc driver - * @regmaps: regmaps associated with the llcc device - * @bcast_regmap: regmap associated with llcc broadcast OR offset - * @bcast_and_regmap: regmap associated with llcc broadcast AND offset + * struct llcc_drv_data - Data associated with the LLCC driver + * @dev: device back-pointer for this LLCC instance + * @regmaps: regmaps associated with the LLCC device + * @bcast_regmap: regmap associated with LLCC broadcast OR offset + * @bcast_and_regmap: regmap associated with LLCC broadcast AND offset * @cfg: pointer to the data structure for slice configuration * @edac_reg_offset: Offset of the LLCC EDAC registers * @lock: mutex associated with each slice * @cfg_size: size of the config data table - * @max_slices: max slices as read from device tree - * @num_banks: Number of llcc banks - * @bitmap: Bit map to track the active slice ids - * @ecc_irq: interrupt for llcc cache error detection and reporting + * @num_banks: Number of LLCC banks + * @ecc_irq: interrupt for LLCC cache error detection and reporting * @ecc_irq_configured: 'True' if firmware has already configured the irq propagation * @version: Indicates the LLCC version + * @desc: Array pointer of pre-allocated LLCC slice descriptors */ struct llcc_drv_data { + struct device *dev; struct regmap **regmaps; struct regmap *bcast_regmap; struct regmap *bcast_and_regmap; @@ -167,48 +172,47 @@ struct llcc_drv_data { const struct llcc_edac_reg_offset *edac_reg_offset; struct mutex lock; u32 cfg_size; - u32 max_slices; u32 num_banks; - unsigned long *bitmap; int ecc_irq; bool ecc_irq_configured; u32 version; + struct llcc_slice_desc *desc; }; #if IS_ENABLED(CONFIG_QCOM_LLCC) /** - * llcc_slice_getd - get llcc slice descriptor + * llcc_slice_getd - get LLCC slice descriptor * @uid: usecase_id of the client */ struct llcc_slice_desc *llcc_slice_getd(u32 uid); /** - * llcc_slice_putd - llcc slice descritpor - * @desc: Pointer to llcc slice descriptor + * llcc_slice_putd - LLCC slice descriptor + * @desc: Pointer to LLCC slice descriptor */ void llcc_slice_putd(struct llcc_slice_desc *desc); /** * llcc_get_slice_id - get slice id - * @desc: Pointer to llcc slice descriptor + * @desc: Pointer to LLCC slice descriptor */ int llcc_get_slice_id(struct llcc_slice_desc *desc); /** - * llcc_get_slice_size - llcc slice size - * @desc: Pointer to llcc slice descriptor + * llcc_get_slice_size - LLCC slice size + * @desc: Pointer to LLCC slice descriptor */ size_t llcc_get_slice_size(struct llcc_slice_desc *desc); /** - * llcc_slice_activate - Activate the llcc slice - * @desc: Pointer to llcc slice descriptor + * llcc_slice_activate - Activate the LLCC slice + * @desc: Pointer to LLCC slice descriptor */ int llcc_slice_activate(struct llcc_slice_desc *desc); /** - * llcc_slice_deactivate - Deactivate the llcc slice - * @desc: Pointer to llcc slice descriptor + * llcc_slice_deactivate - Deactivate the LLCC slice + * @desc: Pointer to LLCC slice descriptor */ int llcc_slice_deactivate(struct llcc_slice_desc *desc); diff --git a/include/linux/soc/qcom/mdt_loader.h b/include/linux/soc/qcom/mdt_loader.h index 82372e0db0a1..74886f772d46 100644 --- a/include/linux/soc/qcom/mdt_loader.h +++ b/include/linux/soc/qcom/mdt_loader.h @@ -10,7 +10,7 @@ struct device; struct firmware; -struct qcom_scm_pas_context; +struct qcom_pas_context; #if IS_ENABLED(CONFIG_QCOM_MDT_LOADER) @@ -20,8 +20,8 @@ int qcom_mdt_load(struct device *dev, const struct firmware *fw, phys_addr_t mem_phys, size_t mem_size, phys_addr_t *reloc_base); -int qcom_mdt_pas_load(struct qcom_scm_pas_context *ctx, const struct firmware *fw, - const char *firmware, void *mem_region, phys_addr_t *reloc_base); +int qcom_mdt_pas_load(struct qcom_pas_context *ctx, const struct firmware *fw, + const char *firmware, phys_addr_t *reloc_base); int qcom_mdt_load_no_init(struct device *dev, const struct firmware *fw, const char *fw_name, void *mem_region, @@ -45,9 +45,9 @@ static inline int qcom_mdt_load(struct device *dev, const struct firmware *fw, return -ENODEV; } -static inline int qcom_mdt_pas_load(struct qcom_scm_pas_context *ctx, +static inline int qcom_mdt_pas_load(struct qcom_pas_context *ctx, const struct firmware *fw, const char *firmware, - void *mem_region, phys_addr_t *reloc_base) + phys_addr_t *reloc_base) { return -ENODEV; } diff --git a/include/linux/soc/qcom/pdr.h b/include/linux/soc/qcom/pdr.h index 83a8ea612e69..2b7691e47c2a 100644 --- a/include/linux/soc/qcom/pdr.h +++ b/include/linux/soc/qcom/pdr.h @@ -5,6 +5,7 @@ #include <linux/soc/qcom/qmi.h> #define SERVREG_NAME_LENGTH 64 +#define SERVREG_PFR_LENGTH 256 struct pdr_service; struct pdr_handle; diff --git a/include/linux/soc/qcom/qmi.h b/include/linux/soc/qcom/qmi.h index 291cdc7ef49c..b9dcb437a0be 100644 --- a/include/linux/soc/qcom/qmi.h +++ b/include/linux/soc/qcom/qmi.h @@ -92,6 +92,18 @@ struct qmi_elem_info { #define QMI_ERR_INCOMPATIBLE_STATE_V01 90 #define QMI_ERR_NOT_SUPPORTED_V01 94 +/* + * Enumerate the IDs of the QMI services + */ +#define QMI_SERVICE_ID_TEST 0x0f /* 15 */ +#define QMI_SERVICE_ID_SSCTL 0x2b /* 43 */ +#define QMI_SERVICE_ID_IPA 0x31 /* 49 */ +#define QMI_SERVICE_ID_SERVREG_LOC 0x40 /* 64 */ +#define QMI_SERVICE_ID_SERVREG_NOTIF 0x42 /* 66 */ +#define QMI_SERVICE_ID_WLFW 0x45 /* 69 */ +#define QMI_SERVICE_ID_SLIMBUS 0x301 /* 769 */ +#define QMI_SERVICE_ID_USB_AUDIO_STREAM 0x41d /* 1053 */ + /** * struct qmi_response_type_v01 - common response header (decoded) * @result: result of the transaction diff --git a/include/linux/soc/qcom/smem.h b/include/linux/soc/qcom/smem.h index f946e3beca21..3249cbd2016b 100644 --- a/include/linux/soc/qcom/smem.h +++ b/include/linux/soc/qcom/smem.h @@ -17,4 +17,6 @@ int qcom_smem_get_feature_code(u32 *code); int qcom_smem_bust_hwspin_lock_by_host(unsigned int host); +int qcom_smem_dram_get_hbb(void); + #endif diff --git a/include/linux/soc/qcom/ubwc.h b/include/linux/soc/qcom/ubwc.h index f052e241736c..f3a70360b177 100644 --- a/include/linux/soc/qcom/ubwc.h +++ b/include/linux/soc/qcom/ubwc.h @@ -8,25 +8,12 @@ #define __QCOM_UBWC_H__ #include <linux/bits.h> +#include <linux/err.h> #include <linux/printk.h> #include <linux/types.h> struct qcom_ubwc_cfg_data { u32 ubwc_enc_version; - /* Can be read from MDSS_BASE + 0x58 */ - u32 ubwc_dec_version; - - /** - * @ubwc_swizzle: Whether to enable level 1, 2 & 3 bank swizzling. - * - * UBWC 1.0 always enables all three levels. - * UBWC 2.0 removes level 1 bank swizzling, leaving levels 2 & 3. - * UBWC 4.0 adds the optional ability to disable levels 2 & 3. - */ - u32 ubwc_swizzle; -#define UBWC_SWIZZLE_ENABLE_LVL1 BIT(0) -#define UBWC_SWIZZLE_ENABLE_LVL2 BIT(1) -#define UBWC_SWIZZLE_ENABLE_LVL3 BIT(2) /** * @highest_bank_bit: Highest Bank Bit @@ -35,21 +22,16 @@ struct qcom_ubwc_cfg_data { * DDR bank. This should ideally use DRAM type detection. */ int highest_bank_bit; - bool ubwc_bank_spread; - /** - * @macrotile_mode: Macrotile Mode - * - * Whether to use 4-channel macrotiling mode or the newer - * 8-channel macrotiling mode introduced in UBWC 3.1. 0 is - * 4-channel and 1 is 8-channel. - */ - bool macrotile_mode; + unsigned int flags; +#define UBWC_FLAG_DISABLE_SWIZZLE_LVL2 BIT(0) +#define UBWC_FLAG_DISABLE_SWIZZLE_LVL3 BIT(1) }; #define UBWC_1_0 0x10000000 #define UBWC_2_0 0x20000000 #define UBWC_3_0 0x30000000 +#define UBWC_3_1 0x30010000 /* UBWC 3.0 + Macrotile mode */ #define UBWC_4_0 0x40000000 #define UBWC_4_3 0x40030000 #define UBWC_5_0 0x50000000 @@ -66,12 +48,83 @@ static inline const struct qcom_ubwc_cfg_data *qcom_ubwc_config_get_data(void) static inline bool qcom_ubwc_get_ubwc_mode(const struct qcom_ubwc_cfg_data *cfg) { - bool ret = cfg->ubwc_enc_version == UBWC_1_0; + return cfg->ubwc_enc_version == UBWC_1_0; +} - if (ret && !(cfg->ubwc_swizzle & UBWC_SWIZZLE_ENABLE_LVL1)) - pr_err("UBWC config discrepancy - level 1 swizzling disabled on UBWC 1.0\n"); +static inline bool qcom_ubwc_min_acc_length_64b(const struct qcom_ubwc_cfg_data *cfg) +{ + return cfg->ubwc_enc_version == UBWC_1_0; +} + +/* + * @qcom_ubwc_macrotile_mode: whether to use 4-channel or 8-channel macrotiling + * + * The 8-channel macrotiling mode was introduced in UBWC 3.1. + * + * Returns: false for the 4-channel and true for 8-channel. + */ +static inline bool qcom_ubwc_macrotile_mode(const struct qcom_ubwc_cfg_data *cfg) +{ + return cfg->ubwc_enc_version >= UBWC_3_1; +} - return ret; +static inline bool qcom_ubwc_bank_spread(const struct qcom_ubwc_cfg_data *cfg) +{ + return true; +} + +#define UBWC_SWIZZLE_ENABLE_LVL1 BIT(0) +#define UBWC_SWIZZLE_ENABLE_LVL2 BIT(1) +#define UBWC_SWIZZLE_ENABLE_LVL3 BIT(2) + +/** + * @qcom_ubwc_swizzle: Whether to enable level 1, 2 & 3 bank swizzling. + * + * UBWC 1.0 always enables all three levels. + * UBWC 2.0 removes level 1 bank swizzling, leaving levels 2 & 3. + * UBWC 4.0 adds the optional ability to disable levels 2 & 3. + */ +static inline u32 qcom_ubwc_swizzle(const struct qcom_ubwc_cfg_data *cfg) +{ + if (cfg->ubwc_enc_version == 0) + return 0; + + if (cfg->ubwc_enc_version == UBWC_1_0) + return UBWC_SWIZZLE_ENABLE_LVL1 | + UBWC_SWIZZLE_ENABLE_LVL2 | + UBWC_SWIZZLE_ENABLE_LVL3; + + u32 ubwc_swizzle = UBWC_SWIZZLE_ENABLE_LVL2 | + UBWC_SWIZZLE_ENABLE_LVL3; + + if (cfg->flags & UBWC_FLAG_DISABLE_SWIZZLE_LVL2) + ubwc_swizzle &= ~UBWC_SWIZZLE_ENABLE_LVL2; + + if (cfg->flags & UBWC_FLAG_DISABLE_SWIZZLE_LVL3) + ubwc_swizzle &= ~UBWC_SWIZZLE_ENABLE_LVL3; + + return ubwc_swizzle; +} + +static inline u32 qcom_ubwc_version_tag(const struct qcom_ubwc_cfg_data *cfg) +{ + if (cfg->ubwc_enc_version >= UBWC_6_0) + return 5; + if (cfg->ubwc_enc_version >= UBWC_5_0) + return 4; + if (cfg->ubwc_enc_version >= UBWC_4_3) + return 3; + if (cfg->ubwc_enc_version >= UBWC_4_0) + return 2; + if (cfg->ubwc_enc_version >= UBWC_3_0) + return 1; + + return 0; +} + +static inline bool qcom_ubwc_enable_amsbc(const struct qcom_ubwc_cfg_data *cfg) +{ + return cfg->ubwc_enc_version >= UBWC_3_0; } #endif /* __QCOM_UBWC_H__ */ diff --git a/include/linux/soc/ti/knav_dma.h b/include/linux/soc/ti/knav_dma.h index 18d806a8e52c..eb1e6b014eaf 100644 --- a/include/linux/soc/ti/knav_dma.h +++ b/include/linux/soc/ti/knav_dma.h @@ -75,7 +75,7 @@ enum knav_dma_desc_type { * struct knav_dma_tx_cfg: Tx channel configuration * @filt_einfo: Filter extended packet info * @filt_pswords: Filter PS words present - * @knav_dma_tx_priority: Tx channel scheduling priority + * @priority: Tx channel scheduling priority */ struct knav_dma_tx_cfg { bool filt_einfo; @@ -87,13 +87,13 @@ struct knav_dma_tx_cfg { * struct knav_dma_rx_cfg: Rx flow configuration * @einfo_present: Extended packet info present * @psinfo_present: PS words present - * @knav_dma_rx_err_mode: Error during buffer starvation - * @knav_dma_desc_type: Host or Monolithic desc + * @err_mode: Error during buffer starvation + * @desc_type: Host or Monolithic desc * @psinfo_at_sop: PS word located at start of packet * @sop_offset: Start of packet offset * @dst_q: Destination queue for a given flow * @thresh: Rx flow size threshold - * @fdq[]: Free desc Queue array + * @fdq: Free desc Queue array * @sz_thresh0: RX packet size threshold 0 * @sz_thresh1: RX packet size threshold 1 * @sz_thresh2: RX packet size threshold 2 @@ -115,7 +115,8 @@ struct knav_dma_rx_cfg { /** * struct knav_dma_cfg: Pktdma channel configuration - * @sl_cfg: Slave configuration + * @direction: DMA transfer mode and direction + * @u: union containing @tx or @rx * @tx: Tx channel configuration * @rx: Rx flow configuration */ diff --git a/include/linux/socket.h b/include/linux/socket.h index ec715ad4bf25..5a5eb1250103 100644 --- a/include/linux/socket.h +++ b/include/linux/socket.h @@ -89,7 +89,6 @@ struct msghdr { bool msg_get_inq : 1;/* return INQ after receive */ unsigned int msg_flags; /* flags on received message */ __kernel_size_t msg_controllen; /* ancillary data buffer length */ - struct kiocb *msg_iocb; /* ptr to iocb for async requests */ struct ubuf_info *msg_ubuf; int (*sg_from_iter)(struct sk_buff *skb, struct iov_iter *from, size_t length); @@ -357,7 +356,7 @@ struct ucred { /* Flags to be cleared on entry by sendmsg and sendmmsg syscalls */ #define MSG_INTERNAL_SENDMSG_FLAGS \ - (MSG_SPLICE_PAGES | MSG_SENDPAGE_NOPOLICY | MSG_SENDPAGE_DECRYPTED) + (MSG_SPLICE_PAGES | MSG_SENDPAGE_NOPOLICY | MSG_SENDPAGE_DECRYPTED | MSG_NO_SHARED_FRAGS) /* Setsockoptions(2) level. Thanks to BSD these must match IPPROTO_xxx */ #define SOL_IP 0 @@ -415,7 +414,7 @@ struct __kernel_timespec; struct old_timespec32; struct scm_timestamping_internal { - struct timespec64 ts[3]; + ktime_t ts[3]; }; extern void put_cmsg_scm_timestamping64(struct msghdr *msg, struct scm_timestamping_internal *tss); @@ -462,6 +461,8 @@ extern struct file *__sys_socket_file(int family, int type, int protocol); extern int __sys_bind(int fd, struct sockaddr __user *umyaddr, int addrlen); extern int __sys_bind_socket(struct socket *sock, struct sockaddr_storage *address, int addrlen); +int connect_socket(struct socket *sock, struct sockaddr_storage *addr, + int addrlen, int flags); extern int __sys_connect_file(struct file *file, struct sockaddr_storage *addr, int addrlen, int file_flags); extern int __sys_connect(int fd, struct sockaddr __user *uservaddr, diff --git a/include/linux/sockptr.h b/include/linux/sockptr.h index 3e6c8e9d67ae..9c2429c1a570 100644 --- a/include/linux/sockptr.h +++ b/include/linux/sockptr.h @@ -87,24 +87,10 @@ static inline int copy_safe_from_sockptr(void *dst, size_t ksize, static inline int copy_struct_from_sockptr(void *dst, size_t ksize, sockptr_t src, size_t usize) { - size_t size = min(ksize, usize); - size_t rest = max(ksize, usize) - size; - if (!sockptr_is_kernel(src)) - return copy_struct_from_user(dst, ksize, src.user, size); - - if (usize < ksize) { - memset(dst + size, 0, rest); - } else if (usize > ksize) { - char *p = src.kernel; + return copy_struct_from_user(dst, ksize, src.user, usize); - while (rest--) { - if (*p++) - return -E2BIG; - } - } - memcpy(dst, src.kernel, size); - return 0; + return copy_struct_from_bounce_buffer(dst, ksize, src.kernel, usize); } static inline int copy_to_sockptr_offset(sockptr_t dst, size_t offset, @@ -121,6 +107,16 @@ static inline int copy_to_sockptr(sockptr_t dst, const void *src, size_t size) return copy_to_sockptr_offset(dst, 0, src, size); } +static inline int +copy_struct_to_sockptr(sockptr_t dst, size_t usize, const void *src, + size_t ksize, bool *ignored_trailing) +{ + if (!sockptr_is_kernel(dst)) + return copy_struct_to_user(dst.user, usize, src, ksize, ignored_trailing); + + return copy_struct_to_bounce_buffer(dst.kernel, usize, src, ksize, ignored_trailing); +} + static inline void *memdup_sockptr_noprof(sockptr_t src, size_t len) { void *p = kmalloc_track_caller_noprof(len, GFP_USER | __GFP_NOWARN); diff --git a/include/linux/sonet.h b/include/linux/sonet.h deleted file mode 100644 index 2b802b6d12ad..000000000000 --- a/include/linux/sonet.h +++ /dev/null @@ -1,20 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* sonet.h - SONET/SHD physical layer control */ -#ifndef LINUX_SONET_H -#define LINUX_SONET_H - - -#include <linux/atomic.h> -#include <uapi/linux/sonet.h> - -struct k_sonet_stats { -#define __HANDLE_ITEM(i) atomic_t i - __SONET_ITEMS -#undef __HANDLE_ITEM -}; - -extern void sonet_copy_stats(struct k_sonet_stats *from,struct sonet_stats *to); -extern void sonet_subtract_stats(struct k_sonet_stats *from, - struct sonet_stats *to); - -#endif diff --git a/include/linux/soundwire/sdw.h b/include/linux/soundwire/sdw.h index f462717acf20..707ca6bd1e23 100644 --- a/include/linux/soundwire/sdw.h +++ b/include/linux/soundwire/sdw.h @@ -11,8 +11,9 @@ #include <linux/idr.h> #include <linux/irq.h> #include <linux/irqdomain.h> +#include <linux/jiffies.h> #include <linux/lockdep_types.h> -#include <linux/mod_devicetable.h> +#include <linux/device-id/sdw.h> #include <linux/mutex.h> #include <linux/types.h> #include <sound/sdca.h> @@ -364,6 +365,10 @@ struct sdw_dpn_prop { * @commit_register_supported: is PCP_Commit register supported * @scp_int1_mask: SCP_INT1_MASK desired settings * @lane_maps: Lane mapping for the slave, only valid if lane_control_support is set + * @bra_block_alignment: If non-zero the length of data in a BRA frame must be + * a multiple of this number of bytes. + * @bra_max_data_per_frame: If non-zero the maximum data payload size (in bytes per + * frame excluding header, CRC, and footer) for this BRA Mode * @clock_reg_supported: the Peripheral implements the clock base and scale * registers introduced with the SoundWire 1.2 specification. SDCA devices * do not need to set this boolean property as the registers are required. @@ -394,6 +399,8 @@ struct sdw_slave_prop { u8 commit_register_supported; u8 scp_int1_mask; u8 lane_maps[SDW_MAX_LANES]; + u32 bra_block_alignment; + u32 bra_max_data_per_frame; bool clock_reg_supported; bool use_domain_irq; }; @@ -532,7 +539,7 @@ struct sdw_slave_intr_status { }; /** - * sdw_reg_bank - SoundWire register banks + * enum sdw_reg_bank - SoundWire register banks * @SDW_BANK0: Soundwire register bank 0 * @SDW_BANK1: Soundwire register bank 1 */ @@ -751,7 +758,7 @@ struct sdw_port_params { * struct sdw_transport_params: Data Port Transport Parameters * * @blk_grp_ctrl_valid: Port implements block group control - * @num: Port number + * @port_num: Port number * @blk_grp_ctrl: Block group control value * @sample_interval: Sample interval * @offset1: Blockoffset of the payload data @@ -782,7 +789,7 @@ struct sdw_transport_params { /** * struct sdw_enable_ch: Enable/disable Data Port channel * - * @num: Port number + * @port_num: Port number * @ch_mask: Active channel mask * @enable: Enable (true) /disable (false) channel */ @@ -836,6 +843,14 @@ struct sdw_defer { */ #define SDW_BPT_MSG_MAX_BYTES (1024 * 1024) +/* + * According to mipi SoundWire DisCo Specification_v2-1, + * this maximum value shall not exceed 470. + * Note that the largest number of bytes accessible by a single BRA operation is limited to 470 + * bytes when using lane 0, but goes up to 502 bytes when using one of the optional extra lanes. + */ +#define SDW_BRA_MAX_BYTES_PER_FRAME 470 + struct sdw_bpt_msg; /** @@ -885,7 +900,7 @@ void sdw_bus_master_delete(struct sdw_bus *bus); void sdw_show_ping_status(struct sdw_bus *bus, bool sync_delay); /** - * sdw_port_config: Master or Slave Port configuration + * struct sdw_port_config: Master or Slave Port configuration * * @num: Port number * @ch_mask: channels mask for port @@ -896,7 +911,7 @@ struct sdw_port_config { }; /** - * sdw_stream_config: Master or Slave stream configuration + * struct sdw_stream_config: Master or Slave stream configuration * * @frame_rate: Audio frame rate of the stream, in Hz * @ch_count: Channel count of the stream @@ -913,7 +928,7 @@ struct sdw_stream_config { }; /** - * sdw_stream_state: Stream states + * enum sdw_stream_state: Stream states * * @SDW_STREAM_ALLOCATED: New stream allocated. * @SDW_STREAM_CONFIGURED: Stream configured @@ -934,7 +949,7 @@ enum sdw_stream_state { }; /** - * sdw_stream_params: Stream parameters + * struct sdw_stream_params: Stream parameters * * @rate: Sampling frequency, in Hz * @ch_count: Number of channels @@ -947,7 +962,7 @@ struct sdw_stream_params { }; /** - * sdw_stream_runtime: Runtime stream parameters + * struct sdw_stream_runtime: Runtime stream parameters * * @name: SoundWire stream name * @params: Stream parameters @@ -983,7 +998,7 @@ struct sdw_stream_runtime { * @defer_msg: Defer message * @params: Current bus parameters * @stream_refcount: number of streams currently using this bus - * @btp_stream_refcount: number of BTP streams currently using this bus (should + * @bpt_stream_refcount: number of BTP streams currently using this bus (should * be zero or one, multiple streams per link is not supported). * @bpt_stream: pointer stored to handle BTP streams. * @ops: Master callback ops @@ -1199,4 +1214,33 @@ static inline int sdw_update_no_pm(struct sdw_slave *slave, u32 addr, u8 mask, u #endif /* CONFIG_SOUNDWIRE */ +/** + * sdw_slave_wait_for_init - Wait for device initialisation + * @slave: Pointer to the SoundWire peripheral. + * @timeout_ms: Timeout in milliseconds. + * + * Wait for a peripheral device to enumerate and be initialised by the + * SoundWire core. + * + * Return: Zero on success, and a negative error code on failure. + */ +static inline int sdw_slave_wait_for_init(struct sdw_slave *slave, int timeout_ms) +{ + unsigned long time; + + if (!slave) + return 0; + + time = wait_for_completion_timeout(&slave->initialization_complete, + msecs_to_jiffies(timeout_ms)); + if (!time) { + dev_err(&slave->dev, "Initialization not complete\n"); + return -ETIMEDOUT; + } + + slave->unattach_request = 0; + + return 0; +} + #endif /* __SOUNDWIRE_H */ diff --git a/include/linux/soundwire/sdw_amd.h b/include/linux/soundwire/sdw_amd.h index fe31773d5210..470360a2723c 100644 --- a/include/linux/soundwire/sdw_amd.h +++ b/include/linux/soundwire/sdw_amd.h @@ -66,8 +66,10 @@ struct sdw_amd_dai_runtime { * @status: peripheral devices status array * @num_din_ports: number of input ports * @num_dout_ports: number of output ports + * @max_ports: total number of input ports and output ports * @cols_index: Column index in frame shape * @rows_index: Rows index in frame shape + * @port_offset_map: dynamic array to map port block offset * @instance: SoundWire manager instance * @quirks: SoundWire manager quirks * @wake_en_mask: wake enable mask per SoundWire manager @@ -92,10 +94,12 @@ struct amd_sdw_manager { int num_din_ports; int num_dout_ports; + int max_ports; int cols_index; int rows_index; + int *port_offset_map; u32 instance; u32 quirks; u32 wake_en_mask; diff --git a/include/linux/spi/spi-mem.h b/include/linux/spi/spi-mem.h index 5774e554c0f0..0cce6b57242a 100644 --- a/include/linux/spi/spi-mem.h +++ b/include/linux/spi/spi-mem.h @@ -28,6 +28,14 @@ .dtr = true, \ } +#define SPI_MEM_DTR_OP_PACKED_CMD(__opcode, __addr, __buswidth) \ + { \ + .nbytes = 2, \ + .opcode = __opcode << 8 | __addr, \ + .buswidth = __buswidth, \ + .dtr = true, \ + } + #define SPI_MEM_OP_ADDR(__nbytes, __val, __buswidth) \ { \ .nbytes = __nbytes, \ @@ -130,11 +138,13 @@ enum spi_mem_data_dir { /** * struct spi_mem_op - describes a SPI memory operation + * @cmd: the complete command * @cmd.nbytes: number of opcode bytes (only 1 or 2 are valid). The opcode is * sent MSB-first. * @cmd.buswidth: number of IO lines used to transmit the command * @cmd.opcode: operation opcode * @cmd.dtr: whether the command opcode should be sent in DTR mode or not + * @addr: the address attributes * @addr.nbytes: number of address bytes to send. Can be zero if the operation * does not need to send an address * @addr.buswidth: number of IO lines used to transmit the address cycles @@ -143,10 +153,12 @@ enum spi_mem_data_dir { * Note that only @addr.nbytes are taken into account in this * address value, so users should make sure the value fits in the * assigned number of bytes. + * @dummy: data for dummy operation * @dummy.nbytes: number of dummy bytes to send after an opcode or address. Can * be zero if the operation does not require dummy bytes * @dummy.buswidth: number of IO lanes used to transmit the dummy bytes * @dummy.dtr: whether the dummy bytes should be sent in DTR mode or not + * @data: the data attributes * @data.buswidth: number of IO lanes used to send/receive the data * @data.dtr: whether the data should be sent in DTR mode or not * @data.ecc: whether error correction is required or not @@ -215,6 +227,8 @@ struct spi_mem_op { * struct spi_mem_dirmap_info - Direct mapping information * @op_tmpl: operation template that should be used by the direct mapping when * the memory device is accessed + * @secondary_op_tmpl: secondary template, may be used as an alternative to the + * primary template (decided by the upper layer) * @offset: absolute offset this direct mapping is pointing to * @length: length in byte of this direct mapping * @@ -225,7 +239,9 @@ struct spi_mem_op { * direction is directly encoded in the ->op_tmpl.data.dir field. */ struct spi_mem_dirmap_info { - struct spi_mem_op op_tmpl; + struct spi_mem_op *op_tmpl; + struct spi_mem_op primary_op_tmpl; + struct spi_mem_op secondary_op_tmpl; u64 offset; u64 length; }; @@ -259,6 +275,7 @@ struct spi_mem_dirmap_desc { * @spi: the underlying SPI device * @drvpriv: spi_mem_driver private data * @name: name of the SPI memory device + * @dqs: extra data trobe pin available for high frequency read operations * * Extra information that describe the SPI memory device and may be needed by * the controller to properly handle this device should be placed here. @@ -270,10 +287,11 @@ struct spi_mem { struct spi_device *spi; void *drvpriv; const char *name; + bool dqs; }; /** - * struct spi_mem_set_drvdata() - attach driver private data to a SPI mem + * spi_mem_set_drvdata() - attach driver private data to a SPI mem * device * @mem: memory device * @data: data to attach to the memory device @@ -284,7 +302,7 @@ static inline void spi_mem_set_drvdata(struct spi_mem *mem, void *data) } /** - * struct spi_mem_get_drvdata() - get driver private data attached to a SPI mem + * spi_mem_get_drvdata() - get driver private data attached to a SPI mem * device * @mem: memory device * @@ -369,12 +387,19 @@ struct spi_controller_mem_ops { * @swap16: Supports swapping bytes on a 16 bit boundary when configured in * Octal DTR * @per_op_freq: Supports per operation frequency switching + * @secondary_op_tmpl: Supports leveraging a secondary memory operation template + * @no_cs_assertion: The controller may automatically deassert the CS if there + * is a pause in the transfer (eg. internal bus contention or + * DMA arbitration on an interconnect). Features such as NAND + * continuous reads shall not be leveraged. */ struct spi_controller_mem_caps { bool dtr; bool ecc; bool swap16; bool per_op_freq; + bool secondary_op_tmpl; + bool no_cs_assertion; }; #define spi_mem_controller_is_capable(ctlr, cap) \ @@ -438,6 +463,8 @@ bool spi_mem_default_supports_op(struct spi_mem *mem, } #endif /* CONFIG_SPI_MEM */ +void spi_mem_set_dqs(struct spi_mem *mem); +bool spi_mem_has_dqs(struct spi_mem *mem); int spi_mem_adjust_op_size(struct spi_mem *mem, struct spi_mem_op *op); void spi_mem_adjust_op_freq(struct spi_mem *mem, struct spi_mem_op *op); u64 spi_mem_calc_op_duration(struct spi_mem *mem, struct spi_mem_op *op); diff --git a/include/linux/spi/spi.h b/include/linux/spi/spi.h index af7cfee7b8f6..88d17fce02dc 100644 --- a/include/linux/spi/spi.h +++ b/include/linux/spi/spi.h @@ -12,7 +12,9 @@ #include <linux/device.h> #include <linux/gpio/consumer.h> #include <linux/kthread.h> -#include <linux/mod_devicetable.h> +#include <linux/device-id/acpi.h> +#include <linux/device-id/of.h> +#include <linux/device-id/spi.h> #include <linux/overflow.h> #include <linux/scatterlist.h> #include <linux/slab.h> @@ -159,10 +161,6 @@ extern void spi_transfer_cs_change_delay_exec(struct spi_message *msg, * @modalias: Name of the driver to use with this device, or an alias * for that name. This appears in the sysfs "modalias" attribute * for driver coldplugging, and in uevents used for hotplugging - * @driver_override: If the name of a driver is written to this attribute, then - * the device will bind to the named driver and only the named driver. - * Do not set directly, because core frees it; use driver_set_override() to - * set or clear it. * @pcpu_statistics: statistics for the spi_device * @word_delay: delay to be inserted between consecutive * words of a transfer @@ -181,6 +179,8 @@ extern void spi_transfer_cs_change_delay_exec(struct spi_message *msg, * @num_tx_lanes: Number of transmit lanes wired up. * @rx_lane_map: Map of peripheral lanes (index) to controller lanes (value). * @num_rx_lanes: Number of receive lanes wired up. + * @userspace_node: entry on the parent controller's userspace_clients list + * when this device was instantiated via the sysfs new_device interface * * A @spi_device is used to interchange data between an SPI target device * (usually a discrete chip) and CPU memory. @@ -224,7 +224,6 @@ struct spi_device { void *controller_state; void *controller_data; char modalias[SPI_NAME_SIZE]; - const char *driver_override; /* The statistics */ struct spi_statistics __percpu *pcpu_statistics; @@ -255,6 +254,10 @@ struct spi_device { u8 rx_lane_map[SPI_DEVICE_DATA_LANE_CNT_MAX]; u8 num_rx_lanes; +#if IS_ENABLED(CONFIG_SPI_DYNAMIC) + struct list_head userspace_node; +#endif + /* * Likely need more hooks for more protocol options affecting how * the controller talks to each chip, like: @@ -387,6 +390,7 @@ static inline void spi_unregister_driver(struct spi_driver *sdrv) } extern struct spi_device *spi_new_ancillary_device(struct spi_device *spi, u8 chip_select); +extern struct spi_device *devm_spi_new_ancillary_device(struct spi_device *spi, u8 chip_select); /* Use a define to avoid include chaining to get THIS_MODULE */ #define spi_register_driver(driver) \ @@ -428,7 +432,6 @@ extern struct spi_device *spi_new_ancillary_device(struct spi_device *spi, u8 ch * @flags: other constraints relevant to this driver * @slave: indicates that this is an SPI slave controller * @target: indicates that this is an SPI target controller - * @devm_allocated: whether the allocation of this struct is devres-managed * @max_transfer_size: function that returns the max transfer size for * a &spi_device; may be %NULL, so the default %SIZE_MAX will be used. * @max_message_size: function that returns the max message size for @@ -558,6 +561,11 @@ extern struct spi_device *spi_new_ancillary_device(struct spi_device *spi, u8 ch * @defer_optimize_message: set to true if controller cannot pre-optimize messages * and needs to defer the optimization step until the message is actually * being transferred + * @userspace_clients: list of SPI devices instantiated from userspace via + * the sysfs new_device interface; protected by @add_lock + * @userspace_registered: true once the new_device/delete_device sysfs + * group has been added by spi_register_controller(); used by + * spi_unregister_controller() to know whether to remove it * * Each SPI controller can communicate with one or more @spi_device * children. These make a small bus, sharing MOSI, MISO and SCK signals @@ -633,9 +641,6 @@ struct spi_controller { */ #define SPI_CONTROLLER_MULTI_CS BIT(7) - /* Flag indicating if the allocation of this struct is devres-managed */ - bool devm_allocated; - union { /* Flag indicating this is an SPI slave controller */ bool slave; @@ -705,7 +710,7 @@ struct spi_controller { int (*transfer)(struct spi_device *spi, struct spi_message *mesg); - /* Called on release() to free memory provided by spi_controller */ + /* Called on deregistration to free memory provided by spi_controller */ void (*cleanup)(struct spi_device *spi); /* @@ -813,6 +818,13 @@ struct spi_controller { bool queue_empty; bool must_async; bool defer_optimize_message; + +#if IS_ENABLED(CONFIG_SPI_DYNAMIC) + /* List of userspace-instantiated devices; protected by @add_lock */ + struct list_head userspace_clients; + /* True after new_device/delete_device sysfs group is created */ + bool userspace_registered; +#endif }; static inline void *spi_controller_get_devdata(struct spi_controller *ctlr) @@ -1023,7 +1035,7 @@ struct spi_res { * this value may have changed compared to what was requested, depending * on the available snapshotting resolution (DMA transfer, * @ptp_sts_supported is false, etc). - * @ptp_sts_word_post: See @ptp_sts_word_post. The two can be equal (meaning + * @ptp_sts_word_post: See @ptp_sts_word_pre. The two can be equal (meaning * that a single byte should be snapshotted). * If the core takes care of the timestamp (if @ptp_sts_supported is false * for this controller), it will set @ptp_sts_word_pre to 0, and diff --git a/include/linux/spinlock.h b/include/linux/spinlock.h index e1e2f144af9b..3d405cc4c121 100644 --- a/include/linux/spinlock.h +++ b/include/linux/spinlock.h @@ -57,6 +57,7 @@ #include <linux/linkage.h> #include <linux/compiler.h> #include <linux/irqflags.h> +#include <linux/interrupt_rc.h> #include <linux/thread_info.h> #include <linux/stringify.h> #include <linux/bottom_half.h> @@ -178,7 +179,7 @@ do { \ #ifdef CONFIG_DEBUG_SPINLOCK extern void do_raw_spin_lock(raw_spinlock_t *lock) __acquires(lock); - extern int do_raw_spin_trylock(raw_spinlock_t *lock); + extern int do_raw_spin_trylock(raw_spinlock_t *lock) __cond_acquires(true, lock); extern void do_raw_spin_unlock(raw_spinlock_t *lock) __releases(lock); #else static inline void do_raw_spin_lock(raw_spinlock_t *lock) __acquires(lock) @@ -189,6 +190,7 @@ static inline void do_raw_spin_lock(raw_spinlock_t *lock) __acquires(lock) } static inline int do_raw_spin_trylock(raw_spinlock_t *lock) + __cond_acquires(true, lock) { int ret = arch_spin_trylock(&(lock)->raw_lock); @@ -272,9 +274,11 @@ static inline void do_raw_spin_unlock(raw_spinlock_t *lock) __releases(lock) #endif #define raw_spin_lock_irq(lock) _raw_spin_lock_irq(lock) +#define raw_spin_lock_irq_disable(lock) _raw_spin_lock_irq_disable(lock) #define raw_spin_lock_bh(lock) _raw_spin_lock_bh(lock) #define raw_spin_unlock(lock) _raw_spin_unlock(lock) #define raw_spin_unlock_irq(lock) _raw_spin_unlock_irq(lock) +#define raw_spin_unlock_irq_enable(lock) _raw_spin_unlock_irq_enable(lock) #define raw_spin_unlock_irqrestore(lock, flags) \ do { \ @@ -289,6 +293,8 @@ static inline void do_raw_spin_unlock(raw_spinlock_t *lock) __releases(lock) #define raw_spin_trylock_irqsave(lock, flags) _raw_spin_trylock_irqsave(lock, &(flags)) +#define raw_spin_trylock_irq_disable(lock) _raw_spin_trylock_irq_disable(lock) + #ifndef CONFIG_PREEMPT_RT /* Include rwlock functions for !RT */ #include <linux/rwlock.h> @@ -371,6 +377,12 @@ static __always_inline void spin_lock_irq(spinlock_t *lock) raw_spin_lock_irq(&lock->rlock); } +static __always_inline void spin_lock_irq_disable(spinlock_t *lock) + __acquires(lock) __no_context_analysis +{ + raw_spin_lock_irq_disable(&lock->rlock); +} + #define spin_lock_irqsave(lock, flags) \ do { \ raw_spin_lock_irqsave(spinlock_check(lock), flags); \ @@ -401,6 +413,12 @@ static __always_inline void spin_unlock_irq(spinlock_t *lock) raw_spin_unlock_irq(&lock->rlock); } +static __always_inline void spin_unlock_irq_enable(spinlock_t *lock) + __releases(lock) __no_context_analysis +{ + raw_spin_unlock_irq_enable(&lock->rlock); +} + static __always_inline void spin_unlock_irqrestore(spinlock_t *lock, unsigned long flags) __releases(lock) __no_context_analysis { @@ -426,6 +444,12 @@ static __always_inline bool _spin_trylock_irqsave(spinlock_t *lock, unsigned lon } #define spin_trylock_irqsave(lock, flags) _spin_trylock_irqsave(lock, &(flags)) +static __always_inline int spin_trylock_irq_disable(spinlock_t *lock) + __cond_acquires(true, lock) __no_context_analysis +{ + return raw_spin_trylock_irq_disable(&lock->rlock); +} + /** * spin_is_locked() - Check whether a spinlock is locked. * @lock: Pointer to the spinlock. diff --git a/include/linux/spinlock_api_smp.h b/include/linux/spinlock_api_smp.h index bda5e7a390cd..90909d933dab 100644 --- a/include/linux/spinlock_api_smp.h +++ b/include/linux/spinlock_api_smp.h @@ -28,6 +28,8 @@ _raw_spin_lock_nest_lock(raw_spinlock_t *lock, struct lockdep_map *map) void __lockfunc _raw_spin_lock_bh(raw_spinlock_t *lock) __acquires(lock); void __lockfunc _raw_spin_lock_irq(raw_spinlock_t *lock) __acquires(lock); +void __lockfunc _raw_spin_lock_irq_disable(raw_spinlock_t *lock) + __acquires(lock); unsigned long __lockfunc _raw_spin_lock_irqsave(raw_spinlock_t *lock) __acquires(lock); @@ -39,6 +41,7 @@ int __lockfunc _raw_spin_trylock_bh(raw_spinlock_t *lock) __cond_acquires(true, void __lockfunc _raw_spin_unlock(raw_spinlock_t *lock) __releases(lock); void __lockfunc _raw_spin_unlock_bh(raw_spinlock_t *lock) __releases(lock); void __lockfunc _raw_spin_unlock_irq(raw_spinlock_t *lock) __releases(lock); +void __lockfunc _raw_spin_unlock_irq_enable(raw_spinlock_t *lock) __releases(lock); void __lockfunc _raw_spin_unlock_irqrestore(raw_spinlock_t *lock, unsigned long flags) __releases(lock); @@ -55,6 +58,11 @@ _raw_spin_unlock_irqrestore(raw_spinlock_t *lock, unsigned long flags) #define _raw_spin_lock_irq(lock) __raw_spin_lock_irq(lock) #endif +/* Use the same config as spin_lock_irq() temporarily. */ +#ifdef CONFIG_INLINE_SPIN_LOCK_IRQ +#define _raw_spin_lock_irq_disable(lock) __raw_spin_lock_irq_disable(lock) +#endif + #ifdef CONFIG_INLINE_SPIN_LOCK_IRQSAVE #define _raw_spin_lock_irqsave(lock) __raw_spin_lock_irqsave(lock) #endif @@ -79,6 +87,11 @@ _raw_spin_unlock_irqrestore(raw_spinlock_t *lock, unsigned long flags) #define _raw_spin_unlock_irq(lock) __raw_spin_unlock_irq(lock) #endif +/* Use the same config as spin_unlock_irq() temporarily. */ +#ifdef CONFIG_INLINE_SPIN_UNLOCK_IRQ +#define _raw_spin_unlock_irq_enable(lock) __raw_spin_unlock_irq_enable(lock) +#endif + #ifdef CONFIG_INLINE_SPIN_UNLOCK_IRQRESTORE #define _raw_spin_unlock_irqrestore(lock, flags) __raw_spin_unlock_irqrestore(lock, flags) #endif @@ -105,6 +118,16 @@ static __always_inline bool _raw_spin_trylock_irq(raw_spinlock_t *lock) return false; } +static __always_inline bool _raw_spin_trylock_irq_disable(raw_spinlock_t *lock) + __cond_acquires(true, lock) +{ + local_interrupt_disable(); + if (_raw_spin_trylock(lock)) + return true; + local_interrupt_enable(); + return false; +} + static __always_inline bool _raw_spin_trylock_irqsave(raw_spinlock_t *lock, unsigned long *flags) __cond_acquires(true, lock) { @@ -143,6 +166,15 @@ static inline void __raw_spin_lock_irq(raw_spinlock_t *lock) LOCK_CONTENDED(lock, do_raw_spin_trylock, do_raw_spin_lock); } +static inline void __raw_spin_lock_irq_disable(raw_spinlock_t *lock) + __acquires(lock) __no_context_analysis +{ + local_interrupt_disable(); + preempt_disable(); + spin_acquire(&lock->dep_map, 0, 0, _RET_IP_); + LOCK_CONTENDED(lock, do_raw_spin_trylock, do_raw_spin_lock); +} + static inline void __raw_spin_lock_bh(raw_spinlock_t *lock) __acquires(lock) __no_context_analysis { @@ -188,6 +220,15 @@ static inline void __raw_spin_unlock_irq(raw_spinlock_t *lock) preempt_enable(); } +static inline void __raw_spin_unlock_irq_enable(raw_spinlock_t *lock) + __releases(lock) +{ + spin_release(&lock->dep_map, _RET_IP_); + do_raw_spin_unlock(lock); + local_interrupt_enable(); + preempt_enable(); +} + static inline void __raw_spin_unlock_bh(raw_spinlock_t *lock) __releases(lock) { diff --git a/include/linux/spinlock_api_up.h b/include/linux/spinlock_api_up.h index a9d5c7c66e03..d03d3065ee04 100644 --- a/include/linux/spinlock_api_up.h +++ b/include/linux/spinlock_api_up.h @@ -42,6 +42,9 @@ #define __LOCK_IRQSAVE(lock, flags, ...) \ do { local_irq_save(flags); __LOCK(lock, ##__VA_ARGS__); } while (0) +#define __LOCK_IRQ_DISABLE(lock, ...) \ + do { local_interrupt_disable(); __LOCK(lock, ##__VA_ARGS__); } while (0) + #define ___UNLOCK_(lock) \ do { __release(lock); (void)(lock); } while (0) @@ -61,6 +64,9 @@ #define __UNLOCK_IRQRESTORE(lock, flags, ...) \ do { local_irq_restore(flags); __UNLOCK(lock, ##__VA_ARGS__); } while (0) +#define __UNLOCK_IRQ_ENABLE(lock, ...) \ + do { __UNLOCK(lock, ##__VA_ARGS__); local_interrupt_enable(); } while (0) + #define _raw_spin_lock(lock) __LOCK(lock) #define _raw_spin_lock_nested(lock, subclass) __LOCK(lock) #define _raw_read_lock(lock) __LOCK(lock, shared) @@ -70,6 +76,7 @@ #define _raw_read_lock_bh(lock) __LOCK_BH(lock, shared) #define _raw_write_lock_bh(lock) __LOCK_BH(lock) #define _raw_spin_lock_irq(lock) __LOCK_IRQ(lock) +#define _raw_spin_lock_irq_disable(lock) __LOCK_IRQ_DISABLE(lock) #define _raw_read_lock_irq(lock) __LOCK_IRQ(lock, shared) #define _raw_write_lock_irq(lock) __LOCK_IRQ(lock) #define _raw_spin_lock_irqsave(lock, flags) __LOCK_IRQSAVE(lock, flags) @@ -97,6 +104,13 @@ static __always_inline int _raw_spin_trylock_irq(raw_spinlock_t *lock) return 1; } +static __always_inline int _raw_spin_trylock_irq_disable(raw_spinlock_t *lock) + __cond_acquires(true, lock) +{ + __LOCK_IRQ_DISABLE(lock); + return 1; +} + static __always_inline int _raw_spin_trylock_irqsave(raw_spinlock_t *lock, unsigned long *flags) __cond_acquires(true, lock) { @@ -132,6 +146,7 @@ static __always_inline int _raw_write_trylock_irqsave(rwlock_t *lock, unsigned l #define _raw_write_unlock_bh(lock) __UNLOCK_BH(lock) #define _raw_read_unlock_bh(lock) __UNLOCK_BH(lock, shared) #define _raw_spin_unlock_irq(lock) __UNLOCK_IRQ(lock) +#define _raw_spin_unlock_irq_enable(lock) __UNLOCK_IRQ_ENABLE(lock) #define _raw_read_unlock_irq(lock) __UNLOCK_IRQ(lock, shared) #define _raw_write_unlock_irq(lock) __UNLOCK_IRQ(lock) #define _raw_spin_unlock_irqrestore(lock, flags) \ diff --git a/include/linux/spinlock_rt.h b/include/linux/spinlock_rt.h index 373618a4243c..560d06384e0c 100644 --- a/include/linux/spinlock_rt.h +++ b/include/linux/spinlock_rt.h @@ -96,6 +96,12 @@ static __always_inline void spin_lock_irq(spinlock_t *lock) rt_spin_lock(lock); } +static __always_inline void spin_lock_irq_disable(spinlock_t *lock) + __acquires(lock) +{ + rt_spin_lock(lock); +} + #define spin_lock_irqsave(lock, flags) \ do { \ typecheck(unsigned long, flags); \ @@ -122,6 +128,12 @@ static __always_inline void spin_unlock_irq(spinlock_t *lock) rt_spin_unlock(lock); } +static __always_inline void spin_unlock_irq_enable(spinlock_t *lock) + __releases(lock) +{ + rt_spin_unlock(lock); +} + static __always_inline void spin_unlock_irqrestore(spinlock_t *lock, unsigned long flags) __releases(lock) @@ -131,6 +143,12 @@ static __always_inline void spin_unlock_irqrestore(spinlock_t *lock, #define spin_trylock(lock) rt_spin_trylock(lock) +static __always_inline int spin_trylock_irq_disable(spinlock_t *lock) + __cond_acquires(true, lock) +{ + return rt_spin_trylock(lock); +} + #define spin_trylock_bh(lock) rt_spin_trylock_bh(lock) #define spin_trylock_irq(lock) rt_spin_trylock(lock) diff --git a/include/linux/spinlock_up.h b/include/linux/spinlock_up.h index 1e84e71ca495..3a50976471d7 100644 --- a/include/linux/spinlock_up.h +++ b/include/linux/spinlock_up.h @@ -48,16 +48,6 @@ static inline void arch_spin_unlock(arch_spinlock_t *lock) lock->slock = 1; } -/* - * Read-write spinlocks. No debug version. - */ -#define arch_read_lock(lock) do { barrier(); (void)(lock); } while (0) -#define arch_write_lock(lock) do { barrier(); (void)(lock); } while (0) -#define arch_read_trylock(lock) ({ barrier(); (void)(lock); 1; }) -#define arch_write_trylock(lock) ({ barrier(); (void)(lock); 1; }) -#define arch_read_unlock(lock) do { barrier(); (void)(lock); } while (0) -#define arch_write_unlock(lock) do { barrier(); (void)(lock); } while (0) - #else /* DEBUG_SPINLOCK */ #define arch_spin_is_locked(lock) ((void)(lock), 0) /* for sched/core.c and kernel_lock.c: */ @@ -68,4 +58,14 @@ static inline void arch_spin_unlock(arch_spinlock_t *lock) #define arch_spin_is_contended(lock) (((void)(lock), 0)) +/* + * Read-write spinlocks. No debug version. + */ +#define arch_read_lock(lock) do { barrier(); (void)(lock); } while (0) +#define arch_write_lock(lock) do { barrier(); (void)(lock); } while (0) +#define arch_read_trylock(lock) ({ barrier(); (void)(lock); 1; }) +#define arch_write_trylock(lock) ({ barrier(); (void)(lock); 1; }) +#define arch_read_unlock(lock) do { barrier(); (void)(lock); } while (0) +#define arch_write_unlock(lock) do { barrier(); (void)(lock); } while (0) + #endif /* __LINUX_SPINLOCK_UP_H */ diff --git a/include/linux/spmi.h b/include/linux/spmi.h index 28e8c8bd3944..4eb9564a7fb3 100644 --- a/include/linux/spmi.h +++ b/include/linux/spmi.h @@ -76,6 +76,7 @@ void spmi_device_remove(struct spmi_device *sdev); * @cmd: sends a non-data command sequence on the SPMI bus. * @read_cmd: sends a register read command sequence on the SPMI bus. * @write_cmd: sends a register write command sequence on the SPMI bus. + * @priv: array of private data. */ struct spmi_controller { struct device dev; @@ -85,6 +86,7 @@ struct spmi_controller { u8 sid, u16 addr, u8 *buf, size_t len); int (*write_cmd)(struct spmi_controller *ctrl, u8 opcode, u8 sid, u16 addr, const u8 *buf, size_t len); + u8 priv[]; }; static inline struct spmi_controller *to_spmi_controller(struct device *d) @@ -109,7 +111,7 @@ struct spmi_controller *spmi_controller_alloc(struct device *parent, /** * spmi_controller_put() - decrement controller refcount - * @ctrl SPMI controller. + * @ctrl: SPMI controller. */ static inline void spmi_controller_put(struct spmi_controller *ctrl) { @@ -129,6 +131,7 @@ int devm_spmi_controller_add(struct device *parent, struct spmi_controller *ctrl * this structure. * @probe: binds this driver to a SPMI device. * @remove: unbinds this driver from the SPMI device. + * @shutdown: shuts down this driver. * * If PM runtime support is desired for a slave, a device driver can call * pm_runtime_put() from their probe() routine (and a balancing diff --git a/include/linux/srcu.h b/include/linux/srcu.h index bb44a0bd7696..7d9bc06df98d 100644 --- a/include/linux/srcu.h +++ b/include/linux/srcu.h @@ -25,20 +25,19 @@ context_lock_struct(srcu_struct, __reentrant_ctx_lock); #ifdef CONFIG_DEBUG_LOCK_ALLOC -int __init_srcu_struct(struct srcu_struct *ssp, const char *name, struct lock_class_key *key); +int init_srcu_struct_lockdep(struct srcu_struct *ssp, const char *name, + struct lock_class_key *key); +static inline int __init_srcu_struct(struct srcu_struct *ssp, const char *name, + struct lock_class_key *key) +{ + return init_srcu_struct_lockdep(ssp, name, key); +} #ifndef CONFIG_TINY_SRCU int __init_srcu_struct_fast(struct srcu_struct *ssp, const char *name, struct lock_class_key *key); int __init_srcu_struct_fast_updown(struct srcu_struct *ssp, const char *name, struct lock_class_key *key); #endif // #ifndef CONFIG_TINY_SRCU -#define init_srcu_struct(ssp) \ -({ \ - static struct lock_class_key __srcu_key; \ - \ - __init_srcu_struct((ssp), #ssp, &__srcu_key); \ -}) - #define init_srcu_struct_fast(ssp) \ ({ \ static struct lock_class_key __srcu_key; \ @@ -56,7 +55,12 @@ int __init_srcu_struct_fast_updown(struct srcu_struct *ssp, const char *name, #define __SRCU_DEP_MAP_INIT(srcu_name) .dep_map = { .name = #srcu_name }, #else /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */ -int init_srcu_struct(struct srcu_struct *ssp); +int init_srcu_struct_generic(struct srcu_struct *ssp); +static inline int __init_srcu_struct(struct srcu_struct *ssp, const char *name, + struct lock_class_key *key) +{ + return init_srcu_struct_generic(ssp); +} #ifndef CONFIG_TINY_SRCU int init_srcu_struct_fast(struct srcu_struct *ssp); int init_srcu_struct_fast_updown(struct srcu_struct *ssp); @@ -65,12 +69,19 @@ int init_srcu_struct_fast_updown(struct srcu_struct *ssp); #define __SRCU_DEP_MAP_INIT(srcu_name) #endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */ +#define init_srcu_struct(ssp) \ +({ \ + static struct lock_class_key __srcu_key; \ + \ + __init_srcu_struct((ssp), #ssp, &__srcu_key); \ +}) + /* Values for SRCU Tree srcu_data ->srcu_reader_flavor, but also used by rcutorture. */ #define SRCU_READ_FLAVOR_NORMAL 0x1 // srcu_read_lock(). #define SRCU_READ_FLAVOR_NMI 0x2 // srcu_read_lock_nmisafe(). // 0x4 // SRCU-lite is no longer with us. -#define SRCU_READ_FLAVOR_FAST 0x4 // srcu_read_lock_fast(). -#define SRCU_READ_FLAVOR_FAST_UPDOWN 0x8 // srcu_read_lock_fast(). +#define SRCU_READ_FLAVOR_FAST 0x4 // srcu_read_lock_fast(), also NMI-safe. +#define SRCU_READ_FLAVOR_FAST_UPDOWN 0x8 // srcu_read_lock_fast_updown(). #define SRCU_READ_FLAVOR_ALL (SRCU_READ_FLAVOR_NORMAL | SRCU_READ_FLAVOR_NMI | \ SRCU_READ_FLAVOR_FAST | SRCU_READ_FLAVOR_FAST_UPDOWN) // All of the above. @@ -397,7 +408,7 @@ static inline struct srcu_ctr __percpu *srcu_read_lock_fast_notrace(struct srcu_ * * The same srcu_struct may be used concurrently by srcu_down_read_fast() * and srcu_read_lock_fast(). However, the same definition/initialization - * requirements called out for srcu_read_lock_safe() apply. + * requirements called out for srcu_read_lock_fast_updown() apply. */ static inline struct srcu_ctr __percpu *srcu_down_read_fast(struct srcu_struct *ssp) __acquires_shared(ssp) { @@ -638,4 +649,11 @@ DEFINE_LOCK_GUARD_1(srcu_fast_notrace, struct srcu_struct, DECLARE_LOCK_GUARD_1_ATTRS(srcu_fast_notrace, __acquires_shared(_T), __releases_shared(*(struct srcu_struct **)_T)) #define class_srcu_fast_notrace_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(srcu_fast_notrace, _T) +DEFINE_LOCK_GUARD_1(srcu_fast_updown, struct srcu_struct, + _T->scp = srcu_read_lock_fast_updown(_T->lock), + srcu_read_unlock_fast_updown(_T->lock, _T->scp), + struct srcu_ctr __percpu *scp) +DECLARE_LOCK_GUARD_1_ATTRS(srcu_fast_updown, __acquires_shared(_T), __releases_shared(*(struct srcu_struct **)_T)) +#define class_srcu_fast_updown_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(srcu_fast_updown, _T) + #endif diff --git a/include/linux/srcutiny.h b/include/linux/srcutiny.h index dec7cbe015aa..fbcf13bc12d1 100644 --- a/include/linux/srcutiny.h +++ b/include/linux/srcutiny.h @@ -11,6 +11,7 @@ #ifndef _LINUX_SRCU_TINY_H #define _LINUX_SRCU_TINY_H +#include <linux/irq_work_types.h> #include <linux/swait.h> struct srcu_struct { @@ -24,18 +25,21 @@ struct srcu_struct { struct rcu_head *srcu_cb_head; /* Pending callbacks: Head. */ struct rcu_head **srcu_cb_tail; /* Pending callbacks: Tail. */ struct work_struct srcu_work; /* For driving grace periods. */ + struct irq_work srcu_irq_work; /* Defer schedule_work() to irq work. */ #ifdef CONFIG_DEBUG_LOCK_ALLOC struct lockdep_map dep_map; #endif /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */ }; void srcu_drive_gp(struct work_struct *wp); +void srcu_tiny_irq_work(struct irq_work *irq_work); #define __SRCU_STRUCT_INIT(name, __ignored, ___ignored, ____ignored) \ { \ .srcu_wq = __SWAIT_QUEUE_HEAD_INITIALIZER(name.srcu_wq), \ .srcu_cb_tail = &name.srcu_cb_head, \ .srcu_work = __WORK_INITIALIZER(name.srcu_work, srcu_drive_gp), \ + .srcu_irq_work = { .func = srcu_tiny_irq_work }, \ __SRCU_DEP_MAP_INIT(name) \ } @@ -150,4 +154,17 @@ static inline void srcu_torture_stats_print(struct srcu_struct *ssp, data_race(READ_ONCE(ssp->srcu_idx_max))); } +/** + * srcu_readers_active - returns true if there are readers. and false otherwise. + * @ssp: which srcu_struct to count active readers (holding srcu_read_lock). + * + * Note that this is not an atomic primitive, and can therefore suffer + * severe errors when invoked on an active srcu_struct. That said, it + * can be useful as an error check at cleanup time. + */ +static inline bool srcu_readers_active(struct srcu_struct *ssp) +{ + return READ_ONCE(ssp->srcu_lock_nesting[0]) || READ_ONCE(ssp->srcu_lock_nesting[1]); +} + #endif diff --git a/include/linux/srcutree.h b/include/linux/srcutree.h index 958cb7ef41cb..75e54e4f963f 100644 --- a/include/linux/srcutree.h +++ b/include/linux/srcutree.h @@ -34,7 +34,7 @@ struct srcu_data { /* Values: SRCU_READ_FLAVOR_.* */ /* Update-side state. */ - spinlock_t __private lock ____cacheline_internodealigned_in_smp; + raw_spinlock_t __private lock ____cacheline_internodealigned_in_smp; struct rcu_segcblist srcu_cblist; /* List of callbacks.*/ unsigned long srcu_gp_seq_needed; /* Furthest future GP needed. */ unsigned long srcu_gp_seq_needed_exp; /* Furthest future exp GP. */ @@ -55,7 +55,7 @@ struct srcu_data { * Node in SRCU combining tree, similar in function to rcu_data. */ struct srcu_node { - spinlock_t __private lock; + raw_spinlock_t __private lock; unsigned long srcu_have_cbs[4]; /* GP seq for children having CBs, but only */ /* if greater than ->srcu_gp_seq. */ unsigned long srcu_data_have_cbs[4]; /* Which srcu_data structs have CBs for given GP? */ @@ -74,7 +74,7 @@ struct srcu_usage { /* First node at each level. */ int srcu_size_state; /* Small-to-big transition state. */ struct mutex srcu_cb_mutex; /* Serialize CB preparation. */ - spinlock_t __private lock; /* Protect counters and size state. */ + raw_spinlock_t __private lock; /* Protect counters and size state. */ struct mutex srcu_gp_mutex; /* Serialize GP work. */ unsigned long srcu_gp_seq; /* Grace-period seq #. */ unsigned long srcu_gp_seq_needed; /* Latest gp_seq needed. */ @@ -95,6 +95,7 @@ struct srcu_usage { unsigned long reschedule_jiffies; unsigned long reschedule_count; struct delayed_work work; + struct irq_work irq_work; struct srcu_struct *srcu_ssp; }; @@ -156,7 +157,7 @@ struct srcu_struct { #define __SRCU_USAGE_INIT(name) \ { \ - .lock = __SPIN_LOCK_UNLOCKED(name.lock), \ + .lock = __RAW_SPIN_LOCK_UNLOCKED(name.lock), \ .srcu_gp_seq = SRCU_GP_SEQ_INITIAL_VAL, \ .srcu_gp_seq_needed = SRCU_GP_SEQ_INITIAL_VAL_WITH_STATE, \ .srcu_gp_seq_needed_exp = SRCU_GP_SEQ_INITIAL_VAL, \ @@ -259,7 +260,7 @@ static inline struct srcu_ctr __percpu *__srcu_ctr_to_ptr(struct srcu_struct *ss * srcu_read_unlock_fast(). * * Note that both this_cpu_inc() and atomic_long_inc() are RCU read-side - * critical sections either because they disables interrupts, because + * critical sections either because they disable interrupts, because * they are a single instruction, or because they are read-modify-write * atomic operations, depending on the whims of the architecture. * This matters because the SRCU-fast grace-period mechanism uses either @@ -373,4 +374,28 @@ static inline void srcu_check_read_flavor(struct srcu_struct *ssp, int read_flav __srcu_check_read_flavor(ssp, read_flavor); } +/** + * srcu_readers_active - returns true if there are readers. and false otherwise. + * @ssp: which srcu_struct to count active readers (holding srcu_read_lock). + * + * Note that this is not an atomic primitive, and can therefore suffer + * severe errors when invoked on an active srcu_struct. That said, it + * can be useful as an error check at cleanup time. + */ +static inline bool srcu_readers_active(struct srcu_struct *ssp) +{ + int cpu; + unsigned long sum = 0; + + for_each_possible_cpu(cpu) { + struct srcu_data *sdp = per_cpu_ptr(ssp->sda, cpu); + + sum += atomic_long_read(&sdp->srcu_ctrs[0].srcu_locks); + sum += atomic_long_read(&sdp->srcu_ctrs[1].srcu_locks); + sum -= atomic_long_read(&sdp->srcu_ctrs[0].srcu_unlocks); + sum -= atomic_long_read(&sdp->srcu_ctrs[1].srcu_unlocks); + } + return sum; +} + #endif diff --git a/include/linux/ssb/ssb.h b/include/linux/ssb/ssb.h index e1fb11e0f12c..67cb66f6f5ed 100644 --- a/include/linux/ssb/ssb.h +++ b/include/linux/ssb/ssb.h @@ -8,7 +8,7 @@ #include <linux/spinlock.h> #include <linux/pci.h> #include <linux/gpio/driver.h> -#include <linux/mod_devicetable.h> +#include <linux/device-id/ssb.h> #include <linux/dma-mapping.h> #include <linux/platform_device.h> @@ -671,4 +671,6 @@ int ssb_pcibios_plat_dev_init(struct pci_dev *dev); int ssb_pcibios_map_irq(const struct pci_dev *dev, u8 slot, u8 pin); #endif /* CONFIG_SSB_EMBEDDED */ +extern const struct software_node ssb_gpio_swnode; + #endif /* LINUX_SSB_H_ */ diff --git a/include/linux/stacktrace.h b/include/linux/stacktrace.h index 97455880ac41..525cf60673fe 100644 --- a/include/linux/stacktrace.h +++ b/include/linux/stacktrace.h @@ -11,7 +11,7 @@ struct pt_regs; #ifdef CONFIG_ARCH_STACKWALK /** - * stack_trace_consume_fn - Callback for arch_stack_walk() + * typedef stack_trace_consume_fn - Callback for arch_stack_walk() * @cookie: Caller supplied pointer handed back by arch_stack_walk() * @addr: The stack entry address to consume * @@ -48,9 +48,9 @@ void arch_stack_walk(stack_trace_consume_fn consume_entry, void *cookie, * @consume_entry * @task: Pointer to a task struct, can be NULL * - * This function returns an error if it detects any unreliable + * Returns: a negative error code if it detects any unreliable * features of the stack. Otherwise it guarantees that the stack - * trace is reliable. + * trace is reliable and returns %0. * * If the task is not 'current', the caller *must* ensure the task is * inactive and its stack is pinned. diff --git a/include/linux/stat.h b/include/linux/stat.h index e3d00e7bb26d..9c5709132862 100644 --- a/include/linux/stat.h +++ b/include/linux/stat.h @@ -41,8 +41,8 @@ struct kstat { u64 ino; dev_t dev; dev_t rdev; - kuid_t uid; - kgid_t gid; + kuid_t uid; /* This is logically a vfsuid_t. */ + kgid_t gid; /* This is logically a vfsgid_t. */ loff_t size; struct timespec64 atime; struct timespec64 mtime; diff --git a/include/linux/static_call_types.h b/include/linux/static_call_types.h index cfb6ddeb292b..ac55bc966a56 100644 --- a/include/linux/static_call_types.h +++ b/include/linux/static_call_types.h @@ -25,7 +25,7 @@ #define STATIC_CALL_SITE_INIT 2UL /* init section */ #define STATIC_CALL_SITE_FLAGS 3UL -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ /* * The static call site table needs to be created by external tooling (objtool @@ -102,6 +102,6 @@ struct static_call_key { #endif /* CONFIG_HAVE_STATIC_CALL */ -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ #endif /* _STATIC_CALL_TYPES_H */ diff --git a/include/linux/stddef.h b/include/linux/stddef.h index 80b6bfb944f0..e1851c50c89b 100644 --- a/include/linux/stddef.h +++ b/include/linux/stddef.h @@ -100,6 +100,71 @@ enum { * Creates a union between a flexible-array member (FAM) in a struct and a set * of additional members that would otherwise follow it. * + * Beware that, as this helper encloses TYPE NAME and MEMBERS in the same + * union, designated initializers for MEMBERS may overwrite portions + * previously initialized through NAME. + * + * For example:: + * + * struct flex { + * size_t count; + * u8 fam[]; + * }; + * + * struct composite { + * ... + * __TRAILING_OVERLAP(struct flex, flex, fam, __packed, + * u8 data; + * ); + * } __packed; + * + * static struct composite comp = { + * .flex = { + * .count = 1, + * }, + * .data = 2, + * }; + * + * In the example above, .flex and .data initialize different views of the same + * union storage. Since .data is initialized last, it _may_ overwrite portions + * previously initialized through .flex, leading to .flex.count being zeroed + * out. + * + * A couple of alternatives are shown below. + * + * a) Initialize only one view of the overlapped storage and assign the rest + * at runtime:: + * + * static struct composite comp = { + * .flex = { + * .count = 1, + * }, + * }; + * + * static void foo(void) + * { + * comp.data = 2; + * ... + * } + * + * b) Alternatively, replace designated initializers with runtime assignments:: + * + * static void foo(void) + * { + * struct composite comp; + * + * comp.flex.count = 1; + * comp.data = 2; + * ... + * } + * + * Compiler Explorer test code: https://godbolt.org/z/voM4E36dT + * + * For another example of the above see commit 5e54510a9389 ("acpi: nfit: + * intel: avoid multiple -Wflex-array-member-not-at-end warnings") + * + * Link: https://git.kernel.org/linus/5e54510a9389caa9 + * * @TYPE: Flexible structure type name, including "struct" keyword. * @NAME: Name for a variable to define. * @FAM: The flexible-array member within @TYPE diff --git a/include/linux/stmmac.h b/include/linux/stmmac.h index 32352a216567..4430b967abde 100644 --- a/include/linux/stmmac.h +++ b/include/linux/stmmac.h @@ -84,61 +84,78 @@ struct stmmac_priv; /* Platfrom data for platform device structure's platform_data field */ struct stmmac_mdio_bus_data { - unsigned int phy_mask; - unsigned int pcs_mask; - unsigned int default_an_inband; + u32 phy_mask; + u32 pcs_mask; int *irqs; int probed_phy_irq; bool needs_reset; }; struct stmmac_dma_cfg { + /* pbl: programmable burst limit + * txpbl: transmit programmable burst limit + * rxpbl: receive programmable burst limit + * If txpbl or rxpbl are zero, the value of pbl will be substituted. + * Range 0 - 63. + */ int pbl; int txpbl; int rxpbl; + /* pblx8: multiplies pbl, txpbl, rxpbl by a factor of 8 for dwmac >= + * 3.50a, or a factor of 4 for previous versions. + */ bool pblx8; - int fixed_burst; - int mixed_burst; + /* fixed_burst: + * when set, AXI bursts defined by axi_blen_regval are permitted. + * AHB uses SINGLE, INCR4, INCR8 or INCR16 during burst transfers. + * when clear, AXI and AHB use SINGLE or INCR bursts. + */ + bool fixed_burst; + /* mixed_burst: + * when set and fixed_burst is clear, AHB uses INCR for bursts > 16 + * and SINGLE or INCRx for bursts <= 16. + */ + bool mixed_burst; + /* aal: address aligned bursts for AHB and AXI master interface */ bool aal; + bool dche; bool eame; + /* multi_msi_en: stmmac core internal */ bool multi_msi_en; - bool dche; + /* atds: stmmac core internal */ bool atds; }; #define AXI_BLEN 7 struct stmmac_axi { - bool axi_lpi_en; - bool axi_xit_frm; u32 axi_wr_osr_lmt; u32 axi_rd_osr_lmt; - bool axi_kbbe; u32 axi_blen_regval; + bool axi_lpi_en; + bool axi_xit_frm; bool axi_fb; - bool axi_mb; - bool axi_rb; }; struct stmmac_rxq_cfg { - u8 mode_to_use; u32 chan; + u32 prio; + u8 mode_to_use; u8 pkt_route; bool use_prio; - u32 prio; }; struct stmmac_txq_cfg { u32 weight; - bool coe_unsupported; - u8 mode_to_use; /* Credit Base Shaper parameters */ u32 send_slope; u32 idle_slope; u32 high_credit; u32 low_credit; - bool use_prio; u32 prio; int tbs_en; + bool use_prio; + bool coe_unsupported; + u8 mode_to_use; }; struct stmmac_safety_feature_cfg { @@ -187,11 +204,13 @@ enum dwmac_core_type { #define STMMAC_FLAG_MULTI_MSI_EN BIT(7) #define STMMAC_FLAG_EXT_SNAPSHOT_EN BIT(8) #define STMMAC_FLAG_INT_SNAPSHOT_EN BIT(9) -#define STMMAC_FLAG_RX_CLK_RUNS_IN_LPI BIT(10) -#define STMMAC_FLAG_EN_TX_LPI_CLOCKGATING BIT(11) -#define STMMAC_FLAG_EN_TX_LPI_CLK_PHY_CAP BIT(12) -#define STMMAC_FLAG_HWTSTAMP_CORRECT_LATENCY BIT(13) -#define STMMAC_FLAG_KEEP_PREAMBLE_BEFORE_SFD BIT(14) +#define STMMAC_FLAG_EEE_DISABLE BIT(10) +#define STMMAC_FLAG_RX_CLK_RUNS_IN_LPI BIT(11) +#define STMMAC_FLAG_EN_TX_LPI_CLOCKGATING BIT(12) +#define STMMAC_FLAG_EN_TX_LPI_CLK_PHY_CAP BIT(13) +#define STMMAC_FLAG_HWTSTAMP_CORRECT_LATENCY BIT(14) +#define STMMAC_FLAG_KEEP_PREAMBLE_BEFORE_SFD BIT(15) +#define STMMAC_FLAG_SERDES_SUPPORTS_2500M BIT(16) struct mac_device_info; @@ -225,28 +244,28 @@ struct plat_stmmacenet_data { phy_interface_t phy_interface; struct stmmac_mdio_bus_data *mdio_bus_data; struct device_node *phy_node; - struct fwnode_handle *port_node; struct device_node *mdio_node; struct stmmac_dma_cfg *dma_cfg; struct stmmac_safety_feature_cfg *safety_feat_cfg; int clk_csr; - int enh_desc; - int tx_coe; + bool default_an_inband; + bool enh_desc; + bool tx_coe; + bool bugged_jumbo; + bool pmt; + bool force_sf_dma_mode; + bool force_thresh_dma_mode; + bool riwt_off; int rx_coe; - int bugged_jumbo; - int pmt; - int force_sf_dma_mode; - int force_thresh_dma_mode; - int riwt_off; int max_speed; int maxmtu; int multicast_filter_bins; int unicast_filter_entries; int tx_fifo_size; int rx_fifo_size; - u32 host_dma_width; - u32 rx_queues_to_use; - u32 tx_queues_to_use; + u8 host_dma_width; + u8 rx_queues_to_use; + u8 tx_queues_to_use; u8 rx_sched_algorithm; u8 tx_sched_algorithm; struct stmmac_rxq_cfg rx_queues_cfg[MTL_MAX_RX_QUEUES]; @@ -256,7 +275,8 @@ struct plat_stmmacenet_data { int (*set_phy_intf_sel)(void *priv, u8 phy_intf_sel); int (*set_clk_tx_rate)(void *priv, struct clk *clk_tx_i, phy_interface_t interface, int speed); - void (*fix_mac_speed)(void *priv, int speed, unsigned int mode); + void (*fix_mac_speed)(void *priv, phy_interface_t interface, + int speed, unsigned int mode); int (*fix_soc_reset)(struct stmmac_priv *priv); int (*serdes_powerup)(struct net_device *ndev, void *priv); void (*serdes_powerdown)(struct net_device *ndev, void *priv); @@ -279,10 +299,41 @@ struct plat_stmmacenet_data { struct phylink_pcs *(*select_pcs)(struct stmmac_priv *priv, phy_interface_t interface); void *bsp_priv; + + /* stmmac clocks: + * stmmac_clk: CSR clock (which can be hclk_i, clk_csr_i, aclk_i, + * or clk_app_i depending on GMAC configuration). This clock + * generates the MDC clock. + * + * pclk: introduced for Imagination Technologies Pistachio board - + * see 5f9755d26fbf ("stmmac: Add an optional register interface + * clock"). This is probably used for cases where separate clocks + * are provided for the host interface and register interface. In + * this case, as the MDC clock is derived from stmmac_clk, pclk + * can only really be the "application clock" for the "host + * interface" and not the "register interface" aka CSR clock as + * it is never used when determining the divider for the MDC + * clock. + * + * clk_ptp_ref: optional PTP reference clock (clk_ptp_ref_i). When + * present, this clock increments the timestamp value. Otherwise, + * the rate of stmmac_clk will be used. + * + * clk_tx_i: MAC transmit clock, which will be 2.5MHz for 10M, + * 25MHz for 100M, or 125MHz for 1G irrespective of the interface + * mode. For the DWMAC PHY interface modes: + * + * GMII/MII PHY's transmit clock for 10M (2.5MHz) or 100M (25MHz), + * or 125MHz local clock for 1G mode + * RMII 50MHz RMII clock divided by 2 or 20. + * RGMII 125MHz local clock divided by 1, 5, or 50. + * SGMII 125MHz SerDes clock divided by 1, 5, or 50. + * TBI/RTBI 125MHz SerDes clock + */ struct clk *stmmac_clk; struct clk *pclk; struct clk *clk_ptp_ref; - struct clk *clk_tx_i; /* clk_tx_i to MAC core */ + struct clk *clk_tx_i; unsigned long clk_ptp_rate; unsigned long clk_ref_rate; struct clk_bulk_data *clks; @@ -296,7 +347,7 @@ struct plat_stmmacenet_data { int rss_en; int mac_port_sel_speed; u8 vlan_fail_q; - struct pci_dev *pdev; + bool provide_bus_info; int int_snapshot_num; int msi_mac_vec; int msi_wol_vec; @@ -306,5 +357,6 @@ struct plat_stmmacenet_data { int msi_tx_base_vec; const struct dwmac4_addrs *dwmac4_addrs; unsigned int flags; + struct stmmac_dma_cfg __dma_cfg; }; #endif diff --git a/include/linux/stop_machine.h b/include/linux/stop_machine.h index 72820503514c..84e7fb627ba4 100644 --- a/include/linux/stop_machine.h +++ b/include/linux/stop_machine.h @@ -2,6 +2,7 @@ #ifndef _LINUX_STOP_MACHINE #define _LINUX_STOP_MACHINE +#include <linux/bug.h> #include <linux/cpu.h> #include <linux/cpumask_types.h> #include <linux/smp.h> @@ -31,7 +32,7 @@ struct cpu_stop_work { int stop_one_cpu(unsigned int cpu, cpu_stop_fn_t fn, void *arg); int stop_two_cpus(unsigned int cpu1, unsigned int cpu2, cpu_stop_fn_t fn, void *arg); -bool stop_one_cpu_nowait(unsigned int cpu, cpu_stop_fn_t fn, void *arg, +void stop_one_cpu_nowait(unsigned int cpu, cpu_stop_fn_t fn, void *arg, struct cpu_stop_work *work_buf); void stop_machine_park(int cpu); void stop_machine_unpark(int cpu); @@ -68,19 +69,17 @@ static void stop_one_cpu_nowait_workfn(struct work_struct *work) preempt_enable(); } -static inline bool stop_one_cpu_nowait(unsigned int cpu, +static inline void stop_one_cpu_nowait(unsigned int cpu, cpu_stop_fn_t fn, void *arg, struct cpu_stop_work *work_buf) { - if (cpu == smp_processor_id()) { - INIT_WORK(&work_buf->work, stop_one_cpu_nowait_workfn); - work_buf->fn = fn; - work_buf->arg = arg; - schedule_work(&work_buf->work); - return true; - } - - return false; + if (WARN_ON_ONCE(cpu != smp_processor_id())) + return; + + INIT_WORK(&work_buf->work, stop_one_cpu_nowait_workfn); + work_buf->fn = fn; + work_buf->arg = arg; + schedule_work(&work_buf->work); } static inline void print_stop_info(const char *log_lvl, struct task_struct *task) { } @@ -99,7 +98,7 @@ static inline void print_stop_info(const char *log_lvl, struct task_struct *task * stop_machine: freeze the machine on all CPUs and run this function * @fn: the function to run * @data: the data ptr to pass to @fn() - * @cpus: the cpus to run @fn() on (NULL = run on each online CPU) + * @cpus: the cpus to run @fn() on (NULL = one unspecified online CPU) * * Description: This causes a thread to be scheduled on every CPU, which * will run with interrupts disabled. Each CPU specified by @cpus will @@ -133,7 +132,7 @@ int stop_machine(cpu_stop_fn_t fn, void *data, const struct cpumask *cpus); * stop_machine_cpuslocked: freeze the machine on all CPUs and run this function * @fn: the function to run * @data: the data ptr to pass to @fn() - * @cpus: the cpus to run @fn() on (NULL = run on each online CPU) + * @cpus: the cpus to run @fn() on (NULL = one unspecified online CPU) * * Same as above. Avoids nested calls to cpus_read_lock(). * diff --git a/include/linux/string.h b/include/linux/string.h index b850bd91b3d8..5702daca4326 100644 --- a/include/linux/string.h +++ b/include/linux/string.h @@ -67,9 +67,6 @@ void *vmemdup_array_user(const void __user *src, size_t n, size_t size) #ifndef __HAVE_ARCH_STRCPY extern char * strcpy(char *,const char *); #endif -#ifndef __HAVE_ARCH_STRNCPY -extern char * strncpy(char *,const char *, __kernel_size_t); -#endif ssize_t sized_strscpy(char *, const char *, size_t); /* diff --git a/include/linux/sunrpc/bc_xprt.h b/include/linux/sunrpc/bc_xprt.h index 98939cb664cf..59d0cc889beb 100644 --- a/include/linux/sunrpc/bc_xprt.h +++ b/include/linux/sunrpc/bc_xprt.h @@ -32,6 +32,7 @@ int xprt_setup_bc(struct rpc_xprt *xprt, unsigned int min_reqs); void xprt_destroy_bc(struct rpc_xprt *xprt, unsigned int max_reqs); void xprt_free_bc_rqst(struct rpc_rqst *req); unsigned int xprt_bc_max_slots(struct rpc_xprt *xprt); +void xprt_svc_shutdown_bc(struct rpc_xprt *xprt); void xprt_svc_destroy_nullify_bc(struct rpc_xprt *xprt, struct svc_serv **serv); /* @@ -71,6 +72,10 @@ static inline void xprt_free_bc_request(struct rpc_rqst *req) { } +static inline void xprt_svc_shutdown_bc(struct rpc_xprt *xprt) +{ +} + static inline void xprt_svc_destroy_nullify_bc(struct rpc_xprt *xprt, struct svc_serv **serv) { svc_destroy(serv); diff --git a/include/linux/sunrpc/cache.h b/include/linux/sunrpc/cache.h index e783132e481f..2735c332ddb7 100644 --- a/include/linux/sunrpc/cache.h +++ b/include/linux/sunrpc/cache.h @@ -16,6 +16,7 @@ #include <linux/atomic.h> #include <linux/kstrtox.h> #include <linux/proc_fs.h> +#include <linux/wait.h> /* * Each cache requires: @@ -79,6 +80,9 @@ struct cache_detail { int (*cache_upcall)(struct cache_detail *, struct cache_head *); + int (*cache_notify)(struct cache_detail *cd, + struct cache_head *h); + void (*cache_request)(struct cache_detail *cd, struct cache_head *ch, char **bpp, int *blen); @@ -112,7 +116,11 @@ struct cache_detail { int entries; /* fields for communication over channel */ - struct list_head queue; + struct list_head requests; + struct list_head readers; + spinlock_t queue_lock; + wait_queue_head_t queue_wait; + u64 next_seqno; atomic_t writers; /* how many time is /channel open */ time64_t last_close; /* if no writers, when did last close */ @@ -184,9 +192,9 @@ sunrpc_cache_update(struct cache_detail *detail, struct cache_head *new, struct cache_head *old, int hash); extern int -sunrpc_cache_pipe_upcall(struct cache_detail *detail, struct cache_head *h); +sunrpc_cache_upcall(struct cache_detail *detail, struct cache_head *h); extern int -sunrpc_cache_pipe_upcall_timeout(struct cache_detail *detail, +sunrpc_cache_upcall_warn(struct cache_detail *detail, struct cache_head *h); @@ -243,6 +251,14 @@ extern int sunrpc_cache_register_pipefs(struct dentry *parent, const char *, extern void sunrpc_cache_unregister_pipefs(struct cache_detail *); extern void sunrpc_cache_unhash(struct cache_detail *, struct cache_head *); +int sunrpc_cache_requests_count(struct cache_detail *cd); +int sunrpc_cache_requests_snapshot(struct cache_detail *cd, + struct cache_head **items, + u64 *seqnos, int max, + u64 min_seqno); +int sunrpc_cache_notify(struct cache_detail *cd, struct cache_head *h, + u32 cache_type); + /* Must store cache_detail in seq_file->private if using next three functions */ extern void *cache_seq_start_rcu(struct seq_file *file, loff_t *pos); extern void *cache_seq_next_rcu(struct seq_file *file, void *p, loff_t *pos); diff --git a/include/linux/sunrpc/clnt.h b/include/linux/sunrpc/clnt.h index f8b406b0a1af..3c2b8c355ab3 100644 --- a/include/linux/sunrpc/clnt.h +++ b/include/linux/sunrpc/clnt.h @@ -190,6 +190,7 @@ int rpc_switch_client_transport(struct rpc_clnt *, const struct rpc_timeout *); void rpc_shutdown_client(struct rpc_clnt *); +void rpc_hold_client(struct rpc_clnt *); void rpc_release_client(struct rpc_clnt *); void rpc_task_release_transport(struct rpc_task *); void rpc_task_release_client(struct rpc_task *); diff --git a/include/linux/sunrpc/debug.h b/include/linux/sunrpc/debug.h index eb4bd62df319..ab61bed2f7af 100644 --- a/include/linux/sunrpc/debug.h +++ b/include/linux/sunrpc/debug.h @@ -38,6 +38,8 @@ extern unsigned int nlm_debug; do { \ ifdebug(fac) \ __sunrpc_printk(fmt, ##__VA_ARGS__); \ + else \ + no_printk(fmt, ##__VA_ARGS__); \ } while (0) # define dfprintk_rcu(fac, fmt, ...) \ @@ -46,15 +48,15 @@ do { \ rcu_read_lock(); \ __sunrpc_printk(fmt, ##__VA_ARGS__); \ rcu_read_unlock(); \ + } else { \ + no_printk(fmt, ##__VA_ARGS__); \ } \ } while (0) -# define RPC_IFDEBUG(x) x #else # define ifdebug(fac) if (0) -# define dfprintk(fac, fmt, ...) do {} while (0) -# define dfprintk_rcu(fac, fmt, ...) do {} while (0) -# define RPC_IFDEBUG(x) +# define dfprintk(fac, fmt, ...) no_printk(fmt, ##__VA_ARGS__) +# define dfprintk_rcu(fac, fmt, ...) no_printk(fmt, ##__VA_ARGS__) #endif /* diff --git a/include/linux/sunrpc/gss_krb5.h b/include/linux/sunrpc/gss_krb5.h index 43950b5237c8..1cd452ed1db5 100644 --- a/include/linux/sunrpc/gss_krb5.h +++ b/include/linux/sunrpc/gss_krb5.h @@ -37,13 +37,9 @@ #ifndef _LINUX_SUNRPC_GSS_KRB5_H #define _LINUX_SUNRPC_GSS_KRB5_H -#include <crypto/skcipher.h> #include <linux/sunrpc/auth_gss.h> #include <linux/sunrpc/gss_err.h> -/* Length of constant used in key derivation */ -#define GSS_KRB5_K5CLENGTH (5) - /* Maximum key length (in bytes) for the supported crypto algorithms */ #define GSS_KRB5_MAX_KEYLEN (32) @@ -56,11 +52,6 @@ /* The length of the Kerberos GSS token header */ #define GSS_KRB5_TOK_HDR_LEN (16) -#define KG_TOK_MIC_MSG 0x0101 -#define KG_TOK_WRAP_MSG 0x0201 - -#define KG2_TOK_INITIAL 0x0101 -#define KG2_TOK_RESPONSE 0x0202 #define KG2_TOK_MIC 0x0404 #define KG2_TOK_WRAP 0x0504 @@ -68,102 +59,6 @@ #define KG2_TOKEN_FLAG_SEALED 0x02 #define KG2_TOKEN_FLAG_ACCEPTORSUBKEY 0x04 -#define KG2_RESP_FLAG_ERROR 0x0001 -#define KG2_RESP_FLAG_DELEG_OK 0x0002 - -enum sgn_alg { - SGN_ALG_DES_MAC_MD5 = 0x0000, - SGN_ALG_MD2_5 = 0x0001, - SGN_ALG_DES_MAC = 0x0002, - SGN_ALG_3 = 0x0003, /* not published */ - SGN_ALG_HMAC_SHA1_DES3_KD = 0x0004 -}; -enum seal_alg { - SEAL_ALG_NONE = 0xffff, - SEAL_ALG_DES = 0x0000, - SEAL_ALG_1 = 0x0001, /* not published */ - SEAL_ALG_DES3KD = 0x0002 -}; - -/* - * These values are assigned by IANA and published via the - * subregistry at the link below: - * - * https://www.iana.org/assignments/kerberos-parameters/kerberos-parameters.xhtml#kerberos-parameters-2 - */ -#define CKSUMTYPE_CRC32 0x0001 -#define CKSUMTYPE_RSA_MD4 0x0002 -#define CKSUMTYPE_RSA_MD4_DES 0x0003 -#define CKSUMTYPE_DESCBC 0x0004 -#define CKSUMTYPE_RSA_MD5 0x0007 -#define CKSUMTYPE_RSA_MD5_DES 0x0008 -#define CKSUMTYPE_NIST_SHA 0x0009 -#define CKSUMTYPE_HMAC_SHA1_DES3 0x000c -#define CKSUMTYPE_HMAC_SHA1_96_AES128 0x000f -#define CKSUMTYPE_HMAC_SHA1_96_AES256 0x0010 -#define CKSUMTYPE_CMAC_CAMELLIA128 0x0011 -#define CKSUMTYPE_CMAC_CAMELLIA256 0x0012 -#define CKSUMTYPE_HMAC_SHA256_128_AES128 0x0013 -#define CKSUMTYPE_HMAC_SHA384_192_AES256 0x0014 -#define CKSUMTYPE_HMAC_MD5_ARCFOUR -138 /* Microsoft md5 hmac cksumtype */ - -/* from gssapi_err_krb5.h */ -#define KG_CCACHE_NOMATCH (39756032L) -#define KG_KEYTAB_NOMATCH (39756033L) -#define KG_TGT_MISSING (39756034L) -#define KG_NO_SUBKEY (39756035L) -#define KG_CONTEXT_ESTABLISHED (39756036L) -#define KG_BAD_SIGN_TYPE (39756037L) -#define KG_BAD_LENGTH (39756038L) -#define KG_CTX_INCOMPLETE (39756039L) -#define KG_CONTEXT (39756040L) -#define KG_CRED (39756041L) -#define KG_ENC_DESC (39756042L) -#define KG_BAD_SEQ (39756043L) -#define KG_EMPTY_CCACHE (39756044L) -#define KG_NO_CTYPES (39756045L) - -/* per Kerberos v5 protocol spec crypto types from the wire. - * these get mapped to linux kernel crypto routines. - * - * These values are assigned by IANA and published via the - * subregistry at the link below: - * - * https://www.iana.org/assignments/kerberos-parameters/kerberos-parameters.xhtml#kerberos-parameters-1 - */ -#define ENCTYPE_NULL 0x0000 -#define ENCTYPE_DES_CBC_CRC 0x0001 /* DES cbc mode with CRC-32 */ -#define ENCTYPE_DES_CBC_MD4 0x0002 /* DES cbc mode with RSA-MD4 */ -#define ENCTYPE_DES_CBC_MD5 0x0003 /* DES cbc mode with RSA-MD5 */ -#define ENCTYPE_DES_CBC_RAW 0x0004 /* DES cbc mode raw */ -/* XXX deprecated? */ -#define ENCTYPE_DES3_CBC_SHA 0x0005 /* DES-3 cbc mode with NIST-SHA */ -#define ENCTYPE_DES3_CBC_RAW 0x0006 /* DES-3 cbc mode raw */ -#define ENCTYPE_DES_HMAC_SHA1 0x0008 -#define ENCTYPE_DES3_CBC_SHA1 0x0010 -#define ENCTYPE_AES128_CTS_HMAC_SHA1_96 0x0011 -#define ENCTYPE_AES256_CTS_HMAC_SHA1_96 0x0012 -#define ENCTYPE_AES128_CTS_HMAC_SHA256_128 0x0013 -#define ENCTYPE_AES256_CTS_HMAC_SHA384_192 0x0014 -#define ENCTYPE_ARCFOUR_HMAC 0x0017 -#define ENCTYPE_ARCFOUR_HMAC_EXP 0x0018 -#define ENCTYPE_CAMELLIA128_CTS_CMAC 0x0019 -#define ENCTYPE_CAMELLIA256_CTS_CMAC 0x001A -#define ENCTYPE_UNKNOWN 0x01ff - -/* - * Constants used for key derivation - */ -/* for 3DES */ -#define KG_USAGE_SEAL (22) -#define KG_USAGE_SIGN (23) -#define KG_USAGE_SEQ (24) - -/* from rfc3961 */ -#define KEY_USAGE_SEED_CHECKSUM (0x99) -#define KEY_USAGE_SEED_ENCRYPTION (0xAA) -#define KEY_USAGE_SEED_INTEGRITY (0x55) - /* from rfc4121 */ #define KG_USAGE_ACCEPTOR_SEAL (22) #define KG_USAGE_ACCEPTOR_SIGN (23) diff --git a/include/linux/sunrpc/sched.h b/include/linux/sunrpc/sched.h index ccba79ebf893..0dbdf3722537 100644 --- a/include/linux/sunrpc/sched.h +++ b/include/linux/sunrpc/sched.h @@ -95,10 +95,7 @@ struct rpc_task { int tk_rpc_status; /* Result of last RPC operation */ unsigned short tk_flags; /* misc flags */ unsigned short tk_timeouts; /* maj timeouts */ - -#if IS_ENABLED(CONFIG_SUNRPC_DEBUG) || IS_ENABLED(CONFIG_TRACEPOINTS) unsigned short tk_pid; /* debugging aid */ -#endif unsigned char tk_priority : 2,/* Task priority */ tk_garb_retry : 2, tk_cred_retry : 2; diff --git a/include/linux/sunrpc/stats.h b/include/linux/sunrpc/stats.h index 3ce1550d1beb..087ade905e29 100644 --- a/include/linux/sunrpc/stats.h +++ b/include/linux/sunrpc/stats.h @@ -37,9 +37,15 @@ struct svc_stat { rpcbadfmt, rpcbadauth, rpcbadclnt; + + /* Per-version per-procedure call counts (per-cpu, per-netns) */ + unsigned long __percpu **vs_count; }; struct net; +int svc_stat_alloc_counts(struct svc_stat *statp); +void svc_stat_free_counts(struct svc_stat *statp); + #ifdef CONFIG_PROC_FS int rpc_proc_init(struct net *); void rpc_proc_exit(struct net *); diff --git a/include/linux/sunrpc/svc.h b/include/linux/sunrpc/svc.h index 4dc14c7a711b..2db1b9ec5658 100644 --- a/include/linux/sunrpc/svc.h +++ b/include/linux/sunrpc/svc.h @@ -20,7 +20,7 @@ #include <linux/lwq.h> #include <linux/wait.h> #include <linux/mm.h> -#include <linux/pagevec.h> +#include <linux/folio_batch.h> #include <linux/kthread.h> /* @@ -85,7 +85,6 @@ struct svc_serv { char * sv_name; /* service name */ - unsigned int sv_nrpools; /* number of thread pools */ bool sv_is_pooled; /* is this a pooled service? */ struct svc_pool * sv_pools; /* array of thread pools */ int (*sv_threadfn)(void *data); @@ -134,25 +133,37 @@ enum { extern u32 svc_max_payload(const struct svc_rqst *rqstp); /* - * RPC Requests and replies are stored in one or more pages. - * We maintain an array of pages for each server thread. - * Requests are copied into these pages as they arrive. Remaining - * pages are available to write the reply into. + * RPC Call and Reply messages each have their own page array. + * rq_pages holds the incoming Call message; rq_respages holds + * the outgoing Reply message. Both arrays are sized to + * svc_serv_maxpages() entries and are allocated dynamically. * - * Pages are sent using ->sendmsg with MSG_SPLICE_PAGES so each server thread - * needs to allocate more to replace those used in sending. To help keep track - * of these pages we have a receive list where all pages initialy live, and a - * send list where pages are moved to when there are to be part of a reply. + * Pages are sent using ->sendmsg with MSG_SPLICE_PAGES so each + * server thread needs to allocate more to replace those used in + * sending. * - * We use xdr_buf for holding responses as it fits well with NFS - * read responses (that have a header, and some data pages, and possibly - * a tail) and means we can share some client side routines. + * rq_pages request page contract: * - * The xdr_buf.head kvec always points to the first page in the rq_*pages - * list. The xdr_buf.pages pointer points to the second page on that - * list. xdr_buf.tail points to the end of the first page. - * This assumes that the non-page part of an rpc reply will fit - * in a page - NFSd ensures this. lockd also has no trouble. + * Transport receive paths that move request data pages out of + * rq_pages -- TCP multi-fragment reassembly (svc_tcp_save_pages) + * and RDMA Read I/O (svc_rdma_clear_rqst_pages) -- NULL those + * entries to prevent svc_rqst_release_pages() from freeing pages + * still in transport use, and set rq_pages_nfree to the count. + * svc_alloc_arg() refills only that many rq_pages entries. + * + * For rq_respages, svc_rqst_release_pages() NULLs entries in + * [rq_respages, rq_next_page) after each RPC. svc_alloc_arg() + * refills only that range. + * + * xdr_buf holds responses; the structure fits NFS read responses + * (header, data pages, optional tail) and enables sharing of + * client-side routines. + * + * The xdr_buf.head kvec always points to the first page in the + * rq_*pages list. The xdr_buf.pages pointer points to the second + * page on that list. xdr_buf.tail points to the end of the first + * page. This assumes that the non-page part of an rpc reply will + * fit in a page - NFSd ensures this. lockd also has no trouble. */ /** @@ -162,10 +173,10 @@ extern u32 svc_max_payload(const struct svc_rqst *rqstp); * Returns a count of pages or vectors that can hold the maximum * size RPC message for @serv. * - * Each request/reply pair can have at most one "payload", plus two - * pages, one for the request, and one for the reply. - * nfsd_splice_actor() might need an extra page when a READ payload - * is not page-aligned. + * Each page array can hold at most one payload plus two + * overhead pages (one for the RPC header, one for tail data). + * nfsd_splice_actor() might need an extra page when a READ + * payload is not page-aligned. */ static inline unsigned long svc_serv_maxpages(const struct svc_serv *serv) { @@ -175,6 +186,9 @@ static inline unsigned long svc_serv_maxpages(const struct svc_serv *serv) /* * The context of a single thread, including the request currently being * processed. + * + * RPC programs are free to use rq_private to stash thread-local information. + * The sunrpc layer will not access it. */ struct svc_rqst { struct list_head rq_all; /* all threads list */ @@ -201,11 +215,12 @@ struct svc_rqst { struct xdr_stream rq_res_stream; struct folio *rq_scratch_folio; struct xdr_buf rq_res; - unsigned long rq_maxpages; /* num of entries in rq_pages */ - struct page * *rq_pages; - struct page * *rq_respages; /* points into rq_pages */ + unsigned long rq_maxpages; /* entries per page array */ + unsigned long rq_pages_nfree; /* rq_pages entries NULLed by transport */ + struct page * *rq_pages; /* Call buffer pages */ + struct page * *rq_respages; /* Reply buffer pages */ struct page * *rq_next_page; /* next reply page to use */ - struct page * *rq_page_end; /* one past the last page */ + struct page * *rq_page_end; /* one past the last reply page */ struct folio_batch rq_fbatch; struct bio_vec *rq_bvec; @@ -215,7 +230,6 @@ struct svc_rqst { u32 rq_vers; /* program version */ u32 rq_proc; /* procedure number */ u32 rq_prot; /* IP protocol */ - int rq_cachetype; /* catering to nfsd */ unsigned long rq_flags; /* flags field */ ktime_t rq_qtime; /* enqueue time */ @@ -251,7 +265,7 @@ struct svc_rqst { unsigned long bc_to_initval; unsigned int bc_to_retries; unsigned int rq_status_counter; /* RPC processing counter */ - void **rq_lease_breaker; /* The v4 client breaking a lease */ + void *rq_private; /* For use by the service thread */ }; /* bits for rq_flags */ @@ -393,7 +407,6 @@ struct svc_version { u32 vs_vers; /* version number */ u32 vs_nproc; /* number of procedures */ const struct svc_procedure *vs_proc; /* per-procedure info */ - unsigned long __percpu *vs_count; /* call counts */ u32 vs_xdrsize; /* xdrsize needed for this version */ /* Don't register with rpcbind */ @@ -454,6 +467,7 @@ int svc_set_pool_threads(struct svc_serv *serv, struct svc_pool *pool, unsigned int min_threads, unsigned int max_threads); int svc_set_num_threads(struct svc_serv *serv, unsigned int min_threads, unsigned int nrservs); +unsigned int svc_serv_maxthreads(const struct svc_serv *serv); int svc_pool_stats_open(struct svc_info *si, struct file *file); void svc_process(struct svc_rqst *rqstp); void svc_process_bc(struct rpc_rqst *req, struct svc_rqst *rqstp); @@ -464,6 +478,7 @@ void svc_wake_up(struct svc_serv *); void svc_reserve(struct svc_rqst *rqstp, int space); void svc_pool_wake_idle_thread(struct svc_pool *pool); struct svc_pool *svc_pool_for_cpu(struct svc_serv *serv); +unsigned int svc_serv_nrpools(const struct svc_serv *serv); char * svc_print_addr(struct svc_rqst *, char *, size_t); const char * svc_proc_name(const struct svc_rqst *rqstp); int svc_encode_result_payload(struct svc_rqst *rqstp, @@ -483,6 +498,21 @@ int svc_generic_rpcbind_set(struct net *net, #define RPC_MAX_ADDRBUFLEN (63U) +/** + * svc_rqst_page_release - release a page associated with an RPC transaction + * @rqstp: RPC transaction context + * @page: page to release + * + * Released pages are batched and freed together, reducing + * allocator pressure under heavy RPC workloads. + */ +static inline void svc_rqst_page_release(struct svc_rqst *rqstp, + struct page *page) +{ + if (!folio_batch_add(&rqstp->rq_fbatch, page_folio(page))) + __folio_batch_release(&rqstp->rq_fbatch); +} + /* * When we want to reduce the size of the reserved space in the response * buffer, we need to take into account the size of any checksum data that diff --git a/include/linux/sunrpc/svc_rdma.h b/include/linux/sunrpc/svc_rdma.h index 57f4fd94166a..76aa5ec4ab40 100644 --- a/include/linux/sunrpc/svc_rdma.h +++ b/include/linux/sunrpc/svc_rdma.h @@ -66,7 +66,6 @@ extern unsigned int svcrdma_ord; extern unsigned int svcrdma_max_requests; extern unsigned int svcrdma_max_bc_requests; extern unsigned int svcrdma_max_req_size; -extern struct workqueue_struct *svcrdma_wq; extern struct percpu_counter svcrdma_stat_read; extern struct percpu_counter svcrdma_stat_recv; @@ -78,12 +77,15 @@ struct svcxprt_rdma { struct rdma_cm_id *sc_cm_id; /* RDMA connection id */ struct list_head sc_accept_q; /* Conn. waiting accept */ struct rpcrdma_notification sc_rn; /* removal notification */ - int sc_ord; /* RDMA read limit */ - int sc_max_send_sges; + u32 sc_ord; /* RDMA read limit */ + unsigned int sc_max_send_sges; bool sc_snd_w_inv; /* OK to use Send With Invalidate */ atomic_t sc_sq_avail; /* SQEs ready to be consumed */ unsigned int sc_sq_depth; /* Depth of SQ */ + atomic_t sc_sq_ticket_head; /* Next ticket to issue */ + atomic_t sc_sq_ticket_tail; /* Ticket currently serving */ + wait_queue_head_t sc_sq_ticket_wait; /* Ticket ordering waitlist */ __be32 sc_fc_credits; /* Forward credits */ u32 sc_max_requests; /* Max requests */ u32 sc_max_bc_requests;/* Backward credits */ @@ -114,6 +116,8 @@ struct svcxprt_rdma { struct llist_head sc_recv_ctxts; + struct llist_head sc_send_release_list; + atomic_t sc_completion_ids; }; /* sc_flags */ @@ -213,6 +217,7 @@ struct svc_rdma_recv_ctxt { */ struct svc_rdma_write_info { struct svcxprt_rdma *wi_rdma; + struct list_head wi_list; const struct svc_rdma_chunk *wi_chunk; @@ -226,13 +231,11 @@ struct svc_rdma_write_info { unsigned int wi_next_off; struct svc_rdma_chunk_ctxt wi_cc; - struct work_struct wi_work; }; struct svc_rdma_send_ctxt { struct llist_node sc_node; struct rpc_rdma_cid sc_cid; - struct work_struct sc_work; struct svcxprt_rdma *sc_rdma; struct ib_send_wr sc_send_wr; @@ -241,7 +244,10 @@ struct svc_rdma_send_ctxt { struct ib_cqe sc_cqe; struct xdr_buf sc_hdrbuf; struct xdr_stream sc_stream; + + struct list_head sc_write_info_list; struct svc_rdma_write_info sc_reply_info; + void *sc_xprt_buf; int sc_page_count; int sc_cur_sge_no; @@ -274,11 +280,14 @@ extern void svc_rdma_cc_init(struct svcxprt_rdma *rdma, extern void svc_rdma_cc_release(struct svcxprt_rdma *rdma, struct svc_rdma_chunk_ctxt *cc, enum dma_data_direction dir); +extern void svc_rdma_write_chunk_release(struct svcxprt_rdma *rdma, + struct svc_rdma_send_ctxt *ctxt); extern void svc_rdma_reply_chunk_release(struct svcxprt_rdma *rdma, struct svc_rdma_send_ctxt *ctxt); -extern int svc_rdma_send_write_list(struct svcxprt_rdma *rdma, - const struct svc_rdma_recv_ctxt *rctxt, - const struct xdr_buf *xdr); +extern int svc_rdma_prepare_write_list(struct svcxprt_rdma *rdma, + const struct svc_rdma_recv_ctxt *rctxt, + struct svc_rdma_send_ctxt *sctxt, + const struct xdr_buf *xdr); extern int svc_rdma_prepare_reply_chunk(struct svcxprt_rdma *rdma, const struct svc_rdma_pcl *write_pcl, const struct svc_rdma_pcl *reply_pcl, @@ -290,6 +299,7 @@ extern int svc_rdma_process_read_list(struct svcxprt_rdma *rdma, /* svc_rdma_sendto.c */ extern void svc_rdma_send_ctxts_destroy(struct svcxprt_rdma *rdma); +extern void svc_rdma_send_ctxts_drain(struct svcxprt_rdma *rdma); extern struct svc_rdma_send_ctxt * svc_rdma_send_ctxt_get(struct svcxprt_rdma *rdma); extern void svc_rdma_send_ctxt_put(struct svcxprt_rdma *rdma, @@ -306,11 +316,19 @@ extern void svc_rdma_send_error_msg(struct svcxprt_rdma *rdma, struct svc_rdma_recv_ctxt *rctxt, int status); extern void svc_rdma_wake_send_waiters(struct svcxprt_rdma *rdma, int avail); +extern int svc_rdma_sq_wait(struct svcxprt_rdma *rdma, + const struct rpc_rdma_cid *cid, int sqecount); +extern int svc_rdma_post_send_err(struct svcxprt_rdma *rdma, + const struct rpc_rdma_cid *cid, + const struct ib_send_wr *bad_wr, + const struct ib_send_wr *first_wr, + int sqecount, int ret); extern int svc_rdma_sendto(struct svc_rqst *); extern int svc_rdma_result_payload(struct svc_rqst *rqstp, unsigned int offset, unsigned int length); /* svc_rdma_transport.c */ +extern void svc_rdma_xprt_deferred_close(struct svcxprt_rdma *rdma); extern struct svc_xprt_class svc_rdma_class; #ifdef CONFIG_SUNRPC_BACKCHANNEL extern struct svc_xprt_class svc_rdma_bc_class; diff --git a/include/linux/sunrpc/svc_rdma_pcl.h b/include/linux/sunrpc/svc_rdma_pcl.h index 7516ad0fae80..6346d8cf2587 100644 --- a/include/linux/sunrpc/svc_rdma_pcl.h +++ b/include/linux/sunrpc/svc_rdma_pcl.h @@ -97,7 +97,7 @@ pcl_next_chunk(const struct svc_rdma_pcl *pcl, struct svc_rdma_chunk *chunk) */ #define pcl_for_each_segment(pos, chunk) \ for (pos = &(chunk)->ch_segments[0]; \ - pos <= &(chunk)->ch_segments[(chunk)->ch_segcount - 1]; \ + pos < &(chunk)->ch_segments[(chunk)->ch_segcount]; \ pos++) /** @@ -119,6 +119,8 @@ extern bool pcl_alloc_call(struct svc_rdma_recv_ctxt *rctxt, __be32 *p); extern bool pcl_alloc_read(struct svc_rdma_recv_ctxt *rctxt, __be32 *p); extern bool pcl_alloc_write(struct svc_rdma_recv_ctxt *rctxt, struct svc_rdma_pcl *pcl, __be32 *p); +extern bool pcl_check_read_chunk_positions(struct svc_rdma_recv_ctxt *rctxt, + unsigned int inline_len); extern int pcl_process_nonpayloads(const struct svc_rdma_pcl *pcl, const struct xdr_buf *xdr, int (*actor)(const struct xdr_buf *, diff --git a/include/linux/sunrpc/xdr.h b/include/linux/sunrpc/xdr.h index 152597750f55..b102b4f21e6b 100644 --- a/include/linux/sunrpc/xdr.h +++ b/include/linux/sunrpc/xdr.h @@ -138,8 +138,23 @@ void xdr_terminate_string(const struct xdr_buf *, const u32); size_t xdr_buf_pagecount(const struct xdr_buf *buf); int xdr_alloc_bvec(struct xdr_buf *buf, gfp_t gfp); void xdr_free_bvec(struct xdr_buf *buf); -unsigned int xdr_buf_to_bvec(struct bio_vec *bvec, unsigned int bvec_size, - const struct xdr_buf *xdr); +int xdr_buf_to_bvec(struct bio_vec *bvec, unsigned int bvec_size, + const struct xdr_buf *xdr); +int xdr_buf_to_sg(const struct xdr_buf *buf, unsigned int offset, + unsigned int len, struct scatterlist *sg, unsigned int nsg); +int xdr_buf_to_sg_alloc(const struct xdr_buf *buf, unsigned int offset, + unsigned int len, struct scatterlist *sg_head, + unsigned int sg_head_nents, + struct scatterlist **sg_overflow, gfp_t gfp); + +/* + * Inline scatterlist entries for xdr_buf_to_sg_alloc(). Sized to cover the + * head kvec, tail kvec, and a few page fragments without any heap allocation. + */ +enum { + XDR_BUF_TO_SG_NENTS = 8, +}; + static inline __be32 *xdr_encode_array(__be32 *p, const void *s, unsigned int len) { @@ -260,7 +275,6 @@ extern void xdr_finish_decode(struct xdr_stream *xdr); extern __be32 *xdr_inline_decode(struct xdr_stream *xdr, size_t nbytes); extern unsigned int xdr_read_pages(struct xdr_stream *xdr, unsigned int len); extern void xdr_enter_page(struct xdr_stream *xdr, unsigned int len); -extern int xdr_process_buf(const struct xdr_buf *buf, unsigned int offset, unsigned int len, int (*actor)(struct scatterlist *, void *), void *data); extern void xdr_set_pagelen(struct xdr_stream *, unsigned int len); extern bool xdr_stream_subsegment(struct xdr_stream *xdr, struct xdr_buf *subbuf, unsigned int len); @@ -290,7 +304,7 @@ xdr_set_scratch_buffer(struct xdr_stream *xdr, void *buf, size_t buflen) /** * xdr_set_scratch_folio - Attach a scratch buffer for decoding data * @xdr: pointer to xdr_stream struct - * @page: an anonymous folio + * @folio: an anonymous folio * * See xdr_set_scratch_buffer(). */ @@ -330,7 +344,7 @@ static inline void xdr_commit_encode(struct xdr_stream *xdr) * xdr_stream_remaining - Return the number of bytes remaining in the stream * @xdr: pointer to struct xdr_stream * - * Return value: + * Returns: * Number of bytes remaining in @xdr before xdr->end */ static inline size_t @@ -350,7 +364,7 @@ ssize_t xdr_stream_encode_opaque_auth(struct xdr_stream *xdr, u32 flavor, * xdr_align_size - Calculate padded size of an object * @n: Size of an object being XDR encoded (in bytes) * - * Return value: + * Returns: * Size (in bytes) of the object including xdr padding */ static inline size_t @@ -368,7 +382,7 @@ xdr_align_size(size_t n) * This implementation avoids the need for conditional * branches or modulo division. * - * Return value: + * Returns: * Size (in bytes) of the needed XDR pad */ static inline size_t xdr_pad_size(size_t n) @@ -380,7 +394,7 @@ static inline size_t xdr_pad_size(size_t n) * xdr_stream_encode_item_present - Encode a "present" list item * @xdr: pointer to xdr_stream * - * Return values: + * Returns: * On success, returns length in bytes of XDR buffer consumed * %-EMSGSIZE on XDR buffer overflow */ @@ -399,7 +413,7 @@ static inline ssize_t xdr_stream_encode_item_present(struct xdr_stream *xdr) * xdr_stream_encode_item_absent - Encode a "not present" list item * @xdr: pointer to xdr_stream * - * Return values: + * Returns: * On success, returns length in bytes of XDR buffer consumed * %-EMSGSIZE on XDR buffer overflow */ @@ -419,7 +433,7 @@ static inline int xdr_stream_encode_item_absent(struct xdr_stream *xdr) * @p: address in a buffer into which to encode * @n: boolean value to encode * - * Return value: + * Returns: * Address of item following the encoded boolean */ static inline __be32 *xdr_encode_bool(__be32 *p, u32 n) @@ -433,7 +447,7 @@ static inline __be32 *xdr_encode_bool(__be32 *p, u32 n) * @xdr: pointer to xdr_stream * @n: boolean value to encode * - * Return values: + * Returns: * On success, returns length in bytes of XDR buffer consumed * %-EMSGSIZE on XDR buffer overflow */ @@ -453,7 +467,7 @@ static inline int xdr_stream_encode_bool(struct xdr_stream *xdr, __u32 n) * @xdr: pointer to xdr_stream * @n: integer to encode * - * Return values: + * Returns: * On success, returns length in bytes of XDR buffer consumed * %-EMSGSIZE on XDR buffer overflow */ @@ -474,7 +488,7 @@ xdr_stream_encode_u32(struct xdr_stream *xdr, __u32 n) * @xdr: pointer to xdr_stream * @n: integer to encode * - * Return values: + * Returns: * On success, returns length in bytes of XDR buffer consumed * %-EMSGSIZE on XDR buffer overflow */ @@ -495,7 +509,7 @@ xdr_stream_encode_be32(struct xdr_stream *xdr, __be32 n) * @xdr: pointer to xdr_stream * @n: 64-bit integer to encode * - * Return values: + * Returns: * On success, returns length in bytes of XDR buffer consumed * %-EMSGSIZE on XDR buffer overflow */ @@ -517,7 +531,7 @@ xdr_stream_encode_u64(struct xdr_stream *xdr, __u64 n) * @ptr: pointer to void pointer * @len: size of object * - * Return values: + * Returns: * On success, returns length in bytes of XDR buffer consumed * %-EMSGSIZE on XDR buffer overflow */ @@ -542,7 +556,7 @@ xdr_stream_encode_opaque_inline(struct xdr_stream *xdr, void **ptr, size_t len) * @ptr: pointer to opaque data object * @len: size of object pointed to by @ptr * - * Return values: + * Returns: * On success, returns length in bytes of XDR buffer consumed * %-EMSGSIZE on XDR buffer overflow */ @@ -563,7 +577,7 @@ xdr_stream_encode_opaque_fixed(struct xdr_stream *xdr, const void *ptr, size_t l * @ptr: pointer to opaque data object * @len: size of object pointed to by @ptr * - * Return values: + * Returns: * On success, returns length in bytes of XDR buffer consumed * %-EMSGSIZE on XDR buffer overflow */ @@ -585,7 +599,7 @@ xdr_stream_encode_opaque(struct xdr_stream *xdr, const void *ptr, size_t len) * @array: array of integers * @array_size: number of elements in @array * - * Return values: + * Returns: * On success, returns length in bytes of XDR buffer consumed * %-EMSGSIZE on XDR buffer overflow */ @@ -608,7 +622,7 @@ xdr_stream_encode_uint32_array(struct xdr_stream *xdr, * xdr_item_is_absent - symbolically handle XDR discriminators * @p: pointer to undecoded discriminator * - * Return values: + * Returns: * %true if the following XDR item is absent * %false if the following XDR item is present */ @@ -621,7 +635,7 @@ static inline bool xdr_item_is_absent(const __be32 *p) * xdr_item_is_present - symbolically handle XDR discriminators * @p: pointer to undecoded discriminator * - * Return values: + * Returns: * %true if the following XDR item is present * %false if the following XDR item is absent */ @@ -635,7 +649,7 @@ static inline bool xdr_item_is_present(const __be32 *p) * @xdr: pointer to xdr_stream * @ptr: pointer to a u32 in which to store the result * - * Return values: + * Returns: * %0 on success * %-EBADMSG on XDR buffer overflow */ @@ -656,7 +670,7 @@ xdr_stream_decode_bool(struct xdr_stream *xdr, __u32 *ptr) * @xdr: pointer to xdr_stream * @ptr: location to store integer * - * Return values: + * Returns: * %0 on success * %-EBADMSG on XDR buffer overflow */ @@ -677,7 +691,7 @@ xdr_stream_decode_u32(struct xdr_stream *xdr, __u32 *ptr) * @xdr: pointer to xdr_stream * @ptr: location to store integer * - * Return values: + * Returns: * %0 on success * %-EBADMSG on XDR buffer overflow */ @@ -698,7 +712,7 @@ xdr_stream_decode_be32(struct xdr_stream *xdr, __be32 *ptr) * @xdr: pointer to xdr_stream * @ptr: location to store 64-bit integer * - * Return values: + * Returns: * %0 on success * %-EBADMSG on XDR buffer overflow */ @@ -720,7 +734,7 @@ xdr_stream_decode_u64(struct xdr_stream *xdr, __u64 *ptr) * @ptr: location to store data * @len: size of buffer pointed to by @ptr * - * Return values: + * Returns: * %0 on success * %-EBADMSG on XDR buffer overflow */ @@ -746,7 +760,7 @@ xdr_stream_decode_opaque_fixed(struct xdr_stream *xdr, void *ptr, size_t len) * on @xdr. It is therefore expected that the object it points to should * be processed immediately. * - * Return values: + * Returns: * On success, returns size of object stored in *@ptr * %-EBADMSG on XDR buffer overflow * %-EMSGSIZE if the size of the object would exceed @maxlen @@ -777,7 +791,7 @@ xdr_stream_decode_opaque_inline(struct xdr_stream *xdr, void **ptr, size_t maxle * @array: location to store the integer array or NULL * @array_size: number of elements to store * - * Return values: + * Returns: * On success, returns number of elements stored in @array * %-EBADMSG on XDR buffer overflow * %-EMSGSIZE if the size of the array exceeds @array_size diff --git a/include/linux/sunrpc/xdrgen/_builtins.h b/include/linux/sunrpc/xdrgen/_builtins.h index a723fb1da9c8..c9033eb1c829 100644 --- a/include/linux/sunrpc/xdrgen/_builtins.h +++ b/include/linux/sunrpc/xdrgen/_builtins.h @@ -296,4 +296,36 @@ xdrgen_encode_opaque(struct xdr_stream *xdr, opaque val) return true; } +struct svc_rqst; + +/** + * xdrgen_svc_decode_void - Decode a void argument + * @rqstp: RPC transaction context + * @xdr: source XDR data stream + * + * Return values: + * %true: procedure arguments decoded successfully + * %false: decode failed + */ +static inline bool +xdrgen_svc_decode_void(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + return xdrgen_decode_void(xdr); +} + +/** + * xdrgen_svc_encode_void - Encode a void result + * @rqstp: RPC transaction context + * @xdr: target XDR data stream + * + * Return values: + * %true: procedure results encoded successfully + * %false: encode failed + */ +static inline bool +xdrgen_svc_encode_void(struct svc_rqst *rqstp, struct xdr_stream *xdr) +{ + return xdrgen_encode_void(xdr); +} + #endif /* _SUNRPC_XDRGEN__BUILTINS_H_ */ diff --git a/include/linux/sunrpc/xdrgen/_defs.h b/include/linux/sunrpc/xdrgen/_defs.h index 20c7270aa64d..8f3776ef3229 100644 --- a/include/linux/sunrpc/xdrgen/_defs.h +++ b/include/linux/sunrpc/xdrgen/_defs.h @@ -25,6 +25,8 @@ typedef struct { #define XDR_void (0) #define XDR_bool (1) +#define XDR_short (1) +#define XDR_unsigned_short (1) #define XDR_int (1) #define XDR_unsigned_int (1) #define XDR_long (1) diff --git a/include/linux/sunrpc/xdrgen/nfs4_1.h b/include/linux/sunrpc/xdrgen/nfs4_1.h index 4ac54bdbd335..bd3289a1d66a 100644 --- a/include/linux/sunrpc/xdrgen/nfs4_1.h +++ b/include/linux/sunrpc/xdrgen/nfs4_1.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 */ /* Generated by xdrgen. Manual edits will be lost. */ /* XDR specification file: ../../Documentation/sunrpc/xdr/nfs4_1.x */ -/* XDR specification modification time: Thu Jan 8 23:12:07 2026 */ +/* XDR specification modification time: Tue Jun 30 11:57:21 2026 */ #ifndef _LINUX_XDRGEN_NFS4_1_DEF_H #define _LINUX_XDRGEN_NFS4_1_DEF_H @@ -9,15 +9,149 @@ #include <linux/types.h> #include <linux/sunrpc/xdrgen/_defs.h> -typedef s64 int64_t; +typedef s32 int32_t; typedef u32 uint32_t; +typedef s64 int64_t; + +typedef u64 uint64_t; + +enum { NFS4_VERIFIER_SIZE = 8 }; + +enum { NFS4_FHSIZE = 128 }; + +enum nfsstat4 { + NFS4_OK = 0, + NFS4ERR_PERM = 1, + NFS4ERR_NOENT = 2, + NFS4ERR_IO = 5, + NFS4ERR_NXIO = 6, + NFS4ERR_ACCESS = 13, + NFS4ERR_EXIST = 17, + NFS4ERR_XDEV = 18, + NFS4ERR_NOTDIR = 20, + NFS4ERR_ISDIR = 21, + NFS4ERR_INVAL = 22, + NFS4ERR_FBIG = 27, + NFS4ERR_NOSPC = 28, + NFS4ERR_ROFS = 30, + NFS4ERR_MLINK = 31, + NFS4ERR_NAMETOOLONG = 63, + NFS4ERR_NOTEMPTY = 66, + NFS4ERR_DQUOT = 69, + NFS4ERR_STALE = 70, + NFS4ERR_BADHANDLE = 10001, + NFS4ERR_BAD_COOKIE = 10003, + NFS4ERR_NOTSUPP = 10004, + NFS4ERR_TOOSMALL = 10005, + NFS4ERR_SERVERFAULT = 10006, + NFS4ERR_BADTYPE = 10007, + NFS4ERR_DELAY = 10008, + NFS4ERR_SAME = 10009, + NFS4ERR_DENIED = 10010, + NFS4ERR_EXPIRED = 10011, + NFS4ERR_LOCKED = 10012, + NFS4ERR_GRACE = 10013, + NFS4ERR_FHEXPIRED = 10014, + NFS4ERR_SHARE_DENIED = 10015, + NFS4ERR_WRONGSEC = 10016, + NFS4ERR_CLID_INUSE = 10017, + NFS4ERR_RESOURCE = 10018, + NFS4ERR_MOVED = 10019, + NFS4ERR_NOFILEHANDLE = 10020, + NFS4ERR_MINOR_VERS_MISMATCH = 10021, + NFS4ERR_STALE_CLIENTID = 10022, + NFS4ERR_STALE_STATEID = 10023, + NFS4ERR_OLD_STATEID = 10024, + NFS4ERR_BAD_STATEID = 10025, + NFS4ERR_BAD_SEQID = 10026, + NFS4ERR_NOT_SAME = 10027, + NFS4ERR_LOCK_RANGE = 10028, + NFS4ERR_SYMLINK = 10029, + NFS4ERR_RESTOREFH = 10030, + NFS4ERR_LEASE_MOVED = 10031, + NFS4ERR_ATTRNOTSUPP = 10032, + NFS4ERR_NO_GRACE = 10033, + NFS4ERR_RECLAIM_BAD = 10034, + NFS4ERR_RECLAIM_CONFLICT = 10035, + NFS4ERR_BADXDR = 10036, + NFS4ERR_LOCKS_HELD = 10037, + NFS4ERR_OPENMODE = 10038, + NFS4ERR_BADOWNER = 10039, + NFS4ERR_BADCHAR = 10040, + NFS4ERR_BADNAME = 10041, + NFS4ERR_BAD_RANGE = 10042, + NFS4ERR_LOCK_NOTSUPP = 10043, + NFS4ERR_OP_ILLEGAL = 10044, + NFS4ERR_DEADLOCK = 10045, + NFS4ERR_FILE_OPEN = 10046, + NFS4ERR_ADMIN_REVOKED = 10047, + NFS4ERR_CB_PATH_DOWN = 10048, + NFS4ERR_BADIOMODE = 10049, + NFS4ERR_BADLAYOUT = 10050, + NFS4ERR_BAD_SESSION_DIGEST = 10051, + NFS4ERR_BADSESSION = 10052, + NFS4ERR_BADSLOT = 10053, + NFS4ERR_COMPLETE_ALREADY = 10054, + NFS4ERR_CONN_NOT_BOUND_TO_SESSION = 10055, + NFS4ERR_DELEG_ALREADY_WANTED = 10056, + NFS4ERR_BACK_CHAN_BUSY = 10057, + NFS4ERR_LAYOUTTRYLATER = 10058, + NFS4ERR_LAYOUTUNAVAILABLE = 10059, + NFS4ERR_NOMATCHING_LAYOUT = 10060, + NFS4ERR_RECALLCONFLICT = 10061, + NFS4ERR_UNKNOWN_LAYOUTTYPE = 10062, + NFS4ERR_SEQ_MISORDERED = 10063, + NFS4ERR_SEQUENCE_POS = 10064, + NFS4ERR_REQ_TOO_BIG = 10065, + NFS4ERR_REP_TOO_BIG = 10066, + NFS4ERR_REP_TOO_BIG_TO_CACHE = 10067, + NFS4ERR_RETRY_UNCACHED_REP = 10068, + NFS4ERR_UNSAFE_COMPOUND = 10069, + NFS4ERR_TOO_MANY_OPS = 10070, + NFS4ERR_OP_NOT_IN_SESSION = 10071, + NFS4ERR_HASH_ALG_UNSUPP = 10072, + NFS4ERR_CLIENTID_BUSY = 10074, + NFS4ERR_PNFS_IO_HOLE = 10075, + NFS4ERR_SEQ_FALSE_RETRY = 10076, + NFS4ERR_BAD_HIGH_SLOT = 10077, + NFS4ERR_DEADSESSION = 10078, + NFS4ERR_ENCR_ALG_UNSUPP = 10079, + NFS4ERR_PNFS_NO_LAYOUT = 10080, + NFS4ERR_NOT_ONLY_OP = 10081, + NFS4ERR_WRONG_CRED = 10082, + NFS4ERR_WRONG_TYPE = 10083, + NFS4ERR_DIRDELEG_UNAVAIL = 10084, + NFS4ERR_REJECT_DELEG = 10085, + NFS4ERR_RETURNCONFLICT = 10086, + NFS4ERR_DELEG_REVOKED = 10087, + NFS4ERR_PARTNER_NOTSUPP = 10088, + NFS4ERR_PARTNER_NO_AUTH = 10089, + NFS4ERR_UNION_NOTSUPP = 10090, + NFS4ERR_OFFLOAD_DENIED = 10091, + NFS4ERR_WRONG_LFS = 10092, + NFS4ERR_BADLABEL = 10093, + NFS4ERR_OFFLOAD_NO_REQS = 10094, + NFS4ERR_NOXATTR = 10095, + NFS4ERR_XATTR2BIG = 10096, +}; + +typedef enum nfsstat4 nfsstat4; + +typedef opaque attrlist4; + typedef struct { u32 count; uint32_t *element; } bitmap4; +typedef u8 verifier4[NFS4_VERIFIER_SIZE]; + +typedef uint64_t nfs_cookie4; + +typedef opaque nfs_fh4; + typedef opaque utf8string; typedef utf8string utf8str_cis; @@ -26,11 +160,30 @@ typedef utf8string utf8str_cs; typedef utf8string utf8str_mixed; +typedef utf8str_cs component4; + +typedef utf8str_cs linktext4; + +typedef struct { + u32 count; + component4 *element; +} pathname4; + struct nfstime4 { int64_t seconds; uint32_t nseconds; }; +struct fattr4 { + bitmap4 attrmask; + attrlist4 attr_vals; +}; + +struct stateid4 { + uint32_t seqid; + u8 other[12]; +}; + typedef bool fattr4_offline; enum { FATTR4_OFFLINE = 83 }; @@ -216,11 +369,109 @@ enum { FATTR4_POSIX_DEFAULT_ACL = 91 }; enum { FATTR4_POSIX_ACCESS_ACL = 92 }; -#define NFS4_int64_t_sz \ - (XDR_hyper) +enum notify_type4 { + NOTIFY4_CHANGE_CHILD_ATTRS = 0, + NOTIFY4_CHANGE_DIR_ATTRS = 1, + NOTIFY4_REMOVE_ENTRY = 2, + NOTIFY4_ADD_ENTRY = 3, + NOTIFY4_RENAME_ENTRY = 4, + NOTIFY4_CHANGE_COOKIE_VERIFIER = 5, + NOTIFY4_GFLAG_EXTEND = 6, + NOTIFY4_AUFLAG_VALID = 7, + NOTIFY4_AUFLAG_USER = 8, + NOTIFY4_AUFLAG_GROUP = 9, + NOTIFY4_AUFLAG_OTHER = 10, + NOTIFY4_CHANGE_AUTH = 11, + NOTIFY4_CFLAG_ORDER = 12, + NOTIFY4_AUFLAG_GANOW = 13, + NOTIFY4_AUFLAG_GALATER = 14, + NOTIFY4_CHANGE_GA = 15, + NOTIFY4_CHANGE_AMASK = 16, +}; + +typedef enum notify_type4 notify_type4; + +struct notify_entry4 { + component4 ne_file; + struct fattr4 ne_attrs; +}; + +struct prev_entry4 { + struct notify_entry4 pe_prev_entry; + nfs_cookie4 pe_prev_entry_cookie; +}; + +struct notify_remove4 { + struct notify_entry4 nrm_old_entry; + nfs_cookie4 nrm_old_entry_cookie; +}; + +struct notify_add4 { + struct { + u32 count; + struct notify_remove4 *element; + } nad_old_entry; + struct notify_entry4 nad_new_entry; + struct { + u32 count; + nfs_cookie4 *element; + } nad_new_entry_cookie; + struct { + u32 count; + struct prev_entry4 *element; + } nad_prev_entry; + bool nad_last_entry; +}; + +struct notify_attr4 { + struct notify_entry4 na_changed_entry; +}; + +struct notify_rename4 { + struct notify_remove4 nrn_old_entry; + struct notify_add4 nrn_new_entry; +}; + +struct notify_verifier4 { + verifier4 nv_old_cookieverf; + verifier4 nv_new_cookieverf; +}; + +typedef opaque notifylist4; + +struct notify4 { + bitmap4 notify_mask; + notifylist4 notify_vals; +}; + +struct CB_NOTIFY4args { + struct stateid4 cna_stateid; + nfs_fh4 cna_fh; + struct { + u32 count; + struct notify4 *element; + } cna_changes; +}; + +struct CB_NOTIFY4res { + nfsstat4 cnr_status; +}; + +#define NFS4_int32_t_sz \ + (XDR_int) #define NFS4_uint32_t_sz \ (XDR_unsigned_int) +#define NFS4_int64_t_sz \ + (XDR_hyper) +#define NFS4_uint64_t_sz \ + (XDR_unsigned_hyper) +#define NFS4_nfsstat4_sz (XDR_int) +#define NFS4_attrlist4_sz (XDR_unsigned_int) #define NFS4_bitmap4_sz (XDR_unsigned_int) +#define NFS4_verifier4_sz (XDR_QUADLEN(NFS4_VERIFIER_SIZE)) +#define NFS4_nfs_cookie4_sz \ + (NFS4_uint64_t_sz) +#define NFS4_nfs_fh4_sz (XDR_unsigned_int + XDR_QUADLEN(NFS4_FHSIZE)) #define NFS4_utf8string_sz (XDR_unsigned_int) #define NFS4_utf8str_cis_sz \ (NFS4_utf8string_sz) @@ -228,8 +479,17 @@ enum { FATTR4_POSIX_ACCESS_ACL = 92 }; (NFS4_utf8string_sz) #define NFS4_utf8str_mixed_sz \ (NFS4_utf8string_sz) +#define NFS4_component4_sz \ + (NFS4_utf8str_cs_sz) +#define NFS4_linktext4_sz \ + (NFS4_utf8str_cs_sz) +#define NFS4_pathname4_sz (XDR_unsigned_int) #define NFS4_nfstime4_sz \ (NFS4_int64_t_sz + NFS4_uint32_t_sz) +#define NFS4_fattr4_sz \ + (NFS4_bitmap4_sz + NFS4_attrlist4_sz) +#define NFS4_stateid4_sz \ + (NFS4_uint32_t_sz + XDR_QUADLEN(12)) #define NFS4_fattr4_offline_sz \ (XDR_bool) #define NFS4_open_arguments4_sz \ @@ -259,5 +519,27 @@ enum { FATTR4_POSIX_ACCESS_ACL = 92 }; (NFS4_aclscope4_sz) #define NFS4_fattr4_posix_default_acl_sz (XDR_unsigned_int) #define NFS4_fattr4_posix_access_acl_sz (XDR_unsigned_int) +#define NFS4_notify_type4_sz (XDR_int) +#define NFS4_notify_entry4_sz \ + (NFS4_component4_sz + NFS4_fattr4_sz) +#define NFS4_prev_entry4_sz \ + (NFS4_notify_entry4_sz + NFS4_nfs_cookie4_sz) +#define NFS4_notify_remove4_sz \ + (NFS4_notify_entry4_sz + NFS4_nfs_cookie4_sz) +#define NFS4_notify_add4_sz \ + (XDR_unsigned_int + (1 * (NFS4_notify_remove4_sz)) + NFS4_notify_entry4_sz + XDR_unsigned_int + (1 * (NFS4_nfs_cookie4_sz)) + XDR_unsigned_int + (1 * (NFS4_prev_entry4_sz)) + XDR_bool) +#define NFS4_notify_attr4_sz \ + (NFS4_notify_entry4_sz) +#define NFS4_notify_rename4_sz \ + (NFS4_notify_remove4_sz + NFS4_notify_add4_sz) +#define NFS4_notify_verifier4_sz \ + (NFS4_verifier4_sz + NFS4_verifier4_sz) +#define NFS4_notifylist4_sz (XDR_unsigned_int) +#define NFS4_notify4_sz \ + (NFS4_bitmap4_sz + NFS4_notifylist4_sz) +#define NFS4_CB_NOTIFY4args_sz \ + (NFS4_stateid4_sz + NFS4_nfs_fh4_sz + XDR_unsigned_int) +#define NFS4_CB_NOTIFY4res_sz \ + (NFS4_nfsstat4_sz) #endif /* _LINUX_XDRGEN_NFS4_1_DEF_H */ diff --git a/include/linux/sunrpc/xdrgen/nlm3.h b/include/linux/sunrpc/xdrgen/nlm3.h new file mode 100644 index 000000000000..0fc627031d8a --- /dev/null +++ b/include/linux/sunrpc/xdrgen/nlm3.h @@ -0,0 +1,210 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Generated by xdrgen. Manual edits will be lost. */ +/* XDR specification file: ../../Documentation/sunrpc/xdr/nlm3.x */ +/* XDR specification modification time: Mon Jun 29 20:42:29 2026 */ + +#ifndef _LINUX_XDRGEN_NLM3_DEF_H +#define _LINUX_XDRGEN_NLM3_DEF_H + +#include <linux/types.h> +#include <linux/sunrpc/xdrgen/_defs.h> + +enum { LM_MAXSTRLEN = 1024 }; + +enum { LM_MAXNAMELEN = 1025 }; + +enum { MAXNETOBJ_SZ = 1024 }; + +typedef opaque netobj; + +enum nlm_stats { + LCK_GRANTED = 0, + LCK_DENIED = 1, + LCK_DENIED_NOLOCKS = 2, + LCK_BLOCKED = 3, + LCK_DENIED_GRACE_PERIOD = 4, +}; + +typedef __be32 nlm_stats; + +struct nlm_stat { + nlm_stats stat; +}; + +struct nlm_res { + netobj cookie; + struct nlm_stat stat; +}; + +struct nlm_holder { + bool exclusive; + s32 uppid; + netobj oh; + u32 l_offset; + u32 l_len; +}; + +struct nlm_testrply { + nlm_stats stat; + union { + struct nlm_holder holder; + } u; +}; + +struct nlm_testres { + netobj cookie; + struct nlm_testrply test_stat; +}; + +struct nlm_lock { + string caller_name; + netobj fh; + netobj oh; + s32 uppid; + u32 l_offset; + u32 l_len; +}; + +struct nlm_lockargs { + netobj cookie; + bool block; + bool exclusive; + struct nlm_lock alock; + bool reclaim; + s32 state; +}; + +struct nlm_cancargs { + netobj cookie; + bool block; + bool exclusive; + struct nlm_lock alock; +}; + +struct nlm_testargs { + netobj cookie; + bool exclusive; + struct nlm_lock alock; +}; + +struct nlm_unlockargs { + netobj cookie; + struct nlm_lock alock; +}; + +enum fsh_mode { + fsm_DN = 0, + fsm_DR = 1, + fsm_DW = 2, + fsm_DRW = 3, +}; + +typedef enum fsh_mode fsh_mode; + +enum fsh_access { + fsa_NONE = 0, + fsa_R = 1, + fsa_W = 2, + fsa_RW = 3, +}; + +typedef enum fsh_access fsh_access; + +struct nlm_share { + string caller_name; + netobj fh; + netobj oh; + fsh_mode mode; + fsh_access access; +}; + +struct nlm_shareargs { + netobj cookie; + struct nlm_share share; + bool reclaim; +}; + +struct nlm_shareres { + netobj cookie; + nlm_stats stat; + s32 sequence; +}; + +struct nlm_notify { + string name; + s32 state; +}; + +enum { SM_PRIV_SIZE = 16 }; + +struct nlm_notifyargs { + struct nlm_notify notify; + u8 private[SM_PRIV_SIZE]; +}; + +enum { + NLM_NULL = 0, + NLM_TEST = 1, + NLM_LOCK = 2, + NLM_CANCEL = 3, + NLM_UNLOCK = 4, + NLM_GRANTED = 5, + NLM_TEST_MSG = 6, + NLM_LOCK_MSG = 7, + NLM_CANCEL_MSG = 8, + NLM_UNLOCK_MSG = 9, + NLM_GRANTED_MSG = 10, + NLM_TEST_RES = 11, + NLM_LOCK_RES = 12, + NLM_CANCEL_RES = 13, + NLM_UNLOCK_RES = 14, + NLM_GRANTED_RES = 15, + NLM_SM_NOTIFY = 16, + NLM_SHARE = 20, + NLM_UNSHARE = 21, + NLM_NM_LOCK = 22, + NLM_FREE_ALL = 23, +}; + +#ifndef NLM_PROG +#define NLM_PROG (100021) +#endif + +#define NLM3_netobj_sz (XDR_unsigned_int + XDR_QUADLEN(MAXNETOBJ_SZ)) +#define NLM3_nlm_stats_sz (XDR_int) +#define NLM3_nlm_stat_sz \ + (NLM3_nlm_stats_sz) +#define NLM3_nlm_res_sz \ + (NLM3_netobj_sz + NLM3_nlm_stat_sz) +#define NLM3_nlm_holder_sz \ + (XDR_bool + XDR_int + NLM3_netobj_sz + XDR_unsigned_int + XDR_unsigned_int) +#define NLM3_nlm_testrply_sz \ + (NLM3_nlm_stats_sz + NLM3_nlm_holder_sz) +#define NLM3_nlm_testres_sz \ + (NLM3_netobj_sz + NLM3_nlm_testrply_sz) +#define NLM3_nlm_lock_sz \ + (XDR_unsigned_int + XDR_QUADLEN(LM_MAXSTRLEN) + NLM3_netobj_sz + NLM3_netobj_sz + XDR_int + XDR_unsigned_int + XDR_unsigned_int) +#define NLM3_nlm_lockargs_sz \ + (NLM3_netobj_sz + XDR_bool + XDR_bool + NLM3_nlm_lock_sz + XDR_bool + XDR_int) +#define NLM3_nlm_cancargs_sz \ + (NLM3_netobj_sz + XDR_bool + XDR_bool + NLM3_nlm_lock_sz) +#define NLM3_nlm_testargs_sz \ + (NLM3_netobj_sz + XDR_bool + NLM3_nlm_lock_sz) +#define NLM3_nlm_unlockargs_sz \ + (NLM3_netobj_sz + NLM3_nlm_lock_sz) +#define NLM3_fsh_mode_sz (XDR_int) +#define NLM3_fsh_access_sz (XDR_int) +#define NLM3_nlm_share_sz \ + (XDR_unsigned_int + XDR_QUADLEN(LM_MAXSTRLEN) + NLM3_netobj_sz + NLM3_netobj_sz + NLM3_fsh_mode_sz + NLM3_fsh_access_sz) +#define NLM3_nlm_shareargs_sz \ + (NLM3_netobj_sz + NLM3_nlm_share_sz + XDR_bool) +#define NLM3_nlm_shareres_sz \ + (NLM3_netobj_sz + NLM3_nlm_stats_sz + XDR_int) +#define NLM3_nlm_notify_sz \ + (XDR_unsigned_int + XDR_QUADLEN(LM_MAXNAMELEN) + XDR_long) +#define NLM3_nlm_notifyargs_sz \ + (NLM3_nlm_notify_sz + XDR_QUADLEN(SM_PRIV_SIZE)) +#define NLM3_MAX_ARGS_SZ \ + (NLM3_nlm_lockargs_sz) + +#endif /* _LINUX_XDRGEN_NLM3_DEF_H */ diff --git a/include/linux/sunrpc/xdrgen/nlm4.h b/include/linux/sunrpc/xdrgen/nlm4.h new file mode 100644 index 000000000000..77860a3d1c1e --- /dev/null +++ b/include/linux/sunrpc/xdrgen/nlm4.h @@ -0,0 +1,233 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Generated by xdrgen. Manual edits will be lost. */ +/* XDR specification file: ../../Documentation/sunrpc/xdr/nlm4.x */ +/* XDR specification modification time: Mon Jun 29 20:42:29 2026 */ + +#ifndef _LINUX_XDRGEN_NLM4_DEF_H +#define _LINUX_XDRGEN_NLM4_DEF_H + +#include <linux/types.h> +#include <linux/sunrpc/xdrgen/_defs.h> + +enum { LM_MAXSTRLEN = 1024 }; + +enum { LM_MAXNAMELEN = 1025 }; + +enum { MAXNETOBJ_SZ = 1024 }; + +typedef opaque netobj; + +enum fsh4_mode { + fsm_DN = 0, + fsm_DR = 1, + fsm_DW = 2, + fsm_DRW = 3, +}; + +typedef enum fsh4_mode fsh4_mode; + +enum fsh4_access { + fsa_NONE = 0, + fsa_R = 1, + fsa_W = 2, + fsa_RW = 3, +}; + +typedef enum fsh4_access fsh4_access; + +enum { SM_MAXSTRLEN = 1024 }; + +typedef u64 uint64; + +typedef s64 int64; + +typedef u32 uint32; + +typedef s32 int32; + +enum nlm4_stats { + NLM4_GRANTED = 0, + NLM4_DENIED = 1, + NLM4_DENIED_NOLOCKS = 2, + NLM4_BLOCKED = 3, + NLM4_DENIED_GRACE_PERIOD = 4, + NLM4_DEADLCK = 5, + NLM4_ROFS = 6, + NLM4_STALE_FH = 7, + NLM4_FBIG = 8, + NLM4_FAILED = 9, +}; + +typedef __be32 nlm4_stats; + +struct nlm4_holder { + bool exclusive; + int32 svid; + netobj oh; + uint64 l_offset; + uint64 l_len; +}; + +struct nlm4_testrply { + nlm4_stats stat; + union { + struct nlm4_holder holder; + } u; +}; + +struct nlm4_stat { + nlm4_stats stat; +}; + +struct nlm4_res { + netobj cookie; + struct nlm4_stat stat; +}; + +struct nlm4_testres { + netobj cookie; + struct nlm4_testrply stat; +}; + +struct nlm4_lock { + string caller_name; + netobj fh; + netobj oh; + int32 svid; + uint64 l_offset; + uint64 l_len; +}; + +struct nlm4_lockargs { + netobj cookie; + bool block; + bool exclusive; + struct nlm4_lock alock; + bool reclaim; + int32 state; +}; + +struct nlm4_cancargs { + netobj cookie; + bool block; + bool exclusive; + struct nlm4_lock alock; +}; + +struct nlm4_testargs { + netobj cookie; + bool exclusive; + struct nlm4_lock alock; +}; + +struct nlm4_unlockargs { + netobj cookie; + struct nlm4_lock alock; +}; + +struct nlm4_share { + string caller_name; + netobj fh; + netobj oh; + fsh4_mode mode; + fsh4_access access; +}; + +struct nlm4_shareargs { + netobj cookie; + struct nlm4_share share; + bool reclaim; +}; + +struct nlm4_shareres { + netobj cookie; + nlm4_stats stat; + int32 sequence; +}; + +struct nlm4_notify { + string name; + int32 state; +}; + +enum { SM_PRIV_SIZE = 16 }; + +struct nlm4_notifyargs { + struct nlm4_notify notify; + u8 private[SM_PRIV_SIZE]; +}; + +enum { + NLMPROC4_NULL = 0, + NLMPROC4_TEST = 1, + NLMPROC4_LOCK = 2, + NLMPROC4_CANCEL = 3, + NLMPROC4_UNLOCK = 4, + NLMPROC4_GRANTED = 5, + NLMPROC4_TEST_MSG = 6, + NLMPROC4_LOCK_MSG = 7, + NLMPROC4_CANCEL_MSG = 8, + NLMPROC4_UNLOCK_MSG = 9, + NLMPROC4_GRANTED_MSG = 10, + NLMPROC4_TEST_RES = 11, + NLMPROC4_LOCK_RES = 12, + NLMPROC4_CANCEL_RES = 13, + NLMPROC4_UNLOCK_RES = 14, + NLMPROC4_GRANTED_RES = 15, + NLMPROC4_SM_NOTIFY = 16, + NLMPROC4_SHARE = 20, + NLMPROC4_UNSHARE = 21, + NLMPROC4_NM_LOCK = 22, + NLMPROC4_FREE_ALL = 23, +}; + +#ifndef NLM4_PROG +#define NLM4_PROG (100021) +#endif + +#define NLM4_netobj_sz (XDR_unsigned_int + XDR_QUADLEN(MAXNETOBJ_SZ)) +#define NLM4_fsh4_mode_sz (XDR_int) +#define NLM4_fsh4_access_sz (XDR_int) +#define NLM4_uint64_sz \ + (XDR_unsigned_hyper) +#define NLM4_int64_sz \ + (XDR_hyper) +#define NLM4_uint32_sz \ + (XDR_unsigned_long) +#define NLM4_int32_sz \ + (XDR_long) +#define NLM4_nlm4_stats_sz (XDR_int) +#define NLM4_nlm4_holder_sz \ + (XDR_bool + NLM4_int32_sz + NLM4_netobj_sz + NLM4_uint64_sz + NLM4_uint64_sz) +#define NLM4_nlm4_testrply_sz \ + (NLM4_nlm4_stats_sz + NLM4_nlm4_holder_sz) +#define NLM4_nlm4_stat_sz \ + (NLM4_nlm4_stats_sz) +#define NLM4_nlm4_res_sz \ + (NLM4_netobj_sz + NLM4_nlm4_stat_sz) +#define NLM4_nlm4_testres_sz \ + (NLM4_netobj_sz + NLM4_nlm4_testrply_sz) +#define NLM4_nlm4_lock_sz \ + (XDR_unsigned_int + XDR_QUADLEN(LM_MAXSTRLEN) + NLM4_netobj_sz + NLM4_netobj_sz + NLM4_int32_sz + NLM4_uint64_sz + NLM4_uint64_sz) +#define NLM4_nlm4_lockargs_sz \ + (NLM4_netobj_sz + XDR_bool + XDR_bool + NLM4_nlm4_lock_sz + XDR_bool + NLM4_int32_sz) +#define NLM4_nlm4_cancargs_sz \ + (NLM4_netobj_sz + XDR_bool + XDR_bool + NLM4_nlm4_lock_sz) +#define NLM4_nlm4_testargs_sz \ + (NLM4_netobj_sz + XDR_bool + NLM4_nlm4_lock_sz) +#define NLM4_nlm4_unlockargs_sz \ + (NLM4_netobj_sz + NLM4_nlm4_lock_sz) +#define NLM4_nlm4_share_sz \ + (XDR_unsigned_int + XDR_QUADLEN(LM_MAXSTRLEN) + NLM4_netobj_sz + NLM4_netobj_sz + NLM4_fsh4_mode_sz + NLM4_fsh4_access_sz) +#define NLM4_nlm4_shareargs_sz \ + (NLM4_netobj_sz + NLM4_nlm4_share_sz + XDR_bool) +#define NLM4_nlm4_shareres_sz \ + (NLM4_netobj_sz + NLM4_nlm4_stats_sz + NLM4_int32_sz) +#define NLM4_nlm4_notify_sz \ + (XDR_unsigned_int + XDR_QUADLEN(LM_MAXNAMELEN) + NLM4_int32_sz) +#define NLM4_nlm4_notifyargs_sz \ + (NLM4_nlm4_notify_sz + XDR_QUADLEN(SM_PRIV_SIZE)) +#define NLM4_MAX_ARGS_SZ \ + (NLM4_nlm4_lockargs_sz) + +#endif /* _LINUX_XDRGEN_NLM4_DEF_H */ diff --git a/include/linux/sunrpc/xprt.h b/include/linux/sunrpc/xprt.h index f46d1fb8f71a..a82045804d34 100644 --- a/include/linux/sunrpc/xprt.h +++ b/include/linux/sunrpc/xprt.h @@ -404,6 +404,8 @@ struct rpc_xprt * xprt_alloc(struct net *net, size_t size, unsigned int max_req); void xprt_free(struct rpc_xprt *); void xprt_add_backlog(struct rpc_xprt *xprt, struct rpc_task *task); +void xprt_add_backlog_noncongested(struct rpc_xprt *xprt, + struct rpc_task *task); bool xprt_wake_up_backlog(struct rpc_xprt *xprt, struct rpc_rqst *req); void xprt_cleanup_ids(void); diff --git a/include/linux/surface_aggregator/device.h b/include/linux/surface_aggregator/device.h index 8cd8c38cf3f3..ed6b271e5a73 100644 --- a/include/linux/surface_aggregator/device.h +++ b/include/linux/surface_aggregator/device.h @@ -14,7 +14,7 @@ #define _LINUX_SURFACE_AGGREGATOR_DEVICE_H #include <linux/device.h> -#include <linux/mod_devicetable.h> +#include <linux/device-id/ssam.h> #include <linux/property.h> #include <linux/types.h> diff --git a/include/linux/swap.h b/include/linux/swap.h index 62fc7499b408..5658a1634b85 100644 --- a/include/linux/swap.h +++ b/include/linux/swap.h @@ -16,12 +16,6 @@ #include <uapi/linux/mempolicy.h> #include <asm/page.h> -struct notifier_block; - -struct bio; - -struct pagevec; - #define SWAP_FLAG_PREFER 0x8000 /* set if swap priority specified */ #define SWAP_FLAG_PRIO_MASK 0x7fff #define SWAP_FLAG_DISCARD 0x10000 /* enable discard for swap */ @@ -31,7 +25,6 @@ struct pagevec; #define SWAP_FLAGS_VALID (SWAP_FLAG_PRIO_MASK | SWAP_FLAG_PREFER | \ SWAP_FLAG_DISCARD | SWAP_FLAG_DISCARD_ONCE | \ SWAP_FLAG_DISCARD_PAGES) -#define SWAP_BATCH 64 static inline int current_is_kswapd(void) { @@ -177,7 +170,6 @@ static inline void mm_account_reclaimed_pages(unsigned long pages) struct address_space; struct sysinfo; -struct writeback_control; struct zone; /* @@ -208,14 +200,13 @@ enum { SWP_DISCARDABLE = (1 << 2), /* blkdev support discard */ SWP_DISCARDING = (1 << 3), /* now discarding a free cluster */ SWP_SOLIDSTATE = (1 << 4), /* blkdev seeks are cheap */ - SWP_CONTINUED = (1 << 5), /* swap_map has count continuation */ SWP_BLKDEV = (1 << 6), /* its a block device */ SWP_ACTIVATED = (1 << 7), /* set after swap_activate success */ - SWP_FS_OPS = (1 << 8), /* swapfile operations go through fs */ SWP_AREA_DISCARD = (1 << 9), /* single-time swap area discards */ SWP_PAGE_DISCARD = (1 << 10), /* freed swap page-cluster discards */ SWP_STABLE_WRITES = (1 << 11), /* no overwrite PG_writeback pages */ SWP_SYNCHRONOUS_IO = (1 << 12), /* synchronous IO is efficient */ + SWP_HIBERNATION = (1 << 13), /* pinned for hibernation */ /* add others here before... */ }; @@ -223,16 +214,6 @@ enum { #define SWAP_CLUSTER_MAX_SKIPPED (SWAP_CLUSTER_MAX << 10) #define COMPACT_CLUSTER_MAX SWAP_CLUSTER_MAX -/* Bit flag in swap_map */ -#define COUNT_CONTINUED 0x80 /* Flag swap_map continuation for full count */ - -/* Special value in first swap_map */ -#define SWAP_MAP_MAX 0x3e /* Max count */ -#define SWAP_MAP_BAD 0x3f /* Note page is bad */ - -/* Special value in each swap_map continuation */ -#define SWAP_CONT_MAX 0x7f /* Max count */ - /* * The first page in the swap file is the swap header, which is always marked * bad to prevent it from being allocated as an entry. This also prevents the @@ -264,9 +245,7 @@ struct swap_info_struct { signed short prio; /* swap priority of this type */ struct plist_node list; /* entry in swap_active_head */ signed char type; /* strange name for an index */ - unsigned int max; /* extent of the swap_map */ - unsigned char *swap_map; /* vmalloc'ed array of usage counts */ - unsigned long *zeromap; /* kvmalloc'ed bitmap to track zero pages */ + unsigned int max; /* size of this swap device */ struct swap_cluster_info *cluster_info; /* cluster info. Only for SSD */ struct list_head free_clusters; /* free clusters list */ struct list_head full_clusters; /* full clusters list */ @@ -284,22 +263,19 @@ struct swap_info_struct { struct completion comp; /* seldom referenced */ spinlock_t lock; /* * protect map scan related fields like - * swap_map, inuse_pages and all cluster - * lists. other fields are only changed + * inuse_pages and all cluster lists. + * Other fields are only changed * at swapon/swapoff, so are protected * by swap_lock. changing flags need * hold this lock and swap_lock. If * both locks need hold, hold swap_lock * first. */ - spinlock_t cont_lock; /* - * protect swap count continuation page - * list. - */ struct work_struct discard_work; /* discard worker */ struct work_struct reclaim_work; /* reclaim worker */ struct list_head discard_clusters; /* discard clusters list */ struct plist_node avail_list; /* entry in swap_avail_head */ + const struct swap_ops *ops; }; static inline swp_entry_t page_swap_entry(struct page *page) @@ -311,40 +287,27 @@ static inline swp_entry_t page_swap_entry(struct page *page) return entry; } -/* linux/mm/workingset.c */ -bool workingset_test_recent(void *shadow, bool file, bool *workingset, - bool flush); -void workingset_age_nonresident(struct lruvec *lruvec, unsigned long nr_pages); -void *workingset_eviction(struct folio *folio, struct mem_cgroup *target_memcg); -void workingset_refault(struct folio *folio, void *shadow); -void workingset_activation(struct folio *folio); - /* linux/mm/page_alloc.c */ extern unsigned long totalreserve_pages; /* Definition of global_zone_page_state not available yet */ #define nr_free_pages() global_zone_page_state(NR_FREE_PAGES) +/* linux/mm/folio.c */ +void folio_add_lru(struct folio *folio); +void folio_mark_accessed(struct folio *folio); +void lru_add_drain_all(void); -/* linux/mm/swap.c */ -void lru_note_cost_unlock_irq(struct lruvec *lruvec, bool file, - unsigned int nr_io, unsigned int nr_rotated) - __releases(lruvec->lru_lock); -void lru_note_cost_refault(struct folio *); -void folio_add_lru(struct folio *); -void folio_add_lru_vma(struct folio *, struct vm_area_struct *); -void mark_page_accessed(struct page *); -void folio_mark_accessed(struct folio *); +enum lru_cache_drained { + LRU_CACHE_NOT_DRAINED, + LRU_CACHE_DRAINED, + LRU_CACHE_DRAINED_ALL, +}; +void lru_cache_drain_for_folio(const struct folio *folio, + unsigned int extra_refs, enum lru_cache_drained *drained); -static inline bool folio_may_be_lru_cached(struct folio *folio) -{ - /* - * Holding PMD-sized folios in per-CPU LRU cache unbalances accounting. - * Holding small numbers of low-order mTHP folios in per-CPU LRU cache - * will be sensible, but nobody has implemented and tested that yet. - */ - return !folio_test_large(folio); -} +/* linux/mm/folio-compat.c */ +void mark_page_accessed(struct page *page); extern atomic_t lru_disable_count; @@ -353,44 +316,7 @@ static inline bool lru_cache_disabled(void) return atomic_read(&lru_disable_count); } -static inline void lru_cache_enable(void) -{ - atomic_dec(&lru_disable_count); -} - -extern void lru_cache_disable(void); -extern void lru_add_drain(void); -extern void lru_add_drain_cpu(int cpu); -extern void lru_add_drain_cpu_zone(struct zone *zone); -extern void lru_add_drain_all(void); -void folio_deactivate(struct folio *folio); -void folio_mark_lazyfree(struct folio *folio); -extern void swap_setup(void); - -/* linux/mm/vmscan.c */ -extern unsigned long zone_reclaimable_pages(struct zone *zone); -extern unsigned long try_to_free_pages(struct zonelist *zonelist, int order, - gfp_t gfp_mask, nodemask_t *mask); - -#define MEMCG_RECLAIM_MAY_SWAP (1 << 1) -#define MEMCG_RECLAIM_PROACTIVE (1 << 2) -#define MIN_SWAPPINESS 0 -#define MAX_SWAPPINESS 200 - -/* Just reclaim from anon folios in proactive memory reclaim */ -#define SWAPPINESS_ANON_ONLY (MAX_SWAPPINESS + 1) - -extern unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg, - unsigned long nr_pages, - gfp_t gfp_mask, - unsigned int reclaim_options, - int *swappiness); -extern unsigned long mem_cgroup_shrink_node(struct mem_cgroup *mem, - gfp_t gfp_mask, bool noswap, - pg_data_t *pgdat, - unsigned long *nr_scanned); extern unsigned long shrink_all_memory(unsigned long nr_pages); -extern int vm_swappiness; long remove_mapping(struct address_space *mapping, struct folio *folio); #if defined(CONFIG_SYSFS) && defined(CONFIG_NUMA) @@ -409,18 +335,12 @@ static inline void reclaim_unregister_node(struct node *node) } #endif /* CONFIG_SYSFS && CONFIG_NUMA */ -#ifdef CONFIG_NUMA -extern int sysctl_min_unmapped_ratio; -extern int sysctl_min_slab_ratio; -#endif - void check_move_unevictable_folios(struct folio_batch *fbatch); extern void __meminit kswapd_run(int nid); extern void __meminit kswapd_stop(int nid); #ifdef CONFIG_SWAP - int add_swap_extent(struct swap_info_struct *sis, unsigned long start_page, unsigned long nr_pages, sector_t start_block); int generic_swapfile_activate(struct swap_info_struct *, struct file *, @@ -451,15 +371,15 @@ static inline long get_nr_swap_pages(void) } extern void si_swapinfo(struct sysinfo *); -extern int add_swap_count_continuation(swp_entry_t, gfp_t); -int swap_type_of(dev_t device, sector_t offset); +extern int pin_hibernation_swap_type(dev_t device, sector_t offset); +extern void unpin_hibernation_swap_type(int type); +extern int find_hibernation_swap_type(dev_t device, sector_t offset); int find_first_swap(dev_t *device); extern unsigned int count_swap_pages(int, int); extern sector_t swapdev_block(int, pgoff_t); extern int __swap_count(swp_entry_t entry); extern bool swap_entry_swapped(struct swap_info_struct *si, swp_entry_t entry); extern int swp_swapcount(swp_entry_t entry); -struct backing_dev_info; extern struct swap_info_struct *get_swap_device(swp_entry_t entry); sector_t swap_folio_sector(struct folio *folio); @@ -517,11 +437,6 @@ static inline void free_swap_cache(struct folio *folio) { } -static inline int add_swap_count_continuation(swp_entry_t swp, gfp_t gfp_mask) -{ - return 0; -} - static inline int swap_dup_entry_direct(swp_entry_t ent) { return 0; @@ -559,23 +474,7 @@ static inline int add_swap_extent(struct swap_info_struct *sis, } #endif /* CONFIG_SWAP */ #ifdef CONFIG_MEMCG -static inline int mem_cgroup_swappiness(struct mem_cgroup *memcg) -{ - /* Cgroup2 doesn't have per-cgroup swappiness */ - if (cgroup_subsys_on_dfl(memory_cgrp_subsys)) - return READ_ONCE(vm_swappiness); - - /* root ? */ - if (mem_cgroup_disabled() || mem_cgroup_is_root(memcg)) - return READ_ONCE(vm_swappiness); - - return READ_ONCE(memcg->swappiness); -} -#else -static inline int mem_cgroup_swappiness(struct mem_cgroup *mem) -{ - return READ_ONCE(vm_swappiness); -} +void lru_reparent_memcg(struct mem_cgroup *memcg, struct mem_cgroup *parent, int nid); #endif #if defined(CONFIG_SWAP) && defined(CONFIG_MEMCG) && defined(CONFIG_BLK_CGROUP) @@ -593,33 +492,31 @@ static inline void folio_throttle_swaprate(struct folio *folio, gfp_t gfp) #endif #if defined(CONFIG_MEMCG) && defined(CONFIG_SWAP) -int __mem_cgroup_try_charge_swap(struct folio *folio, swp_entry_t entry); -static inline int mem_cgroup_try_charge_swap(struct folio *folio, - swp_entry_t entry) +int __mem_cgroup_try_charge_swap(struct folio *folio); +static inline int mem_cgroup_try_charge_swap(struct folio *folio) { if (mem_cgroup_disabled()) return 0; - return __mem_cgroup_try_charge_swap(folio, entry); + return __mem_cgroup_try_charge_swap(folio); } -extern void __mem_cgroup_uncharge_swap(swp_entry_t entry, unsigned int nr_pages); -static inline void mem_cgroup_uncharge_swap(swp_entry_t entry, unsigned int nr_pages) +extern void __mem_cgroup_uncharge_swap(unsigned short id, unsigned int nr_pages); +static inline void mem_cgroup_uncharge_swap(unsigned short id, unsigned int nr_pages) { if (mem_cgroup_disabled()) return; - __mem_cgroup_uncharge_swap(entry, nr_pages); + __mem_cgroup_uncharge_swap(id, nr_pages); } extern long mem_cgroup_get_nr_swap_pages(struct mem_cgroup *memcg); extern bool mem_cgroup_swap_full(struct folio *folio); #else -static inline int mem_cgroup_try_charge_swap(struct folio *folio, - swp_entry_t entry) +static inline int mem_cgroup_try_charge_swap(struct folio *folio) { return 0; } -static inline void mem_cgroup_uncharge_swap(swp_entry_t entry, +static inline void mem_cgroup_uncharge_swap(unsigned short id, unsigned int nr_pages) { } @@ -635,5 +532,24 @@ static inline bool mem_cgroup_swap_full(struct folio *folio) } #endif +/* for_each_managed_zone_pgdat - helper macro to iterate over all managed zones in a pgdat up to + * and including the specified highidx + * @zone: The current zone in the iterator + * @pgdat: The pgdat which node_zones are being iterated + * @idx: The index variable + * @highidx: The index of the highest zone to return + * + * This macro iterates through all managed zones up to and including the specified highidx. + * The zone iterator enters an invalid state after macro call and must be reinitialized + * before it can be used again. + */ +#define for_each_managed_zone_pgdat(zone, pgdat, idx, highidx) \ + for ((idx) = 0, (zone) = (pgdat)->node_zones; \ + (idx) <= (highidx); \ + (idx)++, (zone)++) \ + if (!managed_zone(zone)) \ + continue; \ + else + #endif /* __KERNEL__*/ #endif /* _LINUX_SWAP_H */ diff --git a/include/linux/swap_cgroup.h b/include/linux/swap_cgroup.h deleted file mode 100644 index 91cdf12190a0..000000000000 --- a/include/linux/swap_cgroup.h +++ /dev/null @@ -1,47 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef __LINUX_SWAP_CGROUP_H -#define __LINUX_SWAP_CGROUP_H - -#include <linux/swap.h> - -#if defined(CONFIG_MEMCG) && defined(CONFIG_SWAP) - -extern void swap_cgroup_record(struct folio *folio, unsigned short id, swp_entry_t ent); -extern unsigned short swap_cgroup_clear(swp_entry_t ent, unsigned int nr_ents); -extern unsigned short lookup_swap_cgroup_id(swp_entry_t ent); -extern int swap_cgroup_swapon(int type, unsigned long max_pages); -extern void swap_cgroup_swapoff(int type); - -#else - -static inline -void swap_cgroup_record(struct folio *folio, unsigned short id, swp_entry_t ent) -{ -} - -static inline -unsigned short swap_cgroup_clear(swp_entry_t ent, unsigned int nr_ents) -{ - return 0; -} - -static inline -unsigned short lookup_swap_cgroup_id(swp_entry_t ent) -{ - return 0; -} - -static inline int -swap_cgroup_swapon(int type, unsigned long max_pages) -{ - return 0; -} - -static inline void swap_cgroup_swapoff(int type) -{ - return; -} - -#endif - -#endif /* __LINUX_SWAP_CGROUP_H */ diff --git a/include/linux/swap_ops.h b/include/linux/swap_ops.h new file mode 100644 index 000000000000..57ac6c703f68 --- /dev/null +++ b/include/linux/swap_ops.h @@ -0,0 +1,44 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _MM_SWAP_OPS_H +#define _MM_SWAP_OPS_H + +#include <linux/swap.h> /* for SWAP_CLUSTER_MAX */ + +struct swap_iocb { + union { + struct kiocb iocb; + struct bio bio; + }; + struct bio_vec bvecs[SWAP_CLUSTER_MAX]; + int nr_bvecs; + int len; +}; + +struct swap_io_ctx { + struct swap_iocb *sio; + struct swap_info_struct *sis; +}; + +/* + * SWAP_OPS_F_REQUIRE_NOFS: + * When set, all reclaim operations must operated as GFS_NOFS and not + * just GFP_NOIO, as GFP_NOIO allocations could recourse into the + * file system backing this swap file. + */ +#define SWAP_OPS_F_REQUIRE_NOFS (1U << 0) + +struct swap_ops { + unsigned int flags; + + bool (*can_merge)(struct folio *folio, struct folio *prev_folio, + size_t prev_folio_size, int rw); + void (*submit_write)(struct swap_io_ctx *ctx); + void (*submit_read)(struct swap_io_ctx *ctx); +}; + +void swap_fs_prepare_rw(struct swap_io_ctx *ctx, int rw, struct iov_iter *iter); +bool swap_fs_can_merge(struct folio *folio, struct folio *prev_folio, + size_t prev_folio_size, int rw); +int swap_fs_activate(struct swap_info_struct *sis, const struct swap_ops *ops); + +#endif /* _MM_SWAP_OPS_H */ diff --git a/include/linux/swapops.h b/include/linux/swapops.h index 8cfc966eae48..e7d0d529f3e0 100644 --- a/include/linux/swapops.h +++ b/include/linux/swapops.h @@ -5,6 +5,7 @@ #include <linux/radix-tree.h> #include <linux/bug.h> #include <linux/mm_types.h> +#include <linux/swap.h> #ifdef CONFIG_MMU @@ -73,8 +74,8 @@ static inline pte_t pte_swp_clear_flags(pte_t pte) pte = pte_swp_clear_exclusive(pte); if (pte_swp_soft_dirty(pte)) pte = pte_swp_clear_soft_dirty(pte); - if (pte_swp_uffd_wp(pte)) - pte = pte_swp_clear_uffd_wp(pte); + if (pte_swp_uffd(pte)) + pte = pte_swp_clear_uffd(pte); return pte; } @@ -321,12 +322,12 @@ static inline swp_entry_t make_guard_swp_entry(void) struct page_vma_mapped_walk; -#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifdef CONFIG_ARCH_HAS_PMD_SOFTLEAVES extern int set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw, struct page *page); -extern void remove_migration_pmd(struct page_vma_mapped_walk *pvmw, - struct page *new); +void remove_migration_pmd(struct page_vma_mapped_walk *pvmw, + struct folio *folio); extern void pmd_migration_entry_wait(struct mm_struct *mm, pmd_t *pmd); @@ -338,7 +339,7 @@ static inline pmd_t swp_entry_to_pmd(swp_entry_t entry) return __swp_entry_to_pmd(arch_entry); } -#else /* CONFIG_ARCH_ENABLE_THP_MIGRATION */ +#else /* CONFIG_ARCH_HAS_PMD_SOFTLEAVES */ static inline int set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw, struct page *page) { @@ -346,7 +347,7 @@ static inline int set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw, } static inline void remove_migration_pmd(struct page_vma_mapped_walk *pvmw, - struct page *new) + struct folio *folio) { BUILD_BUG(); } @@ -358,7 +359,7 @@ static inline pmd_t swp_entry_to_pmd(swp_entry_t entry) return __pmd(0); } -#endif /* CONFIG_ARCH_ENABLE_THP_MIGRATION */ +#endif /* CONFIG_ARCH_HAS_PMD_SOFTLEAVES */ #endif /* CONFIG_MMU */ #endif /* _LINUX_SWAPOPS_H */ diff --git a/include/linux/swiotlb.h b/include/linux/swiotlb.h index 3dae0f592063..2a421241b980 100644 --- a/include/linux/swiotlb.h +++ b/include/linux/swiotlb.h @@ -15,8 +15,7 @@ struct page; struct scatterlist; #define SWIOTLB_VERBOSE (1 << 0) /* verbose initialization */ -#define SWIOTLB_FORCE (1 << 1) /* force bounce buffering */ -#define SWIOTLB_ANY (1 << 2) /* allow any memory for the buffer */ +#define SWIOTLB_ANY (1 << 1) /* allow any memory for the buffer */ /* * Maximum allowable number of contiguous slabs to map, @@ -32,8 +31,12 @@ struct scatterlist; #define IO_TLB_SHIFT 11 #define IO_TLB_SIZE (1 << IO_TLB_SHIFT) -/* default to 64MB */ -#define IO_TLB_DEFAULT_SIZE (64UL<<20) +/* compile-time default; overridable via CONFIG_SWIOTLB_DEFAULT_SIZE_MB */ +#ifdef CONFIG_SWIOTLB +#define IO_TLB_DEFAULT_SIZE ((unsigned long)CONFIG_SWIOTLB_DEFAULT_SIZE_MB << 20) +#else +#define IO_TLB_DEFAULT_SIZE (64UL << 20) +#endif unsigned long swiotlb_size_or_default(void); void __init swiotlb_init_remap(bool addressing_limit, unsigned int flags, @@ -64,8 +67,9 @@ extern void __init swiotlb_update_mem_attributes(void); * @areas: Array of memory area descriptors. * @slots: Array of slot descriptors. * @node: Member of the IO TLB memory pool list. - * @rcu: RCU head for swiotlb_dyn_free(). + * @dyn_free: RCU work item used to free the pool from process context. * @transient: %true if transient memory pool. + * @cc_shared: %true if the pool memory is shared for confidential computing. */ struct io_tlb_pool { phys_addr_t start; @@ -79,8 +83,9 @@ struct io_tlb_pool { struct io_tlb_slot *slots; #ifdef CONFIG_SWIOTLB_DYNAMIC struct list_head node; - struct rcu_head rcu; + struct rcu_work dyn_free; bool transient; + bool cc_shared; #endif }; @@ -92,16 +97,17 @@ struct io_tlb_pool { * @debugfs: The dentry to debugfs. * @force_bounce: %true if swiotlb bouncing is forced * @for_alloc: %true if the pool is used for memory allocation + * @cc_shared: %true if the pool memory is shared for confidential computing. * @can_grow: %true if more pools can be allocated dynamically. * @phys_limit: Maximum allowed physical address. * @lock: Lock to synchronize changes to the list. * @pools: List of IO TLB memory pool descriptors (if dynamic). * @dyn_alloc: Dynamic IO TLB pool allocation work. * @total_used: The total number of slots in the pool that are currently used - * across all areas. Used only for calculating used_hiwater in - * debugfs. - * @used_hiwater: The high water mark for total_used. Used only for reporting - * in debugfs. + * across all areas. Used only for calculating used_hiwater via boot + * parameter swiotlb=track_hiwater and exposed via debugfs. + * @used_hiwater: The high water mark for total_used. Can be enabled at boot + * time via swiotlb=track_hiwater and exposed via debugfs. * @transient_nslabs: The total number of slots in all transient pools that * are currently used across all areas. */ @@ -111,6 +117,7 @@ struct io_tlb_mem { struct dentry *debugfs; bool force_bounce; bool for_alloc; + bool cc_shared; #ifdef CONFIG_SWIOTLB_DYNAMIC bool can_grow; u64 phys_limit; @@ -238,7 +245,7 @@ static inline phys_addr_t default_swiotlb_limit(void) phys_addr_t swiotlb_tbl_map_single(struct device *hwdev, phys_addr_t phys, size_t mapping_size, unsigned int alloc_aligned_mask, - enum dma_data_direction dir, unsigned long attrs); + enum dma_data_direction dir, unsigned long *attrs); dma_addr_t swiotlb_map(struct device *dev, phys_addr_t phys, size_t size, enum dma_data_direction dir, unsigned long attrs); @@ -282,15 +289,19 @@ static inline void swiotlb_sync_single_for_cpu(struct device *dev, extern void swiotlb_print_info(void); #ifdef CONFIG_DMA_RESTRICTED_POOL -struct page *swiotlb_alloc(struct device *dev, size_t size); +struct page *swiotlb_alloc(struct device *dev, size_t size, + unsigned long attrs); bool swiotlb_free(struct device *dev, struct page *page, size_t size); +void swiotlb_free_from_pool(struct device *dev, + phys_addr_t tlb_addr, struct io_tlb_pool *pool); static inline bool is_swiotlb_for_alloc(struct device *dev) { return dev->dma_io_tlb_mem->for_alloc; } #else -static inline struct page *swiotlb_alloc(struct device *dev, size_t size) +static inline struct page *swiotlb_alloc(struct device *dev, size_t size, + unsigned long attrs) { return NULL; } @@ -299,6 +310,10 @@ static inline bool swiotlb_free(struct device *dev, struct page *page, { return false; } +static inline void swiotlb_free_from_pool(struct device *dev, + phys_addr_t tlb_addr, struct io_tlb_pool *pool) +{ +} static inline bool is_swiotlb_for_alloc(struct device *dev) { return false; diff --git a/include/linux/switchtec.h b/include/linux/switchtec.h index cdb58d61c152..724da6c08bf7 100644 --- a/include/linux/switchtec.h +++ b/include/linux/switchtec.h @@ -42,6 +42,7 @@ enum switchtec_gen { SWITCHTEC_GEN3, SWITCHTEC_GEN4, SWITCHTEC_GEN5, + SWITCHTEC_GEN6, }; struct mrpc_regs { diff --git a/include/linux/synclink.h b/include/linux/synclink.h deleted file mode 100644 index f1405b1c71ba..000000000000 --- a/include/linux/synclink.h +++ /dev/null @@ -1,37 +0,0 @@ -/* - * SyncLink Multiprotocol Serial Adapter Driver - * - * $Id: synclink.h,v 3.14 2006/07/17 20:15:43 paulkf Exp $ - * - * Copyright (C) 1998-2000 by Microgate Corporation - * - * Redistribution of this file is permitted under - * the terms of the GNU Public License (GPL) - */ -#ifndef _SYNCLINK_H_ -#define _SYNCLINK_H_ - -#include <uapi/linux/synclink.h> - -/* provide 32 bit ioctl compatibility on 64 bit systems */ -#ifdef CONFIG_COMPAT -#include <linux/compat.h> -struct MGSL_PARAMS32 { - compat_ulong_t mode; - unsigned char loopback; - unsigned short flags; - unsigned char encoding; - compat_ulong_t clock_speed; - unsigned char addr_filter; - unsigned short crc_type; - unsigned char preamble_length; - unsigned char preamble; - compat_ulong_t data_rate; - unsigned char data_bits; - unsigned char stop_bits; - unsigned char parity; -}; -#define MGSL_IOCSPARAMS32 _IOW(MGSL_MAGIC_IOC,0,struct MGSL_PARAMS32) -#define MGSL_IOCGPARAMS32 _IOR(MGSL_MAGIC_IOC,1,struct MGSL_PARAMS32) -#endif -#endif /* _SYNCLINK_H_ */ diff --git a/include/linux/sys_soc.h b/include/linux/sys_soc.h index d9b3cf0f410c..cedabf177f47 100644 --- a/include/linux/sys_soc.h +++ b/include/linux/sys_soc.h @@ -21,22 +21,39 @@ struct soc_device_attribute { /** * soc_device_register - register SoC as a device * @soc_plat_dev_attr: Attributes passed from platform to be attributed to a SoC + * + * Returns: + * - %NULL if the SoC bus is not yet registered; + * - on success, the newly allocated &struct soc_device pointer; + * - on failure, a negative error code as an ERR_PTR(). */ struct soc_device *soc_device_register( struct soc_device_attribute *soc_plat_dev_attr); /** * soc_device_unregister - unregister SoC device - * @dev: SoC device to be unregistered + * @soc_dev: SoC device to be unregistered */ void soc_device_unregister(struct soc_device *soc_dev); /** * soc_device_to_device - helper function to fetch struct device * @soc: Previously registered SoC device container + * + * Returns: &struct device pointer for this @soc */ struct device *soc_device_to_device(struct soc_device *soc); +/** + * soc_attr_read_machine - retrieve the machine model and store it in + * the soc_device_attribute structure + * @soc_dev_attr: SoC attribute structure to store the model in + * + * Returns: + * 0 on success, negative error number on failure. + */ +int soc_attr_read_machine(struct soc_device_attribute *soc_dev_attr); + #ifdef CONFIG_SOC_BUS const struct soc_device_attribute *soc_device_match( const struct soc_device_attribute *matches); diff --git a/include/linux/syscall_user_dispatch.h b/include/linux/syscall_user_dispatch.h index 3858a6ffdd5c..3dd30f4b2799 100644 --- a/include/linux/syscall_user_dispatch.h +++ b/include/linux/syscall_user_dispatch.h @@ -6,9 +6,23 @@ #define _SYSCALL_USER_DISPATCH_H #include <linux/thread_info.h> +#include <linux/sched.h> #include <linux/syscall_user_dispatch_types.h> -#ifdef CONFIG_GENERIC_ENTRY +struct pt_regs; + +#ifdef CONFIG_SYSCALL_USER_DISPATCH + +bool syscall_user_dispatch(struct pt_regs *regs); + +static __always_inline bool syscall_user_dispatch_clear_on_dispatch(void) +{ + if (likely(!current->syscall_dispatch.on_dispatch)) + return false; + + current->syscall_dispatch.on_dispatch = false; + return true; +} int set_syscall_user_dispatch(unsigned long mode, unsigned long offset, unsigned long len, char __user *selector); @@ -24,6 +38,16 @@ int syscall_user_dispatch_set_config(struct task_struct *task, unsigned long siz #else +static __always_inline bool syscall_user_dispatch(struct pt_regs *regs) +{ + return false; +} + +static __always_inline bool syscall_user_dispatch_clear_on_dispatch(void) +{ + return false; +} + static inline int set_syscall_user_dispatch(unsigned long mode, unsigned long offset, unsigned long len, char __user *selector) { @@ -46,6 +70,6 @@ static inline int syscall_user_dispatch_set_config(struct task_struct *task, return -EINVAL; } -#endif /* CONFIG_GENERIC_ENTRY */ +#endif /* CONFIG_SYSCALL_USER_DISPATCH */ #endif /* _SYSCALL_USER_DISPATCH_H */ diff --git a/include/linux/syscall_user_dispatch_types.h b/include/linux/syscall_user_dispatch_types.h index 3be36b06c7d7..c0bdd4f760d3 100644 --- a/include/linux/syscall_user_dispatch_types.h +++ b/include/linux/syscall_user_dispatch_types.h @@ -4,7 +4,7 @@ #include <linux/types.h> -#ifdef CONFIG_GENERIC_ENTRY +#ifdef CONFIG_SYSCALL_USER_DISPATCH struct syscall_user_dispatch { char __user *selector; diff --git a/include/linux/syscalls.h b/include/linux/syscalls.h index 02bd6ddb6278..8413b624ad47 100644 --- a/include/linux/syscalls.h +++ b/include/linux/syscalls.h @@ -247,6 +247,10 @@ static inline int is_syscall_trace_event(struct trace_event_call *tp_event) __diag_push(); \ __diag_ignore(GCC, 8, "-Wattribute-alias", \ "Type aliasing is used to sanitize syscall arguments");\ + __diag_ignore(clang, 23, "-Wunknown-warning-option", \ + "Avoid breaking versions without -Wattribute-alias");\ + __diag_ignore(clang, 23, "-Wattribute-alias", \ + "Type aliasing is used to sanitize syscall arguments");\ asmlinkage long sys##name(__MAP(x,__SC_DECL,__VA_ARGS__)) \ __attribute__((alias(__stringify(__se_sys##name)))); \ ALLOW_ERROR_INJECTION(sys##name, ERRNO); \ @@ -453,6 +457,7 @@ asmlinkage long sys_faccessat2(int dfd, const char __user *filename, int mode, asmlinkage long sys_chdir(const char __user *filename); asmlinkage long sys_fchdir(unsigned int fd); asmlinkage long sys_chroot(const char __user *filename); +asmlinkage long sys_fchroot(int fd, unsigned int flags); asmlinkage long sys_fchmod(unsigned int fd, umode_t mode); asmlinkage long sys_fchmodat(int dfd, const char __user *filename, umode_t mode); @@ -936,7 +941,8 @@ asmlinkage long sys_seccomp(unsigned int op, unsigned int flags, asmlinkage long sys_getrandom(char __user *buf, size_t count, unsigned int flags); asmlinkage long sys_memfd_create(const char __user *uname_ptr, unsigned int flags); -asmlinkage long sys_bpf(int cmd, union bpf_attr __user *attr, unsigned int size); +asmlinkage long sys_bpf(int cmd, union bpf_attr __user *attr, unsigned int size, + struct bpf_common_attr __user *attr_common, unsigned int size_common); asmlinkage long sys_execveat(int dfd, const char __user *filename, const char __user *const __user *argv, const char __user *const __user *envp, int flags); @@ -1283,19 +1289,9 @@ static inline long ksys_lchown(const char __user *filename, uid_t user, AT_SYMLINK_NOFOLLOW); } -int do_sys_ftruncate(unsigned int fd, loff_t length, int small); - -static inline long ksys_ftruncate(unsigned int fd, loff_t length) -{ - return do_sys_ftruncate(fd, length, 1); -} - -int do_sys_truncate(const char __user *pathname, loff_t length); - -static inline long ksys_truncate(const char __user *pathname, loff_t length) -{ - return do_sys_truncate(pathname, length); -} +#define FTRUNCATE_LFS (1u << 0) /* allow truncating > 32-bit */ +int ksys_ftruncate(unsigned int fd, loff_t length, unsigned int flags); +int ksys_truncate(const char __user *pathname, loff_t length); static inline unsigned int ksys_personality(unsigned int personality) { diff --git a/include/linux/sysctl.h b/include/linux/sysctl.h index 2886fbceb5d6..e5d7226ab6f5 100644 --- a/include/linux/sysctl.h +++ b/include/linux/sysctl.h @@ -59,7 +59,7 @@ extern const int sysctl_vals[]; #define SYSCTL_LONG_ONE ((void *)&sysctl_long_vals[1]) #define SYSCTL_LONG_MAX ((void *)&sysctl_long_vals[2]) -/** +/* * * "dir" originates from read_iter (dir = 0) or write_iter (dir = 1) * in the file_operations struct at proc/proc_sysctl.c. Its value means @@ -74,53 +74,108 @@ extern const int sysctl_vals[]; extern const unsigned long sysctl_long_vals[]; -typedef int proc_handler(const struct ctl_table *ctl, int write, void *buffer, - size_t *lenp, loff_t *ppos); - -int proc_dostring(const struct ctl_table *, int, void *, size_t *, loff_t *); -int proc_dobool(const struct ctl_table *table, int write, void *buffer, - size_t *lenp, loff_t *ppos); +typedef int proc_handler(const struct ctl_table *ctl, int dir, void *buf, + size_t *lenp, loff_t *ppos); -int proc_dointvec(const struct ctl_table *, int, void *, size_t *, loff_t *); -int proc_dointvec_minmax(const struct ctl_table *table, int dir, void *buffer, +/* proc_handler functions */ +int proc_dostring(const struct ctl_table *ctl, int dir, void *buf, size_t *lenp, + loff_t *ppos); +int proc_dobool(const struct ctl_table *ctl, int dir, void *buf, size_t *lenp, + loff_t *ppos); +int proc_dointvec(const struct ctl_table *ctl, int dir, void *buf, size_t *lenp, + loff_t *ppos); +int proc_dointvec_minmax(const struct ctl_table *ctl, int dir, void *buf, + size_t *lenp, loff_t *ppos); +int proc_douintvec(const struct ctl_table *ctl, int dir, void *buf, size_t *lenp, + loff_t *ppos); +int proc_douintvec_minmax(const struct ctl_table *ctl, int dir, void *buf, + size_t *lenp, loff_t *ppos); +int proc_dou8vec_minmax(const struct ctl_table *ctl, int dir, void *buf, + size_t *lenp, loff_t *ppos); +int proc_doulongvec_minmax(const struct ctl_table *ctl, int dir, void *buf, + size_t *lenp, loff_t *ppos); +int proc_do_large_bitmap(const struct ctl_table *ctl, int dir, void *buf, size_t *lenp, loff_t *ppos); -int proc_dointvec_conv(const struct ctl_table *table, int dir, void *buffer, +int proc_do_static_key(const struct ctl_table *ctl, int dir, void *buf, + size_t *lenp, loff_t *ppos); + +/* + * proc_handler aggregators + * + * Create a proc_handler with a custom converter. Use when the user space + * value is a transformation of the kernel value. Cannot be passed as + * proc_handlers. + * + * Example of creating your custom proc handler: + * int custom_converter(bool *negp, ulong *u_ptr, uint *k_ptr, + * int dir, const struct ctl_table *ctl) {...} + * int custom_proc_handler(const struct ctl_table *ctl, int dir, + * void *buf, size_t *lenp, loff_t *ppos + * { return proc_dointvec_conv(ctl, dir, buf, lenp, ppos, custom_converter); } + */ +int proc_dointvec_conv(const struct ctl_table *ctl, int dir, void *buf, size_t *lenp, loff_t *ppos, int (*conv)(bool *negp, unsigned long *u_ptr, int *k_ptr, - int dir, const struct ctl_table *table)); -int proc_int_k2u_conv_kop(ulong *u_ptr, const int *k_ptr, bool *negp, - ulong (*k_ptr_op)(const ulong)); -int proc_int_u2k_conv_uop(const ulong *u_ptr, int *k_ptr, const bool *negp, - ulong (*u_ptr_op)(const ulong)); + int dir, const struct ctl_table *ctl)); +int proc_douintvec_conv(const struct ctl_table *ctl, int dir, void *buf, + size_t *lenp, loff_t *ppos, + int (*conv)(bool *negp, ulong *u_ptr, uint *k_ptr, + int dir, const struct ctl_table *ctl)); +int proc_doulongvec_conv(const struct ctl_table *ctl, int dir, void *buf, + size_t *lenp, loff_t *ppos, + int (*conv)(bool *negp, ulong *u_ptr, ulong *k_ptr, + int dir, const struct ctl_table *ctl)); + +/* + * bi-directional converter functions + * + * Specify the converter function for both directions (user to kernel & kernel + * to user). Use when you want to change the value of the variable before + * assignment. Used to create custom proc_handler aggregators. + * + * Example of creating your custom bi-directional converter: + * int custom_u2k(ulong *u_ptr, const uint *k_ptr) { ... } + * int custom_converter(bool *negp, ulong *u_ptr, uint *k_ptr, + * int dir, const struct ctl_table *ctl) + * { return proc_uint_conv(u_ptr, k_ptr, dir, ctl, true, + * custom_u2k, proc_uint_k2u_conv} + */ int proc_int_conv(bool *negp, ulong *u_ptr, int *k_ptr, int dir, const struct ctl_table *tbl, bool k_ptr_range_check, int (*user_to_kern)(const bool *negp, const ulong *u_ptr, int *k_ptr), int (*kern_to_user)(bool *negp, ulong *u_ptr, const int *k_ptr)); - -int proc_douintvec(const struct ctl_table *, int, void *, size_t *, loff_t *); -int proc_douintvec_minmax(const struct ctl_table *table, int write, void *buffer, - size_t *lenp, loff_t *ppos); -int proc_douintvec_conv(const struct ctl_table *table, int write, void *buffer, - size_t *lenp, loff_t *ppos, - int (*conv)(unsigned long *lvalp, unsigned int *valp, - int write, const struct ctl_table *table)); -int proc_uint_k2u_conv(ulong *u_ptr, const uint *k_ptr); -int proc_uint_u2k_conv_uop(const ulong *u_ptr, uint *k_ptr, - ulong (*u_ptr_op)(const ulong)); int proc_uint_conv(ulong *u_ptr, uint *k_ptr, int dir, const struct ctl_table *tbl, bool k_ptr_range_check, int (*user_to_kern)(const ulong *u_ptr, uint *k_ptr), int (*kern_to_user)(ulong *u_ptr, const uint *k_ptr)); +int proc_ulong_conv(ulong *u_ptr, ulong *k_ptr, int dir, + const struct ctl_table *tbl, bool k_ptr_range_check, + int (*user_to_kern)(const ulong *u_ptr, ulong *k_ptr), + int (*kern_to_user)(ulong *u_ptr, const ulong *k_ptr)); -int proc_dou8vec_minmax(const struct ctl_table *table, int write, void *buffer, - size_t *lenp, loff_t *ppos); -int proc_doulongvec_minmax(const struct ctl_table *, int, void *, size_t *, loff_t *); -int proc_doulongvec_minmax_conv(const struct ctl_table *table, int dir, - void *buffer, size_t *lenp, loff_t *ppos, - unsigned long convmul, unsigned long convdiv); -int proc_do_large_bitmap(const struct ctl_table *, int, void *, size_t *, loff_t *); -int proc_do_static_key(const struct ctl_table *table, int write, void *buffer, - size_t *lenp, loff_t *ppos); +/* + * uni-directional converter functions + * + * Specify the converter function for one directions (user to kernel or + * kernel to user). Use to call the actual value conversion. Used to Create + * bi-directional converters. + * + * Example of creating a uni-directional converter: + * ulong op(const ulong val) { ... } + * int custom_unidir_conv(ulong *u_ptr, const uint *k_ptr) + * { return proc_uint_k2u_conv_kop(u_ptr, k_ptr, op); } + */ +int proc_int_k2u_conv_kop(ulong *u_ptr, const int *k_ptr, bool *negp, + ulong (*k_ptr_op)(const ulong)); +int proc_int_u2k_conv_uop(const ulong *u_ptr, int *k_ptr, const bool *negp, + ulong (*u_ptr_op)(const ulong)); +int proc_uint_k2u_conv(ulong *u_ptr, const uint *k_ptr); +int proc_uint_u2k_conv_uop(const ulong *u_ptr, uint *k_ptr, + ulong (*u_ptr_op)(const ulong)); +int proc_ulong_u2k_conv_uop(const ulong *u_ptr, ulong *k_ptr, + ulong (*u_ptr_op)(const ulong)); +int proc_ulong_k2u_conv_kop(ulong *u_ptr, const ulong *k_ptr, + ulong (*k_ptr_op)(const ulong)); /* * Register a set of sysctl names by calling register_sysctl @@ -190,9 +245,9 @@ struct ctl_node { * @nreg: When nreg drops to 0 the ctl_table_header will be unregistered. * @rcu: Delays the freeing of the inode. Introduced with "unfuck proc_sysctl ->d_compare()" * - * @type: Enumeration to differentiate between ctl target types - * @type.SYSCTL_TABLE_TYPE_DEFAULT: ctl target with no special considerations - * @type.SYSCTL_TABLE_TYPE_PERMANENTLY_EMPTY: Identifies a permanently empty dir + * @type: Enumeration to differentiate between ctl target types: + * type.SYSCTL_TABLE_TYPE_DEFAULT: ctl target with no special considerations + * type.SYSCTL_TABLE_TYPE_PERMANENTLY_EMPTY: Identifies a permanently empty dir * target to serve as a mount point */ struct ctl_table_header { diff --git a/include/linux/sysfb.h b/include/linux/sysfb.h index 5226efde9ad4..ed23d6516223 100644 --- a/include/linux/sysfb.h +++ b/include/linux/sysfb.h @@ -118,7 +118,7 @@ struct platform_device *sysfb_create_simplefb(const struct screen_info *si, const struct simplefb_platform_data *mode, struct device *parent); -#else /* CONFIG_SYSFB_SIMPLE */ +#else /* CONFIG_SYSFB_SIMPLEFB */ static inline bool sysfb_parse_mode(const struct screen_info *si, struct simplefb_platform_data *mode) @@ -133,6 +133,6 @@ static inline struct platform_device *sysfb_create_simplefb(const struct screen_ return ERR_PTR(-EINVAL); } -#endif /* CONFIG_SYSFB_SIMPLE */ +#endif /* CONFIG_SYSFB_SIMPLEFB */ #endif /* _LINUX_SYSFB_H */ diff --git a/include/linux/sysfs.h b/include/linux/sysfs.h index 99b775f3ff46..b1a3a1e6ad09 100644 --- a/include/linux/sysfs.h +++ b/include/linux/sysfs.h @@ -396,13 +396,13 @@ struct sysfs_ops { #ifdef CONFIG_SYSFS -int __must_check sysfs_create_dir_ns(struct kobject *kobj, const void *ns); +int __must_check sysfs_create_dir_ns(struct kobject *kobj, const struct ns_common *ns); void sysfs_remove_dir(struct kobject *kobj); int __must_check sysfs_rename_dir_ns(struct kobject *kobj, const char *new_name, - const void *new_ns); + const struct ns_common *new_ns); int __must_check sysfs_move_dir_ns(struct kobject *kobj, struct kobject *new_parent_kobj, - const void *new_ns); + const struct ns_common *new_ns); int __must_check sysfs_create_mount_point(struct kobject *parent_kobj, const char *name); void sysfs_remove_mount_point(struct kobject *parent_kobj, @@ -410,7 +410,7 @@ void sysfs_remove_mount_point(struct kobject *parent_kobj, int __must_check sysfs_create_file_ns(struct kobject *kobj, const struct attribute *attr, - const void *ns); + const struct ns_common *ns); int __must_check sysfs_create_files(struct kobject *kobj, const struct attribute * const *attr); int __must_check sysfs_chmod_file(struct kobject *kobj, @@ -419,7 +419,7 @@ struct kernfs_node *sysfs_break_active_protection(struct kobject *kobj, const struct attribute *attr); void sysfs_unbreak_active_protection(struct kernfs_node *kn); void sysfs_remove_file_ns(struct kobject *kobj, const struct attribute *attr, - const void *ns); + const struct ns_common *ns); bool sysfs_remove_file_self(struct kobject *kobj, const struct attribute *attr); void sysfs_remove_files(struct kobject *kobj, const struct attribute * const *attr); @@ -437,7 +437,7 @@ void sysfs_remove_link(struct kobject *kobj, const char *name); int sysfs_rename_link_ns(struct kobject *kobj, struct kobject *target, const char *old_name, const char *new_name, - const void *new_ns); + const struct ns_common *new_ns); void sysfs_delete_link(struct kobject *dir, struct kobject *targ, const char *name); @@ -445,15 +445,15 @@ void sysfs_delete_link(struct kobject *dir, struct kobject *targ, int __must_check sysfs_create_group(struct kobject *kobj, const struct attribute_group *grp); int __must_check sysfs_create_groups(struct kobject *kobj, - const struct attribute_group **groups); + const struct attribute_group *const *groups); int __must_check sysfs_update_groups(struct kobject *kobj, - const struct attribute_group **groups); + const struct attribute_group *const *groups); int sysfs_update_group(struct kobject *kobj, const struct attribute_group *grp); void sysfs_remove_group(struct kobject *kobj, const struct attribute_group *grp); void sysfs_remove_groups(struct kobject *kobj, - const struct attribute_group **groups); + const struct attribute_group *const *groups); int sysfs_add_file_to_group(struct kobject *kobj, const struct attribute *attr, const char *group); void sysfs_remove_file_from_group(struct kobject *kobj, @@ -486,7 +486,7 @@ int sysfs_change_owner(struct kobject *kobj, kuid_t kuid, kgid_t kgid); int sysfs_link_change_owner(struct kobject *kobj, struct kobject *targ, const char *name, kuid_t kuid, kgid_t kgid); int sysfs_groups_change_owner(struct kobject *kobj, - const struct attribute_group **groups, + const struct attribute_group *const *groups, kuid_t kuid, kgid_t kgid); int sysfs_group_change_owner(struct kobject *kobj, const struct attribute_group *groups, kuid_t kuid, @@ -502,7 +502,7 @@ ssize_t sysfs_bin_attr_simple_read(struct file *file, struct kobject *kobj, #else /* CONFIG_SYSFS */ -static inline int sysfs_create_dir_ns(struct kobject *kobj, const void *ns) +static inline int sysfs_create_dir_ns(struct kobject *kobj, const struct ns_common *ns) { return 0; } @@ -512,14 +512,14 @@ static inline void sysfs_remove_dir(struct kobject *kobj) } static inline int sysfs_rename_dir_ns(struct kobject *kobj, - const char *new_name, const void *new_ns) + const char *new_name, const struct ns_common *new_ns) { return 0; } static inline int sysfs_move_dir_ns(struct kobject *kobj, struct kobject *new_parent_kobj, - const void *new_ns) + const struct ns_common *new_ns) { return 0; } @@ -537,7 +537,7 @@ static inline void sysfs_remove_mount_point(struct kobject *parent_kobj, static inline int sysfs_create_file_ns(struct kobject *kobj, const struct attribute *attr, - const void *ns) + const struct ns_common *ns) { return 0; } @@ -567,7 +567,7 @@ static inline void sysfs_unbreak_active_protection(struct kernfs_node *kn) static inline void sysfs_remove_file_ns(struct kobject *kobj, const struct attribute *attr, - const void *ns) + const struct ns_common *ns) { } @@ -612,7 +612,7 @@ static inline void sysfs_remove_link(struct kobject *kobj, const char *name) static inline int sysfs_rename_link_ns(struct kobject *k, struct kobject *t, const char *old_name, - const char *new_name, const void *ns) + const char *new_name, const struct ns_common *ns) { return 0; } @@ -629,13 +629,13 @@ static inline int sysfs_create_group(struct kobject *kobj, } static inline int sysfs_create_groups(struct kobject *kobj, - const struct attribute_group **groups) + const struct attribute_group *const *groups) { return 0; } static inline int sysfs_update_groups(struct kobject *kobj, - const struct attribute_group **groups) + const struct attribute_group *const *groups) { return 0; } @@ -652,7 +652,7 @@ static inline void sysfs_remove_group(struct kobject *kobj, } static inline void sysfs_remove_groups(struct kobject *kobj, - const struct attribute_group **groups) + const struct attribute_group *const *groups) { } @@ -733,7 +733,7 @@ static inline int sysfs_change_owner(struct kobject *kobj, kuid_t kuid, kgid_t k } static inline int sysfs_groups_change_owner(struct kobject *kobj, - const struct attribute_group **groups, + const struct attribute_group *const *groups, kuid_t kuid, kgid_t kgid) { return 0; diff --git a/include/linux/t10-pi.h b/include/linux/t10-pi.h index 2c59fe3efcd4..b6c2496866ea 100644 --- a/include/linux/t10-pi.h +++ b/include/linux/t10-pi.h @@ -4,6 +4,7 @@ #include <linux/types.h> #include <linux/blk-mq.h> +#include <linux/wordpart.h> /* * A T10 PI-capable target device can be formatted with different @@ -25,6 +26,16 @@ enum t10_dif_type { T10_PI_TYPE3_PROTECTION = 0x3, }; +static inline u64 full_pi_ref_tag(const struct request *rq) +{ + unsigned int shift = ilog2(queue_logical_block_size(rq->q)); + + if (IS_ENABLED(CONFIG_BLK_DEV_INTEGRITY) && + rq->q->limits.integrity.interval_exp) + shift = rq->q->limits.integrity.interval_exp; + return blk_rq_pos(rq) >> (shift - SECTOR_SHIFT); +} + /* * T10 Protection Information tuple. */ @@ -39,12 +50,7 @@ struct t10_pi_tuple { static inline u32 t10_pi_ref_tag(struct request *rq) { - unsigned int shift = ilog2(queue_logical_block_size(rq->q)); - - if (IS_ENABLED(CONFIG_BLK_DEV_INTEGRITY) && - rq->q->limits.integrity.interval_exp) - shift = rq->q->limits.integrity.interval_exp; - return blk_rq_pos(rq) >> (shift - SECTOR_SHIFT) & 0xffffffff; + return lower_32_bits(full_pi_ref_tag(rq)); } struct crc64_pi_tuple { @@ -64,12 +70,7 @@ static inline u64 lower_48_bits(u64 n) static inline u64 ext_pi_ref_tag(struct request *rq) { - unsigned int shift = ilog2(queue_logical_block_size(rq->q)); - - if (IS_ENABLED(CONFIG_BLK_DEV_INTEGRITY) && - rq->q->limits.integrity.interval_exp) - shift = rq->q->limits.integrity.interval_exp; - return lower_48_bits(blk_rq_pos(rq) >> (shift - SECTOR_SHIFT)); + return lower_48_bits(full_pi_ref_tag(rq)); } #endif diff --git a/include/linux/taskstats_kern.h b/include/linux/taskstats_kern.h index dbb4d124c7d7..995cd19b56c1 100644 --- a/include/linux/taskstats_kern.h +++ b/include/linux/taskstats_kern.h @@ -14,7 +14,6 @@ #ifdef CONFIG_TASKSTATS extern struct kmem_cache *taskstats_cache; -extern struct mutex taskstats_exit_mutex; static inline void taskstats_tgid_free(struct signal_struct *sig) { diff --git a/include/linux/tcp.h b/include/linux/tcp.h index f72eef31fa23..6a8c77719322 100644 --- a/include/linux/tcp.h +++ b/include/linux/tcp.h @@ -228,8 +228,7 @@ struct tcp_sock { u32 sacked_out; /* SACK'd packets */ u16 tcp_header_len; /* Bytes of tcp header to send */ u8 scaling_ratio; /* see tcp_win_from_space() */ - u8 chrono_type : 2, /* current chronograph type */ - repair : 1, + u8 repair : 1, tcp_usec_ts : 1, /* TSval values in usec */ is_sack_reneg:1, /* in recovery from loss with SACK reneg? */ is_cwnd_limited:1,/* forward progress limited by snd_cwnd? */ @@ -254,23 +253,33 @@ struct tcp_sock { /* TX read-write hotpath cache lines */ __cacheline_group_begin(tcp_sock_write_tx) ____cacheline_aligned; - u32 segs_out; /* RFC4898 tcpEStatsPerfSegsOut - * The total number of segments sent. - */ - u32 data_segs_out; /* RFC4898 tcpEStatsPerfDataSegsOut - * total number of data segments sent. + u32 delivered; /* Total data packets delivered incl. rexmits */ + u32 delivered_ce; /* Like the above but only ECE marked packets */ + u64 bytes_acked; /* RFC4898 tcpEStatsAppHCThruOctetsAcked + * sum(delta(snd_una)), or how many bytes + * were acked. */ u64 bytes_sent; /* RFC4898 tcpEStatsPerfHCDataOctetsOut * total number of data bytes sent. */ + u64 first_tx_mstamp; /* start of window send phase */ + u64 delivered_mstamp; /* time we reached "delivered" */ + u32 data_segs_out; /* RFC4898 tcpEStatsPerfDataSegsOut + * total number of data segments sent. + */ u32 snd_sml; /* Last byte of the most recently transmitted small packet */ + u8 chrono_type; /* current chronograph type */ u32 chrono_start; /* Start time in jiffies of a TCP chrono */ u32 chrono_stat[3]; /* Time in jiffies for chrono_stat stats */ u32 write_seq; /* Tail(+1) of data held in tcp send buffer */ u32 pushed_seq; /* Last pushed seq, required to talk to windows */ u32 lsndtime; /* timestamp of last sent data packet (for restart window) */ - u32 mdev_us; /* medium deviation */ + u32 mdev_us; /* mean deviation of RTT, scaled by 4 (<< 2) in usecs */ u32 rtt_seq; /* sequence number to update rttvar */ + u32 max_packets_out; /* max packets_out in last window */ + u32 cwnd_usage_seq; /* right edge of cwnd usage tracking flight */ + u32 rate_delivered; /* saved rate sample: packets delivered */ + u32 rate_interval_us; /* saved rate sample: time elapsed */ u64 tcp_wstamp_ns; /* departure time for next sent data packet */ u64 accecn_opt_tstamp; /* Last AccECN option sent timestamp */ struct list_head tsorted_sent_queue; /* time-sorted sent but un-SACKed skbs */ @@ -307,8 +316,6 @@ struct tcp_sock { u32 srtt_us; /* smoothed round trip time << 3 in usecs */ u32 packets_out; /* Packets which are "in flight" */ u32 snd_up; /* Urgent pointer */ - u32 delivered; /* Total data packets delivered incl. rexmits */ - u32 delivered_ce; /* Like the above but only ECE marked packets */ u32 received_ce; /* Like the above but for rcvd CE marked pkts */ u32 received_ecn_bytes[3]; /* received byte counters for three ECN * types: INET_ECN_ECT_1, INET_ECN_ECT_0, @@ -316,11 +323,20 @@ struct tcp_sock { */ u32 app_limited; /* limited until "delivered" reaches this val */ u32 rcv_wnd; /* Current receiver window */ + u32 rcv_mwnd_seq; /* Maximum window sequence number (RFC 7323, + * section 2.4, receiver requirements) + */ u32 rcv_tstamp; /* timestamp of last received ACK (for keepalives) */ /* * Options received (usually on last packet, some only on SYN packets). */ struct tcp_options_received rx_opt; + u32 segs_in; /* RFC4898 tcpEStatsPerfSegsIn + * total number of segments in. + */ + u32 segs_out; /* RFC4898 tcpEStatsPerfSegsOut + * The total number of segments sent. + */ __cacheline_group_end(tcp_sock_write_txrx); /* RX read-write hotpath cache lines */ @@ -330,26 +346,13 @@ struct tcp_sock { * sum(delta(rcv_nxt)), or how many bytes * were acked. */ - u32 segs_in; /* RFC4898 tcpEStatsPerfSegsIn - * total number of segments in. - */ u32 data_segs_in; /* RFC4898 tcpEStatsPerfDataSegsIn * total number of data segments in. */ u32 rcv_wup; /* rcv_nxt on last window update sent */ - u32 max_packets_out; /* max packets_out in last window */ - u32 cwnd_usage_seq; /* right edge of cwnd usage tracking flight */ - u32 rate_delivered; /* saved rate sample: packets delivered */ - u32 rate_interval_us; /* saved rate sample: time elapsed */ u32 rcv_rtt_last_tsecr; u32 delivered_ecn_bytes[3]; u16 pkts_acked_ewma;/* Pkts acked EWMA for AccECN cep heuristic */ - u64 first_tx_mstamp; /* start of window send phase */ - u64 delivered_mstamp; /* time we reached "delivered" */ - u64 bytes_acked; /* RFC4898 tcpEStatsAppHCThruOctetsAcked - * sum(delta(snd_una)), or how many bytes - * were acked. - */ struct { u32 rtt_us; u32 seq; @@ -548,6 +551,13 @@ enum tsq_flags { TCPF_ACK_DEFERRED = BIT(TCP_ACK_DEFERRED), }; +/* Flags of interest for tcp_release_cb() */ +#define TCP_DEFERRED_ALL (TCPF_TSQ_DEFERRED | \ + TCPF_WRITE_TIMER_DEFERRED | \ + TCPF_DELACK_TIMER_DEFERRED | \ + TCPF_MTU_REDUCED_DEFERRED | \ + TCPF_ACK_DEFERRED) + #define tcp_sk(ptr) container_of_const(ptr, struct tcp_sock, inet_conn.icsk_inet.sk) /* Variant of tcp_sk() upgrading a const sock to a read/write tcp socket. diff --git a/include/linux/tee_core.h b/include/linux/tee_core.h index ee5f0bd41f43..f993d5118edd 100644 --- a/include/linux/tee_core.h +++ b/include/linux/tee_core.h @@ -50,7 +50,7 @@ enum tee_dma_heap_id { * @dev: embedded basic device structure * @cdev: embedded cdev * @num_users: number of active users of this device - * @c_no_user: completion used when unregistering the device + * @c_no_users: completion used when unregistering the device * @mutex: mutex protecting @num_users and @idr * @idr: register of user space shared memory objects allocated or * registered on this device @@ -132,6 +132,7 @@ struct tee_driver_ops { /* Size for TEE revision string buffer used by get_tee_revision(). */ #define TEE_REVISION_STR_SIZE 128 +#define TEE_DESC_PRIVILEGED 0x1 /** * struct tee_desc - Describes the TEE driver to the subsystem * @name: name of driver @@ -139,7 +140,6 @@ struct tee_driver_ops { * @owner: module providing the driver * @flags: Extra properties of driver, defined by TEE_DESC_* below */ -#define TEE_DESC_PRIVILEGED 0x1 struct tee_desc { const char *name; const struct tee_driver_ops *ops; @@ -187,7 +187,7 @@ struct tee_protmem_pool_ops { * Allocates a new struct tee_device instance. The device is * removed by tee_device_unregister(). * - * @returns a pointer to a 'struct tee_device' or an ERR_PTR on failure + * @returns: a pointer to a 'struct tee_device' or an ERR_PTR on failure */ struct tee_device *tee_device_alloc(const struct tee_desc *teedesc, struct device *dev, @@ -201,7 +201,7 @@ struct tee_device *tee_device_alloc(const struct tee_desc *teedesc, * tee_device_unregister() need to be called to remove the @teedev if * this function fails. * - * @returns < 0 on failure + * @returns: < 0 on failure */ int tee_device_register(struct tee_device *teedev); @@ -254,14 +254,14 @@ void tee_device_set_dev_groups(struct tee_device *teedev, * tee_session_calc_client_uuid() - Calculates client UUID for session * @uuid: Resulting UUID * @connection_method: Connection method for session (TEE_IOCTL_LOGIN_*) - * @connectuon_data: Connection data for opening session + * @connection_data: Connection data for opening session * * Based on connection method calculates UUIDv5 based client UUID. * * For group based logins verifies that calling process has specified * credentials. * - * @return < 0 on failure + * @returns: < 0 on failure */ int tee_session_calc_client_uuid(uuid_t *uuid, u32 connection_method, const u8 connection_data[TEE_IOCTL_UUID_LEN]); @@ -295,7 +295,7 @@ struct tee_shm_pool_ops { * @paddr: Physical address of start of pool * @size: Size in bytes of the pool * - * @returns pointer to a 'struct tee_shm_pool' or an ERR_PTR on failure. + * @returns: pointer to a 'struct tee_shm_pool' or an ERR_PTR on failure. */ struct tee_shm_pool *tee_shm_pool_alloc_res_mem(unsigned long vaddr, phys_addr_t paddr, size_t size, @@ -318,14 +318,16 @@ static inline void tee_shm_pool_free(struct tee_shm_pool *pool) * @paddr: Physical address of start of pool * @size: Size in bytes of the pool * - * @returns pointer to a 'struct tee_protmem_pool' or an ERR_PTR on failure. + * @returns: pointer to a 'struct tee_protmem_pool' or an ERR_PTR on failure. */ struct tee_protmem_pool *tee_protmem_static_pool_alloc(phys_addr_t paddr, size_t size); /** * tee_get_drvdata() - Return driver_data pointer - * @returns the driver_data pointer supplied to tee_register(). + * @teedev: Pointer to the tee_device + * + * @returns: the driver_data pointer supplied to tee_register(). */ void *tee_get_drvdata(struct tee_device *teedev); @@ -334,7 +336,7 @@ void *tee_get_drvdata(struct tee_device *teedev); * TEE driver * @ctx: The TEE context for shared memory allocation * @size: Shared memory allocation size - * @returns a pointer to 'struct tee_shm' on success or an ERR_PTR on failure + * @returns: a pointer to 'struct tee_shm' on success or an ERR_PTR on failure */ struct tee_shm *tee_shm_alloc_priv_buf(struct tee_context *ctx, size_t size); @@ -354,7 +356,7 @@ void tee_dyn_shm_free_helper(struct tee_shm *shm, /** * tee_shm_is_dynamic() - Check if shared memory object is of the dynamic kind * @shm: Shared memory handle - * @returns true if object is dynamic shared memory + * @returns: true if object is dynamic shared memory */ static inline bool tee_shm_is_dynamic(struct tee_shm *shm) { @@ -370,7 +372,7 @@ void tee_shm_put(struct tee_shm *shm); /** * tee_shm_get_id() - Get id of a shared memory object * @shm: Shared memory handle - * @returns id + * @returns: id */ static inline int tee_shm_get_id(struct tee_shm *shm) { @@ -382,7 +384,7 @@ static inline int tee_shm_get_id(struct tee_shm *shm) * count * @ctx: Context owning the shared memory * @id: Id of shared memory object - * @returns a pointer to 'struct tee_shm' on success or an ERR_PTR on failure + * @returns: a pointer to 'struct tee_shm' on success or an ERR_PTR on failure */ struct tee_shm *tee_shm_get_from_id(struct tee_context *ctx, int id); @@ -402,7 +404,7 @@ static inline bool tee_param_is_memref(struct tee_param *param) * teedev_open() - Open a struct tee_device * @teedev: Device to open * - * @return a pointer to struct tee_context on success or an ERR_PTR on failure. + * @returns: pointer to struct tee_context on success or an ERR_PTR on failure. */ struct tee_context *teedev_open(struct tee_device *teedev); diff --git a/include/linux/tee_drv.h b/include/linux/tee_drv.h index e561a26f537a..f3c5e106d853 100644 --- a/include/linux/tee_drv.h +++ b/include/linux/tee_drv.h @@ -9,7 +9,7 @@ #include <linux/device.h> #include <linux/kref.h> #include <linux/list.h> -#include <linux/mod_devicetable.h> +#include <linux/device-id/tee_client.h> #include <linux/tee.h> #include <linux/types.h> diff --git a/include/linux/tegra-mipi-cal.h b/include/linux/tegra-mipi-cal.h new file mode 100644 index 000000000000..2a540b50f65d --- /dev/null +++ b/include/linux/tegra-mipi-cal.h @@ -0,0 +1,57 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef __TEGRA_MIPI_CAL_H_ +#define __TEGRA_MIPI_CAL_H_ + +struct tegra_mipi_device { + const struct tegra_mipi_ops *ops; + struct platform_device *pdev; + unsigned long pads; +}; + +/** + * Operations for Tegra MIPI calibration device + */ +struct tegra_mipi_ops { + /** + * @enable: + * + * Enable MIPI calibration device + */ + int (*enable)(struct tegra_mipi_device *device); + + /** + * @disable: + * + * Disable MIPI calibration device + */ + int (*disable)(struct tegra_mipi_device *device); + + /** + * @start_calibration: + * + * Start MIPI calibration + */ + int (*start_calibration)(struct tegra_mipi_device *device); + + /** + * @finish_calibration: + * + * Finish MIPI calibration + */ + int (*finish_calibration)(struct tegra_mipi_device *device); +}; + +int devm_tegra_mipi_add_provider(struct device *device, struct device_node *np, + const struct tegra_mipi_ops *ops); + +struct tegra_mipi_device *tegra_mipi_request(struct device *device, + struct device_node *np); +void tegra_mipi_free(struct tegra_mipi_device *device); + +int tegra_mipi_enable(struct tegra_mipi_device *device); +int tegra_mipi_disable(struct tegra_mipi_device *device); +int tegra_mipi_start_calibration(struct tegra_mipi_device *device); +int tegra_mipi_finish_calibration(struct tegra_mipi_device *device); + +#endif /* __TEGRA_MIPI_CAL_H_ */ diff --git a/include/linux/thermal.h b/include/linux/thermal.h index 0b5ed6821080..083b4f533933 100644 --- a/include/linux/thermal.h +++ b/include/linux/thermal.h @@ -125,7 +125,6 @@ struct thermal_cooling_device { const char *type; unsigned long max_state; struct device device; - struct device_node *np; void *devdata; void *stats; const struct thermal_cooling_device_ops *ops; @@ -133,6 +132,10 @@ struct thermal_cooling_device { struct mutex lock; /* protect thermal_instances list */ struct list_head thermal_instances; struct list_head node; +#ifdef CONFIG_THERMAL_OF + struct device_node *np; + u32 cdev_id; +#endif #ifdef CONFIG_THERMAL_DEBUGFS struct thermal_debugfs *debugfs; #endif @@ -198,6 +201,21 @@ struct thermal_zone_device *devm_thermal_of_zone_register(struct device *dev, in void devm_thermal_of_zone_unregister(struct device *dev, struct thermal_zone_device *tz); +struct thermal_cooling_device * +thermal_of_cooling_device_register(struct device_node *np, u32 cdev_id, + const char *type, void *data, + const struct thermal_cooling_device_ops *ops); + +struct thermal_cooling_device * +devm_thermal_of_cooling_device_register(struct device *dev, u32 cdev_id, + const char *type, void *devdata, + const struct thermal_cooling_device_ops *ops); + +struct thermal_cooling_device * +devm_thermal_of_child_cooling_device_register(struct device *dev, + struct device_node *np, + const char *type, void *devdata, + const struct thermal_cooling_device_ops *ops); #else static inline @@ -211,6 +229,31 @@ static inline void devm_thermal_of_zone_unregister(struct device *dev, struct thermal_zone_device *tz) { } + +static inline struct thermal_cooling_device * +thermal_of_cooling_device_register(struct device_node *np, u32 cdev_id, + const char *type, void *devdata, + const struct thermal_cooling_device_ops *ops) +{ + return ERR_PTR(-ENODEV); +} + +static inline struct thermal_cooling_device * +devm_thermal_of_cooling_device_register(struct device *dev, u32 cdev_id, + const char *type, void *devdata, + const struct thermal_cooling_device_ops *ops) +{ + return ERR_PTR(-ENODEV); +} + +static inline struct thermal_cooling_device * +devm_thermal_of_child_cooling_device_register(struct device *dev, + struct device_node *np, + const char *type, void *devdata, + const struct thermal_cooling_device_ops *ops) +{ + return ERR_PTR(-ENODEV); +} #endif int for_each_thermal_trip(struct thermal_zone_device *tz, @@ -252,14 +295,11 @@ void thermal_zone_device_update(struct thermal_zone_device *, struct thermal_cooling_device *thermal_cooling_device_register(const char *, void *, const struct thermal_cooling_device_ops *); + struct thermal_cooling_device * -thermal_of_cooling_device_register(struct device_node *np, const char *, void *, - const struct thermal_cooling_device_ops *); -struct thermal_cooling_device * -devm_thermal_of_cooling_device_register(struct device *dev, - struct device_node *np, - const char *type, void *devdata, - const struct thermal_cooling_device_ops *ops); +devm_thermal_cooling_device_register(struct device *dev, const char *type, void *devdata, + const struct thermal_cooling_device_ops *ops); + void thermal_cooling_device_update(struct thermal_cooling_device *); void thermal_cooling_device_unregister(struct thermal_cooling_device *); struct thermal_zone_device *thermal_zone_get_zone_by_name(const char *name); @@ -273,6 +313,9 @@ bool thermal_trip_is_bound_to_cdev(struct thermal_zone_device *tz, int thermal_zone_device_enable(struct thermal_zone_device *tz); int thermal_zone_device_disable(struct thermal_zone_device *tz); void thermal_zone_device_critical(struct thermal_zone_device *tz); + +void thermal_pm_prepare(void); +void thermal_pm_complete(void); #else static inline struct thermal_zone_device *thermal_zone_device_register_with_trips( const char *type, @@ -301,19 +344,12 @@ static inline struct thermal_cooling_device * thermal_cooling_device_register(const char *type, void *devdata, const struct thermal_cooling_device_ops *ops) { return ERR_PTR(-ENODEV); } + static inline struct thermal_cooling_device * -thermal_of_cooling_device_register(struct device_node *np, - const char *type, void *devdata, - const struct thermal_cooling_device_ops *ops) +devm_thermal_cooling_device_register(struct device *dev, const char *type, void *devdata, + const struct thermal_cooling_device_ops *ops) { return ERR_PTR(-ENODEV); } -static inline struct thermal_cooling_device * -devm_thermal_of_cooling_device_register(struct device *dev, - struct device_node *np, - const char *type, void *devdata, - const struct thermal_cooling_device_ops *ops) -{ - return ERR_PTR(-ENODEV); -} + static inline void thermal_cooling_device_unregister( struct thermal_cooling_device *cdev) { } @@ -350,6 +386,9 @@ static inline int thermal_zone_device_enable(struct thermal_zone_device *tz) static inline int thermal_zone_device_disable(struct thermal_zone_device *tz) { return -ENODEV; } + +static inline void thermal_pm_prepare(void) {} +static inline void thermal_pm_complete(void) {} #endif /* CONFIG_THERMAL */ #endif /* __THERMAL_H__ */ diff --git a/include/linux/thread_info.h b/include/linux/thread_info.h index 051e42902690..307b8390fc67 100644 --- a/include/linux/thread_info.h +++ b/include/linux/thread_info.h @@ -92,7 +92,7 @@ static inline long set_restart_fn(struct restart_block *restart, #define THREAD_ALIGN THREAD_SIZE #endif -#define THREADINFO_GFP (GFP_KERNEL_ACCOUNT | __GFP_ZERO) +#define THREADINFO_GFP (GFP_KERNEL_ACCOUNT | __GFP_ZERO | __GFP_SKIP_KASAN) /* * flag set/clear/test wrappers diff --git a/include/linux/thunderbolt.h b/include/linux/thunderbolt.h index 0ba112175bb3..d48623fda79b 100644 --- a/include/linux/thunderbolt.h +++ b/include/linux/thunderbolt.h @@ -13,6 +13,7 @@ #include <linux/types.h> +struct config_group; struct fwnode_handle; struct device; @@ -22,7 +23,7 @@ struct device; #include <linux/idr.h> #include <linux/list.h> #include <linux/mutex.h> -#include <linux/mod_devicetable.h> +#include <linux/device-id/tb.h> #include <linux/pci.h> #include <linux/uuid.h> #include <linux/workqueue.h> @@ -153,6 +154,9 @@ struct tb_property_dir *tb_property_parse_dir(const u32 *block, ssize_t tb_property_format_dir(const struct tb_property_dir *dir, u32 *block, size_t block_len); struct tb_property_dir *tb_property_copy_dir(const struct tb_property_dir *dir); +int tb_property_merge_dir(struct tb_property_dir *parent, + const struct tb_property_dir *dir, + bool replace); struct tb_property_dir *tb_property_create_dir(const uuid_t *uuid); void tb_property_free_dir(struct tb_property_dir *dir); int tb_property_add_immediate(struct tb_property_dir *parent, const char *key, @@ -209,6 +213,8 @@ enum tb_link_width { * @link_width: Width of the downstream facing link * @link_usb4: Downstream link is USB4 * @is_unplugged: The XDomain is unplugged + * @removing: Set by tb_xdomain_remove() under @lock to prevent + * concurrent delayed work queueing * @needs_uuid: If the XDomain does not have @remote_uuid it will be * queried first * @service_ids: Used to generate IDs for the services @@ -228,6 +234,7 @@ enum tb_link_width { * changed notification * @bonding_possible: True if lane bonding is possible on local side * @target_link_width: Target link width from the remote host + * @ntunnels: Keeps track of how many tunnels go through this XDomain * @link: Root switch link the remote domain is connected (ICM only) * @depth: Depth in the chain the remote domain is connected (ICM only) * @@ -257,6 +264,7 @@ struct tb_xdomain { enum tb_link_width link_width; bool link_usb4; bool is_unplugged; + bool removing; bool needs_uuid; struct ida service_ids; struct ida in_hopids; @@ -273,6 +281,7 @@ struct tb_xdomain { int properties_changed_retries; bool bonding_possible; u8 target_link_width; + atomic_t ntunnels; u8 link; u8 depth; }; @@ -392,6 +401,10 @@ void tb_unregister_protocol_handler(struct tb_protocol_handler *handler); * @prtcvers: Protocol version from the properties directory * @prtcrevs: Protocol software revision from the properties directory * @prtcstns: Protocol settings mask from the properties directory + * @lock: Protects this structure + * @local_properties: Properties owned by the service driver + * @remote_properties: Properties read from the remote service. These + * are read-only. * @debugfs_dir: Pointer to the service debugfs directory. Always created * when debugfs is enabled. Can be used by service drivers to * add their own entries under the service. @@ -399,6 +412,9 @@ void tb_unregister_protocol_handler(struct tb_protocol_handler *handler); * Each domain exposes set of services it supports as collection of * properties. For each service there will be one corresponding * &struct tb_service. Service drivers are bound to these. + * + * Service drivers can add their own dynamic properties to + * @local_properties but whenever they do so @lock must be held. */ struct tb_service { struct device dev; @@ -408,6 +424,9 @@ struct tb_service { u32 prtcvers; u32 prtcrevs; u32 prtcstns; + struct mutex lock; + struct tb_property_dir *local_properties; + struct tb_property_dir *remote_properties; struct dentry *debugfs_dir; }; @@ -446,7 +465,7 @@ static inline struct tb_service *tb_to_service(struct device *dev) */ struct tb_service_driver { struct device_driver driver; - int (*probe)(struct tb_service *svc, const struct tb_service_id *id); + int (*probe)(struct tb_service *svc); void (*remove)(struct tb_service *svc); void (*shutdown)(struct tb_service *svc); const struct tb_service_id *id_table; @@ -476,16 +495,17 @@ static inline struct tb_xdomain *tb_service_parent(struct tb_service *svc) return tb_to_xdomain(svc->dev.parent); } +void tb_service_properties_changed(struct tb_service *svc); + /** * struct tb_nhi - thunderbolt native host interface * @lock: Must be held during ring creation/destruction. Is acquired by * interrupt_work when dispatching interrupts to individual rings. - * @pdev: Pointer to the PCI device + * @dev: Device associated with this NHI instance * @ops: NHI specific optional ops * @iobase: MMIO space of the NHI * @tx_rings: All Tx rings available on this host controller * @rx_rings: All Rx rings available on this host controller - * @msix_ida: Used to allocate MSI-X vectors for rings * @going_away: The host controller device is about to disappear so when * this flag is set, avoid touching the hardware anymore. * @iommu_dma_protection: An IOMMU will isolate external-facing ports. @@ -493,20 +513,27 @@ static inline struct tb_xdomain *tb_service_parent(struct tb_service *svc) * MSI-X is used. * @hop_count: Number of rings (end point hops) supported by NHI. * @quirks: NHI specific quirks if any + * @domain_released: Completed when domain has been fully released + * @host_reset: Host router was reset on driver load, or forced on system + * shutdown/reboot. When set, tb_stop() asserts DPR on connected + * downstream ports to signal disconnect before tearing down the + * router tree. Only Thunderbolt 3 devices are reset; USB4 + * routers are skipped. */ struct tb_nhi { spinlock_t lock; - struct pci_dev *pdev; + struct device *dev; const struct tb_nhi_ops *ops; void __iomem *iobase; struct tb_ring **tx_rings; struct tb_ring **rx_rings; - struct ida msix_ida; bool going_away; bool iommu_dma_protection; struct work_struct interrupt_work; u32 hop_count; unsigned long quirks; + struct completion domain_released; + bool host_reset; }; /** @@ -535,6 +562,9 @@ struct tb_nhi { * @start_poll: Called when ring interrupt is triggered to start * polling. Passing %NULL keeps the ring in interrupt mode. * @poll_data: Data passed to @start_poll + * @interval_nsec: Interval counter if interrupt throttling is to be + * used with this ring (in ns) + * @wait: Used to signal that the ring may be empty now */ struct tb_ring { spinlock_t lock; @@ -558,6 +588,8 @@ struct tb_ring { u16 eof_mask; void (*start_poll)(void *data); void *poll_data; + unsigned int interval_nsec; + wait_queue_head_t wait; }; /* Leave ring interrupt enabled on suspend */ @@ -566,6 +598,8 @@ struct tb_ring { #define RING_FLAG_FRAME BIT(1) /* Enable end-to-end flow control */ #define RING_FLAG_E2E BIT(2) +/* Do not enable interrupt for the ring */ +#define RING_FLAG_NO_INTERRUPT BIT(3) struct ring_frame; typedef void (*ring_cb)(struct tb_ring *, struct ring_frame *, bool canceled); @@ -609,7 +643,20 @@ struct ring_frame { }; /* Minimum size for ring_rx */ -#define TB_FRAME_SIZE 0x100 +#define TB_FRAME_SIZE 256 +#define TB_MAX_FRAME_SIZE 4096 + +static inline size_t tb_ring_frame_size(const struct ring_frame *frame) +{ + if (frame->size) + return frame->size; + return TB_MAX_FRAME_SIZE; +} + +static inline size_t tb_ring_size(const struct tb_ring *ring) +{ + return ring->size; +} struct tb_ring *tb_ring_alloc_tx(struct tb_nhi *nhi, int hop, int size, unsigned int flags); @@ -618,6 +665,7 @@ struct tb_ring *tb_ring_alloc_rx(struct tb_nhi *nhi, int hop, int size, u16 sof_mask, u16 eof_mask, void (*start_poll)(void *), void *poll_data); void tb_ring_start(struct tb_ring *ring); +bool tb_ring_flush(struct tb_ring *ring, unsigned int timeout_msec); void tb_ring_stop(struct tb_ring *ring); void tb_ring_free(struct tb_ring *ring); @@ -670,6 +718,8 @@ static inline int tb_ring_tx(struct tb_ring *ring, struct ring_frame *frame) struct ring_frame *tb_ring_poll(struct tb_ring *ring); void tb_ring_poll_complete(struct tb_ring *ring); +int tb_ring_throttling(struct tb_ring *ring, unsigned int interval_nsec); + /** * tb_ring_dma_device() - Return device used for DMA mapping * @ring: Ring whose DMA device is retrieved @@ -681,12 +731,17 @@ void tb_ring_poll_complete(struct tb_ring *ring); */ static inline struct device *tb_ring_dma_device(struct tb_ring *ring) { - return &ring->nhi->pdev->dev; + return ring->nhi->dev; } bool usb4_usb3_port_match(struct device *usb4_port_dev, const struct fwnode_handle *usb3_port_fwnode); +#if IS_REACHABLE(CONFIG_CONFIGFS_FS) +int tb_configfs_register_group(struct config_group *group); +void tb_configfs_unregister_group(struct config_group *group); +#endif + #else /* CONFIG_USB4 */ static inline bool usb4_usb3_port_match(struct device *usb4_port_dev, const struct fwnode_handle *usb3_port_fwnode) diff --git a/include/linux/tick.h b/include/linux/tick.h index 738007d6f577..b121c5d53e30 100644 --- a/include/linux/tick.h +++ b/include/linux/tick.h @@ -7,6 +7,8 @@ #include <linux/clockchips.h> #include <linux/irqflags.h> +#include <linux/jiffies.h> +#include <linux/ktime.h> #include <linux/percpu.h> #include <linux/context_tracking_state.h> #include <linux/cpumask.h> @@ -44,9 +46,6 @@ static inline void tick_unfreeze(void) { } #ifdef CONFIG_TICK_ONESHOT extern void tick_irq_enter(void); -# ifndef arch_needs_cpu -# define arch_needs_cpu() (0) -# endif # else static inline void tick_irq_enter(void) { } #endif @@ -139,8 +138,6 @@ extern bool tick_nohz_idle_got_tick(void); extern ktime_t tick_nohz_get_next_hrtimer(void); extern ktime_t tick_nohz_get_sleep_length(ktime_t *delta_next); extern unsigned long tick_nohz_get_idle_calls_cpu(int cpu); -extern u64 get_cpu_idle_time_us(int cpu, u64 *last_update_time); -extern u64 get_cpu_iowait_time_us(int cpu, u64 *last_update_time); #else /* !CONFIG_NO_HZ_COMMON */ #define tick_nohz_enabled (0) static inline bool tick_nohz_is_active(void) { return false; } @@ -162,8 +159,6 @@ static inline ktime_t tick_nohz_get_sleep_length(ktime_t *delta_next) *delta_next = TICK_NSEC; return *delta_next; } -static inline u64 get_cpu_idle_time_us(int cpu, u64 *unused) { return -1; } -static inline u64 get_cpu_iowait_time_us(int cpu, u64 *unused) { return -1; } #endif /* !CONFIG_NO_HZ_COMMON */ /* diff --git a/include/linux/tifm.h b/include/linux/tifm.h index 44073d06710f..752fcfae27fe 100644 --- a/include/linux/tifm.h +++ b/include/linux/tifm.h @@ -97,7 +97,7 @@ struct tifm_dev { }; struct tifm_driver { - struct tifm_device_id *id_table; + const struct tifm_device_id *id_table; int (*probe)(struct tifm_dev *dev); void (*remove)(struct tifm_dev *dev); int (*suspend)(struct tifm_dev *dev, diff --git a/include/linux/timb_gpio.h b/include/linux/timb_gpio.h deleted file mode 100644 index 3faf5a6bb13e..000000000000 --- a/include/linux/timb_gpio.h +++ /dev/null @@ -1,25 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * timb_gpio.h timberdale FPGA GPIO driver, platform data definition - * Copyright (c) 2009 Intel Corporation - */ - -#ifndef _LINUX_TIMB_GPIO_H -#define _LINUX_TIMB_GPIO_H - -/** - * struct timbgpio_platform_data - Platform data of the Timberdale GPIO driver - * @gpio_base The number of the first GPIO pin, set to -1 for - * dynamic number allocation. - * @nr_pins Number of pins that is supported by the hardware (1-32) - * @irq_base If IRQ is supported by the hardware, this is the base - * number of IRQ:s. One IRQ per pin will be used. Set to - * -1 if IRQ:s is not supported. - */ -struct timbgpio_platform_data { - int gpio_base; - int nr_pins; - int irq_base; -}; - -#endif diff --git a/include/linux/time_namespace.h b/include/linux/time_namespace.h index c514d0e5a45c..58bd9728df58 100644 --- a/include/linux/time_namespace.h +++ b/include/linux/time_namespace.h @@ -8,6 +8,7 @@ #include <linux/ns_common.h> #include <linux/err.h> #include <linux/time64.h> +#include <linux/cleanup.h> struct user_namespace; extern struct user_namespace init_user_ns; @@ -25,7 +26,9 @@ struct time_namespace { struct ucounts *ucounts; struct ns_common ns; struct timens_offsets offsets; +#ifdef CONFIG_TIME_NS_VDSO struct page *vvar_page; +#endif /* If set prevents changing offsets after any task joined namespace. */ bool frozen_offsets; } __randomize_layout; @@ -38,9 +41,6 @@ static inline struct time_namespace *to_time_ns(struct ns_common *ns) return container_of(ns, struct time_namespace, ns); } void __init time_ns_init(void); -extern int vdso_join_timens(struct task_struct *task, - struct time_namespace *ns); -extern void timens_commit(struct task_struct *tsk, struct time_namespace *ns); static inline struct time_namespace *get_time_ns(struct time_namespace *ns) { @@ -53,7 +53,6 @@ struct time_namespace *copy_time_ns(u64 flags, struct time_namespace *old_ns); void free_time_ns(struct time_namespace *ns); void timens_on_fork(struct nsproxy *nsproxy, struct task_struct *tsk); -struct page *find_timens_vvar_page(struct vm_area_struct *vma); static inline void put_time_ns(struct time_namespace *ns) { @@ -117,17 +116,6 @@ static inline void __init time_ns_init(void) { } -static inline int vdso_join_timens(struct task_struct *task, - struct time_namespace *ns) -{ - return 0; -} - -static inline void timens_commit(struct task_struct *tsk, - struct time_namespace *ns) -{ -} - static inline struct time_namespace *get_time_ns(struct time_namespace *ns) { return NULL; @@ -154,11 +142,6 @@ static inline void timens_on_fork(struct nsproxy *nsproxy, return; } -static inline struct page *find_timens_vvar_page(struct vm_area_struct *vma) -{ - return NULL; -} - static inline void timens_add_monotonic(struct timespec64 *ts) { } static inline void timens_add_boottime(struct timespec64 *ts) { } @@ -175,4 +158,20 @@ static inline ktime_t timens_ktime_to_host(clockid_t clockid, ktime_t tim) } #endif +#ifdef CONFIG_TIME_NS_VDSO +extern void timens_commit(struct task_struct *tsk, struct time_namespace *ns); +struct page *find_timens_vvar_page(struct vm_area_struct *vma); +#else /* !CONFIG_TIME_NS_VDSO */ +static inline void timens_commit(struct task_struct *tsk, struct time_namespace *ns) +{ +} + +static inline struct page *find_timens_vvar_page(struct vm_area_struct *vma) +{ + return NULL; +} +#endif /* CONFIG_TIME_NS_VDSO */ + +DEFINE_FREE(time_ns, struct time_namespace *, if (_T) put_time_ns(_T)) + #endif /* _LINUX_TIMENS_H */ diff --git a/include/linux/timekeeper_internal.h b/include/linux/timekeeper_internal.h index b8ae89ea28ab..fe077d97b5f8 100644 --- a/include/linux/timekeeper_internal.h +++ b/include/linux/timekeeper_internal.h @@ -72,6 +72,10 @@ struct tk_read_base { * @id: The timekeeper ID * @tkr_raw: The readout base structure for CLOCK_MONOTONIC_RAW * @raw_sec: CLOCK_MONOTONIC_RAW time in seconds + * @cs_id: The ID of the current clocksource + * @cs_ns_to_cyc_mult: Multiplicator for nanoseconds to cycles conversion + * @cs_ns_to_cyc_shift: Shift value for nanoseconds to cycles conversion + * @cs_ns_to_cyc_maxns: Maximum nanoseconds to cyles conversion range * @clock_was_set_seq: The sequence number of clock was set events * @cs_was_changed_seq: The sequence number of clocksource change events * @clock_valid: Indicator for valid clock @@ -80,8 +84,6 @@ struct tk_read_base { * @cycle_interval: Number of clock cycles in one NTP interval * @xtime_interval: Number of clock shifted nano seconds in one NTP * interval. - * @xtime_remainder: Shifted nano seconds left over when rounding - * @cycle_interval * @raw_interval: Shifted raw nano seconds accumulated per NTP interval. * @next_leap_ktime: CLOCK_MONOTONIC time value of a pending leap-second * @ntp_tick: The ntp_tick_length() value currently being @@ -95,6 +97,10 @@ struct tk_read_base { * @ntp_error_shift: Shift conversion between clock shifted nano seconds and * ntp shifted nano seconds. * @ntp_err_mult: Multiplication factor for scaled math conversion + * @cs_tick_adj: Per-second adjustment handed to NTP via ntp_clear() + * accounting for the difference between the nominal + * NTP interval and the real time taken by the + * clocksource's integer @cycle_interval (upscaled). * @skip_second_overflow: Flag used to avoid updating NTP twice with same second * @tai_offset: The current UTC to TAI offset in seconds * @@ -159,7 +165,11 @@ struct timekeeper { u64 raw_sec; /* Cachline 3 and 4 (timekeeping internal variables): */ - unsigned int clock_was_set_seq; + enum clocksource_ids cs_id; + u32 cs_ns_to_cyc_mult; + u32 cs_ns_to_cyc_shift; + u64 cs_ns_to_cyc_maxns; + u32 clock_was_set_seq; u8 cs_was_changed_seq; u8 clock_valid; @@ -170,7 +180,6 @@ struct timekeeper { u64 cycle_interval; u64 xtime_interval; - s64 xtime_remainder; u64 raw_interval; ktime_t next_leap_ktime; @@ -178,29 +187,10 @@ struct timekeeper { s64 ntp_error; u32 ntp_error_shift; u32 ntp_err_mult; + s64 cs_tick_adj; u32 skip_second_overflow; + s64 skew_delta; s32 tai_offset; }; -#ifdef CONFIG_GENERIC_TIME_VSYSCALL - -extern void update_vsyscall(struct timekeeper *tk); -extern void update_vsyscall_tz(void); - -#else - -static inline void update_vsyscall(struct timekeeper *tk) -{ -} -static inline void update_vsyscall_tz(void) -{ -} -#endif - -#if defined(CONFIG_GENERIC_GETTIMEOFDAY) && defined(CONFIG_POSIX_AUX_CLOCKS) -extern void vdso_time_update_aux(struct timekeeper *tk); -#else -static inline void vdso_time_update_aux(struct timekeeper *tk) { } -#endif - #endif /* _LINUX_TIMEKEEPER_INTERNAL_H */ diff --git a/include/linux/timekeeping.h b/include/linux/timekeeping.h index aee2c1a46e47..efa9442aaeef 100644 --- a/include/linux/timekeeping.h +++ b/include/linux/timekeeping.h @@ -44,7 +44,6 @@ extern void ktime_get_ts64(struct timespec64 *ts); extern void ktime_get_real_ts64(struct timespec64 *tv); extern void ktime_get_coarse_ts64(struct timespec64 *ts); extern void ktime_get_coarse_real_ts64(struct timespec64 *ts); -extern void ktime_get_clock_ts64(clockid_t id, struct timespec64 *ts); /* Multigrain timestamp interfaces */ extern void ktime_get_coarse_real_ts64_mg(struct timespec64 *ts); @@ -268,45 +267,45 @@ extern void timekeeping_inject_sleeptime64(const struct timespec64 *delta); * Auxiliary clock interfaces */ #ifdef CONFIG_POSIX_AUX_CLOCKS -extern bool ktime_get_aux(clockid_t id, ktime_t *kt); -extern bool ktime_get_aux_ts64(clockid_t id, struct timespec64 *kt); +extern bool __must_check ktime_get_aux(clockid_t id, ktime_t *kt); +extern bool __must_check ktime_get_aux_ts64(clockid_t id, struct timespec64 *kt); #else -static inline bool ktime_get_aux(clockid_t id, ktime_t *kt) { return false; } -static inline bool ktime_get_aux_ts64(clockid_t id, struct timespec64 *kt) { return false; } +static inline bool __must_check ktime_get_aux(clockid_t id, ktime_t *kt) { return false; } +static inline bool __must_check ktime_get_aux_ts64(clockid_t id, struct timespec64 *kt) +{ + return false; +} #endif /** - * struct system_time_snapshot - simultaneous raw/real time capture with - * counter value - * @cycles: Clocksource counter value to produce the system times - * @real: Realtime system time - * @boot: Boot time - * @raw: Monotonic raw system time - * @cs_id: Clocksource ID + * struct system_time_snapshot - Simultaneous time capture of monotonic raw time, + * a selected CLOCK_* and the clocksource counter value + * @cycles: Clocksource counter value to produce the system times + * @hw_cycles: For derived clocksources, the hardware counter value from + * which @cycles was derived + * @systime: The system time of the selected CLOCK ID + * @monoraw: Monotonic raw system time + * @cs_id: Clocksource ID + * @hw_csid: Clocksource ID of the underlying hardware counter for derived + * clocksources which implement the read_snapshot() callback. * @clock_was_set_seq: The sequence number of clock-was-set events * @cs_was_changed_seq: The sequence number of clocksource change events + * @valid: True if the snapshot is valid + * + * @monoraw is CLOCK_MONOTONIC_RAW for system time CLOCK ids. For CLOCK_AUX$N + * clock ids it's the monotonic raw time related to the AUX clock, which is + * CLOCK_MONOTONIC_RAW plus a AUX clock specific offset. */ struct system_time_snapshot { u64 cycles; - ktime_t real; - ktime_t boot; - ktime_t raw; + u64 hw_cycles; + ktime_t systime; + ktime_t monoraw; enum clocksource_ids cs_id; - unsigned int clock_was_set_seq; + enum clocksource_ids hw_csid; + u32 clock_was_set_seq; u8 cs_was_changed_seq; -}; - -/** - * struct system_device_crosststamp - system/device cross-timestamp - * (synchronized capture) - * @device: Device time - * @sys_realtime: Realtime simultaneous with device time - * @sys_monoraw: Monotonic raw simultaneous with device time - */ -struct system_device_crosststamp { - ktime_t device; - ktime_t sys_realtime; - ktime_t sys_monoraw; + u8 valid; }; /** @@ -325,6 +324,27 @@ struct system_counterval_t { bool use_nsecs; }; +/** + * struct system_device_crosststamp - system/device cross-timestamp + * (synchronized capture) + * @clock_id: System time Clock ID to capture + * @device: Device time + * @sys_counter: Clocksource counter value simultaneous with device time + * @sys_systime: System time for @clock_id + * @sys_monoraw: Monotonic raw simultaneous with device time + * + * @sys_monoraw is CLOCK_MONOTONIC_RAW for system time CLOCK ids. For + * CLOCK_AUX$N clock ids it's the monotonic raw time related to the AUX clock, + * which is CLOCK_MONOTONIC_RAW plus a AUX clock specific offset. + */ +struct system_device_crosststamp { + clockid_t clock_id; + ktime_t device; + struct system_counterval_t sys_counter; + ktime_t sys_systime; + ktime_t sys_monoraw; +}; + extern bool ktime_real_to_base_clock(ktime_t treal, enum clocksource_ids base_id, u64 *cycles); extern bool timekeeping_clocksource_has_base(enum clocksource_ids id); @@ -341,9 +361,10 @@ extern int get_device_system_crosststamp( struct system_device_crosststamp *xtstamp); /* - * Simultaneously snapshot realtime and monotonic raw clocks + * Simultaneously snapshot a given clock with MONOTONIC_RAW and the underlying + * clocksource counter value. */ -extern void ktime_get_snapshot(struct system_time_snapshot *systime_snapshot); +extern void ktime_get_snapshot_id(clockid_t clock_id, struct system_time_snapshot *systime_snapshot); /* * Persistent clock related interfaces diff --git a/include/linux/timerqueue.h b/include/linux/timerqueue.h index d306d9dd2207..7d0aaa766580 100644 --- a/include/linux/timerqueue.h +++ b/include/linux/timerqueue.h @@ -5,12 +5,11 @@ #include <linux/rbtree.h> #include <linux/timerqueue_types.h> -extern bool timerqueue_add(struct timerqueue_head *head, - struct timerqueue_node *node); -extern bool timerqueue_del(struct timerqueue_head *head, - struct timerqueue_node *node); -extern struct timerqueue_node *timerqueue_iterate_next( - struct timerqueue_node *node); +bool timerqueue_add(struct timerqueue_head *head, struct timerqueue_node *node); +bool timerqueue_del(struct timerqueue_head *head, struct timerqueue_node *node); +struct timerqueue_node *timerqueue_iterate_next(struct timerqueue_node *node); + +bool timerqueue_linked_add(struct timerqueue_linked_head *head, struct timerqueue_linked_node *node); /** * timerqueue_getnext - Returns the timer with the earliest expiration time @@ -19,8 +18,7 @@ extern struct timerqueue_node *timerqueue_iterate_next( * * Returns a pointer to the timer node that has the earliest expiration time. */ -static inline -struct timerqueue_node *timerqueue_getnext(struct timerqueue_head *head) +static inline struct timerqueue_node *timerqueue_getnext(struct timerqueue_head *head) { struct rb_node *leftmost = rb_first_cached(&head->rb_root); @@ -41,4 +39,46 @@ static inline void timerqueue_init_head(struct timerqueue_head *head) { head->rb_root = RB_ROOT_CACHED; } + +/* Timer queues with linked nodes */ + +static __always_inline +struct timerqueue_linked_node *timerqueue_linked_first(struct timerqueue_linked_head *head) +{ + return rb_entry_safe(head->rb_root.rb_leftmost, struct timerqueue_linked_node, node); +} + +static __always_inline +struct timerqueue_linked_node *timerqueue_linked_next(struct timerqueue_linked_node *node) +{ + return rb_entry_safe(node->node.next, struct timerqueue_linked_node, node); +} + +static __always_inline +struct timerqueue_linked_node *timerqueue_linked_prev(struct timerqueue_linked_node *node) +{ + return rb_entry_safe(node->node.prev, struct timerqueue_linked_node, node); +} + +static __always_inline +bool timerqueue_linked_del(struct timerqueue_linked_head *head, struct timerqueue_linked_node *node) +{ + return rb_erase_linked(&node->node, &head->rb_root); +} + +static __always_inline void timerqueue_linked_init(struct timerqueue_linked_node *node) +{ + RB_CLEAR_LINKED_NODE(&node->node); +} + +static __always_inline bool timerqueue_linked_node_queued(struct timerqueue_linked_node *node) +{ + return !RB_EMPTY_LINKED_NODE(&node->node); +} + +static __always_inline void timerqueue_linked_init_head(struct timerqueue_linked_head *head) +{ + head->rb_root = RB_ROOT_LINKED; +} + #endif /* _LINUX_TIMERQUEUE_H */ diff --git a/include/linux/timerqueue_types.h b/include/linux/timerqueue_types.h index dc298d0923e3..be2218b147c4 100644 --- a/include/linux/timerqueue_types.h +++ b/include/linux/timerqueue_types.h @@ -6,12 +6,21 @@ #include <linux/types.h> struct timerqueue_node { - struct rb_node node; - ktime_t expires; + struct rb_node node; + ktime_t expires; }; struct timerqueue_head { - struct rb_root_cached rb_root; + struct rb_root_cached rb_root; +}; + +struct timerqueue_linked_node { + struct rb_node_linked node; + ktime_t expires; +}; + +struct timerqueue_linked_head { + struct rb_root_linked rb_root; }; #endif /* _LINUX_TIMERQUEUE_TYPES_H */ diff --git a/include/linux/timex.h b/include/linux/timex.h index 4ee32eff3f22..7014ccc638fc 100644 --- a/include/linux/timex.h +++ b/include/linux/timex.h @@ -156,8 +156,6 @@ extern int do_clock_adjtime(const clockid_t which_clock, struct __kernel_timex * extern void hardpps(const struct timespec64 *, const struct timespec64 *); -int read_current_timer(unsigned long *timer_val); - /* The clock frequency of the i8253/i8254 PIT */ #define PIT_TICK_RATE 1193182ul diff --git a/include/linux/topology.h b/include/linux/topology.h index 6575af39fd10..709a2dcf4c73 100644 --- a/include/linux/topology.h +++ b/include/linux/topology.h @@ -230,11 +230,24 @@ static inline int cpu_to_mem(int cpu) #define topology_drawer_cpumask(cpu) cpumask_of(cpu) #endif -#if defined(CONFIG_SCHED_SMT) && !defined(cpu_smt_mask) +/* + * Defining cpu_smt_mask as cpumask_of that CPU helps to get + * rid of lot of ifdeffery all around the codebase in case of + * CONFIG_SCHED_SMT=n. It just means there are no other siblings, which + * is what is expected. + */ +#if defined(CONFIG_SCHED_SMT) +# if !defined(cpu_smt_mask) static inline const struct cpumask *cpu_smt_mask(int cpu) { return topology_sibling_cpumask(cpu); } +# endif +#else /* !CONFIG_SCHED_SMT */ +static inline const struct cpumask *cpu_smt_mask(int cpu) +{ + return cpumask_of(cpu); +} #endif #ifndef topology_is_primary_thread diff --git a/include/linux/torture.h b/include/linux/torture.h index 1b59056c3b18..c9b47d138302 100644 --- a/include/linux/torture.h +++ b/include/linux/torture.h @@ -129,6 +129,7 @@ void _torture_stop_kthread(char *m, struct task_struct **tp); #else #define torture_preempt_schedule() do { } while (0) #endif +void torture_sched_set_normal(struct task_struct *t, int nice); #if IS_ENABLED(CONFIG_RCU_TORTURE_TEST) || IS_MODULE(CONFIG_RCU_TORTURE_TEST) || IS_ENABLED(CONFIG_LOCK_TORTURE_TEST) || IS_MODULE(CONFIG_LOCK_TORTURE_TEST) long torture_sched_setaffinity(pid_t pid, const struct cpumask *in_mask, bool dowarn); diff --git a/include/linux/tpm.h b/include/linux/tpm.h index 202da079d500..0db277af45c3 100644 --- a/include/linux/tpm.h +++ b/include/linux/tpm.h @@ -25,10 +25,8 @@ #include <crypto/hash_info.h> #include <crypto/aes.h> -#define TPM_DIGEST_SIZE 20 /* Max TPM v1.2 PCR size */ - -#define TPM2_MAX_DIGEST_SIZE SHA512_DIGEST_SIZE -#define TPM2_MAX_PCR_BANKS 8 +#include <linux/tpm_command.h> +#include <linux/tpm_buf.h> struct tpm_chip; struct trusted_key_payload; @@ -36,49 +34,6 @@ struct trusted_key_options; /* opaque structure, holds auth session parameters like the session key */ struct tpm2_auth; -enum tpm2_session_types { - TPM2_SE_HMAC = 0x00, - TPM2_SE_POLICY = 0x01, - TPM2_SE_TRIAL = 0x02, -}; - -/* if you add a new hash to this, increment TPM_MAX_HASHES below */ -enum tpm_algorithms { - TPM_ALG_ERROR = 0x0000, - TPM_ALG_SHA1 = 0x0004, - TPM_ALG_AES = 0x0006, - TPM_ALG_KEYEDHASH = 0x0008, - TPM_ALG_SHA256 = 0x000B, - TPM_ALG_SHA384 = 0x000C, - TPM_ALG_SHA512 = 0x000D, - TPM_ALG_NULL = 0x0010, - TPM_ALG_SM3_256 = 0x0012, - TPM_ALG_ECC = 0x0023, - TPM_ALG_CFB = 0x0043, -}; - -/* - * maximum number of hashing algorithms a TPM can have. This is - * basically a count of every hash in tpm_algorithms above - */ -#define TPM_MAX_HASHES 5 - -enum tpm2_curves { - TPM2_ECC_NONE = 0x0000, - TPM2_ECC_NIST_P256 = 0x0003, -}; - -struct tpm_digest { - u16 alg_id; - u8 digest[TPM2_MAX_DIGEST_SIZE]; -} __packed; - -struct tpm_bank_info { - u16 alg_id; - u16 digest_size; - u16 crypto_id; -}; - enum TPM_OPS_FLAGS { TPM_OPS_AUTO_STARTUP = BIT(0), }; @@ -136,7 +91,7 @@ struct tpm_chip_seqops { const struct seq_operations *seqops; }; -/* fixed define for the curve we use which is NIST_P256 */ +/* Fixed define for the curve we use which is NIST_P256 */ #define EC_PT_SZ 32 /* @@ -218,124 +173,11 @@ struct tpm_chip { #endif }; -#define TPM_HEADER_SIZE 10 - -enum tpm2_const { - TPM2_PLATFORM_PCR = 24, - TPM2_PCR_SELECT_MIN = ((TPM2_PLATFORM_PCR + 7) / 8), -}; - -enum tpm2_timeouts { - TPM2_TIMEOUT_A = 750, - TPM2_TIMEOUT_B = 4000, - TPM2_TIMEOUT_C = 200, - TPM2_TIMEOUT_D = 30, -}; - -enum tpm2_durations { - TPM2_DURATION_SHORT = 20, - TPM2_DURATION_LONG = 2000, - TPM2_DURATION_DEFAULT = 120000, -}; - -enum tpm2_structures { - TPM2_ST_NO_SESSIONS = 0x8001, - TPM2_ST_SESSIONS = 0x8002, - TPM2_ST_CREATION = 0x8021, -}; - -/* Indicates from what layer of the software stack the error comes from */ -#define TSS2_RC_LAYER_SHIFT 16 -#define TSS2_RESMGR_TPM_RC_LAYER (11 << TSS2_RC_LAYER_SHIFT) - -enum tpm2_return_codes { - TPM2_RC_SUCCESS = 0x0000, - TPM2_RC_HASH = 0x0083, /* RC_FMT1 */ - TPM2_RC_HANDLE = 0x008B, - TPM2_RC_INTEGRITY = 0x009F, - TPM2_RC_INITIALIZE = 0x0100, /* RC_VER1 */ - TPM2_RC_FAILURE = 0x0101, - TPM2_RC_DISABLED = 0x0120, - TPM2_RC_UPGRADE = 0x012D, - TPM2_RC_COMMAND_CODE = 0x0143, - TPM2_RC_TESTING = 0x090A, /* RC_WARN */ - TPM2_RC_REFERENCE_H0 = 0x0910, - TPM2_RC_RETRY = 0x0922, - TPM2_RC_SESSION_MEMORY = 0x0903, -}; - -enum tpm2_command_codes { - TPM2_CC_FIRST = 0x011F, - TPM2_CC_HIERARCHY_CONTROL = 0x0121, - TPM2_CC_HIERARCHY_CHANGE_AUTH = 0x0129, - TPM2_CC_CREATE_PRIMARY = 0x0131, - TPM2_CC_SEQUENCE_COMPLETE = 0x013E, - TPM2_CC_SELF_TEST = 0x0143, - TPM2_CC_STARTUP = 0x0144, - TPM2_CC_SHUTDOWN = 0x0145, - TPM2_CC_NV_READ = 0x014E, - TPM2_CC_CREATE = 0x0153, - TPM2_CC_LOAD = 0x0157, - TPM2_CC_SEQUENCE_UPDATE = 0x015C, - TPM2_CC_UNSEAL = 0x015E, - TPM2_CC_CONTEXT_LOAD = 0x0161, - TPM2_CC_CONTEXT_SAVE = 0x0162, - TPM2_CC_FLUSH_CONTEXT = 0x0165, - TPM2_CC_READ_PUBLIC = 0x0173, - TPM2_CC_START_AUTH_SESS = 0x0176, - TPM2_CC_VERIFY_SIGNATURE = 0x0177, - TPM2_CC_GET_CAPABILITY = 0x017A, - TPM2_CC_GET_RANDOM = 0x017B, - TPM2_CC_PCR_READ = 0x017E, - TPM2_CC_PCR_EXTEND = 0x0182, - TPM2_CC_EVENT_SEQUENCE_COMPLETE = 0x0185, - TPM2_CC_HASH_SEQUENCE_START = 0x0186, - TPM2_CC_CREATE_LOADED = 0x0191, - TPM2_CC_LAST = 0x0193, /* Spec 1.36 */ -}; - -enum tpm2_permanent_handles { - TPM2_RH_NULL = 0x40000007, - TPM2_RS_PW = 0x40000009, -}; - -/* Most Significant Octet for key types */ -enum tpm2_mso_type { - TPM2_MSO_NVRAM = 0x01, - TPM2_MSO_SESSION = 0x02, - TPM2_MSO_POLICY = 0x03, - TPM2_MSO_PERMANENT = 0x40, - TPM2_MSO_VOLATILE = 0x80, - TPM2_MSO_PERSISTENT = 0x81, -}; - static inline enum tpm2_mso_type tpm2_handle_mso(u32 handle) { return handle >> 24; } -enum tpm2_capabilities { - TPM2_CAP_HANDLES = 1, - TPM2_CAP_COMMANDS = 2, - TPM2_CAP_PCRS = 5, - TPM2_CAP_TPM_PROPERTIES = 6, -}; - -enum tpm2_properties { - TPM_PT_TOTAL_COMMANDS = 0x0129, -}; - -enum tpm2_startup_types { - TPM2_SU_CLEAR = 0x0000, - TPM2_SU_STATE = 0x0001, -}; - -enum tpm2_cc_attrs { - TPM2_CC_ATTR_CHANDLES = 25, - TPM2_CC_ATTR_RHANDLE = 28, - TPM2_CC_ATTR_VENDOR = 29, -}; - #define TPM_VID_INTEL 0x8086 #define TPM_VID_WINBOND 0x1050 #define TPM_VID_STM 0x104A @@ -359,77 +201,11 @@ enum tpm_chip_flags { #define to_tpm_chip(d) container_of(d, struct tpm_chip, dev) -struct tpm_header { - __be16 tag; - __be32 length; - union { - __be32 ordinal; - __be32 return_code; - }; -} __packed; - -enum tpm_buf_flags { - /* the capacity exceeded: */ - TPM_BUF_OVERFLOW = BIT(0), - /* TPM2B format: */ - TPM_BUF_TPM2B = BIT(1), - /* read out of boundary: */ - TPM_BUF_BOUNDARY_ERROR = BIT(2), -}; - -/* - * A string buffer type for constructing TPM commands. - */ -struct tpm_buf { - u32 flags; - u32 length; - u8 *data; - u8 handles; -}; - -enum tpm2_object_attributes { - TPM2_OA_FIXED_TPM = BIT(1), - TPM2_OA_ST_CLEAR = BIT(2), - TPM2_OA_FIXED_PARENT = BIT(4), - TPM2_OA_SENSITIVE_DATA_ORIGIN = BIT(5), - TPM2_OA_USER_WITH_AUTH = BIT(6), - TPM2_OA_ADMIN_WITH_POLICY = BIT(7), - TPM2_OA_NO_DA = BIT(10), - TPM2_OA_ENCRYPTED_DUPLICATION = BIT(11), - TPM2_OA_RESTRICTED = BIT(16), - TPM2_OA_DECRYPT = BIT(17), - TPM2_OA_SIGN = BIT(18), -}; - -enum tpm2_session_attributes { - TPM2_SA_CONTINUE_SESSION = BIT(0), - TPM2_SA_AUDIT_EXCLUSIVE = BIT(1), - TPM2_SA_AUDIT_RESET = BIT(3), - TPM2_SA_DECRYPT = BIT(5), - TPM2_SA_ENCRYPT = BIT(6), - TPM2_SA_AUDIT = BIT(7), -}; - struct tpm2_hash { unsigned int crypto_id; unsigned int tpm_id; }; -int tpm_buf_init(struct tpm_buf *buf, u16 tag, u32 ordinal); -void tpm_buf_reset(struct tpm_buf *buf, u16 tag, u32 ordinal); -int tpm_buf_init_sized(struct tpm_buf *buf); -void tpm_buf_reset_sized(struct tpm_buf *buf); -void tpm_buf_destroy(struct tpm_buf *buf); -u32 tpm_buf_length(struct tpm_buf *buf); -void tpm_buf_append(struct tpm_buf *buf, const u8 *new_data, u16 new_length); -void tpm_buf_append_u8(struct tpm_buf *buf, const u8 value); -void tpm_buf_append_u16(struct tpm_buf *buf, const u16 value); -void tpm_buf_append_u32(struct tpm_buf *buf, const u32 value); -u8 tpm_buf_read_u8(struct tpm_buf *buf, off_t *offset); -u16 tpm_buf_read_u16(struct tpm_buf *buf, off_t *offset); -u32 tpm_buf_read_u32(struct tpm_buf *buf, off_t *offset); -void tpm_buf_append_handle(struct tpm_chip *chip, struct tpm_buf *buf, u32 handle); - /* * Check if TPM device is in the firmware upgrade mode. */ diff --git a/include/linux/tpm_buf.h b/include/linux/tpm_buf.h new file mode 100644 index 000000000000..f8c105d8b8bf --- /dev/null +++ b/include/linux/tpm_buf.h @@ -0,0 +1,60 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Following copyright information was take from the original file + * <include/linux/tpm.h> where the definitions were moved from: + * + * Copyright (C) 2004,2007,2008 IBM Corporation + * + * Authors: + * Leendert van Doorn <leendert@watson.ibm.com> + * Dave Safford <safford@watson.ibm.com> + * Reiner Sailer <sailer@watson.ibm.com> + * Kylene Hall <kjhall@us.ibm.com> + * Debora Velarde <dvelarde@us.ibm.com> + * + * Maintained by: <tpmdd_devel@lists.sourceforge.net> + * + * Device driver for TCG/TCPA TPM (trusted platform module). + * Specifications at www.trustedcomputinggroup.org + */ + +#ifndef __LINUX_TPM_BUF_H__ +#define __LINUX_TPM_BUF_H__ + +#include <linux/types.h> +#include <linux/bits.h> + +enum tpm_buf_flags { + /* TPM2B format: */ + TPM_BUF_TPM2B = BIT(0), + /* The buffer is in invalid and unusable state: */ + TPM_BUF_INVALID = BIT(1), +}; + +/* + * A buffer for constructing and parsing TPM commands, responses and sized + * (TPM2B) buffers. + */ +struct tpm_buf { + u8 flags; + u8 handles; + u16 length; + u16 capacity; + u8 data[]; +}; + +void tpm_buf_init(struct tpm_buf *buf, u16 buf_size); +void tpm_buf_init_sized(struct tpm_buf *buf, u16 buf_size); +void tpm_buf_reset(struct tpm_buf *buf, u16 tag, u32 ordinal); +void tpm_buf_reset_sized(struct tpm_buf *buf); +u16 tpm_buf_length(struct tpm_buf *buf); +void tpm_buf_append(struct tpm_buf *buf, const u8 *new_data, u16 new_length); +void tpm_buf_append_u8(struct tpm_buf *buf, const u8 value); +void tpm_buf_append_u16(struct tpm_buf *buf, const u16 value); +void tpm_buf_append_u32(struct tpm_buf *buf, const u32 value); +u8 tpm_buf_read_u8(struct tpm_buf *buf, off_t *offset); +u16 tpm_buf_read_u16(struct tpm_buf *buf, off_t *offset); +u32 tpm_buf_read_u32(struct tpm_buf *buf, off_t *offset); +void tpm_buf_append_handle(struct tpm_buf *buf, u32 handle); + +#endif diff --git a/include/linux/tpm_command.h b/include/linux/tpm_command.h index f5c03e9c3913..fc446a1282e2 100644 --- a/include/linux/tpm_command.h +++ b/include/linux/tpm_command.h @@ -2,28 +2,520 @@ #ifndef __LINUX_TPM_COMMAND_H__ #define __LINUX_TPM_COMMAND_H__ +#include <crypto/sha2.h> + /* - * TPM Command constants from specifications at - * http://www.trustedcomputinggroup.org + * == TPM 1 Family Chips == + * + * TPM 1.2 Main Specification: + * https://trustedcomputinggroup.org/resource/tpm-main-specification/ */ +#define TPM_MAX_ORDINAL 243 + /* Command TAGS */ -#define TPM_TAG_RQU_COMMAND 193 -#define TPM_TAG_RQU_AUTH1_COMMAND 194 -#define TPM_TAG_RQU_AUTH2_COMMAND 195 -#define TPM_TAG_RSP_COMMAND 196 -#define TPM_TAG_RSP_AUTH1_COMMAND 197 -#define TPM_TAG_RSP_AUTH2_COMMAND 198 +enum tpm_command_tags { + TPM_TAG_RQU_COMMAND = 193, + TPM_TAG_RQU_AUTH1_COMMAND = 194, + TPM_TAG_RQU_AUTH2_COMMAND = 195, + TPM_TAG_RSP_COMMAND = 196, + TPM_TAG_RSP_AUTH1_COMMAND = 197, + TPM_TAG_RSP_AUTH2_COMMAND = 198, +}; /* Command Ordinals */ -#define TPM_ORD_GETRANDOM 70 -#define TPM_ORD_OSAP 11 -#define TPM_ORD_OIAP 10 -#define TPM_ORD_SEAL 23 -#define TPM_ORD_UNSEAL 24 +enum tpm_command_ordinals { + TPM_ORD_CONTINUE_SELFTEST = 83, + TPM_ORD_GET_CAP = 101, + TPM_ORD_GET_RANDOM = 70, + TPM_ORD_PCR_EXTEND = 20, + TPM_ORD_PCR_READ = 21, + TPM_ORD_OSAP = 11, + TPM_ORD_OIAP = 10, + TPM_ORD_SAVESTATE = 152, + TPM_ORD_SEAL = 23, + TPM_ORD_STARTUP = 153, + TPM_ORD_UNSEAL = 24, +}; + +enum tpm_capabilities { + TPM_CAP_FLAG = 4, + TPM_CAP_PROP = 5, + TPM_CAP_VERSION_1_1 = 0x06, + TPM_CAP_VERSION_1_2 = 0x1A, +}; + +enum tpm_sub_capabilities { + TPM_CAP_PROP_PCR = 0x101, + TPM_CAP_PROP_MANUFACTURER = 0x103, + TPM_CAP_FLAG_PERM = 0x108, + TPM_CAP_FLAG_VOL = 0x109, + TPM_CAP_PROP_OWNER = 0x111, + TPM_CAP_PROP_TIS_TIMEOUT = 0x115, + TPM_CAP_PROP_TIS_DURATION = 0x120, +}; + +/* Return Codes */ +enum tpm_return_codes { + TPM_BASE_MASK = 0, + TPM_NON_FATAL_MASK = 0x00000800, + TPM_SUCCESS = TPM_BASE_MASK + 0, + TPM_ERR_DEACTIVATED = TPM_BASE_MASK + 6, + TPM_ERR_DISABLED = TPM_BASE_MASK + 7, + TPM_ERR_FAIL = TPM_BASE_MASK + 9, + TPM_ERR_FAILEDSELFTEST = TPM_BASE_MASK + 28, + TPM_ERR_INVALID_POSTINIT = TPM_BASE_MASK + 38, + TPM_ERR_INVALID_FAMILY = TPM_BASE_MASK + 55, + TPM_WARN_RETRY = TPM_BASE_MASK + TPM_NON_FATAL_MASK + 0, + TPM_WARN_DOING_SELFTEST = TPM_BASE_MASK + TPM_NON_FATAL_MASK + 2, +}; + +struct stclear_flags_t { + __be16 tag; + u8 deactivated; + u8 disableForceClear; + u8 physicalPresence; + u8 physicalPresenceLock; + u8 bGlobalLock; +} __packed; + +struct tpm1_version { + u8 major; + u8 minor; + u8 rev_major; + u8 rev_minor; +} __packed; + +struct tpm1_version2 { + __be16 tag; + struct tpm1_version version; +} __packed; + +struct timeout_t { + __be32 a; + __be32 b; + __be32 c; + __be32 d; +} __packed; + +struct duration_t { + __be32 tpm_short; + __be32 tpm_medium; + __be32 tpm_long; +} __packed; + +struct permanent_flags_t { + __be16 tag; + u8 disable; + u8 ownership; + u8 deactivated; + u8 readPubek; + u8 disableOwnerClear; + u8 allowMaintenance; + u8 physicalPresenceLifetimeLock; + u8 physicalPresenceHWEnable; + u8 physicalPresenceCMDEnable; + u8 CEKPUsed; + u8 TPMpost; + u8 TPMpostLock; + u8 FIPS; + u8 operator; + u8 enableRevokeEK; + u8 nvLocked; + u8 readSRKPub; + u8 tpmEstablished; + u8 maintenanceDone; + u8 disableFullDALogicInfo; +} __packed; + +typedef union { + struct permanent_flags_t perm_flags; + struct stclear_flags_t stclear_flags; + __u8 owned; + __be32 num_pcrs; + struct tpm1_version version1; + struct tpm1_version2 version2; + __be32 manufacturer_id; + struct timeout_t timeout; + struct duration_t duration; +} cap_t; + +/* + * 128 bytes is an arbitrary cap. This could be as large as TPM_BUFSIZE - 18 + * bytes, but 128 is still a relatively large number of random bytes and + * anything much bigger causes users of struct tpm_cmd_t to start getting + * compiler warnings about stack frame size. + */ +#define TPM_MAX_RNG_DATA 128 + +struct tpm1_get_random_out { + __be32 rng_data_len; + u8 rng_data[TPM_MAX_RNG_DATA]; +} __packed; /* Other constants */ #define SRKHANDLE 0x40000000 #define TPM_NONCE_SIZE 20 +#define TPM_ST_CLEAR 1 + +/* + * == TPM 2 Family Chips == + * + * TPM 2.0 Library + * https://trustedcomputinggroup.org/resource/tpm-library-specification/ + */ + +/* TPM2 specific constants. */ +#define TPM2_SPACE_BUFFER_SIZE 16384 /* 16 kB */ + +enum tpm2_session_types { + TPM2_SE_HMAC = 0x00, + TPM2_SE_POLICY = 0x01, + TPM2_SE_TRIAL = 0x02, +}; + +enum tpm2_timeouts { + TPM2_TIMEOUT_A = 750, + TPM2_TIMEOUT_B = 4000, + TPM2_TIMEOUT_C = 200, + TPM2_TIMEOUT_D = 30, + TPM2_DURATION_SHORT = 20, + TPM2_DURATION_MEDIUM = 750, + TPM2_DURATION_LONG = 2000, + TPM2_DURATION_LONG_LONG = 300000, + TPM2_DURATION_DEFAULT = 120000, +}; + +enum tpm2_structures { + TPM2_ST_NO_SESSIONS = 0x8001, + TPM2_ST_SESSIONS = 0x8002, + TPM2_ST_CREATION = 0x8021, +}; + +/* Indicates from what layer of the software stack the error comes from */ +#define TSS2_RC_LAYER_SHIFT 16 +#define TSS2_RESMGR_TPM_RC_LAYER (11 << TSS2_RC_LAYER_SHIFT) + +enum tpm2_return_codes { + TPM2_RC_SUCCESS = 0x0000, + TPM2_RC_HASH = 0x0083, /* RC_FMT1 */ + TPM2_RC_HANDLE = 0x008B, + TPM2_RC_INTEGRITY = 0x009F, + TPM2_RC_INITIALIZE = 0x0100, /* RC_VER1 */ + TPM2_RC_FAILURE = 0x0101, + TPM2_RC_DISABLED = 0x0120, + TPM2_RC_UPGRADE = 0x012D, + TPM2_RC_COMMAND_CODE = 0x0143, + TPM2_RC_TESTING = 0x090A, /* RC_WARN */ + TPM2_RC_REFERENCE_H0 = 0x0910, + TPM2_RC_RETRY = 0x0922, + TPM2_RC_SESSION_MEMORY = 0x0903, +}; + +enum tpm2_command_codes { + TPM2_CC_FIRST = 0x011F, + TPM2_CC_HIERARCHY_CONTROL = 0x0121, + TPM2_CC_HIERARCHY_CHANGE_AUTH = 0x0129, + TPM2_CC_CREATE_PRIMARY = 0x0131, + TPM2_CC_SEQUENCE_COMPLETE = 0x013E, + TPM2_CC_SELF_TEST = 0x0143, + TPM2_CC_STARTUP = 0x0144, + TPM2_CC_SHUTDOWN = 0x0145, + TPM2_CC_NV_READ = 0x014E, + TPM2_CC_CREATE = 0x0153, + TPM2_CC_LOAD = 0x0157, + TPM2_CC_SEQUENCE_UPDATE = 0x015C, + TPM2_CC_UNSEAL = 0x015E, + TPM2_CC_CONTEXT_LOAD = 0x0161, + TPM2_CC_CONTEXT_SAVE = 0x0162, + TPM2_CC_FLUSH_CONTEXT = 0x0165, + TPM2_CC_READ_PUBLIC = 0x0173, + TPM2_CC_START_AUTH_SESS = 0x0176, + TPM2_CC_VERIFY_SIGNATURE = 0x0177, + TPM2_CC_GET_CAPABILITY = 0x017A, + TPM2_CC_GET_RANDOM = 0x017B, + TPM2_CC_PCR_READ = 0x017E, + TPM2_CC_PCR_EXTEND = 0x0182, + TPM2_CC_EVENT_SEQUENCE_COMPLETE = 0x0185, + TPM2_CC_HASH_SEQUENCE_START = 0x0186, + TPM2_CC_CREATE_LOADED = 0x0191, + TPM2_CC_LAST = 0x0193, /* Spec 1.36 */ +}; + +enum tpm2_capabilities { + TPM2_CAP_HANDLES = 1, + TPM2_CAP_COMMANDS = 2, + TPM2_CAP_PCRS = 5, + TPM2_CAP_TPM_PROPERTIES = 6, +}; + +enum tpm2_properties { + TPM_PT_TOTAL_COMMANDS = 0x0129, +}; + +enum tpm2_startup_types { + TPM2_SU_CLEAR = 0x0000, + TPM2_SU_STATE = 0x0001, +}; + +enum tpm2_cc_attrs { + TPM2_CC_ATTR_CHANDLES = 25, + TPM2_CC_ATTR_RHANDLE = 28, + TPM2_CC_ATTR_VENDOR = 29, +}; + +enum tpm2_permanent_handles { + TPM2_RH_NULL = 0x40000007, + TPM2_RS_PW = 0x40000009, +}; + +/* Most Significant Octet for key types */ +enum tpm2_mso_type { + TPM2_MSO_NVRAM = 0x01, + TPM2_MSO_SESSION = 0x02, + TPM2_MSO_POLICY = 0x03, + TPM2_MSO_PERMANENT = 0x40, + TPM2_MSO_VOLATILE = 0x80, + TPM2_MSO_PERSISTENT = 0x81, +}; + +enum tpm2_curves { + TPM2_ECC_NONE = 0x0000, + TPM2_ECC_NIST_P256 = 0x0003, +}; + +enum tpm2_object_attributes { + TPM2_OA_FIXED_TPM = BIT(1), + TPM2_OA_ST_CLEAR = BIT(2), + TPM2_OA_FIXED_PARENT = BIT(4), + TPM2_OA_SENSITIVE_DATA_ORIGIN = BIT(5), + TPM2_OA_USER_WITH_AUTH = BIT(6), + TPM2_OA_ADMIN_WITH_POLICY = BIT(7), + TPM2_OA_NO_DA = BIT(10), + TPM2_OA_ENCRYPTED_DUPLICATION = BIT(11), + TPM2_OA_RESTRICTED = BIT(16), + TPM2_OA_DECRYPT = BIT(17), + TPM2_OA_SIGN = BIT(18), +}; + +enum tpm2_session_attributes { + TPM2_SA_CONTINUE_SESSION = BIT(0), + TPM2_SA_AUDIT_EXCLUSIVE = BIT(1), + TPM2_SA_AUDIT_RESET = BIT(3), + TPM2_SA_DECRYPT = BIT(5), + TPM2_SA_ENCRYPT = BIT(6), + TPM2_SA_AUDIT = BIT(7), +}; + +enum tpm2_pcr_select { + TPM2_PLATFORM_PCR = 24, + TPM2_PCR_SELECT_MIN = ((TPM2_PLATFORM_PCR + 7) / 8), +}; + +enum tpm2_handle_types { + TPM2_HT_HMAC_SESSION = 0x02000000, + TPM2_HT_POLICY_SESSION = 0x03000000, + TPM2_HT_TRANSIENT = 0x80000000, +}; + +enum tpm2_pt_props { + TPM2_PT_NONE = 0x00000000, + TPM2_PT_GROUP = 0x00000100, + TPM2_PT_FIXED = TPM2_PT_GROUP * 1, + TPM2_PT_FAMILY_INDICATOR = TPM2_PT_FIXED + 0, + TPM2_PT_LEVEL = TPM2_PT_FIXED + 1, + TPM2_PT_REVISION = TPM2_PT_FIXED + 2, + TPM2_PT_DAY_OF_YEAR = TPM2_PT_FIXED + 3, + TPM2_PT_YEAR = TPM2_PT_FIXED + 4, + TPM2_PT_MANUFACTURER = TPM2_PT_FIXED + 5, + TPM2_PT_VENDOR_STRING_1 = TPM2_PT_FIXED + 6, + TPM2_PT_VENDOR_STRING_2 = TPM2_PT_FIXED + 7, + TPM2_PT_VENDOR_STRING_3 = TPM2_PT_FIXED + 8, + TPM2_PT_VENDOR_STRING_4 = TPM2_PT_FIXED + 9, + TPM2_PT_VENDOR_TPM_TYPE = TPM2_PT_FIXED + 10, + TPM2_PT_FIRMWARE_VERSION_1 = TPM2_PT_FIXED + 11, + TPM2_PT_FIRMWARE_VERSION_2 = TPM2_PT_FIXED + 12, + TPM2_PT_INPUT_BUFFER = TPM2_PT_FIXED + 13, + TPM2_PT_HR_TRANSIENT_MIN = TPM2_PT_FIXED + 14, + TPM2_PT_HR_PERSISTENT_MIN = TPM2_PT_FIXED + 15, + TPM2_PT_HR_LOADED_MIN = TPM2_PT_FIXED + 16, + TPM2_PT_ACTIVE_SESSIONS_MAX = TPM2_PT_FIXED + 17, + TPM2_PT_PCR_COUNT = TPM2_PT_FIXED + 18, + TPM2_PT_PCR_SELECT_MIN = TPM2_PT_FIXED + 19, + TPM2_PT_CONTEXT_GAP_MAX = TPM2_PT_FIXED + 20, + TPM2_PT_NV_COUNTERS_MAX = TPM2_PT_FIXED + 22, + TPM2_PT_NV_INDEX_MAX = TPM2_PT_FIXED + 23, + TPM2_PT_MEMORY = TPM2_PT_FIXED + 24, + TPM2_PT_CLOCK_UPDATE = TPM2_PT_FIXED + 25, + TPM2_PT_CONTEXT_HASH = TPM2_PT_FIXED + 26, + TPM2_PT_CONTEXT_SYM = TPM2_PT_FIXED + 27, + TPM2_PT_CONTEXT_SYM_SIZE = TPM2_PT_FIXED + 28, + TPM2_PT_ORDERLY_COUNT = TPM2_PT_FIXED + 29, + TPM2_PT_MAX_COMMAND_SIZE = TPM2_PT_FIXED + 30, + TPM2_PT_MAX_RESPONSE_SIZE = TPM2_PT_FIXED + 31, + TPM2_PT_MAX_DIGEST = TPM2_PT_FIXED + 32, + TPM2_PT_MAX_OBJECT_CONTEXT = TPM2_PT_FIXED + 33, + TPM2_PT_MAX_SESSION_CONTEXT = TPM2_PT_FIXED + 34, + TPM2_PT_PS_FAMILY_INDICATOR = TPM2_PT_FIXED + 35, + TPM2_PT_PS_LEVEL = TPM2_PT_FIXED + 36, + TPM2_PT_PS_REVISION = TPM2_PT_FIXED + 37, + TPM2_PT_PS_DAY_OF_YEAR = TPM2_PT_FIXED + 38, + TPM2_PT_PS_YEAR = TPM2_PT_FIXED + 39, + TPM2_PT_SPLIT_MAX = TPM2_PT_FIXED + 40, + TPM2_PT_TOTAL_COMMANDS = TPM2_PT_FIXED + 41, + TPM2_PT_LIBRARY_COMMANDS = TPM2_PT_FIXED + 42, + TPM2_PT_VENDOR_COMMANDS = TPM2_PT_FIXED + 43, + TPM2_PT_NV_BUFFER_MAX = TPM2_PT_FIXED + 44, + TPM2_PT_MODES = TPM2_PT_FIXED + 45, + TPM2_PT_MAX_CAP_BUFFER = TPM2_PT_FIXED + 46, + TPM2_PT_VAR = TPM2_PT_GROUP * 2, + TPM2_PT_PERMANENT = TPM2_PT_VAR + 0, + TPM2_PT_STARTUP_CLEAR = TPM2_PT_VAR + 1, + TPM2_PT_HR_NV_INDEX = TPM2_PT_VAR + 2, + TPM2_PT_HR_LOADED = TPM2_PT_VAR + 3, + TPM2_PT_HR_LOADED_AVAIL = TPM2_PT_VAR + 4, + TPM2_PT_HR_ACTIVE = TPM2_PT_VAR + 5, + TPM2_PT_HR_ACTIVE_AVAIL = TPM2_PT_VAR + 6, + TPM2_PT_HR_TRANSIENT_AVAIL = TPM2_PT_VAR + 7, + TPM2_PT_HR_PERSISTENT = TPM2_PT_VAR + 8, + TPM2_PT_HR_PERSISTENT_AVAIL = TPM2_PT_VAR + 9, + TPM2_PT_NV_COUNTERS = TPM2_PT_VAR + 10, + TPM2_PT_NV_COUNTERS_AVAIL = TPM2_PT_VAR + 11, + TPM2_PT_ALGORITHM_SET = TPM2_PT_VAR + 12, + TPM2_PT_LOADED_CURVES = TPM2_PT_VAR + 13, + TPM2_PT_LOCKOUT_COUNTER = TPM2_PT_VAR + 14, + TPM2_PT_MAX_AUTH_FAIL = TPM2_PT_VAR + 15, + TPM2_PT_LOCKOUT_INTERVAL = TPM2_PT_VAR + 16, + TPM2_PT_LOCKOUT_RECOVERY = TPM2_PT_VAR + 17, + TPM2_PT_NV_WRITE_RECOVERY = TPM2_PT_VAR + 18, + TPM2_PT_AUDIT_COUNTER_0 = TPM2_PT_VAR + 19, + TPM2_PT_AUDIT_COUNTER_1 = TPM2_PT_VAR + 20, +}; + +struct tpm2_pcr_read_out { + __be32 update_cnt; + __be32 pcr_selects_cnt; + __be16 hash_alg; + u8 pcr_select_size; + u8 pcr_select[TPM2_PCR_SELECT_MIN]; + __be32 digests_cnt; + __be16 digest_size; + u8 digest[]; +} __packed; + +struct tpm2_get_random_out { + __be16 size; + u8 buffer[TPM_MAX_RNG_DATA]; +} __packed; + +struct tpm2_get_cap_out { + u8 more_data; + __be32 subcap_id; + __be32 property_cnt; + __be32 property_id; + __be32 value; +} __packed; + +struct tpm2_pcr_selection { + __be16 hash_alg; + u8 size_of_select; + u8 pcr_select[3]; +} __packed; + +struct tpm2_context { + __be64 sequence; + __be32 saved_handle; + __be32 hierarchy; + __be16 blob_size; +} __packed; + +/* + * == TPM Common Defs == + */ + +#define TPM_DIGEST_SIZE 20 /* Max TPM v1.2 PCR size */ +#define TPM_BUFSIZE 4096 + +/* + * SHA-512 is, as of today, the largest digest in the TCG algorithm repository. + */ +#define TPM2_MAX_DIGEST_SIZE SHA512_DIGEST_SIZE + +/* + * A TPM name digest i.e., TPMT_HA, is a concatenation of TPM_ALG_ID of the + * name algorithm and hash of TPMT_PUBLIC. + */ +#define TPM2_MAX_NAME_SIZE (TPM2_MAX_DIGEST_SIZE + 2) + +/* + * Fixed define for the size of a name. This is actually HASHALG size + * plus 2, so 32 for SHA256 + */ +#define TPM2_NULL_NAME_SIZE 34 + +/* + * The maximum number of PCR banks. + */ +#define TPM2_MAX_PCR_BANKS 8 + +/* If you add a new hash to this, increment TPM_MAX_HASHES below */ +enum tpm_algorithms { + TPM_ALG_ERROR = 0x0000, + TPM_ALG_SHA1 = 0x0004, + TPM_ALG_AES = 0x0006, + TPM_ALG_KEYEDHASH = 0x0008, + TPM_ALG_SHA256 = 0x000B, + TPM_ALG_SHA384 = 0x000C, + TPM_ALG_SHA512 = 0x000D, + TPM_ALG_NULL = 0x0010, + TPM_ALG_SM3_256 = 0x0012, + TPM_ALG_ECC = 0x0023, + TPM_ALG_CFB = 0x0043, +}; + +/* + * The locality (0 - 4) for a TPM, as defined in section 3.2 of the + * Client Platform Profile Specification. + */ +enum tpm_localities { + TPM_LOCALITY_0 = 0, /* Static RTM */ + TPM_LOCALITY_1 = 1, /* Dynamic OS */ + TPM_LOCALITY_2 = 2, /* DRTM Environment */ + TPM_LOCALITY_3 = 3, /* Aux Components */ + TPM_LOCALITY_4 = 4, /* CPU DRTM Establishment */ + TPM_MAX_LOCALITY = TPM_LOCALITY_4 +}; + +/* + * Structure to represent active PCR algorithm banks usable by the + * TPM chip. + */ +struct tpm_bank_info { + u16 alg_id; + u16 digest_size; + u16 crypto_id; +}; + +/* + * Maximum number of hashing algorithms a TPM can have. This is + * basically a count of every hash in tpm_algorithms above + */ +#define TPM_MAX_HASHES 5 + +struct tpm_digest { + u16 alg_id; + u8 digest[TPM2_MAX_DIGEST_SIZE]; +} __packed; + +#define TPM_HEADER_SIZE 10 + +struct tpm_header { + __be16 tag; + __be32 length; + union { + __be32 ordinal; + __be32 return_code; + }; +} __packed; #endif diff --git a/include/linux/tpm_eventlog.h b/include/linux/tpm_eventlog.h index 891368e82558..40fe92eb7deb 100644 --- a/include/linux/tpm_eventlog.h +++ b/include/linux/tpm_eventlog.h @@ -3,7 +3,7 @@ #ifndef __LINUX_TPM_EVENTLOG_H__ #define __LINUX_TPM_EVENTLOG_H__ -#include <linux/tpm.h> +#include <linux/tpm_command.h> #define TCG_EVENT_NAME_LEN_MAX 255 #define MAX_TEXT_EVENT 1000 /* Max event string length */ @@ -131,11 +131,16 @@ struct tcg_algorithm_info { }; #ifndef TPM_MEMREMAP -#define TPM_MEMREMAP(start, size) NULL +static inline void *TPM_MEMREMAP(unsigned long start, size_t size) +{ + return NULL; +} #endif #ifndef TPM_MEMUNMAP -#define TPM_MEMUNMAP(start, size) do{} while(0) +static inline void TPM_MEMUNMAP(void *mapping, size_t size) +{ +} #endif /** diff --git a/include/linux/tpm_ptp.h b/include/linux/tpm_ptp.h new file mode 100644 index 000000000000..e80ed810a269 --- /dev/null +++ b/include/linux/tpm_ptp.h @@ -0,0 +1,133 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Following copyright information was take from the original file + * <drivers/char/tpm/tpm_tis_core.h> where the definitions were moved + * from: + * + * Copyright (C) 2005, 2006 IBM Corporation + * Copyright (C) 2014, 2015 Intel Corporation + * + * Authors: + * Leendert van Doorn <leendert@watson.ibm.com> + * Kylene Hall <kjhall@us.ibm.com> + * + * Maintained by: <tpmdd-devel@lists.sourceforge.net> + * + * Device driver for TCG/TCPA TPM (trusted platform module). + * Specifications at www.trustedcomputinggroup.org + * + * This device driver implements the TPM interface as defined in + * the TCG TPM Interface Spec version 1.2, revision 1.0. + */ + +#ifndef __LINUX_TPM_PTP_H__ +#define __LINUX_TPM_PTP_H__ + +/* + * TCG PC Client Platform TPM Profile (PTP) Specification + * https://trustedcomputinggroup.org/resource/pc-client-platform-tpm-profile-ptp-specification/ + */ + +/* TIS/FIFO macros and definitions */ + +enum tis_access { + TPM_ACCESS_VALID = 0x80, + TPM_ACCESS_ACTIVE_LOCALITY = 0x20, /* (R) */ + TPM_ACCESS_RELINQUISH_LOCALITY = 0x20, /* (W) */ + TPM_ACCESS_REQUEST_PENDING = 0x04, /* (W) */ + TPM_ACCESS_REQUEST_USE = 0x02, /* (W) */ +}; + +enum tis_status { + TPM_STS_VALID = 0x80, /* (R) */ + TPM_STS_COMMAND_READY = 0x40, /* (R) */ + TPM_STS_DATA_AVAIL = 0x10, /* (R) */ + TPM_STS_DATA_EXPECT = 0x08, /* (R) */ + TPM_STS_GO = 0x20, /* (W) */ + TPM_STS_RESPONSE_RETRY = 0x02, /* (R) */ + TPM_STS_READ_ZERO = 0x23, /* bits that must be zero on read */ +}; + +enum tis_int_flags { + TPM_GLOBAL_INT_ENABLE = 0x80000000, + TPM_INTF_BURST_COUNT_STATIC = 0x100, + TPM_INTF_CMD_READY_INT = 0x080, + TPM_INTF_INT_EDGE_FALLING = 0x040, + TPM_INTF_INT_EDGE_RISING = 0x020, + TPM_INTF_INT_LEVEL_LOW = 0x010, + TPM_INTF_INT_LEVEL_HIGH = 0x008, + TPM_INTF_LOCALITY_CHANGE_INT = 0x004, + TPM_INTF_STS_VALID_INT = 0x002, + TPM_INTF_DATA_AVAIL_INT = 0x001, +}; + +enum tis_defaults { + TIS_MEM_LEN = 0x5000, + TIS_SHORT_TIMEOUT = 750, /* ms */ + TIS_LONG_TIMEOUT = 4000, /* 4 secs */ + TIS_TIMEOUT_MIN_ATML = 14700, /* usecs */ + TIS_TIMEOUT_MAX_ATML = 15000, /* usecs */ +}; + +#define TIS_MEM_X86_LPC_BASE 0xFED40000 +#define INTEL_LEGACY_BLK_BASE_ADDR 0xFED08000 + +enum tis_x86_defaults { + TIS_MEM_X86_LEN = 0x5000, + ILB_REMAP_SIZE = 0x100, + LPC_CNTRL_OFFSET = 0x84, + LPC_CLKRUN_EN = (1 << 2), +}; + +/* + * Some timeout values are needed before it is known whether the chip is + * TPM 1.0 or TPM 2.0. + */ +#define TIS_TIMEOUT_A_MAX max_t(int, TIS_SHORT_TIMEOUT, TPM2_TIMEOUT_A) +#define TIS_TIMEOUT_B_MAX max_t(int, TIS_LONG_TIMEOUT, TPM2_TIMEOUT_B) +#define TIS_TIMEOUT_C_MAX max_t(int, TIS_SHORT_TIMEOUT, TPM2_TIMEOUT_C) +#define TIS_TIMEOUT_D_MAX max_t(int, TIS_SHORT_TIMEOUT, TPM2_TIMEOUT_D) + +#define TPM_ACCESS(l) (0x0000 | ((l) << 12)) +#define TPM_INT_ENABLE(l) (0x0008 | ((l) << 12)) +#define TPM_INT_VECTOR(l) (0x000C | ((l) << 12)) +#define TPM_INT_STATUS(l) (0x0010 | ((l) << 12)) +#define TPM_INTF_CAPS(l) (0x0014 | ((l) << 12)) +#define TPM_STS(l) (0x0018 | ((l) << 12)) +#define TPM_STS3(l) (0x001b | ((l) << 12)) +#define TPM_DATA_FIFO(l) (0x0024 | ((l) << 12)) +#define TPM_INTF_ID(l) (0x0030 | ((l) << 12)) + +#define TPM_DID_VID(l) (0x0F00 | ((l) << 12)) +#define TPM_RID(l) (0x0F04 | ((l) << 12)) + +/* TPM HW Interface and Capabilities */ +#define TPM_TIS_INTF_ACTIVE 0x00 +#define TPM_CRB_INTF_ACTIVE 0x01 + +#define TPM_INTID_INTERFACE_TYPE GENMASK(3, 0) +#define TPM_INTID_INTERFACE_VERSION GENMASK(7, 4) +#define TPM_INTID_CAP_LOCALITY BIT(8) +#define TPM_INTID_CAP_TIS BIT(13) +#define TPM_INTID_CAP_CRB BIT(14) +#define TPM_INTID_CAP_IF_RES GENMASK(16, 15) +#define TPM_INTID_INTERFACE_SELECTOR GENMASK(18, 17) +#define TPM_INTID_INTF_SEL_LOCK BIT(19) + +#define TPM_TIS_INTF_12 0x00 +#define TPM_TIS_INTF_13 0x02 +#define TPM2_TIS_INTF_13 0x03 + +#define TPM_INTF_DATA_AVAIL_INT_SUPPORT BIT(0) +#define TPM_INTF_STS_VALID_INT_SUPPORT BIT(1) +#define TPM_INTF_LOCALITY_CHANGE_INT_SUPPORT BIT(2) +#define TPM_INTF_INTERRUPT_LEVEL_HIGH BIT(3) +#define TPM_INTF_INTERRUPT_LEVEL_LOW BIT(4) +#define TPM_INTF_INTERRUPT_EDGE_RISING BIT(5) +#define TPM_INTF_INTERRUPT_EDGE_FALLING BIT(6) +#define TPM_INTF_COMMAND_READY_INT_SUPPORT BIT(7) +#define TPM_INTF_BURST_COUNT_STATIC BIT(8) +#define TPM_INTF_DATA_TRANSFER_SIZE_SUPPORT GENMASK(10, 9) +#define TPM_INTF_INTERFACE_VERSION GENMASK(30, 28) + +#endif diff --git a/include/linux/trace_events.h b/include/linux/trace_events.h index 37eb2f0f3dd8..5cbd09c8be8d 100644 --- a/include/linux/trace_events.h +++ b/include/linux/trace_events.h @@ -22,20 +22,23 @@ union bpf_attr; const char *trace_print_flags_seq(struct trace_seq *p, const char *delim, unsigned long flags, - const struct trace_print_flags *flag_array); + const struct trace_print_flags *flag_array, + size_t flag_array_size); const char *trace_print_symbols_seq(struct trace_seq *p, unsigned long val, - const struct trace_print_flags *symbol_array); + const struct trace_print_flags *symbol_array, + size_t symbol_array_size); #if BITS_PER_LONG == 32 const char *trace_print_flags_seq_u64(struct trace_seq *p, const char *delim, unsigned long long flags, - const struct trace_print_flags_u64 *flag_array); + const struct trace_print_flags_u64 *flag_array, + size_t flag_array_size); const char *trace_print_symbols_seq_u64(struct trace_seq *p, unsigned long long val, - const struct trace_print_flags_u64 - *symbol_array); + const struct trace_print_flags_u64 *symbol_array, + size_t symbol_array_size); #endif struct trace_iterator; @@ -295,6 +298,15 @@ struct trace_event_class { struct list_head *(*get_fields)(struct trace_event_call *); struct list_head fields; int (*raw_init)(struct trace_event_call *); +#ifdef CONFIG_BPF_EVENTS + /* + * Per-template BTF ids set by DECLARE_EVENT_CLASS via BTF_ID() and + * patched by resolve_btfids at link time. NULL for handcrafted classes. + * [0] FUNC __bpf_trace_<template> + * [1] STRUCT trace_event_raw_<template> + */ + const u32 *btf_ids; +#endif }; extern int trace_event_reg(struct trace_event_call *event, @@ -767,6 +779,7 @@ trace_trigger_soft_disabled(struct trace_event_file *file) #ifdef CONFIG_BPF_EVENTS unsigned int trace_call_bpf(struct trace_event_call *call, void *ctx); +unsigned int trace_call_bpf_faultable(struct trace_event_call *call, void *ctx); int perf_event_attach_bpf_prog(struct perf_event *event, struct bpf_prog *prog, u64 bpf_cookie); void perf_event_detach_bpf_prog(struct perf_event *event); int perf_event_query_prog_array(struct perf_event *event, void __user *info); @@ -783,12 +796,18 @@ int bpf_get_perf_event_info(const struct perf_event *event, u32 *prog_id, unsigned long *missed); int bpf_kprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *prog); int bpf_uprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *prog); +int bpf_tracing_multi_attach(struct bpf_prog *prog, const union bpf_attr *attr); #else static inline unsigned int trace_call_bpf(struct trace_event_call *call, void *ctx) { return 1; } +static inline unsigned int trace_call_bpf_faultable(struct trace_event_call *call, void *ctx) +{ + return 1; +} + static inline int perf_event_attach_bpf_prog(struct perf_event *event, struct bpf_prog *prog, u64 bpf_cookie) { @@ -835,6 +854,11 @@ bpf_uprobe_multi_link_attach(const union bpf_attr *attr, struct bpf_prog *prog) { return -EOPNOTSUPP; } +static inline int +bpf_tracing_multi_attach(struct bpf_prog *prog, const union bpf_attr *attr) +{ + return -EOPNOTSUPP; +} #endif enum { diff --git a/include/linux/trace_printk.h b/include/linux/trace_printk.h index bb5874097f24..3d54f440dccf 100644 --- a/include/linux/trace_printk.h +++ b/include/linux/trace_printk.h @@ -86,8 +86,7 @@ do { \ #define trace_printk(fmt, ...) \ do { \ - char _______STR[] = __stringify((__VA_ARGS__)); \ - if (sizeof(_______STR) > 3) \ + if (sizeof __stringify((__VA_ARGS__)) > 3) \ do_trace_printk(fmt, ##__VA_ARGS__); \ else \ trace_puts(fmt); \ @@ -107,7 +106,6 @@ do { \ __trace_printk(_THIS_IP_, fmt, ##args); \ } while (0) -extern __printf(2, 3) int __trace_bprintk(unsigned long ip, const char *fmt, ...); extern __printf(2, 3) diff --git a/include/linux/trace_remote.h b/include/linux/trace_remote.h new file mode 100644 index 000000000000..fcd1d46ea466 --- /dev/null +++ b/include/linux/trace_remote.h @@ -0,0 +1,48 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef _LINUX_TRACE_REMOTE_H +#define _LINUX_TRACE_REMOTE_H + +#include <linux/dcache.h> +#include <linux/ring_buffer.h> +#include <linux/trace_remote_event.h> + +/** + * struct trace_remote_callbacks - Callbacks used by Tracefs to control the remote + * @init: Called once the remote has been registered. Allows the + * caller to extend the Tracefs remote directory + * @load_trace_buffer: Called before Tracefs accesses the trace buffer for the first + * time. Must return a &trace_buffer_desc + * (most likely filled with trace_remote_alloc_buffer()) + * @unload_trace_buffer: + * Called once Tracefs has no use for the trace buffer + * (most likely call trace_remote_free_buffer()) + * @enable_tracing: Called on Tracefs tracing_on. It is expected from the + * remote to allow writing. + * @swap_reader_page: Called when Tracefs consumes a new page from a + * ring-buffer. It is expected from the remote to isolate a + * @reset: Called on `echo 0 > trace`. It is expected from the + * remote to reset all ring-buffer pages. + * new reader-page from the @cpu ring-buffer. + * @enable_event: Called on events/event_name/enable. It is expected from + * the remote to allow the writing event @id. + */ +struct trace_remote_callbacks { + int (*init)(struct dentry *d, void *priv); + struct trace_buffer_desc *(*load_trace_buffer)(unsigned long size, void *priv); + void (*unload_trace_buffer)(struct trace_buffer_desc *desc, void *priv); + int (*enable_tracing)(bool enable, void *priv); + int (*swap_reader_page)(unsigned int cpu, void *priv); + int (*reset)(unsigned int cpu, void *priv); + int (*enable_event)(unsigned short id, bool enable, void *priv); +}; + +int trace_remote_register(const char *name, struct trace_remote_callbacks *cbs, void *priv, + struct remote_event *events, size_t nr_events); + +int trace_remote_alloc_buffer(struct trace_buffer_desc *desc, size_t desc_size, size_t buffer_size, + const struct cpumask *cpumask); + +void trace_remote_free_buffer(struct trace_buffer_desc *desc); + +#endif diff --git a/include/linux/trace_remote_event.h b/include/linux/trace_remote_event.h new file mode 100644 index 000000000000..c8ae1e1f5e72 --- /dev/null +++ b/include/linux/trace_remote_event.h @@ -0,0 +1,33 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef _LINUX_TRACE_REMOTE_EVENTS_H +#define _LINUX_TRACE_REMOTE_EVENTS_H + +struct trace_remote; +struct trace_event_fields; +struct trace_seq; + +struct remote_event_hdr { + unsigned short id; +}; + +#define REMOTE_EVENT_NAME_MAX 30 +struct remote_event { + char name[REMOTE_EVENT_NAME_MAX]; + unsigned short id; + bool enabled; + struct trace_remote *remote; + struct trace_event_fields *fields; + char *print_fmt; + void (*print)(void *evt, struct trace_seq *seq); +}; + +#define RE_STRUCT(__args...) __args +#define re_field(__type, __field) __type __field; + +#define REMOTE_EVENT_FORMAT(__name, __struct) \ + struct remote_event_format_##__name { \ + struct remote_event_hdr hdr; \ + __struct \ + } +#endif diff --git a/include/linux/tracefs.h b/include/linux/tracefs.h index d03f74658716..bc354d340046 100644 --- a/include/linux/tracefs.h +++ b/include/linux/tracefs.h @@ -30,7 +30,7 @@ struct eventfs_file; * @data: data to pass to the created file ops * @fops: the file operations of the created file * - * The evetnfs files are dynamically created. The struct eventfs_entry array + * The eventfs files are dynamically created. The struct eventfs_entry array * is passed to eventfs_create_dir() or eventfs_create_events_dir() that will * be used to create the files within those directories. When a lookup * or access to a file within the directory is made, the struct eventfs_entry diff --git a/include/linux/tracepoint.h b/include/linux/tracepoint.h index 22ca1c8b54f3..e0d838c9ce93 100644 --- a/include/linux/tracepoint.h +++ b/include/linux/tracepoint.h @@ -20,6 +20,7 @@ #include <linux/rcupdate_trace.h> #include <linux/tracepoint-defs.h> #include <linux/static_call.h> +#include <linux/cfi.h> struct module; struct tracepoint; @@ -122,6 +123,22 @@ static inline bool tracepoint_is_faultable(struct tracepoint *tp) { return tp->ext && tp->ext->faultable; } +/* + * Run RCU callback with the appropriate grace period wait for non-faultable + * tracepoints, e.g., those used in atomic context. + */ +static inline void call_tracepoint_unregister_atomic(struct rcu_head *rcu, rcu_callback_t func) +{ + call_srcu(&tracepoint_srcu, rcu, func); +} +/* + * Run RCU callback with the appropriate grace period wait for faultable + * tracepoints, e.g., those used in syscall context. + */ +static inline void call_tracepoint_unregister_syscall(struct rcu_head *rcu, rcu_callback_t func) +{ + call_rcu_tasks_trace(rcu, func); +} #else static inline void tracepoint_synchronize_unregister(void) { } @@ -129,6 +146,10 @@ static inline bool tracepoint_is_faultable(struct tracepoint *tp) { return false; } +static inline void call_tracepoint_unregister_atomic(struct rcu_head *rcu, rcu_callback_t func) +{ } +static inline void call_tracepoint_unregister_syscall(struct rcu_head *rcu, rcu_callback_t func) +{ } #endif #ifdef CONFIG_HAVE_SYSCALL_TRACEPOINTS @@ -182,7 +203,7 @@ static inline struct tracepoint *tracepoint_ptr_deref(tracepoint_ptr_t *p) #define TP_CONDITION(args...) args /* - * Individual subsystem my have a separate configuration to + * Individual subsystem may have a separate configuration to * enable their tracepoints. By default, this file will create * the tracepoints if CONFIG_TRACEPOINTS is defined. If a subsystem * wants to be able to disable its tracepoints from being created @@ -271,9 +292,18 @@ static inline struct tracepoint *tracepoint_ptr_deref(tracepoint_ptr_t *p) { \ } \ static inline bool \ - trace_##name##_enabled(void) \ + __trace_##name##_enabled(void) \ { \ return static_branch_unlikely(&__tracepoint_##name.key);\ + } \ + static inline bool \ + trace_##name##_enabled(void) \ + { \ + if (IS_ENABLED(CONFIG_LOCKDEP)) { \ + WARN_ONCE(!rcu_is_watching(), \ + "RCU not watching for tracepoint"); \ + } \ + return __trace_##name##_enabled(); \ } #define __DECLARE_TRACE(name, proto, args, cond, data_proto) \ @@ -294,6 +324,10 @@ static inline struct tracepoint *tracepoint_ptr_deref(tracepoint_ptr_t *p) WARN_ONCE(!rcu_is_watching(), \ "RCU not watching for tracepoint"); \ } \ + } \ + static inline void trace_call__##name(proto) \ + { \ + __do_trace_##name(args); \ } #define __DECLARE_TRACE_SYSCALL(name, proto, args, data_proto) \ @@ -313,6 +347,11 @@ static inline struct tracepoint *tracepoint_ptr_deref(tracepoint_ptr_t *p) WARN_ONCE(!rcu_is_watching(), \ "RCU not watching for tracepoint"); \ } \ + } \ + static inline void trace_call__##name(proto) \ + { \ + might_fault(); \ + __do_trace_##name(args); \ } /* @@ -360,6 +399,13 @@ static inline struct tracepoint *tracepoint_ptr_deref(tracepoint_ptr_t *p) void __probestub_##_name(void *__data, proto) \ { \ } \ + /* \ + * Annotate the probestub 'CFI_NOSEAL' to stop objtool from \ + * requesting the kernel remove the ENDBR, because the only \ + * references to the function are in the __tracepoint section, \ + * that objtool doesn't scan. \ + */ \ + CFI_NOSEAL(__probestub_##_name); \ DEFINE_STATIC_CALL(tp_func_##_name, __traceiter_##_name); \ DEFINE_RUST_DO_TRACE(_name, TP_PROTO(proto), TP_ARGS(args)) @@ -398,6 +444,8 @@ static inline struct tracepoint *tracepoint_ptr_deref(tracepoint_ptr_t *p) #define __DECLARE_TRACE_COMMON(name, proto, args, data_proto) \ static inline void trace_##name(proto) \ { } \ + static inline void trace_call__##name(proto) \ + { } \ static inline int \ register_trace_##name(void (*probe)(data_proto), \ void *data) \ @@ -414,6 +462,11 @@ static inline struct tracepoint *tracepoint_ptr_deref(tracepoint_ptr_t *p) { \ } \ static inline bool \ + __trace_##name##_enabled(void) \ + { \ + return false; \ + } \ + static inline bool \ trace_##name##_enabled(void) \ { \ return false; \ diff --git a/include/linux/tty_buffer.h b/include/linux/tty_buffer.h index 31125e3be3c5..48adcb0e8ff3 100644 --- a/include/linux/tty_buffer.h +++ b/include/linux/tty_buffer.h @@ -34,6 +34,7 @@ static inline u8 *flag_buf_ptr(struct tty_buffer *b, unsigned int ofs) struct tty_bufhead { struct tty_buffer *head; /* Queue head */ + struct workqueue_struct *flip_wq; struct work_struct work; struct mutex lock; atomic_t priority; diff --git a/include/linux/tty_driver.h b/include/linux/tty_driver.h index 188ee9b768eb..1f2896e56e77 100644 --- a/include/linux/tty_driver.h +++ b/include/linux/tty_driver.h @@ -69,6 +69,10 @@ struct serial_struct; * Do not create numbered ``/dev`` nodes. For example, create * ``/dev/ttyprintk`` and not ``/dev/ttyprintk0``. Applicable only when a * driver for a single tty device is being allocated. + * + * @TTY_DRIVER_NO_WORKQUEUE: + * Do not create workqueue when tty_register_driver(). Whenever set, flip + * buffer workqueue can be set by tty_port_link_wq() for every port. */ enum tty_driver_flag { TTY_DRIVER_INSTALLED = BIT(0), @@ -79,6 +83,7 @@ enum tty_driver_flag { TTY_DRIVER_HARDWARE_BREAK = BIT(5), TTY_DRIVER_DYNAMIC_ALLOC = BIT(6), TTY_DRIVER_UNNUMBERED_NODE = BIT(7), + TTY_DRIVER_NO_WORKQUEUE = BIT(8), }; enum tty_driver_type { @@ -506,6 +511,7 @@ struct tty_operations { * @flags: tty driver flags (%TTY_DRIVER_) * @proc_entry: proc fs entry, used internally * @other: driver of the linked tty; only used for the PTY driver + * @flip_wq: workqueue to queue flip buffer work on * @ttys: array of active &struct tty_struct, set by tty_standard_install() * @ports: array of &struct tty_port; can be set during initialization by * tty_port_link_device() and similar @@ -539,6 +545,7 @@ struct tty_driver { unsigned long flags; struct proc_dir_entry *proc_entry; struct tty_driver *other; + struct workqueue_struct *flip_wq; /* * Pointer to the tty data structures diff --git a/include/linux/tty_ldisc.h b/include/linux/tty_ldisc.h index c5cccc3fc1e8..d227a58e3e49 100644 --- a/include/linux/tty_ldisc.h +++ b/include/linux/tty_ldisc.h @@ -266,7 +266,7 @@ struct tty_ldisc_ops { }; struct tty_ldisc { - struct tty_ldisc_ops *ops; + const struct tty_ldisc_ops *ops; struct tty_struct *tty; }; @@ -281,8 +281,8 @@ struct tty_ldisc *tty_ldisc_ref_wait(struct tty_struct *); void tty_ldisc_flush(struct tty_struct *tty); -int tty_register_ldisc(struct tty_ldisc_ops *new_ldisc); -void tty_unregister_ldisc(struct tty_ldisc_ops *ldisc); +int tty_register_ldisc(const struct tty_ldisc_ops *new_ldisc); +void tty_unregister_ldisc(const struct tty_ldisc_ops *ldisc); int tty_set_ldisc(struct tty_struct *tty, int disc); #endif /* _LINUX_TTY_LDISC_H */ diff --git a/include/linux/tty_port.h b/include/linux/tty_port.h index 660c254f1efe..23cad403bb8f 100644 --- a/include/linux/tty_port.h +++ b/include/linux/tty_port.h @@ -6,10 +6,10 @@ #include <linux/kref.h> #include <linux/mutex.h> #include <linux/tty_buffer.h> +#include <linux/tty_driver.h> #include <linux/wait.h> struct attribute_group; -struct tty_driver; struct tty_port; struct tty_struct; @@ -138,6 +138,7 @@ struct tty_port { kernel */ void tty_port_init(struct tty_port *port); +void tty_port_link_wq(struct tty_port *port, struct workqueue_struct *flip_wq); void tty_port_link_device(struct tty_port *port, struct tty_driver *driver, unsigned index); struct device *tty_port_register_device(struct tty_port *port, @@ -165,6 +166,18 @@ static inline struct tty_port *tty_port_get(struct tty_port *port) return NULL; } +/* + * Never overwrite the workqueue set by tty_port_link_wq(). + * No effect when %TTY_DRIVER_NO_WORKQUEUE is set, as driver->flip_wq is + * %NULL. + */ +static inline void tty_port_link_driver_wq(struct tty_port *port, + struct tty_driver *driver) +{ + if (!port->buf.flip_wq) + tty_port_link_wq(port, driver->flip_wq); +} + /* If the cts flow control is enabled, return true. */ static inline bool tty_port_cts_enabled(const struct tty_port *port) { diff --git a/include/linux/types.h b/include/linux/types.h index 7e71d260763c..53e0adca4b9f 100644 --- a/include/linux/types.h +++ b/include/linux/types.h @@ -163,6 +163,8 @@ typedef u32 dma_addr_t; typedef unsigned int __bitwise gfp_t; typedef unsigned int __bitwise slab_flags_t; typedef unsigned int __bitwise fmode_t; +typedef unsigned int __bitwise blk_mode_t; +typedef unsigned int __bitwise fop_flags_t; #ifdef CONFIG_PHYS_ADDR_T_64BIT typedef u64 phys_addr_t; @@ -224,6 +226,12 @@ struct ustat { char f_fpack[6]; }; +struct kcov_common_handle_id { +#ifdef CONFIG_KCOV + u64 val; +#endif +}; + /** * struct callback_head - callback structure for use with RCU and task_work * @next: next update requests in a list @@ -239,7 +247,7 @@ struct ustat { * * This guarantee is important for few reasons: * - future call_rcu_lazy() will make use of lower bits in the pointer; - * - the structure shares storage space in struct page with @compound_head, + * - the structure shares storage space in struct page with @compound_info, * which encode PageTail() in bit 0. The guarantee is needed to avoid * false-positive PageTail(). */ @@ -249,6 +257,16 @@ struct callback_head { } __attribute__((aligned(sizeof(void *)))); #define rcu_head callback_head +#ifdef CONFIG_KVFREE_RCU_BATCHED +struct kvfree_rcu_head { + struct kvfree_rcu_head *next; +}; +#else +struct kvfree_rcu_head { + struct rcu_head head; +}; +#endif + typedef void (*rcu_callback_t)(struct rcu_head *head); typedef void (*call_rcu_func_t)(struct rcu_head *head, rcu_callback_t func); diff --git a/include/linux/uaccess.h b/include/linux/uaccess.h index 4fe63169d5a2..eddbbb65ccc4 100644 --- a/include/linux/uaccess.h +++ b/include/linux/uaccess.h @@ -84,7 +84,7 @@ * the 6 functions (copy_{to,from}_user(), __copy_{to,from}_user_inatomic()) * that are used instead. Out of those, __... ones are inlined. Plain * copy_{to,from}_user() might or might not be inlined. If you want them - * inlined, have asm/uaccess.h define INLINE_COPY_{TO,FROM}_USER. + * inlined, have asm/uaccess.h define INLINE_COPY_USER. * * NOTE: only copy_from_user() zero-pads the destination in case of short copy. * Neither __copy_from_user() nor __copy_from_user_inatomic() zero anything @@ -157,7 +157,7 @@ __copy_to_user(void __user *to, const void *from, unsigned long n) } /* - * Architectures that #define INLINE_COPY_TO_USER use this function + * Architectures that #define INLINE_COPY_USER use this function * directly in the normal copy_to/from_user(), the other ones go * through an extern _copy_to/from_user(), which expands the same code * here. @@ -190,10 +190,6 @@ fail: memset(to + (n - res), 0, res); return res; } -#ifndef INLINE_COPY_FROM_USER -extern __must_check unsigned long -_copy_from_user(void *, const void __user *, unsigned long); -#endif static inline __must_check unsigned long _inline_copy_to_user(void __user *to, const void *from, unsigned long n) @@ -207,7 +203,13 @@ _inline_copy_to_user(void __user *to, const void *from, unsigned long n) } return n; } -#ifndef INLINE_COPY_TO_USER +#ifdef INLINE_COPY_USER +# define _copy_to_user _inline_copy_to_user +# define _copy_from_user _inline_copy_from_user +#else +extern __must_check unsigned long +_copy_from_user(void *, const void __user *, unsigned long); + extern __must_check unsigned long _copy_to_user(void __user *, const void *, unsigned long); #endif @@ -217,11 +219,7 @@ copy_from_user(void *to, const void __user *from, unsigned long n) { if (!check_copy_size(to, n, false)) return n; -#ifdef INLINE_COPY_FROM_USER - return _inline_copy_from_user(to, from, n); -#else return _copy_from_user(to, from, n); -#endif } static __always_inline unsigned long __must_check @@ -229,12 +227,7 @@ copy_to_user(void __user *to, const void *from, unsigned long n) { if (!check_copy_size(from, n, true)) return n; - -#ifdef INLINE_COPY_TO_USER - return _inline_copy_to_user(to, from, n); -#else return _copy_to_user(to, from, n); -#endif } #ifndef copy_mc_to_kernel @@ -331,16 +324,21 @@ static inline size_t probe_subpage_writeable(char __user *uaddr, size_t size) #endif /* CONFIG_ARCH_HAS_SUBPAGE_FAULTS */ -#ifndef ARCH_HAS_NOCACHE_UACCESS +#ifndef ARCH_HAS_NONTEMPORAL_UACCESS static inline __must_check unsigned long -__copy_from_user_inatomic_nocache(void *to, const void __user *from, +copy_from_user_inatomic_nontemporal(void *to, const void __user *from, unsigned long n) { + if (can_do_masked_user_access()) + from = mask_user_address(from); + else + if (!access_ok(from, n)) + return n; return __copy_from_user_inatomic(to, from, n); } -#endif /* ARCH_HAS_NOCACHE_UACCESS */ +#endif /* ARCH_HAS_NONTEMPORAL_UACCESS */ extern __must_check int check_zeroed_user(const void __user *from, size_t size); @@ -505,7 +503,7 @@ copy_struct_to_user(void __user *dst, size_t usize, const void *src, return -EFAULT; } if (ignored_trailing) - *ignored_trailing = ksize < usize && + *ignored_trailing = usize < ksize && memchr_inv(src + size, 0, rest) != NULL; /* Copy the interoperable parts of the struct. */ if (copy_to_user(dst, src, size)) @@ -513,6 +511,69 @@ copy_struct_to_user(void __user *dst, size_t usize, const void *src, return 0; } +static __always_inline void +__copy_struct_generic_bounce_buffer(void *dst, size_t dstsize, + const void *src, size_t srcsize, + bool *ignored_trailing) +{ + size_t size = min(dstsize, srcsize); + size_t rest = max(dstsize, srcsize) - size; + + /* Deal with trailing bytes. */ + if (dstsize > srcsize) + memset(dst + size, 0, rest); + if (ignored_trailing) + *ignored_trailing = dstsize < srcsize && + memchr_inv(src + size, 0, rest) != NULL; + /* Copy the interoperable parts of the struct. */ + memcpy(dst, src, size); +} + +/** + * This is like copy_struct_from_user(), but the + * src buffer was already copied into a kernel + * bounce buffer, so it will never return -EFAULT. + */ +static __always_inline __must_check int +copy_struct_from_bounce_buffer(void *dst, size_t dstsize, + const void *src, size_t srcsize) +{ + bool ignored_trailing; + + /* Double check if ksize is larger than a known object size. */ + if (WARN_ON_ONCE(dstsize > __builtin_object_size(dst, 1))) + return -E2BIG; + + __copy_struct_generic_bounce_buffer(dst, dstsize, + src, srcsize, + &ignored_trailing); + if (unlikely(ignored_trailing)) + return -E2BIG; + + return 0; +} + +/** + * This is like copy_struct_to_user(), but the + * dst buffer is a kernel bounce buffer instead + * of a direct userspace buffer, so it will never return -EFAULT. + */ +static __always_inline __must_check int +copy_struct_to_bounce_buffer(void *dst, size_t dstsize, + const void *src, + size_t srcsize, + bool *ignored_trailing) +{ + /* Double check if srcsize is larger than a known object size. */ + if (WARN_ON_ONCE(srcsize > __builtin_object_size(src, 1))) + return -E2BIG; + + __copy_struct_generic_bounce_buffer(dst, dstsize, + src, srcsize, + ignored_trailing); + return 0; +} + bool copy_from_kernel_nofault_allowed(const void *unsafe_src, size_t size); long copy_from_kernel_nofault(void *dst, const void *src, size_t size); @@ -644,6 +705,17 @@ static inline void user_access_restore(unsigned long flags) { } #define user_read_access_end user_access_end #endif +#ifndef unsafe_atomic_store_release_user +# define unsafe_atomic_store_release_user(val, uptr, elbl) \ + do { \ + if (!IS_ENABLED(CONFIG_ARCH_MEMORY_ORDER_TSO)) \ + smp_mb(); \ + else \ + barrier(); \ + unsafe_put_user(val, uptr, elbl); \ + } while (0) +#endif + /* Define RW variant so the below _mode macro expansion works */ #define masked_user_rw_access_begin(u) masked_user_access_begin(u) #define user_rw_access_begin(u, s) user_access_begin(u, s) diff --git a/include/linux/udp.h b/include/linux/udp.h index 1cbf6b4d3aab..998906ec3b32 100644 --- a/include/linux/udp.h +++ b/include/linux/udp.h @@ -23,6 +23,33 @@ static inline struct udphdr *udp_hdr(const struct sk_buff *skb) return (struct udphdr *)skb_transport_header(skb); } +static inline unsigned int udp_get_len(const struct sk_buff *skb, + const struct udphdr *uh, + unsigned int dataoff) +{ + if (uh->len) + return ntohs(uh->len); + if (skb_is_gso(skb)) /* BIG TCP */ + return skb->len - dataoff; + return 0; +} + +static inline unsigned int udp_get_len_short(const struct udphdr *uh) +{ + return ntohs(uh->len); +} + +static inline void udp_set_len(struct udphdr *uh, unsigned int len) +{ + uh->len = len < GRO_LEGACY_MAX_SIZE ? htons(len) : 0; +} + +static inline void udp_set_len_short(struct udphdr *uh, unsigned int len) +{ + DEBUG_NET_WARN_ON_ONCE(len >= GRO_LEGACY_MAX_SIZE); + uh->len = htons(len); +} + #define UDP_HTABLE_SIZE_MIN_PERNET 128 #define UDP_HTABLE_SIZE_MIN (IS_ENABLED(CONFIG_BASE_SMALL) ? 128 : 256) #define UDP_HTABLE_SIZE_MAX 65536 @@ -40,8 +67,6 @@ enum { UDP_FLAGS_ACCEPT_FRAGLIST, UDP_FLAGS_ACCEPT_L4, UDP_FLAGS_ENCAP_ENABLED, /* This socket enabled encap */ - UDP_FLAGS_UDPLITE_SEND_CC, /* set via udplite setsockopt */ - UDP_FLAGS_UDPLITE_RECV_CC, /* set via udplite setsockopt */ }; /* per NUMA structure for lockless producer usage. */ @@ -74,11 +99,7 @@ struct udp_sock { */ __u16 len; /* total length of pending frames */ __u16 gso_size; - /* - * Fields specific to UDP-Lite. - */ - __u16 pcslen; - __u16 pcrlen; + /* * For encapsulation sockets. */ @@ -236,8 +257,6 @@ static inline void udp_allow_gso(struct sock *sk) hlist_nulls_for_each_entry_rcu(__up, node, list, udp_lrpa_node) #endif -#define IS_UDPLITE(__sk) (unlikely(__sk->sk_protocol == IPPROTO_UDPLITE)) - static inline struct sock *udp_tunnel_sk(const struct net *net, bool is_ipv6) { #if IS_ENABLED(CONFIG_NET_UDP_TUNNEL) diff --git a/include/linux/uio.h b/include/linux/uio.h index a9bc5b3067e3..fe2e985d74d2 100644 --- a/include/linux/uio.h +++ b/include/linux/uio.h @@ -389,9 +389,17 @@ ssize_t iov_iter_extract_pages(struct iov_iter *i, struct page ***pages, size_t maxsize, unsigned int maxpages, iov_iter_extraction_t extraction_flags, size_t *offset0); +/* + * Block-layer consumers (e.g. bio_iov_iter_get_pages()) require that the + * segments of an ITER_BVEC iterator are already aligned to the target device's + * DMA alignment, and forward them as-is. In-kernel users that build their own + * bvecs must not create sub-aligned segments; iov_iter_extract_bvecs() enforces + * the same for the segments it extracts via @mem_align_mask. + */ ssize_t iov_iter_extract_bvecs(struct iov_iter *iter, struct bio_vec *bv, size_t max_size, unsigned short *nr_vecs, - unsigned short max_vecs, iov_iter_extraction_t extraction_flags); + unsigned short max_vecs, unsigned mem_align_mask, + iov_iter_extraction_t extraction_flags); /** * iov_iter_extract_will_pin - Indicate how pages from the iterator will be retained diff --git a/include/linux/uio_driver.h b/include/linux/uio_driver.h index 334641e20fb1..02eaac47ac44 100644 --- a/include/linux/uio_driver.h +++ b/include/linux/uio_driver.h @@ -97,7 +97,7 @@ struct uio_device { * @irq_flags: flags for request_irq() * @priv: optional private data * @handler: the device's irq handler - * @mmap: mmap operation for this uio device + * @mmap_prepare: mmap_prepare operation for this uio device * @open: open operation for this uio device * @release: release operation for this uio device * @irqcontrol: disable/enable irqs when 0/1 is written to /dev/uioX @@ -112,7 +112,7 @@ struct uio_info { unsigned long irq_flags; void *priv; irqreturn_t (*handler)(int irq, struct uio_info *dev_info); - int (*mmap)(struct uio_info *info, struct vm_area_struct *vma); + int (*mmap_prepare)(struct uio_info *info, struct vm_area_desc *desc); int (*open)(struct uio_info *info, struct inode *inode); int (*release)(struct uio_info *info, struct inode *inode); int (*irqcontrol)(struct uio_info *info, s32 irq_on); diff --git a/include/linux/ulpi/driver.h b/include/linux/ulpi/driver.h index a8cb617a3028..c668d9c8d876 100644 --- a/include/linux/ulpi/driver.h +++ b/include/linux/ulpi/driver.h @@ -2,7 +2,7 @@ #ifndef __LINUX_ULPI_DRIVER_H #define __LINUX_ULPI_DRIVER_H -#include <linux/mod_devicetable.h> +#include <linux/device-id/ulpi.h> #include <linux/device.h> diff --git a/include/linux/unicode.h b/include/linux/unicode.h index 5e6b212a2aed..64fa44fe180c 100644 --- a/include/linux/unicode.h +++ b/include/linux/unicode.h @@ -66,9 +66,6 @@ int utf8_strncasecmp_folded(const struct unicode_map *um, const struct qstr *cf, const struct qstr *s1); -int utf8_normalize(const struct unicode_map *um, const struct qstr *str, - unsigned char *dest, size_t dlen); - int utf8_casefold(const struct unicode_map *um, const struct qstr *str, unsigned char *dest, size_t dlen); diff --git a/include/linux/units.h b/include/linux/units.h index 80d57c50b9e3..c6d78988613a 100644 --- a/include/linux/units.h +++ b/include/linux/units.h @@ -57,6 +57,9 @@ #define MICROWATT_PER_MILLIWATT 1000UL #define MICROWATT_PER_WATT 1000000UL +#define MICROJOULE_PER_JOULE 1000000UL +#define NANOJOULE_PER_JOULE 1000000000UL + #define BYTES_PER_KBIT (KILO / BITS_PER_BYTE) #define BYTES_PER_MBIT (MEGA / BITS_PER_BYTE) #define BYTES_PER_GBIT (GIGA / BITS_PER_BYTE) diff --git a/include/linux/uprobes.h b/include/linux/uprobes.h index f548fea2adec..d34dbc0fbbfe 100644 --- a/include/linux/uprobes.h +++ b/include/linux/uprobes.h @@ -25,6 +25,7 @@ struct mm_struct; struct inode; struct notifier_block; struct page; +struct srcu_ctr; /* * Allowed return values from uprobe consumer's handler callback @@ -106,7 +107,7 @@ enum hprobe_state { * underlying uprobe is not guaranteed anymore. __UPROBE_DEAD is just an * internal marker and is handled transparently by hprobe_fetch() helper. * - * When uprobe is SRCU-protected, we also record srcu_idx value, necessary for + * When uprobe is SRCU-protected, we also record srcu_scp value, necessary for * SRCU unlocking. * * See hprobe_expire() and hprobe_fetch() for details of race-free uprobe @@ -115,7 +116,7 @@ enum hprobe_state { */ struct hprobe { enum hprobe_state state; - int srcu_idx; + struct srcu_ctr __percpu *srcu_scp; struct uprobe *uprobe; }; @@ -186,9 +187,6 @@ struct xol_area; struct uprobes_state { struct xol_area *xol_area; -#ifdef CONFIG_X86_64 - struct hlist_head head_tramps; -#endif }; typedef int (*uprobe_write_verify_t)(struct page *page, unsigned long vaddr, @@ -238,8 +236,6 @@ extern void uprobe_handle_trampoline(struct pt_regs *regs); extern void *arch_uretprobe_trampoline(unsigned long *psize); extern unsigned long uprobe_get_trampoline_vaddr(void); extern void uprobe_copy_from_page(struct page *page, unsigned long vaddr, void *dst, int len); -extern void arch_uprobe_clear_state(struct mm_struct *mm); -extern void arch_uprobe_init_state(struct mm_struct *mm); extern void handle_syscall_uprobe(struct pt_regs *regs, unsigned long bp_vaddr); extern void arch_uprobe_optimize(struct arch_uprobe *auprobe, unsigned long vaddr); extern unsigned long arch_uprobe_get_xol_area(void); diff --git a/include/linux/usb.h b/include/linux/usb.h index 04277af4bb9d..49ab8dbb885f 100644 --- a/include/linux/usb.h +++ b/include/linux/usb.h @@ -2,7 +2,7 @@ #ifndef __LINUX_USB_H #define __LINUX_USB_H -#include <linux/mod_devicetable.h> +#include <linux/device-id/usb.h> #include <linux/usb/ch9.h> #define USB_MAJOR 180 @@ -21,6 +21,7 @@ #include <linux/completion.h> /* for struct completion */ #include <linux/sched.h> /* for current && schedule_timeout */ #include <linux/mutex.h> /* for struct mutex */ +#include <linux/spinlock.h> /* for spinlock_t */ #include <linux/pm_runtime.h> /* for runtime PM */ struct usb_device; @@ -54,7 +55,8 @@ struct ep_device; * @eusb2_isoc_ep_comp: eUSB2 isoc companion descriptor for this endpoint * @urb_list: urbs queued to this endpoint; maintained by usbcore * @hcpriv: for use by HCD; typically holds hardware dma queue head (QH) - * with one or more transfer descriptors (TDs) per urb + * with one or more transfer descriptors (TDs) per urb; must be preserved + * by core while BW is allocated for the endpoint * @ep_dev: ep_device for sysfs info * @extra: descriptors following this endpoint in the configuration * @extralen: how many bytes of "extra" are valid @@ -636,8 +638,9 @@ struct usb3_lpm_parameters { * @do_remote_wakeup: remote wakeup should be enabled * @reset_resume: needs reset instead of resume * @port_is_suspended: the upstream port is suspended (L2 or U3) - * @offload_at_suspend: offload activities during suspend is enabled. + * @offload_pm_locked: prevents offload_usage changes during PM transitions. * @offload_usage: number of offload activities happening on this usb device. + * @offload_lock: protects offload_usage and offload_pm_locked * @slot_id: Slot ID assigned by xHCI * @l1_params: best effor service latency for USB2 L1 LPM state, and L1 timeout. * @u1_params: exit latencies for USB3 U1 LPM state, and hub-initiated timeout. @@ -726,8 +729,9 @@ struct usb_device { unsigned do_remote_wakeup:1; unsigned reset_resume:1; unsigned port_is_suspended:1; - unsigned offload_at_suspend:1; + unsigned offload_pm_locked:1; int offload_usage; + spinlock_t offload_lock; enum usb_link_tunnel_mode tunnel_mode; struct device_link *usb4_link; @@ -849,6 +853,7 @@ static inline void usb_mark_last_busy(struct usb_device *udev) int usb_offload_get(struct usb_device *udev); int usb_offload_put(struct usb_device *udev); bool usb_offload_check(struct usb_device *udev); +void usb_offload_set_pm_locked(struct usb_device *udev, bool locked); #else static inline int usb_offload_get(struct usb_device *udev) @@ -857,6 +862,8 @@ static inline int usb_offload_put(struct usb_device *udev) { return 0; } static inline bool usb_offload_check(struct usb_device *udev) { return false; } +static inline void usb_offload_set_pm_locked(struct usb_device *udev, bool locked) +{ } #endif extern int usb_disable_lpm(struct usb_device *udev); @@ -1178,7 +1185,8 @@ extern ssize_t usb_show_dynids(struct usb_dynids *dynids, char *buf); * interface. It may also use usb_set_interface() to specify the * appropriate altsetting. If unwilling to manage the interface, * return -ENODEV, if genuine IO errors occurred, an appropriate - * negative errno value. + * negative errno value. The usb_device_id parameter is only valid during + * probe. * @disconnect: Called when the interface is no longer accessible, usually * because its device has been (or is being) disconnected or the * driver module is being unloaded. @@ -2075,6 +2083,8 @@ static inline int usb_translate_errors(int error_code) case -ENODEV: case -EOPNOTSUPP: return error_code; + case -ENOSPC: + return -EBUSY; default: return -EIO; } diff --git a/include/linux/usb/cdc_ncm.h b/include/linux/usb/cdc_ncm.h index 4ac082a63173..97ef37a1ff4a 100644 --- a/include/linux/usb/cdc_ncm.h +++ b/include/linux/usb/cdc_ncm.h @@ -118,8 +118,8 @@ struct cdc_ncm_ctx { u32 timer_interval; u32 max_ndp_size; - u8 is_ndp16; - u8 filtering_supported; + bool is_ndp16; + bool filtering_supported; union { struct usb_cdc_ncm_ndp16 *delayed_ndp16; struct usb_cdc_ncm_ndp32 *delayed_ndp32; diff --git a/include/linux/usb/mctp-usb.h b/include/linux/usb/mctp-usb.h index a2f6f1e04efb..4bb04a371105 100644 --- a/include/linux/usb/mctp-usb.h +++ b/include/linux/usb/mctp-usb.h @@ -2,7 +2,7 @@ /* * mctp-usb.h - MCTP USB transport binding: common definitions, * based on DMTF0283 specification: - * https://www.dmtf.org/sites/default/files/standards/documents/DSP0283_1.0.1.pdf + * https://www.dmtf.org/sites/default/files/standards/documents/DSP0283_1.1.0.pdf * * These are protocol-level definitions, that may be shared between host * and gadget drivers. @@ -13,18 +13,96 @@ #ifndef __LINUX_USB_MCTP_USB_H #define __LINUX_USB_MCTP_USB_H +#include <linux/netdevice.h> +#include <linux/skbuff.h> #include <linux/types.h> +/* + * MCTP-over-USB transport header. DSP0283 v1.0 has an 8-bit length field + * (preceded by 8 reserved bits), v1.1 has a 13-bit length field (preceded by + * 3 reserved bits). We use a be16 for our length to handle the larger v1.1 + * representation, and mask as appropriate. + */ struct mctp_usb_hdr { __be16 id; - u8 rsvd; - u8 len; + __be16 len; } __packed; -#define MCTP_USB_XFER_SIZE 512 +/* max transfer size for DSP0283 v1.0 */ +#define MCTP_USB_1_0_XFER_SIZE 512 #define MCTP_USB_BTU 68 #define MCTP_USB_MTU_MIN MCTP_USB_BTU -#define MCTP_USB_MTU_MAX (U8_MAX - sizeof(struct mctp_usb_hdr)) +#define MCTP_USB_1_0_PKTLEN_MAX U8_MAX +#define MCTP_USB_1_0_MTU_MAX (MCTP_USB_1_0_PKTLEN_MAX - sizeof(struct mctp_usb_hdr)) +#define MCTP_USB_1_1_PKTLEN_MAX GENMASK(12, 0) +#define MCTP_USB_1_1_MTU_MAX (MCTP_USB_1_1_PKTLEN_MAX - sizeof(struct mctp_usb_hdr)) #define MCTP_USB_DMTF_ID 0x1ab4 +/* mctp-usblib */ + +/* + * RX handle: drivers will typically create one on init, which persists for + * the life of the driver. The same handle is used for progressive + * prepare -> complete operations (for each incoming USB transfer), which + * result in netif_rx()-ing the MCTP packets received + */ +struct mctp_usblib_rx { + struct sk_buff *skb; + u16 ep_pktlen; + bool span; +}; + +int mctp_usblib_rx_init(struct mctp_usblib_rx *rx, u16 ep_pktlen, bool span); +void mctp_usblib_rx_fini(struct mctp_usblib_rx *rx); + +int mctp_usblib_rx_prepare(struct net_device *netdev, + struct mctp_usblib_rx *rx, + void **bufp, size_t *lenp, gfp_t gfp); + +int mctp_usblib_rx_complete(struct net_device *netdev, + struct mctp_usblib_rx *rx, size_t len); + +void mctp_usblib_rx_cancel(struct mctp_usblib_rx *rx); + +/* + * TX handle: created by mctp_usblib_tx_push() during the tx path, and + * may persist across multiple packet transmits. + */ +struct mctp_usblib_tx_ctx; + +struct mctp_usblib_tx_ops { + /* Start a USB TX for @data. On returning success, the implementation + * must arrange for mctp_usblib_tx_send_complete() to be called at some + * later point (eg., on urb completion). + */ + int (*send)(struct mctp_usblib_tx_ctx *tx_ctx, void *data, size_t len); +}; + +struct mctp_usblib_tx { + struct mctp_usblib_tx_ops ops; + void *priv; + bool span; + /* protects access to cur_ctx */ + spinlock_t lock; + /* context to which we are adding packets, cleared on send */ + struct mctp_usblib_tx_ctx *cur_ctx; +}; + +void mctp_usblib_tx_init(struct mctp_usblib_tx *tx, + const struct mctp_usblib_tx_ops *ops, void *priv, + bool span); +void mctp_usblib_tx_fini(struct mctp_usblib_tx *tx); + +void *mctp_usblib_tx_ctx_priv(struct mctp_usblib_tx_ctx *tx_ctx); + +int mctp_usblib_tx_push(struct net_device *dev, + struct mctp_usblib_tx *tx, + struct sk_buff *skb, bool more); + +void mctp_usblib_tx_send_complete(struct mctp_usblib_tx_ctx *tx_ctx, + struct net_device *dev, bool ok); + +void mctp_usblib_tx_cancel(struct mctp_usblib_tx *tx, struct net_device *dev, + enum skb_drop_reason reason); + #endif /* __LINUX_USB_MCTP_USB_H */ diff --git a/include/linux/usb/pd.h b/include/linux/usb/pd.h index 6ccd1b2af993..914e78f8acfd 100644 --- a/include/linux/usb/pd.h +++ b/include/linux/usb/pd.h @@ -34,7 +34,8 @@ enum pd_ctrl_msg_type { PD_CTRL_FR_SWAP = 19, PD_CTRL_GET_PPS_STATUS = 20, PD_CTRL_GET_COUNTRY_CODES = 21, - /* 22-23 Reserved */ + PD_CTRL_GET_SINK_CAP_EXT = 22, + /* 23 Reserved */ PD_CTRL_GET_REVISION = 24, /* 25-31 Reserved */ }; @@ -72,7 +73,8 @@ enum pd_ext_msg_type { PD_EXT_PPS_STATUS = 12, PD_EXT_COUNTRY_INFO = 13, PD_EXT_COUNTRY_CODES = 14, - /* 15-31 Reserved */ + PD_EXT_SINK_CAP_EXT = 15, + /* 16-31 Reserved */ }; #define PD_REV10 0x0 @@ -104,6 +106,9 @@ enum pd_ext_msg_type { #define PD_HEADER_LE(type, pwr, data, rev, id, cnt) \ cpu_to_le16(PD_HEADER((type), (pwr), (data), (rev), (id), (cnt), (0))) +#define PD_HEADER_EXT_LE(type, pwr, data, rev, id, cnt) \ + cpu_to_le16(PD_HEADER((type), (pwr), (data), (rev), (id), (cnt), (1))) + static inline unsigned int pd_header_cnt(u16 header) { return (header >> PD_HEADER_CNT_SHIFT) & PD_HEADER_CNT_MASK; @@ -205,6 +210,91 @@ struct pd_message { }; } __packed; +/* + * count_chunked_data_objs - Helper to calculate number of Data Objects on a 4 + * byte boundary. + * @size: Size of data block for extended message. Should *not* include extended + * header size. + */ +static inline u8 count_chunked_data_objs(u32 size) +{ + size += offsetof(struct pd_chunked_ext_message_data, data); + return ((size / 4) + (size % 4 ? 1 : 0)); +} + +/** + * batt_cap_ext_msg - Battery capability extended PD message + * @vid: Battery Vendor ID (assigned by USB-IF) + * @pid: Battery Product ID (assigned by battery or device vendor) + * @batt_design_cap: Battery design capacity in 0.1Wh + * @batt_last_chg_cap: Battery last full charge capacity in 0.1Wh + * @batt_type: Battery Type. bit0 when set indicates invalid battery reference. + * Rest of the bits are reserved. + */ +struct batt_cap_ext_msg { + __le16 vid; + __le16 pid; + __le16 batt_design_cap; + __le16 batt_last_chg_cap; + u8 batt_type; +} __packed; + +#define BATT_CAP_BATT_TYPE_INVALID_REF BIT(0) + +/* Sink Caps Extended Data Block Version */ +#define SKEDB_VER_1_0 1 + +/* Sink Caps Extended Sink Modes */ +#define SINK_MODE_PPS BIT(0) +#define SINK_MODE_VBUS BIT(1) +#define SINK_MODE_AC_SUPPLY BIT(2) +#define SINK_MODE_BATT BIT(3) +#define SINK_MODE_BATT_UL BIT(4) /* Unlimited battery power supply */ +#define SINK_MODE_AVS BIT(5) + +/** + * struct sink_caps_ext_msg - Sink extended capability PD message + * @vid: Vendor ID + * @pid: Product ID + * @xid: Value assigned by USB-IF for product + * @fw: Firmware version + * @hw: Hardware version + * @skedb_ver: Sink Caps Extended Data Block (SKEDB) Version + * @load_step: Indicates the load step slew rate. + * @load_char: Sink overload characteristics + * @compliance: Types of sources the sink has been tested & certified on + * @touch_temp: Indicates the IEC standard to which the touch temperature + * conforms to (if applicable). + * @batt_info: Indicates number batteries and hot swappable ports + * @modes: Charging caps & power sources supported + * @spr_min_pdp: Sink Minimum PDP for SPR mode + * @spr_op_pdp: Sink Operational PDP for SPR mode + * @spr_max_pdp: Sink Maximum PDP for SPR mode + * @epr_min_pdp: Sink Minimum PDP for EPR mode + * @epr_op_pdp: Sink Operational PDP for EPR mode + * @epr_max_pdp: Sink Maximum PDP for EPR mode + */ +struct sink_caps_ext_msg { + __le16 vid; + __le16 pid; + __le32 xid; + u8 fw; + u8 hw; + u8 skedb_ver; + u8 load_step; + __le16 load_char; + u8 compliance; + u8 touch_temp; + u8 batt_info; + u8 modes; + u8 spr_min_pdp; + u8 spr_op_pdp; + u8 spr_max_pdp; + u8 epr_min_pdp; + u8 epr_op_pdp; + u8 epr_max_pdp; +} __packed; + /* PDO: Power Data Object */ #define PDO_MAX_OBJECTS 7 @@ -329,6 +419,32 @@ enum pd_apdo_type { #define PDO_SPR_AVS_APDO_9V_TO_15V_MAX_CURR GENMASK(19, 10) /* 10mA unit */ #define PDO_SPR_AVS_APDO_15V_TO_20V_MAX_CURR GENMASK(9, 0) /* 10mA unit */ +/* SPR AVS has two different current ranges 9V - 15V, 15V - 20V */ +#define SPR_AVS_TIER1_MIN_VOLT_MV 9000 +#define SPR_AVS_TIER1_MAX_VOLT_MV 15000 +#define SPR_AVS_TIER2_MAX_VOLT_MV 20000 + +#define SPR_AVS_AVS_SMALL_STEP_V 1 +/* vAvsStep - 100mv */ +#define SPR_AVS_VOLT_MV_STEP 100 +/* SPR AVS RDO Operating Current is in 50mA step */ +#define RDO_SPR_AVS_CURR_MA_STEP 50 +/* SPR AVS RDO Output voltage is in 25mV step */ +#define RDO_SPR_AVS_OUT_VOLT_MV_STEP 25 + +#define RDO_SPR_AVS_VOLT GENMASK(20, 9) +#define RDO_SPR_AVS_CURR GENMASK(6, 0) + +#define RDO_SPR_AVS_OUT_VOLT(mv) \ + FIELD_PREP(RDO_SPR_AVS_VOLT, ((mv) / RDO_SPR_AVS_OUT_VOLT_MV_STEP)) + +#define RDO_SPR_AVS_OP_CURR(ma) \ + FIELD_PREP(RDO_SPR_AVS_CURR, ((ma) / RDO_SPR_AVS_CURR_MA_STEP)) + +#define RDO_AVS(idx, out_mv, op_ma, flags) \ + (RDO_OBJ(idx) | (flags) | \ + RDO_SPR_AVS_OUT_VOLT(out_mv) | RDO_SPR_AVS_OP_CURR(op_ma)) + static inline enum pd_pdo_type pdo_type(u32 pdo) { return (pdo >> PDO_TYPE_SHIFT) & PDO_TYPE_MASK; @@ -339,6 +455,11 @@ static inline unsigned int pdo_fixed_voltage(u32 pdo) return ((pdo >> PDO_FIXED_VOLT_SHIFT) & PDO_VOLT_MASK) * 50; } +static inline unsigned int pdo_fixed_current(u32 pdo) +{ + return ((pdo >> PDO_FIXED_CURR_SHIFT) & PDO_CURR_MASK) * 10; +} + static inline unsigned int pdo_min_voltage(u32 pdo) { return ((pdo >> PDO_VAR_MIN_VOLT_SHIFT) & PDO_VOLT_MASK) * 50; @@ -394,7 +515,7 @@ static inline unsigned int pdo_epr_avs_apdo_min_voltage_mv(u32 pdo) static inline unsigned int pdo_epr_avs_apdo_max_voltage_mv(u32 pdo) { - return FIELD_GET(PDO_EPR_AVS_APDO_MIN_VOLT, pdo) * 100; + return FIELD_GET(PDO_EPR_AVS_APDO_MAX_VOLT, pdo) * 100; } static inline unsigned int pdo_epr_avs_apdo_pdp_w(u32 pdo) @@ -582,6 +703,11 @@ static inline unsigned int rdo_max_power(u32 rdo) #define PD_P_SNK_STDBY_MW 2500 /* 2500 mW */ +#define PD_I_SNK_STBY_MA 500 /* 500 mA */ + +#define PD_T_AVS_SRC_TRANS_SMALL 50 /* 50 ms */ +#define PD_T_AVS_SRC_TRANS_LARGE 700 /* 700 ms */ + #if IS_ENABLED(CONFIG_TYPEC) struct usb_power_delivery; @@ -620,4 +746,33 @@ void usb_power_delivery_unlink_device(struct usb_power_delivery *pd, struct devi #endif /* CONFIG_TYPEC */ +/* Battery Status Data Object */ +#define BSDO_PRESENT_CAPACITY GENMASK(31, 16) +#define BSDO_CHG_STATUS GENMASK(11, 10) +#define BSDO_BATTERY_PRESENT BIT(9) +#define BSDO_INVALID_BATTERY_REFERENCE BIT(8) + +/* + * Battery Charge Status: Battery Charging Status Values as defined in + * "USB PD Spec Rev3.1 Ver1.8", "Table 6-46 Battery Status Data Object (BSDO)". + */ +#define BSDO_BATTERY_INFO_CHARGING 0x0 +#define BSDO_BATTERY_INFO_DISCHARGING 0x1 +#define BSDO_BATTERY_INFO_IDLE 0x2 +#define BSDO_BATTERY_INFO_RSVD 0x3 + +/** + * BSDO() - Pack data into Battery Status Data Object format. + * @batt_charge: Battery's present state of charge in 0.1WH increment. + * @chg_status: Battery charge status. + * @batt_present: Indicates that battery is present/attached when set else absent when unset. + * @invalid_ref: Indicates that an invalid battery reference was made in the Get_Battery_Status + * request. + */ +#define BSDO(batt_charge, chg_status, batt_present, invalid_ref) \ + ((FIELD_PREP(BSDO_PRESENT_CAPACITY, batt_charge)) | \ + (FIELD_PREP(BSDO_CHG_STATUS, chg_status)) | \ + ((batt_present) ? BSDO_BATTERY_PRESENT : 0) | \ + ((invalid_ref) ? BSDO_INVALID_BATTERY_REFERENCE : 0)) + #endif /* __LINUX_USB_PD_H */ diff --git a/include/linux/usb/quirks.h b/include/linux/usb/quirks.h index b3cc7beab4a3..a4043b33c2c2 100644 --- a/include/linux/usb/quirks.h +++ b/include/linux/usb/quirks.h @@ -81,4 +81,7 @@ /* Device claims zero configurations, forcing to 1 */ #define USB_QUIRK_FORCE_ONE_CONFIG BIT(18) +/* Use a 255 bytes config descriptor request mirroring windows behavior */ +#define USB_QUIRK_WINDOWS_CONFIG_REQ_SIZE BIT(19) + #endif /* __LINUX_USB_QUIRKS_H */ diff --git a/include/linux/usb/serial.h b/include/linux/usb/serial.h index 75b2b763f1ba..534e6650e2aa 100644 --- a/include/linux/usb/serial.h +++ b/include/linux/usb/serial.h @@ -13,6 +13,7 @@ #include <linux/mutex.h> #include <linux/serial.h> #include <linux/kfifo.h> +#include <linux/usb.h> /* The maximum number of ports one device can grab at once */ #define MAX_NUM_PORTS 16 diff --git a/include/linux/usb/tcpci.h b/include/linux/usb/tcpci.h index f7f5cfbdef12..9b46a6bc762c 100644 --- a/include/linux/usb/tcpci.h +++ b/include/linux/usb/tcpci.h @@ -144,6 +144,7 @@ #define TCPC_RX_BUF_FRAME_TYPE 0x31 #define TCPC_RX_BUF_FRAME_TYPE_SOP 0 #define TCPC_RX_BUF_FRAME_TYPE_SOP1 1 +#define TCPC_RX_BUF_FRAME_TYPE_MASK GENMASK(2, 0) #define TCPC_RX_HDR 0x32 #define TCPC_RX_DATA 0x34 /* through 0x4f */ diff --git a/include/linux/usb/tcpm.h b/include/linux/usb/tcpm.h index b22e659f81ba..93079450bba0 100644 --- a/include/linux/usb/tcpm.h +++ b/include/linux/usb/tcpm.h @@ -31,7 +31,7 @@ enum typec_cc_polarity { /* Time to wait for TCPC to complete transmit */ #define PD_T_TCPC_TX_TIMEOUT 100 /* in ms */ #define PD_ROLE_SWAP_TIMEOUT (MSEC_PER_SEC * 10) -#define PD_PPS_CTRL_TIMEOUT (MSEC_PER_SEC * 10) +#define PD_AUG_PSY_CTRL_TIMEOUT (MSEC_PER_SEC * 10) enum tcpm_transmit_status { TCPC_TX_SUCCESS = 0, diff --git a/include/linux/usb/typec.h b/include/linux/usb/typec.h index d61ec38216fa..10a783b738ef 100644 --- a/include/linux/usb/typec.h +++ b/include/linux/usb/typec.h @@ -337,6 +337,7 @@ void typec_unregister_cable(struct typec_cable *cable); struct typec_cable *typec_cable_get(struct typec_port *port); void typec_cable_put(struct typec_cable *cable); int typec_cable_is_active(struct typec_cable *cable); +bool typec_cable_altmode_unsupported(struct typec_altmode *alt); struct typec_plug *typec_register_plug(struct typec_cable *cable, struct typec_plug_desc *desc); diff --git a/include/linux/usb/typec_altmode.h b/include/linux/usb/typec_altmode.h index 0513d333b797..ef21ead551be 100644 --- a/include/linux/usb/typec_altmode.h +++ b/include/linux/usb/typec_altmode.h @@ -3,7 +3,7 @@ #ifndef __USB_TYPEC_ALTMODE_H #define __USB_TYPEC_ALTMODE_H -#include <linux/mod_devicetable.h> +#include <linux/device-id/typec.h> #include <linux/usb/typec.h> #include <linux/device.h> @@ -26,6 +26,9 @@ struct typec_altmode_ops; * @mode: Index of the Mode * @vdo: VDO returned by Discover Modes USB PD command * @active: Tells has the mode been entered or not + * @priority: Priority used by the automatic alternate mode selection process + * @mode_selection: Whether entry to this alternate mode is managed by the + * automatic alternate mode selection process or by the specific driver * @desc: Optional human readable description of the mode * @ops: Operations vector from the driver * @cable_ops: Cable operations vector from the driver. diff --git a/include/linux/usb/uvc.h b/include/linux/usb/uvc.h index ea92ac623a45..99b070ab860f 100644 --- a/include/linux/usb/uvc.h +++ b/include/linux/usb/uvc.h @@ -43,6 +43,16 @@ #define UVC_GUID_MSXU_1_5 \ {0xdc, 0x95, 0x3f, 0x0f, 0x32, 0x26, 0x4e, 0x4c, \ 0x92, 0xc9, 0xa0, 0x47, 0x82, 0xf4, 0x3b, 0xc8} +#define UVC_GUID_LOGITECH_MOTOR_CONTROL_V1 \ + {0x82, 0x06, 0x61, 0x63, 0x70, 0x50, 0xab, 0x49, \ + 0xb8, 0xcc, 0xb3, 0x85, 0x5e, 0x8d, 0x22, 0x56 } +#define UVC_GUID_LOGITECH_PERIPHERAL \ + {0x21, 0x2d, 0xe5, 0xff, 0x30, 0x80, 0x2c, 0x4e, \ + 0x82, 0xd9, 0xf5, 0x87, 0xd0, 0x05, 0x40, 0xbd } +#define UVC_GUID_LOGITECH_USER_HW_CONTROL_V1 \ + {0x82, 0x06, 0x61, 0x63, 0x70, 0x50, 0xab, 0x49, \ + 0xb8, 0xcc, 0xb3, 0x85, 0x5e, 0x8d, 0x22, 0x1f } + /* https://learn.microsoft.com/en-us/windows-hardware/drivers/stream/uvc-extensions-1-5#222-extension-unit-controls */ #define UVC_MSXU_CONTROL_FOCUS 0x01 @@ -138,6 +148,9 @@ #define UVC_GUID_FORMAT_M420 \ { 'M', '4', '2', '0', 0x00, 0x00, 0x10, 0x00, \ 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71} +#define UVC_GUID_FORMAT_P010 \ + { 'P', '0', '1', '0', 0x00, 0x00, 0x10, 0x00, \ + 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71} #define UVC_GUID_FORMAT_H264 \ { 'H', '2', '6', '4', 0x00, 0x00, 0x10, 0x00, \ diff --git a/include/linux/user_namespace.h b/include/linux/user_namespace.h index 9c3be157397e..e38d9e60569f 100644 --- a/include/linux/user_namespace.h +++ b/include/linux/user_namespace.h @@ -58,6 +58,9 @@ enum ucount_type { UCOUNT_FANOTIFY_GROUPS, UCOUNT_FANOTIFY_MARKS, #endif +#if IS_ENABLED(CONFIG_BINFMT_MISC) + UCOUNT_BINFMT_MISC_INTERPRETERS, +#endif UCOUNT_COUNTS, }; diff --git a/include/linux/userfaultfd_k.h b/include/linux/userfaultfd_k.h index fd5f42765497..a4351cffc60c 100644 --- a/include/linux/userfaultfd_k.h +++ b/include/linux/userfaultfd_k.h @@ -21,7 +21,11 @@ #include <linux/hugetlb_inline.h> /* The set of all possible UFFD-related VM flags. */ -#define __VM_UFFD_FLAGS (VM_UFFD_MISSING | VM_UFFD_WP | VM_UFFD_MINOR) +#define __VM_UFFD_FLAGS (VM_UFFD_MISSING | VM_UFFD_MINOR | \ + VM_UFFD_WP | VM_UFFD_RWP) + +#define __VMA_UFFD_FLAGS mk_vma_flags_from_masks(VMA_UFFD_MISSING, VMA_UFFD_WP, \ + VMA_UFFD_MINOR, VMA_UFFD_RWP) /* * CAREFUL: Check include/uapi/asm-generic/fcntl.h when defining @@ -80,6 +84,39 @@ struct userfaultfd_ctx { extern vm_fault_t handle_userfault(struct vm_fault *vmf, unsigned long reason); +/* VMA userfaultfd operations */ +struct vm_uffd_ops { + /* Checks if a VMA can support userfaultfd */ + bool (*can_userfault)(struct vm_area_struct *vma, vm_flags_t vm_flags); + /* + * Called to resolve UFFDIO_CONTINUE request. + * Should return the folio found at pgoff in the VMA's pagecache if it + * exists or ERR_PTR otherwise. + * The returned folio is locked and with reference held. + */ + struct folio *(*get_folio_noalloc)(struct inode *inode, pgoff_t pgoff); + /* + * Called during resolution of UFFDIO_COPY request. + * Should allocate and return a folio or NULL if allocation fails. + */ + struct folio *(*alloc_folio)(struct vm_area_struct *vma, + unsigned long addr); + /* + * Called during resolution of UFFDIO_COPY request. + * Should only be called with a folio returned by alloc_folio() above. + * The folio will be set to locked. + * Returns 0 on success, error code on failure. + */ + int (*filemap_add)(struct folio *folio, struct vm_area_struct *vma, + unsigned long addr); + /* + * Called during resolution of UFFDIO_COPY request on the error + * handling path. + * Should revert the operation of ->filemap_add(). + */ + void (*filemap_remove)(struct folio *folio, struct vm_area_struct *vma); +}; + /* A combined operation mode + behavior flags. */ typedef unsigned int __bitwise uffd_flags_t; @@ -111,31 +148,14 @@ static inline uffd_flags_t uffd_flags_set_mode(uffd_flags_t flags, enum mfill_at /* Flags controlling behavior. These behavior changes are mode-independent. */ #define MFILL_ATOMIC_WP MFILL_ATOMIC_FLAG(0) -extern int mfill_atomic_install_pte(pmd_t *dst_pmd, - struct vm_area_struct *dst_vma, - unsigned long dst_addr, struct page *page, - bool newly_allocated, uffd_flags_t flags); - -extern ssize_t mfill_atomic_copy(struct userfaultfd_ctx *ctx, unsigned long dst_start, - unsigned long src_start, unsigned long len, - uffd_flags_t flags); -extern ssize_t mfill_atomic_zeropage(struct userfaultfd_ctx *ctx, - unsigned long dst_start, - unsigned long len); -extern ssize_t mfill_atomic_continue(struct userfaultfd_ctx *ctx, unsigned long dst_start, - unsigned long len, uffd_flags_t flags); -extern ssize_t mfill_atomic_poison(struct userfaultfd_ctx *ctx, unsigned long start, - unsigned long len, uffd_flags_t flags); -extern int mwriteprotect_range(struct userfaultfd_ctx *ctx, unsigned long start, - unsigned long len, bool enable_wp); extern long uffd_wp_range(struct vm_area_struct *vma, unsigned long start, unsigned long len, bool enable_wp); +extern int mrwprotect_range(struct userfaultfd_ctx *ctx, unsigned long start, + unsigned long len, bool enable_rwp); /* move_pages */ void double_pt_lock(spinlock_t *ptl1, spinlock_t *ptl2); void double_pt_unlock(spinlock_t *ptl1, spinlock_t *ptl2); -ssize_t move_pages(struct userfaultfd_ctx *ctx, unsigned long dst_start, - unsigned long src_start, unsigned long len, __u64 flags); int move_pages_huge_pmd(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd, pmd_t dst_pmdval, struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, @@ -151,7 +171,8 @@ static inline bool is_mergeable_vm_userfaultfd_ctx(struct vm_area_struct *vma, /* * Never enable huge pmd sharing on some uffd registered vmas: * - * - VM_UFFD_WP VMAs, because write protect information is per pgtable entry. + * - VM_UFFD_WP and VM_UFFD_RWP VMAs, because the write protect / access + * tracking information is per pgtable entry. * * - VM_UFFD_MINOR VMAs, because otherwise we would never get minor faults for * VMAs which share huge pmds. (If you have two mappings to the same @@ -161,85 +182,86 @@ static inline bool is_mergeable_vm_userfaultfd_ctx(struct vm_area_struct *vma, */ static inline bool uffd_disable_huge_pmd_share(struct vm_area_struct *vma) { - return vma->vm_flags & (VM_UFFD_WP | VM_UFFD_MINOR); + return vma_test_any_mask(vma, + mk_vma_flags_from_masks(VMA_UFFD_WP, VMA_UFFD_RWP, + VMA_UFFD_MINOR)); } /* - * Don't do fault around for either WP or MINOR registered uffd range. For + * Don't do fault around for WP, RWP or MINOR registered uffd range. For * MINOR registered range, fault around will be a total disaster and ptes can * be installed without notifications; for WP it should mostly be fine as long * as the fault around checks for pte_none() before the installation, however - * to be super safe we just forbid it. + * to be super safe we just forbid it; for RWP, pre-faulted neighbours would + * be indistinguishable from accessed pages in PAGEMAP_SCAN (PAGE_IS_ACCESSED) + * and pollute the tracked working set, so each page must be populated by its + * own fault. */ static inline bool uffd_disable_fault_around(struct vm_area_struct *vma) { - return vma->vm_flags & (VM_UFFD_WP | VM_UFFD_MINOR); + return vma_test_any_mask(vma, + mk_vma_flags_from_masks(VMA_UFFD_WP, VMA_UFFD_RWP, + VMA_UFFD_MINOR)); } static inline bool userfaultfd_missing(struct vm_area_struct *vma) { - return vma->vm_flags & VM_UFFD_MISSING; + return vma_test_any_mask(vma, VMA_UFFD_MISSING); } static inline bool userfaultfd_wp(struct vm_area_struct *vma) { - return vma->vm_flags & VM_UFFD_WP; + return vma_test_any_mask(vma, VMA_UFFD_WP); } static inline bool userfaultfd_minor(struct vm_area_struct *vma) { - return vma->vm_flags & VM_UFFD_MINOR; + return vma_test_any_mask(vma, VMA_UFFD_MINOR); +} + +static inline bool userfaultfd_rwp(struct vm_area_struct *vma) +{ + /* + * Callers gate PAGE_NONE usage on this; PAGE_NONE is a BUILD_BUG() + * without CONFIG_ARCH_HAS_PTE_PROTNONE, so fold to false. + */ + if (!IS_ENABLED(CONFIG_ARCH_HAS_PTE_PROTNONE)) + return false; + return vma_test_single_mask(vma, VMA_UFFD_RWP); +} + +static inline bool userfaultfd_protected(struct vm_area_struct *vma) +{ + return userfaultfd_wp(vma) || userfaultfd_rwp(vma); } static inline bool userfaultfd_pte_wp(struct vm_area_struct *vma, pte_t pte) { - return userfaultfd_wp(vma) && pte_uffd_wp(pte); + return userfaultfd_wp(vma) && pte_uffd(pte); } static inline bool userfaultfd_huge_pmd_wp(struct vm_area_struct *vma, pmd_t pmd) { - return userfaultfd_wp(vma) && pmd_uffd_wp(pmd); + return userfaultfd_wp(vma) && pmd_uffd(pmd); } -static inline bool userfaultfd_armed(struct vm_area_struct *vma) +static inline bool userfaultfd_pte_rwp(struct vm_area_struct *vma, + pte_t pte) { - return vma->vm_flags & __VM_UFFD_FLAGS; + return userfaultfd_rwp(vma) && pte_uffd(pte); } -static inline bool vma_can_userfault(struct vm_area_struct *vma, - vm_flags_t vm_flags, - bool wp_async) +static inline bool userfaultfd_huge_pmd_rwp(struct vm_area_struct *vma, + pmd_t pmd) { - vm_flags &= __VM_UFFD_FLAGS; - - if (vma->vm_flags & VM_DROPPABLE) - return false; - - if ((vm_flags & VM_UFFD_MINOR) && - (!is_vm_hugetlb_page(vma) && !vma_is_shmem(vma))) - return false; - - /* - * If wp async enabled, and WP is the only mode enabled, allow any - * memory type. - */ - if (wp_async && (vm_flags == VM_UFFD_WP)) - return true; - - /* - * If user requested uffd-wp but not enabled pte markers for - * uffd-wp, then shmem & hugetlbfs are not supported but only - * anonymous. - */ - if (!uffd_supports_wp_marker() && (vm_flags & VM_UFFD_WP) && - !vma_is_anonymous(vma)) - return false; + return userfaultfd_rwp(vma) && pmd_uffd(pmd); +} - /* By default, allow any of anon|shmem|hugetlb */ - return vma_is_anonymous(vma) || is_vm_hugetlb_page(vma) || - vma_is_shmem(vma); +static inline bool userfaultfd_armed(struct vm_area_struct *vma) +{ + return vma_test_any_mask(vma, __VMA_UFFD_FLAGS); } static inline bool vma_has_uffd_without_event_remap(struct vm_area_struct *vma) @@ -270,25 +292,7 @@ extern void userfaultfd_unmap_complete(struct mm_struct *mm, struct list_head *uf); extern bool userfaultfd_wp_unpopulated(struct vm_area_struct *vma); extern bool userfaultfd_wp_async(struct vm_area_struct *vma); - -void userfaultfd_reset_ctx(struct vm_area_struct *vma); - -struct vm_area_struct *userfaultfd_clear_vma(struct vma_iterator *vmi, - struct vm_area_struct *prev, - struct vm_area_struct *vma, - unsigned long start, - unsigned long end); - -int userfaultfd_register_range(struct userfaultfd_ctx *ctx, - struct vm_area_struct *vma, - vm_flags_t vm_flags, - unsigned long start, unsigned long end, - bool wp_async); - -void userfaultfd_release_new(struct userfaultfd_ctx *ctx); - -void userfaultfd_release_all(struct mm_struct *mm, - struct userfaultfd_ctx *ctx); +extern bool userfaultfd_rwp_async(struct vm_area_struct *vma); static inline bool userfaultfd_wp_use_markers(struct vm_area_struct *vma) { @@ -308,10 +312,10 @@ static inline bool userfaultfd_wp_use_markers(struct vm_area_struct *vma) } /* - * Returns true if this is a swap pte and was uffd-wp wr-protected in either - * forms (pte marker or a normal swap pte), false otherwise. + * Returns true if this swap pte carries uffd-tracked state in either + * form (pte marker or a normal swap pte), false otherwise. */ -static inline bool pte_swp_uffd_wp_any(pte_t pte) +static inline bool pte_swp_uffd_any(pte_t pte) { if (!uffd_supports_wp_marker()) return false; @@ -319,7 +323,7 @@ static inline bool pte_swp_uffd_wp_any(pte_t pte) if (pte_present(pte)) return false; - if (pte_swp_uffd_wp(pte)) + if (pte_swp_uffd(pte)) return true; if (pte_is_uffd_wp_marker(pte)) @@ -364,6 +368,16 @@ static inline bool userfaultfd_minor(struct vm_area_struct *vma) return false; } +static inline bool userfaultfd_rwp(struct vm_area_struct *vma) +{ + return false; +} + +static inline bool userfaultfd_protected(struct vm_area_struct *vma) +{ + return false; +} + static inline bool userfaultfd_pte_wp(struct vm_area_struct *vma, pte_t pte) { @@ -376,6 +390,17 @@ static inline bool userfaultfd_huge_pmd_wp(struct vm_area_struct *vma, return false; } +static inline bool userfaultfd_pte_rwp(struct vm_area_struct *vma, + pte_t pte) +{ + return false; +} + +static inline bool userfaultfd_huge_pmd_rwp(struct vm_area_struct *vma, + pmd_t pmd) +{ + return false; +} static inline bool userfaultfd_armed(struct vm_area_struct *vma) { @@ -446,6 +471,11 @@ static inline bool userfaultfd_wp_async(struct vm_area_struct *vma) return false; } +static inline bool userfaultfd_rwp_async(struct vm_area_struct *vma) +{ + return false; +} + static inline bool vma_has_uffd_without_event_remap(struct vm_area_struct *vma) { return false; @@ -457,10 +487,10 @@ static inline bool userfaultfd_wp_use_markers(struct vm_area_struct *vma) } /* - * Returns true if this is a swap pte and was uffd-wp wr-protected in either - * forms (pte marker or a normal swap pte), false otherwise. + * Returns true if this swap pte carries uffd-tracked state in either + * form (pte marker or a normal swap pte), false otherwise. */ -static inline bool pte_swp_uffd_wp_any(pte_t pte) +static inline bool pte_swp_uffd_any(pte_t pte) { return false; } diff --git a/include/linux/utsname.h b/include/linux/utsname.h index 547bd4439706..dc5594ffffcc 100644 --- a/include/linux/utsname.h +++ b/include/linux/utsname.h @@ -18,7 +18,7 @@ enum uts_proc { UTS_PROC_DOMAINNAME, }; -#ifdef CONFIG_PROC_SYSCTL +#ifdef CONFIG_SYSCTL extern void uts_proc_notify(enum uts_proc proc); #else static inline void uts_proc_notify(enum uts_proc proc) diff --git a/include/linux/vdpa.h b/include/linux/vdpa.h index 2bfe3baa63f4..782c42d25db1 100644 --- a/include/linux/vdpa.h +++ b/include/linux/vdpa.h @@ -72,9 +72,6 @@ struct vdpa_mgmt_dev; * struct vdpa_device - representation of a vDPA device * @dev: underlying device * @vmap: the metadata passed to upper layer to be used for mapping - * @driver_override: driver name to force a match; do not set directly, - * because core frees it; use driver_set_override() to - * set or clear it. * @config: the configuration ops for this device. * @map: the map ops for this device * @cf_lock: Protects get and set access to configuration layout. @@ -90,7 +87,6 @@ struct vdpa_mgmt_dev; struct vdpa_device { struct device dev; union virtio_map vmap; - const char *driver_override; const struct vdpa_config_ops *config; const struct virtio_map_ops *map; struct rw_semaphore cf_lock; /* Protects get/set config */ diff --git a/include/linux/vdso_datastore.h b/include/linux/vdso_datastore.h index a91fa24b06e0..13b01baf3497 100644 --- a/include/linux/vdso_datastore.h +++ b/include/linux/vdso_datastore.h @@ -7,4 +7,10 @@ extern const struct vm_special_mapping vdso_vvar_mapping; struct vm_area_struct *vdso_install_vvar_mapping(struct mm_struct *mm, unsigned long addr); +#ifdef CONFIG_VDSO_DATASTORE +void __init vdso_setup_data_pages(void); +#else /* !CONFIG_VDSO_DATASTORE */ +static inline void vdso_setup_data_pages(void) { } +#endif /* CONFIG_VDSO_DATASTORE */ + #endif /* _LINUX_VDSO_DATASTORE_H */ diff --git a/include/linux/vfio.h b/include/linux/vfio.h index e90859956514..45f08986359e 100644 --- a/include/linux/vfio.h +++ b/include/linux/vfio.h @@ -16,6 +16,7 @@ #include <linux/cdev.h> #include <uapi/linux/vfio.h> #include <linux/iova_bitmap.h> +#include <linux/uaccess.h> struct kvm; struct iommufd_ctx; @@ -52,6 +53,7 @@ struct vfio_device { struct vfio_device_set *dev_set; struct list_head dev_set_list; unsigned int migration_flags; + u8 precopy_info_v2; struct kvm *kvm; /* Members below here are private, not for driver use */ @@ -72,13 +74,12 @@ struct vfio_device { u8 iommufd_attached:1; #endif u8 cdev_opened:1; -#ifdef CONFIG_DEBUG_FS + u8 noiommu:1; /* * debug_root is a static property of the vfio_device * which must be set prior to registering the vfio_device. */ struct dentry *debug_root; -#endif }; /** @@ -284,6 +285,44 @@ static inline int vfio_check_feature(u32 flags, size_t argsz, u32 supported_ops, return 1; } +/** + * vfio_check_precopy_ioctl - Validate user input for the VFIO_MIG_GET_PRECOPY_INFO ioctl + * @vdev: The vfio device + * @cmd: Cmd from the ioctl + * @arg: Arg from the ioctl + * @info: Driver pointer to hold the userspace input to the ioctl + * + * For use in a driver's get_precopy_info. Checks that the inputs to the + * VFIO_MIG_GET_PRECOPY_INFO ioctl are correct. + + * Returns 0 on success, otherwise errno. + */ + +static inline int +vfio_check_precopy_ioctl(struct vfio_device *vdev, unsigned int cmd, + unsigned long arg, struct vfio_precopy_info *info) +{ + unsigned long minsz; + + if (cmd != VFIO_MIG_GET_PRECOPY_INFO) + return -ENOTTY; + + minsz = offsetofend(struct vfio_precopy_info, dirty_bytes); + + if (copy_from_user(info, (void __user *)arg, minsz)) + return -EFAULT; + + if (info->argsz < minsz) + return -EINVAL; + + /* keep v1 behaviour as is for compatibility reasons */ + if (vdev->precopy_info_v2) + /* flags are output, set its initial value to 0 */ + info->flags = 0; + + return 0; +} + struct vfio_device *_vfio_alloc_device(size_t size, struct device *dev, const struct vfio_device_ops *ops); #define vfio_alloc_device(dev_struct, member, dev, ops) \ diff --git a/include/linux/vfio_pci_core.h b/include/linux/vfio_pci_core.h index 2ebba746c18f..9a1674c152aa 100644 --- a/include/linux/vfio_pci_core.h +++ b/include/linux/vfio_pci_core.h @@ -21,7 +21,7 @@ #define VFIO_PCI_CORE_H #define VFIO_PCI_OFFSET_SHIFT 40 -#define VFIO_PCI_OFFSET_TO_INDEX(off) (off >> VFIO_PCI_OFFSET_SHIFT) +#define VFIO_PCI_OFFSET_TO_INDEX(off) ((u64)(off) >> VFIO_PCI_OFFSET_SHIFT) #define VFIO_PCI_INDEX_TO_OFFSET(index) ((u64)(index) << VFIO_PCI_OFFSET_SHIFT) #define VFIO_PCI_OFFSET_MASK (((u64)(1) << VFIO_PCI_OFFSET_SHIFT) - 1) @@ -101,6 +101,9 @@ struct vfio_pci_core_device { const struct vfio_pci_device_ops *pci_ops; void __iomem *barmap[PCI_STD_NUM_BARS]; bool bar_mmap_supported[PCI_STD_NUM_BARS]; + /* Flags modified at runtime - dedicated storage unit */ + bool virq_disabled; + bool bardirty; u8 *pci_config_map; u8 *vconfig; struct perm_bits *msi_perm; @@ -115,18 +118,22 @@ struct vfio_pci_core_device { u16 msix_size; u32 msix_offset; u32 rbar[7]; + /* Flags only modified on setup/release - bitfield ok */ bool has_dyn_msix:1; bool pci_2_3:1; - bool virq_disabled:1; bool reset_works:1; bool extended_caps:1; - bool bardirty:1; bool has_vga:1; - bool needs_reset:1; bool nointx:1; bool needs_pm_restore:1; - bool pm_intx_masked:1; - bool pm_runtime_engaged:1; + bool disable_idle_d3:1; + bool nointxmask:1; + bool disable_vga:1; + /* Flags modified at runtime - dedicated storage unit */ + bool needs_reset; + bool pm_intx_masked; + bool pm_runtime_engaged; + bool sriov_active; struct pci_saved_state *pci_saved_state; struct pci_saved_state *pm_save; int ioeventfds_nr; @@ -156,8 +163,6 @@ int vfio_pci_core_register_dev_region(struct vfio_pci_core_device *vdev, unsigned int type, unsigned int subtype, const struct vfio_pci_regops *ops, size_t size, u32 flags, void *data); -void vfio_pci_core_set_params(bool nointxmask, bool is_disable_vga, - bool is_disable_idle_d3); void vfio_pci_core_close_device(struct vfio_device *core_vdev); int vfio_pci_core_init_dev(struct vfio_device *core_vdev); void vfio_pci_core_release_dev(struct vfio_device *core_vdev); @@ -188,7 +193,6 @@ int vfio_pci_core_match_token_uuid(struct vfio_device *core_vdev, int vfio_pci_core_enable(struct vfio_pci_core_device *vdev); void vfio_pci_core_disable(struct vfio_pci_core_device *vdev); void vfio_pci_core_finish_enable(struct vfio_pci_core_device *vdev); -int vfio_pci_core_setup_barmap(struct vfio_pci_core_device *vdev, int bar); pci_ers_result_t vfio_pci_core_aer_err_detected(struct pci_dev *pdev, pci_channel_state_t state); ssize_t vfio_pci_core_do_io_rw(struct vfio_pci_core_device *vdev, bool test_mem, @@ -234,6 +238,25 @@ static inline bool is_aligned_for_order(struct vm_area_struct *vma, !IS_ALIGNED(pfn, 1 << order))); } +/* + * Returns a BAR's iomap base or an ERR_PTR() if, for example, the + * BAR isn't valid, its resource wasn't acquired, or its iomap + * failed. This shall only be used after vfio_pci_core_enable() + * has set up the BAR maps and before vfio_pci_core_disable() + * tears them down. + */ +static inline void __iomem __must_check * +vfio_pci_core_get_iomap(struct vfio_pci_core_device *vdev, unsigned int bar) +{ + if (WARN_ON_ONCE(bar >= PCI_STD_NUM_BARS)) + return IOMEM_ERR_PTR(-EINVAL); + + if (WARN_ON_ONCE(!vdev->barmap[bar])) + return IOMEM_ERR_PTR(-ENODEV); + + return vdev->barmap[bar]; +} + int vfio_pci_dma_buf_iommufd_map(struct dma_buf_attachment *attachment, struct phys_vec *phys); diff --git a/include/linux/virtio.h b/include/linux/virtio.h index 3bbc4cb6a672..f923e42cfd01 100644 --- a/include/linux/virtio.h +++ b/include/linux/virtio.h @@ -7,7 +7,7 @@ #include <linux/scatterlist.h> #include <linux/spinlock.h> #include <linux/device.h> -#include <linux/mod_devicetable.h> +#include <linux/device-id/virtio.h> #include <linux/gfp.h> #include <linux/dma-mapping.h> #include <linux/completion.h> @@ -157,11 +157,13 @@ struct virtio_admin_cmd { * @id: the device type identification (used to match it with a driver). * @config: the configuration ops for this device. * @vringh_config: configuration ops for host vrings. + * @map: the map operations for mapping virtio device memory. * @vqs: the list of virtqueues for this device. * @features: the 64 lower features supported by both driver and device. * @features_array: the full features space supported by both driver and * device. * @priv: private pointer for the driver's use. + * @vmap: the map container with transport- or device-specific metadata. * @debugfs_dir: debugfs directory entry. * @debugfs_filter_features: features to be filtered set by debugfs. */ @@ -211,6 +213,7 @@ int virtio_device_freeze(struct virtio_device *dev); int virtio_device_restore(struct virtio_device *dev); #endif void virtio_reset_device(struct virtio_device *dev); +void virtio_device_shutdown(struct virtio_device *dev); int virtio_device_reset_prepare(struct virtio_device *dev); int virtio_device_reset_done(struct virtio_device *dev); diff --git a/include/linux/virtio_caif.h b/include/linux/virtio_caif.h deleted file mode 100644 index ea722479510c..000000000000 --- a/include/linux/virtio_caif.h +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright (C) ST-Ericsson AB 2012 - * Author: Sjur Brændeland <sjur.brandeland@stericsson.com> - * - * This header is BSD licensed so - * anyone can use the definitions to implement compatible remote processors - */ - -#ifndef VIRTIO_CAIF_H -#define VIRTIO_CAIF_H - -#include <linux/types.h> -struct virtio_caif_transf_config { - __virtio16 headroom; - __virtio16 tailroom; - __virtio32 mtu; - u8 reserved[4]; -}; - -struct virtio_caif_config { - struct virtio_caif_transf_config uplink, downlink; - u8 reserved[8]; -}; -#endif diff --git a/include/linux/virtio_dma_buf.h b/include/linux/virtio_dma_buf.h index a2fdf217ac62..545ac5f17a54 100644 --- a/include/linux/virtio_dma_buf.h +++ b/include/linux/virtio_dma_buf.h @@ -17,7 +17,7 @@ * @ops: the base dma_buf_ops. ops.attach MUST be virtio_dma_buf_attach. * @device_attach: [optional] callback invoked by virtio_dma_buf_attach during * all attach operations. - * @get_uid: [required] callback to get the uuid of the exported object. + * @get_uuid: [required] callback to get the uuid of the exported object. */ struct virtio_dma_buf_ops { struct dma_buf_ops ops; diff --git a/include/linux/virtio_net.h b/include/linux/virtio_net.h index 75dabb763c65..c381b916c1b5 100644 --- a/include/linux/virtio_net.h +++ b/include/linux/virtio_net.h @@ -6,6 +6,7 @@ #include <linux/ip.h> #include <linux/ipv6.h> #include <linux/udp.h> +#include <net/tcp.h> #include <uapi/linux/tcp.h> #include <uapi/linux/virtio_net.h> @@ -179,6 +180,9 @@ retry: if (skb->ip_summed == CHECKSUM_PARTIAL && skb->csum_offset != offsetof(struct tcphdr, check)) return -EINVAL; + + BUILD_BUG_ON(TCP_MIN_GSO_SIZE * GSO_MAX_SEGS < GSO_MAX_SIZE); + gso_size = max(gso_size, TCP_MIN_GSO_SIZE); break; } @@ -207,6 +211,39 @@ static inline int virtio_net_hdr_to_skb(struct sk_buff *skb, return __virtio_net_hdr_to_skb(skb, hdr, little_endian, hdr->gso_type); } +/* This function must be called after virtio_net_hdr_from_skb(). */ +static inline void __virtio_net_set_hdrlen(const struct sk_buff *skb, + struct virtio_net_hdr *hdr, + bool little_endian) +{ + u16 hdr_len; + + hdr_len = skb_transport_offset(skb); + + if (hdr->gso_type == VIRTIO_NET_HDR_GSO_UDP_L4) + hdr_len += sizeof(struct udphdr); + else + hdr_len += tcp_hdrlen(skb); + + hdr->hdr_len = __cpu_to_virtio16(little_endian, hdr_len); +} + +/* This function must be called after virtio_net_hdr_from_skb(). */ +static inline void __virtio_net_set_tnl_hdrlen(const struct sk_buff *skb, + struct virtio_net_hdr *hdr) +{ + u16 hdr_len; + + hdr_len = skb_inner_transport_offset(skb); + + if (hdr->gso_type == VIRTIO_NET_HDR_GSO_UDP_L4) + hdr_len += sizeof(struct udphdr); + else + hdr_len += inner_tcp_hdrlen(skb); + + hdr->hdr_len = __cpu_to_virtio16(true, hdr_len); +} + static inline int virtio_net_hdr_from_skb(const struct sk_buff *skb, struct virtio_net_hdr *hdr, bool little_endian, @@ -385,7 +422,8 @@ virtio_net_hdr_tnl_from_skb(const struct sk_buff *skb, bool tnl_hdr_negotiated, bool little_endian, int vlan_hlen, - bool has_data_valid) + bool has_data_valid, + bool feature_hdrlen) { struct virtio_net_hdr *hdr = (struct virtio_net_hdr *)vhdr; unsigned int inner_nh, outer_th; @@ -394,9 +432,17 @@ virtio_net_hdr_tnl_from_skb(const struct sk_buff *skb, tnl_gso_type = skb_shinfo(skb)->gso_type & (SKB_GSO_UDP_TUNNEL | SKB_GSO_UDP_TUNNEL_CSUM); - if (!tnl_gso_type) - return virtio_net_hdr_from_skb(skb, hdr, little_endian, - has_data_valid, vlan_hlen); + if (!tnl_gso_type) { + ret = virtio_net_hdr_from_skb(skb, hdr, little_endian, + has_data_valid, vlan_hlen); + if (ret) + return ret; + + if (feature_hdrlen && hdr->hdr_len) + __virtio_net_set_hdrlen(skb, hdr, little_endian); + + return ret; + } /* Tunnel support not negotiated but skb ask for it. */ if (!tnl_hdr_negotiated) @@ -414,6 +460,9 @@ virtio_net_hdr_tnl_from_skb(const struct sk_buff *skb, if (ret) return ret; + if (feature_hdrlen && hdr->hdr_len) + __virtio_net_set_tnl_hdrlen(skb, hdr); + if (skb->protocol == htons(ETH_P_IPV6)) hdr->gso_type |= VIRTIO_NET_HDR_GSO_UDP_TUNNEL_IPV6; else diff --git a/include/linux/vm_event_item.h b/include/linux/vm_event_item.h index 22a139f82d75..2628ccda076a 100644 --- a/include/linux/vm_event_item.h +++ b/include/linux/vm_event_item.h @@ -38,21 +38,8 @@ enum vm_event_item { PGPGIN, PGPGOUT, PSWPIN, PSWPOUT, PGFREE, PGACTIVATE, PGDEACTIVATE, PGLAZYFREE, PGFAULT, PGMAJFAULT, PGLAZYFREED, - PGREFILL, PGREUSE, - PGSTEAL_KSWAPD, - PGSTEAL_DIRECT, - PGSTEAL_KHUGEPAGED, - PGSTEAL_PROACTIVE, - PGSCAN_KSWAPD, - PGSCAN_DIRECT, - PGSCAN_KHUGEPAGED, - PGSCAN_PROACTIVE, PGSCAN_DIRECT_THROTTLE, - PGSCAN_ANON, - PGSCAN_FILE, - PGSTEAL_ANON, - PGSTEAL_FILE, #ifdef CONFIG_NUMA PGSCAN_ZONE_RECLAIM_SUCCESS, PGSCAN_ZONE_RECLAIM_FAILED, @@ -188,6 +175,10 @@ enum vm_event_item { PGPGIN, PGPGOUT, PSWPIN, PSWPOUT, KSTACK_REST, #endif #endif /* CONFIG_DEBUG_STACK_USAGE */ +#ifdef CONFIG_SWAP + NRSWPIN, + NRSWPOUT, +#endif /* CONFIG_SWAP */ NR_VM_EVENT_ITEMS }; diff --git a/include/linux/vmalloc.h b/include/linux/vmalloc.h index e8e94f90d686..aed121d729b0 100644 --- a/include/linux/vmalloc.h +++ b/include/linux/vmalloc.h @@ -62,7 +62,7 @@ struct vm_struct { #ifdef CONFIG_HAVE_ARCH_HUGE_VMALLOC unsigned int page_order; #endif - unsigned int nr_pages; + unsigned long nr_pages; phys_addr_t phys_addr; const void *caller; unsigned long requested_size; @@ -265,7 +265,9 @@ static inline bool is_vm_area_hugepages(const void *addr) * allocated in the vmalloc layer. */ #ifdef CONFIG_HAVE_ARCH_HUGE_VMALLOC - return find_vm_area(addr)->page_order > 0; + struct vm_struct *area = find_vm_area(addr); + + return area && area->page_order > 0; #else return false; #endif @@ -286,8 +288,6 @@ int unregister_vmap_purge_notifier(struct notifier_block *nb); #ifdef CONFIG_MMU #define VMALLOC_TOTAL (VMALLOC_END - VMALLOC_START) -unsigned long vmalloc_nr_pages(void); - int vm_area_map_pages(struct vm_struct *area, unsigned long start, unsigned long end, struct page **pages); void vm_area_unmap_pages(struct vm_struct *area, unsigned long start, @@ -304,21 +304,20 @@ static inline void set_vm_flush_reset_perms(void *addr) #else /* !CONFIG_MMU */ #define VMALLOC_TOTAL 0UL -static inline unsigned long vmalloc_nr_pages(void) { return 0; } static inline void set_vm_flush_reset_perms(void *addr) {} #endif /* CONFIG_MMU */ #if defined(CONFIG_MMU) && defined(CONFIG_SMP) struct vm_struct **pcpu_get_vm_areas(const unsigned long *offsets, const size_t *sizes, int nr_vms, - size_t align); + size_t align, gfp_t gfp); void pcpu_free_vm_areas(struct vm_struct **vms, int nr_vms); # else static inline struct vm_struct ** pcpu_get_vm_areas(const unsigned long *offsets, const size_t *sizes, int nr_vms, - size_t align) + size_t align, gfp_t gfp) { return NULL; } diff --git a/include/linux/vmpressure.h b/include/linux/vmpressure.h index 6a2f51ebbfd3..b4d13457bc2a 100644 --- a/include/linux/vmpressure.h +++ b/include/linux/vmpressure.h @@ -13,39 +13,76 @@ struct vmpressure { unsigned long scanned; unsigned long reclaimed; + /* The lock is used to keep the scanned/reclaimed in sync. */ + spinlock_t sr_lock; +#ifdef CONFIG_MEMCG_V1 + /* + * tree=true accumulators feed the v1 userspace eventfd interface + * (memory.pressure_level). Drained by @work. v2 has no equivalent + * interface, so this state is omitted on CONFIG_MEMCG_V1=n builds. + */ unsigned long tree_scanned; unsigned long tree_reclaimed; - /* The lock is used to keep the scanned/reclaimed above in sync. */ - spinlock_t sr_lock; - /* The list of vmpressure_event structs. */ struct list_head events; /* Have to grab the lock on events traversal or modifications. */ struct mutex events_lock; struct work_struct work; +#endif +}; + +enum vmpressure_levels { + VMPRESSURE_LOW = 0, + VMPRESSURE_MEDIUM, + VMPRESSURE_CRITICAL, + VMPRESSURE_NUM_LEVELS, }; struct mem_cgroup; #ifdef CONFIG_MEMCG -extern void vmpressure(gfp_t gfp, struct mem_cgroup *memcg, bool tree, - unsigned long scanned, unsigned long reclaimed); -extern void vmpressure_prio(gfp_t gfp, struct mem_cgroup *memcg, int prio); - +void vmpressure(gfp_t gfp, int order, struct mem_cgroup *memcg, bool tree, + unsigned long scanned, unsigned long reclaimed); extern void vmpressure_init(struct vmpressure *vmpr); extern void vmpressure_cleanup(struct vmpressure *vmpr); extern struct vmpressure *memcg_to_vmpressure(struct mem_cgroup *memcg); extern struct mem_cgroup *vmpressure_to_memcg(struct vmpressure *vmpr); + +/* Shared with the v1 vmpressure block in mm/memcontrol-v1.c. */ +extern const unsigned long vmpressure_win; +extern enum vmpressure_levels vmpressure_calc_level(unsigned long scanned, + unsigned long reclaimed); + +#ifdef CONFIG_MEMCG_V1 +extern void vmpressure_prio(gfp_t gfp, struct mem_cgroup *memcg, int prio); extern int vmpressure_register_event(struct mem_cgroup *memcg, struct eventfd_ctx *eventfd, const char *args); extern void vmpressure_unregister_event(struct mem_cgroup *memcg, struct eventfd_ctx *eventfd); + +/* v1 hooks called from mm/vmpressure.c; no-ops below when !MEMCG_V1. */ +extern void vmpressure_v1_init(struct vmpressure *vmpr); +extern void vmpressure_v1_cleanup(struct vmpressure *vmpr); +extern void vmpressure_v1_account_tree(struct vmpressure *vmpr, + unsigned long scanned, + unsigned long reclaimed); #else -static inline void vmpressure(gfp_t gfp, struct mem_cgroup *memcg, bool tree, - unsigned long scanned, unsigned long reclaimed) {} +static inline void vmpressure_prio(gfp_t gfp, struct mem_cgroup *memcg, + int prio) {} +static inline void vmpressure_v1_init(struct vmpressure *vmpr) {} +static inline void vmpressure_v1_cleanup(struct vmpressure *vmpr) {} +static inline void vmpressure_v1_account_tree(struct vmpressure *vmpr, + unsigned long scanned, + unsigned long reclaimed) {} +#endif /* CONFIG_MEMCG_V1 */ + +#else /* !CONFIG_MEMCG */ +static inline void vmpressure(gfp_t gfp, int order, struct mem_cgroup *memcg, + bool tree, unsigned long scanned, + unsigned long reclaimed) {} static inline void vmpressure_prio(gfp_t gfp, struct mem_cgroup *memcg, int prio) {} #endif /* CONFIG_MEMCG */ diff --git a/include/linux/vmstat.h b/include/linux/vmstat.h index 3c9c266cf782..5b31d8e7ae40 100644 --- a/include/linux/vmstat.h +++ b/include/linux/vmstat.h @@ -20,7 +20,6 @@ struct reclaim_stat { unsigned nr_congested; unsigned nr_writeback; unsigned nr_immediate; - unsigned nr_pageout; unsigned nr_activate[ANON_AND_FILE]; unsigned nr_ref_keep; unsigned nr_unmap_fail; @@ -194,6 +193,19 @@ unsigned long global_node_page_state_pages(enum node_stat_item item) return x; } +/* + * Non-clamping variant of global_node_page_state() intended for callers that + * snapshot a monotonically-incremented counter and subtract two samples. + * Returns the raw wrapping value so that unsigned modular subtraction stays + * correct across a signed-long overflow (a real hazard on 32-bit) that the + * clamp in global_node_page_state() would otherwise turn into a huge spurious + * delta. Do NOT use for non-monotonic page-count reads. + */ +static inline unsigned long global_node_page_state_monotonic(enum node_stat_item item) +{ + return (unsigned long)atomic_long_read(&vm_node_stat[item]); +} + static inline unsigned long global_node_page_state(enum node_stat_item item) { VM_WARN_ON_ONCE(vmstat_item_in_bytes(item)); @@ -259,11 +271,14 @@ extern unsigned long node_page_state(struct pglist_data *pgdat, enum node_stat_item item); extern unsigned long node_page_state_pages(struct pglist_data *pgdat, enum node_stat_item item); +extern unsigned long node_page_state_monotonic(struct pglist_data *pgdat, + enum node_stat_item item); extern void fold_vm_numa_events(void); #else #define sum_zone_node_page_state(node, item) global_zone_page_state(item) #define node_page_state(node, item) global_node_page_state(item) #define node_page_state_pages(node, item) global_node_page_state_pages(item) +#define node_page_state_monotonic(node, item) global_node_page_state_monotonic(item) static inline void fold_vm_numa_events(void) { } diff --git a/include/linux/vt_buffer.h b/include/linux/vt_buffer.h index b6eeb8cb6070..6c15c6a15f74 100644 --- a/include/linux/vt_buffer.h +++ b/include/linux/vt_buffer.h @@ -16,7 +16,7 @@ #include <linux/string.h> -#if IS_ENABLED(CONFIG_VGA_CONSOLE) || IS_ENABLED(CONFIG_MDA_CONSOLE) +#if IS_ENABLED(CONFIG_VGA_CONSOLE) #include <asm/vga.h> #endif diff --git a/include/linux/vtime.h b/include/linux/vtime.h index 29dd5b91dd7d..82825e775499 100644 --- a/include/linux/vtime.h +++ b/include/linux/vtime.h @@ -10,7 +10,6 @@ */ #ifdef CONFIG_VIRT_CPU_ACCOUNTING extern void vtime_account_kernel(struct task_struct *tsk); -extern void vtime_account_idle(struct task_struct *tsk); #endif /* !CONFIG_VIRT_CPU_ACCOUNTING */ #ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN @@ -27,16 +26,39 @@ static inline void vtime_guest_exit(struct task_struct *tsk) { } static inline void vtime_init_idle(struct task_struct *tsk, int cpu) { } #endif +static inline bool vtime_generic_enabled_cpu(int cpu) +{ + return context_tracking_enabled_cpu(cpu); +} + +static inline bool vtime_generic_enabled_this_cpu(void) +{ + return context_tracking_enabled_this_cpu(); +} + #ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE +extern void vtime_account_idle(struct task_struct *tsk); extern void vtime_account_irq(struct task_struct *tsk, unsigned int offset); extern void vtime_account_softirq(struct task_struct *tsk); extern void vtime_account_hardirq(struct task_struct *tsk); extern void vtime_flush(struct task_struct *tsk); +#ifdef CONFIG_HAVE_VIRT_CPU_ACCOUNTING_IDLE +static inline void vtime_reset(void) { } +static inline void vtime_dyntick_start(void) { } +static inline void vtime_dyntick_stop(void) { } +#else +extern void vtime_reset(void); +extern void vtime_dyntick_start(void); +extern void vtime_dyntick_stop(void); +#endif #else /* !CONFIG_VIRT_CPU_ACCOUNTING_NATIVE */ static inline void vtime_account_irq(struct task_struct *tsk, unsigned int offset) { } static inline void vtime_account_softirq(struct task_struct *tsk) { } static inline void vtime_account_hardirq(struct task_struct *tsk) { } static inline void vtime_flush(struct task_struct *tsk) { } +static inline void vtime_reset(void) { } +static inline void vtime_dyntick_start(void) { } +static inline void vtime_dyntick_stop(void) { } #endif /* @@ -74,12 +96,12 @@ static inline bool vtime_accounting_enabled(void) static inline bool vtime_accounting_enabled_cpu(int cpu) { - return context_tracking_enabled_cpu(cpu); + return vtime_generic_enabled_cpu(cpu); } static inline bool vtime_accounting_enabled_this_cpu(void) { - return context_tracking_enabled_this_cpu(); + return vtime_generic_enabled_this_cpu(); } extern void vtime_task_switch_generic(struct task_struct *prev); diff --git a/include/linux/wait.h b/include/linux/wait.h index dce055e6add3..7e215330199c 100644 --- a/include/linux/wait.h +++ b/include/linux/wait.h @@ -1228,6 +1228,7 @@ long prepare_to_wait_event(struct wait_queue_head *wq_head, struct wait_queue_en void finish_wait(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry); long wait_woken(struct wait_queue_entry *wq_entry, unsigned mode, long timeout); int woken_wake_function(struct wait_queue_entry *wq_entry, unsigned mode, int sync, void *key); +int woken_wake_bit_function(struct wait_queue_entry *wq_entry, unsigned mode, int sync, void *key); int autoremove_wake_function(struct wait_queue_entry *wq_entry, unsigned mode, int sync, void *key); #define DEFINE_WAIT_FUNC(name, function) \ diff --git a/include/linux/wait_bit.h b/include/linux/wait_bit.h index 9e29d79fc790..553d7b23e3ad 100644 --- a/include/linux/wait_bit.h +++ b/include/linux/wait_bit.h @@ -32,6 +32,7 @@ int out_of_line_wait_on_bit_timeout(unsigned long *word, int, wait_bit_action_f int out_of_line_wait_on_bit_lock(unsigned long *word, int, wait_bit_action_f *action, unsigned int mode); struct wait_queue_head *bit_waitqueue(unsigned long *word, int bit); extern void __init wait_bit_init(void); +extern struct wait_bit_key *__var_wake_key(struct wait_queue_entry *wq_entry, void *arg); int wake_bit_function(struct wait_queue_entry *wq_entry, unsigned mode, int sync, void *key); @@ -406,7 +407,7 @@ do { \ schedule()) /** - * wait_var_event_killable - wait for a variable to be updated and notified + * wait_var_event_interruptible - wait for a variable to be updated and notified * @var: the address of variable being waited on * @condition: the condition to wait for * @@ -492,7 +493,7 @@ do { \ * wait_var_event_mutex - wait for a variable to be updated under a mutex * @var: the address of the variable being waited on * @condition: condition to wait for - * @mutex: the mutex which protects updates to the variable + * @lock: the mutex which protects updates to the variable * * Wait for a condition which can only be reliably tested while holding * a mutex. The variables assessed in the condition will normal be diff --git a/include/linux/watchdog.h b/include/linux/watchdog.h index 62cdd26fd025..29cd03686154 100644 --- a/include/linux/watchdog.h +++ b/include/linux/watchdog.h @@ -26,7 +26,8 @@ struct watchdog_device; struct watchdog_core_data; struct watchdog_governor; -/** struct watchdog_ops - The watchdog-devices operations +/** + * struct watchdog_ops - The watchdog-devices operations * * @owner: The module owner. * @start: The routine for starting the watchdog device. @@ -59,7 +60,8 @@ struct watchdog_ops { long (*ioctl)(struct watchdog_device *, unsigned int, unsigned long); }; -/** struct watchdog_device - The structure that defines a watchdog device +/** + * struct watchdog_device - The structure that defines a watchdog device * * @id: The watchdog's ID. (Allocated by watchdog_register_device) * @parent: The parent bus device @@ -83,6 +85,8 @@ struct watchdog_ops { * Replaces max_timeout if specified. * @reboot_nb: The notifier block to stop watchdog on reboot. * @restart_nb: The notifier block to register a restart function. + * @pm_nb: The notifier block to stop watchdog on suspend and restart it + * on resume. * @driver_data:Pointer to the drivers private data. * @wd_data: Pointer to watchdog core internal data. * @status: Field that contains the devices internal status bits. diff --git a/include/linux/wmi.h b/include/linux/wmi.h index 75cb0c7cfe57..defcb624a7e2 100644 --- a/include/linux/wmi.h +++ b/include/linux/wmi.h @@ -11,23 +11,19 @@ #include <linux/compiler_attributes.h> #include <linux/device.h> #include <linux/acpi.h> -#include <linux/mod_devicetable.h> +#include <linux/device-id/wmi.h> #include <linux/types.h> /** * struct wmi_device - WMI device structure * @dev: Device associated with this WMI device * @setable: True for devices implementing the Set Control Method - * @driver_override: Driver name to force a match; do not set directly, - * because core frees it; use driver_set_override() to - * set or clear it. * * This represents WMI devices discovered by the WMI driver core. */ struct wmi_device { struct device dev; bool setable; - const char *driver_override; }; /** @@ -68,9 +64,13 @@ ssize_t wmi_string_from_utf8s(struct wmi_string *str, size_t max_chars, const u8 size_t src_length); int wmidev_invoke_method(struct wmi_device *wdev, u8 instance, u32 method_id, - const struct wmi_buffer *in, struct wmi_buffer *out); + const struct wmi_buffer *in, struct wmi_buffer *out, size_t min_size); -int wmidev_query_block(struct wmi_device *wdev, u8 instance, struct wmi_buffer *out); +int wmidev_invoke_procedure(struct wmi_device *wdev, u8 instance, u32 method_id, + const struct wmi_buffer *in); + +int wmidev_query_block(struct wmi_device *wdev, u8 instance, struct wmi_buffer *out, + size_t min_size); int wmidev_set_block(struct wmi_device *wdev, u8 instance, const struct wmi_buffer *in); @@ -87,7 +87,7 @@ u8 wmidev_instance_count(struct wmi_device *wdev); * struct wmi_driver - WMI driver structure * @driver: Driver model structure * @id_table: List of WMI GUIDs supported by this driver - * @no_notify_data: Driver supports WMI events which provide no event data + * @min_event_size: Minimum event payload size supported by this driver * @no_singleton: Driver can be instantiated multiple times * @probe: Callback for device binding * @remove: Callback for device unbinding @@ -97,11 +97,14 @@ u8 wmidev_instance_count(struct wmi_device *wdev); * * This represents WMI drivers which handle WMI devices. The data inside the buffer * passed to the @notify_new callback is guaranteed to be aligned on a 8-byte boundary. + * The minimum supported size for said buffer can be specified using @min_event_size. + * WMI drivers that still use the deprecated @notify callback can still set @min_event_size + * to 0 in order to signal that they support WMI events which provide no event data. */ struct wmi_driver { struct device_driver driver; const struct wmi_device_id *id_table; - bool no_notify_data; + size_t min_event_size; bool no_singleton; int (*probe)(struct wmi_device *wdev, const void *context); diff --git a/include/linux/workqueue.h b/include/linux/workqueue.h index a4749f56398f..c8a36423cb34 100644 --- a/include/linux/workqueue.h +++ b/include/linux/workqueue.h @@ -131,8 +131,9 @@ struct rcu_work { enum wq_affn_scope { WQ_AFFN_DFL, /* use system default */ WQ_AFFN_CPU, /* one pod per CPU */ - WQ_AFFN_SMT, /* one pod poer SMT */ + WQ_AFFN_SMT, /* one pod per SMT */ WQ_AFFN_CACHE, /* one pod per LLC */ + WQ_AFFN_CACHE_SHARD, /* synthetic sub-LLC shards */ WQ_AFFN_NUMA, /* one pod per NUMA node */ WQ_AFFN_SYSTEM, /* one pod across the whole system */ @@ -408,6 +409,7 @@ enum wq_flags { __WQ_DRAINING = 1 << 16, /* internal: workqueue is draining */ __WQ_ORDERED = 1 << 17, /* internal: workqueue is ordered */ __WQ_LEGACY = 1 << 18, /* internal: create*_workqueue() */ + __WQ_DEPRECATED = 1 << 19, /* internal: workqueue is deprecated */ /* BH wq only allows the following flags */ __WQ_BH_ALLOWS = WQ_BH | WQ_HIGHPRI | WQ_PERCPU, @@ -440,6 +442,9 @@ enum wq_consts { * system_long_wq is similar to system_percpu_wq but may host long running * works. Queue flushing might take relatively long. * + * system_dfl_long_wq is similar to system_dfl_wq but it may host long running + * works. + * * system_dfl_wq is unbound workqueue. Workers are not bound to * any specific CPU, not concurrency managed, and all queued works are * executed immediately as long as max_active limit is not reached and @@ -468,6 +473,7 @@ extern struct workqueue_struct *system_power_efficient_wq; extern struct workqueue_struct *system_freezable_power_efficient_wq; extern struct workqueue_struct *system_bh_wq; extern struct workqueue_struct *system_bh_highpri_wq; +extern struct workqueue_struct *system_dfl_long_wq; void workqueue_softirq_action(bool highpri); void workqueue_softirq_dead(unsigned int cpu); @@ -512,6 +518,28 @@ __printf(1, 4) struct workqueue_struct * alloc_workqueue_noprof(const char *fmt, unsigned int flags, int max_active, ...); #define alloc_workqueue(...) alloc_hooks(alloc_workqueue_noprof(__VA_ARGS__)) +/** + * devm_alloc_workqueue - Resource-managed allocate a workqueue + * @dev: Device to allocate workqueue for + * @fmt: printf format for the name of the workqueue + * @flags: WQ_* flags + * @max_active: max in-flight work items, 0 for default + * @...: args for @fmt + * + * Resource managed workqueue, see alloc_workqueue() for details. + * + * The workqueue will be automatically destroyed on driver detach. Typically + * this should be used in drivers already relying on devm interafaces. + * + * RETURNS: + * Pointer to the allocated workqueue on success, %NULL on failure. + */ +__printf(2, 5) struct workqueue_struct * +devm_alloc_workqueue_noprof(struct device *dev, const char *fmt, + unsigned int flags, int max_active, ...); +#define devm_alloc_workqueue(...) \ + alloc_hooks(devm_alloc_workqueue_noprof(__VA_ARGS__)) + #ifdef CONFIG_LOCKDEP /** * alloc_workqueue_lockdep_map - allocate a workqueue with user-defined lockdep_map @@ -568,6 +596,8 @@ alloc_workqueue_lockdep_map(const char *fmt, unsigned int flags, int max_active, */ #define alloc_ordered_workqueue(fmt, flags, args...) \ alloc_workqueue(fmt, WQ_UNBOUND | __WQ_ORDERED | (flags), 1, ##args) +#define devm_alloc_ordered_workqueue(dev, fmt, flags, args...) \ + devm_alloc_workqueue(dev, fmt, WQ_UNBOUND | __WQ_ORDERED | (flags), 1, ##args) #define create_workqueue(name) \ alloc_workqueue("%s", __WQ_LEGACY | WQ_MEM_RECLAIM | WQ_PERCPU, 1, (name)) @@ -631,6 +661,7 @@ extern void workqueue_set_min_active(struct workqueue_struct *wq, int min_active); extern struct work_struct *current_work(void); extern bool current_is_workqueue_rescuer(void); +extern bool current_is_workqueue_mem_reclaim(void); extern bool workqueue_congested(int cpu, struct workqueue_struct *wq); extern unsigned int work_busy(struct work_struct *work); extern __printf(1, 2) void set_worker_desc(const char *fmt, ...); @@ -712,14 +743,14 @@ static inline bool schedule_work_on(int cpu, struct work_struct *work) } /** - * schedule_work - put work task in global workqueue + * schedule_work - put work task in per-CPU workqueue * @work: job to be done * - * Returns %false if @work was already on the kernel-global workqueue and + * Returns %false if @work was already on the system per-CPU workqueue and * %true otherwise. * - * This puts a job in the kernel-global workqueue if it was not already - * queued and leaves it in the same position on the kernel-global + * This puts a job in the system per-CPU workqueue if it was not already + * queued and leaves it in the same position on the system per-CPU * workqueue otherwise. * * Shares the same memory-ordering properties of queue_work(), cf. the @@ -783,6 +814,8 @@ extern void __warn_flushing_systemwide_wq(void) _wq == system_highpri_wq) || \ (__builtin_constant_p(_wq == system_long_wq) && \ _wq == system_long_wq) || \ + (__builtin_constant_p(_wq == system_dfl_long_wq) && \ + _wq == system_dfl_long_wq) || \ (__builtin_constant_p(_wq == system_dfl_wq) && \ _wq == system_dfl_wq) || \ (__builtin_constant_p(_wq == system_freezable_wq) && \ @@ -796,12 +829,12 @@ extern void __warn_flushing_systemwide_wq(void) }) /** - * schedule_delayed_work_on - queue work in global workqueue on CPU after delay + * schedule_delayed_work_on - queue work in per-CPU workqueue on CPU after delay * @cpu: cpu to use * @dwork: job to be done * @delay: number of jiffies to wait * - * After waiting for a given time this puts a job in the kernel-global + * After waiting for a given time this puts a job in the system per-CPU * workqueue on the specified CPU. */ static inline bool schedule_delayed_work_on(int cpu, struct delayed_work *dwork, @@ -811,11 +844,11 @@ static inline bool schedule_delayed_work_on(int cpu, struct delayed_work *dwork, } /** - * schedule_delayed_work - put work task in global workqueue after delay + * schedule_delayed_work - put work task in per-CPU workqueue after delay * @dwork: job to be done * @delay: number of jiffies to wait or 0 for immediate execution * - * After waiting for a given time this puts a job in the kernel-global + * After waiting for a given time this puts a job in the system per-CPU * workqueue. */ static inline bool schedule_delayed_work(struct delayed_work *dwork, diff --git a/include/linux/writeback.h b/include/linux/writeback.h index e530112c4b3a..b749a9a5a5ee 100644 --- a/include/linux/writeback.h +++ b/include/linux/writeback.h @@ -11,7 +11,7 @@ #include <linux/flex_proportions.h> #include <linux/backing-dev-defs.h> #include <linux/blk_types.h> -#include <linux/pagevec.h> +#include <linux/folio_batch.h> struct bio; @@ -244,7 +244,7 @@ void wbc_attach_fdatawrite_inode(struct writeback_control *wbc, struct inode *inode); /** - * wbc_init_bio - writeback specific initializtion of bio + * wbc_init_bio - writeback specific initialization of bio * @wbc: writeback_control for the writeback in progress * @bio: bio to be initialized * diff --git a/include/linux/ww_mutex.h b/include/linux/ww_mutex.h index 85b1fff02fde..0c95ead5a297 100644 --- a/include/linux/ww_mutex.h +++ b/include/linux/ww_mutex.h @@ -181,7 +181,7 @@ static inline void ww_acquire_init(struct ww_acquire_ctx *ctx, * data structures. */ static inline void ww_acquire_done(struct ww_acquire_ctx *ctx) - __releases(ctx) __acquires_shared(ctx) __no_context_analysis + __must_hold(ctx) { #ifdef DEBUG_WW_MUTEXES lockdep_assert_held(ctx); @@ -199,7 +199,7 @@ static inline void ww_acquire_done(struct ww_acquire_ctx *ctx) * mutexes have been released with ww_mutex_unlock. */ static inline void ww_acquire_fini(struct ww_acquire_ctx *ctx) - __releases_shared(ctx) __no_context_analysis + __releases(ctx) __no_context_analysis { #ifdef CONFIG_DEBUG_LOCK_ALLOC mutex_release(&ctx->first_lock_dep_map, _THIS_IP_); diff --git a/include/linux/xattr.h b/include/linux/xattr.h index 296b5ee5c979..54ac3cbc133f 100644 --- a/include/linux/xattr.h +++ b/include/linux/xattr.h @@ -16,6 +16,7 @@ #include <linux/types.h> #include <linux/spinlock.h> #include <linux/mm.h> +#include <linux/rhashtable-types.h> #include <linux/user_namespace.h> #include <uapi/linux/xattr.h> @@ -105,32 +106,67 @@ static inline const char *xattr_prefix(const struct xattr_handler *handler) return handler->prefix ?: handler->name; } -struct simple_xattrs { - struct rb_root rb_root; - rwlock_t lock; +struct simple_xattr_cache { + struct rhashtable *ht; }; struct simple_xattr { - struct rb_node rb_node; + struct rhash_head hash_node; + struct list_head *parent; + struct list_head node; + struct rcu_head rcu; char *name; size_t size; char value[] __counted_by(size); }; -void simple_xattrs_init(struct simple_xattrs *xattrs); -void simple_xattrs_free(struct simple_xattrs *xattrs, size_t *freed_space); +#define SIMPLE_XATTR_MAX_NR 128 +#define SIMPLE_XATTR_MAX_SIZE (128 << 10) + +struct simple_xattr_limits { + atomic_t nr_xattrs; /* current user.* xattr count */ + atomic_t xattr_size; /* current total user.* value bytes */ +}; + +static inline void simple_xattr_limits_init(struct simple_xattr_limits *limits) +{ + atomic_set(&limits->nr_xattrs, 0); + atomic_set(&limits->xattr_size, 0); +} + +void simple_xattrs_free(struct simple_xattr_cache *cache, struct list_head *xattrs, + size_t *freed_space); size_t simple_xattr_space(const char *name, size_t size); struct simple_xattr *simple_xattr_alloc(const void *value, size_t size); void simple_xattr_free(struct simple_xattr *xattr); -int simple_xattr_get(struct simple_xattrs *xattrs, const char *name, - void *buffer, size_t size); -struct simple_xattr *simple_xattr_set(struct simple_xattrs *xattrs, +void simple_xattr_free_rcu(struct simple_xattr *xattr); +int simple_xattr_get(struct simple_xattr_cache *cache, struct list_head *xattrs, + const char *name, void *buffer, size_t size); +struct simple_xattr *simple_xattr_set(struct simple_xattr_cache *cache, + struct list_head *xattrs, const char *name, const void *value, size_t size, int flags); -ssize_t simple_xattr_list(struct inode *inode, struct simple_xattrs *xattrs, +int simple_xattr_set_limited(struct simple_xattr_cache *cache, + struct list_head *xattrs, + struct simple_xattr_limits *limits, + const char *name, const void *value, + size_t size, int flags); +ssize_t simple_xattr_list(struct inode *inode, struct list_head *xattrs, char *buffer, size_t size); -void simple_xattr_add(struct simple_xattrs *xattrs, - struct simple_xattr *new_xattr); +int simple_xattr_add(struct simple_xattr_cache *cache, struct list_head *xattrs, + struct simple_xattr *new_xattr); +int simple_xattr_add_limited(struct simple_xattr_cache *cache, + struct list_head *xattrs, + struct simple_xattr_limits *limits, + struct simple_xattr *new_xattr); int xattr_list_one(char **buffer, ssize_t *remaining_size, const char *name); +void simple_xattr_cache_cleanup(struct simple_xattr_cache *cache); + +DEFINE_CLASS(simple_xattr, + struct simple_xattr *, + if (!IS_ERR_OR_NULL(_T)) simple_xattr_free(_T), + simple_xattr_alloc(value, size), + const void *value, size_t size) + #endif /* _LINUX_XATTR_H */ diff --git a/include/linux/yam.h b/include/linux/yam.h deleted file mode 100644 index a29b04fa1e66..000000000000 --- a/include/linux/yam.h +++ /dev/null @@ -1,67 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/*****************************************************************************/ - -/* - * yam.h -- YAM radio modem driver. - * - * Copyright (C) 1998 Frederic Rible F1OAT (frible@teaser.fr) - * Adapted from baycom.c driver written by Thomas Sailer (sailer@ife.ee.ethz.ch) - * - * Please note that the GPL allows you to use the driver, NOT the radio. - * In order to use the radio, you need a license from the communications - * authority of your country. - */ - -/*****************************************************************************/ - -#define SIOCYAMRESERVED (0) -#define SIOCYAMSCFG (1) /* Set configuration */ -#define SIOCYAMGCFG (2) /* Get configuration */ -#define SIOCYAMSMCS (3) /* Set mcs data */ - -#define YAM_IOBASE (1 << 0) -#define YAM_IRQ (1 << 1) -#define YAM_BITRATE (1 << 2) /* Bit rate of radio port ->57600 */ -#define YAM_MODE (1 << 3) /* 0=simplex 1=duplex 2=duplex+tempo */ -#define YAM_HOLDDLY (1 << 4) /* duplex tempo (sec) */ -#define YAM_TXDELAY (1 << 5) /* Tx Delay (ms) */ -#define YAM_TXTAIL (1 << 6) /* Tx Tail (ms) */ -#define YAM_PERSIST (1 << 7) /* Persist (ms) */ -#define YAM_SLOTTIME (1 << 8) /* Slottime (ms) */ -#define YAM_BAUDRATE (1 << 9) /* Baud rate of rs232 port ->115200 */ - -#define YAM_MAXBITRATE 57600 -#define YAM_MAXBAUDRATE 115200 -#define YAM_MAXMODE 2 -#define YAM_MAXHOLDDLY 99 -#define YAM_MAXTXDELAY 999 -#define YAM_MAXTXTAIL 999 -#define YAM_MAXPERSIST 255 -#define YAM_MAXSLOTTIME 999 - -#define YAM_FPGA_SIZE 5302 - -struct yamcfg { - unsigned int mask; /* Mask of commands */ - unsigned int iobase; /* IO Base of COM port */ - unsigned int irq; /* IRQ of COM port */ - unsigned int bitrate; /* Bit rate of radio port */ - unsigned int baudrate; /* Baud rate of the RS232 port */ - unsigned int txdelay; /* TxDelay */ - unsigned int txtail; /* TxTail */ - unsigned int persist; /* Persistence */ - unsigned int slottime; /* Slottime */ - unsigned int mode; /* mode 0 (simp), 1(Dupl), 2(Dupl+delay) */ - unsigned int holddly; /* PTT delay in FullDuplex 2 mode */ -}; - -struct yamdrv_ioctl_cfg { - int cmd; - struct yamcfg cfg; -}; - -struct yamdrv_ioctl_mcs { - int cmd; - unsigned int bitrate; - unsigned char bits[YAM_FPGA_SIZE]; -}; diff --git a/include/linux/zorro.h b/include/linux/zorro.h index f36c8d39553d..4514c3109deb 100644 --- a/include/linux/zorro.h +++ b/include/linux/zorro.h @@ -17,7 +17,7 @@ #include <linux/device.h> #include <linux/init.h> #include <linux/ioport.h> -#include <linux/mod_devicetable.h> +#include <linux/device-id/zorro.h> #include <asm/zorro.h> |