diff options
Diffstat (limited to 'tools')
77 files changed, 2619 insertions, 544 deletions
diff --git a/tools/Kconfig b/tools/Kconfig index 667807b3317..5c75af48fe3 100644 --- a/tools/Kconfig +++ b/tools/Kconfig @@ -98,7 +98,7 @@ config TOOLS_SHA512 config TOOLS_MKEFICAPSULE bool "Build efimkcapsule command" - default y if EFI_CAPSULE_ON_DISK + default y if EFI_LOADER help This command allows users to create a UEFI capsule file and, optionally sign that file. If you want to enable UEFI capsule diff --git a/tools/binman/binman.rst b/tools/binman/binman.rst index 230e055667f..0cafc36bdcb 100644 --- a/tools/binman/binman.rst +++ b/tools/binman/binman.rst @@ -711,6 +711,13 @@ missing-msg: information about what needs to be fixed. See missing-blob-help for the message for each tag. +assume-size: + Sets the assumed size of a blob entry if it is missing. This allows for a + check that the rest of the image fits into the available space, even when + the contents are not available. If the entry is missing, Binman will use + this assumed size for the entry size, including creating a fake file of that + size if requested. + no-expanded: By default binman substitutes entries with expanded versions if available, so that a `u-boot` entry type turns into `u-boot-expanded`, for example. The @@ -1204,7 +1211,7 @@ Templates provide a simple way to handle this:: spi-image { filename = "image-spi.bin"; - insert-template = <&fit>; + insert-template = <&common_part>; /* things specific to SPI follow */ footer { @@ -1217,7 +1224,7 @@ Templates provide a simple way to handle this:: mmc-image { filename = "image-mmc.bin"; - insert-template = <&fit>; + insert-template = <&common_part>; /* things specific to MMC follow */ footer { diff --git a/tools/binman/bintools.rst b/tools/binman/bintools.rst index 1336f4d0115..cd05ad8cb26 100644 --- a/tools/binman/bintools.rst +++ b/tools/binman/bintools.rst @@ -10,6 +10,15 @@ binaries. It is fairly easy to create new bintools. Just add a new file to the +Bintool: bootgen: Generate bootable fsbl image for zynq/zynqmp +-------------------------------------------------------------- + +This bintools supports running Xilinx "bootgen" in order +to generate a bootable, authenticated image form an SPL. + + + + Bintool: bzip2: Compression/decompression using the bzip2 algorithm ------------------------------------------------------------------- @@ -37,6 +46,33 @@ Documentation about CBFS is at https://www.coreboot.org/CBFS +Bintool: cst: Image generation for U-Boot +----------------------------------------- + +This bintool supports running `cst` with some basic parameters as +needed by binman. + + + +Bintool: fdt_add_pubkey: Add public key to control dtb (spl or u-boot proper) +----------------------------------------------------------------------------- + +This bintool supports running `fdt_add_pubkey`. + +Normally mkimage adds signature information to the control dtb. However +binman images are built independent from each other. Thus it is required +to add the public key separately from mkimage. + + + +Bintool: fdtgrep: Handles the 'fdtgrep' tool +-------------------------------------------- + +This bintool supports running `fdtgrep` with parameters suitable for +producing SPL devicetrees from the main one. + + + Bintool: fiptool: Image generation for ARM Trusted Firmware ----------------------------------------------------------- @@ -143,11 +179,20 @@ Documentation is available via:: +Bintool: mkeficapsule: Handles the 'mkeficapsule' tool +------------------------------------------------------ + +This bintool is used for generating the EFI capsules. The +capsule generation parameters can either be specified through +commandline, or through a config file. + + + Bintool: mkimage: Image generation for U-Boot --------------------------------------------- This bintool supports running `mkimage` with some basic parameters as -neeed by binman. +needed by binman. Normally binman uses the mkimage built by U-Boot. But when run outside the U-Boot build system, binman can use the version installed in your system. @@ -194,25 +239,3 @@ Documentation is available via:: -Bintool: fdt_add_pubkey: Add public key to device tree ------------------------------------------------------- - -This bintool supports running `fdt_add_pubkey` in order to add a public -key coming from a certificate to a device-tree. - -Normally signing is done using `mkimage` in context of `binman sign`. However, -in this process the public key is not added to the stage before u-boot proper. -Using `fdt_add_pubkey` the key can be injected to the SPL independent of -`mkimage` - - - -Bintool: bootgen: Sign ZynqMP FSBL image ----------------------------------------- - -This bintool supports running `bootgen` in order to sign a SPL for ZynqMP -devices. - -The bintool automatically creates an appropriate input image file (.bif) for -bootgen based on the passed arguments. The output is a bootable, -authenticated `boot.bin` file. diff --git a/tools/binman/btool/cbfstool.py b/tools/binman/btool/cbfstool.py index 29be2d8a2b5..2d8559abb2b 100644 --- a/tools/binman/btool/cbfstool.py +++ b/tools/binman/btool/cbfstool.py @@ -214,6 +214,7 @@ class Bintoolcbfstool(bintool.Bintool): """ if method != bintool.FETCH_BIN: return None + # Version 4.22.01 fname, tmpdir = self.fetch_from_drive( - '1IOnE0Qvy97d-0WOCwF64xBGpKSY2sMtJ') + '1gxNxRuJgD0Iiy9LAPCSB_0959eJCp98g') return fname, tmpdir diff --git a/tools/binman/btool/cst.py b/tools/binman/btool/cst.py new file mode 100644 index 00000000000..30e78bdbbd9 --- /dev/null +++ b/tools/binman/btool/cst.py @@ -0,0 +1,48 @@ +# SPDX-License-Identifier: GPL-2.0+ +# Copyright 2024 Marek Vasut <marex@denx.de> +# +"""Bintool implementation for cst""" + +import re + +from binman import bintool + +class Bintoolcst(bintool.Bintool): + """Image generation for U-Boot + + This bintool supports running `cst` with some basic parameters as + needed by binman. + """ + def __init__(self, name): + super().__init__(name, 'Sign NXP i.MX image') + + # pylint: disable=R0913 + def run(self, output_fname=None): + """Run cst + + Args: + output_fname: Output filename to write to + """ + args = [] + if output_fname: + args += ['-o', output_fname] + return self.run_cmd(*args) + + def fetch(self, method): + """Fetch handler for cst + + This installs cst using the apt utility. + + Args: + method (FETCH_...): Method to use + + Returns: + True if the file was fetched and now installed, None if a method + other than FETCH_BIN was requested + + Raises: + Valuerror: Fetching could not be completed + """ + if method != bintool.FETCH_BIN: + return None + return self.apt_install('imx-code-signing-tool') diff --git a/tools/binman/btool/fdt_add_pubkey.py b/tools/binman/btool/fdt_add_pubkey.py index a50774200c9..75a9716ad72 100644 --- a/tools/binman/btool/fdt_add_pubkey.py +++ b/tools/binman/btool/fdt_add_pubkey.py @@ -32,6 +32,10 @@ class Bintoolfdt_add_pubkey(bintool.Bintool): verified for the image / configuration to be considered valid. algo (str): Cryptographic algorithm. Optional parameter, default value: sha1,rsa2048 + + Returns: + CommandResult: Resulting output from the bintool, or None if the + tool is not present """ args = [] if algo: diff --git a/tools/binman/btool/fdtgrep.py b/tools/binman/btool/fdtgrep.py new file mode 100644 index 00000000000..da1f8c7bf4e --- /dev/null +++ b/tools/binman/btool/fdtgrep.py @@ -0,0 +1,137 @@ +# SPDX-License-Identifier: GPL-2.0+ +# Copyright 2022 Google LLC +# +"""Bintool implementation for fdtgrep + +fdtgrepprovides a way to grep devicetree-binary files to extract or remove +certain elements. + +Usage: fdtgrep - extract portions from device tree + +Usage: + fdtgrep <options> <dt file>|- + +Output formats are: + dts - device tree soure text + dtb - device tree blob (sets -Hmt automatically) + bin - device tree fragment (may not be a valid .dtb) + +Options: -[haAc:b:C:defg:G:HIlLmn:N:o:O:p:P:rRsStTvhV] + -a, --show-address Display address + -A, --colour Show all nodes/tags, colour those that match + -b, --include-node-with-prop <arg> Include contains containing property + -c, --include-compat <arg> Compatible nodes to include in grep + -C, --exclude-compat <arg> Compatible nodes to exclude in grep + -d, --diff Diff: Mark matching nodes with +, others with - + -e, --enter-node Enter direct subnode names of matching nodes + -f, --show-offset Display offset + -g, --include-match <arg> Node/property/compatible string to include in grep + -G, --exclude-match <arg> Node/property/compatible string to exclude in grep + -H, --show-header Output a header + -I, --show-version Put "/dts-v1/;" on first line of dts output + -l, --list-regions Output a region list + -L, --list-strings List strings in string table + -m, --include-mem Include mem_rsvmap section in binary output + -n, --include-node <arg> Node to include in grep + -N, --exclude-node <arg> Node to exclude in grep + -p, --include-prop <arg> Property to include in grep + -P, --exclude-prop <arg> Property to exclude in grep + -r, --remove-strings Remove unused strings from string table + -R, --include-root Include root node and all properties + -s, --show-subnodes Show all subnodes matching nodes + -S, --skip-supernodes Don't include supernodes of matching nodes + -t, --show-stringtab Include string table in binary output + -T, --show-aliases Include matching aliases in output + -o, --out <arg> -o <output file> + -O, --out-format <arg> -O <output format> + -v, --invert-match Invert the sense of matching (select non-matching lines) + -h, --help Print this help and exit + -V, --version Print version and exit +""" + +import tempfile + +from u_boot_pylib import tools +from binman import bintool + +class Bintoolfdtgrep(bintool.Bintool): + """Handles the 'fdtgrep' tool + + This bintool supports running `fdtgrep` with parameters suitable for + producing SPL devicetrees from the main one. + """ + def __init__(self, name): + super().__init__(name, 'Grep devicetree files') + + def create_for_phase(self, infile, phase, outfile, remove_props): + """Create the FDT for a particular phase + + Args: + infile (str): Input filename containing the full FDT contents (with + all nodes and properties) + phase (str): Phase to generate for ('tpl', 'vpl', 'spl') + outfile (str): Output filename to write the grepped FDT contents to + (with only neceesary nodes and properties) + + Returns: + CommandResult: Resulting output from the bintool, or None if the + tool is not present + """ + if phase == 'tpl': + tag = 'bootph-pre-sram' + elif phase == 'vpl': + tag = 'bootph-verify' + elif phase == 'spl': + tag = 'bootph-pre-ram' + else: + raise ValueError(f"Invalid U-Boot phase '{phase}': Use tpl/vpl/spl") + + # These args mirror those in cmd_fdtgrep in scripts/Makefile.lib + # First do the first stage + with tempfile.NamedTemporaryFile(prefix='fdtgrep.tmp', + dir=tools.get_output_dir()) as tmp: + args = [ + infile, + '-o', tmp.name, + '-b', 'bootph-all', + '-b', tag, + '-u', + '-RT', + '-n', '/chosen', + '-n', '/config', + '-O', 'dtb', + ] + self.run_cmd(*args) + args = [ + tmp.name, + '-o', outfile, + '-r', + '-O', 'dtb', + '-P', 'bootph-all', + '-P', 'bootph-pre-ram', + '-P', 'bootph-pre-sram', + '-P', 'bootph-verify', + ] + for prop_name in remove_props: + args += ['-P', prop_name] + return self.run_cmd(*args) + + def fetch(self, method): + """Fetch handler for fdtgrep + + This installs fdtgrep using the apt utility, which assumes that it is + packaged in u-boot tools, which it is not. + + Args: + method (FETCH_...): Method to use + + Returns: + True if the file was fetched and now installed, None if a method + other than FETCH_BIN was requested + + Raises: + Valuerror: Fetching could not be completed + """ + if method != bintool.FETCH_BIN: + return None + return self.apt_install('u-boot-tools') diff --git a/tools/binman/control.py b/tools/binman/control.py index 2f00279232b..542c2b45644 100644 --- a/tools/binman/control.py +++ b/tools/binman/control.py @@ -617,6 +617,50 @@ def PrepareImagesAndDtbs(dtb_fname, select_images, update_fdt, use_expanded): dtb_item.Flush() return images +def CheckForProblems(image): + """Check for problems with image generation + + Shows warning about missing, faked or optional external blobs, as well as + missing bintools. + + Args: + image (Image): Image to process + + Returns: + bool: True if there are any problems which result in a non-functional + image + """ + missing_list = [] + image.CheckMissing(missing_list) + if missing_list: + tout.error("Image '%s' is missing external blobs and is non-functional: %s\n" % + (image.name, ' '.join([e.name for e in missing_list]))) + _ShowHelpForMissingBlobs(tout.ERROR, missing_list) + + faked_list = [] + image.CheckFakedBlobs(faked_list) + if faked_list: + tout.warning( + "Image '%s' has faked external blobs and is non-functional: %s\n" % + (image.name, ' '.join([os.path.basename(e.GetDefaultFilename()) + for e in faked_list]))) + + optional_list = [] + image.CheckOptional(optional_list) + if optional_list: + tout.warning( + "Image '%s' is missing optional external blobs but is still functional: %s\n" % + (image.name, ' '.join([e.name for e in optional_list]))) + _ShowHelpForMissingBlobs(tout.WARNING, optional_list) + + missing_bintool_list = [] + image.check_missing_bintools(missing_bintool_list) + if missing_bintool_list: + tout.warning( + "Image '%s' has missing bintools and is non-functional: %s\n" % + (image.name, ' '.join([os.path.basename(bintool.name) + for bintool in missing_bintool_list]))) + return any([missing_list, faked_list, missing_bintool_list]) def ProcessImage(image, update_fdt, write_map, get_contents=True, allow_resize=True, allow_missing=False, @@ -689,38 +733,11 @@ def ProcessImage(image, update_fdt, write_map, get_contents=True, if write_map: image.WriteMap() - missing_list = [] - image.CheckMissing(missing_list) - if missing_list: - tout.error("Image '%s' is missing external blobs and is non-functional: %s\n" % - (image.name, ' '.join([e.name for e in missing_list]))) - _ShowHelpForMissingBlobs(tout.ERROR, missing_list) - - faked_list = [] - image.CheckFakedBlobs(faked_list) - if faked_list: - tout.warning( - "Image '%s' has faked external blobs and is non-functional: %s\n" % - (image.name, ' '.join([os.path.basename(e.GetDefaultFilename()) - for e in faked_list]))) + has_problems = CheckForProblems(image) - optional_list = [] - image.CheckOptional(optional_list) - if optional_list: - tout.warning( - "Image '%s' is missing optional external blobs but is still functional: %s\n" % - (image.name, ' '.join([e.name for e in optional_list]))) - _ShowHelpForMissingBlobs(tout.WARNING, optional_list) - - missing_bintool_list = [] - image.check_missing_bintools(missing_bintool_list) - if missing_bintool_list: - tout.warning( - "Image '%s' has missing bintools and is non-functional: %s\n" % - (image.name, ' '.join([os.path.basename(bintool.name) - for bintool in missing_bintool_list]))) - return any([missing_list, faked_list, missing_bintool_list]) + image.WriteAlternates() + return has_problems def Binman(args): """The main control code for binman diff --git a/tools/binman/elf.py b/tools/binman/elf.py index 2ecc95f7eb8..a4694056391 100644 --- a/tools/binman/elf.py +++ b/tools/binman/elf.py @@ -275,7 +275,7 @@ def LookupAndWriteSymbols(elf_fname, entry, section, is_elf=False, return 0 base = syms.get(base_sym) if not base and not is_elf: - tout.debug('LookupAndWriteSymbols: no base') + tout.debug(f'LookupAndWriteSymbols: no base: elf_fname={elf_fname}, base_sym={base_sym}, is_elf={is_elf}') return 0 base_addr = 0 if is_elf else base.address count = 0 diff --git a/tools/binman/entries.rst b/tools/binman/entries.rst index 254afe76074..12482703782 100644 --- a/tools/binman/entries.rst +++ b/tools/binman/entries.rst @@ -11,6 +11,48 @@ features to produce new behaviours. +.. _etype_alternates_fdt: + +Entry: alternates-fdt: Entry that generates alternative sections for each devicetree provided +--------------------------------------------------------------------------------------------- + +When creating an image designed to boot on multiple models, each model +requires its own devicetree. This entry deals with selecting the correct +devicetree from a directory containing them. Each one is read in turn, then +used to produce section contents which are written to a file. This results +in a number of images, one for each model. + +For example this produces images for each .dtb file in the 'dtb' directory:: + + alternates-fdt { + fdt-list-dir = "dtb"; + filename-pattern = "NAME.bin"; + fdt-phase = "tpl"; + + section { + u-boot-tpl { + }; + }; + }; + +Each output file is named based on its input file, so an input file of +`model1.dtb` results in an output file of `model1.bin` (i.e. the `NAME` in +the `filename-pattern` property is replaced with the .dtb basename). + +Note that this entry type still produces contents for the 'main' image, in +that case using the normal dtb provided to Binman, e.g. `u-boot-tpl.dtb`. +But that image is unlikely to be useful, since it relates to whatever dtb +happened to be the default when U-Boot builds +(i.e. `CONFIG_DEFAULT_DEVICE_TREE`). However, Binman ensures that the size +of each of the alternates is the same as the 'default' one, so they can in +principle be 'slotted in' to the appropriate place in the main image. + +The optional `fdt-phase` property indicates the phase to build. In this +case, it etype runs fdtgrep to obtain the devicetree subset for that phase, +respecting the `bootph-xxx` tags in the devicetree. + + + .. _etype_atf_bl31: Entry: atf-bl31: ARM Trusted Firmware (ATF) BL31 blob @@ -470,11 +512,11 @@ updating the EC on startup via software sync. .. _etype_efi_capsule: -Entry: capsule: Entry for generating EFI Capsule files ------------------------------------------------------- +Entry: efi-capsule: Generate EFI capsules +----------------------------------------- -The parameters needed for generation of the capsules can be provided -as properties in the entry. +The parameters needed for generation of the capsules can +be provided as properties in the entry. Properties / Entry arguments: - image-index: Unique number for identifying corresponding @@ -495,9 +537,9 @@ Properties / Entry arguments: file. Mandatory property for generating signed capsules. - oem-flags - OEM flags to be passed through capsule header. - Since this is a subclass of Entry_section, all properties of the parent - class also apply here. Except for the properties stated as mandatory, the - rest of the properties are optional. +Since this is a subclass of Entry_section, all properties of the parent +class also apply here. Except for the properties stated as mandatory, the +rest of the properties are optional. For more details on the description of the capsule format, and the capsule update functionality, refer Section 8.5 and Chapter 23 in the `UEFI @@ -510,17 +552,17 @@ provided as a subnode of the capsule entry. A typical capsule entry node would then look something like this:: capsule { - type = "efi-capsule"; - image-index = <0x1>; - /* Image GUID for testing capsule update */ - image-guid = SANDBOX_UBOOT_IMAGE_GUID; - hardware-instance = <0x0>; - private-key = "path/to/the/private/key"; - public-key-cert = "path/to/the/public-key-cert"; - oem-flags = <0x8000>; + type = "efi-capsule"; + image-index = <0x1>; + /* Image GUID for testing capsule update */ + image-guid = SANDBOX_UBOOT_IMAGE_GUID; + hardware-instance = <0x0>; + private-key = "path/to/the/private/key"; + public-key-cert = "path/to/the/public-key-cert"; + oem-flags = <0x8000>; - u-boot { - }; + u-boot { + }; }; In the above example, the capsule payload is the U-Boot image. The @@ -534,8 +576,8 @@ payload using the blob-ext subnode. .. _etype_efi_empty_capsule: -Entry: efi-empty-capsule: Entry for generating EFI Empty Capsule files ----------------------------------------------------------------------- +Entry: efi-empty-capsule: Generate EFI empty capsules +----------------------------------------------------- The parameters needed for generation of the empty capsules can be provided as properties in the entry. @@ -551,22 +593,22 @@ update functionality, refer Section 8.5 and Chapter 23 in the `UEFI specification`_. For more information on the empty capsule, refer the sections 2.3.2 and 2.3.3 in the `Dependable Boot specification`_. -A typical accept empty capsule entry node would then look something -like this:: +A typical accept empty capsule entry node would then look something like +this:: empty-capsule { - type = "efi-empty-capsule"; - /* GUID of the image being accepted */ - image-type-id = SANDBOX_UBOOT_IMAGE_GUID; - capsule-type = "accept"; + type = "efi-empty-capsule"; + /* GUID of image being accepted */ + image-type-id = SANDBOX_UBOOT_IMAGE_GUID; + capsule-type = "accept"; }; -A typical revert empty capsule entry node would then look something -like this:: +A typical revert empty capsule entry node would then look something like +this:: empty-capsule { - type = "efi-empty-capsule"; - capsule-type = "revert"; + type = "efi-empty-capsule"; + capsule-type = "revert"; }; The empty capsules do not have any input payload image. @@ -815,6 +857,13 @@ The top-level 'fit' node supports the following special properties: fit,fdt-list-val = "dtb1", "dtb2"; + fit,fdt-list-dir + As an alternative to fit,fdt-list the list of device tree files + can be provided as a directory. Each .dtb file in the directory is + processed, , e.g.:: + + fit,fdt-list-dir = "arch/arm/dts + Substitutions ~~~~~~~~~~~~~ @@ -890,6 +939,7 @@ You can create config nodes in a similar way:: firmware = "atf"; loadables = "uboot"; fdt = "fdt-SEQ"; + fit,compatible; // optional }; }; @@ -899,6 +949,39 @@ for each of your two files. Note that if no devicetree files are provided (with '-a of-list' as above) then no nodes will be generated. +The 'fit,compatible' property (if present) is replaced with the compatible +string from the root node of the devicetree, so that things work correctly +with FIT's configuration-matching algortihm. + +Dealing with phases +~~~~~~~~~~~~~~~~~~~ + +FIT can be used to load firmware. In this case it may be necessary to run +the devicetree for each model through fdtgrep to remove unwanted properties. +The 'fit,fdt-phase' property can be provided to indicate the phase for which +the devicetree is intended. + +For example this indicates that the FDT should be processed for VPL:: + + images { + @fdt-SEQ { + description = "fdt-NAME"; + type = "flat_dt"; + compression = "none"; + fit,fdt-phase = "vpl"; + }; + }; + +Using this mechanism, it is possible to generate a FIT which can provide VPL +images for multiple models, with TPL selecting the correct model to use. The +same approach can of course be used for SPL images. + +Note that the `of-spl-remove-props` entryarg can be used to indicate +additional properties to remove. It is often used to remove properties like +`clock-names` and `pinctrl-names` which are not needed in SPL builds. + +See :ref:`fdtgrep_filter` for more information. + Generating nodes from an ELF file (split-elf) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -1521,6 +1604,28 @@ byte. +.. _etype_nxp_imx8mcst: + +Entry: nxp-imx8mcst: NXP i.MX8M CST .cfg file generator and cst invoker +----------------------------------------------------------------------- + +Properties / Entry arguments: + - nxp,loader-address - loader address (SPL text base) + + + +.. _etype_nxp_imx8mimage: + +Entry: nxp-imx8mimage: NXP i.MX8M imx8mimage .cfg file generator and mkimage invoker +------------------------------------------------------------------------------------ + +Properties / Entry arguments: + - nxp,boot-from - device to boot from (e.g. 'sd') + - nxp,loader-address - loader address (SPL text base) + - nxp,rom-version - BootROM version ('2' for i.MX8M Nano and Plus) + + + .. _etype_opensbi: Entry: opensbi: RISC-V OpenSBI fw_dynamic blob @@ -1929,6 +2034,12 @@ Properties / Entry arguments: - content: List of phandles to entries to sign - keyfile: Filename of file containing key to sign binary with - sha: Hash function to be used for signing + - auth-in-place: This is an integer field that contains two pieces + of information: + + - Lower Byte - Remains 0x02 as per our use case + ( 0x02: Move the authenticated binary back to the header ) + - Upper Byte - The Host ID of the core owning the firewall Output files: - input.<unique_name> - input file passed to openssl @@ -1937,6 +2048,35 @@ Output files: - cert.<unique_name> - output file generated by openssl (which is used as the entry contents) +Depending on auth-in-place information in the inputs, we read the +firewall nodes that describe the configurations of firewall that TIFS +will be doing after reading the certificate. + +The syntax of the firewall nodes are as such:: + + firewall-257-0 { + id = <257>; /* The ID of the firewall being configured */ + region = <0>; /* Region number to configure */ + + control = /* The control register */ + <(FWCTRL_EN | FWCTRL_LOCK | FWCTRL_BG | FWCTRL_CACHE)>; + + permissions = /* The permission registers */ + <((FWPRIVID_ALL << FWPRIVID_SHIFT) | + FWPERM_SECURE_PRIV_RWCD | + FWPERM_SECURE_USER_RWCD | + FWPERM_NON_SECURE_PRIV_RWCD | + FWPERM_NON_SECURE_USER_RWCD)>; + + /* More defines can be found in k3-security.h */ + + start_address = /* The Start Address of the firewall */ + <0x0 0x0>; + end_address = /* The End Address of the firewall */ + <0xff 0xffffffff>; + }; + + openssl signs the provided data, using the TI templated config file and writes the signature in this entry. This allows verification that the data is genuine. @@ -2233,8 +2373,6 @@ u-boot-spl-dtb SPL can access binman symbols at runtime. See :ref:`binman_fdt`. -in the binman README for more information. - The ELF file 'spl/u-boot-spl' must also be available for this to work, since binman uses that to look up symbols to write into the SPL binary. @@ -2423,8 +2561,6 @@ u-boot-tpl-dtb TPL can access binman symbols at runtime. See :ref:`binman_fdt`. -in the binman README for more information. - The ELF file 'tpl/u-boot-tpl' must also be available for this to work, since binman uses that to look up symbols to write into the TPL binary. @@ -2514,6 +2650,9 @@ in the binman README for more information. The ELF file 'vpl/u-boot-vpl' must also be available for this to work, since binman uses that to look up symbols to write into the VPL binary. +Note that this entry is automatically replaced with u-boot-vpl-expanded +unless --no-expanded is used or the node has a 'no-expanded' property. + .. _etype_u_boot_vpl_bss_pad: @@ -2602,8 +2741,8 @@ Properties / Entry arguments: This is the U-Boot VPL binary, It does not include a device tree blob at the end of it so may not be able to work without it, assuming VPL needs -a device tree to operate on your platform. You can add a u_boot_vpl_dtb -entry after this one, or use a u_boot_vpl entry instead, which normally +a device tree to operate on your platform. You can add a u-boot-vpl-dtb +entry after this one, or use a u-boot-vpl entry instead, which normally expands to a section containing u-boot-vpl-dtb, u-boot-vpl-bss-pad and u-boot-vpl-dtb diff --git a/tools/binman/entry.py b/tools/binman/entry.py index 42e0b7b9145..6d2f3789940 100644 --- a/tools/binman/entry.py +++ b/tools/binman/entry.py @@ -315,6 +315,7 @@ class Entry(object): self.overlap = fdt_util.GetBool(self._node, 'overlap') if self.overlap: self.required_props += ['offset', 'size'] + self.assume_size = fdt_util.GetInt(self._node, 'assume-size', 0) # This is only supported by blobs and sections at present self.compress = fdt_util.GetString(self._node, 'compress', 'none') @@ -812,7 +813,7 @@ class Entry(object): as missing """ print('''Binman Entry Documentation -=========================== +========================== This file describes the entry types supported by binman. These entry types can be placed in an image one by one to build up a final firmware image. It is @@ -1198,6 +1199,9 @@ features to produce new behaviours. self.uncomp_size = len(indata) if self.comp_bintool.is_present(): data = self.comp_bintool.compress(indata) + uniq = self.GetUniqueName() + fname = tools.get_output_filename(f'comp.{uniq}') + tools.write_file(fname, data) else: self.record_missing_bintool(self.comp_bintool) data = tools.get_bytes(0, 1024) @@ -1382,3 +1386,17 @@ features to produce new behaviours. def UpdateSignatures(self, privatekey_fname, algo, input_fname): self.Raise('Updating signatures is not supported with this entry type') + + def FdtContents(self, fdt_etype): + """Get the contents of an FDT for a particular phase + + Args: + fdt_etype (str): Filename of the phase of the FDT to return, e.g. + 'u-boot-tpl-dtb' + + Returns: + tuple: + fname (str): Filename of .dtb + bytes: Contents of FDT (possibly run through fdtgrep) + """ + return self.section.FdtContents(fdt_etype) diff --git a/tools/binman/entry_test.py b/tools/binman/entry_test.py index ac6582cf86a..40d74d401a2 100644 --- a/tools/binman/entry_test.py +++ b/tools/binman/entry_test.py @@ -103,7 +103,7 @@ class TestEntry(unittest.TestCase): ent = entry.Entry.Create(None, self.GetNode(), 'missing', missing_etype=True) self.assertTrue(isinstance(ent, Entry_blob)) - self.assertEquals('missing', ent.etype) + self.assertEqual('missing', ent.etype) def testDecompressData(self): """Test the DecompressData() method of the base class""" @@ -111,8 +111,8 @@ class TestEntry(unittest.TestCase): base.compress = 'lz4' bintools = {} base.comp_bintool = base.AddBintool(bintools, '_testing') - self.assertEquals(tools.get_bytes(0, 1024), base.CompressData(b'abc')) - self.assertEquals(tools.get_bytes(0, 1024), base.DecompressData(b'abc')) + self.assertEqual(tools.get_bytes(0, 1024), base.CompressData(b'abc')) + self.assertEqual(tools.get_bytes(0, 1024), base.DecompressData(b'abc')) def testLookupOffset(self): """Test the lookup_offset() method of the base class""" diff --git a/tools/binman/etype/alternates_fdt.py b/tools/binman/etype/alternates_fdt.py new file mode 100644 index 00000000000..808f535aa1b --- /dev/null +++ b/tools/binman/etype/alternates_fdt.py @@ -0,0 +1,132 @@ +# SPDX-License-Identifier: GPL-2.0+ +# Copyright 2024 Google LLC +# Written by Simon Glass <sjg@chromium.org> + +"""Entry-type module for producing multiple alternate sections""" + +import glob +import os + +from binman.entry import EntryArg +from binman.etype.section import Entry_section +from dtoc import fdt_util +from u_boot_pylib import tools + +class Entry_alternates_fdt(Entry_section): + """Entry that generates alternative sections for each devicetree provided + + When creating an image designed to boot on multiple models, each model + requires its own devicetree. This entry deals with selecting the correct + devicetree from a directory containing them. Each one is read in turn, then + used to produce section contents which are written to a file. This results + in a number of images, one for each model. + + For example this produces images for each .dtb file in the 'dtb' directory:: + + alternates-fdt { + fdt-list-dir = "dtb"; + filename-pattern = "NAME.bin"; + fdt-phase = "tpl"; + + section { + u-boot-tpl { + }; + }; + }; + + Each output file is named based on its input file, so an input file of + `model1.dtb` results in an output file of `model1.bin` (i.e. the `NAME` in + the `filename-pattern` property is replaced with the .dtb basename). + + Note that this entry type still produces contents for the 'main' image, in + that case using the normal dtb provided to Binman, e.g. `u-boot-tpl.dtb`. + But that image is unlikely to be useful, since it relates to whatever dtb + happened to be the default when U-Boot builds + (i.e. `CONFIG_DEFAULT_DEVICE_TREE`). However, Binman ensures that the size + of each of the alternates is the same as the 'default' one, so they can in + principle be 'slotted in' to the appropriate place in the main image. + + The optional `fdt-phase` property indicates the phase to build. In this + case, it etype runs fdtgrep to obtain the devicetree subset for that phase, + respecting the `bootph-xxx` tags in the devicetree. + """ + def __init__(self, section, etype, node): + super().__init__(section, etype, node) + self.fdt_list_dir = None + self.filename_pattern = None + self.required_props = ['fdt-list-dir'] + self._cur_fdt = None + self._fdt_phase = None + self.fdtgrep = None + self._fdt_dir = None + self._fdts = None + self._fname_pattern = None + self._remove_props = None + self.alternates = None + + def ReadNode(self): + """Read properties from the node""" + super().ReadNode() + self._fdt_dir = fdt_util.GetString(self._node, 'fdt-list-dir') + fname = tools.get_input_filename(self._fdt_dir) + fdts = glob.glob('*.dtb', root_dir=fname) + self._fdts = [os.path.splitext(f)[0] for f in fdts] + + self._fdt_phase = fdt_util.GetString(self._node, 'fdt-phase') + + # This is used by Image.WriteAlternates() + self.alternates = self._fdts + + self._fname_pattern = fdt_util.GetString(self._node, 'filename-pattern') + + self._remove_props = [] + props, = self.GetEntryArgsOrProps( + [EntryArg('of-spl-remove-props', str)], required=False) + if props: + self._remove_props = props.split() + + def FdtContents(self, fdt_etype): + # If there is no current FDT, just use the normal one + if not self._cur_fdt: + return self.section.FdtContents(fdt_etype) + + # Find the file to use + fname = os.path.join(self._fdt_dir, f'{self._cur_fdt}.dtb') + infile = tools.get_input_filename(fname) + + # Run fdtgrep if needed, to remove unwanted nodes and properties + if self._fdt_phase: + uniq = self.GetUniqueName() + outfile = tools.get_output_filename( + f'{uniq}.{self._cur_fdt}-{self._fdt_phase}.dtb') + self.fdtgrep.create_for_phase(infile, self._fdt_phase, outfile, + self._remove_props) + return outfile, tools.read_file(outfile) + return fname, tools.read_file(infile) + + def ProcessWithFdt(self, alt): + """Produce the contents of this entry, using a particular FDT blob + + Args: + alt (str): Name of the alternate + + Returns: + tuple: + str: Filename to use for the alternate's .bin file + bytes: Contents of this entry's section, using the selected FDT + """ + pattern = self._fname_pattern or 'NAME.bin' + fname = pattern.replace('NAME', alt) + + data = b'' + try: + self._cur_fdt = alt + self.ProcessContents() + data = self.GetPaddedData() + finally: + self._cur_fdt = None + return fname, data + + def AddBintools(self, btools): + super().AddBintools(btools) + self.fdtgrep = self.AddBintool(btools, 'fdtgrep') diff --git a/tools/binman/etype/blob.py b/tools/binman/etype/blob.py index 064fae50365..041e1122953 100644 --- a/tools/binman/etype/blob.py +++ b/tools/binman/etype/blob.py @@ -48,11 +48,16 @@ class Entry_blob(Entry): self.external and (self.optional or self.section.GetAllowMissing())) # Allow the file to be missing if not self._pathname: + if not fake_size and self.assume_size: + fake_size = self.assume_size self._pathname, faked = self.check_fake_fname(self._filename, fake_size) self.missing = True if not faked: - self.SetContents(b'') + content_size = 0 + if self.assume_size: # Ensure we get test coverage on next line + content_size = self.assume_size + self.SetContents(tools.get_bytes(0, content_size)) return True self.ReadBlobContents() diff --git a/tools/binman/etype/blob_dtb.py b/tools/binman/etype/blob_dtb.py index d543de9f759..b234323d7cf 100644 --- a/tools/binman/etype/blob_dtb.py +++ b/tools/binman/etype/blob_dtb.py @@ -41,12 +41,12 @@ class Entry_blob_dtb(Entry_blob): def ObtainContents(self, fake_size=0): """Get the device-tree from the list held by the 'state' module""" self._filename = self.GetDefaultFilename() - self._pathname, _ = state.GetFdtContents(self.GetFdtEtype()) + self._pathname, _ = self.FdtContents(self.GetFdtEtype()) return super().ReadBlobContents() def ProcessContents(self): """Re-read the DTB contents so that we get any calculated properties""" - _, indata = state.GetFdtContents(self.GetFdtEtype()) + _, indata = self.FdtContents(self.GetFdtEtype()) if self.compress == 'zstd' and self.prepend != 'length': self.Raise('The zstd compression requires a length header') @@ -57,7 +57,9 @@ class Entry_blob_dtb(Entry_blob): def GetFdtEtype(self): """Get the entry type of this device tree - This can be 'u-boot-dtb', 'u-boot-spl-dtb' or 'u-boot-tpl-dtb' + This can be 'u-boot-dtb', 'u-boot-spl-dtb', 'u-boot-tpl-dtb' or + 'u-boot-vpl-dtb' + Returns: Entry type if any, e.g. 'u-boot-dtb' """ diff --git a/tools/binman/etype/efi_capsule.py b/tools/binman/etype/efi_capsule.py index e3203717822..751f654bf31 100644 --- a/tools/binman/etype/efi_capsule.py +++ b/tools/binman/etype/efi_capsule.py @@ -36,23 +36,23 @@ class Entry_efi_capsule(Entry_section): be provided as properties in the entry. Properties / Entry arguments: - - image-index: Unique number for identifying corresponding - payload image. Number between 1 and descriptor count, i.e. - the total number of firmware images that can be updated. Mandatory - property. - - image-guid: Image GUID which will be used for identifying the - updatable image on the board. Mandatory property. - - hardware-instance: Optional number for identifying unique - hardware instance of a device in the system. Default value of 0 - for images where value is not to be used. - - fw-version: Value of image version that can be put on the capsule - through the Firmware Management Protocol(FMP) header. - - monotonic-count: Count used when signing an image. - - private-key: Path to PEM formatted .key private key file. Mandatory - property for generating signed capsules. - - public-key-cert: Path to PEM formatted .crt public key certificate - file. Mandatory property for generating signed capsules. - - oem-flags - OEM flags to be passed through capsule header. + - image-index: Unique number for identifying corresponding + payload image. Number between 1 and descriptor count, i.e. + the total number of firmware images that can be updated. Mandatory + property. + - image-guid: Image GUID which will be used for identifying the + updatable image on the board. Mandatory property. + - hardware-instance: Optional number for identifying unique + hardware instance of a device in the system. Default value of 0 + for images where value is not to be used. + - fw-version: Value of image version that can be put on the capsule + through the Firmware Management Protocol(FMP) header. + - monotonic-count: Count used when signing an image. + - private-key: Path to PEM formatted .key private key file. Mandatory + property for generating signed capsules. + - public-key-cert: Path to PEM formatted .crt public key certificate + file. Mandatory property for generating signed capsules. + - oem-flags - OEM flags to be passed through capsule header. Since this is a subclass of Entry_section, all properties of the parent class also apply here. Except for the properties stated as mandatory, the @@ -66,9 +66,9 @@ class Entry_efi_capsule(Entry_section): properties in the entry. The payload to be used in the capsule is to be provided as a subnode of the capsule entry. - A typical capsule entry node would then look something like this + A typical capsule entry node would then look something like this:: - capsule { + capsule { type = "efi-capsule"; image-index = <0x1>; /* Image GUID for testing capsule update */ @@ -80,7 +80,7 @@ class Entry_efi_capsule(Entry_section): u-boot { }; - }; + }; In the above example, the capsule payload is the U-Boot image. The capsule entry would read the contents of the payload and put them diff --git a/tools/binman/etype/efi_empty_capsule.py b/tools/binman/etype/efi_empty_capsule.py index 064bf9a77f0..1d99fbfb3bb 100644 --- a/tools/binman/etype/efi_empty_capsule.py +++ b/tools/binman/etype/efi_empty_capsule.py @@ -19,31 +19,33 @@ class Entry_efi_empty_capsule(Entry_section): be provided as properties in the entry. Properties / Entry arguments: - - image-guid: Image GUID which will be used for identifying the - updatable image on the board. Mandatory for accept capsule. - - capsule-type - String to indicate type of capsule to generate. Valid - values are 'accept' and 'revert'. + - image-guid: Image GUID which will be used for identifying the + updatable image on the board. Mandatory for accept capsule. + - capsule-type - String to indicate type of capsule to generate. Valid + values are 'accept' and 'revert'. For more details on the description of the capsule format, and the capsule update functionality, refer Section 8.5 and Chapter 23 in the `UEFI specification`_. For more information on the empty capsule, refer the sections 2.3.2 and 2.3.3 in the `Dependable Boot specification`_. - A typical accept empty capsule entry node would then look something like this + A typical accept empty capsule entry node would then look something like + this:: - empty-capsule { + empty-capsule { type = "efi-empty-capsule"; /* GUID of image being accepted */ image-type-id = SANDBOX_UBOOT_IMAGE_GUID; capsule-type = "accept"; - }; + }; - A typical revert empty capsule entry node would then look something like this + A typical revert empty capsule entry node would then look something like + this:: - empty-capsule { + empty-capsule { type = "efi-empty-capsule"; capsule-type = "revert"; - }; + }; The empty capsules do not have any input payload image. diff --git a/tools/binman/etype/fit.py b/tools/binman/etype/fit.py index 2c14b15b03c..ee44e5a1cd6 100644 --- a/tools/binman/etype/fit.py +++ b/tools/binman/etype/fit.py @@ -5,7 +5,9 @@ """Entry-type module for producing a FIT""" +import glob import libfdt +import os from binman.entry import Entry, EntryArg from binman.etype.section import Entry_section @@ -87,6 +89,13 @@ class Entry_fit(Entry_section): fit,fdt-list-val = "dtb1", "dtb2"; + fit,fdt-list-dir + As an alternative to fit,fdt-list the list of device tree files + can be provided as a directory. Each .dtb file in the directory is + processed, , e.g.:: + + fit,fdt-list-dir = "arch/arm/dts + Substitutions ~~~~~~~~~~~~~ @@ -162,6 +171,7 @@ class Entry_fit(Entry_section): firmware = "atf"; loadables = "uboot"; fdt = "fdt-SEQ"; + fit,compatible; // optional }; }; @@ -171,6 +181,40 @@ class Entry_fit(Entry_section): Note that if no devicetree files are provided (with '-a of-list' as above) then no nodes will be generated. + The 'fit,compatible' property (if present) is replaced with the compatible + string from the root node of the devicetree, so that things work correctly + with FIT's configuration-matching algortihm. + + Dealing with phases + ~~~~~~~~~~~~~~~~~~~ + + FIT can be used to load firmware. In this case it may be necessary to run + the devicetree for each model through fdtgrep to remove unwanted properties. + The 'fit,fdt-phase' property can be provided to indicate the phase for which + the devicetree is intended. + + For example this indicates that the FDT should be processed for VPL:: + + images { + @fdt-SEQ { + description = "fdt-NAME"; + type = "flat_dt"; + compression = "none"; + fit,fdt-phase = "vpl"; + }; + }; + + Using this mechanism, it is possible to generate a FIT which can provide VPL + images for multiple models, with TPL selecting the correct model to use. The + same approach can of course be used for SPL images. + + Note that the `of-spl-remove-props` entryarg can be used to indicate + additional properties to remove. It is often used to remove properties like + `clock-names` and `pinctrl-names` which are not needed in SPL builds. This + value is automatically passed to binman by the U-Boot build. + + See :ref:`fdtgrep_filter` for more information. + Generating nodes from an ELF file (split-elf) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -352,9 +396,16 @@ class Entry_fit(Entry_section): self._fit = None self._fit_props = {} self._fdts = None + self._fdt_dir = None self.mkimage = None + self.fdtgrep = None self._priv_entries = {} self._loadables = [] + self._remove_props = [] + props, = self.GetEntryArgsOrProps( + [EntryArg('of-spl-remove-props', str)], required=False) + if props: + self._remove_props = props.split() def ReadNode(self): super().ReadNode() @@ -368,7 +419,14 @@ class Entry_fit(Entry_section): if fdts is not None: self._fdts = fdts.split() else: - self._fdts = fdt_util.GetStringList(self._node, 'fit,fdt-list-val') + self._fdt_dir = fdt_util.GetString(self._node, 'fit,fdt-list-dir') + if self._fdt_dir: + indir = tools.get_input_filename(self._fdt_dir) + fdts = glob.glob('*.dtb', root_dir=indir) + self._fdts = [os.path.splitext(f)[0] for f in sorted(fdts)] + else: + self._fdts = fdt_util.GetStringList(self._node, + 'fit,fdt-list-val') self._fit_default_dt = self.GetEntryArgsOrProps([EntryArg('default-dt', str)])[0] @@ -483,6 +541,19 @@ class Entry_fit(Entry_section): rel_path = node.path[len(self._node.path) + 1:] self.Raise(f"subnode '{rel_path}': {msg}") + def _run_fdtgrep(self, infile, phase, outfile): + """Run fdtgrep to create the dtb for a phase + + Args: + infile (str): Input filename containing the full FDT contents (with + all nodes and properties) + phase (str): Phase to generate for ('tpl', 'vpl', 'spl') + outfile (str): Output filename to write the grepped FDT contents to + (with only neceesary nodes and properties) + """ + return self.fdtgrep.create_for_phase(infile, phase, outfile, + self._remove_props) + def _build_input(self): """Finish the FIT by adding the 'data' properties to it @@ -584,6 +655,7 @@ class Entry_fit(Entry_section): for seq, fdt_fname in enumerate(self._fdts): node_name = node.name[1:].replace('SEQ', str(seq + 1)) fname = tools.get_input_filename(fdt_fname + '.dtb') + fdt_phase = None with fsw.add_node(node_name): for pname, prop in node.props.items(): if pname == 'fit,firmware': @@ -594,6 +666,15 @@ class Entry_fit(Entry_section): fsw.property('loadables', val.encode('utf-8')) elif pname == 'fit,operation': pass + elif pname == 'fit,compatible': + fdt_phase = fdt_util.GetString(node, pname) + data = tools.read_file(fname) + fdt = Fdt.FromData(data) + fdt.Scan() + prop = fdt.GetRoot().props['compatible'] + fsw.property('compatible', prop.bytes) + elif pname == 'fit,fdt-phase': + fdt_phase = fdt_util.GetString(node, pname) elif pname.startswith('fit,'): self._raise_subnode( node, f"Unknown directive '{pname}'") @@ -606,7 +687,14 @@ class Entry_fit(Entry_section): # Add data for 'images' nodes (but not 'config') if depth == 1 and in_images: - fsw.property('data', tools.read_file(fname)) + if fdt_phase: + phase_fname = tools.get_output_filename( + f'{fdt_fname}-{fdt_phase}.dtb') + self._run_fdtgrep(fname, fdt_phase, phase_fname) + data = tools.read_file(phase_fname) + else: + data = tools.read_file(fname) + fsw.property('data', data) for subnode in node.subnodes: with fsw.add_node(subnode.name): @@ -834,6 +922,7 @@ class Entry_fit(Entry_section): def AddBintools(self, btools): super().AddBintools(btools) self.mkimage = self.AddBintool(btools, 'mkimage') + self.fdtgrep = self.AddBintool(btools, 'fdtgrep') def CheckMissing(self, missing_list): # We must use our private entry list for this since generator nodes diff --git a/tools/binman/etype/intel_descriptor.py b/tools/binman/etype/intel_descriptor.py index 7fe88a9ec1a..3ce967fe81a 100644 --- a/tools/binman/etype/intel_descriptor.py +++ b/tools/binman/etype/intel_descriptor.py @@ -59,7 +59,7 @@ class Entry_intel_descriptor(Entry_blob_ext): if self.missing: # Return zero offsets so that these entries get placed somewhere if self.HasSibling('intel-me'): - info['intel-me'] = [0, None] + info['intel-me'] = [0x1000, None] return info offset = self.data.find(FD_SIGNATURE) if offset == -1: diff --git a/tools/binman/etype/nxp_imx8mcst.py b/tools/binman/etype/nxp_imx8mcst.py new file mode 100644 index 00000000000..8221517b0c4 --- /dev/null +++ b/tools/binman/etype/nxp_imx8mcst.py @@ -0,0 +1,164 @@ +# SPDX-License-Identifier: GPL-2.0+ +# Copyright 2023-2024 Marek Vasut <marex@denx.de> +# Written with much help from Simon Glass <sjg@chromium.org> +# +# Entry-type module for generating the i.MX8M code signing tool +# input configuration file and invocation of cst on generated +# input configuration file and input data to be signed. +# + +import configparser +import os +import struct + +from collections import OrderedDict + +from binman.entry import Entry +from binman.etype.mkimage import Entry_mkimage +from binman.etype.section import Entry_section +from binman import elf +from dtoc import fdt_util +from u_boot_pylib import tools + +MAGIC_NXP_IMX_IVT = 0x412000d1 +MAGIC_FITIMAGE = 0xedfe0dd0 + +csf_config_template = """ +[Header] + Version = 4.3 + Hash Algorithm = sha256 + Engine = CAAM + Engine Configuration = 0 + Certificate Format = X509 + Signature Format = CMS + +[Install SRK] + File = "SRK_1_2_3_4_table.bin" + Source index = 0 + +[Install CSFK] + File = "CSF1_1_sha256_4096_65537_v3_usr_crt.pem" + +[Authenticate CSF] + +[Unlock] + Engine = CAAM + Features = MID + +[Install Key] + Verification index = 0 + Target Index = 2 + File = "IMG1_1_sha256_4096_65537_v3_usr_crt.pem" + +[Authenticate Data] + Verification index = 2 + Blocks = 0x1234 0x78 0xabcd "data.bin" +""" + +class Entry_nxp_imx8mcst(Entry_mkimage): + """NXP i.MX8M CST .cfg file generator and cst invoker + + Properties / Entry arguments: + - nxp,loader-address - loader address (SPL text base) + """ + + def __init__(self, section, etype, node): + super().__init__(section, etype, node) + self.required_props = ['nxp,loader-address'] + + def ReadNode(self): + super().ReadNode() + self.loader_address = fdt_util.GetInt(self._node, 'nxp,loader-address') + self.srk_table = os.getenv('SRK_TABLE', fdt_util.GetString(self._node, 'nxp,srk-table', 'SRK_1_2_3_4_table.bin')) + self.csf_crt = os.getenv('CSF_KEY', fdt_util.GetString(self._node, 'nxp,csf-crt', 'CSF1_1_sha256_4096_65537_v3_usr_crt.pem')) + self.img_crt = os.getenv('IMG_KEY', fdt_util.GetString(self._node, 'nxp,img-crt', 'IMG1_1_sha256_4096_65537_v3_usr_crt.pem')) + self.unlock = fdt_util.GetBool(self._node, 'nxp,unlock') + self.ReadEntries() + + def BuildSectionData(self, required): + data, input_fname, uniq = self.collect_contents_to_file( + self._entries.values(), 'input') + + # Parse the input data and figure out what it is that is being signed. + # - If it is mkimage'd imx8mimage, then extract to be signed data size + # from imx8mimage header, and calculate CSF blob offset right past + # the SPL from this information. + # - If it is fitImage, then pad the image to 4k, add generated IVT and + # sign the whole payload, then append CSF blob at the end right past + # the IVT. + signtype = struct.unpack('<I', data[:4])[0] + signbase = self.loader_address + signsize = 0 + if signtype == MAGIC_NXP_IMX_IVT: # SPL/imx8mimage + # Sign the payload including imx8mimage header + # (extra 0x40 bytes before the payload) + signbase -= 0x40 + signsize = struct.unpack('<I', data[24:28])[0] - signbase + # Remove mkimage generated padding from the end of data + data = data[:signsize] + elif signtype == MAGIC_FITIMAGE: # fitImage + # Align fitImage to 4k + signsize = tools.align(len(data), 0x1000) + data += tools.get_bytes(0, signsize - len(data)) + # Add generated IVT + data += struct.pack('<I', MAGIC_NXP_IMX_IVT) + data += struct.pack('<I', signbase + signsize) # IVT base + data += struct.pack('<I', 0) + data += struct.pack('<I', 0) + data += struct.pack('<I', 0) + data += struct.pack('<I', signbase + signsize) # IVT base + data += struct.pack('<I', signbase + signsize + 0x20) # CSF base + data += struct.pack('<I', 0) + else: + # Unknown section type, pass input data through. + return data + + # Write out customized data to be signed + output_dname = tools.get_output_filename(f'nxp.cst-input-data.{uniq}') + tools.write_file(output_dname, data) + + # Generate CST configuration file used to sign payload + cfg_fname = tools.get_output_filename('nxp.csf-config-txt.%s' % uniq) + config = configparser.ConfigParser() + # Do not make key names lowercase + config.optionxform = str + # Load configuration template and modify keys of interest + config.read_string(csf_config_template) + config['Install SRK']['File'] = '"' + self.srk_table + '"' + config['Install CSFK']['File'] = '"' + self.csf_crt + '"' + config['Install Key']['File'] = '"' + self.img_crt + '"' + config['Authenticate Data']['Blocks'] = hex(signbase) + ' 0 ' + hex(len(data)) + ' "' + str(output_dname) + '"' + if not self.unlock: + config.remove_section('Unlock') + with open(cfg_fname, 'w') as cfgf: + config.write(cfgf) + + output_fname = tools.get_output_filename(f'nxp.csf-output-blob.{uniq}') + args = ['-i', cfg_fname, '-o', output_fname] + if self.cst.run_cmd(*args) is not None: + outdata = tools.read_file(output_fname) + return data + outdata + else: + # Bintool is missing; just use the input data as the output + self.record_missing_bintool(self.cst) + return data + + def SetImagePos(self, image_pos): + # Customized SoC specific SetImagePos which skips the mkimage etype + # implementation and removes the 0x48 offset introduced there. That + # offset is only used for uImage/fitImage, which is not the case in + # here. + upto = 0x00 + for entry in super().GetEntries().values(): + entry.SetOffsetSize(upto, None) + + # Give up if any entries lack a size + if entry.size is None: + return + upto += entry.size + + Entry_section.SetImagePos(self, image_pos) + + def AddBintools(self, btools): + super().AddBintools(btools) + self.cst = self.AddBintool(btools, 'cst') diff --git a/tools/binman/etype/ti_secure.py b/tools/binman/etype/ti_secure.py index 704dcf8a381..420ee263e4f 100644 --- a/tools/binman/etype/ti_secure.py +++ b/tools/binman/etype/ti_secure.py @@ -53,10 +53,11 @@ class Entry_ti_secure(Entry_x509_cert): - keyfile: Filename of file containing key to sign binary with - sha: Hash function to be used for signing - auth-in-place: This is an integer field that contains two pieces - of information - Lower Byte - Remains 0x02 as per our use case - ( 0x02: Move the authenticated binary back to the header ) - Upper Byte - The Host ID of the core owning the firewall + of information: + + - Lower Byte - Remains 0x02 as per our use case + ( 0x02: Move the authenticated binary back to the header ) + - Upper Byte - The Host ID of the core owning the firewall Output files: - input.<unique_name> - input file passed to openssl @@ -69,29 +70,29 @@ class Entry_ti_secure(Entry_x509_cert): firewall nodes that describe the configurations of firewall that TIFS will be doing after reading the certificate. - The syntax of the firewall nodes are as such: + The syntax of the firewall nodes are as such:: - firewall-257-0 { - id = <257>; /* The ID of the firewall being configured */ - region = <0>; /* Region number to configure */ + firewall-257-0 { + id = <257>; /* The ID of the firewall being configured */ + region = <0>; /* Region number to configure */ - control = /* The control register */ - <(FWCTRL_EN | FWCTRL_LOCK | FWCTRL_BG | FWCTRL_CACHE)>; + control = /* The control register */ + <(FWCTRL_EN | FWCTRL_LOCK | FWCTRL_BG | FWCTRL_CACHE)>; - permissions = /* The permission registers */ - <((FWPRIVID_ALL << FWPRIVID_SHIFT) | - FWPERM_SECURE_PRIV_RWCD | - FWPERM_SECURE_USER_RWCD | - FWPERM_NON_SECURE_PRIV_RWCD | - FWPERM_NON_SECURE_USER_RWCD)>; + permissions = /* The permission registers */ + <((FWPRIVID_ALL << FWPRIVID_SHIFT) | + FWPERM_SECURE_PRIV_RWCD | + FWPERM_SECURE_USER_RWCD | + FWPERM_NON_SECURE_PRIV_RWCD | + FWPERM_NON_SECURE_USER_RWCD)>; - /* More defines can be found in k3-security.h */ + /* More defines can be found in k3-security.h */ - start_address = /* The Start Address of the firewall */ - <0x0 0x0>; - end_address = /* The End Address of the firewall */ - <0xff 0xffffffff>; - }; + start_address = /* The Start Address of the firewall */ + <0x0 0x0>; + end_address = /* The End Address of the firewall */ + <0xff 0xffffffff>; + }; openssl signs the provided data, using the TI templated config file and diff --git a/tools/binman/etype/u_boot_spl_nodtb.py b/tools/binman/etype/u_boot_spl_nodtb.py index e7ec329c902..0e172aec1b0 100644 --- a/tools/binman/etype/u_boot_spl_nodtb.py +++ b/tools/binman/etype/u_boot_spl_nodtb.py @@ -23,8 +23,6 @@ class Entry_u_boot_spl_nodtb(Entry_blob): SPL can access binman symbols at runtime. See :ref:`binman_fdt`. - in the binman README for more information. - The ELF file 'spl/u-boot-spl' must also be available for this to work, since binman uses that to look up symbols to write into the SPL binary. """ diff --git a/tools/binman/etype/u_boot_tpl_nodtb.py b/tools/binman/etype/u_boot_tpl_nodtb.py index 9bb2b5dda30..e0c8a557d07 100644 --- a/tools/binman/etype/u_boot_tpl_nodtb.py +++ b/tools/binman/etype/u_boot_tpl_nodtb.py @@ -23,8 +23,6 @@ class Entry_u_boot_tpl_nodtb(Entry_blob): TPL can access binman symbols at runtime. See :ref:`binman_fdt`. - in the binman README for more information. - The ELF file 'tpl/u-boot-tpl' must also be available for this to work, since binman uses that to look up symbols to write into the TPL binary. """ diff --git a/tools/binman/etype/u_boot_vpl.py b/tools/binman/etype/u_boot_vpl.py index 31d7e8374e2..0797831688f 100644 --- a/tools/binman/etype/u_boot_vpl.py +++ b/tools/binman/etype/u_boot_vpl.py @@ -27,6 +27,9 @@ class Entry_u_boot_vpl(Entry_blob): The ELF file 'vpl/u-boot-vpl' must also be available for this to work, since binman uses that to look up symbols to write into the VPL binary. + + Note that this entry is automatically replaced with u-boot-vpl-expanded + unless --no-expanded is used or the node has a 'no-expanded' property. """ def __init__(self, section, etype, node): super().__init__(section, etype, node, auto_write_symbols=True) diff --git a/tools/binman/etype/u_boot_vpl_nodtb.py b/tools/binman/etype/u_boot_vpl_nodtb.py index 64c2767488d..765cf53d164 100644 --- a/tools/binman/etype/u_boot_vpl_nodtb.py +++ b/tools/binman/etype/u_boot_vpl_nodtb.py @@ -16,8 +16,8 @@ class Entry_u_boot_vpl_nodtb(Entry_blob): This is the U-Boot VPL binary, It does not include a device tree blob at the end of it so may not be able to work without it, assuming VPL needs - a device tree to operate on your platform. You can add a u_boot_vpl_dtb - entry after this one, or use a u_boot_vpl entry instead, which normally + a device tree to operate on your platform. You can add a u-boot-vpl-dtb + entry after this one, or use a u-boot-vpl entry instead, which normally expands to a section containing u-boot-vpl-dtb, u-boot-vpl-bss-pad and u-boot-vpl-dtb diff --git a/tools/binman/fdt_test.py b/tools/binman/fdt_test.py index 7ef87295463..564c1770820 100644 --- a/tools/binman/fdt_test.py +++ b/tools/binman/fdt_test.py @@ -44,43 +44,43 @@ class TestFdt(unittest.TestCase): fname = self.GetCompiled('045_prop_test.dts') dt = FdtScan(fname) node = dt.GetNode('/binman/intel-me') - self.assertEquals('intel-me', node.name) + self.assertEqual('intel-me', node.name) val = fdt_util.GetString(node, 'filename') - self.assertEquals(str, type(val)) - self.assertEquals('me.bin', val) + self.assertEqual(str, type(val)) + self.assertEqual('me.bin', val) prop = node.props['intval'] - self.assertEquals(fdt.Type.INT, prop.type) - self.assertEquals(3, fdt_util.GetInt(node, 'intval')) + self.assertEqual(fdt.Type.INT, prop.type) + self.assertEqual(3, fdt_util.GetInt(node, 'intval')) prop = node.props['intarray'] - self.assertEquals(fdt.Type.INT, prop.type) - self.assertEquals(list, type(prop.value)) - self.assertEquals(2, len(prop.value)) - self.assertEquals([5, 6], + self.assertEqual(fdt.Type.INT, prop.type) + self.assertEqual(list, type(prop.value)) + self.assertEqual(2, len(prop.value)) + self.assertEqual([5, 6], [fdt_util.fdt32_to_cpu(val) for val in prop.value]) prop = node.props['byteval'] - self.assertEquals(fdt.Type.BYTE, prop.type) - self.assertEquals(chr(8), prop.value) + self.assertEqual(fdt.Type.BYTE, prop.type) + self.assertEqual(chr(8), prop.value) prop = node.props['bytearray'] - self.assertEquals(fdt.Type.BYTE, prop.type) - self.assertEquals(list, type(prop.value)) - self.assertEquals(str, type(prop.value[0])) - self.assertEquals(3, len(prop.value)) - self.assertEquals([chr(1), '#', '4'], prop.value) + self.assertEqual(fdt.Type.BYTE, prop.type) + self.assertEqual(list, type(prop.value)) + self.assertEqual(str, type(prop.value[0])) + self.assertEqual(3, len(prop.value)) + self.assertEqual([chr(1), '#', '4'], prop.value) prop = node.props['longbytearray'] - self.assertEquals(fdt.Type.INT, prop.type) - self.assertEquals(0x090a0b0c, fdt_util.GetInt(node, 'longbytearray')) + self.assertEqual(fdt.Type.INT, prop.type) + self.assertEqual(0x090a0b0c, fdt_util.GetInt(node, 'longbytearray')) prop = node.props['stringval'] - self.assertEquals(fdt.Type.STRING, prop.type) - self.assertEquals('message2', fdt_util.GetString(node, 'stringval')) + self.assertEqual(fdt.Type.STRING, prop.type) + self.assertEqual('message2', fdt_util.GetString(node, 'stringval')) prop = node.props['stringarray'] - self.assertEquals(fdt.Type.STRING, prop.type) - self.assertEquals(list, type(prop.value)) - self.assertEquals(3, len(prop.value)) - self.assertEquals(['another', 'multi-word', 'message'], prop.value) + self.assertEqual(fdt.Type.STRING, prop.type) + self.assertEqual(list, type(prop.value)) + self.assertEqual(3, len(prop.value)) + self.assertEqual(['another', 'multi-word', 'message'], prop.value) diff --git a/tools/binman/ftest.py b/tools/binman/ftest.py index 8a44bc051b3..93f3d22cf57 100644 --- a/tools/binman/ftest.py +++ b/tools/binman/ftest.py @@ -7,6 +7,7 @@ # python -m unittest func_test.TestFunctional.testHelp import collections +import glob import gzip import hashlib from optparse import OptionParser @@ -546,7 +547,7 @@ class TestFunctional(unittest.TestCase): dtb_data = self._SetupDtb(fname) # For testing purposes, make a copy of the DT for SPL and TPL. Add - # a node indicating which it is, so aid verification. + # a node indicating which it is, to aid verification. for name in ['spl', 'tpl', 'vpl']: dtb_fname = '%s/u-boot-%s.dtb' % (name, name) outfile = os.path.join(self._indir, dtb_fname) @@ -2095,7 +2096,7 @@ class TestFunctional(unittest.TestCase): dtb.Scan() props = self._GetPropTree(dtb, ['size', 'uncomp-size']) orig = self._decompress(data) - self.assertEquals(COMPRESS_DATA, orig) + self.assertEqual(COMPRESS_DATA, orig) # Do a sanity check on various fields image = control.images['image'] @@ -2809,9 +2810,9 @@ class TestFunctional(unittest.TestCase): orig_entry = orig_image.GetEntries()['fdtmap'] entry = image.GetEntries()['fdtmap'] - self.assertEquals(orig_entry.offset, entry.offset) - self.assertEquals(orig_entry.size, entry.size) - self.assertEquals(orig_entry.image_pos, entry.image_pos) + self.assertEqual(orig_entry.offset, entry.offset) + self.assertEqual(orig_entry.size, entry.size) + self.assertEqual(orig_entry.image_pos, entry.image_pos) def testReadImageNoHeader(self): """Test accessing an image's FDT map without an image header""" @@ -3895,7 +3896,7 @@ class TestFunctional(unittest.TestCase): mat = re_line.match(line) vals[mat.group(1)].append(mat.group(2)) - self.assertEquals('FIT description: test-desc', lines[0]) + self.assertEqual('FIT description: test-desc', lines[0]) self.assertIn('Created:', lines[1]) self.assertIn('Image 0 (kernel)', vals) self.assertIn('Hash value', vals) @@ -4012,7 +4013,7 @@ class TestFunctional(unittest.TestCase): fit_pos, fdt_util.fdt32_to_cpu(fnode.props['data-position'].value)) - self.assertEquals(expected_size, len(data)) + self.assertEqual(expected_size, len(data)) actual_pos = len(U_BOOT_DATA) + fit_pos self.assertEqual(U_BOOT_DATA + b'aa', data[actual_pos:actual_pos + external_data_size]) @@ -4180,8 +4181,8 @@ class TestFunctional(unittest.TestCase): data = self._DoReadFile('172_scp.dts') self.assertEqual(SCP_DATA, data[:len(SCP_DATA)]) - def testFitFdt(self): - """Test an image with an FIT with multiple FDT images""" + def CheckFitFdt(self, dts='170_fit_fdt.dts', use_fdt_list=True): + """Check an image with an FIT with multiple FDT images""" def _CheckFdt(seq, expected_data): """Check the FDT nodes @@ -4220,11 +4221,12 @@ class TestFunctional(unittest.TestCase): self.assertEqual('fdt-%d' % seq, fnode.props['fdt'].value) entry_args = { - 'of-list': 'test-fdt1 test-fdt2', 'default-dt': 'test-fdt2', } + if use_fdt_list: + entry_args['of-list'] = 'test-fdt1 test-fdt2' data = self._DoReadFileDtb( - '170_fit_fdt.dts', + dts, entry_args=entry_args, extra_indirs=[os.path.join(self._indir, TEST_FDT_SUBDIR)])[0] self.assertEqual(U_BOOT_NODTB_DATA, data[-len(U_BOOT_NODTB_DATA):]) @@ -4243,6 +4245,10 @@ class TestFunctional(unittest.TestCase): _CheckConfig(1, TEST_FDT1_DATA) _CheckConfig(2, TEST_FDT2_DATA) + def testFitFdt(self): + """Test an image with an FIT with multiple FDT images""" + self.CheckFitFdt() + def testFitFdtMissingList(self): """Test handling of a missing 'of-list' entry arg""" with self.assertRaises(ValueError) as e: @@ -4431,7 +4437,7 @@ class TestFunctional(unittest.TestCase): props = self._GetPropTree(dtb, ['offset', 'image-pos', 'size', 'uncomp-size']) orig = self._decompress(data) - self.assertEquals(COMPRESS_DATA + U_BOOT_DATA, orig) + self.assertEqual(COMPRESS_DATA + U_BOOT_DATA, orig) # Do a sanity check on various fields image = control.images['image'] @@ -4475,7 +4481,7 @@ class TestFunctional(unittest.TestCase): 'uncomp-size']) orig = self._decompress(data) - self.assertEquals(COMPRESS_DATA + COMPRESS_DATA + U_BOOT_DATA, orig) + self.assertEqual(COMPRESS_DATA + COMPRESS_DATA + U_BOOT_DATA, orig) # Do a sanity check on various fields image = control.images['image'] @@ -4519,7 +4525,7 @@ class TestFunctional(unittest.TestCase): props = self._GetPropTree(dtb, ['offset', 'image-pos', 'size', 'uncomp-size']) orig = self._decompress(data) - self.assertEquals(COMPRESS_DATA + U_BOOT_DATA, orig) + self.assertEqual(COMPRESS_DATA + U_BOOT_DATA, orig) expected = { 'section/blob:offset': 0, 'section/blob:size': len(COMPRESS_DATA), @@ -4545,7 +4551,7 @@ class TestFunctional(unittest.TestCase): props = self._GetPropTree(dtb, ['offset', 'image-pos', 'size', 'uncomp-size']) orig = self._decompress(data) - self.assertEquals(COMPRESS_DATA + U_BOOT_DATA, orig) + self.assertEqual(COMPRESS_DATA + U_BOOT_DATA, orig) expected = { 'section/blob:offset': 0, 'section/blob:size': len(COMPRESS_DATA), @@ -4580,7 +4586,7 @@ class TestFunctional(unittest.TestCase): 'uncomp-size']) base = data[len(U_BOOT_DATA):] - self.assertEquals(U_BOOT_DATA, base[:len(U_BOOT_DATA)]) + self.assertEqual(U_BOOT_DATA, base[:len(U_BOOT_DATA)]) rest = base[len(U_BOOT_DATA):] # Check compressed data @@ -4588,22 +4594,22 @@ class TestFunctional(unittest.TestCase): expect1 = bintool.compress(COMPRESS_DATA + U_BOOT_DATA) data1 = rest[:len(expect1)] section1 = self._decompress(data1) - self.assertEquals(expect1, data1) - self.assertEquals(COMPRESS_DATA + U_BOOT_DATA, section1) + self.assertEqual(expect1, data1) + self.assertEqual(COMPRESS_DATA + U_BOOT_DATA, section1) rest1 = rest[len(expect1):] expect2 = bintool.compress(COMPRESS_DATA + COMPRESS_DATA) data2 = rest1[:len(expect2)] section2 = self._decompress(data2) - self.assertEquals(expect2, data2) - self.assertEquals(COMPRESS_DATA + COMPRESS_DATA, section2) + self.assertEqual(expect2, data2) + self.assertEqual(COMPRESS_DATA + COMPRESS_DATA, section2) rest2 = rest1[len(expect2):] expect_size = (len(U_BOOT_DATA) + len(U_BOOT_DATA) + len(expect1) + len(expect2) + len(U_BOOT_DATA)) - #self.assertEquals(expect_size, len(data)) + #self.assertEqual(expect_size, len(data)) - #self.assertEquals(U_BOOT_DATA, rest2) + #self.assertEqual(U_BOOT_DATA, rest2) self.maxDiff = None expected = { @@ -4695,7 +4701,7 @@ class TestFunctional(unittest.TestCase): u_boot = image.GetEntries()['section'].GetEntries()['u-boot'] - self.assertEquals(U_BOOT_DATA, u_boot.ReadData()) + self.assertEqual(U_BOOT_DATA, u_boot.ReadData()) def testTplNoDtb(self): """Test that an image with tpl/u-boot-tpl-nodtb.bin can be created""" @@ -5526,7 +5532,7 @@ fdt fdtmap Extract the devicetree blob from the fdtmap segments, entry = elf.read_loadable_segments(elf_data) # We assume there are two segments - self.assertEquals(2, len(segments)) + self.assertEqual(2, len(segments)) atf1 = dtb.GetNode('/images/atf-1') _, start, data = segments[0] @@ -6107,7 +6113,7 @@ fdt fdtmap Extract the devicetree blob from the fdtmap data = bintool.compress(COMPRESS_DATA) self.assertNotEqual(COMPRESS_DATA, data) orig = bintool.decompress(data) - self.assertEquals(COMPRESS_DATA, orig) + self.assertEqual(COMPRESS_DATA, orig) def testCompUtilVersions(self): """Test tool version of compression algorithms""" @@ -6125,7 +6131,7 @@ fdt fdtmap Extract the devicetree blob from the fdtmap self.assertNotEqual(COMPRESS_DATA, data) data += tools.get_bytes(0, 64) orig = bintool.decompress(data) - self.assertEquals(COMPRESS_DATA, orig) + self.assertEqual(COMPRESS_DATA, orig) def testCompressDtbZstd(self): """Test that zstd compress of device-tree files failed""" @@ -7175,27 +7181,24 @@ fdt fdtmap Extract the devicetree blob from the fdtmap def testSplPubkeyDtb(self): - """Test u_boot_spl_pubkey_dtb etype""" - data = tools.read_file(self.TestFile("key.pem")) - self._MakeInputFile("key.crt", data) - self._DoReadFileRealDtb('306_spl_pubkey_dtb.dts') - image = control.images['image'] - entries = image.GetEntries() - dtb_entry = entries['u-boot-spl-pubkey-dtb'] - dtb_data = dtb_entry.GetData() - dtb = fdt.Fdt.FromData(dtb_data) - dtb.Scan() - - signature_node = dtb.GetNode('/signature') - self.assertIsNotNone(signature_node) - key_node = signature_node.FindNode("key-key") - self.assertIsNotNone(key_node) - self.assertEqual(fdt_util.GetString(key_node, "required"), - "conf") - self.assertEqual(fdt_util.GetString(key_node, "algo"), - "sha384,rsa4096") - self.assertEqual(fdt_util.GetString(key_node, "key-name-hint"), - "key") + """Test u_boot_spl_pubkey_dtb etype""" + data = tools.read_file(self.TestFile("key.pem")) + self._MakeInputFile("key.crt", data) + self._DoReadFileRealDtb('306_spl_pubkey_dtb.dts') + image = control.images['image'] + entries = image.GetEntries() + dtb_entry = entries['u-boot-spl-pubkey-dtb'] + dtb_data = dtb_entry.GetData() + dtb = fdt.Fdt.FromData(dtb_data) + dtb.Scan() + + signature_node = dtb.GetNode('/signature') + self.assertIsNotNone(signature_node) + key_node = signature_node.FindNode("key-key") + self.assertIsNotNone(key_node) + self.assertEqual(fdt_util.GetString(key_node, "required"), "conf") + self.assertEqual(fdt_util.GetString(key_node, "algo"), "sha384,rsa4096") + self.assertEqual(fdt_util.GetString(key_node, "key-name-hint"), "key") def testXilinxBootgenSigning(self): """Test xilinx-bootgen etype""" @@ -7460,5 +7463,233 @@ fdt fdtmap Extract the devicetree blob from the fdtmap with self.assertRaises(ValueError) as e: self._DoReadFile('323_capsule_accept_revert_missing.dts') + def test_assume_size(self): + """Test handling of the assume-size property for external blob""" + with self.assertRaises(ValueError) as e: + self._DoTestFile('326_assume_size.dts', allow_missing=True, + allow_fake_blobs=True) + self.assertIn("contents size 0xa (10) exceeds section size 0x9 (9)", + str(e.exception)) + + def test_assume_size_ok(self): + """Test handling of the assume-size where it fits OK""" + with test_util.capture_sys_output() as (stdout, stderr): + self._DoTestFile('327_assume_size_ok.dts', allow_missing=True, + allow_fake_blobs=True) + err = stderr.getvalue() + self.assertRegex( + err, + "Image '.*' has faked external blobs and is non-functional: .*") + + def test_assume_size_no_fake(self): + """Test handling of the assume-size where it fits OK""" + with test_util.capture_sys_output() as (stdout, stderr): + self._DoTestFile('327_assume_size_ok.dts', allow_missing=True) + err = stderr.getvalue() + self.assertRegex( + err, + "Image '.*' is missing external blobs and is non-functional: .*") + + def SetupAlternateDts(self): + """Compile the .dts test files for alternative-fdt + + Returns: + tuple: + str: Test directory created + list of str: '.bin' files which we expect Binman to create + """ + testdir = TestFunctional._MakeInputDir('dtb') + dtb_list = [] + for fname in glob.glob(f'{self.TestFile("alt_dts")}/*.dts'): + tmp_fname = fdt_util.EnsureCompiled(fname, testdir) + base = os.path.splitext(os.path.basename(fname))[0] + dtb_list.append(base + '.bin') + shutil.move(tmp_fname, os.path.join(testdir, base + '.dtb')) + + return testdir, dtb_list + + def CheckAlternates(self, dts, phase, xpl_data): + """Run the test for the alterative-fdt etype + + Args: + dts (str): Devicetree file to process + phase (str): Phase to process ('spl', 'tpl' or 'vpl') + xpl_data (bytes): Expected data for the phase's binary + + Returns: + dict of .dtb files produced + key: str filename + value: Fdt object + """ + dtb_list = self.SetupAlternateDts()[1] + + entry_args = { + f'{phase}-dtb': '1', + f'{phase}-bss-pad': 'y', + 'of-spl-remove-props': 'prop-to-remove another-prop-to-get-rid-of', + } + data = self._DoReadFileDtb(dts, use_real_dtb=True, update_dtb=True, + use_expanded=True, entry_args=entry_args)[0] + self.assertEqual(xpl_data, data[:len(xpl_data)]) + rest = data[len(xpl_data):] + pad_len = 10 + self.assertEqual(tools.get_bytes(0, pad_len), rest[:pad_len]) + + # Check the dtb is using the test file + dtb_data = rest[pad_len:] + dtb = fdt.Fdt.FromData(dtb_data) + dtb.Scan() + fdt_size = dtb.GetFdtObj().totalsize() + self.assertEqual('model-not-set', + fdt_util.GetString(dtb.GetRoot(), 'compatible')) + + pad_len = 10 + + # Check the other output files + dtbs = {} + for fname in dtb_list: + pathname = tools.get_output_filename(fname) + self.assertTrue(os.path.exists(pathname)) + + data = tools.read_file(pathname) + self.assertEqual(xpl_data, data[:len(xpl_data)]) + rest = data[len(xpl_data):] + + self.assertEqual(tools.get_bytes(0, pad_len), rest[:pad_len]) + rest = rest[pad_len:] + + dtb = fdt.Fdt.FromData(rest) + dtb.Scan() + dtbs[fname] = dtb + + expected = 'one' if '1' in fname else 'two' + self.assertEqual(f'u-boot,model-{expected}', + fdt_util.GetString(dtb.GetRoot(), 'compatible')) + + # Make sure the FDT is the same size as the 'main' one + rest = rest[fdt_size:] + + self.assertEqual(b'', rest) + return dtbs + + def testAlternatesFdt(self): + """Test handling of alternates-fdt etype""" + self._SetupTplElf() + dtbs = self.CheckAlternates('328_alternates_fdt.dts', 'tpl', + U_BOOT_TPL_NODTB_DATA) + for dtb in dtbs.values(): + # Check for the node with the tag + node = dtb.GetNode('/node') + self.assertIsNotNone(node) + self.assertEqual(5, len(node.props.keys())) + + # Make sure the other node is still there + self.assertIsNotNone(dtb.GetNode('/node/other-node')) + + def testAlternatesFdtgrep(self): + """Test handling of alternates-fdt etype using fdtgrep""" + self._SetupTplElf() + dtbs = self.CheckAlternates('329_alternates_fdtgrep.dts', 'tpl', + U_BOOT_TPL_NODTB_DATA) + for dtb in dtbs.values(): + # Check for the node with the tag + node = dtb.GetNode('/node') + self.assertIsNotNone(node) + self.assertEqual({'some-prop', 'not-a-prop-to-remove'}, + node.props.keys()) + + # Make sure the other node is gone + self.assertIsNone(dtb.GetNode('/node/other-node')) + + def testAlternatesFdtgrepVpl(self): + """Test handling of alternates-fdt etype using fdtgrep with vpl""" + self._SetupVplElf() + dtbs = self.CheckAlternates('330_alternates_vpl.dts', 'vpl', + U_BOOT_VPL_NODTB_DATA) + + def testAlternatesFdtgrepSpl(self): + """Test handling of alternates-fdt etype using fdtgrep with spl""" + self._SetupSplElf() + dtbs = self.CheckAlternates('331_alternates_spl.dts', 'spl', + U_BOOT_SPL_NODTB_DATA) + + def testAlternatesFdtgrepInval(self): + """Test alternates-fdt etype using fdtgrep with invalid phase""" + self._SetupSplElf() + with self.assertRaises(ValueError) as e: + dtbs = self.CheckAlternates('332_alternates_inval.dts', 'spl', + U_BOOT_SPL_NODTB_DATA) + self.assertIn("Invalid U-Boot phase 'bad-phase': Use tpl/vpl/spl", + str(e.exception)) + + def testFitFdtListDir(self): + """Test an image with an FIT with FDT images using fit,fdt-list-dir""" + self.CheckFitFdt('333_fit_fdt_dir.dts', False) + + def testFitFdtCompat(self): + """Test an image with an FIT with compatible in the config nodes""" + entry_args = { + 'of-list': 'model1 model2', + 'default-dt': 'model2', + } + testdir, dtb_list = self.SetupAlternateDts() + data = self._DoReadFileDtb( + '334_fit_fdt_compat.dts', use_real_dtb=True, update_dtb=True, + entry_args=entry_args, extra_indirs=[testdir])[0] + + fit_data = data[len(U_BOOT_DATA):-len(U_BOOT_NODTB_DATA)] + + fit = fdt.Fdt.FromData(fit_data) + fit.Scan() + + cnode = fit.GetNode('/configurations') + self.assertIn('default', cnode.props) + self.assertEqual('config-2', cnode.props['default'].value) + + for seq in range(1, 2): + name = f'config-{seq}' + fnode = fit.GetNode('/configurations/%s' % name) + self.assertIsNotNone(fnode) + self.assertIn('compatible', fnode.props.keys()) + expected = 'one' if seq == 1 else 'two' + self.assertEqual(f'u-boot,model-{expected}', + fnode.props['compatible'].value) + + def testFitFdtPhase(self): + """Test an image with an FIT with fdt-phase in the fdt nodes""" + phase = 'tpl' + entry_args = { + f'{phase}-dtb': '1', + f'{phase}-bss-pad': 'y', + 'of-spl-remove-props': 'prop-to-remove another-prop-to-get-rid-of', + 'of-list': 'model1 model2', + 'default-dt': 'model2', + } + testdir, dtb_list = self.SetupAlternateDts() + data = self._DoReadFileDtb( + '335_fit_fdt_phase.dts', use_real_dtb=True, update_dtb=True, + entry_args=entry_args, extra_indirs=[testdir])[0] + fit_data = data[len(U_BOOT_DATA):-len(U_BOOT_NODTB_DATA)] + fit = fdt.Fdt.FromData(fit_data) + fit.Scan() + + # Check that each FDT has only the expected properties for the phase + for seq in range(1, 2): + fnode = fit.GetNode(f'/images/fdt-{seq}') + self.assertIsNotNone(fnode) + dtb = fdt.Fdt.FromData(fnode.props['data'].bytes) + dtb.Scan() + + # Make sure that the 'bootph-pre-sram' tag in /node protects it from + # removal + node = dtb.GetNode('/node') + self.assertIsNotNone(node) + self.assertEqual({'some-prop', 'not-a-prop-to-remove'}, + node.props.keys()) + + # Make sure the other node is gone + self.assertIsNone(dtb.GetNode('/node/other-node')) + + if __name__ == "__main__": unittest.main() diff --git a/tools/binman/image.py b/tools/binman/image.py index e77b5d0d97c..702c9055585 100644 --- a/tools/binman/image.py +++ b/tools/binman/image.py @@ -21,6 +21,9 @@ from dtoc import fdt_util from u_boot_pylib import tools from u_boot_pylib import tout +# This is imported if needed +state = None + class Image(section.Entry_section): """A Image, representing an output from binman @@ -75,6 +78,10 @@ class Image(section.Entry_section): def __init__(self, name, node, copy_to_orig=True, test=False, ignore_missing=False, use_expanded=False, missing_etype=False, generate=True): + # Put this here to allow entry-docs and help to work without libfdt + global state + from binman import state + super().__init__(None, 'section', node, test=test) self.copy_to_orig = copy_to_orig self.name = name @@ -186,6 +193,19 @@ class Image(section.Entry_section): os.remove(sname) os.symlink(fname, sname) + def WriteAlternates(self): + """Write out alternative devicetree blobs, each in its own file""" + alt_entry = self.FindEntryType('alternates-fdt') + if not alt_entry: + return + + for alt in alt_entry.alternates: + fname, data = alt_entry.ProcessWithFdt(alt) + pathname = tools.get_output_filename(fname) + tout.info(f"Writing alternate '{alt}' to '{pathname}'") + tools.write_file(pathname, data) + tout.info("Wrote %#x bytes" % len(data)) + def WriteMap(self): """Write a map of the image to a .map file @@ -418,3 +438,7 @@ class Image(section.Entry_section): super().AddBintools(bintools) self.bintools = bintools return bintools + + def FdtContents(self, fdt_etype): + """This base-class implementation simply calls the state function""" + return state.GetFdtContents(fdt_etype) diff --git a/tools/binman/main.py b/tools/binman/main.py index 92d2431aea7..dc817ddcd42 100755 --- a/tools/binman/main.py +++ b/tools/binman/main.py @@ -122,6 +122,8 @@ def RunBinman(args): ret_code = RunTests(args.debug, args.verbosity, args.processes, args.test_preserve_dirs, args.tests, args.toolpath) + if args.debug and not test_util.use_concurrent: + print('Tests can run in parallel: pip install concurrencytest') elif args.cmd == 'bintool-docs': control.write_bintool_docs(bintool.Bintool.get_tool_list()) diff --git a/tools/binman/test/326_assume_size.dts b/tools/binman/test/326_assume_size.dts new file mode 100644 index 00000000000..4c5f8b418d8 --- /dev/null +++ b/tools/binman/test/326_assume_size.dts @@ -0,0 +1,16 @@ +// SPDX-License-Identifier: GPL-2.0+ + +/dts-v1/; + +/ { + #address-cells = <1>; + #size-cells = <1>; + + binman { + size = <9>; + blob-ext { + filename = "assume_blob"; + assume-size = <10>; + }; + }; +}; diff --git a/tools/binman/test/327_assume_size_ok.dts b/tools/binman/test/327_assume_size_ok.dts new file mode 100644 index 00000000000..00ed726f872 --- /dev/null +++ b/tools/binman/test/327_assume_size_ok.dts @@ -0,0 +1,16 @@ +// SPDX-License-Identifier: GPL-2.0+ + +/dts-v1/; + +/ { + #address-cells = <1>; + #size-cells = <1>; + + binman { + size = <10>; + blob-ext { + filename = "assume_blob"; + assume-size = <10>; + }; + }; +}; diff --git a/tools/binman/test/328_alternates_fdt.dts b/tools/binman/test/328_alternates_fdt.dts new file mode 100644 index 00000000000..c913c8e4745 --- /dev/null +++ b/tools/binman/test/328_alternates_fdt.dts @@ -0,0 +1,28 @@ +// SPDX-License-Identifier: GPL-2.0+ +// Copyright 2024 Google LLC +// Written by Simon Glass <sjg@chromium.org> + +/dts-v1/; + +/ { + #address-cells = <1>; + #size-cells = <1>; + + compatible = "model-not-set"; + + binman { + alternates-fdt { + fdt-list-dir = "dtb"; + filename-pattern = "NAME.bin"; + + section { + u-boot-tpl { + }; + }; + }; + + blob { + filename = "blobfile"; + }; + }; +}; diff --git a/tools/binman/test/329_alternates_fdtgrep.dts b/tools/binman/test/329_alternates_fdtgrep.dts new file mode 100644 index 00000000000..41695281456 --- /dev/null +++ b/tools/binman/test/329_alternates_fdtgrep.dts @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: GPL-2.0+ +// Copyright 2024 Google LLC +// Written by Simon Glass <sjg@chromium.org> + +/dts-v1/; + +/ { + #address-cells = <1>; + #size-cells = <1>; + + compatible = "model-not-set"; + + binman { + alternates-fdt { + fdt-list-dir = "dtb"; + filename-pattern = "NAME.bin"; + fdt-phase = "tpl"; + + section { + u-boot-tpl { + }; + }; + }; + + blob { + filename = "blobfile"; + }; + }; +}; diff --git a/tools/binman/test/330_alternates_vpl.dts b/tools/binman/test/330_alternates_vpl.dts new file mode 100644 index 00000000000..5b57069e2ab --- /dev/null +++ b/tools/binman/test/330_alternates_vpl.dts @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: GPL-2.0+ +// Copyright 2024 Google LLC +// Written by Simon Glass <sjg@chromium.org> + +/dts-v1/; + +/ { + #address-cells = <1>; + #size-cells = <1>; + + compatible = "model-not-set"; + + binman { + alternates-fdt { + fdt-list-dir = "dtb"; + filename-pattern = "NAME.bin"; + fdt-phase = "vpl"; + + section { + u-boot-vpl { + }; + }; + }; + + blob { + filename = "blobfile"; + }; + }; +}; diff --git a/tools/binman/test/331_alternates_spl.dts b/tools/binman/test/331_alternates_spl.dts new file mode 100644 index 00000000000..882fefce34a --- /dev/null +++ b/tools/binman/test/331_alternates_spl.dts @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: GPL-2.0+ +// Copyright 2024 Google LLC +// Written by Simon Glass <sjg@chromium.org> + +/dts-v1/; + +/ { + #address-cells = <1>; + #size-cells = <1>; + + compatible = "model-not-set"; + + binman { + alternates-fdt { + fdt-list-dir = "dtb"; + filename-pattern = "NAME.bin"; + fdt-phase = "spl"; + + section { + u-boot-spl { + }; + }; + }; + + blob { + filename = "blobfile"; + }; + }; +}; diff --git a/tools/binman/test/332_alternates_inval.dts b/tools/binman/test/332_alternates_inval.dts new file mode 100644 index 00000000000..8c145dd2449 --- /dev/null +++ b/tools/binman/test/332_alternates_inval.dts @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: GPL-2.0+ +// Copyright 2024 Google LLC +// Written by Simon Glass <sjg@chromium.org> + +/dts-v1/; + +/ { + #address-cells = <1>; + #size-cells = <1>; + + compatible = "model-not-set"; + + binman { + alternates-fdt { + fdt-list-dir = "dtb"; + filename-pattern = "NAME.bin"; + fdt-phase = "bad-phase"; + + section { + u-boot-spl { + }; + }; + }; + + blob { + filename = "blobfile"; + }; + }; +}; diff --git a/tools/binman/test/333_fit_fdt_dir.dts b/tools/binman/test/333_fit_fdt_dir.dts new file mode 100644 index 00000000000..aa778451a4b --- /dev/null +++ b/tools/binman/test/333_fit_fdt_dir.dts @@ -0,0 +1,58 @@ +// SPDX-License-Identifier: GPL-2.0+ + +/dts-v1/; + +/ { + #address-cells = <1>; + #size-cells = <1>; + + binman { + u-boot { + }; + fit { + description = "test-desc"; + #address-cells = <1>; + fit,fdt-list-dir = "fdts"; + + images { + kernel { + description = "Vanilla Linux kernel"; + type = "kernel"; + arch = "ppc"; + os = "linux"; + compression = "gzip"; + load = <00000000>; + entry = <00000000>; + hash-1 { + algo = "crc32"; + }; + hash-2 { + algo = "sha1"; + }; + u-boot { + }; + }; + @fdt-SEQ { + description = "fdt-NAME.dtb"; + type = "flat_dt"; + compression = "none"; + hash { + algo = "sha256"; + }; + }; + }; + + configurations { + default = "@config-DEFAULT-SEQ"; + @config-SEQ { + description = "conf-NAME.dtb"; + firmware = "uboot"; + loadables = "atf"; + fdt = "fdt-SEQ"; + }; + }; + }; + u-boot-nodtb { + }; + }; +}; diff --git a/tools/binman/test/334_fit_fdt_compat.dts b/tools/binman/test/334_fit_fdt_compat.dts new file mode 100644 index 00000000000..3bf45c710db --- /dev/null +++ b/tools/binman/test/334_fit_fdt_compat.dts @@ -0,0 +1,60 @@ +// SPDX-License-Identifier: GPL-2.0+ + +/dts-v1/; + +/ { + #address-cells = <1>; + #size-cells = <1>; + + binman { + u-boot { + }; + fit { + description = "test-desc"; + #address-cells = <1>; + fit,fdt-list = "of-list"; + + images { + kernel { + description = "Vanilla Linux kernel"; + type = "kernel"; + arch = "ppc"; + os = "linux"; + compression = "gzip"; + load = <00000000>; + entry = <00000000>; + hash-1 { + algo = "crc32"; + }; + hash-2 { + algo = "sha1"; + }; + u-boot { + }; + }; + @fdt-SEQ { + description = "fdt-NAME.dtb"; + type = "flat_dt"; + compression = "none"; + hash { + algo = "sha256"; + }; + }; + }; + + configurations { + default = "@config-DEFAULT-SEQ"; + @config-SEQ { + description = "conf-NAME.dtb"; + firmware = "uboot"; + loadables = "atf"; + fdt = "fdt-SEQ"; + fit,firmware = "vpl"; + fit,compatible; + }; + }; + }; + u-boot-nodtb { + }; + }; +}; diff --git a/tools/binman/test/335_fit_fdt_phase.dts b/tools/binman/test/335_fit_fdt_phase.dts new file mode 100644 index 00000000000..f8d0740a394 --- /dev/null +++ b/tools/binman/test/335_fit_fdt_phase.dts @@ -0,0 +1,61 @@ +// SPDX-License-Identifier: GPL-2.0+ + +/dts-v1/; + +/ { + #address-cells = <1>; + #size-cells = <1>; + + binman { + u-boot { + }; + fit { + description = "test-desc"; + #address-cells = <1>; + fit,fdt-list = "of-list"; + + images { + kernel { + description = "Vanilla Linux kernel"; + type = "kernel"; + arch = "ppc"; + os = "linux"; + compression = "gzip"; + load = <00000000>; + entry = <00000000>; + hash-1 { + algo = "crc32"; + }; + hash-2 { + algo = "sha1"; + }; + u-boot { + }; + }; + @fdt-SEQ { + description = "fdt-NAME.dtb"; + type = "flat_dt"; + compression = "none"; + fit,fdt-phase = "tpl"; + hash { + algo = "sha256"; + }; + }; + }; + + configurations { + default = "@config-DEFAULT-SEQ"; + @config-SEQ { + description = "conf-NAME.dtb"; + firmware = "uboot"; + loadables = "atf"; + fdt = "fdt-SEQ"; + fit,firmware = "tpl"; + fit,compatible; + }; + }; + }; + u-boot-nodtb { + }; + }; +}; diff --git a/tools/binman/test/alt_dts/model1.dts b/tools/binman/test/alt_dts/model1.dts new file mode 100644 index 00000000000..01e95e8fabe --- /dev/null +++ b/tools/binman/test/alt_dts/model1.dts @@ -0,0 +1,24 @@ +// SPDX-License-Identifier: GPL-2.0+ +// Copyright 2024 Google LLC +// Written by Simon Glass <sjg@chromium.org> + +/dts-v1/; + +/ { + model = "Model One"; + compatible = "u-boot,model-one"; + + /* this node remains due to bootph-pre-sram tag */ + node { + some-prop; + prop-to-remove; + another-prop-to-get-rid-of; + not-a-prop-to-remove; + bootph-pre-sram; + + /* this node get removed by fdtgrep */ + other-node { + another-prop; + }; + }; +}; diff --git a/tools/binman/test/alt_dts/model2.dts b/tools/binman/test/alt_dts/model2.dts new file mode 100644 index 00000000000..7829c519772 --- /dev/null +++ b/tools/binman/test/alt_dts/model2.dts @@ -0,0 +1,24 @@ +// SPDX-License-Identifier: GPL-2.0+ +// Copyright 2024 Google LLC +// Written by Simon Glass <sjg@chromium.org> + +/dts-v1/; + +/ { + model = "Model Two"; + compatible = "u-boot,model-two"; + + /* this node remains due to bootph-pre-sram tag */ + node { + some-prop; + prop-to-remove; + another-prop-to-get-rid-of; + not-a-prop-to-remove; + bootph-pre-sram; + + /* this node get removed by fdtgrep */ + other-node { + another-prop; + }; + }; +}; diff --git a/tools/buildman/bsettings.py b/tools/buildman/bsettings.py index e225ac2ca0f..aea724fc559 100644 --- a/tools/buildman/bsettings.py +++ b/tools/buildman/bsettings.py @@ -29,7 +29,7 @@ def setup(fname=''): settings.read(config_fname) def add_file(data): - settings.readfp(io.StringIO(data)) + settings.read_file(io.StringIO(data)) def get_items(section): """Get the items from a section of the config. diff --git a/tools/buildman/builder.py b/tools/buildman/builder.py index f35175b4598..c4384f53e8d 100644 --- a/tools/buildman/builder.py +++ b/tools/buildman/builder.py @@ -256,14 +256,14 @@ class Builder: def __init__(self, toolchains, base_dir, git_dir, num_threads, num_jobs, gnu_make='make', checkout=True, show_unknown=True, step=1, no_subdirs=False, full_path=False, verbose_build=False, - mrproper=False, per_board_out_dir=False, - config_only=False, squash_config_y=False, - warnings_as_errors=False, work_in_output=False, - test_thread_exceptions=False, adjust_cfg=None, - allow_missing=False, no_lto=False, reproducible_builds=False, - force_build=False, force_build_failures=False, - force_reconfig=False, in_tree=False, - force_config_on_failure=False, make_func=None): + mrproper=False, fallback_mrproper=False, + per_board_out_dir=False, config_only=False, + squash_config_y=False, warnings_as_errors=False, + work_in_output=False, test_thread_exceptions=False, + adjust_cfg=None, allow_missing=False, no_lto=False, + reproducible_builds=False, force_build=False, + force_build_failures=False, force_reconfig=False, + in_tree=False, force_config_on_failure=False, make_func=None): """Create a new Builder object Args: @@ -283,6 +283,7 @@ class Builder: PATH verbose_build: Run build with V=1 and don't use 'make -s' mrproper: Always run 'make mrproper' when configuring + fallback_mrproper: Run 'make mrproper' and retry on build failure per_board_out_dir: Build in a separate persistent directory per board rather than a thread-specific directory config_only: Only configure each build, don't build it @@ -352,6 +353,7 @@ class Builder: self.force_reconfig = force_reconfig self.in_tree = in_tree self.force_config_on_failure = force_config_on_failure + self.fallback_mrproper = fallback_mrproper if not self.squash_config_y: self.config_filenames += EXTRA_CONFIG_FILENAMES diff --git a/tools/buildman/builderthread.py b/tools/buildman/builderthread.py index a8599c0bb2a..bbe2f6f0d24 100644 --- a/tools/buildman/builderthread.py +++ b/tools/buildman/builderthread.py @@ -240,7 +240,7 @@ class BuilderThread(threading.Thread): return args, cwd, src_dir def _reconfigure(self, commit, brd, cwd, args, env, config_args, config_out, - cmd_list): + cmd_list, mrproper): """Reconfigure the build Args: @@ -251,11 +251,12 @@ class BuilderThread(threading.Thread): env (dict): Environment strings config_args (list of str): defconfig arg for this board cmd_list (list of str): List to add the commands to, for logging + mrproper (bool): True to run mrproper first Returns: CommandResult object """ - if self.mrproper: + if mrproper: result = self.make(commit, brd, 'mrproper', cwd, 'mrproper', *args, env=env) config_out.write(result.combined) @@ -380,7 +381,7 @@ class BuilderThread(threading.Thread): commit = 'current' return commit - def _config_and_build(self, commit_upto, brd, work_dir, do_config, + def _config_and_build(self, commit_upto, brd, work_dir, do_config, mrproper, config_only, adjust_cfg, commit, out_dir, out_rel_dir, result): """Do the build, configuring first if necessary @@ -390,6 +391,7 @@ class BuilderThread(threading.Thread): brd (Board): Board to create arguments for work_dir (str): Directory to which the source will be checked out do_config (bool): True to run a make <board>_defconfig on the source + mrproper (bool): True to run mrproper first config_only (bool): Only configure the source, do not build it adjust_cfg (list of str): See the cfgutil module and run_commit() commit (Commit): Commit only being built @@ -419,7 +421,8 @@ class BuilderThread(threading.Thread): cmd_list = [] if do_config or adjust_cfg: result = self._reconfigure( - commit, brd, cwd, args, env, config_args, config_out, cmd_list) + commit, brd, cwd, args, env, config_args, config_out, cmd_list, + mrproper) do_config = False # No need to configure next time if adjust_cfg: cfgutil.adjust_cfg_file(cfg_file, adjust_cfg) @@ -445,9 +448,9 @@ class BuilderThread(threading.Thread): result.cmd_list = cmd_list return result, do_config - def run_commit(self, commit_upto, brd, work_dir, do_config, config_only, - force_build, force_build_failures, work_in_output, - adjust_cfg): + def run_commit(self, commit_upto, brd, work_dir, do_config, mrproper, + config_only, force_build, force_build_failures, + work_in_output, adjust_cfg): """Build a particular commit. If the build is already done, and we are not forcing a build, we skip @@ -458,6 +461,7 @@ class BuilderThread(threading.Thread): brd (Board): Board to build work_dir (str): Directory to which the source will be checked out do_config (bool): True to run a make <board>_defconfig on the source + mrproper (bool): True to run mrproper first config_only (bool): Only configure the source, do not build it force_build (bool): Force a build even if one was previously done force_build_failures (bool): Force a bulid if the previous result @@ -498,8 +502,9 @@ class BuilderThread(threading.Thread): if self.toolchain: commit = self._checkout(commit_upto, work_dir) result, do_config = self._config_and_build( - commit_upto, brd, work_dir, do_config, config_only, - adjust_cfg, commit, out_dir, out_rel_dir, result) + commit_upto, brd, work_dir, do_config, mrproper, + config_only, adjust_cfg, commit, out_dir, out_rel_dir, + result) result.already_done = False result.toolchain = self.toolchain @@ -688,19 +693,22 @@ class BuilderThread(threading.Thread): force_build = False for commit_upto in range(0, len(job.commits), job.step): result, request_config = self.run_commit(commit_upto, brd, - work_dir, do_config, self.builder.config_only, + work_dir, do_config, self.mrproper, + self.builder.config_only, force_build or self.builder.force_build, self.builder.force_build_failures, job.work_in_output, job.adjust_cfg) failed = result.return_code or result.stderr did_config = do_config - if failed and not do_config: + if failed and not do_config and not self.mrproper: # If our incremental build failed, try building again # with a reconfig. if self.builder.force_config_on_failure: result, request_config = self.run_commit(commit_upto, - brd, work_dir, True, False, True, False, - job.work_in_output, job.adjust_cfg) + brd, work_dir, True, + self.mrproper or self.builder.fallback_mrproper, + False, True, False, job.work_in_output, + job.adjust_cfg) did_config = True if not self.builder.force_reconfig: do_config = request_config @@ -744,7 +752,7 @@ class BuilderThread(threading.Thread): else: # Just build the currently checked-out build result, request_config = self.run_commit(None, brd, work_dir, True, - self.builder.config_only, True, + self.mrproper, self.builder.config_only, True, self.builder.force_build_failures, job.work_in_output, job.adjust_cfg) result.commit_upto = 0 diff --git a/tools/buildman/buildman.rst b/tools/buildman/buildman.rst index aae2477b5c3..b8ff3bf1ab2 100644 --- a/tools/buildman/buildman.rst +++ b/tools/buildman/buildman.rst @@ -995,7 +995,8 @@ By default, buildman doesn't execute 'make mrproper' prior to building the first commit for each board. This reduces the amount of work 'make' does, and hence speeds up the build. To force use of 'make mrproper', use -the -m flag. This flag will slow down any buildman invocation, since it increases the amount -of work done on any build. +of work done on any build. An alternative is to use the --fallback-mrproper +flag, which retries the build with 'make mrproper' only after a build failure. One possible application of buildman is as part of a continual edit, build, edit, build, ... cycle; repeatedly applying buildman to the same change or @@ -1285,6 +1286,11 @@ then buildman hangs. Failing to handle any eventuality is a bug in buildman and should be reported. But you can use -T0 to disable threading and hopefully figure out the root cause of the build failure. +For situations where buildman is invoked from multiple running processes, it is +sometimes useful to have buildman wait until the others have finished. Use the +--process-limit option for this: --process-limit 1 will allow only one buildman +to process jobs at a time. + Build summary ------------- diff --git a/tools/buildman/cmdline.py b/tools/buildman/cmdline.py index 03211bd5aa5..544a391a464 100644 --- a/tools/buildman/cmdline.py +++ b/tools/buildman/cmdline.py @@ -90,7 +90,9 @@ def add_upto_m(parser): parser.add_argument('--list-tool-chains', action='store_true', default=False, help='List available tool chains (use -v to see probing detail)') parser.add_argument('-m', '--mrproper', action='store_true', - default=False, help="Run 'make mrproper before reconfiguring") + default=False, help="Run 'make mrproper' before reconfiguring") + parser.add_argument('--fallback-mrproper', action='store_true', + default=False, help="Run 'make mrproper' and retry on build failure") parser.add_argument( '-M', '--allow-missing', action='store_true', default=False, help='Tell binman to allow missing blobs and generate fake ones as needed') @@ -127,6 +129,8 @@ def add_after_m(parser): default=False, help="Use an O= (output) directory per board rather than per thread") parser.add_argument('--print-arch', action='store_true', default=False, help="Print the architecture for a board (ARCH=)") + parser.add_argument('--process-limit', type=int, + default=0, help='Limit to number of buildmans running at once') parser.add_argument('-r', '--reproducible-builds', action='store_true', help='Set SOURCE_DATE_EPOCH=0 to suuport a reproducible build') parser.add_argument('-R', '--regen-board-list', type=str, diff --git a/tools/buildman/control.py b/tools/buildman/control.py index 8f6850c5211..d3d027f02ab 100644 --- a/tools/buildman/control.py +++ b/tools/buildman/control.py @@ -7,10 +7,13 @@ This holds the main control logic for buildman, when not running tests. """ +import getpass import multiprocessing import os import shutil import sys +import tempfile +import time from buildman import boards from buildman import bsettings @@ -21,10 +24,23 @@ from patman import gitutil from patman import patchstream from u_boot_pylib import command from u_boot_pylib import terminal -from u_boot_pylib.terminal import tprint +from u_boot_pylib import tools +from u_boot_pylib.terminal import print_clear, tprint TEST_BUILDER = None +# Space-separated list of buildman process IDs currently running jobs +RUNNING_FNAME = f'buildmanq.{getpass.getuser()}' + +# Lock file for access to RUNNING_FILE +LOCK_FNAME = f'{RUNNING_FNAME}.lock' + +# Wait time for access to lock (seconds) +LOCK_WAIT_S = 10 + +# Wait time to start running +RUN_WAIT_S = 300 + def get_plural(count): """Returns a plural 's' if count is not 1""" return 's' if count != 1 else '' @@ -108,7 +124,8 @@ def show_actions(series, why_selected, boards_selected, output_dir, print(commit.subject) print() for arg in why_selected: - if arg != 'all': + # When -x is used, only the 'all' member exists + if arg != 'all' or len(why_selected) == 1: print(arg, f': {len(why_selected[arg])} boards') if verbose: print(f" {' '.join(why_selected[arg])}") @@ -578,6 +595,125 @@ def calc_adjust_cfg(adjust_cfg, reproducible_builds): return adjust_cfg +def read_procs(tmpdir=tempfile.gettempdir()): + """Read the list of running buildman processes + + If the list is corrupted, returns an empty list + + Args: + tmpdir (str): Temporary directory to use (for testing only) + """ + running_fname = os.path.join(tmpdir, RUNNING_FNAME) + procs = [] + if os.path.exists(running_fname): + items = tools.read_file(running_fname, binary=False).split() + try: + procs = [int(x) for x in items] + except ValueError: # Handle invalid format + pass + return procs + + +def check_pid(pid): + """Check for existence of a unix PID + + https://stackoverflow.com/questions/568271/how-to-check-if-there-exists-a-process-with-a-given-pid-in-python + + Args: + pid (int): PID to check + + Returns: + True if it exists, else False + """ + try: + os.kill(pid, 0) + except OSError: + return False + else: + return True + + +def write_procs(procs, tmpdir=tempfile.gettempdir()): + """Write the list of running buildman processes + + Args: + tmpdir (str): Temporary directory to use (for testing only) + """ + running_fname = os.path.join(tmpdir, RUNNING_FNAME) + tools.write_file(running_fname, ' '.join([str(p) for p in procs]), + binary=False) + + # Allow another user to access the file + os.chmod(running_fname, 0o666) + +def wait_for_process_limit(limit, tmpdir=tempfile.gettempdir(), + pid=os.getpid()): + """Wait until the number of buildman processes drops to the limit + + This uses FileLock to protect a 'running' file, which contains a list of + PIDs of running buildman processes. The number of PIDs in the file indicates + the number of running processes. + + When buildman starts up, it calls this function to wait until it is OK to + start the build. + + On exit, no attempt is made to remove the PID from the file, since other + buildman processes will notice that the PID is no-longer valid, and ignore + it. + + Two timeouts are provided: + LOCK_WAIT_S: length of time to wait for the lock; if this occurs, the + lock is busted / removed before trying again + RUN_WAIT_S: length of time to wait to be allowed to run; if this occurs, + the build starts, with the PID being added to the file. + + Args: + limit (int): Maximum number of buildman processes, including this one; + must be > 0 + tmpdir (str): Temporary directory to use (for testing only) + pid (int): Current process ID (for testing only) + """ + from filelock import Timeout, FileLock + + running_fname = os.path.join(tmpdir, RUNNING_FNAME) + lock_fname = os.path.join(tmpdir, LOCK_FNAME) + lock = FileLock(lock_fname) + + # Allow another user to access the file + col = terminal.Color() + tprint('Waiting for other buildman processes...', newline=False, + colour=col.RED) + + claimed = False + deadline = time.time() + RUN_WAIT_S + while True: + try: + with lock.acquire(timeout=LOCK_WAIT_S): + os.chmod(lock_fname, 0o666) + procs = read_procs(tmpdir) + + # Drop PIDs which are not running + procs = list(filter(check_pid, procs)) + + # If we haven't hit the limit, add ourself + if len(procs) < limit: + tprint('done...', newline=False) + claimed = True + if time.time() >= deadline: + tprint('timeout...', newline=False) + claimed = True + if claimed: + write_procs(procs + [pid], tmpdir) + break + + except Timeout: + tprint('failed to get lock: busting...', newline=False) + os.remove(lock_fname) + + time.sleep(1) + tprint('starting build', newline=False) + print_clear() + def do_buildman(args, toolchains=None, make_func=None, brds=None, clean_dir=False, test_thread_exceptions=False): """The main control code for buildman @@ -656,6 +792,7 @@ def do_buildman(args, toolchains=None, make_func=None, brds=None, no_subdirs=args.no_subdirs, full_path=args.full_path, verbose_build=args.verbose_build, mrproper=args.mrproper, + fallback_mrproper=args.fallback_mrproper, per_board_out_dir=args.per_board_out_dir, config_only=args.config_only, squash_config_y=not args.preserve_config_y, @@ -676,5 +813,8 @@ def do_buildman(args, toolchains=None, make_func=None, brds=None, TEST_BUILDER = builder + if args.process_limit: + wait_for_process_limit(args.process_limit) + return run_builder(builder, series.commits if series else None, brds.get_selected_dict(), args) diff --git a/tools/buildman/func_test.py b/tools/buildman/func_test.py index 6b88ed815d6..0ac9fc7e44f 100644 --- a/tools/buildman/func_test.py +++ b/tools/buildman/func_test.py @@ -807,27 +807,27 @@ CONFIG_LOCALVERSION=y params, warnings = self._boards.scan_defconfigs(src, src) # We should get two boards - self.assertEquals(2, len(params)) + self.assertEqual(2, len(params)) self.assertFalse(warnings) first = 0 if params[0]['target'] == 'board0' else 1 board0 = params[first] board2 = params[1 - first] - self.assertEquals('arm', board0['arch']) - self.assertEquals('armv7', board0['cpu']) - self.assertEquals('-', board0['soc']) - self.assertEquals('Tester', board0['vendor']) - self.assertEquals('ARM Board 0', board0['board']) - self.assertEquals('config0', board0['config']) - self.assertEquals('board0', board0['target']) - - self.assertEquals('powerpc', board2['arch']) - self.assertEquals('ppc', board2['cpu']) - self.assertEquals('mpc85xx', board2['soc']) - self.assertEquals('Tester', board2['vendor']) - self.assertEquals('PowerPC board 1', board2['board']) - self.assertEquals('config2', board2['config']) - self.assertEquals('board2', board2['target']) + self.assertEqual('arm', board0['arch']) + self.assertEqual('armv7', board0['cpu']) + self.assertEqual('-', board0['soc']) + self.assertEqual('Tester', board0['vendor']) + self.assertEqual('ARM Board 0', board0['board']) + self.assertEqual('config0', board0['config']) + self.assertEqual('board0', board0['target']) + + self.assertEqual('powerpc', board2['arch']) + self.assertEqual('ppc', board2['cpu']) + self.assertEqual('mpc85xx', board2['soc']) + self.assertEqual('Tester', board2['vendor']) + self.assertEqual('PowerPC board 1', board2['board']) + self.assertEqual('config2', board2['config']) + self.assertEqual('board2', board2['target']) def test_output_is_new(self): """Test detecting new changes to Kconfig""" @@ -898,7 +898,7 @@ Active aarch64 armv8 - armltd total_compute board2 params_list, warnings = self._boards.build_board_list(config_dir, src) # There should be two boards no warnings - self.assertEquals(2, len(params_list)) + self.assertEqual(2, len(params_list)) self.assertFalse(warnings) # Set an invalid status line in the file @@ -907,12 +907,12 @@ Active aarch64 armv8 - armltd total_compute board2 for line in orig_data.splitlines(keepends=True)] tools.write_file(main, ''.join(lines), binary=False) params_list, warnings = self._boards.build_board_list(config_dir, src) - self.assertEquals(2, len(params_list)) + self.assertEqual(2, len(params_list)) params = params_list[0] if params['target'] == 'board2': params = params_list[1] - self.assertEquals('-', params['status']) - self.assertEquals(["WARNING: Other: unknown status for 'board0'"], + self.assertEqual('-', params['status']) + self.assertEqual(["WARNING: Other: unknown status for 'board0'"], warnings) # Remove the status line (S:) from a file @@ -920,39 +920,39 @@ Active aarch64 armv8 - armltd total_compute board2 if not line.startswith('S:')] tools.write_file(main, ''.join(lines), binary=False) params_list, warnings = self._boards.build_board_list(config_dir, src) - self.assertEquals(2, len(params_list)) - self.assertEquals(["WARNING: -: unknown status for 'board0'"], warnings) + self.assertEqual(2, len(params_list)) + self.assertEqual(["WARNING: -: unknown status for 'board0'"], warnings) # Remove the configs/ line (F:) from a file - this is the last line data = ''.join(orig_data.splitlines(keepends=True)[:-1]) tools.write_file(main, data, binary=False) params_list, warnings = self._boards.build_board_list(config_dir, src) - self.assertEquals(2, len(params_list)) - self.assertEquals(["WARNING: no maintainers for 'board0'"], warnings) + self.assertEqual(2, len(params_list)) + self.assertEqual(["WARNING: no maintainers for 'board0'"], warnings) # Mark a board as orphaned - this should give a warning lines = ['S: Orphaned' if line.startswith('S') else line for line in orig_data.splitlines(keepends=True)] tools.write_file(main, ''.join(lines), binary=False) params_list, warnings = self._boards.build_board_list(config_dir, src) - self.assertEquals(2, len(params_list)) - self.assertEquals(["WARNING: no maintainers for 'board0'"], warnings) + self.assertEqual(2, len(params_list)) + self.assertEqual(["WARNING: no maintainers for 'board0'"], warnings) # Change the maintainer to '-' - this should give a warning lines = ['M: -' if line.startswith('M') else line for line in orig_data.splitlines(keepends=True)] tools.write_file(main, ''.join(lines), binary=False) params_list, warnings = self._boards.build_board_list(config_dir, src) - self.assertEquals(2, len(params_list)) - self.assertEquals(["WARNING: -: unknown status for 'board0'"], warnings) + self.assertEqual(2, len(params_list)) + self.assertEqual(["WARNING: -: unknown status for 'board0'"], warnings) # Remove the maintainer line (M:) from a file lines = [line for line in orig_data.splitlines(keepends=True) if not line.startswith('M:')] tools.write_file(main, ''.join(lines), binary=False) params_list, warnings = self._boards.build_board_list(config_dir, src) - self.assertEquals(2, len(params_list)) - self.assertEquals(["WARNING: no maintainers for 'board0'"], warnings) + self.assertEqual(2, len(params_list)) + self.assertEqual(["WARNING: no maintainers for 'board0'"], warnings) # Move the contents of the second file into this one, removing the # second file, to check multiple records in a single file. @@ -960,14 +960,14 @@ Active aarch64 armv8 - armltd total_compute board2 tools.write_file(main, both_data, binary=False) os.remove(other) params_list, warnings = self._boards.build_board_list(config_dir, src) - self.assertEquals(2, len(params_list)) + self.assertEqual(2, len(params_list)) self.assertFalse(warnings) # Add another record, this should be ignored with a warning extra = '\n\nAnother\nM: Fred\nF: configs/board9_defconfig\nS: other\n' tools.write_file(main, both_data + extra, binary=False) params_list, warnings = self._boards.build_board_list(config_dir, src) - self.assertEquals(2, len(params_list)) + self.assertEqual(2, len(params_list)) self.assertFalse(warnings) # Add another TARGET to the Kconfig @@ -983,8 +983,8 @@ endif tools.write_file(kc_file, orig_kc_data + extra) params_list, warnings = self._boards.build_board_list(config_dir, src, warn_targets=True) - self.assertEquals(2, len(params_list)) - self.assertEquals( + self.assertEqual(2, len(params_list)) + self.assertEqual( ['WARNING: board2_defconfig: Duplicate TARGET_xxx: board2 and other'], warnings) @@ -994,8 +994,8 @@ endif tools.write_file(kc_file, b''.join(lines)) params_list, warnings = self._boards.build_board_list(config_dir, src, warn_targets=True) - self.assertEquals(2, len(params_list)) - self.assertEquals( + self.assertEqual(2, len(params_list)) + self.assertEqual( ['WARNING: board2_defconfig: No TARGET_BOARD2 enabled'], warnings) tools.write_file(kc_file, orig_kc_data) @@ -1004,7 +1004,7 @@ endif data = ''.join(both_data.splitlines(keepends=True)[:-1]) tools.write_file(main, data + 'N: oa.*2\n', binary=False) params_list, warnings = self._boards.build_board_list(config_dir, src) - self.assertEquals(2, len(params_list)) + self.assertEqual(2, len(params_list)) self.assertFalse(warnings) def testRegenBoards(self): diff --git a/tools/buildman/pyproject.toml b/tools/buildman/pyproject.toml index fe0f6421b53..68bfa45c3f4 100644 --- a/tools/buildman/pyproject.toml +++ b/tools/buildman/pyproject.toml @@ -8,7 +8,11 @@ version = "0.0.6" authors = [ { name="Simon Glass", email="sjg@chromium.org" }, ] -dependencies = ["u_boot_pylib >= 0.0.6", "patch-manager >= 0.0.6"] +dependencies = [ + "filelock >= 3.0.12", + "u_boot_pylib >= 0.0.6", + "patch-manager >= 0.0.6" +] description = "Buildman build tool for U-Boot" readme = "README.rst" requires-python = ">=3.7" diff --git a/tools/buildman/requirements.txt b/tools/buildman/requirements.txt index 4a31e69e4cb..052d0ed5c6f 100644 --- a/tools/buildman/requirements.txt +++ b/tools/buildman/requirements.txt @@ -1,3 +1,5 @@ +coverage==6.2 jsonschema==4.17.3 +pycryptodome==3.20 pyyaml==6.0 yamllint==1.26.3 diff --git a/tools/buildman/test.py b/tools/buildman/test.py index f92add7a7c5..bfad3093030 100644 --- a/tools/buildman/test.py +++ b/tools/buildman/test.py @@ -2,12 +2,14 @@ # Copyright (c) 2012 The Chromium OS Authors. # +from filelock import FileLock import os import shutil import sys import tempfile import time import unittest +from unittest.mock import patch from buildman import board from buildman import boards @@ -156,6 +158,11 @@ class TestBuild(unittest.TestCase): if not os.path.isdir(self.base_dir): os.mkdir(self.base_dir) + self.cur_time = 0 + self.valid_pids = [] + self.finish_time = None + self.finish_pid = None + def tearDown(self): shutil.rmtree(self.base_dir) @@ -584,7 +591,7 @@ class TestBuild(unittest.TestCase): if use_network: with test_util.capture_sys_output() as (stdout, stderr): url = self.toolchains.LocateArchUrl('arm') - self.assertRegexpMatches(url, 'https://www.kernel.org/pub/tools/' + self.assertRegex(url, 'https://www.kernel.org/pub/tools/' 'crosstool/files/bin/x86_64/.*/' 'x86_64-gcc-.*-nolibc[-_]arm-.*linux-gnueabi.tar.xz') @@ -747,6 +754,120 @@ class TestBuild(unittest.TestCase): self.assertEqual([ ['MARY="mary"', 'Missing expected line: CONFIG_MARY="mary"']], result) + def get_procs(self): + running_fname = os.path.join(self.base_dir, control.RUNNING_FNAME) + items = tools.read_file(running_fname, binary=False).split() + return [int(x) for x in items] + + def get_time(self): + return self.cur_time + + def inc_time(self, amount): + self.cur_time += amount + + # Handle a process exiting + if self.finish_time == self.cur_time: + self.valid_pids = [pid for pid in self.valid_pids + if pid != self.finish_pid] + + def kill(self, pid, signal): + if pid not in self.valid_pids: + raise OSError('Invalid PID') + + def test_process_limit(self): + """Test wait_for_process_limit() function""" + tmpdir = self.base_dir + + with (patch('time.time', side_effect=self.get_time), + patch('time.sleep', side_effect=self.inc_time), + patch('os.kill', side_effect=self.kill)): + # Grab the process. Since there is no other profcess, this should + # immediately succeed + control.wait_for_process_limit(1, tmpdir=tmpdir, pid=1) + lines = terminal.get_print_test_lines() + self.assertEqual(0, self.cur_time) + self.assertEqual('Waiting for other buildman processes...', + lines[0].text) + self.assertEqual(self._col.RED, lines[0].colour) + self.assertEqual(False, lines[0].newline) + self.assertEqual(True, lines[0].bright) + + self.assertEqual('done...', lines[1].text) + self.assertEqual(None, lines[1].colour) + self.assertEqual(False, lines[1].newline) + self.assertEqual(True, lines[1].bright) + + self.assertEqual('starting build', lines[2].text) + self.assertEqual([1], control.read_procs(tmpdir)) + self.assertEqual(None, lines[2].colour) + self.assertEqual(False, lines[2].newline) + self.assertEqual(True, lines[2].bright) + + # Try again, with a different PID...this should eventually timeout + # and start the build anyway + self.cur_time = 0 + self.valid_pids = [1] + control.wait_for_process_limit(1, tmpdir=tmpdir, pid=2) + lines = terminal.get_print_test_lines() + self.assertEqual('Waiting for other buildman processes...', + lines[0].text) + self.assertEqual('timeout...', lines[1].text) + self.assertEqual(None, lines[1].colour) + self.assertEqual(False, lines[1].newline) + self.assertEqual(True, lines[1].bright) + self.assertEqual('starting build', lines[2].text) + self.assertEqual([1, 2], control.read_procs(tmpdir)) + self.assertEqual(control.RUN_WAIT_S, self.cur_time) + + # Check lock-busting + self.cur_time = 0 + self.valid_pids = [1, 2] + lock_fname = os.path.join(tmpdir, control.LOCK_FNAME) + lock = FileLock(lock_fname) + lock.acquire(timeout=1) + control.wait_for_process_limit(1, tmpdir=tmpdir, pid=3) + lines = terminal.get_print_test_lines() + self.assertEqual('Waiting for other buildman processes...', + lines[0].text) + self.assertEqual('failed to get lock: busting...', lines[1].text) + self.assertEqual(None, lines[1].colour) + self.assertEqual(False, lines[1].newline) + self.assertEqual(True, lines[1].bright) + self.assertEqual('timeout...', lines[2].text) + self.assertEqual('starting build', lines[3].text) + self.assertEqual([1, 2, 3], control.read_procs(tmpdir)) + self.assertEqual(control.RUN_WAIT_S, self.cur_time) + lock.release() + + # Check handling of dead processes. Here we have PID 2 as a running + # process, even though the PID file contains 1, 2 and 3. So we can + # add one more PID, to make 2 and 4 + self.cur_time = 0 + self.valid_pids = [2] + control.wait_for_process_limit(2, tmpdir=tmpdir, pid=4) + lines = terminal.get_print_test_lines() + self.assertEqual('Waiting for other buildman processes...', + lines[0].text) + self.assertEqual('done...', lines[1].text) + self.assertEqual('starting build', lines[2].text) + self.assertEqual([2, 4], control.read_procs(tmpdir)) + self.assertEqual(0, self.cur_time) + + # Try again, with PID 2 quitting at time 50. This allows the new + # build to start + self.cur_time = 0 + self.valid_pids = [2, 4] + self.finish_pid = 2 + self.finish_time = 50 + control.wait_for_process_limit(2, tmpdir=tmpdir, pid=5) + lines = terminal.get_print_test_lines() + self.assertEqual('Waiting for other buildman processes...', + lines[0].text) + self.assertEqual('done...', lines[1].text) + self.assertEqual('starting build', lines[2].text) + self.assertEqual([4, 5], control.read_procs(tmpdir)) + self.assertEqual(self.finish_time, self.cur_time) + if __name__ == "__main__": unittest.main() diff --git a/tools/buildman/toolchain.py b/tools/buildman/toolchain.py index 79c7c11a110..324ad0e0821 100644 --- a/tools/buildman/toolchain.py +++ b/tools/buildman/toolchain.py @@ -175,9 +175,9 @@ class Toolchain: def MakeEnvironment(self, full_path): """Returns an environment for using the toolchain. - Thie takes the current environment and adds CROSS_COMPILE so that + This takes the current environment and adds CROSS_COMPILE so that the tool chain will operate correctly. This also disables localized - output and possibly unicode encoded output of all build tools by + output and possibly Unicode encoded output of all build tools by adding LC_ALL=C. Note that os.environb is used to obtain the environment, since in some diff --git a/tools/dtoc/dtb_platdata.py b/tools/dtoc/dtb_platdata.py index 39f416cfd80..89066e6403f 100644 --- a/tools/dtoc/dtb_platdata.py +++ b/tools/dtoc/dtb_platdata.py @@ -835,7 +835,6 @@ class DtbPlatdata(): def generate_uclasses(self): self.out('\n') - self.out('#include <common.h>\n') self.out('#include <dm.h>\n') self.out('#include <dt-structs.h>\n') self.out('\n') @@ -1059,7 +1058,6 @@ class DtbPlatdata(): self.out('/* Allow use of U_BOOT_DRVINFO() in this file */\n') self.out('#define DT_PLAT_C\n') self.out('\n') - self.out('#include <common.h>\n') self.out('#include <dm.h>\n') self.out('#include <dt-structs.h>\n') self.out('\n') @@ -1092,7 +1090,6 @@ class DtbPlatdata(): See the documentation in doc/driver-model/of-plat.rst for more information. """ - self.out('#include <common.h>\n') self.out('#include <dm.h>\n') self.out('#include <dt-structs.h>\n') self.out('\n') diff --git a/tools/dtoc/test_dtoc.py b/tools/dtoc/test_dtoc.py index 597c93e8a87..c4a0889aebe 100755 --- a/tools/dtoc/test_dtoc.py +++ b/tools/dtoc/test_dtoc.py @@ -63,7 +63,6 @@ C_HEADER = C_HEADER_PRE + ''' /* Allow use of U_BOOT_DRVINFO() in this file */ #define DT_PLAT_C -#include <common.h> #include <dm.h> #include <dt-structs.h> ''' @@ -417,7 +416,6 @@ U_BOOT_DRVINFO(spl_test3) = { ''' uclass_text_inst = ''' -#include <common.h> #include <dm.h> #include <dt-structs.h> @@ -521,7 +519,6 @@ DM_UCLASS_INST(testfdt) = { * This was generated by dtoc from a .dtb (device tree binary) file. */ -#include <common.h> #include <dm.h> #include <dt-structs.h> diff --git a/tools/env/fw_env.h b/tools/env/fw_env.h index 78c803c9447..060eea542cc 100644 --- a/tools/env/fw_env.h +++ b/tools/env/fw_env.h @@ -94,7 +94,6 @@ int fw_env_set(int argc, char *argv[], struct env_opts *opts); */ int fw_parse_script(char *fname, struct env_opts *opts); - /** * fw_env_open() - read enviroment from flash into RAM cache * @@ -151,7 +150,6 @@ int fw_env_flush(struct env_opts *opts); */ int fw_env_close(struct env_opts *opts); - /** * fw_env_version - return the current version of the library * diff --git a/tools/envcrc.c b/tools/envcrc.c index 550f31038bd..09051800364 100644 --- a/tools/envcrc.c +++ b/tools/envcrc.c @@ -48,7 +48,6 @@ #define ENV_SIZE (CONFIG_ENV_SIZE - ENV_HEADER_SIZE) - #ifdef ENV_IS_EMBEDDED # include <env_internal.h> extern unsigned int env_size; diff --git a/tools/fdtgrep.c b/tools/fdtgrep.c index f1ff1946bd4..037176bc9ef 100644 --- a/tools/fdtgrep.c +++ b/tools/fdtgrep.c @@ -102,7 +102,6 @@ static void print_ansi_colour(FILE *fout, int col) fprintf(fout, "\033[1;%dm", col + 30); } - /** * value_add() - Add a new value to our list of things to grep for * diff --git a/tools/gdb/remote.c b/tools/gdb/remote.c index 3cd04213514..f17bb06fb4c 100644 --- a/tools/gdb/remote.c +++ b/tools/gdb/remote.c @@ -45,7 +45,6 @@ extern "C" { #endif #endif /* alloca not defined. */ - #include "serial.h" #include "error.h" #include "remote.h" @@ -313,7 +312,6 @@ static int remote_binary_checked; ? (REGISTER_BYTES * 2 + 32) \ : 400) - /* This variable sets the number of bytes to be written to the target in a single packet. Normally PBUFSIZ is satisfactory, but some targets need smaller values (perhaps because the receiving end diff --git a/tools/image-host.c b/tools/image-host.c index 7bfc0cb6b18..49ce7436bb9 100644 --- a/tools/image-host.c +++ b/tools/image-host.c @@ -730,7 +730,7 @@ static const char *fit_config_get_image_list(const void *fit, int noffset, int *lenp, int *allow_missingp) { static const char default_list[] = FIT_KERNEL_PROP "\0" - FIT_FDT_PROP; + FIT_FDT_PROP "\0" FIT_SCRIPT_PROP; const char *prop; /* If there is an "sign-image" property, use that */ diff --git a/tools/imagetool.h b/tools/imagetool.h index a766aa2ae91..57be608210a 100644 --- a/tools/imagetool.h +++ b/tools/imagetool.h @@ -256,7 +256,6 @@ time_t imagetool_get_source_date( * for ex. default_image.c, fit_image.c */ - void pbl_load_uboot(int fd, struct image_tool_params *mparams); int zynqmpbif_copy_image(int fd, struct image_tool_params *mparams); int imx8image_copy_image(int fd, struct image_tool_params *mparams); diff --git a/tools/img2srec.c b/tools/img2srec.c index 75efd76e0e3..5a6d3259f3e 100644 --- a/tools/img2srec.c +++ b/tools/img2srec.c @@ -109,7 +109,6 @@ static char* ExtractDecimal (uint32_t* value, char* getPtr) return getPtr; } /* ExtractDecimal */ - static void ExtractNumber (uint32_t* value, char* getPtr) { bool neg = false; @@ -129,7 +128,6 @@ static void ExtractNumber (uint32_t* value, char* getPtr) if (neg) *value = -(*value); } /* ExtractNumber */ - static uint8_t* ExtractWord(uint16_t* value, uint8_t* buffer) { uint16_t x; @@ -139,7 +137,6 @@ static uint8_t* ExtractWord(uint16_t* value, uint8_t* buffer) return buffer; } /* ExtractWord */ - static uint8_t* ExtractLong(uint32_t* value, uint8_t* buffer) { uint32_t x; @@ -151,14 +148,12 @@ static uint8_t* ExtractLong(uint32_t* value, uint8_t* buffer) return buffer; } /* ExtractLong */ - static uint8_t* ExtractBlock(uint16_t count, uint8_t* data, uint8_t* buffer) { while (count--) *data++ = *buffer++; return buffer; } /* ExtractBlock */ - static char* WriteHex(char* pa, uint8_t value, uint16_t* pCheckSum) { uint16_t temp; @@ -173,7 +168,6 @@ static char* WriteHex(char* pa, uint8_t value, uint16_t* pCheckSum) return pa; } - static char* BuildSRecord(char* pa, uint16_t sType, uint32_t addr, const uint8_t* data, int nCount) { @@ -223,7 +217,6 @@ static char* BuildSRecord(char* pa, uint16_t sType, uint32_t addr, return pa; } - static void ConvertELF(char* fileName, uint32_t loadOffset) { FILE* file; @@ -240,7 +233,6 @@ static void ConvertELF(char* fileName, uint32_t loadOffset) char srecLine[128]; char *hdr_name; - /* open file */ if ((file = fopen(fileName,"rb")) == NULL) { fprintf (stderr, "Can't open %s: %s\n", fileName, strerror(errno)); @@ -348,7 +340,6 @@ static void ConvertELF(char* fileName, uint32_t loadOffset) fclose(file); } /* ConvertELF */ - /************************************************************************* | MAIN |*************************************************************************/ diff --git a/tools/imx8mimage.c b/tools/imx8mimage.c index 939f829a9f7..d60d293e649 100644 --- a/tools/imx8mimage.c +++ b/tools/imx8mimage.c @@ -5,7 +5,6 @@ * Peng Fan <peng.fan@nxp.com> */ - #include "imagetool.h" #include <image.h> #include "imximage.h" diff --git a/tools/imximage.c b/tools/imximage.c index 2df4c7dd491..467d9f27d2a 100644 --- a/tools/imximage.c +++ b/tools/imximage.c @@ -793,7 +793,6 @@ static uint32_t parse_cfg_file(struct imx_header *imxhdr, char *name) return dcd_len; } - static int imximage_check_image_types(uint8_t type) { if (type == IH_TYPE_IMXIMAGE) @@ -1057,7 +1056,6 @@ static int imximage_generate(struct image_tool_params *params, return pad_len; } - /* * imximage parameters */ diff --git a/tools/kwbimage.c b/tools/kwbimage.c index 4dce495ff03..d1cbced28fc 100644 --- a/tools/kwbimage.c +++ b/tools/kwbimage.c @@ -804,7 +804,6 @@ static int kwb_sign_and_verify(RSA *key, void *data, int datasz, return 0; } - static int kwb_dump_fuse_cmds_38x(FILE *out, struct secure_hdr_v1 *sec_hdr) { struct hash_v1 kak_pub_hash; diff --git a/tools/mkfwumdata.c b/tools/mkfwumdata.c index 9732a8ddc5a..fbc2067bc12 100644 --- a/tools/mkfwumdata.c +++ b/tools/mkfwumdata.c @@ -10,28 +10,35 @@ #include <stdint.h> #include <stdlib.h> #include <string.h> -#include <u-boot/crc.h> #include <unistd.h> +#include <generated/autoconf.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <u-boot/crc.h> #include <uuid/uuid.h> -/* This will dynamically allocate the fwu_mdata */ -#define CONFIG_FWU_NUM_BANKS 0 -#define CONFIG_FWU_NUM_IMAGES_PER_BANK 0 - -/* Since we can not include fwu.h, redefine version here. */ -#define FWU_MDATA_VERSION 1 - typedef uint8_t u8; typedef int16_t s16; typedef uint16_t u16; typedef uint32_t u32; typedef uint64_t u64; -#include <fwu_mdata.h> +#undef CONFIG_FWU_NUM_BANKS +#undef CONFIG_FWU_NUM_IMAGES_PER_BANK -/* TODO: Endianness conversion may be required for some arch. */ +/* This will dynamically allocate the fwu_mdata */ +#define CONFIG_FWU_NUM_BANKS 0 +#define CONFIG_FWU_NUM_IMAGES_PER_BANK 0 + +/* version 2 supports maximum of 4 banks */ +#define MAX_BANKS_V2 4 + +#define BANK_INVALID (u8)0xFF +#define BANK_ACCEPTED (u8)0xFC -static const char *opts_short = "b:i:a:p:gh"; +#include <fwu_mdata.h> + +static const char *opts_short = "b:i:a:p:v:V:gh"; static struct option options[] = { {"banks", required_argument, NULL, 'b'}, @@ -39,6 +46,8 @@ static struct option options[] = { {"guid", required_argument, NULL, 'g'}, {"active-bank", required_argument, NULL, 'a'}, {"previous-bank", required_argument, NULL, 'p'}, + {"version", required_argument, NULL, 'v'}, + {"vendor-file", required_argument, NULL, 'V'}, {"help", no_argument, NULL, 'h'}, {NULL, 0, NULL, 0}, }; @@ -49,9 +58,11 @@ static void print_usage(void) fprintf(stderr, "Options:\n" "\t-i, --images <num> Number of images (mandatory)\n" "\t-b, --banks <num> Number of banks (mandatory)\n" + "\t-v, --version Metadata version (mandatory)\n" "\t-a, --active-bank <num> Active bank (default=0)\n" "\t-p, --previous-bank <num> Previous active bank (default=active_bank - 1)\n" "\t-g, --guid Use GUID instead of UUID\n" + "\t-V, --vendor-file Vendor data file to append to the metadata\n" "\t-h, --help print a help message\n" ); fprintf(stderr, " UUIDs list syntax:\n" @@ -70,13 +81,28 @@ struct fwu_mdata_object { size_t images; size_t banks; size_t size; + u8 version; + size_t vsize; + void *vbuf; struct fwu_mdata *mdata; }; static int previous_bank, active_bank; static bool __use_guid; -static struct fwu_mdata_object *fwu_alloc_mdata(size_t images, size_t banks) +static bool supported_mdata_version(unsigned long version) +{ + switch (version) { + case 1: + case 2: + return true; + default: + return false; + } +} + +static struct fwu_mdata_object *fwu_alloc_mdata(size_t images, size_t banks, + u8 version, size_t vendor_size) { struct fwu_mdata_object *mobj; @@ -84,19 +110,40 @@ static struct fwu_mdata_object *fwu_alloc_mdata(size_t images, size_t banks) if (!mobj) return NULL; - mobj->size = sizeof(struct fwu_mdata) + - (sizeof(struct fwu_image_entry) + - sizeof(struct fwu_image_bank_info) * banks) * images; + if (version == 1) { + mobj->size = sizeof(struct fwu_mdata) + + (sizeof(struct fwu_image_entry) + + sizeof(struct fwu_image_bank_info) * banks) * images; + } else { + mobj->size = sizeof(struct fwu_mdata) + + sizeof(struct fwu_fw_store_desc) + + (sizeof(struct fwu_image_entry) + + sizeof(struct fwu_image_bank_info) * banks) * images; + + mobj->size += vendor_size; + mobj->vsize = vendor_size; + } + mobj->images = images; mobj->banks = banks; + mobj->version = version; mobj->mdata = calloc(1, mobj->size); - if (!mobj->mdata) { - free(mobj); - return NULL; + if (!mobj->mdata) + goto alloc_err; + + if (vendor_size) { + mobj->vbuf = calloc(1, mobj->vsize); + if (!mobj->vbuf) + goto alloc_err; } return mobj; + +alloc_err: + free(mobj->mdata); + free(mobj); + return NULL; } static struct fwu_image_entry * @@ -104,9 +151,18 @@ fwu_get_image(struct fwu_mdata_object *mobj, size_t idx) { size_t offset; - offset = sizeof(struct fwu_mdata) + - (sizeof(struct fwu_image_entry) + - sizeof(struct fwu_image_bank_info) * mobj->banks) * idx; + if (mobj->version == 1) { + offset = sizeof(struct fwu_mdata) + + (sizeof(struct fwu_image_entry) + + sizeof(struct fwu_image_bank_info) * mobj->banks) * + idx; + } else { + offset = sizeof(struct fwu_mdata) + + sizeof(struct fwu_fw_store_desc) + + (sizeof(struct fwu_image_entry) + + sizeof(struct fwu_image_bank_info) * mobj->banks) * + idx; + } return (struct fwu_image_entry *)((char *)mobj->mdata + offset); } @@ -116,11 +172,20 @@ fwu_get_bank(struct fwu_mdata_object *mobj, size_t img_idx, size_t bnk_idx) { size_t offset; - offset = sizeof(struct fwu_mdata) + - (sizeof(struct fwu_image_entry) + - sizeof(struct fwu_image_bank_info) * mobj->banks) * img_idx + - sizeof(struct fwu_image_entry) + - sizeof(struct fwu_image_bank_info) * bnk_idx; + if (mobj->version == 1) { + offset = sizeof(struct fwu_mdata) + + (sizeof(struct fwu_image_entry) + + sizeof(struct fwu_image_bank_info) * mobj->banks) * + img_idx + sizeof(struct fwu_image_entry) + + sizeof(struct fwu_image_bank_info) * bnk_idx; + } else { + offset = sizeof(struct fwu_mdata) + + sizeof(struct fwu_fw_store_desc) + + (sizeof(struct fwu_image_entry) + + sizeof(struct fwu_image_bank_info) * mobj->banks) * + img_idx + sizeof(struct fwu_image_entry) + + sizeof(struct fwu_image_bank_info) * bnk_idx; + } return (struct fwu_image_bank_info *)((char *)mobj->mdata + offset); } @@ -188,7 +253,7 @@ fwu_parse_fill_image_uuid(struct fwu_mdata_object *mobj, return -EINVAL; if (strcmp(uuid, "0") && - uuid_guid_parse(uuid, (unsigned char *)&image->location_uuid) < 0) + uuid_guid_parse(uuid, (unsigned char *)&image->location_guid) < 0) return -EINVAL; /* Image type UUID */ @@ -196,7 +261,7 @@ fwu_parse_fill_image_uuid(struct fwu_mdata_object *mobj, if (!uuid) return -EINVAL; - if (uuid_guid_parse(uuid, (unsigned char *)&image->image_type_uuid) < 0) + if (uuid_guid_parse(uuid, (unsigned char *)&image->image_type_guid) < 0) return -EINVAL; /* Fill bank image-UUID */ @@ -210,45 +275,118 @@ fwu_parse_fill_image_uuid(struct fwu_mdata_object *mobj, return -EINVAL; if (strcmp(uuid, "0") && - uuid_guid_parse(uuid, (unsigned char *)&bank->image_uuid) < 0) + uuid_guid_parse(uuid, (unsigned char *)&bank->image_guid) < 0) return -EINVAL; } return 0; } +#if defined(CONFIG_FWU_MDATA_V1) +static void fwu_fill_version_specific_mdata(struct fwu_mdata_object *mobj) +{ +} +#else +static void fwu_fill_version_specific_mdata(struct fwu_mdata_object *mobj) +{ + int i; + struct fwu_fw_store_desc *fw_desc; + struct fwu_mdata *mdata = mobj->mdata; + + mdata->metadata_size = mobj->size; + mdata->desc_offset = sizeof(struct fwu_mdata); + + for (i = 0; i < MAX_BANKS_V2; i++) + mdata->bank_state[i] = i < mobj->banks ? + BANK_ACCEPTED : BANK_INVALID; + + fw_desc = (struct fwu_fw_store_desc *)((u8 *)mdata + sizeof(*mdata)); + fw_desc->num_banks = mobj->banks; + fw_desc->num_images = mobj->images; + fw_desc->img_entry_size = sizeof(struct fwu_image_entry) + + (sizeof(struct fwu_image_bank_info) * mobj->banks); + fw_desc->bank_info_entry_size = + sizeof(struct fwu_image_bank_info); +} +#endif /* CONFIG_FWU_MDATA_V1 */ + /* Caller must ensure that @uuids[] has @mobj->images entries. */ static int fwu_parse_fill_uuids(struct fwu_mdata_object *mobj, char *uuids[]) { struct fwu_mdata *mdata = mobj->mdata; + char *vdata; int i, ret; - mdata->version = FWU_MDATA_VERSION; + mdata->version = mobj->version; mdata->active_index = active_bank; mdata->previous_active_index = previous_bank; + fwu_fill_version_specific_mdata(mobj); + for (i = 0; i < mobj->images; i++) { ret = fwu_parse_fill_image_uuid(mobj, i, uuids[i]); if (ret < 0) return ret; } + if (mobj->vsize) { + vdata = (char *)mobj->mdata + (mobj->size - mobj->vsize); + memcpy(vdata, mobj->vbuf, mobj->vsize); + } + mdata->crc32 = crc32(0, (const unsigned char *)&mdata->version, mobj->size - sizeof(uint32_t)); return 0; } -static int -fwu_make_mdata(size_t images, size_t banks, char *uuids[], char *output) +static int fwu_read_vendor_data(struct fwu_mdata_object *mobj, + const char *vendor_file) +{ + int ret = 0; + FILE *vfile = NULL; + + vfile = fopen(vendor_file, "r"); + if (!vfile) { + ret = -1; + goto out; + } + + if (fread(mobj->vbuf, 1, mobj->vsize, vfile) != mobj->vsize) + ret = -1; + +out: + fclose(vfile); + return ret; +} + +static int fwu_make_mdata(size_t images, size_t banks, u8 version, + const char *vendor_file, char *uuids[], + char *output) { - struct fwu_mdata_object *mobj; - FILE *file; int ret; + FILE *file; + struct stat sbuf; + size_t vendor_size = 0; + struct fwu_mdata_object *mobj; + + if (vendor_file) { + ret = stat(vendor_file, &sbuf); + if (ret) + return -errno; - mobj = fwu_alloc_mdata(images, banks); + vendor_size = sbuf.st_size; + } + + mobj = fwu_alloc_mdata(images, banks, version, vendor_size); if (!mobj) return -ENOMEM; + if (vendor_file) { + ret = fwu_read_vendor_data(mobj, vendor_file); + if (ret) + goto done_make; + } + ret = fwu_parse_fill_uuids(mobj, uuids); if (ret < 0) goto done_make; @@ -259,7 +397,7 @@ fwu_make_mdata(size_t images, size_t banks, char *uuids[], char *output) goto done_make; } - ret = fwrite(mobj->mdata, mobj->size, 1, file); + ret = fwrite(mobj->mdata, 1, mobj->size, file); if (ret != mobj->size) ret = -errno; else @@ -269,6 +407,7 @@ fwu_make_mdata(size_t images, size_t banks, char *uuids[], char *output) done_make: free(mobj->mdata); + free(mobj->vbuf); free(mobj); return ret; @@ -276,13 +415,15 @@ done_make: int main(int argc, char *argv[]) { - unsigned long banks = 0, images = 0; + unsigned long banks = 0, images = 0, version = 0; int c, ret; + const char *vendor_file; /* Explicitly initialize defaults */ active_bank = 0; __use_guid = false; previous_bank = INT_MAX; + vendor_file = NULL; do { c = getopt_long(argc, argv, opts_short, options, NULL); @@ -305,6 +446,12 @@ int main(int argc, char *argv[]) case 'a': active_bank = strtoul(optarg, NULL, 0); break; + case 'v': + version = strtoul(optarg, NULL, 0); + break; + case 'V': + vendor_file = optarg; + break; } } while (c != -1); @@ -313,6 +460,17 @@ int main(int argc, char *argv[]) return -EINVAL; } + if (!version || !supported_mdata_version(version)) { + fprintf(stderr, "Error: Version value can only be either 1 or 2, not %ld.\n", + version); + return -EINVAL; + } + + if (version == 1 && vendor_file) { + fprintf(stderr, "Error: Vendor Data can only be appended in version 2 of FWU Metadata.\n"); + return -EINVAL; + } + /* This command takes UUIDs * images and output file. */ if (optind + images + 1 != argc) { fprintf(stderr, "Error: UUID list or output file is not specified or too much.\n"); @@ -325,7 +483,8 @@ int main(int argc, char *argv[]) previous_bank = active_bank > 0 ? active_bank - 1 : banks - 1; } - ret = fwu_make_mdata(images, banks, argv + optind, argv[argc - 1]); + ret = fwu_make_mdata(images, banks, (u8)version, vendor_file, + argv + optind, argv[argc - 1]); if (ret < 0) fprintf(stderr, "Error: Failed to parse and write image: %s\n", strerror(-ret)); diff --git a/tools/mxsimage.c b/tools/mxsimage.c index ead61d0cd63..42df0698ca2 100644 --- a/tools/mxsimage.c +++ b/tools/mxsimage.c @@ -2058,7 +2058,6 @@ static int sb_verify_image_end(struct sb_image_ctx *ictx, return ret; } - static int sb_build_tree_from_img(struct sb_image_ctx *ictx) { long filesize; diff --git a/tools/patman/func_test.py b/tools/patman/func_test.py index e3918497cf4..af6c025a441 100644 --- a/tools/patman/func_test.py +++ b/tools/patman/func_test.py @@ -211,6 +211,7 @@ class TestFunctional(unittest.TestCase): 'u-boot': ['u-boot@lists.denx.de'], 'simon': [self.leb], 'fred': [self.fred], + 'joe': [self.joe], } text = self._get_text('test01.txt') @@ -259,6 +260,7 @@ class TestFunctional(unittest.TestCase): self.assertEqual('Postfix:\t some-branch', next(lines)) self.assertEqual('Cover: 4 lines', next(lines)) self.assertEqual(' Cc: %s' % self.fred, next(lines)) + self.assertEqual(' Cc: %s' % self.joe, next(lines)) self.assertEqual(' Cc: %s' % self.leb, next(lines)) self.assertEqual(' Cc: %s' % mel, next(lines)) @@ -272,7 +274,8 @@ class TestFunctional(unittest.TestCase): self.assertEqual(('%s %s\0%s' % (args[0], rick, stefan)), cc_lines[0]) self.assertEqual( - '%s %s\0%s\0%s\0%s' % (args[1], self.fred, self.leb, rick, stefan), + '%s %s\0%s\0%s\0%s\0%s' % (args[1], self.fred, self.joe, self.leb, + rick, stefan), cc_lines[1]) expected = ''' @@ -290,6 +293,7 @@ Changes in v4: change - Some changes - Some notes for the cover letter +- fdt: Correct cast for sandbox in fdtdec_setup_mem_size_base() Simon Glass (2): pci: Correct cast for sandbox @@ -339,6 +343,7 @@ Changes in v4: - Multi line change +- New - Some changes Changes in v2: @@ -540,7 +545,8 @@ complicated as possible''') with open('.patman', 'w', buffering=1) as f: f.write('[settings]\n' 'get_maintainer_script: dummy-script.sh\n' - 'check_patch: False\n') + 'check_patch: False\n' + 'add_maintainers: True\n') with open('dummy-script.sh', 'w', buffering=1) as f: f.write('#!/usr/bin/env python\n' 'print("hello@there.com")\n') diff --git a/tools/patman/patchstream.py b/tools/patman/patchstream.py index e2e2a83e677..a09ae9c7371 100644 --- a/tools/patman/patchstream.py +++ b/tools/patman/patchstream.py @@ -475,6 +475,13 @@ class PatchStream: elif name == 'changes': self.in_change = 'Commit' self.change_version = self._parse_version(value, line) + elif name == 'cc': + self.commit.add_cc(value.split(',')) + elif name == 'added-in': + version = self._parse_version(value, line) + self.commit.add_change(version, '- New') + self.series.AddChange(version, None, '- %s' % + self.commit.subject) else: self._add_warn('Line %d: Ignoring Commit-%s' % (self.linenum, name)) diff --git a/tools/patman/patman.rst b/tools/patman/patman.rst index f4588c00fc1..63b95a6b161 100644 --- a/tools/patman/patman.rst +++ b/tools/patman/patman.rst @@ -350,7 +350,20 @@ Cover-changes: n - This line will only appear in the cover letter <blank line> -Patch-cc: Their Name <email> +Commit-added-in: n + Add a change noting the version this commit was added in. This is + equivalent to:: + + Commit-changes: n + - New + + Cover-changes: n + - <commit subject> + + It is a convenient shorthand for suppressing the '(no changes in vN)' + message. + +Patch-cc / Commit-cc: Their Name <email> This copies a single patch to another email address. Note that the Cc: used by git send-email is ignored by patman, but will be interpreted by git send-email if you use it. diff --git a/tools/patman/settings.py b/tools/patman/settings.py index 636983e32da..68c93e313b3 100644 --- a/tools/patman/settings.py +++ b/tools/patman/settings.py @@ -59,25 +59,25 @@ class _ProjectConfigParser(ConfigParser.ConfigParser): # Check to make sure that bogus project gets general alias. >>> config = _ProjectConfigParser("zzz") - >>> config.readfp(StringIO(sample_config)) + >>> config.read_file(StringIO(sample_config)) >>> str(config.get("alias", "enemies")) 'Evil <evil@example.com>' # Check to make sure that alias gets overridden by project. >>> config = _ProjectConfigParser("sm") - >>> config.readfp(StringIO(sample_config)) + >>> config.read_file(StringIO(sample_config)) >>> str(config.get("alias", "enemies")) 'Green G. <ugly@example.com>' # Check to make sure that settings get merged with project. >>> config = _ProjectConfigParser("linux") - >>> config.readfp(StringIO(sample_config)) + >>> config.read_file(StringIO(sample_config)) >>> sorted((str(a), str(b)) for (a, b) in config.items("settings")) [('am_hero', 'True'), ('check_patch_use_tree', 'True'), ('process_tags', 'False')] # Check to make sure that settings works with unknown project. >>> config = _ProjectConfigParser("unknown") - >>> config.readfp(StringIO(sample_config)) + >>> config.read_file(StringIO(sample_config)) >>> sorted((str(a), str(b)) for (a, b) in config.items("settings")) [('am_hero', 'True')] """ diff --git a/tools/patman/setup.py b/tools/patman/setup.py index 2ff791da0f7..bcaad69a1c2 100644 --- a/tools/patman/setup.py +++ b/tools/patman/setup.py @@ -3,7 +3,6 @@ from setuptools import setup setup(name='patman', version='1.0', - license='GPL-2.0+', scripts=['patman'], packages=['patman'], package_dir={'patman': ''}, diff --git a/tools/patman/test/0002-fdt-Correct-cast-for-sandbox-in-fdtdec_setup_mem_siz.patch b/tools/patman/test/0002-fdt-Correct-cast-for-sandbox-in-fdtdec_setup_mem_siz.patch index 56278a6ce9b..48ea1793b47 100644 --- a/tools/patman/test/0002-fdt-Correct-cast-for-sandbox-in-fdtdec_setup_mem_siz.patch +++ b/tools/patman/test/0002-fdt-Correct-cast-for-sandbox-in-fdtdec_setup_mem_siz.patch @@ -21,7 +21,9 @@ Series-cc: Stefan Brüns <stefan.bruens@rwth-aachen.de> Cover-letter-cc: Lord Mëlchett <clergy@palace.gov> Series-version: 3 Patch-cc: fred +Commit-cc: joe Series-process-log: sort, uniq +Commit-added-in: 4 Series-changes: 4 - Some changes - Multi diff --git a/tools/patman/test/test01.txt b/tools/patman/test/test01.txt index fc3066e50b4..b2d73c5972c 100644 --- a/tools/patman/test/test01.txt +++ b/tools/patman/test/test01.txt @@ -49,7 +49,9 @@ Date: Sat Apr 15 15:39:08 2017 -0600 Cover-letter-cc: Lord Mëlchett <clergy@palace.gov> Series-version: 3 Patch-cc: fred + Commit-cc: joe Series-process-log: sort, uniq + Commit-added-in: 4 Series-changes: 4 - Some changes - Multi diff --git a/tools/qconfig.py b/tools/qconfig.py index 04118d942da..7b868c7d72c 100755 --- a/tools/qconfig.py +++ b/tools/qconfig.py @@ -1,13 +1,12 @@ #!/usr/bin/env python3 # SPDX-License-Identifier: GPL-2.0+ -# -# Author: Masahiro Yamada <yamada.masahiro@socionext.com> -# -""" -Build and query a Kconfig database for boards. +"""Build and query a Kconfig database for boards. + +See doc/develop/qconfig.rst for documentation. -See doc/develop/moveconfig.rst for documentation. +Author: Masahiro Yamada <yamada.masahiro@socionext.com> +Author: Simon Glass <sjg@chromium.org> """ from argparse import ArgumentParser @@ -29,7 +28,6 @@ import threading import time import unittest -import asteval from buildman import bsettings from buildman import kconfiglib from buildman import toolchain @@ -216,43 +214,29 @@ def read_file(fname, as_lines=True, skip_unicode=False): print(f"Failed on file '{fname}: {exc}") return None -def try_expand(line): - """If value looks like an expression, try expanding it - Otherwise just return the existing value - """ - if line.find('=') == -1: - return line - - try: - aeval = asteval.Interpreter( usersyms=SIZES, minimal=True ) - cfg, val = re.split("=", line) - val= val.strip('\"') - if re.search(r'[*+-/]|<<|SZ_+|\(([^\)]+)\)', val): - newval = hex(aeval(val)) - print(f'\tExpanded expression {val} to {newval}') - return cfg+'='+newval - except: - print(f'\tFailed to expand expression in {line}') - - return line - ### classes ### class Progress: - """Progress Indicator""" def __init__(self, col, total): """Create a new progress indicator. Args: - color_enabled (bool): True for colour output + col (terminal.Color): Colour-output class total (int): A number of defconfig files to process. + + current (int): Number of boards processed so far + failed (int): Number of failed boards + failure_msg (str): Message indicating number of failures, '' if none """ self.col = col + self.total = total + self.current = 0 self.good = 0 - self.total = total + self.failed = None + self.failure_msg = None def inc(self, success): """Increment the number of processed defconfig files. @@ -274,22 +258,28 @@ class Progress: print(f'{line} \r', end='') sys.stdout.flush() + def completed(self): + """Set up extra properties when completed""" + self.failed = self.total - self.good + self.failure_msg = f'{self.failed} failed, ' if self.failed else '' -class KconfigScanner: - """Kconfig scanner.""" - def __init__(self): - """Scan all the Kconfig files and create a Config object.""" - # Define environment variables referenced from Kconfig - os.environ['srctree'] = os.getcwd() - os.environ['UBOOTVERSION'] = 'dummy' - os.environ['KCONFIG_OBJDIR'] = '' - os.environ['CC'] = 'gcc' - self.conf = kconfiglib.Kconfig() +def scan_kconfig(): + """Scan all the Kconfig files and create a Config object + Returns: + Kconfig object + """ + # Define environment variables referenced from Kconfig + os.environ['srctree'] = os.getcwd() + os.environ['UBOOTVERSION'] = 'dummy' + os.environ['KCONFIG_OBJDIR'] = '' + os.environ['CC'] = 'gcc' + return kconfiglib.Kconfig() -class KconfigParser: +# pylint: disable=R0903 +class KconfigParser: """A parser of .config and include/autoconf.mk.""" re_arch = re.compile(r'CONFIG_SYS_ARCH="(.*)"') @@ -374,7 +364,7 @@ class Slot: """ def __init__(self, toolchains, args, progress, devnull, make_cmd, - reference_src_dir, db_queue, col): + reference_src_dir, db_queue): """Create a new process slot. Args: @@ -396,7 +386,7 @@ class Slot: self.make_cmd = (make_cmd, 'O=' + self.build_dir) self.reference_src_dir = reference_src_dir self.db_queue = db_queue - self.col = col + self.col = progress.col self.parser = KconfigParser(args, self.build_dir) self.state = STATE_IDLE self.failed_boards = set() @@ -501,6 +491,7 @@ class Slot: cmd = list(self.make_cmd) cmd.append(self.defconfig) + # pylint: disable=R1732 self.proc = subprocess.Popen(cmd, stdout=self.devnull, stderr=subprocess.PIPE, cwd=self.current_src_dir) @@ -523,6 +514,7 @@ class Slot: cmd = list(self.make_cmd) cmd.append('KCONFIG_IGNORE_DUPLICATES=1') cmd.append(AUTO_CONF_PATH) + # pylint: disable=R1732 self.proc = subprocess.Popen(cmd, stdout=self.devnull, env=env, stderr=subprocess.PIPE, cwd=self.current_src_dir) @@ -546,6 +538,7 @@ class Slot: cmd = list(self.make_cmd) cmd.append('savedefconfig') + # pylint: disable=R1732 self.proc = subprocess.Popen(cmd, stdout=self.devnull, stderr=subprocess.PIPE) self.state = STATE_SAVEDEFCONFIG @@ -601,11 +594,9 @@ class Slot: return self.failed_boards class Slots: - """Controller of the array of subprocess slots.""" - def __init__(self, toolchains, args, progress, reference_src_dir, db_queue, - col): + def __init__(self, toolchains, args, progress, reference_src_dir, db_queue): """Create a new slots controller. Args: @@ -615,17 +606,16 @@ class Slots: reference_src_dir (str): Determine the true starting config state from this source tree (None for none) db_queue (Queue): output queue to write config info for the database - col (terminal.Color): Colour object """ self.args = args self.slots = [] self.progress = progress - self.col = col + self.col = progress.col devnull = subprocess.DEVNULL make_cmd = get_make_cmd() for _ in range(args.jobs): self.slots.append(Slot(toolchains, args, progress, devnull, - make_cmd, reference_src_dir, db_queue, col)) + make_cmd, reference_src_dir, db_queue)) def add(self, defconfig): """Add a new subprocess if a vacant slot is found. @@ -711,18 +701,35 @@ class ReferenceSource: return self.src_dir -def move_config(toolchains, args, db_queue, col): +def move_config(args): """Build database or sync config options to defconfig files. Args: - toolchains (Toolchains): Toolchains to use args (Namespace): Program arguments - db_queue (Queue): Queue for database updates - col (terminal.Color): Colour object Returns: - Progress: Progress indicator + tuple: + config_db (dict of configs for each defconfig): + key: defconfig name, e.g. "MPC8548CDS_legacy_defconfig" + value: dict: + key: CONFIG option + value: Value of option + Progress: Progress indicator """ + config_db = {} + db_queue = queue.Queue() + dbt = DatabaseThread(config_db, db_queue) + dbt.daemon = True + dbt.start() + + check_clean_directory() + bsettings.setup('') + + # Get toolchains to use + toolchains = toolchain.Toolchains() + toolchains.GetSettings() + toolchains.Scan(verbose=False) + if args.git_ref: reference_src = ReferenceSource(args.git_ref) reference_src_dir = reference_src.get_dir() @@ -734,8 +741,10 @@ def move_config(toolchains, args, db_queue, col): else: defconfigs = get_all_defconfigs() + col = terminal.Color(terminal.COLOR_NEVER if args.nocolour + else terminal.COLOR_IF_TERMINAL) progress = Progress(col, len(defconfigs)) - slots = Slots(toolchains, args, progress, reference_src_dir, db_queue, col) + slots = Slots(toolchains, args, progress, reference_src_dir, db_queue) # Main loop to process defconfig files: # Add a new subprocess into a vacant slot. @@ -751,7 +760,9 @@ def move_config(toolchains, args, db_queue, col): time.sleep(SLEEP_TIME) slots.write_failed_boards() - return progress + db_queue.join() + progress.completed() + return config_db, progress def find_kconfig_rules(kconf, config, imply_config): """Check whether a config has a 'select' or 'imply' keyword @@ -772,7 +783,7 @@ def find_kconfig_rules(kconf, config, imply_config): return sym return None -def check_imply_rule(kconf, config, imply_config): +def check_imply_rule(kconf, imply_config): """Check if we can add an 'imply' option This finds imply_config in the Kconfig and looks to see if it is possible @@ -780,7 +791,6 @@ def check_imply_rule(kconf, config, imply_config): Args: kconf (Kconfiglib.Kconfig): Kconfig object - config (str): Name of config to check (without CONFIG_ prefix) imply_config (str): Implying config (without CONFIG_ prefix) which may or may not have an 'imply' for 'config') @@ -873,6 +883,7 @@ def read_database(): all_defconfigs = set() defconfig_db = collections.defaultdict(set) + defconfig = None for line in read_file(CONFIG_DATABASE): line = line.rstrip() if not line: # Separator between defconfigs @@ -914,15 +925,15 @@ def do_imply_config(config_list, add_imply, imply_flags, skip_added, - If imply_defconfigs contains anything not in defconfigs then this config does not imply the target config - Params: - config_list: List of CONFIG options to check (each a string) - add_imply: Automatically add an 'imply' for each config. - imply_flags: Flags which control which implying configs are allowed + Args: + config_list (list of str): List of CONFIG options to check + add_imply (bool): Automatically add an 'imply' for each config. + imply_flags (int): Flags which control which implying configs are allowed (IMPLY_...) - skip_added: Don't show options which already have an imply added. - check_kconfig: Check if implied symbols already have an 'imply' or + skip_added (bool): Don't show options which already have an imply added. + check_kconfig (bool): Check if implied symbols already have an 'imply' or 'select' for the target config, and show this information if so. - find_superset: True to look for configs which are a superset of those + find_superset (bool): True to look for configs which are a superset of those already found. So for example if CONFIG_EXYNOS5 implies an option, but CONFIG_EXYNOS covers a larger set of defconfigs and also implies that option, this will drop the former in favour of the @@ -932,7 +943,7 @@ def do_imply_config(config_list, add_imply, imply_flags, skip_added, config - a CONFIG_XXX options (a string, e.g. 'CONFIG_CMD_EEPROM') defconfig - a defconfig file (a string, e.g. 'configs/snow_defconfig') """ - kconf = KconfigScanner().conf if check_kconfig else None + kconf = scan_kconfig() if check_kconfig else None if add_imply and add_imply != 'all': add_imply = add_imply.split(',') @@ -1051,13 +1062,13 @@ def do_imply_config(config_list, add_imply, imply_flags, skip_added, if add_imply and (add_imply == 'all' or iconfig in add_imply): fname, linenum, kconfig_info = (check_imply_rule(kconf, - config[CONFIG_LEN:], iconfig[CONFIG_LEN:])) + iconfig[CONFIG_LEN:])) if fname: add_list[fname].append(linenum) if show and kconfig_info != 'skip': - print(f'{num_common:5d} : ' - f'{iconfig.ljust(30):-30s}{kconfig_info:-25s} {missing_str}') + print(f'{num_common:5} : ' + f'{iconfig.ljust(30)}{kconfig_info.ljust(25)} {missing_str}') # Having collected a list of things to add, now we add them. We process # each file from the largest line number to the smallest so that @@ -1068,7 +1079,7 @@ def do_imply_config(config_list, add_imply, imply_flags, skip_added, for linenum in sorted(linenums, reverse=True): add_imply_rule(config[CONFIG_LEN:], fname, linenum) -def defconfig_matches(configs, re_match): +def defconfig_matches(configs, re_match, re_val): """Check if any CONFIG option matches a regex The match must be complete, i.e. from the start to end of the CONFIG option. @@ -1078,23 +1089,30 @@ def defconfig_matches(configs, re_match): key: CONFIG option value: Value of option re_match (re.Pattern): Match to check + re_val (re.Pattern): Regular expression to check against value (or None) Returns: bool: True if any CONFIG matches the regex """ - for cfg in configs: + for cfg, val in configs.items(): if re_match.fullmatch(cfg): - return True + if not re_val or re_val.fullmatch(val): + return True return False -def do_find_config(config_list): +def do_find_config(config_list, list_format): """Find boards with a given combination of CONFIGs - Params: - config_list: List of CONFIG options to check (each a regex consisting - of a config option, with or without a CONFIG_ prefix. If an option - is preceded by a tilde (~) then it must be false, otherwise it must - be true) + Args: + config_list (list of str): List of CONFIG options to check (each a regex + consisting of a config option, with or without a CONFIG_ prefix. If + an option is preceded by a tilde (~) then it must be false, + otherwise it must be true) + list_format (bool): True to write in 'list' format, one board name per + line + + Returns: + int: exit code (0 for success) """ _, all_defconfigs, config_db, _ = read_database() @@ -1109,6 +1127,11 @@ def do_find_config(config_list): if cfg[0] == '~': want = False cfg = cfg[1:] + val = None + re_val = None + if '=' in cfg: + cfg, val = cfg.split('=', maxsplit=1) + re_val = re.compile(val) # Search everything that is still in the running. If it has a config # that we want, or doesn't have one that we don't, add it into the @@ -1117,11 +1140,14 @@ def do_find_config(config_list): out = set() re_match = re.compile(cfg) for defc in in_list: - has_cfg = defconfig_matches(config_db[defc], re_match) + has_cfg = defconfig_matches(config_db[defc], re_match, re_val) if has_cfg == want: out.add(defc) - print(f'{len(out)} matches') - print(' '.join(item.split('_defconfig')[0] for item in out)) + if not list_format: + print(f'{len(out)} matches') + sep = '\n' if list_format else ' ' + print(sep.join(item.split('_defconfig')[0] for item in sorted(list(out)))) + return 0 def prefix_config(cfg): @@ -1155,7 +1181,16 @@ RE_C_CONFIGS = re.compile(r'CONFIG_([A-Za-z0-9_]*)') RE_CONFIG_IS = re.compile(r'CONFIG_IS_ENABLED\(([A-Za-z0-9_]*)\)') class ConfigUse: + """Tracks whether a config relates to SPL or not""" def __init__(self, cfg, is_spl, fname, rest): + """Set up a new ConfigUse + + Args: + cfg (str): CONFIG option, without any CONFIG_ or SPL_ prefix + is_spl (bool): True if this option relates to SPL + fname (str): Makefile filename where the CONFIG option was found + rest (str): Line of the Makefile + """ self.cfg = cfg self.is_spl = is_spl self.fname = fname @@ -1362,7 +1397,7 @@ def do_scan_source(path, do_update): print('Scanning Kconfig') - kconf = KconfigScanner().conf + kconf = scan_kconfig() print(f'Scanning source in {path}') args = ['git', 'grep', '-E', r'IS_ENABLED|\bCONFIG'] with subprocess.Popen(args, stdout=subprocess.PIPE) as proc: @@ -1460,9 +1495,17 @@ def do_scan_source(path, do_update): file=out) for item in sorted(proper_not_found): print(item, file=out) + return 0 -def main(): +def parse_args(): + """Parse the program arguments + + Returns: + tuple: + argparse.ArgumentParser: parser + argparse.Namespace: Parsed arguments + """ try: cpu_count = multiprocessing.cpu_count() except NotImplementedError: @@ -1496,6 +1539,8 @@ doc/develop/moveconfig.rst for documentation.''' help='Find boards with a given config combination') parser.add_argument('-i', '--imply', action='store_true', default=False, help='find options which imply others') + parser.add_argument('-l', '--list', action='store_true', default=False, + help='Show a sorted list of board names, one per line') parser.add_argument('-I', '--imply-flags', type=str, default='', help="control the -i option ('help' for help") parser.add_argument('-j', '--jobs', type=int, default=cpu_count, @@ -1521,103 +1566,144 @@ doc/develop/moveconfig.rst for documentation.''' parser.add_argument('configs', nargs='*') args = parser.parse_args() + if not any((args.force_sync, args.build_db, args.imply, args.find, + args.scan_source, args.test)): + parser.print_usage() + sys.exit(1) - if args.test: - sys.argv = [sys.argv[0]] - fail, _ = doctest.testmod() - if fail: - return 1 - unittest.main() + return parser, args - col = terminal.Color(terminal.COLOR_NEVER if args.nocolour - else terminal.COLOR_IF_TERMINAL) - if args.scan_source: - do_scan_source(os.getcwd(), args.update) - return 0 +def imply(args): + """Handle checking for flags which imply others - if not any((args.force_sync, args.build_db, args.imply, args.find)): - parser.print_usage() - sys.exit(1) + Args: + args (argparse.Namespace): Program arguments + + Returns: + int: exit code (0 for success) + """ + imply_flags = 0 + if args.imply_flags == 'all': + imply_flags = -1 + + elif args.imply_flags: + for flag in args.imply_flags.split(','): + bad = flag not in IMPLY_FLAGS + if bad: + print(f"Invalid flag '{flag}'") + if flag == 'help' or bad: + print("Imply flags: (separate with ',')") + for name, info in IMPLY_FLAGS.items(): + print(f' {name:-15s}: {info[1]}') + return 1 + imply_flags |= IMPLY_FLAGS[flag][0] + + do_imply_config(args.configs, args.add_imply, imply_flags, args.skip_added) + return 0 + + +def add_commit(configs): + """Add a commit indicating which CONFIG options were converted + + Args: + configs (list of str) List of CONFIG_... options to process + """ + subprocess.call(['git', 'add', '-u']) + if configs: + part = 'et al ' if len(configs) > 1 else '' + msg = f'Convert {configs[0]} {part}to Kconfig' + msg += ('\n\nThis converts the following to Kconfig:\n %s\n' % + '\n '.join(configs)) + else: + msg = 'configs: Resync with savedefconfig' + msg += '\n\nRsync all defconfig files using moveconfig.py' + subprocess.call(['git', 'commit', '-s', '-m', msg]) + + +def write_db(config_db, progress): + """Write the database to a file + + Args: + config_db (dict of dict): configs for each defconfig + key: defconfig name, e.g. "MPC8548CDS_legacy_defconfig" + value: dict: + key: CONFIG option + value: Value of option + progress (Progress): Progress indicator. + + Returns: + int: exit code (0 for success) + """ + col = progress.col + with open(CONFIG_DATABASE, 'w', encoding='utf-8') as outf: + for defconfig, configs in config_db.items(): + outf.write(f'{defconfig}\n') + for config in sorted(configs.keys()): + outf.write(f' {config}={configs[config]}\n') + outf.write('\n') + print(col.build( + col.RED if progress.failed else col.GREEN, + f'{progress.failure_msg}{len(config_db)} boards written to {CONFIG_DATABASE}')) + return 0 + + +def move_done(progress): + """Write a message indicating that the move is done + + Args: + progress (Progress): Progress indicator. + + Returns: + int: exit code (0 for success) + """ + col = progress.col + if progress.failed: + print(col.build(col.RED, f'{progress.failure_msg}see {FAILED_LIST}', True)) + else: + # Add enough spaces to overwrite the progress indicator + print(col.build( + col.GREEN, f'{progress.total} processed ', bright=True)) + return 0 + +def do_tests(): + """Run doctests and unit tests (so far there are no unit tests)""" + sys.argv = [sys.argv[0]] + fail, _ = doctest.testmod() + if fail: + return 1 + unittest.main() + return 0 - # prefix the option name with CONFIG_ if missing - configs = [prefix_config(cfg) for cfg in args.configs] +def main(): + """Main program""" + parser, args = parse_args() check_top_directory() + # prefix the option name with CONFIG_ if missing + args.configs = [prefix_config(cfg) for cfg in args.configs] + + if args.test: + return do_tests() + if args.scan_source: + return do_scan_source(os.getcwd(), args.update) if args.imply: - imply_flags = 0 - if args.imply_flags == 'all': - imply_flags = -1 - - elif args.imply_flags: - for flag in args.imply_flags.split(','): - bad = flag not in IMPLY_FLAGS - if bad: - print(f"Invalid flag '{flag}'") - if flag == 'help' or bad: - print("Imply flags: (separate with ',')") - for name, info in IMPLY_FLAGS.items(): - print(f' {name:-15s}: {info[1]}') - parser.print_usage() - sys.exit(1) - imply_flags |= IMPLY_FLAGS[flag][0] - - do_imply_config(configs, args.add_imply, imply_flags, args.skip_added) + if imply(args): + parser.print_usage() + sys.exit(1) return 0 - if args.find: - do_find_config(configs) - return 0 + return do_find_config(args.configs, args.list) - # We are either building the database or forcing a sync of defconfigs - config_db = {} - db_queue = queue.Queue() - dbt = DatabaseThread(config_db, db_queue) - dbt.daemon = True - dbt.start() - - check_clean_directory() - bsettings.setup('') - toolchains = toolchain.Toolchains() - toolchains.GetSettings() - toolchains.Scan(verbose=False) - progress = move_config(toolchains, args, db_queue, col) - db_queue.join() + config_db, progress = move_config(args) if args.commit: - subprocess.call(['git', 'add', '-u']) - if configs: - msg = 'Convert %s %sto Kconfig' % (configs[0], - 'et al ' if len(configs) > 1 else '') - msg += ('\n\nThis converts the following to Kconfig:\n %s\n' % - '\n '.join(configs)) - else: - msg = 'configs: Resync with savedefconfig' - msg += '\n\nRsync all defconfig files using moveconfig.py' - subprocess.call(['git', 'commit', '-s', '-m', msg]) + add_commit(args.configs) - failed = progress.total - progress.good - failure = f'{failed} failed, ' if failed else '' if args.build_db: - with open(CONFIG_DATABASE, 'w', encoding='utf-8') as outf: - for defconfig, configs in config_db.items(): - outf.write(f'{defconfig}\n') - for config in sorted(configs.keys()): - outf.write(f' {config}={configs[config]}\n') - outf.write('\n') - print(col.build( - col.RED if failed else col.GREEN, - f'{failure}{len(config_db)} boards written to {CONFIG_DATABASE}')) - else: - if failed: - print(col.build(col.RED, f'{failure}see {FAILED_LIST}', True)) - else: - # Add enough spaces to overwrite the progress indicator - print(col.build( - col.GREEN, f'{progress.total} processed ', bright=True)) - - return 0 + return write_db(config_db, progress) + return move_done(progress) if __name__ == '__main__': diff --git a/tools/u_boot_pylib/terminal.py b/tools/u_boot_pylib/terminal.py index 40d79f8ac07..2cd5a54ab52 100644 --- a/tools/u_boot_pylib/terminal.py +++ b/tools/u_boot_pylib/terminal.py @@ -164,8 +164,11 @@ def print_clear(): global last_print_len if last_print_len: - print('\r%s\r' % (' '* last_print_len), end='', flush=True) - last_print_len = None + if print_test_mode: + print_test_list.append(PrintLine(None, None, None, None)) + else: + print('\r%s\r' % (' '* last_print_len), end='', flush=True) + last_print_len = None def set_print_test_mode(enable=True): """Go into test mode, where all printing is recorded""" diff --git a/tools/u_boot_pylib/test_util.py b/tools/u_boot_pylib/test_util.py index f18d385d995..857ce58c98c 100644 --- a/tools/u_boot_pylib/test_util.py +++ b/tools/u_boot_pylib/test_util.py @@ -60,12 +60,17 @@ def run_test_coverage(prog, filter_fname, exclude_list, build_dir, required=None prefix = '' if build_dir: prefix = 'PYTHONPATH=$PYTHONPATH:%s/sandbox_spl/tools ' % build_dir - cmd = ('%spython3-coverage run ' - '--omit "%s" %s %s %s %s' % (prefix, ','.join(glob_list), + + # Detect a Python virtualenv and use 'coverage' instead + covtool = ('python3-coverage' if sys.prefix == sys.base_prefix else + 'coverage') + + cmd = ('%s%s run ' + '--omit "%s" %s %s %s %s' % (prefix, covtool, ','.join(glob_list), prog, extra_args or '', test_cmd, single_thread or '-P1')) os.system(cmd) - stdout = command.output('python3-coverage', 'report') + stdout = command.output(covtool, 'report') lines = stdout.splitlines() if required: # Convert '/path/to/name.py' just the module name 'name' diff --git a/tools/update-subtree.sh b/tools/update-subtree.sh new file mode 100755 index 00000000000..536b3318573 --- /dev/null +++ b/tools/update-subtree.sh @@ -0,0 +1,85 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0+ +# +# Copyright (c) 2024 Linaro Limited +# +# Usage: from the top level U-Boot source tree, run: +# $ ./tools/update-subtree.sh pull <subtree-name> <release-tag> +# Or: +# $ ./tools/update-subtree.sh pick <subtree-name> <commit-id> +# +# The script will pull changes from subtree repo into U-Boot. +# It will automatically create a squash/merge commit listing the commits +# imported. + +set -e + +print_usage() { + echo "usage: $0 <op> <subtree-name> <ref>" + echo " <op> pull or pick" + echo " <subtree-name> mbedtls or dts or lwip" + echo " <ref> release tag [pull] or commit id [pick]" +} + +if [ $# -ne 3 ]; then + print_usage + exit 1 +fi + +op=$1 +subtree_name=$2 +ref=$3 + +set_params() { + case "$subtree_name" in + mbedtls) + path=lib/mbedtls/external/mbedtls + repo_url=https://github.com/Mbed-TLS/mbedtls.git + remote_name="mbedtls_upstream" + ;; + dts) + path=dts/upstream + repo_url=https://git.kernel.org/pub/scm/linux/kernel/git/devicetree/devicetree-rebasing.git + remote_name="devicetree-rebasing" + ;; + lwip) + path=lib/lwip/lwip + repo_url=https://git.savannah.gnu.org/git/lwip.git + remote_name="lwip_upstream" + ;; + *) + echo "Invalid subtree name: $subtree_name" + print_usage + exit 1 + esac +} + +set_params + +merge_commit_msg=$(cat << EOF +Subtree merge tag '$ref' of $subtree_name repo [1] into $path + +[1] $repo_url +EOF +) + +remote_add_and_fetch() { + if [ -z "$(git remote get-url $remote_name 2>/dev/null)" ]; then + echo "Warning: Script automatically adds new git remote via:" + echo " git remote add $remote_name \\" + echo " $repo_url" + git remote add $remote_name $repo_url + fi + git fetch $remote_name master +} + +if [ "$op" = "pull" ]; then + remote_add_and_fetch + git subtree pull --prefix $path $remote_name "$ref" --squash -m "$merge_commit_msg" +elif [ "$op" = "pick" ]; then + remote_add_and_fetch + git cherry-pick -x --strategy=subtree -Xsubtree=$path/ "$ref" +else + print_usage + exit 1 +fi |