diff options
Diffstat (limited to 'tools')
34 files changed, 670 insertions, 213 deletions
diff --git a/tools/binman/binman.rst b/tools/binman/binman.rst index 230e055667f..872e9746c8c 100644 --- a/tools/binman/binman.rst +++ b/tools/binman/binman.rst @@ -711,6 +711,13 @@ missing-msg: information about what needs to be fixed. See missing-blob-help for the message for each tag. +assume-size: + Sets the assumed size of a blob entry if it is missing. This allows for a + check that the rest of the image fits into the available space, even when + the contents are not available. If the entry is missing, Binman will use + this assumed size for the entry size, including creating a fake file of that + size if requested. + no-expanded: By default binman substitutes entries with expanded versions if available, so that a `u-boot` entry type turns into `u-boot-expanded`, for example. The diff --git a/tools/binman/entries.rst b/tools/binman/entries.rst index 254afe76074..bdda1ef2855 100644 --- a/tools/binman/entries.rst +++ b/tools/binman/entries.rst @@ -470,11 +470,11 @@ updating the EC on startup via software sync. .. _etype_efi_capsule: -Entry: capsule: Entry for generating EFI Capsule files ------------------------------------------------------- +Entry: efi-capsule: Generate EFI capsules +----------------------------------------- -The parameters needed for generation of the capsules can be provided -as properties in the entry. +The parameters needed for generation of the capsules can +be provided as properties in the entry. Properties / Entry arguments: - image-index: Unique number for identifying corresponding @@ -495,9 +495,9 @@ Properties / Entry arguments: file. Mandatory property for generating signed capsules. - oem-flags - OEM flags to be passed through capsule header. - Since this is a subclass of Entry_section, all properties of the parent - class also apply here. Except for the properties stated as mandatory, the - rest of the properties are optional. +Since this is a subclass of Entry_section, all properties of the parent +class also apply here. Except for the properties stated as mandatory, the +rest of the properties are optional. For more details on the description of the capsule format, and the capsule update functionality, refer Section 8.5 and Chapter 23 in the `UEFI @@ -510,17 +510,17 @@ provided as a subnode of the capsule entry. A typical capsule entry node would then look something like this:: capsule { - type = "efi-capsule"; - image-index = <0x1>; - /* Image GUID for testing capsule update */ - image-guid = SANDBOX_UBOOT_IMAGE_GUID; - hardware-instance = <0x0>; - private-key = "path/to/the/private/key"; - public-key-cert = "path/to/the/public-key-cert"; - oem-flags = <0x8000>; + type = "efi-capsule"; + image-index = <0x1>; + /* Image GUID for testing capsule update */ + image-guid = SANDBOX_UBOOT_IMAGE_GUID; + hardware-instance = <0x0>; + private-key = "path/to/the/private/key"; + public-key-cert = "path/to/the/public-key-cert"; + oem-flags = <0x8000>; - u-boot { - }; + u-boot { + }; }; In the above example, the capsule payload is the U-Boot image. The @@ -534,8 +534,8 @@ payload using the blob-ext subnode. .. _etype_efi_empty_capsule: -Entry: efi-empty-capsule: Entry for generating EFI Empty Capsule files ----------------------------------------------------------------------- +Entry: efi-empty-capsule: Generate EFI empty capsules +----------------------------------------------------- The parameters needed for generation of the empty capsules can be provided as properties in the entry. @@ -551,22 +551,22 @@ update functionality, refer Section 8.5 and Chapter 23 in the `UEFI specification`_. For more information on the empty capsule, refer the sections 2.3.2 and 2.3.3 in the `Dependable Boot specification`_. -A typical accept empty capsule entry node would then look something -like this:: +A typical accept empty capsule entry node would then look something like +this:: empty-capsule { - type = "efi-empty-capsule"; - /* GUID of the image being accepted */ - image-type-id = SANDBOX_UBOOT_IMAGE_GUID; - capsule-type = "accept"; + type = "efi-empty-capsule"; + /* GUID of image being accepted */ + image-type-id = SANDBOX_UBOOT_IMAGE_GUID; + capsule-type = "accept"; }; -A typical revert empty capsule entry node would then look something -like this:: +A typical revert empty capsule entry node would then look something like +this:: empty-capsule { - type = "efi-empty-capsule"; - capsule-type = "revert"; + type = "efi-empty-capsule"; + capsule-type = "revert"; }; The empty capsules do not have any input payload image. @@ -1521,6 +1521,28 @@ byte. +.. _etype_nxp_imx8mcst: + +Entry: nxp-imx8mcst: NXP i.MX8M CST .cfg file generator and cst invoker +----------------------------------------------------------------------- + +Properties / Entry arguments: + - nxp,loader-address - loader address (SPL text base) + + + +.. _etype_nxp_imx8mimage: + +Entry: nxp-imx8mimage: NXP i.MX8M imx8mimage .cfg file generator and mkimage invoker +------------------------------------------------------------------------------------ + +Properties / Entry arguments: + - nxp,boot-from - device to boot from (e.g. 'sd') + - nxp,loader-address - loader address (SPL text base) + - nxp,rom-version - BootROM version ('2' for i.MX8M Nano and Plus) + + + .. _etype_opensbi: Entry: opensbi: RISC-V OpenSBI fw_dynamic blob @@ -1929,6 +1951,12 @@ Properties / Entry arguments: - content: List of phandles to entries to sign - keyfile: Filename of file containing key to sign binary with - sha: Hash function to be used for signing + - auth-in-place: This is an integer field that contains two pieces + of information: + + - Lower Byte - Remains 0x02 as per our use case + ( 0x02: Move the authenticated binary back to the header ) + - Upper Byte - The Host ID of the core owning the firewall Output files: - input.<unique_name> - input file passed to openssl @@ -1937,6 +1965,35 @@ Output files: - cert.<unique_name> - output file generated by openssl (which is used as the entry contents) +Depending on auth-in-place information in the inputs, we read the +firewall nodes that describe the configurations of firewall that TIFS +will be doing after reading the certificate. + +The syntax of the firewall nodes are as such:: + + firewall-257-0 { + id = <257>; /* The ID of the firewall being configured */ + region = <0>; /* Region number to configure */ + + control = /* The control register */ + <(FWCTRL_EN | FWCTRL_LOCK | FWCTRL_BG | FWCTRL_CACHE)>; + + permissions = /* The permission registers */ + <((FWPRIVID_ALL << FWPRIVID_SHIFT) | + FWPERM_SECURE_PRIV_RWCD | + FWPERM_SECURE_USER_RWCD | + FWPERM_NON_SECURE_PRIV_RWCD | + FWPERM_NON_SECURE_USER_RWCD)>; + + /* More defines can be found in k3-security.h */ + + start_address = /* The Start Address of the firewall */ + <0x0 0x0>; + end_address = /* The End Address of the firewall */ + <0xff 0xffffffff>; + }; + + openssl signs the provided data, using the TI templated config file and writes the signature in this entry. This allows verification that the data is genuine. diff --git a/tools/binman/entry.py b/tools/binman/entry.py index 42e0b7b9145..219d5dcecab 100644 --- a/tools/binman/entry.py +++ b/tools/binman/entry.py @@ -315,6 +315,7 @@ class Entry(object): self.overlap = fdt_util.GetBool(self._node, 'overlap') if self.overlap: self.required_props += ['offset', 'size'] + self.assume_size = fdt_util.GetInt(self._node, 'assume-size', 0) # This is only supported by blobs and sections at present self.compress = fdt_util.GetString(self._node, 'compress', 'none') @@ -812,7 +813,7 @@ class Entry(object): as missing """ print('''Binman Entry Documentation -=========================== +========================== This file describes the entry types supported by binman. These entry types can be placed in an image one by one to build up a final firmware image. It is diff --git a/tools/binman/entry_test.py b/tools/binman/entry_test.py index ac6582cf86a..40d74d401a2 100644 --- a/tools/binman/entry_test.py +++ b/tools/binman/entry_test.py @@ -103,7 +103,7 @@ class TestEntry(unittest.TestCase): ent = entry.Entry.Create(None, self.GetNode(), 'missing', missing_etype=True) self.assertTrue(isinstance(ent, Entry_blob)) - self.assertEquals('missing', ent.etype) + self.assertEqual('missing', ent.etype) def testDecompressData(self): """Test the DecompressData() method of the base class""" @@ -111,8 +111,8 @@ class TestEntry(unittest.TestCase): base.compress = 'lz4' bintools = {} base.comp_bintool = base.AddBintool(bintools, '_testing') - self.assertEquals(tools.get_bytes(0, 1024), base.CompressData(b'abc')) - self.assertEquals(tools.get_bytes(0, 1024), base.DecompressData(b'abc')) + self.assertEqual(tools.get_bytes(0, 1024), base.CompressData(b'abc')) + self.assertEqual(tools.get_bytes(0, 1024), base.DecompressData(b'abc')) def testLookupOffset(self): """Test the lookup_offset() method of the base class""" diff --git a/tools/binman/etype/blob.py b/tools/binman/etype/blob.py index 064fae50365..041e1122953 100644 --- a/tools/binman/etype/blob.py +++ b/tools/binman/etype/blob.py @@ -48,11 +48,16 @@ class Entry_blob(Entry): self.external and (self.optional or self.section.GetAllowMissing())) # Allow the file to be missing if not self._pathname: + if not fake_size and self.assume_size: + fake_size = self.assume_size self._pathname, faked = self.check_fake_fname(self._filename, fake_size) self.missing = True if not faked: - self.SetContents(b'') + content_size = 0 + if self.assume_size: # Ensure we get test coverage on next line + content_size = self.assume_size + self.SetContents(tools.get_bytes(0, content_size)) return True self.ReadBlobContents() diff --git a/tools/binman/etype/efi_capsule.py b/tools/binman/etype/efi_capsule.py index e3203717822..751f654bf31 100644 --- a/tools/binman/etype/efi_capsule.py +++ b/tools/binman/etype/efi_capsule.py @@ -36,23 +36,23 @@ class Entry_efi_capsule(Entry_section): be provided as properties in the entry. Properties / Entry arguments: - - image-index: Unique number for identifying corresponding - payload image. Number between 1 and descriptor count, i.e. - the total number of firmware images that can be updated. Mandatory - property. - - image-guid: Image GUID which will be used for identifying the - updatable image on the board. Mandatory property. - - hardware-instance: Optional number for identifying unique - hardware instance of a device in the system. Default value of 0 - for images where value is not to be used. - - fw-version: Value of image version that can be put on the capsule - through the Firmware Management Protocol(FMP) header. - - monotonic-count: Count used when signing an image. - - private-key: Path to PEM formatted .key private key file. Mandatory - property for generating signed capsules. - - public-key-cert: Path to PEM formatted .crt public key certificate - file. Mandatory property for generating signed capsules. - - oem-flags - OEM flags to be passed through capsule header. + - image-index: Unique number for identifying corresponding + payload image. Number between 1 and descriptor count, i.e. + the total number of firmware images that can be updated. Mandatory + property. + - image-guid: Image GUID which will be used for identifying the + updatable image on the board. Mandatory property. + - hardware-instance: Optional number for identifying unique + hardware instance of a device in the system. Default value of 0 + for images where value is not to be used. + - fw-version: Value of image version that can be put on the capsule + through the Firmware Management Protocol(FMP) header. + - monotonic-count: Count used when signing an image. + - private-key: Path to PEM formatted .key private key file. Mandatory + property for generating signed capsules. + - public-key-cert: Path to PEM formatted .crt public key certificate + file. Mandatory property for generating signed capsules. + - oem-flags - OEM flags to be passed through capsule header. Since this is a subclass of Entry_section, all properties of the parent class also apply here. Except for the properties stated as mandatory, the @@ -66,9 +66,9 @@ class Entry_efi_capsule(Entry_section): properties in the entry. The payload to be used in the capsule is to be provided as a subnode of the capsule entry. - A typical capsule entry node would then look something like this + A typical capsule entry node would then look something like this:: - capsule { + capsule { type = "efi-capsule"; image-index = <0x1>; /* Image GUID for testing capsule update */ @@ -80,7 +80,7 @@ class Entry_efi_capsule(Entry_section): u-boot { }; - }; + }; In the above example, the capsule payload is the U-Boot image. The capsule entry would read the contents of the payload and put them diff --git a/tools/binman/etype/efi_empty_capsule.py b/tools/binman/etype/efi_empty_capsule.py index 064bf9a77f0..1d99fbfb3bb 100644 --- a/tools/binman/etype/efi_empty_capsule.py +++ b/tools/binman/etype/efi_empty_capsule.py @@ -19,31 +19,33 @@ class Entry_efi_empty_capsule(Entry_section): be provided as properties in the entry. Properties / Entry arguments: - - image-guid: Image GUID which will be used for identifying the - updatable image on the board. Mandatory for accept capsule. - - capsule-type - String to indicate type of capsule to generate. Valid - values are 'accept' and 'revert'. + - image-guid: Image GUID which will be used for identifying the + updatable image on the board. Mandatory for accept capsule. + - capsule-type - String to indicate type of capsule to generate. Valid + values are 'accept' and 'revert'. For more details on the description of the capsule format, and the capsule update functionality, refer Section 8.5 and Chapter 23 in the `UEFI specification`_. For more information on the empty capsule, refer the sections 2.3.2 and 2.3.3 in the `Dependable Boot specification`_. - A typical accept empty capsule entry node would then look something like this + A typical accept empty capsule entry node would then look something like + this:: - empty-capsule { + empty-capsule { type = "efi-empty-capsule"; /* GUID of image being accepted */ image-type-id = SANDBOX_UBOOT_IMAGE_GUID; capsule-type = "accept"; - }; + }; - A typical revert empty capsule entry node would then look something like this + A typical revert empty capsule entry node would then look something like + this:: - empty-capsule { + empty-capsule { type = "efi-empty-capsule"; capsule-type = "revert"; - }; + }; The empty capsules do not have any input payload image. diff --git a/tools/binman/etype/intel_descriptor.py b/tools/binman/etype/intel_descriptor.py index 7fe88a9ec1a..3ce967fe81a 100644 --- a/tools/binman/etype/intel_descriptor.py +++ b/tools/binman/etype/intel_descriptor.py @@ -59,7 +59,7 @@ class Entry_intel_descriptor(Entry_blob_ext): if self.missing: # Return zero offsets so that these entries get placed somewhere if self.HasSibling('intel-me'): - info['intel-me'] = [0, None] + info['intel-me'] = [0x1000, None] return info offset = self.data.find(FD_SIGNATURE) if offset == -1: diff --git a/tools/binman/etype/ti_secure.py b/tools/binman/etype/ti_secure.py index 704dcf8a381..420ee263e4f 100644 --- a/tools/binman/etype/ti_secure.py +++ b/tools/binman/etype/ti_secure.py @@ -53,10 +53,11 @@ class Entry_ti_secure(Entry_x509_cert): - keyfile: Filename of file containing key to sign binary with - sha: Hash function to be used for signing - auth-in-place: This is an integer field that contains two pieces - of information - Lower Byte - Remains 0x02 as per our use case - ( 0x02: Move the authenticated binary back to the header ) - Upper Byte - The Host ID of the core owning the firewall + of information: + + - Lower Byte - Remains 0x02 as per our use case + ( 0x02: Move the authenticated binary back to the header ) + - Upper Byte - The Host ID of the core owning the firewall Output files: - input.<unique_name> - input file passed to openssl @@ -69,29 +70,29 @@ class Entry_ti_secure(Entry_x509_cert): firewall nodes that describe the configurations of firewall that TIFS will be doing after reading the certificate. - The syntax of the firewall nodes are as such: + The syntax of the firewall nodes are as such:: - firewall-257-0 { - id = <257>; /* The ID of the firewall being configured */ - region = <0>; /* Region number to configure */ + firewall-257-0 { + id = <257>; /* The ID of the firewall being configured */ + region = <0>; /* Region number to configure */ - control = /* The control register */ - <(FWCTRL_EN | FWCTRL_LOCK | FWCTRL_BG | FWCTRL_CACHE)>; + control = /* The control register */ + <(FWCTRL_EN | FWCTRL_LOCK | FWCTRL_BG | FWCTRL_CACHE)>; - permissions = /* The permission registers */ - <((FWPRIVID_ALL << FWPRIVID_SHIFT) | - FWPERM_SECURE_PRIV_RWCD | - FWPERM_SECURE_USER_RWCD | - FWPERM_NON_SECURE_PRIV_RWCD | - FWPERM_NON_SECURE_USER_RWCD)>; + permissions = /* The permission registers */ + <((FWPRIVID_ALL << FWPRIVID_SHIFT) | + FWPERM_SECURE_PRIV_RWCD | + FWPERM_SECURE_USER_RWCD | + FWPERM_NON_SECURE_PRIV_RWCD | + FWPERM_NON_SECURE_USER_RWCD)>; - /* More defines can be found in k3-security.h */ + /* More defines can be found in k3-security.h */ - start_address = /* The Start Address of the firewall */ - <0x0 0x0>; - end_address = /* The End Address of the firewall */ - <0xff 0xffffffff>; - }; + start_address = /* The Start Address of the firewall */ + <0x0 0x0>; + end_address = /* The End Address of the firewall */ + <0xff 0xffffffff>; + }; openssl signs the provided data, using the TI templated config file and diff --git a/tools/binman/fdt_test.py b/tools/binman/fdt_test.py index 7ef87295463..564c1770820 100644 --- a/tools/binman/fdt_test.py +++ b/tools/binman/fdt_test.py @@ -44,43 +44,43 @@ class TestFdt(unittest.TestCase): fname = self.GetCompiled('045_prop_test.dts') dt = FdtScan(fname) node = dt.GetNode('/binman/intel-me') - self.assertEquals('intel-me', node.name) + self.assertEqual('intel-me', node.name) val = fdt_util.GetString(node, 'filename') - self.assertEquals(str, type(val)) - self.assertEquals('me.bin', val) + self.assertEqual(str, type(val)) + self.assertEqual('me.bin', val) prop = node.props['intval'] - self.assertEquals(fdt.Type.INT, prop.type) - self.assertEquals(3, fdt_util.GetInt(node, 'intval')) + self.assertEqual(fdt.Type.INT, prop.type) + self.assertEqual(3, fdt_util.GetInt(node, 'intval')) prop = node.props['intarray'] - self.assertEquals(fdt.Type.INT, prop.type) - self.assertEquals(list, type(prop.value)) - self.assertEquals(2, len(prop.value)) - self.assertEquals([5, 6], + self.assertEqual(fdt.Type.INT, prop.type) + self.assertEqual(list, type(prop.value)) + self.assertEqual(2, len(prop.value)) + self.assertEqual([5, 6], [fdt_util.fdt32_to_cpu(val) for val in prop.value]) prop = node.props['byteval'] - self.assertEquals(fdt.Type.BYTE, prop.type) - self.assertEquals(chr(8), prop.value) + self.assertEqual(fdt.Type.BYTE, prop.type) + self.assertEqual(chr(8), prop.value) prop = node.props['bytearray'] - self.assertEquals(fdt.Type.BYTE, prop.type) - self.assertEquals(list, type(prop.value)) - self.assertEquals(str, type(prop.value[0])) - self.assertEquals(3, len(prop.value)) - self.assertEquals([chr(1), '#', '4'], prop.value) + self.assertEqual(fdt.Type.BYTE, prop.type) + self.assertEqual(list, type(prop.value)) + self.assertEqual(str, type(prop.value[0])) + self.assertEqual(3, len(prop.value)) + self.assertEqual([chr(1), '#', '4'], prop.value) prop = node.props['longbytearray'] - self.assertEquals(fdt.Type.INT, prop.type) - self.assertEquals(0x090a0b0c, fdt_util.GetInt(node, 'longbytearray')) + self.assertEqual(fdt.Type.INT, prop.type) + self.assertEqual(0x090a0b0c, fdt_util.GetInt(node, 'longbytearray')) prop = node.props['stringval'] - self.assertEquals(fdt.Type.STRING, prop.type) - self.assertEquals('message2', fdt_util.GetString(node, 'stringval')) + self.assertEqual(fdt.Type.STRING, prop.type) + self.assertEqual('message2', fdt_util.GetString(node, 'stringval')) prop = node.props['stringarray'] - self.assertEquals(fdt.Type.STRING, prop.type) - self.assertEquals(list, type(prop.value)) - self.assertEquals(3, len(prop.value)) - self.assertEquals(['another', 'multi-word', 'message'], prop.value) + self.assertEqual(fdt.Type.STRING, prop.type) + self.assertEqual(list, type(prop.value)) + self.assertEqual(3, len(prop.value)) + self.assertEqual(['another', 'multi-word', 'message'], prop.value) diff --git a/tools/binman/ftest.py b/tools/binman/ftest.py index 8a44bc051b3..e4da04030a5 100644 --- a/tools/binman/ftest.py +++ b/tools/binman/ftest.py @@ -2095,7 +2095,7 @@ class TestFunctional(unittest.TestCase): dtb.Scan() props = self._GetPropTree(dtb, ['size', 'uncomp-size']) orig = self._decompress(data) - self.assertEquals(COMPRESS_DATA, orig) + self.assertEqual(COMPRESS_DATA, orig) # Do a sanity check on various fields image = control.images['image'] @@ -2809,9 +2809,9 @@ class TestFunctional(unittest.TestCase): orig_entry = orig_image.GetEntries()['fdtmap'] entry = image.GetEntries()['fdtmap'] - self.assertEquals(orig_entry.offset, entry.offset) - self.assertEquals(orig_entry.size, entry.size) - self.assertEquals(orig_entry.image_pos, entry.image_pos) + self.assertEqual(orig_entry.offset, entry.offset) + self.assertEqual(orig_entry.size, entry.size) + self.assertEqual(orig_entry.image_pos, entry.image_pos) def testReadImageNoHeader(self): """Test accessing an image's FDT map without an image header""" @@ -3895,7 +3895,7 @@ class TestFunctional(unittest.TestCase): mat = re_line.match(line) vals[mat.group(1)].append(mat.group(2)) - self.assertEquals('FIT description: test-desc', lines[0]) + self.assertEqual('FIT description: test-desc', lines[0]) self.assertIn('Created:', lines[1]) self.assertIn('Image 0 (kernel)', vals) self.assertIn('Hash value', vals) @@ -4012,7 +4012,7 @@ class TestFunctional(unittest.TestCase): fit_pos, fdt_util.fdt32_to_cpu(fnode.props['data-position'].value)) - self.assertEquals(expected_size, len(data)) + self.assertEqual(expected_size, len(data)) actual_pos = len(U_BOOT_DATA) + fit_pos self.assertEqual(U_BOOT_DATA + b'aa', data[actual_pos:actual_pos + external_data_size]) @@ -4431,7 +4431,7 @@ class TestFunctional(unittest.TestCase): props = self._GetPropTree(dtb, ['offset', 'image-pos', 'size', 'uncomp-size']) orig = self._decompress(data) - self.assertEquals(COMPRESS_DATA + U_BOOT_DATA, orig) + self.assertEqual(COMPRESS_DATA + U_BOOT_DATA, orig) # Do a sanity check on various fields image = control.images['image'] @@ -4475,7 +4475,7 @@ class TestFunctional(unittest.TestCase): 'uncomp-size']) orig = self._decompress(data) - self.assertEquals(COMPRESS_DATA + COMPRESS_DATA + U_BOOT_DATA, orig) + self.assertEqual(COMPRESS_DATA + COMPRESS_DATA + U_BOOT_DATA, orig) # Do a sanity check on various fields image = control.images['image'] @@ -4519,7 +4519,7 @@ class TestFunctional(unittest.TestCase): props = self._GetPropTree(dtb, ['offset', 'image-pos', 'size', 'uncomp-size']) orig = self._decompress(data) - self.assertEquals(COMPRESS_DATA + U_BOOT_DATA, orig) + self.assertEqual(COMPRESS_DATA + U_BOOT_DATA, orig) expected = { 'section/blob:offset': 0, 'section/blob:size': len(COMPRESS_DATA), @@ -4545,7 +4545,7 @@ class TestFunctional(unittest.TestCase): props = self._GetPropTree(dtb, ['offset', 'image-pos', 'size', 'uncomp-size']) orig = self._decompress(data) - self.assertEquals(COMPRESS_DATA + U_BOOT_DATA, orig) + self.assertEqual(COMPRESS_DATA + U_BOOT_DATA, orig) expected = { 'section/blob:offset': 0, 'section/blob:size': len(COMPRESS_DATA), @@ -4580,7 +4580,7 @@ class TestFunctional(unittest.TestCase): 'uncomp-size']) base = data[len(U_BOOT_DATA):] - self.assertEquals(U_BOOT_DATA, base[:len(U_BOOT_DATA)]) + self.assertEqual(U_BOOT_DATA, base[:len(U_BOOT_DATA)]) rest = base[len(U_BOOT_DATA):] # Check compressed data @@ -4588,22 +4588,22 @@ class TestFunctional(unittest.TestCase): expect1 = bintool.compress(COMPRESS_DATA + U_BOOT_DATA) data1 = rest[:len(expect1)] section1 = self._decompress(data1) - self.assertEquals(expect1, data1) - self.assertEquals(COMPRESS_DATA + U_BOOT_DATA, section1) + self.assertEqual(expect1, data1) + self.assertEqual(COMPRESS_DATA + U_BOOT_DATA, section1) rest1 = rest[len(expect1):] expect2 = bintool.compress(COMPRESS_DATA + COMPRESS_DATA) data2 = rest1[:len(expect2)] section2 = self._decompress(data2) - self.assertEquals(expect2, data2) - self.assertEquals(COMPRESS_DATA + COMPRESS_DATA, section2) + self.assertEqual(expect2, data2) + self.assertEqual(COMPRESS_DATA + COMPRESS_DATA, section2) rest2 = rest1[len(expect2):] expect_size = (len(U_BOOT_DATA) + len(U_BOOT_DATA) + len(expect1) + len(expect2) + len(U_BOOT_DATA)) - #self.assertEquals(expect_size, len(data)) + #self.assertEqual(expect_size, len(data)) - #self.assertEquals(U_BOOT_DATA, rest2) + #self.assertEqual(U_BOOT_DATA, rest2) self.maxDiff = None expected = { @@ -4695,7 +4695,7 @@ class TestFunctional(unittest.TestCase): u_boot = image.GetEntries()['section'].GetEntries()['u-boot'] - self.assertEquals(U_BOOT_DATA, u_boot.ReadData()) + self.assertEqual(U_BOOT_DATA, u_boot.ReadData()) def testTplNoDtb(self): """Test that an image with tpl/u-boot-tpl-nodtb.bin can be created""" @@ -5526,7 +5526,7 @@ fdt fdtmap Extract the devicetree blob from the fdtmap segments, entry = elf.read_loadable_segments(elf_data) # We assume there are two segments - self.assertEquals(2, len(segments)) + self.assertEqual(2, len(segments)) atf1 = dtb.GetNode('/images/atf-1') _, start, data = segments[0] @@ -6107,7 +6107,7 @@ fdt fdtmap Extract the devicetree blob from the fdtmap data = bintool.compress(COMPRESS_DATA) self.assertNotEqual(COMPRESS_DATA, data) orig = bintool.decompress(data) - self.assertEquals(COMPRESS_DATA, orig) + self.assertEqual(COMPRESS_DATA, orig) def testCompUtilVersions(self): """Test tool version of compression algorithms""" @@ -6125,7 +6125,7 @@ fdt fdtmap Extract the devicetree blob from the fdtmap self.assertNotEqual(COMPRESS_DATA, data) data += tools.get_bytes(0, 64) orig = bintool.decompress(data) - self.assertEquals(COMPRESS_DATA, orig) + self.assertEqual(COMPRESS_DATA, orig) def testCompressDtbZstd(self): """Test that zstd compress of device-tree files failed""" @@ -7460,5 +7460,33 @@ fdt fdtmap Extract the devicetree blob from the fdtmap with self.assertRaises(ValueError) as e: self._DoReadFile('323_capsule_accept_revert_missing.dts') + def test_assume_size(self): + """Test handling of the assume-size property for external blob""" + with self.assertRaises(ValueError) as e: + self._DoTestFile('326_assume_size.dts', allow_missing=True, + allow_fake_blobs=True) + self.assertIn("contents size 0xa (10) exceeds section size 0x9 (9)", + str(e.exception)) + + def test_assume_size_ok(self): + """Test handling of the assume-size where it fits OK""" + with test_util.capture_sys_output() as (stdout, stderr): + self._DoTestFile('327_assume_size_ok.dts', allow_missing=True, + allow_fake_blobs=True) + err = stderr.getvalue() + self.assertRegex( + err, + "Image '.*' has faked external blobs and is non-functional: .*") + + def test_assume_size_no_fake(self): + """Test handling of the assume-size where it fits OK""" + with test_util.capture_sys_output() as (stdout, stderr): + self._DoTestFile('327_assume_size_ok.dts', allow_missing=True) + err = stderr.getvalue() + self.assertRegex( + err, + "Image '.*' is missing external blobs and is non-functional: .*") + + if __name__ == "__main__": unittest.main() diff --git a/tools/binman/test/326_assume_size.dts b/tools/binman/test/326_assume_size.dts new file mode 100644 index 00000000000..4c5f8b418d8 --- /dev/null +++ b/tools/binman/test/326_assume_size.dts @@ -0,0 +1,16 @@ +// SPDX-License-Identifier: GPL-2.0+ + +/dts-v1/; + +/ { + #address-cells = <1>; + #size-cells = <1>; + + binman { + size = <9>; + blob-ext { + filename = "assume_blob"; + assume-size = <10>; + }; + }; +}; diff --git a/tools/binman/test/327_assume_size_ok.dts b/tools/binman/test/327_assume_size_ok.dts new file mode 100644 index 00000000000..00ed726f872 --- /dev/null +++ b/tools/binman/test/327_assume_size_ok.dts @@ -0,0 +1,16 @@ +// SPDX-License-Identifier: GPL-2.0+ + +/dts-v1/; + +/ { + #address-cells = <1>; + #size-cells = <1>; + + binman { + size = <10>; + blob-ext { + filename = "assume_blob"; + assume-size = <10>; + }; + }; +}; diff --git a/tools/buildman/bsettings.py b/tools/buildman/bsettings.py index e225ac2ca0f..aea724fc559 100644 --- a/tools/buildman/bsettings.py +++ b/tools/buildman/bsettings.py @@ -29,7 +29,7 @@ def setup(fname=''): settings.read(config_fname) def add_file(data): - settings.readfp(io.StringIO(data)) + settings.read_file(io.StringIO(data)) def get_items(section): """Get the items from a section of the config. diff --git a/tools/buildman/builder.py b/tools/buildman/builder.py index f35175b4598..c4384f53e8d 100644 --- a/tools/buildman/builder.py +++ b/tools/buildman/builder.py @@ -256,14 +256,14 @@ class Builder: def __init__(self, toolchains, base_dir, git_dir, num_threads, num_jobs, gnu_make='make', checkout=True, show_unknown=True, step=1, no_subdirs=False, full_path=False, verbose_build=False, - mrproper=False, per_board_out_dir=False, - config_only=False, squash_config_y=False, - warnings_as_errors=False, work_in_output=False, - test_thread_exceptions=False, adjust_cfg=None, - allow_missing=False, no_lto=False, reproducible_builds=False, - force_build=False, force_build_failures=False, - force_reconfig=False, in_tree=False, - force_config_on_failure=False, make_func=None): + mrproper=False, fallback_mrproper=False, + per_board_out_dir=False, config_only=False, + squash_config_y=False, warnings_as_errors=False, + work_in_output=False, test_thread_exceptions=False, + adjust_cfg=None, allow_missing=False, no_lto=False, + reproducible_builds=False, force_build=False, + force_build_failures=False, force_reconfig=False, + in_tree=False, force_config_on_failure=False, make_func=None): """Create a new Builder object Args: @@ -283,6 +283,7 @@ class Builder: PATH verbose_build: Run build with V=1 and don't use 'make -s' mrproper: Always run 'make mrproper' when configuring + fallback_mrproper: Run 'make mrproper' and retry on build failure per_board_out_dir: Build in a separate persistent directory per board rather than a thread-specific directory config_only: Only configure each build, don't build it @@ -352,6 +353,7 @@ class Builder: self.force_reconfig = force_reconfig self.in_tree = in_tree self.force_config_on_failure = force_config_on_failure + self.fallback_mrproper = fallback_mrproper if not self.squash_config_y: self.config_filenames += EXTRA_CONFIG_FILENAMES diff --git a/tools/buildman/builderthread.py b/tools/buildman/builderthread.py index a8599c0bb2a..bbe2f6f0d24 100644 --- a/tools/buildman/builderthread.py +++ b/tools/buildman/builderthread.py @@ -240,7 +240,7 @@ class BuilderThread(threading.Thread): return args, cwd, src_dir def _reconfigure(self, commit, brd, cwd, args, env, config_args, config_out, - cmd_list): + cmd_list, mrproper): """Reconfigure the build Args: @@ -251,11 +251,12 @@ class BuilderThread(threading.Thread): env (dict): Environment strings config_args (list of str): defconfig arg for this board cmd_list (list of str): List to add the commands to, for logging + mrproper (bool): True to run mrproper first Returns: CommandResult object """ - if self.mrproper: + if mrproper: result = self.make(commit, brd, 'mrproper', cwd, 'mrproper', *args, env=env) config_out.write(result.combined) @@ -380,7 +381,7 @@ class BuilderThread(threading.Thread): commit = 'current' return commit - def _config_and_build(self, commit_upto, brd, work_dir, do_config, + def _config_and_build(self, commit_upto, brd, work_dir, do_config, mrproper, config_only, adjust_cfg, commit, out_dir, out_rel_dir, result): """Do the build, configuring first if necessary @@ -390,6 +391,7 @@ class BuilderThread(threading.Thread): brd (Board): Board to create arguments for work_dir (str): Directory to which the source will be checked out do_config (bool): True to run a make <board>_defconfig on the source + mrproper (bool): True to run mrproper first config_only (bool): Only configure the source, do not build it adjust_cfg (list of str): See the cfgutil module and run_commit() commit (Commit): Commit only being built @@ -419,7 +421,8 @@ class BuilderThread(threading.Thread): cmd_list = [] if do_config or adjust_cfg: result = self._reconfigure( - commit, brd, cwd, args, env, config_args, config_out, cmd_list) + commit, brd, cwd, args, env, config_args, config_out, cmd_list, + mrproper) do_config = False # No need to configure next time if adjust_cfg: cfgutil.adjust_cfg_file(cfg_file, adjust_cfg) @@ -445,9 +448,9 @@ class BuilderThread(threading.Thread): result.cmd_list = cmd_list return result, do_config - def run_commit(self, commit_upto, brd, work_dir, do_config, config_only, - force_build, force_build_failures, work_in_output, - adjust_cfg): + def run_commit(self, commit_upto, brd, work_dir, do_config, mrproper, + config_only, force_build, force_build_failures, + work_in_output, adjust_cfg): """Build a particular commit. If the build is already done, and we are not forcing a build, we skip @@ -458,6 +461,7 @@ class BuilderThread(threading.Thread): brd (Board): Board to build work_dir (str): Directory to which the source will be checked out do_config (bool): True to run a make <board>_defconfig on the source + mrproper (bool): True to run mrproper first config_only (bool): Only configure the source, do not build it force_build (bool): Force a build even if one was previously done force_build_failures (bool): Force a bulid if the previous result @@ -498,8 +502,9 @@ class BuilderThread(threading.Thread): if self.toolchain: commit = self._checkout(commit_upto, work_dir) result, do_config = self._config_and_build( - commit_upto, brd, work_dir, do_config, config_only, - adjust_cfg, commit, out_dir, out_rel_dir, result) + commit_upto, brd, work_dir, do_config, mrproper, + config_only, adjust_cfg, commit, out_dir, out_rel_dir, + result) result.already_done = False result.toolchain = self.toolchain @@ -688,19 +693,22 @@ class BuilderThread(threading.Thread): force_build = False for commit_upto in range(0, len(job.commits), job.step): result, request_config = self.run_commit(commit_upto, brd, - work_dir, do_config, self.builder.config_only, + work_dir, do_config, self.mrproper, + self.builder.config_only, force_build or self.builder.force_build, self.builder.force_build_failures, job.work_in_output, job.adjust_cfg) failed = result.return_code or result.stderr did_config = do_config - if failed and not do_config: + if failed and not do_config and not self.mrproper: # If our incremental build failed, try building again # with a reconfig. if self.builder.force_config_on_failure: result, request_config = self.run_commit(commit_upto, - brd, work_dir, True, False, True, False, - job.work_in_output, job.adjust_cfg) + brd, work_dir, True, + self.mrproper or self.builder.fallback_mrproper, + False, True, False, job.work_in_output, + job.adjust_cfg) did_config = True if not self.builder.force_reconfig: do_config = request_config @@ -744,7 +752,7 @@ class BuilderThread(threading.Thread): else: # Just build the currently checked-out build result, request_config = self.run_commit(None, brd, work_dir, True, - self.builder.config_only, True, + self.mrproper, self.builder.config_only, True, self.builder.force_build_failures, job.work_in_output, job.adjust_cfg) result.commit_upto = 0 diff --git a/tools/buildman/buildman.rst b/tools/buildman/buildman.rst index aae2477b5c3..b8ff3bf1ab2 100644 --- a/tools/buildman/buildman.rst +++ b/tools/buildman/buildman.rst @@ -995,7 +995,8 @@ By default, buildman doesn't execute 'make mrproper' prior to building the first commit for each board. This reduces the amount of work 'make' does, and hence speeds up the build. To force use of 'make mrproper', use -the -m flag. This flag will slow down any buildman invocation, since it increases the amount -of work done on any build. +of work done on any build. An alternative is to use the --fallback-mrproper +flag, which retries the build with 'make mrproper' only after a build failure. One possible application of buildman is as part of a continual edit, build, edit, build, ... cycle; repeatedly applying buildman to the same change or @@ -1285,6 +1286,11 @@ then buildman hangs. Failing to handle any eventuality is a bug in buildman and should be reported. But you can use -T0 to disable threading and hopefully figure out the root cause of the build failure. +For situations where buildman is invoked from multiple running processes, it is +sometimes useful to have buildman wait until the others have finished. Use the +--process-limit option for this: --process-limit 1 will allow only one buildman +to process jobs at a time. + Build summary ------------- diff --git a/tools/buildman/cmdline.py b/tools/buildman/cmdline.py index 03211bd5aa5..544a391a464 100644 --- a/tools/buildman/cmdline.py +++ b/tools/buildman/cmdline.py @@ -90,7 +90,9 @@ def add_upto_m(parser): parser.add_argument('--list-tool-chains', action='store_true', default=False, help='List available tool chains (use -v to see probing detail)') parser.add_argument('-m', '--mrproper', action='store_true', - default=False, help="Run 'make mrproper before reconfiguring") + default=False, help="Run 'make mrproper' before reconfiguring") + parser.add_argument('--fallback-mrproper', action='store_true', + default=False, help="Run 'make mrproper' and retry on build failure") parser.add_argument( '-M', '--allow-missing', action='store_true', default=False, help='Tell binman to allow missing blobs and generate fake ones as needed') @@ -127,6 +129,8 @@ def add_after_m(parser): default=False, help="Use an O= (output) directory per board rather than per thread") parser.add_argument('--print-arch', action='store_true', default=False, help="Print the architecture for a board (ARCH=)") + parser.add_argument('--process-limit', type=int, + default=0, help='Limit to number of buildmans running at once') parser.add_argument('-r', '--reproducible-builds', action='store_true', help='Set SOURCE_DATE_EPOCH=0 to suuport a reproducible build') parser.add_argument('-R', '--regen-board-list', type=str, diff --git a/tools/buildman/control.py b/tools/buildman/control.py index 8f6850c5211..464835c5be5 100644 --- a/tools/buildman/control.py +++ b/tools/buildman/control.py @@ -7,10 +7,13 @@ This holds the main control logic for buildman, when not running tests. """ +import getpass import multiprocessing import os import shutil import sys +import tempfile +import time from buildman import boards from buildman import bsettings @@ -21,10 +24,23 @@ from patman import gitutil from patman import patchstream from u_boot_pylib import command from u_boot_pylib import terminal -from u_boot_pylib.terminal import tprint +from u_boot_pylib import tools +from u_boot_pylib.terminal import print_clear, tprint TEST_BUILDER = None +# Space-separated list of buildman process IDs currently running jobs +RUNNING_FNAME = f'buildmanq.{getpass.getuser()}' + +# Lock file for access to RUNNING_FILE +LOCK_FNAME = f'{RUNNING_FNAME}.lock' + +# Wait time for access to lock (seconds) +LOCK_WAIT_S = 10 + +# Wait time to start running +RUN_WAIT_S = 300 + def get_plural(count): """Returns a plural 's' if count is not 1""" return 's' if count != 1 else '' @@ -578,6 +594,125 @@ def calc_adjust_cfg(adjust_cfg, reproducible_builds): return adjust_cfg +def read_procs(tmpdir=tempfile.gettempdir()): + """Read the list of running buildman processes + + If the list is corrupted, returns an empty list + + Args: + tmpdir (str): Temporary directory to use (for testing only) + """ + running_fname = os.path.join(tmpdir, RUNNING_FNAME) + procs = [] + if os.path.exists(running_fname): + items = tools.read_file(running_fname, binary=False).split() + try: + procs = [int(x) for x in items] + except ValueError: # Handle invalid format + pass + return procs + + +def check_pid(pid): + """Check for existence of a unix PID + + https://stackoverflow.com/questions/568271/how-to-check-if-there-exists-a-process-with-a-given-pid-in-python + + Args: + pid (int): PID to check + + Returns: + True if it exists, else False + """ + try: + os.kill(pid, 0) + except OSError: + return False + else: + return True + + +def write_procs(procs, tmpdir=tempfile.gettempdir()): + """Write the list of running buildman processes + + Args: + tmpdir (str): Temporary directory to use (for testing only) + """ + running_fname = os.path.join(tmpdir, RUNNING_FNAME) + tools.write_file(running_fname, ' '.join([str(p) for p in procs]), + binary=False) + + # Allow another user to access the file + os.chmod(running_fname, 0o666) + +def wait_for_process_limit(limit, tmpdir=tempfile.gettempdir(), + pid=os.getpid()): + """Wait until the number of buildman processes drops to the limit + + This uses FileLock to protect a 'running' file, which contains a list of + PIDs of running buildman processes. The number of PIDs in the file indicates + the number of running processes. + + When buildman starts up, it calls this function to wait until it is OK to + start the build. + + On exit, no attempt is made to remove the PID from the file, since other + buildman processes will notice that the PID is no-longer valid, and ignore + it. + + Two timeouts are provided: + LOCK_WAIT_S: length of time to wait for the lock; if this occurs, the + lock is busted / removed before trying again + RUN_WAIT_S: length of time to wait to be allowed to run; if this occurs, + the build starts, with the PID being added to the file. + + Args: + limit (int): Maximum number of buildman processes, including this one; + must be > 0 + tmpdir (str): Temporary directory to use (for testing only) + pid (int): Current process ID (for testing only) + """ + from filelock import Timeout, FileLock + + running_fname = os.path.join(tmpdir, RUNNING_FNAME) + lock_fname = os.path.join(tmpdir, LOCK_FNAME) + lock = FileLock(lock_fname) + + # Allow another user to access the file + col = terminal.Color() + tprint('Waiting for other buildman processes...', newline=False, + colour=col.RED) + + claimed = False + deadline = time.time() + RUN_WAIT_S + while True: + try: + with lock.acquire(timeout=LOCK_WAIT_S): + os.chmod(lock_fname, 0o666) + procs = read_procs(tmpdir) + + # Drop PIDs which are not running + procs = list(filter(check_pid, procs)) + + # If we haven't hit the limit, add ourself + if len(procs) < limit: + tprint('done...', newline=False) + claimed = True + if time.time() >= deadline: + tprint('timeout...', newline=False) + claimed = True + if claimed: + write_procs(procs + [pid], tmpdir) + break + + except Timeout: + tprint('failed to get lock: busting...', newline=False) + os.remove(lock_fname) + + time.sleep(1) + tprint('starting build', newline=False) + print_clear() + def do_buildman(args, toolchains=None, make_func=None, brds=None, clean_dir=False, test_thread_exceptions=False): """The main control code for buildman @@ -656,6 +791,7 @@ def do_buildman(args, toolchains=None, make_func=None, brds=None, no_subdirs=args.no_subdirs, full_path=args.full_path, verbose_build=args.verbose_build, mrproper=args.mrproper, + fallback_mrproper=args.fallback_mrproper, per_board_out_dir=args.per_board_out_dir, config_only=args.config_only, squash_config_y=not args.preserve_config_y, @@ -676,5 +812,8 @@ def do_buildman(args, toolchains=None, make_func=None, brds=None, TEST_BUILDER = builder + if args.process_limit: + wait_for_process_limit(args.process_limit) + return run_builder(builder, series.commits if series else None, brds.get_selected_dict(), args) diff --git a/tools/buildman/func_test.py b/tools/buildman/func_test.py index 6b88ed815d6..0ac9fc7e44f 100644 --- a/tools/buildman/func_test.py +++ b/tools/buildman/func_test.py @@ -807,27 +807,27 @@ CONFIG_LOCALVERSION=y params, warnings = self._boards.scan_defconfigs(src, src) # We should get two boards - self.assertEquals(2, len(params)) + self.assertEqual(2, len(params)) self.assertFalse(warnings) first = 0 if params[0]['target'] == 'board0' else 1 board0 = params[first] board2 = params[1 - first] - self.assertEquals('arm', board0['arch']) - self.assertEquals('armv7', board0['cpu']) - self.assertEquals('-', board0['soc']) - self.assertEquals('Tester', board0['vendor']) - self.assertEquals('ARM Board 0', board0['board']) - self.assertEquals('config0', board0['config']) - self.assertEquals('board0', board0['target']) - - self.assertEquals('powerpc', board2['arch']) - self.assertEquals('ppc', board2['cpu']) - self.assertEquals('mpc85xx', board2['soc']) - self.assertEquals('Tester', board2['vendor']) - self.assertEquals('PowerPC board 1', board2['board']) - self.assertEquals('config2', board2['config']) - self.assertEquals('board2', board2['target']) + self.assertEqual('arm', board0['arch']) + self.assertEqual('armv7', board0['cpu']) + self.assertEqual('-', board0['soc']) + self.assertEqual('Tester', board0['vendor']) + self.assertEqual('ARM Board 0', board0['board']) + self.assertEqual('config0', board0['config']) + self.assertEqual('board0', board0['target']) + + self.assertEqual('powerpc', board2['arch']) + self.assertEqual('ppc', board2['cpu']) + self.assertEqual('mpc85xx', board2['soc']) + self.assertEqual('Tester', board2['vendor']) + self.assertEqual('PowerPC board 1', board2['board']) + self.assertEqual('config2', board2['config']) + self.assertEqual('board2', board2['target']) def test_output_is_new(self): """Test detecting new changes to Kconfig""" @@ -898,7 +898,7 @@ Active aarch64 armv8 - armltd total_compute board2 params_list, warnings = self._boards.build_board_list(config_dir, src) # There should be two boards no warnings - self.assertEquals(2, len(params_list)) + self.assertEqual(2, len(params_list)) self.assertFalse(warnings) # Set an invalid status line in the file @@ -907,12 +907,12 @@ Active aarch64 armv8 - armltd total_compute board2 for line in orig_data.splitlines(keepends=True)] tools.write_file(main, ''.join(lines), binary=False) params_list, warnings = self._boards.build_board_list(config_dir, src) - self.assertEquals(2, len(params_list)) + self.assertEqual(2, len(params_list)) params = params_list[0] if params['target'] == 'board2': params = params_list[1] - self.assertEquals('-', params['status']) - self.assertEquals(["WARNING: Other: unknown status for 'board0'"], + self.assertEqual('-', params['status']) + self.assertEqual(["WARNING: Other: unknown status for 'board0'"], warnings) # Remove the status line (S:) from a file @@ -920,39 +920,39 @@ Active aarch64 armv8 - armltd total_compute board2 if not line.startswith('S:')] tools.write_file(main, ''.join(lines), binary=False) params_list, warnings = self._boards.build_board_list(config_dir, src) - self.assertEquals(2, len(params_list)) - self.assertEquals(["WARNING: -: unknown status for 'board0'"], warnings) + self.assertEqual(2, len(params_list)) + self.assertEqual(["WARNING: -: unknown status for 'board0'"], warnings) # Remove the configs/ line (F:) from a file - this is the last line data = ''.join(orig_data.splitlines(keepends=True)[:-1]) tools.write_file(main, data, binary=False) params_list, warnings = self._boards.build_board_list(config_dir, src) - self.assertEquals(2, len(params_list)) - self.assertEquals(["WARNING: no maintainers for 'board0'"], warnings) + self.assertEqual(2, len(params_list)) + self.assertEqual(["WARNING: no maintainers for 'board0'"], warnings) # Mark a board as orphaned - this should give a warning lines = ['S: Orphaned' if line.startswith('S') else line for line in orig_data.splitlines(keepends=True)] tools.write_file(main, ''.join(lines), binary=False) params_list, warnings = self._boards.build_board_list(config_dir, src) - self.assertEquals(2, len(params_list)) - self.assertEquals(["WARNING: no maintainers for 'board0'"], warnings) + self.assertEqual(2, len(params_list)) + self.assertEqual(["WARNING: no maintainers for 'board0'"], warnings) # Change the maintainer to '-' - this should give a warning lines = ['M: -' if line.startswith('M') else line for line in orig_data.splitlines(keepends=True)] tools.write_file(main, ''.join(lines), binary=False) params_list, warnings = self._boards.build_board_list(config_dir, src) - self.assertEquals(2, len(params_list)) - self.assertEquals(["WARNING: -: unknown status for 'board0'"], warnings) + self.assertEqual(2, len(params_list)) + self.assertEqual(["WARNING: -: unknown status for 'board0'"], warnings) # Remove the maintainer line (M:) from a file lines = [line for line in orig_data.splitlines(keepends=True) if not line.startswith('M:')] tools.write_file(main, ''.join(lines), binary=False) params_list, warnings = self._boards.build_board_list(config_dir, src) - self.assertEquals(2, len(params_list)) - self.assertEquals(["WARNING: no maintainers for 'board0'"], warnings) + self.assertEqual(2, len(params_list)) + self.assertEqual(["WARNING: no maintainers for 'board0'"], warnings) # Move the contents of the second file into this one, removing the # second file, to check multiple records in a single file. @@ -960,14 +960,14 @@ Active aarch64 armv8 - armltd total_compute board2 tools.write_file(main, both_data, binary=False) os.remove(other) params_list, warnings = self._boards.build_board_list(config_dir, src) - self.assertEquals(2, len(params_list)) + self.assertEqual(2, len(params_list)) self.assertFalse(warnings) # Add another record, this should be ignored with a warning extra = '\n\nAnother\nM: Fred\nF: configs/board9_defconfig\nS: other\n' tools.write_file(main, both_data + extra, binary=False) params_list, warnings = self._boards.build_board_list(config_dir, src) - self.assertEquals(2, len(params_list)) + self.assertEqual(2, len(params_list)) self.assertFalse(warnings) # Add another TARGET to the Kconfig @@ -983,8 +983,8 @@ endif tools.write_file(kc_file, orig_kc_data + extra) params_list, warnings = self._boards.build_board_list(config_dir, src, warn_targets=True) - self.assertEquals(2, len(params_list)) - self.assertEquals( + self.assertEqual(2, len(params_list)) + self.assertEqual( ['WARNING: board2_defconfig: Duplicate TARGET_xxx: board2 and other'], warnings) @@ -994,8 +994,8 @@ endif tools.write_file(kc_file, b''.join(lines)) params_list, warnings = self._boards.build_board_list(config_dir, src, warn_targets=True) - self.assertEquals(2, len(params_list)) - self.assertEquals( + self.assertEqual(2, len(params_list)) + self.assertEqual( ['WARNING: board2_defconfig: No TARGET_BOARD2 enabled'], warnings) tools.write_file(kc_file, orig_kc_data) @@ -1004,7 +1004,7 @@ endif data = ''.join(both_data.splitlines(keepends=True)[:-1]) tools.write_file(main, data + 'N: oa.*2\n', binary=False) params_list, warnings = self._boards.build_board_list(config_dir, src) - self.assertEquals(2, len(params_list)) + self.assertEqual(2, len(params_list)) self.assertFalse(warnings) def testRegenBoards(self): diff --git a/tools/buildman/pyproject.toml b/tools/buildman/pyproject.toml index fe0f6421b53..68bfa45c3f4 100644 --- a/tools/buildman/pyproject.toml +++ b/tools/buildman/pyproject.toml @@ -8,7 +8,11 @@ version = "0.0.6" authors = [ { name="Simon Glass", email="sjg@chromium.org" }, ] -dependencies = ["u_boot_pylib >= 0.0.6", "patch-manager >= 0.0.6"] +dependencies = [ + "filelock >= 3.0.12", + "u_boot_pylib >= 0.0.6", + "patch-manager >= 0.0.6" +] description = "Buildman build tool for U-Boot" readme = "README.rst" requires-python = ">=3.7" diff --git a/tools/buildman/requirements.txt b/tools/buildman/requirements.txt index 4a31e69e4cb..052d0ed5c6f 100644 --- a/tools/buildman/requirements.txt +++ b/tools/buildman/requirements.txt @@ -1,3 +1,5 @@ +coverage==6.2 jsonschema==4.17.3 +pycryptodome==3.20 pyyaml==6.0 yamllint==1.26.3 diff --git a/tools/buildman/test.py b/tools/buildman/test.py index f92add7a7c5..bfad3093030 100644 --- a/tools/buildman/test.py +++ b/tools/buildman/test.py @@ -2,12 +2,14 @@ # Copyright (c) 2012 The Chromium OS Authors. # +from filelock import FileLock import os import shutil import sys import tempfile import time import unittest +from unittest.mock import patch from buildman import board from buildman import boards @@ -156,6 +158,11 @@ class TestBuild(unittest.TestCase): if not os.path.isdir(self.base_dir): os.mkdir(self.base_dir) + self.cur_time = 0 + self.valid_pids = [] + self.finish_time = None + self.finish_pid = None + def tearDown(self): shutil.rmtree(self.base_dir) @@ -584,7 +591,7 @@ class TestBuild(unittest.TestCase): if use_network: with test_util.capture_sys_output() as (stdout, stderr): url = self.toolchains.LocateArchUrl('arm') - self.assertRegexpMatches(url, 'https://www.kernel.org/pub/tools/' + self.assertRegex(url, 'https://www.kernel.org/pub/tools/' 'crosstool/files/bin/x86_64/.*/' 'x86_64-gcc-.*-nolibc[-_]arm-.*linux-gnueabi.tar.xz') @@ -747,6 +754,120 @@ class TestBuild(unittest.TestCase): self.assertEqual([ ['MARY="mary"', 'Missing expected line: CONFIG_MARY="mary"']], result) + def get_procs(self): + running_fname = os.path.join(self.base_dir, control.RUNNING_FNAME) + items = tools.read_file(running_fname, binary=False).split() + return [int(x) for x in items] + + def get_time(self): + return self.cur_time + + def inc_time(self, amount): + self.cur_time += amount + + # Handle a process exiting + if self.finish_time == self.cur_time: + self.valid_pids = [pid for pid in self.valid_pids + if pid != self.finish_pid] + + def kill(self, pid, signal): + if pid not in self.valid_pids: + raise OSError('Invalid PID') + + def test_process_limit(self): + """Test wait_for_process_limit() function""" + tmpdir = self.base_dir + + with (patch('time.time', side_effect=self.get_time), + patch('time.sleep', side_effect=self.inc_time), + patch('os.kill', side_effect=self.kill)): + # Grab the process. Since there is no other profcess, this should + # immediately succeed + control.wait_for_process_limit(1, tmpdir=tmpdir, pid=1) + lines = terminal.get_print_test_lines() + self.assertEqual(0, self.cur_time) + self.assertEqual('Waiting for other buildman processes...', + lines[0].text) + self.assertEqual(self._col.RED, lines[0].colour) + self.assertEqual(False, lines[0].newline) + self.assertEqual(True, lines[0].bright) + + self.assertEqual('done...', lines[1].text) + self.assertEqual(None, lines[1].colour) + self.assertEqual(False, lines[1].newline) + self.assertEqual(True, lines[1].bright) + + self.assertEqual('starting build', lines[2].text) + self.assertEqual([1], control.read_procs(tmpdir)) + self.assertEqual(None, lines[2].colour) + self.assertEqual(False, lines[2].newline) + self.assertEqual(True, lines[2].bright) + + # Try again, with a different PID...this should eventually timeout + # and start the build anyway + self.cur_time = 0 + self.valid_pids = [1] + control.wait_for_process_limit(1, tmpdir=tmpdir, pid=2) + lines = terminal.get_print_test_lines() + self.assertEqual('Waiting for other buildman processes...', + lines[0].text) + self.assertEqual('timeout...', lines[1].text) + self.assertEqual(None, lines[1].colour) + self.assertEqual(False, lines[1].newline) + self.assertEqual(True, lines[1].bright) + self.assertEqual('starting build', lines[2].text) + self.assertEqual([1, 2], control.read_procs(tmpdir)) + self.assertEqual(control.RUN_WAIT_S, self.cur_time) + + # Check lock-busting + self.cur_time = 0 + self.valid_pids = [1, 2] + lock_fname = os.path.join(tmpdir, control.LOCK_FNAME) + lock = FileLock(lock_fname) + lock.acquire(timeout=1) + control.wait_for_process_limit(1, tmpdir=tmpdir, pid=3) + lines = terminal.get_print_test_lines() + self.assertEqual('Waiting for other buildman processes...', + lines[0].text) + self.assertEqual('failed to get lock: busting...', lines[1].text) + self.assertEqual(None, lines[1].colour) + self.assertEqual(False, lines[1].newline) + self.assertEqual(True, lines[1].bright) + self.assertEqual('timeout...', lines[2].text) + self.assertEqual('starting build', lines[3].text) + self.assertEqual([1, 2, 3], control.read_procs(tmpdir)) + self.assertEqual(control.RUN_WAIT_S, self.cur_time) + lock.release() + + # Check handling of dead processes. Here we have PID 2 as a running + # process, even though the PID file contains 1, 2 and 3. So we can + # add one more PID, to make 2 and 4 + self.cur_time = 0 + self.valid_pids = [2] + control.wait_for_process_limit(2, tmpdir=tmpdir, pid=4) + lines = terminal.get_print_test_lines() + self.assertEqual('Waiting for other buildman processes...', + lines[0].text) + self.assertEqual('done...', lines[1].text) + self.assertEqual('starting build', lines[2].text) + self.assertEqual([2, 4], control.read_procs(tmpdir)) + self.assertEqual(0, self.cur_time) + + # Try again, with PID 2 quitting at time 50. This allows the new + # build to start + self.cur_time = 0 + self.valid_pids = [2, 4] + self.finish_pid = 2 + self.finish_time = 50 + control.wait_for_process_limit(2, tmpdir=tmpdir, pid=5) + lines = terminal.get_print_test_lines() + self.assertEqual('Waiting for other buildman processes...', + lines[0].text) + self.assertEqual('done...', lines[1].text) + self.assertEqual('starting build', lines[2].text) + self.assertEqual([4, 5], control.read_procs(tmpdir)) + self.assertEqual(self.finish_time, self.cur_time) + if __name__ == "__main__": unittest.main() diff --git a/tools/buildman/toolchain.py b/tools/buildman/toolchain.py index 79c7c11a110..324ad0e0821 100644 --- a/tools/buildman/toolchain.py +++ b/tools/buildman/toolchain.py @@ -175,9 +175,9 @@ class Toolchain: def MakeEnvironment(self, full_path): """Returns an environment for using the toolchain. - Thie takes the current environment and adds CROSS_COMPILE so that + This takes the current environment and adds CROSS_COMPILE so that the tool chain will operate correctly. This also disables localized - output and possibly unicode encoded output of all build tools by + output and possibly Unicode encoded output of all build tools by adding LC_ALL=C. Note that os.environb is used to obtain the environment, since in some diff --git a/tools/image-host.c b/tools/image-host.c index 7bfc0cb6b18..49ce7436bb9 100644 --- a/tools/image-host.c +++ b/tools/image-host.c @@ -730,7 +730,7 @@ static const char *fit_config_get_image_list(const void *fit, int noffset, int *lenp, int *allow_missingp) { static const char default_list[] = FIT_KERNEL_PROP "\0" - FIT_FDT_PROP; + FIT_FDT_PROP "\0" FIT_SCRIPT_PROP; const char *prop; /* If there is an "sign-image" property, use that */ diff --git a/tools/patman/func_test.py b/tools/patman/func_test.py index e3918497cf4..af6c025a441 100644 --- a/tools/patman/func_test.py +++ b/tools/patman/func_test.py @@ -211,6 +211,7 @@ class TestFunctional(unittest.TestCase): 'u-boot': ['u-boot@lists.denx.de'], 'simon': [self.leb], 'fred': [self.fred], + 'joe': [self.joe], } text = self._get_text('test01.txt') @@ -259,6 +260,7 @@ class TestFunctional(unittest.TestCase): self.assertEqual('Postfix:\t some-branch', next(lines)) self.assertEqual('Cover: 4 lines', next(lines)) self.assertEqual(' Cc: %s' % self.fred, next(lines)) + self.assertEqual(' Cc: %s' % self.joe, next(lines)) self.assertEqual(' Cc: %s' % self.leb, next(lines)) self.assertEqual(' Cc: %s' % mel, next(lines)) @@ -272,7 +274,8 @@ class TestFunctional(unittest.TestCase): self.assertEqual(('%s %s\0%s' % (args[0], rick, stefan)), cc_lines[0]) self.assertEqual( - '%s %s\0%s\0%s\0%s' % (args[1], self.fred, self.leb, rick, stefan), + '%s %s\0%s\0%s\0%s\0%s' % (args[1], self.fred, self.joe, self.leb, + rick, stefan), cc_lines[1]) expected = ''' @@ -290,6 +293,7 @@ Changes in v4: change - Some changes - Some notes for the cover letter +- fdt: Correct cast for sandbox in fdtdec_setup_mem_size_base() Simon Glass (2): pci: Correct cast for sandbox @@ -339,6 +343,7 @@ Changes in v4: - Multi line change +- New - Some changes Changes in v2: @@ -540,7 +545,8 @@ complicated as possible''') with open('.patman', 'w', buffering=1) as f: f.write('[settings]\n' 'get_maintainer_script: dummy-script.sh\n' - 'check_patch: False\n') + 'check_patch: False\n' + 'add_maintainers: True\n') with open('dummy-script.sh', 'w', buffering=1) as f: f.write('#!/usr/bin/env python\n' 'print("hello@there.com")\n') diff --git a/tools/patman/patchstream.py b/tools/patman/patchstream.py index e2e2a83e677..a09ae9c7371 100644 --- a/tools/patman/patchstream.py +++ b/tools/patman/patchstream.py @@ -475,6 +475,13 @@ class PatchStream: elif name == 'changes': self.in_change = 'Commit' self.change_version = self._parse_version(value, line) + elif name == 'cc': + self.commit.add_cc(value.split(',')) + elif name == 'added-in': + version = self._parse_version(value, line) + self.commit.add_change(version, '- New') + self.series.AddChange(version, None, '- %s' % + self.commit.subject) else: self._add_warn('Line %d: Ignoring Commit-%s' % (self.linenum, name)) diff --git a/tools/patman/patman.rst b/tools/patman/patman.rst index f4588c00fc1..63b95a6b161 100644 --- a/tools/patman/patman.rst +++ b/tools/patman/patman.rst @@ -350,7 +350,20 @@ Cover-changes: n - This line will only appear in the cover letter <blank line> -Patch-cc: Their Name <email> +Commit-added-in: n + Add a change noting the version this commit was added in. This is + equivalent to:: + + Commit-changes: n + - New + + Cover-changes: n + - <commit subject> + + It is a convenient shorthand for suppressing the '(no changes in vN)' + message. + +Patch-cc / Commit-cc: Their Name <email> This copies a single patch to another email address. Note that the Cc: used by git send-email is ignored by patman, but will be interpreted by git send-email if you use it. diff --git a/tools/patman/settings.py b/tools/patman/settings.py index 636983e32da..68c93e313b3 100644 --- a/tools/patman/settings.py +++ b/tools/patman/settings.py @@ -59,25 +59,25 @@ class _ProjectConfigParser(ConfigParser.ConfigParser): # Check to make sure that bogus project gets general alias. >>> config = _ProjectConfigParser("zzz") - >>> config.readfp(StringIO(sample_config)) + >>> config.read_file(StringIO(sample_config)) >>> str(config.get("alias", "enemies")) 'Evil <evil@example.com>' # Check to make sure that alias gets overridden by project. >>> config = _ProjectConfigParser("sm") - >>> config.readfp(StringIO(sample_config)) + >>> config.read_file(StringIO(sample_config)) >>> str(config.get("alias", "enemies")) 'Green G. <ugly@example.com>' # Check to make sure that settings get merged with project. >>> config = _ProjectConfigParser("linux") - >>> config.readfp(StringIO(sample_config)) + >>> config.read_file(StringIO(sample_config)) >>> sorted((str(a), str(b)) for (a, b) in config.items("settings")) [('am_hero', 'True'), ('check_patch_use_tree', 'True'), ('process_tags', 'False')] # Check to make sure that settings works with unknown project. >>> config = _ProjectConfigParser("unknown") - >>> config.readfp(StringIO(sample_config)) + >>> config.read_file(StringIO(sample_config)) >>> sorted((str(a), str(b)) for (a, b) in config.items("settings")) [('am_hero', 'True')] """ diff --git a/tools/patman/test/0002-fdt-Correct-cast-for-sandbox-in-fdtdec_setup_mem_siz.patch b/tools/patman/test/0002-fdt-Correct-cast-for-sandbox-in-fdtdec_setup_mem_siz.patch index 56278a6ce9b..48ea1793b47 100644 --- a/tools/patman/test/0002-fdt-Correct-cast-for-sandbox-in-fdtdec_setup_mem_siz.patch +++ b/tools/patman/test/0002-fdt-Correct-cast-for-sandbox-in-fdtdec_setup_mem_siz.patch @@ -21,7 +21,9 @@ Series-cc: Stefan Brüns <stefan.bruens@rwth-aachen.de> Cover-letter-cc: Lord Mëlchett <clergy@palace.gov> Series-version: 3 Patch-cc: fred +Commit-cc: joe Series-process-log: sort, uniq +Commit-added-in: 4 Series-changes: 4 - Some changes - Multi diff --git a/tools/patman/test/test01.txt b/tools/patman/test/test01.txt index fc3066e50b4..b2d73c5972c 100644 --- a/tools/patman/test/test01.txt +++ b/tools/patman/test/test01.txt @@ -49,7 +49,9 @@ Date: Sat Apr 15 15:39:08 2017 -0600 Cover-letter-cc: Lord Mëlchett <clergy@palace.gov> Series-version: 3 Patch-cc: fred + Commit-cc: joe Series-process-log: sort, uniq + Commit-added-in: 4 Series-changes: 4 - Some changes - Multi diff --git a/tools/rkcommon.c b/tools/rkcommon.c index 12c27b34eaa..3e52236b15a 100644 --- a/tools/rkcommon.c +++ b/tools/rkcommon.c @@ -470,7 +470,7 @@ int rkcommon_verify_header(unsigned char *buf, int size, * If no 'imagename' is specified via the commandline (e.g. if this is * 'dumpimage -l' w/o any further constraints), we accept any spl_info. */ - if (params->imagename == NULL) + if (params->imagename == NULL || !strlen(params->imagename)) return 0; /* Match the 'imagename' against the 'spl_hdr' found */ diff --git a/tools/u_boot_pylib/terminal.py b/tools/u_boot_pylib/terminal.py index 40d79f8ac07..2cd5a54ab52 100644 --- a/tools/u_boot_pylib/terminal.py +++ b/tools/u_boot_pylib/terminal.py @@ -164,8 +164,11 @@ def print_clear(): global last_print_len if last_print_len: - print('\r%s\r' % (' '* last_print_len), end='', flush=True) - last_print_len = None + if print_test_mode: + print_test_list.append(PrintLine(None, None, None, None)) + else: + print('\r%s\r' % (' '* last_print_len), end='', flush=True) + last_print_len = None def set_print_test_mode(enable=True): """Go into test mode, where all printing is recorded""" diff --git a/tools/u_boot_pylib/test_util.py b/tools/u_boot_pylib/test_util.py index f18d385d995..857ce58c98c 100644 --- a/tools/u_boot_pylib/test_util.py +++ b/tools/u_boot_pylib/test_util.py @@ -60,12 +60,17 @@ def run_test_coverage(prog, filter_fname, exclude_list, build_dir, required=None prefix = '' if build_dir: prefix = 'PYTHONPATH=$PYTHONPATH:%s/sandbox_spl/tools ' % build_dir - cmd = ('%spython3-coverage run ' - '--omit "%s" %s %s %s %s' % (prefix, ','.join(glob_list), + + # Detect a Python virtualenv and use 'coverage' instead + covtool = ('python3-coverage' if sys.prefix == sys.base_prefix else + 'coverage') + + cmd = ('%s%s run ' + '--omit "%s" %s %s %s %s' % (prefix, covtool, ','.join(glob_list), prog, extra_args or '', test_cmd, single_thread or '-P1')) os.system(cmd) - stdout = command.output('python3-coverage', 'report') + stdout = command.output(covtool, 'report') lines = stdout.splitlines() if required: # Convert '/path/to/name.py' just the module name 'name' |