From a0dcaf2049056348b8b603116ed1d8556851e951 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Mon, 8 Jul 2019 14:25:35 -0600 Subject: binman: Add a return value to ProcessContentsUpdate() At present if this function tries to update the contents such that the size changes, it raises an error. We plan to add the ability to change the size of entries after packing is completed, since in some cases it is not possible to determine the size in advance. An example of this is with a compressed device tree, where the values of the device tree change in SetCalculatedProperties() or ProcessEntryContents(). While the device tree itself does not change size, since placeholders for any new properties have already bee added by AddMissingProperties(), we cannot predict the size of the device tree after compression. If a value changes from 0 to 0x1234 (say), then the compressed device tree may expand. As a first step towards supporting this, make ProcessContentsUpdate() return a value indicating whether the content size is OK. For now this is always True (since otherwise binman raises an error), but later patches will adjust this. Signed-off-by: Simon Glass --- tools/binman/image.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'tools/binman/image.py') diff --git a/tools/binman/image.py b/tools/binman/image.py index f237ae302df..c8bce394aa1 100644 --- a/tools/binman/image.py +++ b/tools/binman/image.py @@ -122,8 +122,11 @@ class Image: """Call the ProcessContents() method for each entry This is intended to adjust the contents as needed by the entry type. + + Returns: + True if the new data size is OK, False if expansion is needed """ - self._section.ProcessEntryContents() + return self._section.ProcessEntryContents() def WriteSymbols(self): """Write symbol values into binary files for access at run time""" -- cgit v1.2.3 From c52c9e7da809e36001d125891e594c4740235055 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Mon, 8 Jul 2019 14:25:37 -0600 Subject: binman: Allow entries to expand after packing Add support for detecting entries that change size after they have already been packed, and re-running packing when it happens. This removes the limitation that entry size cannot change after PackEntries() is called. Signed-off-by: Simon Glass --- tools/binman/image.py | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'tools/binman/image.py') diff --git a/tools/binman/image.py b/tools/binman/image.py index c8bce394aa1..6339d020e76 100644 --- a/tools/binman/image.py +++ b/tools/binman/image.py @@ -55,6 +55,10 @@ class Image: self._filename = filename self._section = bsection.Section('main-section', None, self._node, self) + def Raise(self, msg): + """Convenience function to raise an error referencing an image""" + raise ValueError("Image '%s': %s" % (self._node.path, msg)) + def GetFdtSet(self): """Get the set of device tree files used by this image""" return self._section.GetFdtSet() @@ -100,6 +104,10 @@ class Image: """ self._section.GetEntryOffsets() + def ResetForPack(self): + """Reset offset/size fields so that packing can be done again""" + self._section.ResetForPack() + def PackEntries(self): """Pack all entries into the image""" self._section.PackEntries() -- cgit v1.2.3 From 41b8ba090ced0bb54733dc3dd32e945e7e332801 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Mon, 8 Jul 2019 14:25:43 -0600 Subject: binman: Allow listing the entries in an image It is useful to be able to summarise all the entries in an image, e.g. to display this to this user. Add a new ListEntries() method to Entry, and set up a way to call it through the Image class. Signed-off-by: Simon Glass --- tools/binman/image.py | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'tools/binman/image.py') diff --git a/tools/binman/image.py b/tools/binman/image.py index 6339d020e76..6f4bd5d37b2 100644 --- a/tools/binman/image.py +++ b/tools/binman/image.py @@ -162,3 +162,13 @@ class Image: file=fd) self._section.WriteMap(fd, 0) return fname + + def BuildEntryList(self): + """List the files in an image + + Returns: + List of entry.EntryInfo objects describing all entries in the image + """ + entries = [] + self._section.ListEntries(entries, 0) + return entries -- cgit v1.2.3 From ffded7527ad5272249ae728a88585373182cc7f4 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Mon, 8 Jul 2019 14:25:46 -0600 Subject: binman: Support reading an image into an Image object It is possible to read an Image, locate its FDT map and then read it into the binman data structures. This allows full access to the entries that were written to the image. Add support for this. Signed-off-by: Simon Glass --- tools/binman/image.py | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) (limited to 'tools/binman/image.py') diff --git a/tools/binman/image.py b/tools/binman/image.py index 6f4bd5d37b2..f890350a8d0 100644 --- a/tools/binman/image.py +++ b/tools/binman/image.py @@ -12,6 +12,9 @@ from operator import attrgetter import re import sys +from etype import fdtmap +from etype import image_header +import fdt import fdt_util import bsection import tools @@ -47,6 +50,41 @@ class Image: else: self._ReadNode() + @classmethod + def FromFile(cls, fname): + """Convert an image file into an Image for use in binman + + Args: + fname: Filename of image file to read + + Returns: + Image object on success + + Raises: + ValueError if something goes wrong + """ + data = tools.ReadFile(fname) + size = len(data) + + # First look for an image header + pos = image_header.LocateHeaderOffset(data) + if pos is None: + # Look for the FDT map + pos = fdtmap.LocateFdtmap(data) + if pos is None: + raise ValueError('Cannot find FDT map in image') + + # We don't know the FDT size, so check its header first + probe_dtb = fdt.Fdt.FromData( + data[pos + fdtmap.FDTMAP_HDR_LEN:pos + 256]) + dtb_size = probe_dtb.GetFdtObj().totalsize() + fdtmap_data = data[pos:pos + dtb_size + fdtmap.FDTMAP_HDR_LEN] + dtb = fdt.Fdt.FromData(fdtmap_data[fdtmap.FDTMAP_HDR_LEN:]) + dtb.Scan() + + # Return an Image with the associated nodes + return Image('image', dtb.GetRoot()) + def _ReadNode(self): """Read properties from the image node""" self._size = fdt_util.GetInt(self._node, 'size') -- cgit v1.2.3 From 8beb11ea6e09726996350a21bedba110f234d983 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Mon, 8 Jul 2019 14:25:47 -0600 Subject: binman: Convert Image to a subclass of Entry When support for sections (and thus hierarchical images) was added to binman, the decision was made to create a new Section class which could be used by both Image and an Entry_section class. The decision between using inheritance and composition was tricky to make, but in the end it was decided that Image was different enough from Entry that it made sense to put the implementation of sections in an entirely separate class. It also has the advantage that core Image code does have to rely on an entry class in the etype directory. This work was mostly completed in commit: 8f1da50ccc "binman: Refactor much of the image code into 'section' As a result of this, the Section class has its own version of things like offset and size and these must be kept in sync with the parent Entry_section class in some cases. In the last year it has become apparent that the cost of keeping things in sync is larger than expected, since more and more code wants to access these properties. An alternative approach, previously considered and rejected, now seems better. Adjust Image to be a subclass of Entry_section. Move the code from Section (in bsection.py) to Entry_section and delete Section. Update all tests accordingly. This requires substantial changes to Image. Overall the changes reduce code size by about 240 lines. While much of that is just boilerplate from Section, there are quite a few functions in Entry_section which now do not need to be overiden from Entry. This suggests the change is beneficial even without further functionality being added. A side benefit is that the properties of sections are now consistent with other entries. This fixes a problem in testListCmd() where some properties are missing for sections. Unfortunately this is a very large commit since it is not feasible to do the migration piecemeal. Given the substantial tests available and the 100% code coverage of binman, we should be able to do this safely. Signed-off-by: Simon Glass --- tools/binman/image.py | 129 +++++++++++++------------------------------------- 1 file changed, 33 insertions(+), 96 deletions(-) (limited to 'tools/binman/image.py') diff --git a/tools/binman/image.py b/tools/binman/image.py index f890350a8d0..487290b6c4e 100644 --- a/tools/binman/image.py +++ b/tools/binman/image.py @@ -12,14 +12,15 @@ from operator import attrgetter import re import sys +from entry import Entry from etype import fdtmap from etype import image_header +from etype import section import fdt import fdt_util -import bsection import tools -class Image: +class Image(section.Entry_section): """A Image, representing an output from binman An image is comprised of a collection of entries each containing binary @@ -27,12 +28,8 @@ class Image: This class implements the various operations needed for images. - Atrtributes: - _node: Node object that contains the image definition in device tree - _name: Image name - _size: Image size in bytes, or None if not known yet - _filename: Output filename for image - _sections: Sections present in this image (may be one or more) + Attributes: + filename: Output filename for image Args: test: True if this is being called from a test of Images. This this case @@ -40,15 +37,15 @@ class Image: we create a section manually. """ def __init__(self, name, node, test=False): - self._node = node - self._name = name - self._size = None - self._filename = '%s.bin' % self._name - if test: - self._section = bsection.Section('main-section', None, self._node, - self, True) - else: - self._ReadNode() + self.image = self + section.Entry_section.__init__(self, None, 'section', node, test) + self.name = 'main-section' + self.image_name = name + self._filename = '%s.bin' % self.image_name + if not test: + filename = fdt_util.GetString(self._node, 'filename') + if filename: + self._filename = filename @classmethod def FromFile(cls, fname): @@ -85,84 +82,17 @@ class Image: # Return an Image with the associated nodes return Image('image', dtb.GetRoot()) - def _ReadNode(self): - """Read properties from the image node""" - self._size = fdt_util.GetInt(self._node, 'size') - filename = fdt_util.GetString(self._node, 'filename') - if filename: - self._filename = filename - self._section = bsection.Section('main-section', None, self._node, self) - def Raise(self, msg): """Convenience function to raise an error referencing an image""" raise ValueError("Image '%s': %s" % (self._node.path, msg)) - def GetFdtSet(self): - """Get the set of device tree files used by this image""" - return self._section.GetFdtSet() - - def ExpandEntries(self): - """Expand out any entries which have calculated sub-entries - - Some entries are expanded out at runtime, e.g. 'files', which produces - a section containing a list of files. Process these entries so that - this information is added to the device tree. - """ - self._section.ExpandEntries() - - def AddMissingProperties(self): - """Add properties that are not present in the device tree - - When binman has completed packing the entries the offset and size of - each entry are known. But before this the device tree may not specify - these. Add any missing properties, with a dummy value, so that the - size of the entry is correct. That way we can insert the correct values - later. - """ - self._section.AddMissingProperties() - - def ProcessFdt(self, fdt): - """Allow entries to adjust the device tree - - Some entries need to adjust the device tree for their purposes. This - may involve adding or deleting properties. - """ - return self._section.ProcessFdt(fdt) - - def GetEntryContents(self): - """Call ObtainContents() for the section - """ - self._section.GetEntryContents() - - def GetEntryOffsets(self): - """Handle entries that want to set the offset/size of other entries - - This calls each entry's GetOffsets() method. If it returns a list - of entries to update, it updates them. - """ - self._section.GetEntryOffsets() - - def ResetForPack(self): - """Reset offset/size fields so that packing can be done again""" - self._section.ResetForPack() - def PackEntries(self): """Pack all entries into the image""" - self._section.PackEntries() - - def CheckSize(self): - """Check that the image contents does not exceed its size, etc.""" - self._size = self._section.CheckSize() - - def CheckEntries(self): - """Check that entries do not overlap or extend outside the image""" - self._section.CheckEntries() - - def SetCalculatedProperties(self): - self._section.SetCalculatedProperties() + section.Entry_section.Pack(self, 0) def SetImagePos(self): - self._section.SetImagePos(0) + # This first section in the image so it starts at 0 + section.Entry_section.SetImagePos(self, 0) def ProcessEntryContents(self): """Call the ProcessContents() method for each entry @@ -172,20 +102,27 @@ class Image: Returns: True if the new data size is OK, False if expansion is needed """ - return self._section.ProcessEntryContents() + sizes_ok = True + for entry in self._entries.values(): + if not entry.ProcessContents(): + sizes_ok = False + print("Entry '%s' size change" % self._node.path) + return sizes_ok def WriteSymbols(self): """Write symbol values into binary files for access at run time""" - self._section.WriteSymbols() + section.Entry_section.WriteSymbols(self, self) + + def BuildSection(self, fd, base_offset): + """Write the section to a file""" + fd.seek(base_offset) + fd.write(self.GetData()) def BuildImage(self): """Write the image to a file""" fname = tools.GetOutputFilename(self._filename) with open(fname, 'wb') as fd: - self._section.BuildSection(fd, 0) - - def GetEntries(self): - return self._section.GetEntries() + self.BuildSection(fd, 0) def WriteMap(self): """Write a map of the image to a .map file @@ -193,12 +130,12 @@ class Image: Returns: Filename of map file written """ - filename = '%s.map' % self._name + filename = '%s.map' % self.image_name fname = tools.GetOutputFilename(filename) with open(fname, 'w') as fd: print('%8s %8s %8s %s' % ('ImagePos', 'Offset', 'Size', 'Name'), file=fd) - self._section.WriteMap(fd, 0) + section.Entry_section.WriteMap(self, fd, 0) return fname def BuildEntryList(self): @@ -208,5 +145,5 @@ class Image: List of entry.EntryInfo objects describing all entries in the image """ entries = [] - self._section.ListEntries(entries, 0) + self.ListEntries(entries, 0) return entries -- cgit v1.2.3 From 61f564d15f35e5f5600ed639201b257efa09d1f1 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Mon, 8 Jul 2019 14:25:48 -0600 Subject: binman: Support listing an image Add support for listing the entries in an image. This relies on the image having an FDT map. Signed-off-by: Simon Glass --- tools/binman/image.py | 150 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 150 insertions(+) (limited to 'tools/binman/image.py') diff --git a/tools/binman/image.py b/tools/binman/image.py index 487290b6c4e..2c5668e2a96 100644 --- a/tools/binman/image.py +++ b/tools/binman/image.py @@ -8,6 +8,7 @@ from __future__ import print_function from collections import OrderedDict +import fnmatch from operator import attrgetter import re import sys @@ -147,3 +148,152 @@ class Image(section.Entry_section): entries = [] self.ListEntries(entries, 0) return entries + + def FindEntryPath(self, entry_path): + """Find an entry at a given path in the image + + Args: + entry_path: Path to entry (e.g. /ro-section/u-boot') + + Returns: + Entry object corresponding to that past + + Raises: + ValueError if no entry found + """ + parts = entry_path.split('/') + entries = self.GetEntries() + parent = '/' + for part in parts: + entry = entries.get(part) + if not entry: + raise ValueError("Entry '%s' not found in '%s'" % + (part, parent)) + parent = entry.GetPath() + entries = entry.GetEntries() + return entry + + def ReadData(self, decomp=True): + return self._data + + def GetListEntries(self, entry_paths): + """List the entries in an image + + This decodes the supplied image and returns a list of entries from that + image, preceded by a header. + + Args: + entry_paths: List of paths to match (each can have wildcards). Only + entries whose names match one of these paths will be printed + + Returns: + String error message if something went wrong, otherwise + 3-Tuple: + List of EntryInfo objects + List of lines, each + List of text columns, each a string + List of widths of each column + """ + def _EntryToStrings(entry): + """Convert an entry to a list of strings, one for each column + + Args: + entry: EntryInfo object containing information to output + + Returns: + List of strings, one for each field in entry + """ + def _AppendHex(val): + """Append a hex value, or an empty string if val is None + + Args: + val: Integer value, or None if none + """ + args.append('' if val is None else '>%x' % val) + + args = [' ' * entry.indent + entry.name] + _AppendHex(entry.image_pos) + _AppendHex(entry.size) + args.append(entry.etype) + _AppendHex(entry.offset) + _AppendHex(entry.uncomp_size) + return args + + def _DoLine(lines, line): + """Add a line to the output list + + This adds a line (a list of columns) to the output list. It also updates + the widths[] array with the maximum width of each column + + Args: + lines: List of lines to add to + line: List of strings, one for each column + """ + for i, item in enumerate(line): + widths[i] = max(widths[i], len(item)) + lines.append(line) + + def _NameInPaths(fname, entry_paths): + """Check if a filename is in a list of wildcarded paths + + Args: + fname: Filename to check + entry_paths: List of wildcarded paths (e.g. ['*dtb*', 'u-boot*', + 'section/u-boot']) + + Returns: + True if any wildcard matches the filename (using Unix filename + pattern matching, not regular expressions) + False if not + """ + for path in entry_paths: + if fnmatch.fnmatch(fname, path): + return True + return False + + entries = self.BuildEntryList() + + # This is our list of lines. Each item in the list is a list of strings, one + # for each column + lines = [] + HEADER = ['Name', 'Image-pos', 'Size', 'Entry-type', 'Offset', + 'Uncomp-size'] + num_columns = len(HEADER) + + # This records the width of each column, calculated as the maximum width of + # all the strings in that column + widths = [0] * num_columns + _DoLine(lines, HEADER) + + # We won't print anything unless it has at least this indent. So at the + # start we will print nothing, unless a path matches (or there are no + # entry paths) + MAX_INDENT = 100 + min_indent = MAX_INDENT + path_stack = [] + path = '' + indent = 0 + selected_entries = [] + for entry in entries: + if entry.indent > indent: + path_stack.append(path) + elif entry.indent < indent: + path_stack.pop() + if path_stack: + path = path_stack[-1] + '/' + entry.name + indent = entry.indent + + # If there are entry paths to match and we are not looking at a + # sub-entry of a previously matched entry, we need to check the path + if entry_paths and indent <= min_indent: + if _NameInPaths(path[1:], entry_paths): + # Print this entry and all sub-entries (=higher indent) + min_indent = indent + else: + # Don't print this entry, nor any following entries until we get + # a path match + min_indent = MAX_INDENT + continue + _DoLine(lines, _EntryToStrings(entry)) + selected_entries.append(entry) + return selected_entries, lines, widths -- cgit v1.2.3 From eea264ead3ca198ed66f62a78dc4940075621ae7 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Mon, 8 Jul 2019 14:25:49 -0600 Subject: binman: Allow for logging information to be displayed Binman generally operates silently but in some cases it is useful to see what Binman is actually doing at each step. Enable some logging output with different logging levels selectable via the -v flag. Signed-off-by: Simon Glass --- tools/binman/image.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'tools/binman/image.py') diff --git a/tools/binman/image.py b/tools/binman/image.py index 2c5668e2a96..bbb5e23c3b2 100644 --- a/tools/binman/image.py +++ b/tools/binman/image.py @@ -20,6 +20,7 @@ from etype import section import fdt import fdt_util import tools +import tout class Image(section.Entry_section): """A Image, representing an output from binman @@ -107,7 +108,7 @@ class Image(section.Entry_section): for entry in self._entries.values(): if not entry.ProcessContents(): sizes_ok = False - print("Entry '%s' size change" % self._node.path) + tout.Debug("Entry '%s' size change" % self._node.path) return sizes_ok def WriteSymbols(self): -- cgit v1.2.3 From f667e45b1c0a7f21d433ee8f3ec18858d87dd2e5 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Mon, 8 Jul 2019 14:25:50 -0600 Subject: binman: Allow reading an entry from an image It is useful to be able to extract entry contents from an image to see what is inside. Add a simple function to read the contents of an entry, decompressing it by default. Signed-off-by: Simon Glass --- tools/binman/image.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'tools/binman/image.py') diff --git a/tools/binman/image.py b/tools/binman/image.py index bbb5e23c3b2..fb6e591ca60 100644 --- a/tools/binman/image.py +++ b/tools/binman/image.py @@ -82,7 +82,9 @@ class Image(section.Entry_section): dtb.Scan() # Return an Image with the associated nodes - return Image('image', dtb.GetRoot()) + image = Image('image', dtb.GetRoot()) + image._data = data + return image def Raise(self, msg): """Convenience function to raise an error referencing an image""" -- cgit v1.2.3