diff options
Diffstat (limited to 'tools')
53 files changed, 620 insertions, 165 deletions
diff --git a/tools/.gitignore b/tools/.gitignore index 0108c567309..6a5c613f772 100644 --- a/tools/.gitignore +++ b/tools/.gitignore @@ -29,6 +29,7 @@ /mxsboot /ncb /prelink-riscv +/preload_check_sign /printinitialenv /proftool /relocate-rela diff --git a/tools/Kconfig b/tools/Kconfig index 01ff0fcf748..8e272ee99a8 100644 --- a/tools/Kconfig +++ b/tools/Kconfig @@ -9,6 +9,11 @@ config MKIMAGE_DTC_PATH some cases the system dtc may not support all required features and the path to a different version should be given here. +config TOOLS_IMAGE_PRE_LOAD + def_bool y + help + Enable pre-load signature support in the tools builds. + config TOOLS_CRC16 def_bool y help diff --git a/tools/Makefile b/tools/Makefile index 237fa900a24..e5f5eea47c7 100644 --- a/tools/Makefile +++ b/tools/Makefile @@ -66,6 +66,7 @@ mkenvimage-objs := mkenvimage.o os_support.o generated/lib/crc32.o hostprogs-y += dumpimage mkimage hostprogs-$(CONFIG_TOOLS_LIBCRYPTO) += fit_info fit_check_sign hostprogs-$(CONFIG_TOOLS_LIBCRYPTO) += fdt_add_pubkey +hostprogs-$(CONFIG_TOOLS_LIBCRYPTO) += preload_check_sign ifneq ($(CONFIG_CMD_BOOTEFI_SELFTEST)$(CONFIG_FWU_MDATA_GPT_BLK),) hostprogs-y += file2include @@ -89,6 +90,8 @@ ECDSA_OBJS-$(CONFIG_TOOLS_LIBCRYPTO) := $(addprefix generated/lib/ecdsa/, ecdsa- AES_OBJS-$(CONFIG_TOOLS_LIBCRYPTO) := $(addprefix generated/lib/aes/, \ aes-encrypt.o aes-decrypt.o) +PRELOAD_OBJS-$(CONFIG_TOOLS_LIBCRYPTO) := generated/boot/image-pre-load.o + # Cryptographic helpers and image types that depend on openssl/libcrypto LIBCRYPTO_OBJS-$(CONFIG_TOOLS_LIBCRYPTO) := \ generated/lib/fdt-libcrypto.o \ @@ -158,6 +161,7 @@ fit_info-objs := $(dumpimage-mkimage-objs) fit_info.o fit_check_sign-objs := $(dumpimage-mkimage-objs) fit_check_sign.o fdt_add_pubkey-objs := $(dumpimage-mkimage-objs) fdt_add_pubkey.o file2include-objs := file2include.o +preload_check_sign-objs := $(dumpimage-mkimage-objs) $(PRELOAD_OBJS-y) preload_check_sign.o ifneq ($(CONFIG_MX23)$(CONFIG_MX28)$(CONFIG_TOOLS_LIBCRYPTO),) # Add CFG_MXS into host CFLAGS, so we can check whether or not register @@ -195,6 +199,7 @@ HOSTLDLIBS_dumpimage := $(HOSTLDLIBS_mkimage) HOSTLDLIBS_fit_info := $(HOSTLDLIBS_mkimage) HOSTLDLIBS_fit_check_sign := $(HOSTLDLIBS_mkimage) HOSTLDLIBS_fdt_add_pubkey := $(HOSTLDLIBS_mkimage) +HOSTLDLIBS_preload_check_sign := $(HOSTLDLIBS_mkimage) hostprogs-$(CONFIG_EXYNOS5250) += mkexynosspl hostprogs-$(CONFIG_EXYNOS5420) += mkexynosspl diff --git a/tools/binman/etype/fdtmap.py b/tools/binman/etype/fdtmap.py index f1f6217940f..2259404180c 100644 --- a/tools/binman/etype/fdtmap.py +++ b/tools/binman/etype/fdtmap.py @@ -106,6 +106,9 @@ class Entry_fdtmap(Entry): Returns: FDT map binary data """ + fsw = libfdt.FdtSw() + fsw.finish_reservemap() + def _AddNode(node): """Add a node to the FDT map""" for pname, prop in node.props.items(): @@ -134,8 +137,6 @@ class Entry_fdtmap(Entry): # Build a new tree with all nodes and properties starting from that # node - fsw = libfdt.FdtSw() - fsw.finish_reservemap() with fsw.add_node(''): fsw.property_string('image-node', node.name) _AddNode(node) diff --git a/tools/binman/etype/image_header.py b/tools/binman/etype/image_header.py index 24011884958..2114df8159f 100644 --- a/tools/binman/etype/image_header.py +++ b/tools/binman/etype/image_header.py @@ -62,6 +62,7 @@ class Entry_image_header(Entry): def _GetHeader(self): image_pos = self.GetSiblingImagePos('fdtmap') + offset = None if image_pos == False: self.Raise("'image_header' section must have an 'fdtmap' sibling") elif image_pos is None: diff --git a/tools/binman/etype/pre_load.py b/tools/binman/etype/pre_load.py index 2e4c72359ff..00f1a896767 100644 --- a/tools/binman/etype/pre_load.py +++ b/tools/binman/etype/pre_load.py @@ -112,6 +112,8 @@ class Entry_pre_load(Entry_collection): # Compute the signature if padding_name is None: padding_name = "pkcs-1.5" + padding = None + padding_args = None if padding_name == "pss": salt_len = key.size_in_bytes() - hash_image.digest_size - 2 padding = pss diff --git a/tools/binman/etype/ti_board_config.py b/tools/binman/etype/ti_board_config.py index c10d66edcb1..7c6773ac7bc 100644 --- a/tools/binman/etype/ti_board_config.py +++ b/tools/binman/etype/ti_board_config.py @@ -119,12 +119,14 @@ class Entry_ti_board_config(Entry_section): array of bytes representing value """ size = 0 + br = bytearray() if (data_type == '#/definitions/u8'): size = 1 elif (data_type == '#/definitions/u16'): size = 2 else: size = 4 + br = None if type(val) == int: br = val.to_bytes(size, byteorder='little') return br diff --git a/tools/binman/etype/x509_cert.py b/tools/binman/etype/x509_cert.py index 29630d1b86c..25e6808b7f9 100644 --- a/tools/binman/etype/x509_cert.py +++ b/tools/binman/etype/x509_cert.py @@ -84,6 +84,7 @@ class Entry_x509_cert(Entry_collection): input_fname = tools.get_output_filename('input.%s' % uniq) config_fname = tools.get_output_filename('config.%s' % uniq) tools.write_file(input_fname, input_data) + stdout = None if type == 'generic': stdout = self.openssl.x509_cert( cert_fname=output_fname, diff --git a/tools/binman/ftest.py b/tools/binman/ftest.py index 733169b99f6..1a92a99b511 100644 --- a/tools/binman/ftest.py +++ b/tools/binman/ftest.py @@ -303,7 +303,7 @@ class TestFunctional(unittest.TestCase): def setUp(self): # Enable this to turn on debugging output # tout.init(tout.DEBUG) - command.test_result = None + command.TEST_RESULT = None def tearDown(self): """Remove the temporary output directory""" @@ -345,8 +345,9 @@ class TestFunctional(unittest.TestCase): Arguments to pass, as a list of strings kwargs: Arguments to pass to Command.RunPipe() """ - result = command.run_pipe([[self._binman_pathname] + list(args)], - capture=True, capture_stderr=True, raise_on_error=False) + all_args = [self._binman_pathname] + list(args) + result = command.run_one(*all_args, capture=True, capture_stderr=True, + raise_on_error=False) if result.return_code and kwargs.get('raise_on_error', True): raise Exception("Error running '%s': %s" % (' '.join(args), result.stdout + result.stderr)) @@ -762,6 +763,16 @@ class TestFunctional(unittest.TestCase): return False return True + def _CheckPreload(self, image, key, algo="sha256,rsa2048", + padding="pkcs-1.5"): + try: + tools.run('preload_check_sign', '-k', key, '-a', algo, '-p', + padding, '-f', image) + except: + self.fail('Expected image signed with a pre-load') + return False + return True + def testRun(self): """Test a basic run with valid args""" result = self._RunBinman('-h') @@ -780,11 +791,11 @@ class TestFunctional(unittest.TestCase): def testFullHelpInternal(self): """Test that the full help is displayed with -H""" try: - command.test_result = command.CommandResult() + command.TEST_RESULT = command.CommandResult() result = self._DoBinman('-H') help_file = os.path.join(self._binman_dir, 'README.rst') finally: - command.test_result = None + command.TEST_RESULT = None def testHelp(self): """Test that the basic help is displayed with -h""" @@ -1872,7 +1883,7 @@ class TestFunctional(unittest.TestCase): def testGbb(self): """Test for the Chromium OS Google Binary Block""" - command.test_result = self._HandleGbbCommand + command.TEST_RESULT = self._HandleGbbCommand entry_args = { 'keydir': 'devkeys', 'bmpblk': 'bmpblk.bin', @@ -1941,7 +1952,7 @@ class TestFunctional(unittest.TestCase): def testVblock(self): """Test for the Chromium OS Verified Boot Block""" self._hash_data = False - command.test_result = self._HandleVblockCommand + command.TEST_RESULT = self._HandleVblockCommand entry_args = { 'keydir': 'devkeys', } @@ -1974,7 +1985,7 @@ class TestFunctional(unittest.TestCase): def testVblockContent(self): """Test that the vblock signs the right data""" self._hash_data = True - command.test_result = self._HandleVblockCommand + command.TEST_RESULT = self._HandleVblockCommand entry_args = { 'keydir': 'devkeys', } @@ -5498,7 +5509,7 @@ fdt fdtmap Extract the devicetree blob from the fdtmap def testFitSubentryUsesBintool(self): """Test that binman FIT subentries can use bintools""" - command.test_result = self._HandleGbbCommand + command.TEST_RESULT = self._HandleGbbCommand entry_args = { 'keydir': 'devkeys', 'bmpblk': 'bmpblk.bin', @@ -5783,9 +5794,14 @@ fdt fdtmap Extract the devicetree blob from the fdtmap data = self._DoReadFileDtb( '230_pre_load.dts', entry_args=entry_args, extra_indirs=[os.path.join(self._binman_dir, 'test')])[0] + + image_fname = tools.get_output_filename('image.bin') + is_signed = self._CheckPreload(image_fname, self.TestFile("dev.key")) + self.assertEqual(PRE_LOAD_MAGIC, data[:len(PRE_LOAD_MAGIC)]) self.assertEqual(PRE_LOAD_VERSION, data[4:4 + len(PRE_LOAD_VERSION)]) self.assertEqual(PRE_LOAD_HDR_SIZE, data[8:8 + len(PRE_LOAD_HDR_SIZE)]) + self.assertEqual(is_signed, True) def testPreLoadNoKey(self): """Test an image with a pre-load heade0r with missing key""" @@ -6383,6 +6399,7 @@ fdt fdtmap Extract the devicetree blob from the fdtmap ename, prop = entry_m.group(1), entry_m.group(3) entry, entry_name, prop_name = image.LookupEntry(entries, name, msg) + expect_val = None if prop_name == 'offset': expect_val = entry.offset elif prop_name == 'image_pos': diff --git a/tools/binman/requirements.txt b/tools/binman/requirements.txt new file mode 100644 index 00000000000..f068ef75a30 --- /dev/null +++ b/tools/binman/requirements.txt @@ -0,0 +1,5 @@ +importlib_resources==6.5.2 +jsonschema==4.23.0 +pycryptodomex==3.21.0 +pyelftools==0.31 +yamllint==1.35.1 diff --git a/tools/binman/setup.py b/tools/binman/setup.py index 9a9206eb044..bec078a3d9b 100644 --- a/tools/binman/setup.py +++ b/tools/binman/setup.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0+ -from distutils.core import setup +from setuptools import setup setup(name='binman', version='1.0', license='GPL-2.0+', diff --git a/tools/binman/state.py b/tools/binman/state.py index 45bae40c525..6772d3678fe 100644 --- a/tools/binman/state.py +++ b/tools/binman/state.py @@ -406,10 +406,13 @@ def CheckSetHashValue(node, get_data_func): hash_node = node.FindNode('hash') if hash_node: algo = hash_node.props.get('algo').value + data = None if algo == 'sha256': m = hashlib.sha256() m.update(get_data_func()) data = m.digest() + if data is None: + raise ValueError(f"Node '{node.path}': Unknown hash algorithm '{algo}'") for n in GetUpdateNodes(hash_node): n.SetData('value', data) diff --git a/tools/buildman/boards.py b/tools/buildman/boards.py index e7aa0d85a58..2fe43c3fc89 100644 --- a/tools/buildman/boards.py +++ b/tools/buildman/boards.py @@ -251,9 +251,9 @@ class KconfigScanner: '-undef', '-x', 'assembler-with-cpp', defconfig] - result = command.run_pipe([cmd], capture=True, capture_stderr=True) + stdout = command.output(*cmd, capture_stderr=True) temp = tempfile.NamedTemporaryFile(prefix='buildman-') - tools.write_file(temp.name, result.stdout, False) + tools.write_file(temp.name, stdout, False) fname = temp.name tout.info(f'Processing #include to produce {defconfig}') else: diff --git a/tools/buildman/builder.py b/tools/buildman/builder.py index cbf1345281b..4bea0a02b78 100644 --- a/tools/buildman/builder.py +++ b/tools/buildman/builder.py @@ -19,8 +19,8 @@ import time from buildman import builderthread from buildman import toolchain -from patman import gitutil from u_boot_pylib import command +from u_boot_pylib import gitutil from u_boot_pylib import terminal from u_boot_pylib import tools from u_boot_pylib.terminal import tprint @@ -510,7 +510,7 @@ class Builder: stage: Stage that we are at (mrproper, config, oldconfig, build) cwd: Directory where make should be run args: Arguments to pass to make - kwargs: Arguments to pass to command.run_pipe() + kwargs: Arguments to pass to command.run_one() """ def check_output(stream, data): @@ -531,11 +531,12 @@ class Builder: return False self._restarting_config = False - self._terminated = False + self._terminated = False cmd = [self.gnu_make] + list(args) - result = command.run_pipe([cmd], capture=True, capture_stderr=True, - cwd=cwd, raise_on_error=False, infile='/dev/null', - output_func=check_output, **kwargs) + result = command.run_one(*cmd, capture=True, capture_stderr=True, + cwd=cwd, raise_on_error=False, + infile='/dev/null', output_func=check_output, + **kwargs) if self._terminated: # Try to be helpful @@ -1095,14 +1096,13 @@ class Builder: diff = result[name] if name.startswith('_'): continue - if diff != 0: - color = self.col.RED if diff > 0 else self.col.GREEN + colour = self.col.RED if diff > 0 else self.col.GREEN msg = ' %s %+d' % (name, diff) if not printed_target: tprint('%10s %-15s:' % ('', result['_target']), newline=False) printed_target = True - tprint(msg, colour=color, newline=False) + tprint(msg, colour=colour, newline=False) if printed_target: tprint() if show_bloat: @@ -1353,6 +1353,7 @@ class Builder: for line in lines: if not line: continue + col = None if line[0] == '+': col = self.col.GREEN elif line[0] == '-': diff --git a/tools/buildman/builderthread.py b/tools/buildman/builderthread.py index 29e6cf32af1..b8578d5b97b 100644 --- a/tools/buildman/builderthread.py +++ b/tools/buildman/builderthread.py @@ -17,8 +17,8 @@ import sys import threading from buildman import cfgutil -from patman import gitutil from u_boot_pylib import command +from u_boot_pylib import gitutil from u_boot_pylib import tools RETURN_CODE_RETRY = -1 @@ -179,13 +179,12 @@ class BuilderThread(threading.Thread): cwd (str): Working directory to set, or None to leave it alone *args (list of str): Arguments to pass to 'make' **kwargs (dict): A list of keyword arguments to pass to - command.run_pipe() + command.run_one() Returns: CommandResult object """ - return self.builder.do_make(commit, brd, stage, cwd, *args, - **kwargs) + return self.builder.do_make(commit, brd, stage, cwd, *args, **kwargs) def _build_args(self, brd, out_dir, out_rel_dir, work_dir, commit_upto): """Set up arguments to the args list based on the settings @@ -588,9 +587,10 @@ class BuilderThread(threading.Thread): lines = [] for fname in BASE_ELF_FILENAMES: cmd = [f'{self.toolchain.cross}nm', '--size-sort', fname] - nm_result = command.run_pipe([cmd], capture=True, - capture_stderr=True, cwd=result.out_dir, - raise_on_error=False, env=env) + nm_result = command.run_one(*cmd, capture=True, + capture_stderr=True, + cwd=result.out_dir, + raise_on_error=False, env=env) if nm_result.stdout: nm_fname = self.builder.get_func_sizes_file( result.commit_upto, result.brd.target, fname) @@ -598,9 +598,10 @@ class BuilderThread(threading.Thread): print(nm_result.stdout, end=' ', file=outf) cmd = [f'{self.toolchain.cross}objdump', '-h', fname] - dump_result = command.run_pipe([cmd], capture=True, - capture_stderr=True, cwd=result.out_dir, - raise_on_error=False, env=env) + dump_result = command.run_one(*cmd, capture=True, + capture_stderr=True, + cwd=result.out_dir, + raise_on_error=False, env=env) rodata_size = '' if dump_result.stdout: objdump = self.builder.get_objdump_file(result.commit_upto, @@ -613,9 +614,10 @@ class BuilderThread(threading.Thread): rodata_size = fields[2] cmd = [f'{self.toolchain.cross}size', fname] - size_result = command.run_pipe([cmd], capture=True, - capture_stderr=True, cwd=result.out_dir, - raise_on_error=False, env=env) + size_result = command.run_one(*cmd, capture=True, + capture_stderr=True, + cwd=result.out_dir, + raise_on_error=False, env=env) if size_result.stdout: lines.append(size_result.stdout.splitlines()[1] + ' ' + rodata_size) @@ -624,9 +626,8 @@ class BuilderThread(threading.Thread): cmd = [f'{self.toolchain.cross}objcopy', '-O', 'binary', '-j', '.rodata.default_environment', 'env/built-in.o', 'uboot.env'] - command.run_pipe([cmd], capture=True, - capture_stderr=True, cwd=result.out_dir, - raise_on_error=False, env=env) + command.run_one(*cmd, capture=True, capture_stderr=True, + cwd=result.out_dir, raise_on_error=False, env=env) if not work_in_output: copy_files(result.out_dir, build_dir, '', ['uboot.env']) diff --git a/tools/buildman/control.py b/tools/buildman/control.py index 55d4d770c5c..5109b1cd5ce 100644 --- a/tools/buildman/control.py +++ b/tools/buildman/control.py @@ -20,9 +20,9 @@ from buildman import bsettings from buildman import cfgutil from buildman import toolchain from buildman.builder import Builder -from patman import gitutil from patman import patchstream from u_boot_pylib import command +from u_boot_pylib import gitutil from u_boot_pylib import terminal from u_boot_pylib import tools from u_boot_pylib.terminal import print_clear, tprint diff --git a/tools/buildman/func_test.py b/tools/buildman/func_test.py index 4e12c671a3d..d779040c74e 100644 --- a/tools/buildman/func_test.py +++ b/tools/buildman/func_test.py @@ -18,8 +18,8 @@ from buildman import bsettings from buildman import cmdline from buildman import control from buildman import toolchain -from patman import gitutil from u_boot_pylib import command +from u_boot_pylib import gitutil from u_boot_pylib import terminal from u_boot_pylib import test_util from u_boot_pylib import tools @@ -187,7 +187,7 @@ class TestFunctional(unittest.TestCase): self._git_dir = os.path.join(self._base_dir, 'src') self._buildman_pathname = sys.argv[0] self._buildman_dir = os.path.dirname(os.path.realpath(sys.argv[0])) - command.test_result = self._HandleCommand + command.TEST_RESULT = self._HandleCommand bsettings.setup(None) bsettings.add_file(settings_data) self.setupToolchains() @@ -232,8 +232,8 @@ class TestFunctional(unittest.TestCase): self._toolchains.Add('gcc', test=False) def _RunBuildman(self, *args): - return command.run_pipe([[self._buildman_pathname] + list(args)], - capture=True, capture_stderr=True) + all_args = [self._buildman_pathname] + list(args) + return command.run_one(*all_args, capture=True, capture_stderr=True) def _RunControl(self, *args, brds=False, clean_dir=False, test_thread_exceptions=False, get_builder=True): @@ -266,7 +266,7 @@ class TestFunctional(unittest.TestCase): return result def testFullHelp(self): - command.test_result = None + command.TEST_RESULT = None result = self._RunBuildman('-H') help_file = os.path.join(self._buildman_dir, 'README.rst') # Remove possible extraneous strings @@ -277,7 +277,7 @@ class TestFunctional(unittest.TestCase): self.assertEqual(0, result.return_code) def testHelp(self): - command.test_result = None + command.TEST_RESULT = None result = self._RunBuildman('-h') help_file = os.path.join(self._buildman_dir, 'README.rst') self.assertTrue(len(result.stdout) > 1000) @@ -286,11 +286,11 @@ class TestFunctional(unittest.TestCase): def testGitSetup(self): """Test gitutils.Setup(), from outside the module itself""" - command.test_result = command.CommandResult(return_code=1) + command.TEST_RESULT = command.CommandResult(return_code=1) gitutil.setup() self.assertEqual(gitutil.use_no_decorate, False) - command.test_result = command.CommandResult(return_code=0) + command.TEST_RESULT = command.CommandResult(return_code=0) gitutil.setup() self.assertEqual(gitutil.use_no_decorate, True) @@ -445,7 +445,7 @@ class TestFunctional(unittest.TestCase): stage: Stage that we are at (mrproper, config, build) cwd: Directory where make should be run args: Arguments to pass to make - kwargs: Arguments to pass to command.run_pipe() + kwargs: Arguments to pass to command.run_one() """ self._make_calls += 1 out_dir = '' diff --git a/tools/buildman/main.py b/tools/buildman/main.py index a948f36d9c0..72571b226d9 100755 --- a/tools/buildman/main.py +++ b/tools/buildman/main.py @@ -50,8 +50,7 @@ def run_tests(skip_net_tests, debug, verbose, args): # 'entry' module. result = test_util.run_test_suites( 'buildman', debug, verbose, False, args.threads, test_name, [], - [test.TestBuild, func_test.TestFunctional, - 'buildman.toolchain', 'patman.gitutil']) + [test.TestBuild, func_test.TestFunctional, 'buildman.toolchain']) return (0 if result.wasSuccessful() else 1) diff --git a/tools/buildman/requirements.txt b/tools/buildman/requirements.txt index 052d0ed5c6f..d48650cd1e5 100644 --- a/tools/buildman/requirements.txt +++ b/tools/buildman/requirements.txt @@ -1,5 +1,2 @@ -coverage==6.2 -jsonschema==4.17.3 -pycryptodome==3.20 -pyyaml==6.0 -yamllint==1.26.3 +filelock==3.0.12 +importlib_resources==6.5.2 diff --git a/tools/buildman/test.py b/tools/buildman/test.py index 385a34e5254..c5feb74a105 100644 --- a/tools/buildman/test.py +++ b/tools/buildman/test.py @@ -836,6 +836,7 @@ class TestBuild(unittest.TestCase): tmpdir = self.base_dir with (patch('time.time', side_effect=self.get_time), + patch('time.perf_counter', side_effect=self.get_time), patch('time.monotonic', side_effect=self.get_time), patch('time.sleep', side_effect=self.inc_time), patch('os.kill', side_effect=self.kill)): diff --git a/tools/buildman/toolchain.py b/tools/buildman/toolchain.py index 958f36f9f61..5d051e005da 100644 --- a/tools/buildman/toolchain.py +++ b/tools/buildman/toolchain.py @@ -100,7 +100,7 @@ class Toolchain: else: self.priority = priority if test: - result = command.run_pipe([cmd], capture=True, env=env, + result = command.run_one(*cmd, capture=True, env=env, raise_on_error=False) self.ok = result.return_code == 0 if verbose: diff --git a/tools/docker/Dockerfile b/tools/docker/Dockerfile index d2848ab85f3..85d67848327 100644 --- a/tools/docker/Dockerfile +++ b/tools/docker/Dockerfile @@ -122,7 +122,6 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ python3 \ python3-dev \ python3-pip \ - python3-pyelftools \ python3-sphinx \ python3-virtualenv \ rpm2cpio \ @@ -308,12 +307,18 @@ USER uboot:uboot # COPY / ADD directives don't work as we need them to. RUN wget -O /tmp/pytest-requirements.txt https://source.denx.de/u-boot/u-boot/-/raw/master/test/py/requirements.txt RUN wget -O /tmp/sphinx-requirements.txt https://source.denx.de/u-boot/u-boot/-/raw/master/doc/sphinx/requirements.txt +RUN wget -O /tmp/binman-requirements.txt https://source.denx.de/u-boot/u-boot/-/raw/master/tools/binman/requirements.txt RUN wget -O /tmp/buildman-requirements.txt https://source.denx.de/u-boot/u-boot/-/raw/master/tools/buildman/requirements.txt +RUN wget -O /tmp/patman-requirements.txt https://source.denx.de/u-boot/u-boot/-/raw/master/tools/patman/requirements.txt +RUN wget -O /tmp/u_boot_pylib-requirements.txt https://source.denx.de/u-boot/u-boot/-/raw/master/tools/u_boot_pylib/requirements.txt RUN virtualenv -p /usr/bin/python3 /tmp/venv && \ . /tmp/venv/bin/activate && \ pip install -r /tmp/pytest-requirements.txt \ -r /tmp/sphinx-requirements.txt \ - -r /tmp/buildman-requirements.txt && \ + -r /tmp/binman-requirements.txt \ + -r /tmp/buildman-requirements.txt \ + -r /tmp/patman-requirements.txt \ + -r /tmp/u_boot_pylib-requirements.txt && \ deactivate && \ rm -rf /tmp/venv /tmp/*-requirements.txt diff --git a/tools/dtoc/setup.py b/tools/dtoc/setup.py index 5e092fe0872..ae9ad043b01 100644 --- a/tools/dtoc/setup.py +++ b/tools/dtoc/setup.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0+ -from distutils.core import setup +from setuptools import setup setup(name='dtoc', version='1.0', license='GPL-2.0+', diff --git a/tools/image-host.c b/tools/image-host.c index e6de34fa059..a9b86902763 100644 --- a/tools/image-host.c +++ b/tools/image-host.c @@ -19,6 +19,11 @@ #include <openssl/evp.h> #endif +#if CONFIG_IS_ENABLED(IMAGE_PRE_LOAD) +#include <openssl/rsa.h> +#include <openssl/err.h> +#endif + /** * fit_set_hash_value - set hash value in requested has node * @fit: pointer to the FIT format image header @@ -729,11 +734,20 @@ static int strlist_add(struct strlist *list, const char *str) { char *dup; + if (!list || !str) + return -1; + dup = strdup(str); + if(!dup) + return -1; + list->strings = realloc(list->strings, (list->count + 1) * sizeof(char *)); - if (!list || !str) + if (!list->strings) { + free(dup); return -1; + } + list->strings[list->count++] = dup; return 0; @@ -1401,3 +1415,139 @@ int fit_check_sign(const void *fit, const void *key, return ret; } #endif + +#if CONFIG_IS_ENABLED(IMAGE_PRE_LOAD) +/** + * rsa_verify_openssl() - Verify a signature against some data with openssl API + * + * Verify a RSA PKCS1.5/PSS signature against an expected hash. + * + * @info: Specifies the key and algorithms + * @region: Pointer to the input data + * @region_count: Number of region + * @sig: Signature + * @sig_len: Number of bytes in the signature + * Return: 0 if verified, -ve on error + */ +int rsa_verify_openssl(struct image_sign_info *info, + const struct image_region region[], int region_count, + uint8_t *sig, uint sig_len) +{ + EVP_PKEY *pkey = NULL; + EVP_PKEY_CTX *ckey = NULL; + EVP_MD_CTX *ctx = NULL; + int pad; + int size; + int i; + int ret = 0; + + if (!info) { + fprintf(stderr, "No info provided\n"); + ret = -EINVAL; + goto out; + } + + if (!info->key) { + fprintf(stderr, "No key provided\n"); + ret = -EINVAL; + goto out; + } + + if (!info->checksum) { + fprintf(stderr, "No checksum information\n"); + ret = -EINVAL; + goto out; + } + + if (!info->padding) { + fprintf(stderr, "No padding information\n"); + ret = -EINVAL; + goto out; + } + + if (region_count < 1) { + fprintf(stderr, "Invalid value for region_count: %d\n", region_count); + ret = -EINVAL; + goto out; + } + + pkey = (EVP_PKEY *)info->key; + + ckey = EVP_PKEY_CTX_new(pkey, NULL); + if (!ckey) { + ret = -ENOMEM; + fprintf(stderr, "EVK key context setup failed: %s\n", + ERR_error_string(ERR_get_error(), NULL)); + goto out; + } + + size = EVP_PKEY_size(pkey); + if (size > sig_len) { + fprintf(stderr, "Invalid signature size (%d bytes)\n", + size); + ret = -EINVAL; + goto out; + } + + ctx = EVP_MD_CTX_new(); + if (!ctx) { + ret = -ENOMEM; + fprintf(stderr, "EVP context creation failed: %s\n", + ERR_error_string(ERR_get_error(), NULL)); + goto out; + } + EVP_MD_CTX_init(ctx); + + if (EVP_DigestVerifyInit(ctx, &ckey, + EVP_get_digestbyname(info->checksum->name), + NULL, pkey) <= 0) { + ret = -EINVAL; + fprintf(stderr, "Verifier setup failed: %s\n", + ERR_error_string(ERR_get_error(), NULL)); + goto out; + } + + if (!strcmp(info->padding->name, "pkcs-1.5")) { + pad = RSA_PKCS1_PADDING; + } else if (!strcmp(info->padding->name, "pss")) { + pad = RSA_PKCS1_PSS_PADDING; + } else { + ret = -ENOMSG; + fprintf(stderr, "Unsupported padding: %s\n", + info->padding->name); + goto out; + } + + if (EVP_PKEY_CTX_set_rsa_padding(ckey, pad) <= 0) { + ret = -EINVAL; + fprintf(stderr, "padding setup has failed: %s\n", + ERR_error_string(ERR_get_error(), NULL)); + goto out; + } + + for (i=0 ; i < region_count ; ++i) { + if (EVP_DigestVerifyUpdate(ctx, region[i].data, + region[i].size) <= 0) { + ret = -EINVAL; + fprintf(stderr, "Hashing data failed: %s\n", + ERR_error_string(ERR_get_error(), NULL)); + goto out; + } + } + + if (EVP_DigestVerifyFinal(ctx, sig, sig_len) <= 0) { + ret = -EINVAL; + fprintf(stderr, "Verifying digest failed: %s\n", + ERR_error_string(ERR_get_error(), NULL)); + goto out; + } +out: + if (ctx) + EVP_MD_CTX_free(ctx); + + if (ret) + fprintf(stderr, "Failed to verify signature\n"); + + return ret; +} +#endif diff --git a/tools/imx8image.c b/tools/imx8image.c index 15510d3e712..0135b190951 100644 --- a/tools/imx8image.c +++ b/tools/imx8image.c @@ -290,6 +290,7 @@ static uint32_t parse_cfg_file(image_t *param_stack, char *name) } } + free(line); fclose(fd); return 0; } diff --git a/tools/imx8mimage.c b/tools/imx8mimage.c index d60d293e649..0f24ba75c0f 100644 --- a/tools/imx8mimage.c +++ b/tools/imx8mimage.c @@ -206,6 +206,7 @@ static uint32_t parse_cfg_file(char *name) } } + free(line); fclose(fd); return 0; } diff --git a/tools/imximage.c b/tools/imximage.c index 467d9f27d2a..55231caf8f3 100644 --- a/tools/imximage.c +++ b/tools/imximage.c @@ -783,6 +783,7 @@ static uint32_t parse_cfg_file(struct imx_header *imxhdr, char *name) } (*set_dcd_rst)(imxhdr, dcd_len, name, lineno); + free(line); fclose(fd); /* Exit if there is no BOOT_FROM field specifying the flash_offset */ diff --git a/tools/kwbimage.c b/tools/kwbimage.c index d1cbced28fc..3dcf5ba66b9 100644 --- a/tools/kwbimage.c +++ b/tools/kwbimage.c @@ -1653,6 +1653,12 @@ static int image_create_config_parse_oneline(char *line, char *unknown_msg = "Ignoring unknown line '%s'\n"; keyword = strtok_r(line, delimiters, &saveptr); + + if (!keyword) { + fprintf(stderr, "Parameter missing in line '%s'\n", line); + return -1; + } + keyword_id = recognize_keyword(keyword); if (!keyword_id) { diff --git a/tools/microcode-tool.py b/tools/microcode-tool.py index 24c02c4fca1..b726794751a 100755 --- a/tools/microcode-tool.py +++ b/tools/microcode-tool.py @@ -279,6 +279,9 @@ def MicrocodeTool(): if (not not options.mcfile) != (not not options.mcfile): parser.error("You must specify either header files or a microcode file, not both") + date = None + microcodes = None + license_text = None if options.headerfile: date, license_text, microcodes = ParseHeaderFiles(options.headerfile) elif options.mcfile: diff --git a/tools/mkimage.h b/tools/mkimage.h index 15741f250fd..5d6bcc9301a 100644 --- a/tools/mkimage.h +++ b/tools/mkimage.h @@ -37,6 +37,10 @@ static inline void *map_sysmem(ulong paddr, unsigned long len) return (void *)(uintptr_t)paddr; } +static inline void unmap_sysmem(const void *vaddr) +{ +} + static inline ulong map_to_sysmem(const void *ptr) { return (ulong)(uintptr_t)ptr; diff --git a/tools/patman/__init__.py b/tools/patman/__init__.py index 08eeffdf6d2..6de0e9fba10 100644 --- a/tools/patman/__init__.py +++ b/tools/patman/__init__.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: GPL-2.0+ __all__ = ['checkpatch', 'commit', 'control', 'func_test', 'get_maintainer', - 'gitutil', '__main__', 'patchstream', 'project', 'series', - 'settings','setup', 'status', 'test_checkpatch', 'test_settings'] + '__main__', 'patchstream', 'project', 'series', + 'settings', 'setup', 'status', 'test_checkpatch', 'test_settings'] diff --git a/tools/patman/__main__.py b/tools/patman/__main__.py index f645b38b647..36f1c08507c 100755 --- a/tools/patman/__main__.py +++ b/tools/patman/__main__.py @@ -49,7 +49,7 @@ def run_patman(): result = test_util.run_test_suites( 'patman', False, False, False, None, None, None, [test_checkpatch.TestPatch, func_test.TestFunctional, - 'gitutil', 'settings']) + 'settings']) sys.exit(0 if result.wasSuccessful() else 1) diff --git a/tools/patman/checkpatch.py b/tools/patman/checkpatch.py index e03cac115e4..2975881705c 100644 --- a/tools/patman/checkpatch.py +++ b/tools/patman/checkpatch.py @@ -8,8 +8,8 @@ import os import re import sys -from patman import gitutil from u_boot_pylib import command +from u_boot_pylib import gitutil from u_boot_pylib import terminal EMACS_PREFIX = r'(?:[0-9]{4}.*\.patch:[0-9]+: )?' diff --git a/tools/patman/cmdline.py b/tools/patman/cmdline.py index d6496c0cb78..562bc823f60 100644 --- a/tools/patman/cmdline.py +++ b/tools/patman/cmdline.py @@ -13,8 +13,8 @@ import os import pathlib import sys -from patman import gitutil from patman import project +from u_boot_pylib import gitutil from patman import settings PATMAN_DIR = pathlib.Path(__file__).parent diff --git a/tools/patman/control.py b/tools/patman/control.py index b292da9dc27..fb5a4246ced 100644 --- a/tools/patman/control.py +++ b/tools/patman/control.py @@ -12,8 +12,8 @@ import os import sys from patman import checkpatch -from patman import gitutil from patman import patchstream +from u_boot_pylib import gitutil from u_boot_pylib import terminal diff --git a/tools/patman/func_test.py b/tools/patman/func_test.py index af6c025a441..bf333dc557b 100644 --- a/tools/patman/func_test.py +++ b/tools/patman/func_test.py @@ -18,11 +18,11 @@ import unittest from patman.commit import Commit from patman import control -from patman import gitutil from patman import patchstream from patman.patchstream import PatchStream from patman.series import Series from patman import settings +from u_boot_pylib import gitutil from u_boot_pylib import terminal from u_boot_pylib import tools from u_boot_pylib.test_util import capture_sys_output diff --git a/tools/patman/get_maintainer.py b/tools/patman/get_maintainer.py index 8df3d124bac..200ee96551d 100644 --- a/tools/patman/get_maintainer.py +++ b/tools/patman/get_maintainer.py @@ -7,8 +7,8 @@ import os import shlex import shutil -from patman import gitutil from u_boot_pylib import command +from u_boot_pylib import gitutil def find_get_maintainer(script_file_name): diff --git a/tools/patman/patchstream.py b/tools/patman/patchstream.py index 4955f6aaab9..490d382045b 100644 --- a/tools/patman/patchstream.py +++ b/tools/patman/patchstream.py @@ -15,9 +15,9 @@ import shutil import tempfile from patman import commit -from patman import gitutil from patman.series import Series from u_boot_pylib import command +from u_boot_pylib import gitutil # Tags that we detect and remove RE_REMOVE = re.compile(r'^BUG=|^TEST=|^BRANCH=|^Review URL:' @@ -711,7 +711,7 @@ def get_list(commit_range, git_dir=None, count=None): """ params = gitutil.log_cmd(commit_range, reverse=True, count=count, git_dir=git_dir) - return command.run_pipe([params], capture=True).stdout + return command.run_one(*params, capture=True).stdout def get_metadata_for_list(commit_range, git_dir=None, count=None, series=None, allow_overwrite=False): diff --git a/tools/patman/project.py b/tools/patman/project.py index 4459042b5d4..d6143a67066 100644 --- a/tools/patman/project.py +++ b/tools/patman/project.py @@ -4,7 +4,7 @@ import os.path -from patman import gitutil +from u_boot_pylib import gitutil def detect_project(): """Autodetect the name of the current project. diff --git a/tools/patman/requirements.txt b/tools/patman/requirements.txt new file mode 100644 index 00000000000..e8cbc6cf0c3 --- /dev/null +++ b/tools/patman/requirements.txt @@ -0,0 +1,5 @@ +ConfigParser==7.1.0 +importlib_resources==6.5.2 +pygit2==1.13.3 +Requests==2.32.3 +setuptools==75.8.0 diff --git a/tools/patman/series.py b/tools/patman/series.py index 6866e1dbd08..d7f2f010f5d 100644 --- a/tools/patman/series.py +++ b/tools/patman/series.py @@ -12,8 +12,8 @@ import sys import time from patman import get_maintainer -from patman import gitutil from patman import settings +from u_boot_pylib import gitutil from u_boot_pylib import terminal from u_boot_pylib import tools diff --git a/tools/patman/settings.py b/tools/patman/settings.py index 68c93e313b3..d66b22be1df 100644 --- a/tools/patman/settings.py +++ b/tools/patman/settings.py @@ -12,7 +12,7 @@ import argparse import os import re -from patman import gitutil +from u_boot_pylib import gitutil """Default settings per-project. diff --git a/tools/patman/test_checkpatch.py b/tools/patman/test_checkpatch.py index db7860f551d..3bf16febbf6 100644 --- a/tools/patman/test_checkpatch.py +++ b/tools/patman/test_checkpatch.py @@ -11,10 +11,10 @@ import tempfile import unittest from patman import checkpatch -from patman import gitutil from patman import patchstream from patman import series from patman import commit +from u_boot_pylib import gitutil class Line: @@ -530,4 +530,3 @@ index 0000000..2234c87 if __name__ == "__main__": unittest.main() - gitutil.RunTests() diff --git a/tools/preload_check_sign.c b/tools/preload_check_sign.c new file mode 100644 index 00000000000..ebead459273 --- /dev/null +++ b/tools/preload_check_sign.c @@ -0,0 +1,160 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Check a file including a preload header including a signature + * + * Copyright (c) 2025 Paul HENRYS <paul.henrys_ext@softathome.com> + * + * Binman makes it possible to generate a preload header signing part or the + * complete file. The tool preload_check_sign allows to verify and authenticate + * a file starting with a preload header. + */ +#include <stdio.h> +#include <unistd.h> +#include <openssl/pem.h> +#include <openssl/evp.h> +#include <openssl/err.h> +#include <image.h> + +extern void image_pre_load_sig_set_info(struct image_sig_info *info); +extern int image_pre_load_sig(ulong addr); + +static void usage(char *cmdname) +{ + fprintf(stderr, "Usage: %s -f file -k PEM key file\n" + " -f ==> set file which should be checked\n" + " -k ==> PEM key file\n" + " -a ==> algo (default: sha256,rsa2048)\n" + " -p ==> padding (default: pkcs-1.5)\n" + " -h ==> help\n", + cmdname); + exit(EXIT_FAILURE); +} + +int main(int argc, char **argv) +{ + int ret = 0; + char cmdname[256]; + char *file = NULL; + char *keyfile = NULL; + int c; + FILE *fp = NULL; + FILE *fp_key = NULL; + size_t bytes; + long filesize; + void *buffer = NULL; + EVP_PKEY *pkey = NULL; + char *algo = "sha256,rsa2048"; + char *padding = "pkcs-1.5"; + struct image_sig_info info = {0}; + + strncpy(cmdname, *argv, sizeof(cmdname) - 1); + cmdname[sizeof(cmdname) - 1] = '\0'; + while ((c = getopt(argc, argv, "f:k:a:p:h")) != -1) + switch (c) { + case 'f': + file = optarg; + break; + case 'k': + keyfile = optarg; + break; + case 'a': + algo = optarg; + break; + case 'p': + padding = optarg; + break; + default: + usage(cmdname); + break; + } + + if (!file) { + fprintf(stderr, "%s: Missing file\n", *argv); + usage(*argv); + } + + if (!keyfile) { + fprintf(stderr, "%s: Missing key file\n", *argv); + usage(*argv); + } + + fp = fopen(file, "r"); + if (!fp) { + fprintf(stderr, "Error opening file: %s\n", file); + ret = EXIT_FAILURE; + goto out; + } + + fseek(fp, 0, SEEK_END); + filesize = ftell(fp); + rewind(fp); + + buffer = malloc(filesize); + if (!buffer) { + fprintf(stderr, "Memory allocation failed"); + ret = EXIT_FAILURE; + goto out; + } + + bytes = fread(buffer, 1, filesize, fp); + if (bytes != filesize) { + fprintf(stderr, "Error reading file\n"); + ret = EXIT_FAILURE; + goto out; + } + + fp_key = fopen(keyfile, "r"); + if (!fp_key) { + fprintf(stderr, "Error opening file: %s\n", keyfile); + ret = EXIT_FAILURE; + goto out; + } + + /* Attempt to read the private key */ + pkey = PEM_read_PrivateKey(fp_key, NULL, NULL, NULL); + if (!pkey) { + /* If private key reading fails, try reading as a public key */ + fseek(fp_key, 0, SEEK_SET); + pkey = PEM_read_PUBKEY(fp_key, NULL, NULL, NULL); + } + if (!pkey) { + fprintf(stderr, "Unable to retrieve the public key: %s\n", + ERR_error_string(ERR_get_error(), NULL)); + ret = EXIT_FAILURE; + goto out; + } + + info.algo_name = algo; + info.padding_name = padding; + info.key = (uint8_t *)pkey; + info.mandatory = 1; + info.sig_size = EVP_PKEY_size(pkey); + if (info.sig_size < 0) { + fprintf(stderr, "Fail to retrieve the signature size: %s\n", + ERR_error_string(ERR_get_error(), NULL)); + ret = EXIT_FAILURE; + goto out; + } + + /* Compute signature information */ + info.sig_info.name = info.algo_name; + info.sig_info.padding = image_get_padding_algo(info.padding_name); + info.sig_info.checksum = image_get_checksum_algo(info.sig_info.name); + info.sig_info.crypto = image_get_crypto_algo(info.sig_info.name); + info.sig_info.key = info.key; + info.sig_info.keylen = info.key_len; + + /* Check the signature */ + image_pre_load_sig_set_info(&info); + ret = image_pre_load_sig((ulong)buffer); +out: + if (fp) + fclose(fp); + if (fp_key) + fclose(fp_key); + if (info.key) + EVP_PKEY_free(pkey); + free(buffer); + + exit(ret); +} diff --git a/tools/proftool.c b/tools/proftool.c index af2cdb6d584..c7b427f3078 100644 --- a/tools/proftool.c +++ b/tools/proftool.c @@ -676,6 +676,7 @@ static int read_trace_config(FILE *fin) if (!tok) { error("Invalid trace config data on line %d\n", linenum); + free(line); return -1; } if (0 == strcmp(tok, "include-func")) { @@ -685,6 +686,7 @@ static int read_trace_config(FILE *fin) } else { error("Unknown command in trace config data line %d\n", linenum); + free(line); return -1; } @@ -692,6 +694,7 @@ static int read_trace_config(FILE *fin) if (!tok) { error("Missing pattern in trace config data line %d\n", linenum); + free(line); return -1; } diff --git a/tools/rmboard.py b/tools/rmboard.py index 0c56b149e0f..594fd89b8d7 100755 --- a/tools/rmboard.py +++ b/tools/rmboard.py @@ -43,18 +43,16 @@ def rm_kconfig_include(path): Args: path: Path to search for and remove """ - cmd = ['git', 'grep', path] - stdout = command.run_pipe([cmd], capture=True, raise_on_error=False).stdout + stdout = command.output('git', 'grep', path, raise_on_error=False) if not stdout: return fname = stdout.split(':')[0] print("Fixing up '%s' to remove reference to '%s'" % (fname, path)) - cmd = ['sed', '-i', '\|%s|d' % path, fname] - stdout = command.run_pipe([cmd], capture=True).stdout + stdout = command.run_one('sed', '-i', rf'\|{path}|d', fname, + capture=True).stdout - cmd = ['git', 'add', fname] - stdout = command.run_pipe([cmd], capture=True).stdout + stdout = command.output('git', 'add', fname) def rm_board(board): """Create a commit which removes a single board @@ -68,8 +66,7 @@ def rm_board(board): """ # Find all MAINTAINERS and Kconfig files which mention the board - cmd = ['git', 'grep', '-l', board] - stdout = command.run_pipe([cmd], capture=True).stdout + stdout = command.output('git', 'grep', '-l', board) maintain = [] kconfig = [] for line in stdout.splitlines(): @@ -109,16 +106,14 @@ def rm_board(board): # Search for Kconfig files in the resulting list. Remove any 'source' lines # which reference Kconfig files we want to remove for path in real: - cmd = ['find', path] - stdout = (command.run_pipe([cmd], capture=True, raise_on_error=False). - stdout) + stdout = command.output('find', path, raise_on_error=False) for fname in stdout.splitlines(): if fname.endswith('Kconfig'): rm_kconfig_include(fname) # Remove unwanted files cmd = ['git', 'rm', '-r'] + real - stdout = command.run_pipe([cmd], capture=True).stdout + stdout = command.output(*cmd, capture=True) ## Change the messages as needed msg = '''arm: Remove %s board @@ -131,13 +126,11 @@ Remove it. msg += 'Patch-cc: %s\n' % name # Create the commit - cmd = ['git', 'commit', '-s', '-m', msg] - stdout = command.run_pipe([cmd], capture=True).stdout + stdout = command.output('git', 'commit', '-s', '-m', msg) # Check if the board is mentioned anywhere else. The user will need to deal # with this - cmd = ['git', 'grep', '-il', board] - print(command.run_pipe([cmd], capture=True, raise_on_error=False).stdout) + print(command.output('git', 'grep', '-il', board, raise_on_error=False)) print(' '.join(cmd)) for board in sys.argv[1:]: diff --git a/tools/u_boot_pylib/__init__.py b/tools/u_boot_pylib/__init__.py index 63c88e85ec0..807a62e0743 100644 --- a/tools/u_boot_pylib/__init__.py +++ b/tools/u_boot_pylib/__init__.py @@ -1,4 +1,4 @@ # SPDX-License-Identifier: GPL-2.0+ -__all__ = ['command', 'cros_subprocess','terminal', 'test_util', 'tools', - 'tout'] +__all__ = ['command', 'cros_subprocess', 'gitutil', 'terminal', 'test_util', + 'tools', 'tout'] diff --git a/tools/u_boot_pylib/__main__.py b/tools/u_boot_pylib/__main__.py index 8f98d7bd9f8..c0762bca733 100755 --- a/tools/u_boot_pylib/__main__.py +++ b/tools/u_boot_pylib/__main__.py @@ -13,7 +13,6 @@ if __name__ == "__main__": sys.path.append(os.path.join(our_path, '..')) # Run tests - from u_boot_pylib import terminal from u_boot_pylib import test_util result = test_util.run_test_suites( diff --git a/tools/u_boot_pylib/command.py b/tools/u_boot_pylib/command.py index bbe95d86122..0e247355ef6 100644 --- a/tools/u_boot_pylib/command.py +++ b/tools/u_boot_pylib/command.py @@ -1,21 +1,44 @@ # SPDX-License-Identifier: GPL-2.0+ -# Copyright (c) 2011 The Chromium OS Authors. -# +""" +Shell command ease-ups for Python -import os +Copyright (c) 2011 The Chromium OS Authors. +""" + +import subprocess from u_boot_pylib import cros_subprocess -"""Shell command ease-ups for Python.""" +# This permits interception of RunPipe for test purposes. If it is set to +# a function, then that function is called with the pipe list being +# executed. Otherwise, it is assumed to be a CommandResult object, and is +# returned as the result for every run_pipe() call. +# When this value is None, commands are executed as normal. +TEST_RESULT = None + + +class CommandExc(Exception): + """Reports an exception to the caller""" + def __init__(self, msg, result): + """Set up a new exception object + + Args: + result (CommandResult): Execution result so far + """ + super().__init__(msg) + self.result = result + class CommandResult: """A class which captures the result of executing a command. Members: - stdout: stdout obtained from command, as a string - stderr: stderr obtained from command, as a string - return_code: Return code from command - exception: Exception received, or None if all ok + stdout (bytes): stdout obtained from command, as a string + stderr (bytes): stderr obtained from command, as a string + combined (bytes): stdout and stderr interleaved + return_code (int): Return code from command + exception (Exception): Exception received, or None if all ok + output (str or None): Returns output as a single line if requested """ def __init__(self, stdout='', stderr='', combined='', return_code=0, exception=None): @@ -24,8 +47,16 @@ class CommandResult: self.combined = combined self.return_code = return_code self.exception = exception + self.output = None def to_output(self, binary): + """Converts binary output to its final form + + Args: + binary (bool): True to report binary output, False to use strings + Returns: + self + """ if not binary: self.stdout = self.stdout.decode('utf-8') self.stderr = self.stderr.decode('utf-8') @@ -33,49 +64,47 @@ class CommandResult: return self -# This permits interception of RunPipe for test purposes. If it is set to -# a function, then that function is called with the pipe list being -# executed. Otherwise, it is assumed to be a CommandResult object, and is -# returned as the result for every run_pipe() call. -# When this value is None, commands are executed as normal. -test_result = None - -def run_pipe(pipe_list, infile=None, outfile=None, - capture=False, capture_stderr=False, oneline=False, - raise_on_error=True, cwd=None, binary=False, - output_func=None, **kwargs): +def run_pipe(pipe_list, infile=None, outfile=None, capture=False, + capture_stderr=False, oneline=False, raise_on_error=True, cwd=None, + binary=False, output_func=None, **kwargs): """ Perform a command pipeline, with optional input/output filenames. Args: - pipe_list: List of command lines to execute. Each command line is - piped into the next, and is itself a list of strings. For + pipe_list (list of list): List of command lines to execute. Each command + line is piped into the next, and is itself a list of strings. For example [ ['ls', '.git'] ['wc'] ] will pipe the output of 'ls .git' into 'wc'. - infile: File to provide stdin to the pipeline - outfile: File to store stdout - capture: True to capture output - capture_stderr: True to capture stderr - oneline: True to strip newline chars from output - output_func: Output function to call with each output fragment - (if it returns True the function terminates) - kwargs: Additional keyword arguments to cros_subprocess.Popen() + infile (str): File to provide stdin to the pipeline + outfile (str): File to store stdout + capture (bool): True to capture output + capture_stderr (bool): True to capture stderr + oneline (bool): True to strip newline chars from output + raise_on_error (bool): True to raise on an error, False to return it in + the CommandResult + cwd (str or None): Directory to run the command in + binary (bool): True to report binary output, False to use strings + output_func (function): Output function to call with each output + fragment (if it returns True the function terminates) + **kwargs: Additional keyword arguments to cros_subprocess.Popen() Returns: CommandResult object + Raises: + CommandExc if an exception happens """ - if test_result: - if hasattr(test_result, '__call__'): + if TEST_RESULT: + if hasattr(TEST_RESULT, '__call__'): # pylint: disable=E1102 - result = test_result(pipe_list=pipe_list) + result = TEST_RESULT(pipe_list=pipe_list) if result: return result else: - return test_result + return TEST_RESULT # No result: fall through to normal processing result = CommandResult(b'', b'', b'') last_pipe = None pipeline = list(pipe_list) - user_pipestr = '|'.join([' '.join(pipe) for pipe in pipe_list]) + user_pipestr = '|'.join([' '.join(pipe) for pipe in pipe_list]) kwargs['stdout'] = None kwargs['stderr'] = None while pipeline: @@ -96,7 +125,8 @@ def run_pipe(pipe_list, infile=None, outfile=None, except Exception as err: result.exception = err if raise_on_error: - raise Exception("Error running '%s': %s" % (user_pipestr, str)) + raise CommandExc(f"Error running '{user_pipestr}': {err}", + result) from err result.return_code = 255 return result.to_output(binary) @@ -107,31 +137,84 @@ def run_pipe(pipe_list, infile=None, outfile=None, result.output = result.stdout.rstrip(b'\r\n') result.return_code = last_pipe.wait() if raise_on_error and result.return_code: - raise Exception("Error running '%s'" % user_pipestr) + raise CommandExc(f"Error running '{user_pipestr}'", result) return result.to_output(binary) + def output(*cmd, **kwargs): + """Run a command and return its output + + Args: + *cmd (list of str): Command to run + **kwargs (dict of args): Extra arguments to pass in + + Returns: + str: command output + """ kwargs['raise_on_error'] = kwargs.get('raise_on_error', True) return run_pipe([cmd], capture=True, **kwargs).stdout + def output_one_line(*cmd, **kwargs): """Run a command and output it as a single-line string - The command us expected to produce a single line of output + The command is expected to produce a single line of output + + Args: + *cmd (list of str): Command to run + **kwargs (dict of args): Extra arguments to pass in Returns: - String containing output of command + str: output of command with all newlines removed """ raise_on_error = kwargs.pop('raise_on_error', True) result = run_pipe([cmd], capture=True, oneline=True, - raise_on_error=raise_on_error, **kwargs).stdout.strip() + raise_on_error=raise_on_error, **kwargs).stdout.strip() return result + def run(*cmd, **kwargs): + """Run a command + + Note that you must add 'capture' to kwargs to obtain non-empty output + + Args: + *cmd (list of str): Command to run + **kwargs (dict of args): Extra arguments to pass in + + Returns: + str: output of command + """ return run_pipe([cmd], **kwargs).stdout + +def run_one(*cmd, **kwargs): + """Run a single command + + Note that you must add 'capture' to kwargs to obtain non-empty output + + Args: + *cmd (list of str): Command to run + **kwargs (dict of args): Extra arguments to pass in + + Returns: + CommandResult: output of command + """ + return run_pipe([cmd], **kwargs) + + def run_list(cmd): + """Run a command and return its output + + Args: + cmd (list of str): Command to run + + Returns: + str: output of command + """ return run_pipe([cmd], capture=True).stdout + def stop_all(): + """Stop all subprocesses initiated with cros_subprocess""" cros_subprocess.stay_alive = False diff --git a/tools/patman/gitutil.py b/tools/u_boot_pylib/gitutil.py index 10ea5ff39f5..6d6a7eedecc 100644 --- a/tools/patman/gitutil.py +++ b/tools/u_boot_pylib/gitutil.py @@ -65,9 +65,9 @@ def count_commits_to_branch(branch): rev_range = '%s..%s' % (us, branch) else: rev_range = '@{upstream}..' - pipe = [log_cmd(rev_range, oneline=True)] - result = command.run_pipe(pipe, capture=True, capture_stderr=True, - oneline=True, raise_on_error=False) + cmd = log_cmd(rev_range, oneline=True) + result = command.run_one(*cmd, capture=True, capture_stderr=True, + oneline=True, raise_on_error=False) if result.return_code: raise ValueError('Failed to determine upstream: %s' % result.stderr.strip()) @@ -84,8 +84,7 @@ def name_revision(commit_hash): Return: Name of revision, if any, else None """ - pipe = ['git', 'name-rev', commit_hash] - stdout = command.run_pipe([pipe], capture=True, oneline=True).stdout + stdout = command.output_one_line('git', 'name-rev', commit_hash) # We expect a commit, a space, then a revision name name = stdout.split(' ')[1].strip() @@ -108,9 +107,9 @@ def guess_upstream(git_dir, branch): Name of upstream branch (e.g. 'upstream/master') or None if none Warning/error message, or None if none """ - pipe = [log_cmd(branch, git_dir=git_dir, oneline=True, count=100)] - result = command.run_pipe(pipe, capture=True, capture_stderr=True, - raise_on_error=False) + cmd = log_cmd(branch, git_dir=git_dir, oneline=True, count=100) + result = command.run_one(*cmd, capture=True, capture_stderr=True, + raise_on_error=False) if result.return_code: return None, "Branch '%s' not found" % branch for line in result.stdout.splitlines()[1:]: @@ -140,7 +139,7 @@ def get_upstream(git_dir, branch): 'branch.%s.remote' % branch) merge = command.output_one_line('git', '--git-dir', git_dir, 'config', 'branch.%s.merge' % branch) - except Exception: + except command.CommandExc: upstream, msg = guess_upstream(git_dir, branch) return upstream, msg @@ -183,9 +182,9 @@ def count_commits_in_range(git_dir, range_expr): Number of patches that exist in the supplied range or None if none were found """ - pipe = [log_cmd(range_expr, git_dir=git_dir, oneline=True)] - result = command.run_pipe(pipe, capture=True, capture_stderr=True, - raise_on_error=False) + cmd = log_cmd(range_expr, git_dir=git_dir, oneline=True) + result = command.run_one(*cmd, capture=True, capture_stderr=True, + raise_on_error=False) if result.return_code: return None, "Range '%s' not found or is invalid" % range_expr patch_count = len(result.stdout.splitlines()) @@ -250,9 +249,8 @@ def clone(git_dir, output_dir): Args: commit_hash: Commit hash to check out """ - pipe = ['git', 'clone', git_dir, '.'] - result = command.run_pipe([pipe], capture=True, cwd=output_dir, - capture_stderr=True) + result = command.run_one('git', 'clone', git_dir, '.', capture=True, + cwd=output_dir, capture_stderr=True) if result.return_code != 0: raise OSError('git clone: %s' % result.stderr) @@ -263,13 +261,13 @@ def fetch(git_dir=None, work_tree=None): Args: commit_hash: Commit hash to check out """ - pipe = ['git'] + cmd = ['git'] if git_dir: - pipe.extend(['--git-dir', git_dir]) + cmd.extend(['--git-dir', git_dir]) if work_tree: - pipe.extend(['--work-tree', work_tree]) - pipe.append('fetch') - result = command.run_pipe([pipe], capture=True, capture_stderr=True) + cmd.extend(['--work-tree', work_tree]) + cmd.append('fetch') + result = command.run_one(*cmd, capture=True, capture_stderr=True) if result.return_code != 0: raise OSError('git fetch: %s' % result.stderr) @@ -283,9 +281,9 @@ def check_worktree_is_available(git_dir): Returns: True if git-worktree commands will work, False otherwise. """ - pipe = ['git', '--git-dir', git_dir, 'worktree', 'list'] - result = command.run_pipe([pipe], capture=True, capture_stderr=True, - raise_on_error=False) + result = command.run_one('git', '--git-dir', git_dir, 'worktree', 'list', + capture=True, capture_stderr=True, + raise_on_error=False) return result.return_code == 0 @@ -298,11 +296,11 @@ def add_worktree(git_dir, output_dir, commit_hash=None): commit_hash: Commit hash to checkout """ # We need to pass --detach to avoid creating a new branch - pipe = ['git', '--git-dir', git_dir, 'worktree', 'add', '.', '--detach'] + cmd = ['git', '--git-dir', git_dir, 'worktree', 'add', '.', '--detach'] if commit_hash: - pipe.append(commit_hash) - result = command.run_pipe([pipe], capture=True, cwd=output_dir, - capture_stderr=True) + cmd.append(commit_hash) + result = command.run_one(*cmd, capture=True, cwd=output_dir, + capture_stderr=True) if result.return_code != 0: raise OSError('git worktree add: %s' % result.stderr) @@ -313,8 +311,8 @@ def prune_worktrees(git_dir): Args: git_dir: The repository whose deleted worktrees should be pruned """ - pipe = ['git', '--git-dir', git_dir, 'worktree', 'prune'] - result = command.run_pipe([pipe], capture=True, capture_stderr=True) + result = command.run_one('git', '--git-dir', git_dir, 'worktree', 'prune', + capture=True, capture_stderr=True) if result.return_code != 0: raise OSError('git worktree prune: %s' % result.stderr) @@ -687,7 +685,7 @@ def setup(): if alias_fname: settings.ReadGitAliases(alias_fname) cmd = log_cmd(None, count=0) - use_no_decorate = (command.run_pipe([cmd], raise_on_error=False) + use_no_decorate = (command.run_one(*cmd, raise_on_error=False) .return_code == 0) diff --git a/tools/u_boot_pylib/requirements.txt b/tools/u_boot_pylib/requirements.txt new file mode 100644 index 00000000000..1087e6f2857 --- /dev/null +++ b/tools/u_boot_pylib/requirements.txt @@ -0,0 +1 @@ +concurrencytest==0.1.2 diff --git a/tools/u_boot_pylib/tools.py b/tools/u_boot_pylib/tools.py index 0499a75526f..1afd289eadd 100644 --- a/tools/u_boot_pylib/tools.py +++ b/tools/u_boot_pylib/tools.py @@ -376,7 +376,7 @@ def run_result(name, *args, **kwargs): args = tuple(extra_args) + args name = os.path.expanduser(name) # Expand paths containing ~ all_args = (name,) + args - result = command.run_pipe([all_args], capture=True, capture_stderr=True, + result = command.run_one(*all_args, capture=True, capture_stderr=True, env=env, raise_on_error=False, binary=binary) if result.return_code: if raise_on_error: diff --git a/tools/ublimage.c b/tools/ublimage.c index 8f9b58c7983..a1bd807bfa0 100644 --- a/tools/ublimage.c +++ b/tools/ublimage.c @@ -178,6 +178,7 @@ static uint32_t parse_cfg_file(struct ubl_header *ublhdr, char *name) lineno, fld, &dcd_len); } } + free(line); fclose(fd); return dcd_len; |