Merge branch 'next'

Signed-off-by: Tom Rini <trini@konsulko.com>

11 files changed, 210 insertions, 53 deletions

diff --git a/tools/buildman/builder.py b/tools/buildman/builder.py
index c2a69027f88..d81752e9943 100644
--- a/tools/buildman/builder.py
+++ b/tools/buildman/builder.py
@@ -19,10 +19,10 @@ import time
 
 from buildman import builderthread
 from buildman import toolchain
-from patman import command
 from patman import gitutil
-from patman import terminal
-from patman.terminal import tprint
+from u_boot_pylib import command
+from u_boot_pylib import terminal
+from u_boot_pylib.terminal import tprint
 
 # This indicates an new int or hex Kconfig property with no default
 # It hangs the build since the 'conf' tool cannot proceed without valid input.
@@ -194,6 +194,8 @@ class Builder:
         work_in_output: Use the output directory as the work directory and
             don't write to a separate output directory.
         thread_exceptions: List of exceptions raised by thread jobs
+        no_lto (bool): True to set the NO_LTO flag when building
+        reproducible_builds (bool): True to set SOURCE_DATE_EPOCH=0 for builds
 
     Private members:
         _base_board_dict: Last-summarised Dict of boards
@@ -253,7 +255,7 @@ class Builder:
                  config_only=False, squash_config_y=False,
                  warnings_as_errors=False, work_in_output=False,
                  test_thread_exceptions=False, adjust_cfg=None,
-                 allow_missing=False):
+                 allow_missing=False, no_lto=False, reproducible_builds=False):
         """Create a new Builder object
 
         Args:
@@ -292,6 +294,7 @@ class Builder:
                     C=val to set the value of C (val must have quotes if C is
                         a string Kconfig
             allow_missing: Run build with BINMAN_ALLOW_MISSING=1
+            no_lto (bool): True to set the NO_LTO flag when building
 
         """
         self.toolchains = toolchains
@@ -331,6 +334,8 @@ class Builder:
         self.adjust_cfg = adjust_cfg
         self.allow_missing = allow_missing
         self._ide = False
+        self.no_lto = no_lto
+        self.reproducible_builds = reproducible_builds
 
         if not self.squash_config_y:
             self.config_filenames += EXTRA_CONFIG_FILENAMES
diff --git a/tools/buildman/builderthread.py b/tools/buildman/builderthread.py
index 680efae02d7..879ff138ad7 100644
--- a/tools/buildman/builderthread.py
+++ b/tools/buildman/builderthread.py
@@ -10,8 +10,8 @@ import sys
 import threading
 
 from buildman import cfgutil
-from patman import command
 from patman import gitutil
+from u_boot_pylib import command
 
 RETURN_CODE_RETRY = -1
 BASE_ELF_FILENAMES = ['u-boot', 'spl/u-boot-spl', 'tpl/u-boot-tpl']
@@ -255,6 +255,10 @@ class BuilderThread(threading.Thread):
                     args.append('KCFLAGS=-Werror')
                 if self.builder.allow_missing:
                     args.append('BINMAN_ALLOW_MISSING=1')
+                if self.builder.no_lto:
+                    args.append('NO_LTO=1')
+                if self.builder.reproducible_builds:
+                    args.append('SOURCE_DATE_EPOCH=0')
                 config_args = ['%s_defconfig' % brd.target]
                 config_out = ''
                 args.extend(self.builder.toolchains.GetMakeArguments(brd))
@@ -273,14 +277,19 @@ class BuilderThread(threading.Thread):
 
                 # If we need to reconfigure, do that now
                 cfg_file = os.path.join(out_dir, '.config')
+                cmd_list = []
                 if do_config or adjust_cfg:
                     config_out = ''
                     if self.mrproper:
                         result = self.Make(commit, brd, 'mrproper', cwd,
                                 'mrproper', *args, env=env)
                         config_out += result.combined
+                        cmd_list.append([self.builder.gnu_make, 'mrproper',
+                                         *args])
                     result = self.Make(commit, brd, 'config', cwd,
                             *(args + config_args), env=env)
+                    cmd_list.append([self.builder.gnu_make] + args +
+                                    config_args)
                     config_out += result.combined
                     do_config = False   # No need to configure next time
                     if adjust_cfg:
@@ -290,6 +299,7 @@ class BuilderThread(threading.Thread):
                         args.append('cfg')
                     result = self.Make(commit, brd, 'build', cwd, *args,
                             env=env)
+                    cmd_list.append([self.builder.gnu_make] + args)
                     if (result.return_code == 2 and
                         ('Some images are invalid' in result.stderr)):
                         # This is handled later by the check for output in
@@ -303,6 +313,7 @@ class BuilderThread(threading.Thread):
                 result.stderr = result.stderr.replace(src_dir + '/', '')
                 if self.builder.verbose_build:
                     result.stdout = config_out + result.stdout
+                result.cmd_list = cmd_list
             else:
                 result.return_code = 1
                 result.stderr = 'No tool chain for %s\n' % brd.arch
@@ -378,6 +389,12 @@ class BuilderThread(threading.Thread):
             with open(os.path.join(build_dir, 'out-env'), 'wb') as fd:
                 for var in sorted(env.keys()):
                     fd.write(b'%s="%s"' % (var, env[var]))
+
+            with open(os.path.join(build_dir, 'out-cmd'), 'w',
+                      encoding='utf-8') as fd:
+                for cmd in result.cmd_list:
+                    print(' '.join(cmd), file=fd)
+
             lines = []
             for fname in BASE_ELF_FILENAMES:
                 cmd = ['%snm' % self.toolchain.cross, '--size-sort', fname]
diff --git a/tools/buildman/buildman.rst b/tools/buildman/buildman.rst
index 2a83cb7e4f8..c8b0db3d8b9 100644
--- a/tools/buildman/buildman.rst
+++ b/tools/buildman/buildman.rst
@@ -1023,14 +1023,15 @@ U-Boot's build system embeds information such as a build timestamp into the
 final binary. This information varies each time U-Boot is built. This causes
 various files to be rebuilt even if no source changes are made, which in turn
 requires that the final U-Boot binary be re-linked. This unnecessary work can
-be avoided by turning off the timestamp feature. This can be achieved by
-setting the SOURCE_DATE_EPOCH environment variable to 0.
+be avoided by turning off the timestamp feature. This can be achieved using
+the `-r` flag, which enables reproducible builds by setting
+`SOURCE_DATE_EPOCH=0` when building.
 
 Combining all of these options together yields the command-line shown below.
 This will provide the quickest possible feedback regarding the current content
 of the source tree, thus allowing rapid tested evolution of the code::
 
-    SOURCE_DATE_EPOCH=0 ./tools/buildman/buildman -P tegra
+    ./tools/buildman/buildman -Pr tegra
 
 
 Checking configuration
@@ -1108,6 +1109,8 @@ and 'brppt1_spi', removing a trailing semicolon. 'brppt1_nand' gained an a
 value for 'altbootcmd', but lost one for ' altbootcmd'.
 
 The -U option uses the u-boot.env files which are produced by a build.
+Internally, buildman writes out an out-env file into the build directory for
+later comparison.
 
 
 Building with clang
@@ -1121,6 +1124,20 @@ toolchain. For example:
    buildman -O clang-7 --board sandbox
 
 
+Building without LTO
+--------------------
+
+Link-time optimisation (LTO) is designed to reduce code size by globally
+optimising the U-Boot build. Unfortunately this can dramatically slow down
+builds. This is particularly noticeable when running a lot of builds.
+
+Use the -L (--no-lto) flag to disable LTO.
+
+.. code-block:: bash
+
+   buildman -L --board sandbox
+
+
 Doing a simple build
 --------------------
 
@@ -1298,6 +1315,14 @@ You should use 'buildman -nv <criteria>' instead of greoing the boards.cfg file,
 since it may be dropped altogether in future.
 
 
+Checking the command
+--------------------
+
+Buildman writes out the toolchain information to a `toolchain` file within the
+output directory. It also writes the commands used to build U-Boot in an
+`out-cmd` file. You can check these if you suspect something strange is
+happening.
+
 TODO
 ----
 
diff --git a/tools/buildman/cfgutil.py b/tools/buildman/cfgutil.py
index ab74a8ef062..a340e01cb6b 100644
--- a/tools/buildman/cfgutil.py
+++ b/tools/buildman/cfgutil.py
@@ -7,7 +7,7 @@
 
 import re
 
-from patman import tools
+from u_boot_pylib import tools
 
 RE_LINE = re.compile(r'(# )?CONFIG_([A-Z0-9_]+)(=(.*)| is not set)')
 RE_CFG = re.compile(r'(~?)(CONFIG_)?([A-Z0-9_]+)(=.*)?')
diff --git a/tools/buildman/cmdline.py b/tools/buildman/cmdline.py
index c485994e9fe..a9cda249572 100644
--- a/tools/buildman/cmdline.py
+++ b/tools/buildman/cmdline.py
@@ -3,6 +3,11 @@
 #
 
 from optparse import OptionParser
+import os
+import pathlib
+
+BUILDMAN_DIR = pathlib.Path(__file__).parent
+HAS_TESTS = os.path.exists(BUILDMAN_DIR / "test.py")
 
 def ParseArgs():
     """Parse command line arguments from sys.argv[]
@@ -71,6 +76,8 @@ def ParseArgs():
           default=False, help="Don't convert y to 1 in configs")
     parser.add_option('-l', '--list-error-boards', action='store_true',
           default=False, help='Show a list of boards next to each error/warning')
+    parser.add_option('-L', '--no-lto', action='store_true',
+          default=False, help='Disable Link-time Optimisation (LTO) for builds')
     parser.add_option('--list-tool-chains', action='store_true', default=False,
           help='List available tool chains (use -v to see probing detail)')
     parser.add_option('-m', '--mrproper', action='store_true',
@@ -95,18 +102,21 @@ def ParseArgs():
           default=False, help="Use full toolchain path in CROSS_COMPILE")
     parser.add_option('-P', '--per-board-out-dir', action='store_true',
           default=False, help="Use an O= (output) directory per board rather than per thread")
+    parser.add_option('-r', '--reproducible-builds', action='store_true',
+          help='Set SOURCE_DATE_EPOCH=0 to suuport a reproducible build')
     parser.add_option('-R', '--regen-board-list', action='store_true',
           help='Force regeneration of the list of boards, like the old boards.cfg file')
     parser.add_option('-s', '--summary', action='store_true',
           default=False, help='Show a build summary')
     parser.add_option('-S', '--show-sizes', action='store_true',
           default=False, help='Show image size variation in summary')
-    parser.add_option('--skip-net-tests', action='store_true', default=False,
-                      help='Skip tests which need the network')
     parser.add_option('--step', type='int',
           default=1, help='Only build every n commits (0=just first and last)')
-    parser.add_option('-t', '--test', action='store_true', dest='test',
-                      default=False, help='run tests')
+    if HAS_TESTS:
+        parser.add_option('--skip-net-tests', action='store_true', default=False,
+                          help='Skip tests which need the network')
+        parser.add_option('-t', '--test', action='store_true', dest='test',
+                          default=False, help='run tests')
     parser.add_option('-T', '--threads', type='int',
           default=None,
           help='Number of builder threads to use (0=single-thread)')
diff --git a/tools/buildman/control.py b/tools/buildman/control.py
index 87e7d0e2012..35f44c0cf3d 100644
--- a/tools/buildman/control.py
+++ b/tools/buildman/control.py
@@ -3,6 +3,7 @@
 #
 
 import multiprocessing
+import importlib.resources
 import os
 import shutil
 import subprocess
@@ -13,12 +14,12 @@ from buildman import bsettings
 from buildman import cfgutil
 from buildman import toolchain
 from buildman.builder import Builder
-from patman import command
 from patman import gitutil
 from patman import patchstream
-from patman import terminal
-from patman import tools
-from patman.terminal import tprint
+from u_boot_pylib import command
+from u_boot_pylib import terminal
+from u_boot_pylib import tools
+from u_boot_pylib.terminal import tprint
 
 def GetPlural(count):
     """Returns a plural 's' if count is not 1"""
@@ -152,9 +153,8 @@ def DoBuildman(options, args, toolchains=None, make_func=None, brds=None,
     global builder
 
     if options.full_help:
-        tools.print_full_help(
-            os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])),
-                         'README.rst'))
+        with importlib.resources.path('buildman', 'README.rst') as readme:
+            tools.print_full_help(str(readme))
         return 0
 
     gitutil.setup()
@@ -261,9 +261,9 @@ def DoBuildman(options, args, toolchains=None, make_func=None, brds=None,
             count += 1   # Build upstream commit also
 
     if not count:
-        str = ("No commits found to process in branch '%s': "
+        msg = ("No commits found to process in branch '%s': "
                "set branch's upstream or use -c flag" % options.branch)
-        sys.exit(col.build(col.RED, str))
+        sys.exit(col.build(col.RED, msg))
     if options.work_in_output:
         if len(selected) != 1:
             sys.exit(col.build(col.RED,
@@ -338,6 +338,14 @@ def DoBuildman(options, args, toolchains=None, make_func=None, brds=None,
             shutil.rmtree(output_dir)
     adjust_cfg = cfgutil.convert_list_to_dict(options.adjust_cfg)
 
+    # Drop LOCALVERSION_AUTO since it changes the version string on every commit
+    if options.reproducible_builds:
+        # If these are mentioned, leave the local version alone
+        if 'LOCALVERSION' in adjust_cfg or 'LOCALVERSION_AUTO' in adjust_cfg:
+            print('Not dropping LOCALVERSION_AUTO for reproducible build')
+        else:
+            adjust_cfg['LOCALVERSION_AUTO'] = '~'
+
     builder = Builder(toolchains, output_dir, options.git_dir,
             options.threads, options.jobs, gnu_make=gnu_make, checkout=True,
             show_unknown=options.show_unknown, step=options.step,
@@ -351,7 +359,8 @@ def DoBuildman(options, args, toolchains=None, make_func=None, brds=None,
             work_in_output=options.work_in_output,
             test_thread_exceptions=test_thread_exceptions,
             adjust_cfg=adjust_cfg,
-            allow_missing=allow_missing)
+            allow_missing=allow_missing, no_lto=options.no_lto,
+            reproducible_builds=options.reproducible_builds)
     builder.force_config_on_failure = not options.quick
     if make_func:
         builder.do_make = make_func
diff --git a/tools/buildman/func_test.py b/tools/buildman/func_test.py
index 559e4edf74b..ebd78f225e1 100644
--- a/tools/buildman/func_test.py
+++ b/tools/buildman/func_test.py
@@ -14,11 +14,11 @@ from buildman import bsettings
 from buildman import cmdline
 from buildman import control
 from buildman import toolchain
-from patman import command
 from patman import gitutil
-from patman import terminal
-from patman import test_util
-from patman import tools
+from u_boot_pylib import command
+from u_boot_pylib import terminal
+from u_boot_pylib import test_util
+from u_boot_pylib import tools
 
 settings_data = '''
 # Buildman settings file
@@ -415,17 +415,19 @@ class TestFunctional(unittest.TestCase):
             kwargs: Arguments to pass to command.run_pipe()
         """
         self._make_calls += 1
+        out_dir = ''
+        for arg in args:
+            if arg.startswith('O='):
+                out_dir = arg[2:]
         if stage == 'mrproper':
             return command.CommandResult(return_code=0)
         elif stage == 'config':
+            fname = os.path.join(cwd or '', out_dir, '.config')
+            tools.write_file(fname, b'CONFIG_SOMETHING=1')
             return command.CommandResult(return_code=0,
                     combined='Test configuration complete')
         elif stage == 'build':
             stderr = ''
-            out_dir = ''
-            for arg in args:
-                if arg.startswith('O='):
-                    out_dir = arg[2:]
             fname = os.path.join(cwd or '', out_dir, 'u-boot')
             tools.write_file(fname, b'U-Boot')
 
@@ -723,3 +725,57 @@ Some images are invalid'''
                          control.get_allow_missing(False, False, 2, True))
         self.assertEqual(False,
                          control.get_allow_missing(False, True, 2, True))
+
+    def check_command(self, *extra_args):
+        """Run a command with the extra arguments and return the commands used
+
+        Args:
+            extra_args (list of str): List of extra arguments
+
+        Returns:
+            list of str: Lines returned in the out-cmd file
+        """
+        self._RunControl('-o', self._output_dir, *extra_args)
+        board0_dir = os.path.join(self._output_dir, 'current', 'board0')
+        self.assertTrue(os.path.exists(os.path.join(board0_dir, 'done')))
+        cmd_fname = os.path.join(board0_dir, 'out-cmd')
+        self.assertTrue(os.path.exists(cmd_fname))
+        data = tools.read_file(cmd_fname)
+
+        config_fname = os.path.join(board0_dir, '.config')
+        self.assertTrue(os.path.exists(config_fname))
+        cfg_data = tools.read_file(config_fname)
+
+        return data.splitlines(), cfg_data
+
+    def testCmdFile(self):
+        """Test that the -cmd-out file is produced"""
+        lines = self.check_command()[0]
+        self.assertEqual(2, len(lines))
+        self.assertRegex(lines[0], b'make O=/.*board0_defconfig')
+        self.assertRegex(lines[0], b'make O=/.*-s.*')
+
+    def testNoLto(self):
+        """Test that the --no-lto flag works"""
+        lines = self.check_command('-L')[0]
+        self.assertIn(b'NO_LTO=1', lines[0])
+
+    def testReproducible(self):
+        """Test that the -r flag works"""
+        lines, cfg_data = self.check_command('-r')
+        self.assertIn(b'SOURCE_DATE_EPOCH=0', lines[0])
+
+        # We should see CONFIG_LOCALVERSION_AUTO unset
+        self.assertEqual(b'''CONFIG_SOMETHING=1
+# CONFIG_LOCALVERSION_AUTO is not set
+''', cfg_data)
+
+        with test_util.capture_sys_output() as (stdout, stderr):
+            lines, cfg_data = self.check_command('-r', '-a', 'LOCALVERSION')
+        self.assertIn(b'SOURCE_DATE_EPOCH=0', lines[0])
+
+        # We should see CONFIG_LOCALVERSION_AUTO unset
+        self.assertEqual(b'''CONFIG_SOMETHING=1
+CONFIG_LOCALVERSION=y
+''', cfg_data)
+        self.assertIn('Not dropping LOCALVERSION_AUTO', stdout.getvalue())
diff --git a/tools/buildman/main.py b/tools/buildman/main.py
index 67c560c48d3..5e1f68d8235 100755
--- a/tools/buildman/main.py
+++ b/tools/buildman/main.py
@@ -25,8 +25,8 @@ from buildman import control
 from buildman import toolchain
 from patman import patchstream
 from patman import gitutil
-from patman import terminal
-from patman import test_util
+from u_boot_pylib import terminal
+from u_boot_pylib import test_util
 
 def RunTests(skip_net_tests, verboose, args):
     from buildman import func_test
@@ -46,17 +46,22 @@ def RunTests(skip_net_tests, verboose, args):
 
     return (0 if result.wasSuccessful() else 1)
 
-options, args = cmdline.ParseArgs()
+def run_buildman():
+    options, args = cmdline.ParseArgs()
 
-if not options.debug:
-    sys.tracebacklimit = 0
+    if not options.debug:
+        sys.tracebacklimit = 0
 
-# Run our meagre tests
-if options.test:
-    RunTests(options.skip_net_tests, options.verbose, args)
+    # Run our meagre tests
+    if cmdline.HAS_TESTS and options.test:
+        RunTests(options.skip_net_tests, options.verbose, args)
 
-# Build selected commits for selected boards
-else:
-    bsettings.Setup(options.config_file)
-    ret_code = control.DoBuildman(options, args)
-    sys.exit(ret_code)
+    # Build selected commits for selected boards
+    else:
+        bsettings.Setup(options.config_file)
+        ret_code = control.DoBuildman(options, args)
+        sys.exit(ret_code)
+
+
+if __name__ == "__main__":
+    run_buildman()
diff --git a/tools/buildman/pyproject.toml b/tools/buildman/pyproject.toml
new file mode 100644
index 00000000000..4d75e772ee1
--- /dev/null
+++ b/tools/buildman/pyproject.toml
@@ -0,0 +1,29 @@
+[build-system]
+requires = ["setuptools>=61.0"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "buildman"
+version = "0.0.2"
+authors = [
+  { name="Simon Glass", email="sjg@chromium.org" },
+]
+dependencies = ["u_boot_pylib", "patch-manager"]
+description = "Buildman build tool for U-Boot"
+readme = "README.rst"
+requires-python = ">=3.7"
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "License :: OSI Approved :: GNU General Public License v2 or later (GPLv2+)",
+    "Operating System :: OS Independent",
+]
+
+[project.urls]
+"Homepage" = "https://u-boot.readthedocs.io/en/latest/build/buildman.html"
+"Bug Tracker" = "https://source.denx.de/groups/u-boot/-/issues"
+
+[project.scripts]
+buildman = "buildman.main:run_buildman"
+
+[tool.setuptools.package-data]
+buildman = ["*.rst"]
diff --git a/tools/buildman/test.py b/tools/buildman/test.py
index daf5467503e..9fa6445b798 100644
--- a/tools/buildman/test.py
+++ b/tools/buildman/test.py
@@ -17,10 +17,10 @@ from buildman import cfgutil
 from buildman import control
 from buildman import toolchain
 from patman import commit
-from patman import command
-from patman import terminal
-from patman import test_util
-from patman import tools
+from u_boot_pylib import command
+from u_boot_pylib import terminal
+from u_boot_pylib import test_util
+from u_boot_pylib import tools
 
 use_network = True
 
diff --git a/tools/buildman/toolchain.py b/tools/buildman/toolchain.py
index 6bae9131971..0ecd8458b91 100644
--- a/tools/buildman/toolchain.py
+++ b/tools/buildman/toolchain.py
@@ -11,9 +11,9 @@ import tempfile
 import urllib.request, urllib.error, urllib.parse
 
 from buildman import bsettings
-from patman import command
-from patman import terminal
-from patman import tools
+from u_boot_pylib import command
+from u_boot_pylib import terminal
+from u_boot_pylib import tools
 
 (PRIORITY_FULL_PREFIX, PRIORITY_PREFIX_GCC, PRIORITY_PREFIX_GCC_PATH,
     PRIORITY_CALC) = list(range(4))
@@ -156,9 +156,10 @@ class Toolchain:
         Returns:
             Value of that environment variable or arguments
         """
-        wrapper = self.GetWrapper()
         if which == VAR_CROSS_COMPILE:
-            return wrapper + os.path.join(self.path, self.cross)
+            wrapper = self.GetWrapper()
+            base = '' if self.arch == 'sandbox' else self.path
+            return wrapper + os.path.join(base, self.cross)
         elif which == VAR_PATH:
             return self.path
         elif which == VAR_ARCH: