From 63adc40d4c18b3a39df68289cd6cf4d38e8dd5ad Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Fri, 7 Feb 2025 11:30:37 -0700 Subject: test: Leave out the prefix when printing test names When tests are all in the same suite it is annoying to have to read all the common text after each name. Skip this to help the user. Signed-off-by: Simon Glass --- test/py/tests/test_suite.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'test/py/tests/test_suite.py') diff --git a/test/py/tests/test_suite.py b/test/py/tests/test_suite.py index 73c185349b4..ae127301fd7 100644 --- a/test/py/tests/test_suite.py +++ b/test/py/tests/test_suite.py @@ -66,11 +66,12 @@ def collect_info(cons, output): msg = m.group(3) if DEBUG_ME: cons.log.info(f"test_name {test_name} msg '{msg}'") - if msg == ' (flat tree)' and test_name not in tests: - tests.add(test_name) + full_name = f'{cur_suite}.{test_name}' + if msg == ' (flat tree)' and full_name not in tests: + tests.add(full_name) test_count += 1 if not msg or 'skipped as it is manual' in msg: - tests.add(test_name) + tests.add(full_name) test_count += 1 if DEBUG_ME: cons.log.info(f'test_count {test_count}') -- cgit v1.2.3 From ea29bad9cff2fd88d172276b58859f01649fe444 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Fri, 7 Feb 2025 11:30:39 -0700 Subject: test: Tweak FDT-overlay tests Use fdt_overlay consistently in the identifiers and file/dir names. Signed-off-by: Simon Glass --- test/py/tests/test_suite.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'test/py/tests/test_suite.py') diff --git a/test/py/tests/test_suite.py b/test/py/tests/test_suite.py index ae127301fd7..1e02d67efe2 100644 --- a/test/py/tests/test_suite.py +++ b/test/py/tests/test_suite.py @@ -7,10 +7,10 @@ import re # List of test suites we expect to find with 'ut info' and 'ut all' EXPECTED_SUITES = [ 'addrmap', 'bdinfo', 'bloblist', 'bootm', 'bootstd', - 'cmd', 'common', 'dm', 'env', 'exit', + 'cmd', 'common', 'dm', 'env', 'exit', 'fdt_overlay', 'fdt', 'font', 'hush', 'lib', 'loadm', 'log', 'mbr', 'measurement', 'mem', - 'overlay', 'pci_mps', 'setexpr', 'upl', + 'pci_mps', 'setexpr', 'upl', ] -- cgit v1.2.3 From fa0b68d22add6416fa56a5907c752a9348ae1a45 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Fri, 7 Feb 2025 11:30:57 -0700 Subject: test: Allow running a selection of suites Enhance the ut command to accept a comma-separated list of test suites to run. Report the summary information for these at the end. Signed-off-by: Simon Glass --- test/py/tests/test_suite.py | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'test/py/tests/test_suite.py') diff --git a/test/py/tests/test_suite.py b/test/py/tests/test_suite.py index 1e02d67efe2..d0025a7ba30 100644 --- a/test/py/tests/test_suite.py +++ b/test/py/tests/test_suite.py @@ -187,3 +187,8 @@ def xtest_suite(u_boot_console, u_boot_config): assert suite_count == len(EXPECTED_SUITES) assert total_test_count == len(all_tests) + + # Run three suites + with cons.log.section('Check multiple suites'): + output = cons.run_command('ut bloblist,setexpr,mem') + assert 'Suites run: 3' in output -- cgit v1.2.3 From 540bf7881a1a1c79c4d256b68396ae699e3a3157 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Fri, 7 Feb 2025 11:30:59 -0700 Subject: test: Do flag-processing in the correct place At present the 'ut' command handles its flags in a strange way, in that they must come after the subcommand. So, we must use 'ut bloblist -r2' to run the bloblist tests twice. This is an artefact of the way tests were run, through subcommands. It is now possible to correct this, by doing flag-processing before running the suite. Update the code to handle this, so that 'ut -r2 bloblist' works. Update the 'test_suite' test to check the new arguments. Add a sanity-check for -I while we are here. Signed-off-by: Simon Glass --- test/py/tests/test_suite.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) (limited to 'test/py/tests/test_suite.py') diff --git a/test/py/tests/test_suite.py b/test/py/tests/test_suite.py index d0025a7ba30..9ddc883394b 100644 --- a/test/py/tests/test_suite.py +++ b/test/py/tests/test_suite.py @@ -135,7 +135,7 @@ def xtest_suite(u_boot_console, u_boot_config): - The number of suites matches that reported by the 'ut info' - Where available, the number of tests is each suite matches that - reported by 'ut info -s' + reported by 'ut -s info' - The total number of tests adds up to the total that are actually run with 'ut all' - All suites are run with 'ut all' @@ -167,7 +167,7 @@ def xtest_suite(u_boot_console, u_boot_config): # Run 'ut info' and compare with the log results with cons.log.section('Check suite test-counts'): - output = cons.run_command('ut info -s') + output = cons.run_command('ut -s info') suite_count, total_test_count, test_count = process_ut_info(cons, output) @@ -191,4 +191,18 @@ def xtest_suite(u_boot_console, u_boot_config): # Run three suites with cons.log.section('Check multiple suites'): output = cons.run_command('ut bloblist,setexpr,mem') - assert 'Suites run: 3' in output + assert 'Suites run: 3' in output + + # Run a particular test + with cons.log.section('Check single test'): + output = cons.run_command('ut bloblist reloc') + assert 'Test: reloc: bloblist.c' in output + + # Run tests multiple times + with cons.log.section('Check multiple runs'): + output = cons.run_command('ut -r2 bloblist') + lines = output.splitlines() + run = len([line for line in lines if 'Test:' in line]) + count = re.search(r'Tests run: (\d*)', lines[-1]).group(1) + + assert run == 2 * int(count) -- cgit v1.2.3 From 752c3769874596d012cd8325099d2ae20123f989 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Sun, 9 Feb 2025 09:07:14 -0700 Subject: test/py: Shorten u_boot_console This fixture name is quite long and results in lots of verbose code. We know this is U-Boot so the 'u_boot_' part is not necessary. But it is also a bit of a misnomer, since it provides access to all the information available to tests. It is not just the console. It would be too confusing to use con as it would be confused with config and it is probably too short. So shorten it to 'ubman'. Signed-off-by: Simon Glass Link: https://lore.kernel.org/u-boot/CAFLszTgPa4aT_J9h9pqeTtLCVn4x2JvLWRcWRD8NaN3uoSAtyA@mail.gmail.com/ --- test/py/tests/test_suite.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'test/py/tests/test_suite.py') diff --git a/test/py/tests/test_suite.py b/test/py/tests/test_suite.py index 9ddc883394b..c7f4dedba51 100644 --- a/test/py/tests/test_suite.py +++ b/test/py/tests/test_suite.py @@ -130,7 +130,7 @@ def process_ut_info(cons, output): @pytest.mark.notbuildconfigspec('sandbox_spl') @pytest.mark.notbuildconfigspec('sandbox64') # This test is disabled since it fails; remove the leading 'x' to try it -def xtest_suite(u_boot_console, u_boot_config): +def xtest_suite(ubman, u_boot_config): """Perform various checks on the unit tests, including: - The number of suites matches that reported by the 'ut info' @@ -142,11 +142,11 @@ def xtest_suite(u_boot_console, u_boot_config): - The expected set of suites is run (the list is hard-coded in this test) """ - cons = u_boot_console + cons = ubman buildconfig = u_boot_config.buildconfig with cons.log.section('Run all unit tests'): # ut hush hush_test_simple_dollar prints "Unknown command" on purpose. - with u_boot_console.disable_check('unknown_command'): + with ubman.disable_check('unknown_command'): output = cons.run_command('ut all') # Process the output from the run -- cgit v1.2.3 From d08653d3699c1aafada3418c9f74b887bfb21a65 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Sun, 9 Feb 2025 09:07:17 -0700 Subject: test/py: Drop assigning ubman to cons Now that we have a shorter name, we don't need this sort of thing. Just use ubman instead. Signed-off-by: Simon Glass --- test/py/tests/test_suite.py | 67 ++++++++++++++++++++++----------------------- 1 file changed, 33 insertions(+), 34 deletions(-) (limited to 'test/py/tests/test_suite.py') diff --git a/test/py/tests/test_suite.py b/test/py/tests/test_suite.py index c7f4dedba51..7fe9a90dfd3 100644 --- a/test/py/tests/test_suite.py +++ b/test/py/tests/test_suite.py @@ -18,11 +18,11 @@ EXPECTED_SUITES = [ DEBUG_ME = False -def collect_info(cons, output): +def collect_info(ubman, output): """Process the output from 'ut all' Args: - cons: U-Boot console object + ubman: U-Boot console object output: Output from running 'ut all' Returns: @@ -45,15 +45,15 @@ def collect_info(cons, output): for line in output.splitlines(): line = line.rstrip() if DEBUG_ME: - cons.log.info(f'line: {line}') + ubman.log.info(f'line: {line}') m = re.search('----Running ([^ ]*) tests----', line) if m: if DEBUG_ME and cur_suite and cur_suite != 'info': - cons.log.info(f'suite: {cur_suite} expected {exp_test_count[cur_suite]} found {test_count}') + ubman.log.info(f'suite: {cur_suite} expected {exp_test_count[cur_suite]} found {test_count}') cur_suite = m.group(1) if DEBUG_ME: - cons.log.info(f'cur_suite: {cur_suite}') + ubman.log.info(f'cur_suite: {cur_suite}') suites.add(cur_suite) test_count = 0 @@ -65,7 +65,7 @@ def collect_info(cons, output): test_name = m.group(1) msg = m.group(3) if DEBUG_ME: - cons.log.info(f"test_name {test_name} msg '{msg}'") + ubman.log.info(f"test_name {test_name} msg '{msg}'") full_name = f'{cur_suite}.{test_name}' if msg == ' (flat tree)' and full_name not in tests: tests.add(full_name) @@ -74,10 +74,10 @@ def collect_info(cons, output): tests.add(full_name) test_count += 1 if DEBUG_ME: - cons.log.info(f'test_count {test_count}') + ubman.log.info(f'test_count {test_count}') if DEBUG_ME: - cons.log.info(f'suite: {cur_suite} expected {exp_test_count[cur_suite]} found {test_count}') - cons.log.info(f"Tests: {' '.join(sorted(list(tests)))}") + ubman.log.info(f'suite: {cur_suite} expected {exp_test_count[cur_suite]} found {test_count}') + ubman.log.info(f"Tests: {' '.join(sorted(list(tests)))}") # Figure out what is missing, or extra missing = set() @@ -91,11 +91,11 @@ def collect_info(cons, output): return suites, tests, exp_test_count, missing, extra -def process_ut_info(cons, output): +def process_ut_info(ubman, output): """Process the output of the 'ut info' command Args: - cons: U-Boot console object + ubman: U-Boot console object output: Output from running 'ut all' Returns: @@ -113,7 +113,7 @@ def process_ut_info(cons, output): for line in output.splitlines(): line = line.rstrip() if DEBUG_ME: - cons.log.info(f'line: {line}') + ubman.log.info(f'line: {line}') m = re.match(r'Test suites: (.*)', line) if m: suite_count = int(m.group(1)) @@ -142,45 +142,44 @@ def xtest_suite(ubman, u_boot_config): - The expected set of suites is run (the list is hard-coded in this test) """ - cons = ubman buildconfig = u_boot_config.buildconfig - with cons.log.section('Run all unit tests'): + with ubman.log.section('Run all unit tests'): # ut hush hush_test_simple_dollar prints "Unknown command" on purpose. with ubman.disable_check('unknown_command'): - output = cons.run_command('ut all') + output = ubman.run_command('ut all') # Process the output from the run - with cons.log.section('Check output'): - suites, all_tests, exp_test_count, missing, extra = collect_info(cons, + with ubman.log.section('Check output'): + suites, all_tests, exp_test_count, missing, extra = collect_info(ubman, output) - cons.log.info(f'missing {missing}') - cons.log.info(f'extra {extra}') + ubman.log.info(f'missing {missing}') + ubman.log.info(f'extra {extra}') # Make sure we got a test count for each suite assert not (suites - exp_test_count.keys()) # Deal with missing suites - with cons.log.section('Check missing suites'): + with ubman.log.section('Check missing suites'): if 'config_cmd_seama' not in buildconfig: - cons.log.info("CMD_SEAMA not enabled: Ignoring suite 'seama'") + ubman.log.info("CMD_SEAMA not enabled: Ignoring suite 'seama'") missing.discard('seama') # Run 'ut info' and compare with the log results - with cons.log.section('Check suite test-counts'): - output = cons.run_command('ut -s info') + with ubman.log.section('Check suite test-counts'): + output = ubman.run_command('ut -s info') - suite_count, total_test_count, test_count = process_ut_info(cons, + suite_count, total_test_count, test_count = process_ut_info(ubman, output) if missing or extra: - cons.log.info(f"suites: {' '.join(sorted(list(suites)))}") - cons.log.error(f'missing: {sorted(list(missing))}') - cons.log.error(f'extra: {sorted(list(extra))}') + ubman.log.info(f"suites: {' '.join(sorted(list(suites)))}") + ubman.log.error(f'missing: {sorted(list(missing))}') + ubman.log.error(f'extra: {sorted(list(extra))}') assert not missing, f'Missing suites {missing}' assert not extra, f'Extra suites {extra}' - cons.log.info(str(exp_test_count)) + ubman.log.info(str(exp_test_count)) for suite in EXPECTED_SUITES: assert test_count[suite] in ['?', str(exp_test_count[suite])], \ f'suite {suite} expected {exp_test_count[suite]}' @@ -189,18 +188,18 @@ def xtest_suite(ubman, u_boot_config): assert total_test_count == len(all_tests) # Run three suites - with cons.log.section('Check multiple suites'): - output = cons.run_command('ut bloblist,setexpr,mem') + with ubman.log.section('Check multiple suites'): + output = ubman.run_command('ut bloblist,setexpr,mem') assert 'Suites run: 3' in output # Run a particular test - with cons.log.section('Check single test'): - output = cons.run_command('ut bloblist reloc') + with ubman.log.section('Check single test'): + output = ubman.run_command('ut bloblist reloc') assert 'Test: reloc: bloblist.c' in output # Run tests multiple times - with cons.log.section('Check multiple runs'): - output = cons.run_command('ut -r2 bloblist') + with ubman.log.section('Check multiple runs'): + output = ubman.run_command('ut -r2 bloblist') lines = output.splitlines() run = len([line for line in lines if 'Test:' in line]) count = re.search(r'Tests run: (\d*)', lines[-1]).group(1) -- cgit v1.2.3