summaryrefslogtreecommitdiff
path: root/test/py/tests
diff options
context:
space:
mode:
Diffstat (limited to 'test/py/tests')
-rw-r--r--test/py/tests/test_extension.py4
-rw-r--r--test/py/tests/test_smbios.py18
-rw-r--r--test/py/tests/test_spl.py2
-rw-r--r--test/py/tests/test_suite.py188
-rw-r--r--test/py/tests/test_upl.py4
-rw-r--r--test/py/tests/test_usb.py2
-rw-r--r--test/py/tests/test_ut.py5
-rw-r--r--test/py/tests/test_vbe.py2
-rw-r--r--test/py/tests/test_vpl.py2
9 files changed, 217 insertions, 10 deletions
diff --git a/test/py/tests/test_extension.py b/test/py/tests/test_extension.py
index 267cf2ff27c..2a3c5116171 100644
--- a/test/py/tests/test_extension.py
+++ b/test/py/tests/test_extension.py
@@ -26,7 +26,9 @@ def test_extension(u_boot_console):
load_dtb(u_boot_console)
output = u_boot_console.run_command('extension list')
- assert('No extension' in output)
+ # extension_bootdev_hunt may have already run.
+ # Without reboot we cannot make any assumption here.
+ # assert('No extension' in output)
output = u_boot_console.run_command('extension scan')
assert output == 'Found 2 extension board(s).'
diff --git a/test/py/tests/test_smbios.py b/test/py/tests/test_smbios.py
index 82b0b689830..0405a9b9d38 100644
--- a/test/py/tests/test_smbios.py
+++ b/test/py/tests/test_smbios.py
@@ -32,10 +32,26 @@ def test_cmd_smbios_sandbox(u_boot_console):
"""Run the smbios command on the sandbox"""
output = u_boot_console.run_command('smbios')
assert 'DMI type 0,' in output
- assert 'String 1: U-Boot' in output
+ assert 'Vendor: U-Boot' in output
assert 'DMI type 1,' in output
assert 'Manufacturer: sandbox' in output
assert 'DMI type 2,' in output
assert 'DMI type 3,' in output
assert 'DMI type 4,' in output
assert 'DMI type 127,' in output
+
+@pytest.mark.buildconfigspec('cmd_smbios')
+@pytest.mark.buildconfigspec('sysinfo_smbios')
+@pytest.mark.buildconfigspec('generate_smbios_table_verbose')
+def test_cmd_smbios_sysinfo_verbose(u_boot_console):
+ """Run the smbios command"""
+ output = u_boot_console.run_command('smbios')
+ assert 'DMI type 0,' in output
+ assert 'Vendor: U-Boot' in output
+ assert 'DMI type 1,' in output
+ assert 'Manufacturer: linux' in output
+ assert 'DMI type 2,' in output
+ assert 'DMI type 3,' in output
+ assert 'DMI type 7,' in output
+ assert 'DMI type 4,' in output
+ assert 'DMI type 127,' in output
diff --git a/test/py/tests/test_spl.py b/test/py/tests/test_spl.py
index 42e4c4342b2..474f430a344 100644
--- a/test/py/tests/test_spl.py
+++ b/test/py/tests/test_spl.py
@@ -36,7 +36,7 @@ def test_spl(u_boot_console, ut_spl_subtest):
cons = u_boot_console
cons.restart_uboot_with_flags(['-u', '-k', ut_spl_subtest.split()[1]])
output = cons.get_spawn_output().replace('\r', '')
- assert 'Failures: 0' in output
+ assert 'failures: 0' in output
finally:
# Restart afterward in case a non-SPL test is run next. This should not
# happen since SPL tests are run in their own invocation of test.py, but
diff --git a/test/py/tests/test_suite.py b/test/py/tests/test_suite.py
new file mode 100644
index 00000000000..73c185349b4
--- /dev/null
+++ b/test/py/tests/test_suite.py
@@ -0,0 +1,188 @@
+# SPDX-License-Identifier: GPL-2.0
+# Copyright 2024 Google LLC
+
+import pytest
+import re
+
+# List of test suites we expect to find with 'ut info' and 'ut all'
+EXPECTED_SUITES = [
+ 'addrmap', 'bdinfo', 'bloblist', 'bootm', 'bootstd',
+ 'cmd', 'common', 'dm', 'env', 'exit',
+ 'fdt', 'font', 'hush', 'lib',
+ 'loadm', 'log', 'mbr', 'measurement', 'mem',
+ 'overlay', 'pci_mps', 'setexpr', 'upl',
+ ]
+
+
+# Set this to True to aid debugging of tests
+DEBUG_ME = False
+
+
+def collect_info(cons, output):
+ """Process the output from 'ut all'
+
+ Args:
+ cons: U-Boot console object
+ output: Output from running 'ut all'
+
+ Returns:
+ tuple:
+ set: suite names that were found in output
+ set: test names that were found in output
+ dict: test count for each suite:
+ key: suite name
+ value: number of tests for the suite found in output
+ set: missing suites (compared to EXPECTED_SUITES)
+ set: extra suites (compared to EXPECTED_SUITES)
+ """
+ suites = set()
+ tests = set()
+ cur_suite = None
+ test_count = None
+ exp_test_count = {}
+
+ # Collect suites{}
+ for line in output.splitlines():
+ line = line.rstrip()
+ if DEBUG_ME:
+ cons.log.info(f'line: {line}')
+ m = re.search('----Running ([^ ]*) tests----', line)
+ if m:
+ if DEBUG_ME and cur_suite and cur_suite != 'info':
+ cons.log.info(f'suite: {cur_suite} expected {exp_test_count[cur_suite]} found {test_count}')
+
+ cur_suite = m.group(1)
+ if DEBUG_ME:
+ cons.log.info(f'cur_suite: {cur_suite}')
+ suites.add(cur_suite)
+
+ test_count = 0
+ m = re.match(rf'Running (\d+) {cur_suite} tests', line)
+ if m:
+ exp_test_count[cur_suite] = int(m.group(1))
+ m = re.search(r'Test: (\w*): ([-a-z0-9_]*\.c)?( .*)?', line)
+ if m:
+ test_name = m.group(1)
+ msg = m.group(3)
+ if DEBUG_ME:
+ cons.log.info(f"test_name {test_name} msg '{msg}'")
+ if msg == ' (flat tree)' and test_name not in tests:
+ tests.add(test_name)
+ test_count += 1
+ if not msg or 'skipped as it is manual' in msg:
+ tests.add(test_name)
+ test_count += 1
+ if DEBUG_ME:
+ cons.log.info(f'test_count {test_count}')
+ if DEBUG_ME:
+ cons.log.info(f'suite: {cur_suite} expected {exp_test_count[cur_suite]} found {test_count}')
+ cons.log.info(f"Tests: {' '.join(sorted(list(tests)))}")
+
+ # Figure out what is missing, or extra
+ missing = set()
+ extra = set(suites)
+ for suite in EXPECTED_SUITES:
+ if suite in extra:
+ extra.remove(suite)
+ else:
+ missing.add(suite)
+
+ return suites, tests, exp_test_count, missing, extra
+
+
+def process_ut_info(cons, output):
+ """Process the output of the 'ut info' command
+
+ Args:
+ cons: U-Boot console object
+ output: Output from running 'ut all'
+
+ Returns:
+ tuple:
+ int: Number of suites reported
+ int: Number of tests reported
+ dict: test count for each suite:
+ key: suite name
+ value: number of tests reported for the suite
+
+ """
+ suite_count = None
+ total_test_count = None
+ test_count = {}
+ for line in output.splitlines():
+ line = line.rstrip()
+ if DEBUG_ME:
+ cons.log.info(f'line: {line}')
+ m = re.match(r'Test suites: (.*)', line)
+ if m:
+ suite_count = int(m.group(1))
+ m = re.match(r'Total tests: (.*)', line)
+ if m:
+ total_test_count = int(m.group(1))
+ m = re.match(r' *([0-9?]*) (\w*)', line)
+ if m:
+ test_count[m.group(2)] = m.group(1)
+ return suite_count, total_test_count, test_count
+
+
+@pytest.mark.buildconfigspec('sandbox')
+@pytest.mark.notbuildconfigspec('sandbox_spl')
+@pytest.mark.notbuildconfigspec('sandbox64')
+# This test is disabled since it fails; remove the leading 'x' to try it
+def xtest_suite(u_boot_console, u_boot_config):
+ """Perform various checks on the unit tests, including:
+
+ - The number of suites matches that reported by the 'ut info'
+ - Where available, the number of tests is each suite matches that
+ reported by 'ut info -s'
+ - The total number of tests adds up to the total that are actually run
+ with 'ut all'
+ - All suites are run with 'ut all'
+ - The expected set of suites is run (the list is hard-coded in this test)
+
+ """
+ cons = u_boot_console
+ buildconfig = u_boot_config.buildconfig
+ with cons.log.section('Run all unit tests'):
+ # ut hush hush_test_simple_dollar prints "Unknown command" on purpose.
+ with u_boot_console.disable_check('unknown_command'):
+ output = cons.run_command('ut all')
+
+ # Process the output from the run
+ with cons.log.section('Check output'):
+ suites, all_tests, exp_test_count, missing, extra = collect_info(cons,
+ output)
+ cons.log.info(f'missing {missing}')
+ cons.log.info(f'extra {extra}')
+
+ # Make sure we got a test count for each suite
+ assert not (suites - exp_test_count.keys())
+
+ # Deal with missing suites
+ with cons.log.section('Check missing suites'):
+ if 'config_cmd_seama' not in buildconfig:
+ cons.log.info("CMD_SEAMA not enabled: Ignoring suite 'seama'")
+ missing.discard('seama')
+
+ # Run 'ut info' and compare with the log results
+ with cons.log.section('Check suite test-counts'):
+ output = cons.run_command('ut info -s')
+
+ suite_count, total_test_count, test_count = process_ut_info(cons,
+ output)
+
+ if missing or extra:
+ cons.log.info(f"suites: {' '.join(sorted(list(suites)))}")
+ cons.log.error(f'missing: {sorted(list(missing))}')
+ cons.log.error(f'extra: {sorted(list(extra))}')
+
+ assert not missing, f'Missing suites {missing}'
+ assert not extra, f'Extra suites {extra}'
+
+ cons.log.info(str(exp_test_count))
+ for suite in EXPECTED_SUITES:
+ assert test_count[suite] in ['?', str(exp_test_count[suite])], \
+ f'suite {suite} expected {exp_test_count[suite]}'
+
+ assert suite_count == len(EXPECTED_SUITES)
+ assert total_test_count == len(all_tests)
diff --git a/test/py/tests/test_upl.py b/test/py/tests/test_upl.py
index 3164bda6b71..a1ccc8df233 100644
--- a/test/py/tests/test_upl.py
+++ b/test/py/tests/test_upl.py
@@ -17,7 +17,7 @@ def test_upl_handoff(u_boot_console):
proper and runs a test to check that the parameters are correct.
The entire FIT is loaded into memory in SPL (in upl_load_from_image()) so
- that it can be inpected in upl_test_info_norun
+ that it can be inspected in upl_test_info_norun
"""
cons = u_boot_console
ram = os.path.join(cons.config.build_dir, 'ram.bin')
@@ -35,4 +35,4 @@ def test_upl_handoff(u_boot_console):
# Check the FIT offsets look correct
output = cons.run_command('ut upl -f upl_test_info_norun')
- assert 'Failures: 0' in output
+ assert 'failures: 0' in output
diff --git a/test/py/tests/test_usb.py b/test/py/tests/test_usb.py
index e1f203b5cbc..566d73b7c64 100644
--- a/test/py/tests/test_usb.py
+++ b/test/py/tests/test_usb.py
@@ -242,7 +242,7 @@ def test_usb_part(u_boot_console):
elif part_type == '83':
print('ext(2/4) detected')
output = u_boot_console.run_command(
- 'fstype usb %d:%d' % i, part_id
+ 'fstype usb %d:%d' % (i, part_id)
)
if 'ext2' in output:
part_ext2.append(part_id)
diff --git a/test/py/tests/test_ut.py b/test/py/tests/test_ut.py
index 10ec7e582e0..d2d8ce10755 100644
--- a/test/py/tests/test_ut.py
+++ b/test/py/tests/test_ut.py
@@ -343,9 +343,10 @@ def setup_cros_image(cons):
start, size, num, name = line.split(maxsplit=3)
parts[int(num)] = Partition(int(start), int(size), name)
+ # Set up the kernel command-line
dummy = os.path.join(cons.config.result_dir, 'dummy.txt')
with open(dummy, 'wb') as outf:
- outf.write(b'dummy\n')
+ outf.write(b'BOOT_IMAGE=/vmlinuz-5.15.0-121-generic root=/dev/nvme0n1p1 ro quiet splash vt.handoff=7')
# For now we just use dummy kernels. This limits testing to just detecting
# a signed kernel. We could add support for the x86 data structures so that
@@ -606,4 +607,4 @@ def test_ut(u_boot_console, ut_subtest):
assert 'Unknown command \'quux\' - try \'help\'' in output
else:
output = u_boot_console.run_command('ut ' + ut_subtest)
- assert output.endswith('Failures: 0')
+ assert output.endswith('failures: 0')
diff --git a/test/py/tests/test_vbe.py b/test/py/tests/test_vbe.py
index 50b6c1cd911..861df3f8266 100644
--- a/test/py/tests/test_vbe.py
+++ b/test/py/tests/test_vbe.py
@@ -117,4 +117,4 @@ def test_vbe(u_boot_console):
with cons.log.section('Kernel load'):
output = cons.run_command_list(cmd.splitlines())
- assert 'Failures: 0' in output[-1]
+ assert 'failures: 0' in output[-1]
diff --git a/test/py/tests/test_vpl.py b/test/py/tests/test_vpl.py
index 4af578b9173..8c472ca7a92 100644
--- a/test/py/tests/test_vpl.py
+++ b/test/py/tests/test_vpl.py
@@ -26,7 +26,7 @@ def test_vpl(u_boot_console, ut_vpl_subtest):
cons = u_boot_console
cons.restart_uboot_with_flags(['-u', '-k', ut_vpl_subtest.split()[1]])
output = cons.get_spawn_output().replace('\r', '')
- assert 'Failures: 0' in output
+ assert 'failures: 0' in output
finally:
# Restart afterward in case a non-VPL test is run next. This should not
# happen since VPL tests are run in their own invocation of test.py, but
generated by cgit v1.2.3 (git 2.0.4) at 2025-08-01 21:03:00 +0000