diff options
Diffstat (limited to 'test/py')
| -rw-r--r-- | test/py/conftest.py | 87 | ||||
| -rw-r--r-- | test/py/requirements.txt | 28 | ||||
| -rw-r--r-- | test/py/tests/test_suite.py | 34 | ||||
| -rw-r--r-- | test/py/tests/test_usb.py | 2 |
4 files changed, 116 insertions, 35 deletions
diff --git a/test/py/conftest.py b/test/py/conftest.py index 3bd333bfd24..31043a697e2 100644 --- a/test/py/conftest.py +++ b/test/py/conftest.py @@ -25,6 +25,7 @@ import re from _pytest.runner import runtestprotocol import subprocess import sys +import time from u_boot_spawn import BootFail, Timeout, Unexpected, handle_exception # Globals: The HTML log file, and the connection to the U-Boot console. @@ -91,6 +92,9 @@ def pytest_addoption(parser): parser.addoption('--role', help='U-Boot board role (for Labgrid-sjg)') parser.addoption('--use-running-system', default=False, action='store_true', help="Assume that U-Boot is ready and don't wait for a prompt") + parser.addoption('--timing', default=False, action='store_true', + help='Show info on test timing') + def run_build(config, source_dir, build_dir, board_type, log): """run_build: Build U-Boot @@ -322,6 +326,7 @@ def pytest_configure(config): ubconfig.use_running_system = config.getoption('use_running_system') ubconfig.dtb = build_dir + '/arch/sandbox/dts/test.dtb' ubconfig.connection_ok = True + ubconfig.timing = config.getoption('timing') env_vars = ( 'board_type', @@ -516,6 +521,12 @@ tests_skipped = [] tests_warning = [] tests_passed = [] +# Duration of each test: +# key (string): test name +# value (float): duration in ms +test_durations = {} + + def pytest_itemcollected(item): """pytest hook: Called once for each test found during collection. @@ -531,6 +542,73 @@ def pytest_itemcollected(item): tests_not_run.append(item.name) + +def show_timings(): + """Write timings for each test, along with a histogram""" + + def get_time_delta(msecs): + """Convert milliseconds into a user-friendly string""" + if msecs >= 1000: + return f'{msecs / 1000:.1f}s' + else: + return f'{msecs:.0f}ms' + + def show_bar(key, msecs, value): + """Show a single bar (line) of the histogram + + Args: + key (str): Key to write on the left + value (int): Value to display, i.e. the relative length of the bar + """ + if value: + bar_length = int((value / max_count) * max_bar_length) + print(f"{key:>8} : {get_time_delta(msecs):>7} |{'#' * bar_length} {value}", file=buf) + + # Create the buckets we will use, each has a count and a total time + bucket = {} + for power in range(5): + for i in [1, 2, 3, 4, 5, 7.5]: + bucket[i * 10 ** power] = {'count': 0, 'msecs': 0.0} + max_dur = max(bucket.keys()) + + # Collect counts for each bucket; if outside the range, add to too_long + # Also show a sorted list of test timings from longest to shortest + too_long = 0 + too_long_msecs = 0.0 + max_count = 0 + with log.section('Timing Report', 'timing_report'): + for name, dur in sorted(test_durations.items(), key=lambda kv: kv[1], + reverse=True): + log.info(f'{get_time_delta(dur):>8} {name}') + greater = [k for k in bucket.keys() if dur <= k] + if greater: + buck = bucket[min(greater)] + buck['count'] += 1 + max_count = max(max_count, buck['count']) + buck['msecs'] += dur + else: + too_long += 1 + too_long_msecs += dur + + # Set the maximum length of a histogram bar, in characters + max_bar_length = 40 + + # Show a a summary with histogram + buf = io.StringIO() + with log.section('Timing Summary', 'timing_summary'): + print('Duration : Total | Number of tests', file=buf) + print(f'{"=" * 8} : {"=" * 7} |{"=" * max_bar_length}', file=buf) + for dur, buck in bucket.items(): + if buck['count']: + label = get_time_delta(dur) + show_bar(f'<{label}', buck['msecs'], buck['count']) + if too_long: + show_bar(f'>{get_time_delta(max_dur)}', too_long_msecs, too_long) + log.info(buf.getvalue()) + if ubconfig.timing: + print(buf.getvalue(), end='') + + def cleanup(): """Clean up all global state. @@ -580,6 +658,7 @@ def cleanup(): for test in tests_not_run: anchor = anchors.get(test, None) log.status_fail('... ' + test, anchor) + show_timings() log.close() atexit.register(cleanup) @@ -713,7 +792,9 @@ def pytest_runtest_protocol(item, nextitem): log.get_and_reset_warning() ihook = item.ihook ihook.pytest_runtest_logstart(nodeid=item.nodeid, location=item.location) + start = time.monotonic() reports = runtestprotocol(item, nextitem=nextitem) + duration = round((time.monotonic() - start) * 1000, 1) ihook.pytest_runtest_logfinish(nodeid=item.nodeid, location=item.location) was_warning = log.get_and_reset_warning() @@ -726,6 +807,7 @@ def pytest_runtest_protocol(item, nextitem): start_test_section(item) failure_cleanup = False + record_duration = True if not was_warning: test_list = tests_passed msg = 'OK' @@ -756,6 +838,11 @@ def pytest_runtest_protocol(item, nextitem): test_list = tests_skipped msg = 'SKIPPED:\n' + str(report.longrepr) msg_log = log.status_skipped + record_duration = False + + msg += f' {duration} ms' + if record_duration: + test_durations[item.name] = duration if failure_cleanup: console.drain_console() diff --git a/test/py/requirements.txt b/test/py/requirements.txt index 75760f96e56..acfe17dce9f 100644 --- a/test/py/requirements.txt +++ b/test/py/requirements.txt @@ -1,30 +1,4 @@ -atomicwrites==1.4.1 -attrs==19.3.0 -concurrencytest==0.1.2 -coverage==6.2 -extras==1.0.0 filelock==3.0.12 -fixtures==3.0.0 -importlib-metadata==0.23 -linecache2==1.0.0 -more-itertools==7.2.0 -packaging==24.1 -pbr==5.4.3 -pluggy==0.13.0 -py==1.11.0 -pycryptodomex==3.19.1 -pyelftools==0.27 -pygit2==1.13.3 -pyparsing==3.0.7 +pycryptodomex==3.21.0 pytest==6.2.5 pytest-xdist==2.5.0 -python-mimeparse==1.6.0 -python-subunit==1.3.0 -requests==2.32.3 -setuptools==70.3.0 -six==1.16.0 -testtools==2.3.0 -traceback2==1.4.0 -unittest2==1.1.0 -wcwidth==0.1.7 -zipp==3.19.2 diff --git a/test/py/tests/test_suite.py b/test/py/tests/test_suite.py index 73c185349b4..9ddc883394b 100644 --- a/test/py/tests/test_suite.py +++ b/test/py/tests/test_suite.py @@ -7,10 +7,10 @@ import re # List of test suites we expect to find with 'ut info' and 'ut all' EXPECTED_SUITES = [ 'addrmap', 'bdinfo', 'bloblist', 'bootm', 'bootstd', - 'cmd', 'common', 'dm', 'env', 'exit', + 'cmd', 'common', 'dm', 'env', 'exit', 'fdt_overlay', 'fdt', 'font', 'hush', 'lib', 'loadm', 'log', 'mbr', 'measurement', 'mem', - 'overlay', 'pci_mps', 'setexpr', 'upl', + 'pci_mps', 'setexpr', 'upl', ] @@ -66,11 +66,12 @@ def collect_info(cons, output): msg = m.group(3) if DEBUG_ME: cons.log.info(f"test_name {test_name} msg '{msg}'") - if msg == ' (flat tree)' and test_name not in tests: - tests.add(test_name) + full_name = f'{cur_suite}.{test_name}' + if msg == ' (flat tree)' and full_name not in tests: + tests.add(full_name) test_count += 1 if not msg or 'skipped as it is manual' in msg: - tests.add(test_name) + tests.add(full_name) test_count += 1 if DEBUG_ME: cons.log.info(f'test_count {test_count}') @@ -134,7 +135,7 @@ def xtest_suite(u_boot_console, u_boot_config): - The number of suites matches that reported by the 'ut info' - Where available, the number of tests is each suite matches that - reported by 'ut info -s' + reported by 'ut -s info' - The total number of tests adds up to the total that are actually run with 'ut all' - All suites are run with 'ut all' @@ -166,7 +167,7 @@ def xtest_suite(u_boot_console, u_boot_config): # Run 'ut info' and compare with the log results with cons.log.section('Check suite test-counts'): - output = cons.run_command('ut info -s') + output = cons.run_command('ut -s info') suite_count, total_test_count, test_count = process_ut_info(cons, output) @@ -186,3 +187,22 @@ def xtest_suite(u_boot_console, u_boot_config): assert suite_count == len(EXPECTED_SUITES) assert total_test_count == len(all_tests) + + # Run three suites + with cons.log.section('Check multiple suites'): + output = cons.run_command('ut bloblist,setexpr,mem') + assert 'Suites run: 3' in output + + # Run a particular test + with cons.log.section('Check single test'): + output = cons.run_command('ut bloblist reloc') + assert 'Test: reloc: bloblist.c' in output + + # Run tests multiple times + with cons.log.section('Check multiple runs'): + output = cons.run_command('ut -r2 bloblist') + lines = output.splitlines() + run = len([line for line in lines if 'Test:' in line]) + count = re.search(r'Tests run: (\d*)', lines[-1]).group(1) + + assert run == 2 * int(count) diff --git a/test/py/tests/test_usb.py b/test/py/tests/test_usb.py index e1f203b5cbc..566d73b7c64 100644 --- a/test/py/tests/test_usb.py +++ b/test/py/tests/test_usb.py @@ -242,7 +242,7 @@ def test_usb_part(u_boot_console): elif part_type == '83': print('ext(2/4) detected') output = u_boot_console.run_command( - 'fstype usb %d:%d' % i, part_id + 'fstype usb %d:%d' % (i, part_id) ) if 'ext2' in output: part_ext2.append(part_id) |