From 636f38d83a7e0e6ca076ae65e086c800337fb3a3 Mon Sep 17 00:00:00 2001 From: Stephen Warren Date: Fri, 22 Jan 2016 12:30:08 -0700 Subject: test/py: move U-Boot respawn trigger to the test core Prior to this change, U-Boot was lazilly (re-)spawned if/when a test attempted to interact with it, and no active connection existed. This approach was simple, yet had the disadvantage that U-Boot might be spawned in the middle of a test function, e.g. after the test had already performed actions such as creating data files, etc. In that case, this could cause the log to contain the sequence (1) some test logs, (2) U-Boot's boot process, (3) the rest of that test's logs. This isn't optimally readable. This issue will affect the upcoming DFU and enhanced UMS tests. This change converts u_boot_console to be a function-scoped fixture, so that pytest attempts to re-create the object for each test invocation. This allows the fixture factory function to ensure that U-Boot is spawned prior to every test. In practice, the same object is returned each time so there is essentially no additional overhead due to this change. This allows us to remove: - The explicit ensure_spawned() call from test_sleep, since the core now ensures that the spawn happens before the test code is executed. - The laxy calls to ensure_spawned() in the u_boot_console_* implementations. The one downside is that test_env's "state_ttest_env" fixture must be converted to a function-scoped fixture too, since a module-scoped fixture cannot use a function-scoped fixture. To avoid overhead, we use the same trick of returning the same object each time. Signed-off-by: Stephen Warren Acked-by: Simon Glass --- test/py/conftest.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'test/py/conftest.py') diff --git a/test/py/conftest.py b/test/py/conftest.py index e1674dfce05..38aa3f922a8 100644 --- a/test/py/conftest.py +++ b/test/py/conftest.py @@ -227,7 +227,7 @@ def pytest_generate_tests(metafunc): vals = subconfig.get(fn + 's', []) metafunc.parametrize(fn, vals) -@pytest.fixture(scope='session') +@pytest.fixture(scope='function') def u_boot_console(request): '''Generate the value of a test's u_boot_console fixture. @@ -238,6 +238,7 @@ def u_boot_console(request): The fixture value. ''' + console.ensure_spawned() return console tests_not_run = set() -- cgit v1.2.3 From c10eb9d39fb54f435dbd632f71c760e4a887a015 Mon Sep 17 00:00:00 2001 From: Stephen Warren Date: Fri, 22 Jan 2016 12:30:09 -0700 Subject: test/py: drain console log at the end of any failed test Tests may fail for a number of reasons, and in particular for reasons other than a timeout waiting for U-Boot to print expected data. If the last operation that a failed test performs is not waiting for U-Boot to print something, then any trailing output from U-Boot during that test's operation will not be logged as part of that test, but rather either along with the next test, or even thrown away, potentiall hiding clues re: the test failure reason. Solve this by explicitly draining (and hence logging) the U-Boot output in the case of failed tests. Signed-off-by: Stephen Warren Acked-by: Simon Glass --- test/py/conftest.py | 1 + 1 file changed, 1 insertion(+) (limited to 'test/py/conftest.py') diff --git a/test/py/conftest.py b/test/py/conftest.py index 38aa3f922a8..c1f19cee65a 100644 --- a/test/py/conftest.py +++ b/test/py/conftest.py @@ -386,6 +386,7 @@ def pytest_runtest_protocol(item, nextitem): skipped = report if failed: + console.drain_console() tests_failed.add(item.name) elif skipped: tests_skipped.add(item.name) -- cgit v1.2.3 From e8debf394fbba594fcfc267c61f8c6bbca395b06 Mon Sep 17 00:00:00 2001 From: Stephen Warren Date: Tue, 26 Jan 2016 13:41:30 -0700 Subject: test/py: use " for docstrings Python's coding style docs indicate to use " not ' for docstrings. test/py has other violations of the coding style docs, since the docs specify a stranger style than I would expect, but nobody has complained about those yet:-) Signed-off-by: Stephen Warren Reviewed-by: Simon Glass --- test/py/conftest.py | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) (limited to 'test/py/conftest.py') diff --git a/test/py/conftest.py b/test/py/conftest.py index c1f19cee65a..a4e54c66ced 100644 --- a/test/py/conftest.py +++ b/test/py/conftest.py @@ -29,7 +29,7 @@ log = None console = None def mkdir_p(path): - '''Create a directory path. + """Create a directory path. This includes creating any intermediate/parent directories. Any errors caused due to already extant directories are ignored. @@ -39,7 +39,7 @@ def mkdir_p(path): Returns: Nothing. - ''' + """ try: os.makedirs(path) @@ -50,14 +50,14 @@ def mkdir_p(path): raise def pytest_addoption(parser): - '''pytest hook: Add custom command-line options to the cmdline parser. + """pytest hook: Add custom command-line options to the cmdline parser. Args: parser: The pytest command-line parser. Returns: Nothing. - ''' + """ parser.addoption('--build-dir', default=None, help='U-Boot build directory (O=)') @@ -73,14 +73,14 @@ def pytest_addoption(parser): help='Compile U-Boot before running tests') def pytest_configure(config): - '''pytest hook: Perform custom initialization at startup time. + """pytest hook: Perform custom initialization at startup time. Args: config: The pytest configuration. Returns: Nothing. - ''' + """ global log global console @@ -190,7 +190,7 @@ def pytest_configure(config): console = u_boot_console_exec_attach.ConsoleExecAttach(log, ubconfig) def pytest_generate_tests(metafunc): - '''pytest hook: parameterize test functions based on custom rules. + """pytest hook: parameterize test functions based on custom rules. If a test function takes parameter(s) (fixture names) of the form brd__xxx or env__xxx, the brd and env configuration dictionaries are consulted to @@ -202,7 +202,7 @@ def pytest_generate_tests(metafunc): Returns: Nothing. - ''' + """ subconfigs = { 'brd': console.config.brd, @@ -229,14 +229,14 @@ def pytest_generate_tests(metafunc): @pytest.fixture(scope='function') def u_boot_console(request): - '''Generate the value of a test's u_boot_console fixture. + """Generate the value of a test's u_boot_console fixture. Args: request: The pytest request. Returns: The fixture value. - ''' + """ console.ensure_spawned() return console @@ -247,7 +247,7 @@ tests_skipped = set() tests_passed = set() def pytest_itemcollected(item): - '''pytest hook: Called once for each test found during collection. + """pytest hook: Called once for each test found during collection. This enables our custom result analysis code to see the list of all tests that should eventually be run. @@ -257,12 +257,12 @@ def pytest_itemcollected(item): Returns: Nothing. - ''' + """ tests_not_run.add(item.name) def cleanup(): - '''Clean up all global state. + """Clean up all global state. Executed (via atexit) once the entire test process is complete. This includes logging the status of all tests, and the identity of any failed @@ -273,7 +273,7 @@ def cleanup(): Returns: Nothing. - ''' + """ if console: console.close() @@ -295,7 +295,7 @@ def cleanup(): atexit.register(cleanup) def setup_boardspec(item): - '''Process any 'boardspec' marker for a test. + """Process any 'boardspec' marker for a test. Such a marker lists the set of board types that a test does/doesn't support. If tests are being executed on an unsupported board, the test is @@ -306,7 +306,7 @@ def setup_boardspec(item): Returns: Nothing. - ''' + """ mark = item.get_marker('boardspec') if not mark: @@ -323,7 +323,7 @@ def setup_boardspec(item): pytest.skip('board not supported') def setup_buildconfigspec(item): - '''Process any 'buildconfigspec' marker for a test. + """Process any 'buildconfigspec' marker for a test. Such a marker lists some U-Boot configuration feature that the test requires. If tests are being executed on an U-Boot build that doesn't @@ -334,7 +334,7 @@ def setup_buildconfigspec(item): Returns: Nothing. - ''' + """ mark = item.get_marker('buildconfigspec') if not mark: @@ -344,7 +344,7 @@ def setup_buildconfigspec(item): pytest.skip('.config feature not enabled') def pytest_runtest_setup(item): - '''pytest hook: Configure (set up) a test item. + """pytest hook: Configure (set up) a test item. Called once for each test to perform any custom configuration. This hook is used to skip the test if certain conditions apply. @@ -354,14 +354,14 @@ def pytest_runtest_setup(item): Returns: Nothing. - ''' + """ log.start_section(item.name) setup_boardspec(item) setup_buildconfigspec(item) def pytest_runtest_protocol(item, nextitem): - '''pytest hook: Called to execute a test. + """pytest hook: Called to execute a test. This hook wraps the standard pytest runtestprotocol() function in order to acquire visibility into, and record, each test function's result. @@ -372,7 +372,7 @@ def pytest_runtest_protocol(item, nextitem): Returns: A list of pytest reports (test result data). - ''' + """ reports = runtestprotocol(item, nextitem=nextitem) failed = None -- cgit v1.2.3 From d20e5e976f70bd2e230787091a88278dfe6e5192 Mon Sep 17 00:00:00 2001 From: Stephen Warren Date: Tue, 26 Jan 2016 15:26:04 -0700 Subject: test/py: Provide custom IDs when parametrizing tests When pytest generates the name for parametrized tests, simple parameter values (ints, strings) get used directly, but more complex values such as dicts are not handled. This yields test names such as: dfu[env__usb_dev_port0-env__dfu_config0] dfu[env__usb_dev_port0-env__dfu_config1] Add some code to extract a custom fixture ID from the fixture values, so that we end up with meaningful names such as: dfu[micro_b-emmc] dfu[devport2-ram] If the boardenv file doesn't define custom names, the code falls back to the old algorithm. Signed-off-by: Stephen Warren Reviewed-by: Simon Glass --- test/py/conftest.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'test/py/conftest.py') diff --git a/test/py/conftest.py b/test/py/conftest.py index a4e54c66ced..9c9426aebe1 100644 --- a/test/py/conftest.py +++ b/test/py/conftest.py @@ -225,7 +225,13 @@ def pytest_generate_tests(metafunc): # ... otherwise, see if there's a key that contains a list of # values to use instead. vals = subconfig.get(fn + 's', []) - metafunc.parametrize(fn, vals) + def fixture_id(index, val): + try: + return val["fixture_id"] + except: + return fn + str(index) + ids = [fixture_id(index, val) for (index, val) in enumerate(vals)] + metafunc.parametrize(fn, vals, ids=ids) @pytest.fixture(scope='function') def u_boot_console(request): -- cgit v1.2.3 From 78b39cc3e19e698c04c2417ed5f79e324c90595e Mon Sep 17 00:00:00 2001 From: Stephen Warren Date: Wed, 27 Jan 2016 23:57:51 -0700 Subject: test/py: correctly log xfail/xpass tests Tests can complete in passed, skipped, xpass, xfailed, or failed, states. Currently the U-Boot log generation code doesn't handle the xfailed or xpass states since they aren't used. Add support for the remaining states. Without this, tests that xfail end up being reported as skipped. Signed-off-by: Stephen Warren Acked-by: Simon Glass --- test/py/conftest.py | 60 ++++++++++++++++++++++++++++++++++------------------- 1 file changed, 39 insertions(+), 21 deletions(-) (limited to 'test/py/conftest.py') diff --git a/test/py/conftest.py b/test/py/conftest.py index 9c9426aebe1..3e162cafcc4 100644 --- a/test/py/conftest.py +++ b/test/py/conftest.py @@ -249,6 +249,8 @@ def u_boot_console(request): tests_not_run = set() tests_failed = set() +tests_xpassed = set() +tests_xfailed = set() tests_skipped = set() tests_passed = set() @@ -289,6 +291,14 @@ def cleanup(): log.status_skipped('%d skipped' % len(tests_skipped)) for test in tests_skipped: log.status_skipped('... ' + test) + if tests_xpassed: + log.status_xpass('%d xpass' % len(tests_xpassed)) + for test in tests_xpassed: + log.status_xpass('... ' + test) + if tests_xfailed: + log.status_xfail('%d xfail' % len(tests_xfailed)) + for test in tests_xfailed: + log.status_xfail('... ' + test) if tests_failed: log.status_fail('%d failed' % len(tests_failed)) for test in tests_failed: @@ -381,34 +391,42 @@ def pytest_runtest_protocol(item, nextitem): """ reports = runtestprotocol(item, nextitem=nextitem) - failed = None - skipped = None + + failure_cleanup = False + test_list = tests_passed + msg = 'OK' + msg_log = log.status_pass for report in reports: if report.outcome == 'failed': - failed = report + if hasattr(report, 'wasxfail'): + test_list = tests_xpassed + msg = 'XPASSED' + msg_log = log.status_xpass + else: + failure_cleanup = True + test_list = tests_failed + msg = 'FAILED:\n' + str(report.longrepr) + msg_log = log.status_fail break if report.outcome == 'skipped': - if not skipped: - skipped = report - - if failed: + if hasattr(report, 'wasxfail'): + failure_cleanup = True + test_list = tests_xfailed + msg = 'XFAILED:\n' + str(report.longrepr) + msg_log = log.status_xfail + break + test_list = tests_skipped + msg = 'SKIPPED:\n' + str(report.longrepr) + msg_log = log.status_skipped + + if failure_cleanup: console.drain_console() - tests_failed.add(item.name) - elif skipped: - tests_skipped.add(item.name) - else: - tests_passed.add(item.name) + + test_list.add(item.name) tests_not_run.remove(item.name) try: - if failed: - msg = 'FAILED:\n' + str(failed.longrepr) - log.status_fail(msg) - elif skipped: - msg = 'SKIPPED:\n' + str(skipped.longrepr) - log.status_skipped(msg) - else: - log.status_pass('OK') + msg_log(msg) except: # If something went wrong with logging, it's better to let the test # process continue, which may report other exceptions that triggered @@ -424,7 +442,7 @@ def pytest_runtest_protocol(item, nextitem): log.end_section(item.name) - if failed: + if failure_cleanup: console.cleanup_spawn() return reports -- cgit v1.2.3