From 1facaadea1871c1b6962272dd21569a4aafda18c Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Thu, 20 Oct 2022 18:22:48 -0600 Subject: test: Report skippped tests At present it is possible for a test to skip itself by returning -EAGAIN but this is not recorded. An existing example is in test_pre_run() with the "Console recording disabled" check. Keep a track of skipped tests and report the total at the end. Signed-off-by: Simon Glass Acked-by: Heinrich Schuchardt --- test/test-main.c | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) (limited to 'test/test-main.c') diff --git a/test/test-main.c b/test/test-main.c index a98a77d68fc..2323cbaece5 100644 --- a/test/test-main.c +++ b/test/test-main.c @@ -356,6 +356,19 @@ static int test_post_run(struct unit_test_state *uts, struct unit_test *test) return 0; } +/** + * skip_test() - Handle skipping a test + * + * @uts: Test state to update + * @return -EAGAIN (always) + */ +static int skip_test(struct unit_test_state *uts) +{ + uts->skip_count++; + + return -EAGAIN; +} + /** * ut_run_test() - Run a single test * @@ -386,11 +399,13 @@ static int ut_run_test(struct unit_test_state *uts, struct unit_test *test, ret = test_pre_run(uts, test); if (ret == -EAGAIN) - return -EAGAIN; + return skip_test(uts); if (ret) return ret; - test->func(uts); + ret = test->func(uts); + if (ret == -EAGAIN) + skip_test(uts); ret = test_post_run(uts, test); if (ret) @@ -424,7 +439,7 @@ static int ut_run_test_live_flat(struct unit_test_state *uts, int runs; if ((test->flags & UT_TESTF_OTHER_FDT) && !IS_ENABLED(CONFIG_SANDBOX)) - return -EAGAIN; + return skip_test(uts); /* Run with the live tree if possible */ runs = 0; @@ -558,6 +573,8 @@ int ut_run_list(const char *category, const char *prefix, os_free(uts.other_fdt); } + if (uts.skip_count) + printf("Skipped: %d, ", uts.skip_count); if (ret == -ENOENT) printf("Test '%s' not found\n", select_name); else -- cgit v1.2.3 From cbd71fad6d468018727ab04b2bb912989aec0785 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Thu, 20 Oct 2022 18:22:50 -0600 Subject: test: Support tests which can only be run manually At present we normally write tests either in Python or in C. But most Python tests end up doing a lot of checks which would be better done in C. Checks done in C are orders of magnitude faster and it is possible to get full access to U-Boot's internal workings, rather than just relying on the command line. The model is to have a Python test set up some things and then use C code (in a unit test) to check that they were done correctly. But we don't want those checks to happen as part of normal test running, since each C unit tests is dependent on the associate Python tests, so cannot run without it. To acheive this, add a new UT_TESTF_MANUAL flag to use with the C 'check' tests, so that they can be skipped by default when the 'ut' command is used. Require that tests have a name ending with '_norun', so that pytest knows to skip them. Signed-off-by: Simon Glass --- test/test-main.c | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) (limited to 'test/test-main.c') diff --git a/test/test-main.c b/test/test-main.c index 2323cbaece5..ddfd89c089a 100644 --- a/test/test-main.c +++ b/test/test-main.c @@ -508,6 +508,30 @@ static int ut_run_tests(struct unit_test_state *uts, const char *prefix, if (!test_matches(prefix, test_name, select_name)) continue; + + if (test->flags & UT_TESTF_MANUAL) { + int len; + + /* + * manual tests must have a name ending "_norun" as this + * is how pytest knows to skip them. See + * generate_ut_subtest() for this check. + */ + len = strlen(test_name); + if (len < 6 || strcmp(test_name + len - 6, "_norun")) { + printf("Test %s is manual so must have a name ending in _norun\n", + test_name); + uts->fail_count++; + return -EBADF; + } + if (!uts->force_run) { + if (select_name) { + printf("Test %s skipped as it is manual (use -f to run it)\n", + test_name); + } + continue; + } + } old_fail_count = uts->fail_count; for (i = 0; i < uts->runs_per_test; i++) ret = ut_run_test_live_flat(uts, test, select_name); @@ -529,7 +553,7 @@ static int ut_run_tests(struct unit_test_state *uts, const char *prefix, int ut_run_list(const char *category, const char *prefix, struct unit_test *tests, int count, const char *select_name, - int runs_per_test) + int runs_per_test, bool force_run) { struct unit_test_state uts = { .fail_count = 0 }; bool has_dm_tests = false; @@ -563,6 +587,7 @@ int ut_run_list(const char *category, const char *prefix, } memcpy(uts.fdt_copy, gd->fdt_blob, uts.fdt_size); } + uts.force_run = force_run; ret = ut_run_tests(&uts, prefix, tests, count, select_name); /* Best efforts only...ignore errors */ -- cgit v1.2.3