From 680e3312c23b72142a767c91eb1d74d36e1094fe Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Sun, 12 Nov 2017 21:52:08 -0700 Subject: binman: Rename tests to ftest At present these tests use the same filename as patman. This adds confusion when running all tests, since error messages look very similar. In fact binman tries to run the wrong tests at present. Rename the tests. Signed-off-by: Simon Glass --- tools/binman/binman.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'tools/binman/binman.py') diff --git a/tools/binman/binman.py b/tools/binman/binman.py index e75a59d9517..d264bcdfa8c 100755 --- a/tools/binman/binman.py +++ b/tools/binman/binman.py @@ -34,7 +34,7 @@ def RunTests(): """Run the functional tests and any embedded doctests""" import entry_test import fdt_test - import func_test + import ftest import test import doctest @@ -44,7 +44,7 @@ def RunTests(): suite.run(result) sys.argv = [sys.argv[0]] - for module in (func_test.TestFunctional, fdt_test.TestFdt, + for module in (ftest.TestFunctional, fdt_test.TestFdt, entry_test.TestEntry): suite = unittest.TestLoader().loadTestsFromTestCase(module) suite.run(result) -- cgit v1.2.3 From 5a3f2226043cd46d446ab6eba39e47d131b47175 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Sun, 12 Nov 2017 21:52:19 -0700 Subject: binman: Append to PYTHONPATH when running test coverage Rather that overwrite this, append to it, in case the caller has already set up the path correctly. Signed-off-by: Simon Glass --- tools/binman/binman.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'tools/binman/binman.py') diff --git a/tools/binman/binman.py b/tools/binman/binman.py index d264bcdfa8c..963d43a3761 100755 --- a/tools/binman/binman.py +++ b/tools/binman/binman.py @@ -58,7 +58,7 @@ def RunTests(): def RunTestCoverage(): """Run the tests and check that we get 100% coverage""" # This uses the build output from sandbox_spl to get _libfdt.so - cmd = ('PYTHONPATH=%s/sandbox_spl/tools coverage run ' + cmd = ('PYTHONPATH=$PYTHONPATH:%s/sandbox_spl/tools coverage run ' '--include "tools/binman/*.py" --omit "*test*,*binman.py" ' 'tools/binman/binman.py -t' % options.build_dir) os.system(cmd) -- cgit v1.2.3 From 934cdcfb1b1cac6a6c987f3f91e341c713770224 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Sun, 12 Nov 2017 21:52:21 -0700 Subject: binman: Add tests for importlib availability Add a test that the 'entry' module works with or without importlib. The tests are numbered so that they are executed in the correct order. Signed-off-by: Simon Glass --- tools/binman/binman.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'tools/binman/binman.py') diff --git a/tools/binman/binman.py b/tools/binman/binman.py index 963d43a3761..cf83edfd044 100755 --- a/tools/binman/binman.py +++ b/tools/binman/binman.py @@ -44,8 +44,12 @@ def RunTests(): suite.run(result) sys.argv = [sys.argv[0]] - for module in (ftest.TestFunctional, fdt_test.TestFdt, - entry_test.TestEntry): + + # Run the entry tests first ,since these need to be the first to import the + # 'entry' module. + suite = unittest.TestLoader().loadTestsFromTestCase(entry_test.TestEntry) + suite.run(result) + for module in (ftest.TestFunctional, fdt_test.TestFdt): suite = unittest.TestLoader().loadTestsFromTestCase(module) suite.run(result) -- cgit v1.2.3 From a25ebed36fcf95d09629e8d2e95fdf4907798fb5 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Sun, 12 Nov 2017 21:52:24 -0700 Subject: binman: Check for files missing from test coverage Files that are never imported are not shown in the test-coverage report. Detect these and show an error. Signed-off-by: Simon Glass --- tools/binman/binman.py | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) (limited to 'tools/binman/binman.py') diff --git a/tools/binman/binman.py b/tools/binman/binman.py index cf83edfd044..7ad4d3030b9 100755 --- a/tools/binman/binman.py +++ b/tools/binman/binman.py @@ -10,6 +10,7 @@ """See README for more information""" +import glob import os import sys import traceback @@ -67,12 +68,27 @@ def RunTestCoverage(): 'tools/binman/binman.py -t' % options.build_dir) os.system(cmd) stdout = command.Output('coverage', 'report') - coverage = stdout.splitlines()[-1].split(' ')[-1] + lines = stdout.splitlines() + + test_set= set([os.path.basename(line.split()[0]) + for line in lines if '/etype/' in line]) + glob_list = glob.glob(os.path.join(our_path, 'etype/*.py')) + all_set = set([os.path.basename(item) for item in glob_list]) + missing_list = all_set + missing_list.difference_update(test_set) + missing_list.remove('_testing.py') + coverage = lines[-1].split(' ')[-1] + ok = True + if missing_list: + print 'Missing tests for %s' % (', '.join(missing_list)) + ok = False if coverage != '100%': print stdout print "Type 'coverage html' to get a report in htmlcov/index.html" - raise ValueError('Coverage error: %s, but should be 100%%' % coverage) - + print 'Coverage error: %s, but should be 100%%' % coverage + ok = False + if not ok: + raise ValueError('Test coverage failure') def RunBinman(options, args): """Main entry point to binman once arguments are parsed -- cgit v1.2.3 From 9677faa34ee81c7abb4c08b0dc4ce4aace5473fc Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Sun, 12 Nov 2017 21:52:29 -0700 Subject: binman: Return non-zero exit code on test failure Return exit code 1 when test fail so that callers can detect this. Signed-off-by: Simon Glass --- tools/binman/binman.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'tools/binman/binman.py') diff --git a/tools/binman/binman.py b/tools/binman/binman.py index 7ad4d3030b9..3ccf25f1f88 100755 --- a/tools/binman/binman.py +++ b/tools/binman/binman.py @@ -58,7 +58,11 @@ def RunTests(): for test, err in result.errors: print test.id(), err for test, err in result.failures: - print err + print err, result.failures + if result.errors or result.failures: + print 'binman tests FAILED' + return 1 + return 0 def RunTestCoverage(): """Run the tests and check that we get 100% coverage""" @@ -106,7 +110,7 @@ def RunBinman(options, args): sys.tracebacklimit = 0 if options.test: - RunTests() + ret_code = RunTests() elif options.test_coverage: RunTestCoverage() -- cgit v1.2.3