diff options
Diffstat (limited to 'tests/scripts/analyze_outcomes.py')
| -rwxr-xr-x | tests/scripts/analyze_outcomes.py | 720 |
1 files changed, 720 insertions, 0 deletions
diff --git a/tests/scripts/analyze_outcomes.py b/tests/scripts/analyze_outcomes.py new file mode 100755 index 00000000000..5b4deb62987 --- /dev/null +++ b/tests/scripts/analyze_outcomes.py @@ -0,0 +1,720 @@ +#!/usr/bin/env python3 + +"""Analyze the test outcomes from a full CI run. + +This script can also run on outcomes from a partial run, but the results are +less likely to be useful. +""" + +import argparse +import sys +import traceback +import re +import subprocess +import os +import typing + +import check_test_cases + + +# `ComponentOutcomes` is a named tuple which is defined as: +# ComponentOutcomes( +# successes = { +# "<suite_case>", +# ... +# }, +# failures = { +# "<suite_case>", +# ... +# } +# ) +# suite_case = "<suite>;<case>" +ComponentOutcomes = typing.NamedTuple('ComponentOutcomes', + [('successes', typing.Set[str]), + ('failures', typing.Set[str])]) + +# `Outcomes` is a representation of the outcomes file, +# which defined as: +# Outcomes = { +# "<component>": ComponentOutcomes, +# ... +# } +Outcomes = typing.Dict[str, ComponentOutcomes] + + +class Results: + """Process analysis results.""" + + def __init__(self): + self.error_count = 0 + self.warning_count = 0 + + def new_section(self, fmt, *args, **kwargs): + self._print_line('\n*** ' + fmt + ' ***\n', *args, **kwargs) + + def info(self, fmt, *args, **kwargs): + self._print_line('Info: ' + fmt, *args, **kwargs) + + def error(self, fmt, *args, **kwargs): + self.error_count += 1 + self._print_line('Error: ' + fmt, *args, **kwargs) + + def warning(self, fmt, *args, **kwargs): + self.warning_count += 1 + self._print_line('Warning: ' + fmt, *args, **kwargs) + + @staticmethod + def _print_line(fmt, *args, **kwargs): + sys.stderr.write((fmt + '\n').format(*args, **kwargs)) + +def execute_reference_driver_tests(results: Results, ref_component: str, driver_component: str, \ + outcome_file: str) -> None: + """Run the tests specified in ref_component and driver_component. Results + are stored in the output_file and they will be used for the following + coverage analysis""" + results.new_section("Test {} and {}", ref_component, driver_component) + + shell_command = "tests/scripts/all.sh --outcome-file " + outcome_file + \ + " " + ref_component + " " + driver_component + results.info("Running: {}", shell_command) + ret_val = subprocess.run(shell_command.split(), check=False).returncode + + if ret_val != 0: + results.error("failed to run reference/driver components") + +def analyze_coverage(results: Results, outcomes: Outcomes, + allow_list: typing.List[str], full_coverage: bool) -> None: + """Check that all available test cases are executed at least once.""" + available = check_test_cases.collect_available_test_cases() + for suite_case in available: + hit = any(suite_case in comp_outcomes.successes or + suite_case in comp_outcomes.failures + for comp_outcomes in outcomes.values()) + + if not hit and suite_case not in allow_list: + if full_coverage: + results.error('Test case not executed: {}', suite_case) + else: + results.warning('Test case not executed: {}', suite_case) + elif hit and suite_case in allow_list: + # Test Case should be removed from the allow list. + if full_coverage: + results.error('Allow listed test case was executed: {}', suite_case) + else: + results.warning('Allow listed test case was executed: {}', suite_case) + +def name_matches_pattern(name: str, str_or_re) -> bool: + """Check if name matches a pattern, that may be a string or regex. + - If the pattern is a string, name must be equal to match. + - If the pattern is a regex, name must fully match. + """ + # The CI's python is too old for re.Pattern + #if isinstance(str_or_re, re.Pattern): + if not isinstance(str_or_re, str): + return str_or_re.fullmatch(name) is not None + else: + return str_or_re == name + +def analyze_driver_vs_reference(results: Results, outcomes: Outcomes, + component_ref: str, component_driver: str, + ignored_suites: typing.List[str], ignored_tests=None) -> None: + """Check that all tests passing in the reference component are also + passing in the corresponding driver component. + Skip: + - full test suites provided in ignored_suites list + - only some specific test inside a test suite, for which the corresponding + output string is provided + """ + ref_outcomes = outcomes.get("component_" + component_ref) + driver_outcomes = outcomes.get("component_" + component_driver) + + if ref_outcomes is None or driver_outcomes is None: + results.error("required components are missing: bad outcome file?") + return + + if not ref_outcomes.successes: + results.error("no passing test in reference component: bad outcome file?") + return + + for suite_case in ref_outcomes.successes: + # suite_case is like "test_suite_foo.bar;Description of test case" + (full_test_suite, test_string) = suite_case.split(';') + test_suite = full_test_suite.split('.')[0] # retrieve main part of test suite name + + # Immediately skip fully-ignored test suites + if test_suite in ignored_suites or full_test_suite in ignored_suites: + continue + + # For ignored test cases inside test suites, just remember and: + # don't issue an error if they're skipped with drivers, + # but issue an error if they're not (means we have a bad entry). + ignored = False + if full_test_suite in ignored_tests: + for str_or_re in ignored_tests[full_test_suite]: + if name_matches_pattern(test_string, str_or_re): + ignored = True + + if not ignored and not suite_case in driver_outcomes.successes: + results.error("PASS -> SKIP/FAIL: {}", suite_case) + if ignored and suite_case in driver_outcomes.successes: + results.error("uselessly ignored: {}", suite_case) + +def analyze_outcomes(results: Results, outcomes: Outcomes, args) -> None: + """Run all analyses on the given outcome collection.""" + analyze_coverage(results, outcomes, args['allow_list'], + args['full_coverage']) + +def read_outcome_file(outcome_file: str) -> Outcomes: + """Parse an outcome file and return an outcome collection. + """ + outcomes = {} + with open(outcome_file, 'r', encoding='utf-8') as input_file: + for line in input_file: + (_platform, component, suite, case, result, _cause) = line.split(';') + # Note that `component` is not unique. If a test case passes on Linux + # and fails on FreeBSD, it'll end up in both the successes set and + # the failures set. + suite_case = ';'.join([suite, case]) + if component not in outcomes: + outcomes[component] = ComponentOutcomes(set(), set()) + if result == 'PASS': + outcomes[component].successes.add(suite_case) + elif result == 'FAIL': + outcomes[component].failures.add(suite_case) + + return outcomes + +def do_analyze_coverage(results: Results, outcomes: Outcomes, args) -> None: + """Perform coverage analysis.""" + results.new_section("Analyze coverage") + analyze_outcomes(results, outcomes, args) + +def do_analyze_driver_vs_reference(results: Results, outcomes: Outcomes, args) -> None: + """Perform driver vs reference analyze.""" + results.new_section("Analyze driver {} vs reference {}", + args['component_driver'], args['component_ref']) + + ignored_suites = ['test_suite_' + x for x in args['ignored_suites']] + + analyze_driver_vs_reference(results, outcomes, + args['component_ref'], args['component_driver'], + ignored_suites, args['ignored_tests']) + +# List of tasks with a function that can handle this task and additional arguments if required +KNOWN_TASKS = { + 'analyze_coverage': { + 'test_function': do_analyze_coverage, + 'args': { + 'allow_list': [ + # Algorithm not supported yet + 'test_suite_psa_crypto_metadata;Asymmetric signature: pure EdDSA', + # Algorithm not supported yet + 'test_suite_psa_crypto_metadata;Cipher: XTS', + ], + 'full_coverage': False, + } + }, + # There are 2 options to use analyze_driver_vs_reference_xxx locally: + # 1. Run tests and then analysis: + # - tests/scripts/all.sh --outcome-file "$PWD/out.csv" <component_ref> <component_driver> + # - tests/scripts/analyze_outcomes.py out.csv analyze_driver_vs_reference_xxx + # 2. Let this script run both automatically: + # - tests/scripts/analyze_outcomes.py out.csv analyze_driver_vs_reference_xxx + 'analyze_driver_vs_reference_hash': { + 'test_function': do_analyze_driver_vs_reference, + 'args': { + 'component_ref': 'test_psa_crypto_config_reference_hash_use_psa', + 'component_driver': 'test_psa_crypto_config_accel_hash_use_psa', + 'ignored_suites': [ + 'shax', 'mdx', # the software implementations that are being excluded + 'md.psa', # purposefully depends on whether drivers are present + 'psa_crypto_low_hash.generated', # testing the builtins + ], + 'ignored_tests': { + 'test_suite_platform': [ + # Incompatible with sanitizers (e.g. ASan). If the driver + # component uses a sanitizer but the reference component + # doesn't, we have a PASS vs SKIP mismatch. + 'Check mbedtls_calloc overallocation', + ], + } + } + }, + 'analyze_driver_vs_reference_hmac': { + 'test_function': do_analyze_driver_vs_reference, + 'args': { + 'component_ref': 'test_psa_crypto_config_reference_hmac', + 'component_driver': 'test_psa_crypto_config_accel_hmac', + 'ignored_suites': [ + # These suites require legacy hash support, which is disabled + # in the accelerated component. + 'shax', 'mdx', + # This suite tests builtins directly, but these are missing + # in the accelerated case. + 'psa_crypto_low_hash.generated', + ], + 'ignored_tests': { + 'test_suite_md': [ + # Builtin HMAC is not supported in the accelerate component. + re.compile('.*HMAC.*'), + # Following tests make use of functions which are not available + # when MD_C is disabled, as it happens in the accelerated + # test component. + re.compile('generic .* Hash file .*'), + 'MD list', + ], + 'test_suite_md.psa': [ + # "legacy only" tests require hash algorithms to be NOT + # accelerated, but this of course false for the accelerated + # test component. + re.compile('PSA dispatch .* legacy only'), + ], + 'test_suite_platform': [ + # Incompatible with sanitizers (e.g. ASan). If the driver + # component uses a sanitizer but the reference component + # doesn't, we have a PASS vs SKIP mismatch. + 'Check mbedtls_calloc overallocation', + ], + } + } + }, + 'analyze_driver_vs_reference_cipher_aead_cmac': { + 'test_function': do_analyze_driver_vs_reference, + 'args': { + 'component_ref': 'test_psa_crypto_config_reference_cipher_aead_cmac', + 'component_driver': 'test_psa_crypto_config_accel_cipher_aead_cmac', + # Modules replaced by drivers. + 'ignored_suites': [ + # low-level (block/stream) cipher modules + 'aes', 'aria', 'camellia', 'des', 'chacha20', + # AEAD modes and CMAC + 'ccm', 'chachapoly', 'cmac', 'gcm', + # The Cipher abstraction layer + 'cipher', + ], + 'ignored_tests': { + # PEM decryption is not supported so far. + # The rest of PEM (write, unencrypted read) works though. + 'test_suite_pem': [ + re.compile(r'PEM read .*(AES|DES|\bencrypt).*'), + ], + 'test_suite_platform': [ + # Incompatible with sanitizers (e.g. ASan). If the driver + # component uses a sanitizer but the reference component + # doesn't, we have a PASS vs SKIP mismatch. + 'Check mbedtls_calloc overallocation', + ], + # Following tests depend on AES_C/DES_C but are not about + # them really, just need to know some error code is there. + 'test_suite_error': [ + 'Low and high error', + 'Single low error' + ], + # Similar to test_suite_error above. + 'test_suite_version': [ + 'Check for MBEDTLS_AES_C when already present', + ], + # The en/decryption part of PKCS#12 is not supported so far. + # The rest of PKCS#12 (key derivation) works though. + 'test_suite_pkcs12': [ + re.compile(r'PBE Encrypt, .*'), + re.compile(r'PBE Decrypt, .*'), + ], + # The en/decryption part of PKCS#5 is not supported so far. + # The rest of PKCS#5 (PBKDF2) works though. + 'test_suite_pkcs5': [ + re.compile(r'PBES2 Encrypt, .*'), + re.compile(r'PBES2 Decrypt .*'), + ], + # Encrypted keys are not supported so far. + # pylint: disable=line-too-long + 'test_suite_pkparse': [ + 'Key ASN1 (Encrypted key PKCS12, trailing garbage data)', + 'Key ASN1 (Encrypted key PKCS5, trailing garbage data)', + re.compile(r'Parse (RSA|EC) Key .*\(.* ([Ee]ncrypted|password).*\)'), + ], + } + } + }, + 'analyze_driver_vs_reference_ecp_light_only': { + 'test_function': do_analyze_driver_vs_reference, + 'args': { + 'component_ref': 'test_psa_crypto_config_reference_ecc_ecp_light_only', + 'component_driver': 'test_psa_crypto_config_accel_ecc_ecp_light_only', + 'ignored_suites': [ + # Modules replaced by drivers + 'ecdsa', 'ecdh', 'ecjpake', + ], + 'ignored_tests': { + 'test_suite_platform': [ + # Incompatible with sanitizers (e.g. ASan). If the driver + # component uses a sanitizer but the reference component + # doesn't, we have a PASS vs SKIP mismatch. + 'Check mbedtls_calloc overallocation', + ], + # This test wants a legacy function that takes f_rng, p_rng + # arguments, and uses legacy ECDSA for that. The test is + # really about the wrapper around the PSA RNG, not ECDSA. + 'test_suite_random': [ + 'PSA classic wrapper: ECDSA signature (SECP256R1)', + ], + # In the accelerated test ECP_C is not set (only ECP_LIGHT is) + # so we must ignore disparities in the tests for which ECP_C + # is required. + 'test_suite_ecp': [ + re.compile(r'ECP check public-private .*'), + re.compile(r'ECP calculate public: .*'), + re.compile(r'ECP gen keypair .*'), + re.compile(r'ECP point muladd .*'), + re.compile(r'ECP point multiplication .*'), + re.compile(r'ECP test vectors .*'), + ], + 'test_suite_ssl': [ + # This deprecated function is only present when ECP_C is On. + 'Test configuration of groups for DHE through mbedtls_ssl_conf_curves()', + ], + } + } + }, + 'analyze_driver_vs_reference_no_ecp_at_all': { + 'test_function': do_analyze_driver_vs_reference, + 'args': { + 'component_ref': 'test_psa_crypto_config_reference_ecc_no_ecp_at_all', + 'component_driver': 'test_psa_crypto_config_accel_ecc_no_ecp_at_all', + 'ignored_suites': [ + # Modules replaced by drivers + 'ecp', 'ecdsa', 'ecdh', 'ecjpake', + ], + 'ignored_tests': { + 'test_suite_platform': [ + # Incompatible with sanitizers (e.g. ASan). If the driver + # component uses a sanitizer but the reference component + # doesn't, we have a PASS vs SKIP mismatch. + 'Check mbedtls_calloc overallocation', + ], + # See ecp_light_only + 'test_suite_random': [ + 'PSA classic wrapper: ECDSA signature (SECP256R1)', + ], + 'test_suite_pkparse': [ + # When PK_PARSE_C and ECP_C are defined then PK_PARSE_EC_COMPRESSED + # is automatically enabled in build_info.h (backward compatibility) + # even if it is disabled in config_psa_crypto_no_ecp_at_all(). As a + # consequence compressed points are supported in the reference + # component but not in the accelerated one, so they should be skipped + # while checking driver's coverage. + re.compile(r'Parse EC Key .*compressed\)'), + re.compile(r'Parse Public EC Key .*compressed\)'), + ], + # See ecp_light_only + 'test_suite_ssl': [ + 'Test configuration of groups for DHE through mbedtls_ssl_conf_curves()', + ], + } + } + }, + 'analyze_driver_vs_reference_ecc_no_bignum': { + 'test_function': do_analyze_driver_vs_reference, + 'args': { + 'component_ref': 'test_psa_crypto_config_reference_ecc_no_bignum', + 'component_driver': 'test_psa_crypto_config_accel_ecc_no_bignum', + 'ignored_suites': [ + # Modules replaced by drivers + 'ecp', 'ecdsa', 'ecdh', 'ecjpake', + 'bignum_core', 'bignum_random', 'bignum_mod', 'bignum_mod_raw', + 'bignum.generated', 'bignum.misc', + ], + 'ignored_tests': { + 'test_suite_platform': [ + # Incompatible with sanitizers (e.g. ASan). If the driver + # component uses a sanitizer but the reference component + # doesn't, we have a PASS vs SKIP mismatch. + 'Check mbedtls_calloc overallocation', + ], + # See ecp_light_only + 'test_suite_random': [ + 'PSA classic wrapper: ECDSA signature (SECP256R1)', + ], + # See no_ecp_at_all + 'test_suite_pkparse': [ + re.compile(r'Parse EC Key .*compressed\)'), + re.compile(r'Parse Public EC Key .*compressed\)'), + ], + 'test_suite_asn1parse': [ + 'INTEGER too large for mpi', + ], + 'test_suite_asn1write': [ + re.compile(r'ASN.1 Write mpi.*'), + ], + 'test_suite_debug': [ + re.compile(r'Debug print mbedtls_mpi.*'), + ], + # See ecp_light_only + 'test_suite_ssl': [ + 'Test configuration of groups for DHE through mbedtls_ssl_conf_curves()', + ], + } + } + }, + 'analyze_driver_vs_reference_ecc_ffdh_no_bignum': { + 'test_function': do_analyze_driver_vs_reference, + 'args': { + 'component_ref': 'test_psa_crypto_config_reference_ecc_ffdh_no_bignum', + 'component_driver': 'test_psa_crypto_config_accel_ecc_ffdh_no_bignum', + 'ignored_suites': [ + # Modules replaced by drivers + 'ecp', 'ecdsa', 'ecdh', 'ecjpake', 'dhm', + 'bignum_core', 'bignum_random', 'bignum_mod', 'bignum_mod_raw', + 'bignum.generated', 'bignum.misc', + ], + 'ignored_tests': { + 'test_suite_platform': [ + # Incompatible with sanitizers (e.g. ASan). If the driver + # component uses a sanitizer but the reference component + # doesn't, we have a PASS vs SKIP mismatch. + 'Check mbedtls_calloc overallocation', + ], + # See ecp_light_only + 'test_suite_random': [ + 'PSA classic wrapper: ECDSA signature (SECP256R1)', + ], + # See no_ecp_at_all + 'test_suite_pkparse': [ + re.compile(r'Parse EC Key .*compressed\)'), + re.compile(r'Parse Public EC Key .*compressed\)'), + ], + 'test_suite_asn1parse': [ + 'INTEGER too large for mpi', + ], + 'test_suite_asn1write': [ + re.compile(r'ASN.1 Write mpi.*'), + ], + 'test_suite_debug': [ + re.compile(r'Debug print mbedtls_mpi.*'), + ], + # See ecp_light_only + 'test_suite_ssl': [ + 'Test configuration of groups for DHE through mbedtls_ssl_conf_curves()', + ], + } + } + }, + 'analyze_driver_vs_reference_ffdh_alg': { + 'test_function': do_analyze_driver_vs_reference, + 'args': { + 'component_ref': 'test_psa_crypto_config_reference_ffdh', + 'component_driver': 'test_psa_crypto_config_accel_ffdh', + 'ignored_suites': ['dhm'], + 'ignored_tests': { + 'test_suite_platform': [ + # Incompatible with sanitizers (e.g. ASan). If the driver + # component uses a sanitizer but the reference component + # doesn't, we have a PASS vs SKIP mismatch. + 'Check mbedtls_calloc overallocation', + ], + } + } + }, + 'analyze_driver_vs_reference_tfm_config': { + 'test_function': do_analyze_driver_vs_reference, + 'args': { + 'component_ref': 'test_tfm_config', + 'component_driver': 'test_tfm_config_p256m_driver_accel_ec', + 'ignored_suites': [ + # Modules replaced by drivers + 'asn1parse', 'asn1write', + 'ecp', 'ecdsa', 'ecdh', 'ecjpake', + 'bignum_core', 'bignum_random', 'bignum_mod', 'bignum_mod_raw', + 'bignum.generated', 'bignum.misc', + ], + 'ignored_tests': { + 'test_suite_platform': [ + # Incompatible with sanitizers (e.g. ASan). If the driver + # component uses a sanitizer but the reference component + # doesn't, we have a PASS vs SKIP mismatch. + 'Check mbedtls_calloc overallocation', + ], + # See ecp_light_only + 'test_suite_random': [ + 'PSA classic wrapper: ECDSA signature (SECP256R1)', + ], + } + } + }, + 'analyze_driver_vs_reference_rsa': { + 'test_function': do_analyze_driver_vs_reference, + 'args': { + 'component_ref': 'test_psa_crypto_config_reference_rsa_crypto', + 'component_driver': 'test_psa_crypto_config_accel_rsa_crypto', + 'ignored_suites': [ + # Modules replaced by drivers. + 'rsa', 'pkcs1_v15', 'pkcs1_v21', + # We temporarily don't care about PK stuff. + 'pk', 'pkwrite', 'pkparse' + ], + 'ignored_tests': { + 'test_suite_platform': [ + # Incompatible with sanitizers (e.g. ASan). If the driver + # component uses a sanitizer but the reference component + # doesn't, we have a PASS vs SKIP mismatch. + 'Check mbedtls_calloc overallocation', + ], + # Following tests depend on RSA_C but are not about + # them really, just need to know some error code is there. + 'test_suite_error': [ + 'Low and high error', + 'Single high error' + ], + # Constant time operations only used for PKCS1_V15 + 'test_suite_constant_time': [ + re.compile(r'mbedtls_ct_zeroize_if .*'), + re.compile(r'mbedtls_ct_memmove_left .*') + ], + 'test_suite_psa_crypto': [ + # We don't support generate_key_ext entry points + # in drivers yet. + re.compile(r'PSA generate key ext: RSA, e=.*'), + ], + } + } + }, + 'analyze_block_cipher_dispatch': { + 'test_function': do_analyze_driver_vs_reference, + 'args': { + 'component_ref': 'test_full_block_cipher_legacy_dispatch', + 'component_driver': 'test_full_block_cipher_psa_dispatch', + 'ignored_suites': [ + # Skipped in the accelerated component + 'aes', 'aria', 'camellia', + # These require AES_C, ARIA_C or CAMELLIA_C to be enabled in + # order for the cipher module (actually cipher_wrapper) to work + # properly. However these symbols are disabled in the accelerated + # component so we ignore them. + 'cipher.ccm', 'cipher.gcm', 'cipher.aes', 'cipher.aria', + 'cipher.camellia', + ], + 'ignored_tests': { + 'test_suite_cmac': [ + # Following tests require AES_C/ARIA_C/CAMELLIA_C to be enabled, + # but these are not available in the accelerated component. + 'CMAC null arguments', + re.compile('CMAC.* (AES|ARIA|Camellia).*'), + ], + 'test_suite_cipher.padding': [ + # Following tests require AES_C/CAMELLIA_C to be enabled, + # but these are not available in the accelerated component. + re.compile('Set( non-existent)? padding with (AES|CAMELLIA).*'), + ], + 'test_suite_pkcs5': [ + # The AES part of PKCS#5 PBES2 is not yet supported. + # The rest of PKCS#5 (PBKDF2) works, though. + re.compile(r'PBES2 .* AES-.*') + ], + 'test_suite_pkparse': [ + # PEM (called by pkparse) requires AES_C in order to decrypt + # the key, but this is not available in the accelerated + # component. + re.compile('Parse RSA Key.*(password|AES-).*'), + ], + 'test_suite_pem': [ + # Following tests require AES_C, but this is diabled in the + # accelerated component. + re.compile('PEM read .*AES.*'), + 'PEM read (unknown encryption algorithm)', + ], + 'test_suite_error': [ + # Following tests depend on AES_C but are not about them + # really, just need to know some error code is there. + 'Single low error', + 'Low and high error', + ], + 'test_suite_version': [ + # Similar to test_suite_error above. + 'Check for MBEDTLS_AES_C when already present', + ], + 'test_suite_platform': [ + # Incompatible with sanitizers (e.g. ASan). If the driver + # component uses a sanitizer but the reference component + # doesn't, we have a PASS vs SKIP mismatch. + 'Check mbedtls_calloc overallocation', + ], + } + } + } +} + +def main(): + main_results = Results() + + try: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('outcomes', metavar='OUTCOMES.CSV', + help='Outcome file to analyze') + parser.add_argument('specified_tasks', default='all', nargs='?', + help='Analysis to be done. By default, run all tasks. ' + 'With one or more TASK, run only those. ' + 'TASK can be the name of a single task or ' + 'comma/space-separated list of tasks. ') + parser.add_argument('--list', action='store_true', + help='List all available tasks and exit.') + parser.add_argument('--require-full-coverage', action='store_true', + dest='full_coverage', help="Require all available " + "test cases to be executed and issue an error " + "otherwise. This flag is ignored if 'task' is " + "neither 'all' nor 'analyze_coverage'") + options = parser.parse_args() + + if options.list: + for task in KNOWN_TASKS: + print(task) + sys.exit(0) + + if options.specified_tasks == 'all': + tasks_list = KNOWN_TASKS.keys() + else: + tasks_list = re.split(r'[, ]+', options.specified_tasks) + for task in tasks_list: + if task not in KNOWN_TASKS: + sys.stderr.write('invalid task: {}\n'.format(task)) + sys.exit(2) + + KNOWN_TASKS['analyze_coverage']['args']['full_coverage'] = options.full_coverage + + # If the outcome file exists, parse it once and share the result + # among tasks to improve performance. + # Otherwise, it will be generated by execute_reference_driver_tests. + if not os.path.exists(options.outcomes): + if len(tasks_list) > 1: + sys.stderr.write("mutiple tasks found, please provide a valid outcomes file.\n") + sys.exit(2) + + task_name = tasks_list[0] + task = KNOWN_TASKS[task_name] + if task['test_function'] != do_analyze_driver_vs_reference: # pylint: disable=comparison-with-callable + sys.stderr.write("please provide valid outcomes file for {}.\n".format(task_name)) + sys.exit(2) + + execute_reference_driver_tests(main_results, + task['args']['component_ref'], + task['args']['component_driver'], + options.outcomes) + + outcomes = read_outcome_file(options.outcomes) + + for task in tasks_list: + test_function = KNOWN_TASKS[task]['test_function'] + test_args = KNOWN_TASKS[task]['args'] + test_function(main_results, outcomes, test_args) + + main_results.info("Overall results: {} warnings and {} errors", + main_results.warning_count, main_results.error_count) + + sys.exit(0 if (main_results.error_count == 0) else 1) + + except Exception: # pylint: disable=broad-except + # Print the backtrace and exit explicitly with our chosen status. + traceback.print_exc() + sys.exit(120) + +if __name__ == '__main__': + main() |