From 6fc28df4dbdbdf7d4184edbb4678ec70d671edee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Boddenberg?= Date: Sat, 15 Sep 2018 23:52:37 +0200 Subject: [PATCH] ST: Change CLI to be more fine grained in testsuite selection --- serial_e2e_tests/BusVoodoo_e2e_tests.py | 224 ++++++++++-------- ...{BusVoodoo.yml => busvoodoo_e2e_tests.yml} | 18 +- 2 files changed, 136 insertions(+), 106 deletions(-) rename serial_e2e_tests/{BusVoodoo.yml => busvoodoo_e2e_tests.yml} (87%) diff --git a/serial_e2e_tests/BusVoodoo_e2e_tests.py b/serial_e2e_tests/BusVoodoo_e2e_tests.py index f745df7..470c083 100755 --- a/serial_e2e_tests/BusVoodoo_e2e_tests.py +++ b/serial_e2e_tests/BusVoodoo_e2e_tests.py @@ -1,10 +1,11 @@ #!/usr/bin/python # deps: pip install pyserial junit-xml termcolor yaml import argparse +import datetime import itertools import re import serial -import time +import sys import yaml from junitparser import TestCase, TestSuite, JUnitXml, Skipped, Error, IntAttr, Attr @@ -32,19 +33,22 @@ def create_testsuite(name): log('executing %s' % name) return TestSuite(name) -def write_xml_report(testsuites): +def write_xml_report(testsuites, name='busvoodoo_e2e_%s_testreport.xml' % datetime.datetime.now().strftime("%Y-%m-%d_%H:%m")): xml = JUnitXml() for testsuite in testsuites: xml.add_testsuite(testsuite) - xml.write('busvoodoo_%s_testreport.xml' % 'date') + xml.write(name) # bv serial commands helper def bv_send(msg): bv_serial.write(b'%s\r' % msg) -def reset_busvoodoo(): +def softreset_busvoodoo(): + log('soft-resetting BusVoodoo via \'reset\' command') + # end potential protocol config sequence for i in range(10): bv_send('') + # quit to HiZ mode bv_send('q') def open_protocol(protocol): @@ -54,29 +58,22 @@ def open_protocol(protocol): ### GENERIC TEST FUNCTIONNS### def generic_input(input, expectation): - #flush buffer befor sending command + #flush buffer before sending command bv_serial.readlines() + bv_send(input) - output = (bv_serial.readlines()) - output_stripped = ''.join(output) + output = ''.join(bv_serial.readlines()) for e in expectation: - if e is not '' and e not in output_stripped: - return [ 1, output_stripped ] - return [ 0, output_stripped ] + if e is not '' and e not in output: + return [1, output] + return [0, output] def generic_input_test(input, expectation, testname): tc = TestCase(testname) - #flush buffer befor sending command - bv_serial.readlines() - - bv_send(input) - output = (bv_serial.readlines()) - output_stripped = ''.join(output) - - for e in expectation: - if e not in output_stripped: - tc.result = Error(ansi_escape.sub('', output_stripped), 'error') + result = generic_input(input, expectation) + if result[0] > 0: + tc.result = Error(ansi_escape.sub('', result[1]), 'error') failure(testname) return tc success(testname) @@ -88,7 +85,6 @@ def generic_inputs(inputs, expectations): error(err_msg) return [1, err_msg] - reset_busvoodoo() outputs = '' for i in range(len(inputs)): @@ -99,6 +95,24 @@ def generic_inputs(inputs, expectations): return [1, outputs] return [0, outputs] +def get_protocols_based_on_hw_verison(protocols): + # only one protocol specified + if protocols is not 'all' and ',' not in protocols: + return [protocols] + # used for default choices + if protocols is 'all': + protocols = yaml["protocols"] + # comma-separated list of protocols + elif ',' in protocols: + protocols = args.protocol_command_tests.split(',') + + supported_protocols = [] + for protocol in protocols: + supported_versions = yaml["protocols"][protocol]['hardware_version'].split(',') + if args.hardware_version in supported_versions: + supported_protocols.insert(len(supported_protocols),protocol) + return supported_protocols + ### TEST SPECIIC FUNCTIONS def prot_default_settings_test(protocol): # @@ -151,106 +165,110 @@ def pinstest(): print "TBC..." ############### actual script ##################### -# argparse -# so it could look like: BusVoodoo_e2e_tests.py --general_tests true --default_protocols_tests true --protocols_commands_test true -description = 'This is a test program. It demonstrates how to use the argparse module with a program description.' -parser = argparse.ArgumentParser() -parser = argparse.ArgumentParser(description = description) -parser.add_argument("-g", "--general_tests", action='store_true', help="Set to execute general tests") -parser.add_argument("-p", "--protocol_command_tests", help="Execute commands test. Pass 'all' or comma separated string holding protocols 'spi,1-wire,uart'.") -parser.add_argument("-c", "--protocol_combination_tests", help="Execute combination test. Pass 'all' or comma separated string holding protocols 'spi,1-wire,uart'.") -parser.add_argument("-x", "--xml_report", help="Pass 'false' or path to disable report generation or specify absolute path (default is $PWD/busvoodoo_$(date)_test-report.xml)") -parser.add_argument("-f", "--flavor", required=True, help="flavor of BusVoodoo under test (light,full)") -parser.add_argument("-w", "--hardware_version", required=True, help="hardware version of BusVoodoo under test (v0,vA)") -parser.add_argument("-s", "--serial_address", required=True, help="Path to BusVoodoo serial device e.g. '/dev/ttyACM0'") -args = parser.parse_args() -# ./BusVoodoo_e2e_tests.py -w v0 -f light -s /dev/ttyACM0 -g -p all -c all - # bv output is ansi colored, so we need to replace # these colored and XML invalid chars in order to put # the output in the JUnit XML report for Jenkins. ansi_escape = re.compile(r'\x1B\[[0-?]*[ -/]*[@-~]') -print args.general_tests -print args.protocol_command_tests -print args.flavor -print args.hardware_version - -sys.exit(2) - # ... so all testsuites can add themselves after executing. testsuites = [] +description = 'This is a test program. It demonstrates how to use the argparse module with a program description.' +parser = argparse.ArgumentParser() +parser = argparse.ArgumentParser(description = description) +parser.add_argument("-W", "--hardware_version", required=True, help="hardware version of BusVoodoo under test (v0,vA)") +# TODO: Ask whether the flavor is important for testing? +#parser.add_argument("-F", "--flavor", required=True, help="flavor of BusVoodoo under test (light,full)") +parser.add_argument("-S", "--serial_address", required=True, help="Path to BusVoodoo serial device e.g. '/dev/ttyACM0'") +parser.add_argument("-t", "--testboard", help="Set serial address of BusVoodoo testboard to run all general tests.") +parser.add_argument("-g", "--general_tests", action='store_true', help="Set to execute general tests") +parser.add_argument("-d", "--default_protocols_test", action='store_true', help="Testsuite which opens every protocol mode supported by passed 'hardware_version'") +parser.add_argument("-p", "--protocol_command_tests", help="Execute commands test. Pass 'all' or comma separated string holding protocols 'spi,1-wire,uart'.") +parser.add_argument("-c", "--protocol_combination_tests", help="Execute combination test. Pass 'all' or comma separated string holding protocols 'spi,1-wire,uart'.") +parser.add_argument("-x", "--xml_report", help="Pass 'false' or path to disable report generation or specify absolute path (default is $PWD/busvoodoo_$(date)_test-report.xml)") +args = parser.parse_args() -# TODO: write help - +# INIT & SANITY log('opening Busvoodoo configuration YAML file...') -with open("BusVoodoo.yml", 'r') as stream: +with open("busvoodoo_e2e_tests.yml", 'r') as stream: yaml = yaml.load(stream) log('init serial connection to BusVoodoo...') bv_serial = serial.Serial('/dev/ttyACM0', timeout=1) +softreset_busvoodoo() # clean start -#log('...init serial connection to testboard...') -#tb_serial = serial.Serial('/dev/ttyACM1', timeout=1) +log('verifying passed HW_VERSION with device HW_VERSION') +if generic_input('v', ['hardware version: %s' % args.hardware_version])[0] > 0: + error("passed HW_VERSION does NOT match device HW_VERSION!") + sys.exit(1) -reset_busvoodoo() # quit to HiZ mode +if args.testboard: + log('...init serial connection to testboard...') + tb_serial = serial.Serial('/dev/ttyACM1', timeout=1) -# GENERAL COMMANDS TESTS -testsuite = create_testsuite('general commands tests') -for command in yaml["commands"]: - expectation = yaml["commands"][command]['expectation'] - for input in yaml["commands"][command]['input']: - testsuite.add_testcase( - generic_input_test(input, expectation, - # testname - "{0} [{1}]".format(command, input))) -# special test cases of "general commands tests" -for testcase in selftest(): - testsuite.add_testcase(testcase) -#for testcase in pinstest():it -# testsuite.add_testcase(testcase) -add_testsuite(testsuite) - -# DEFAULT PROTOCOL TESTS -testsuite = create_testsuite('default protocol tests') -for protocol in yaml["protocols"]: - testsuite.add_testcase(prot_default_settings_test(protocol)) -add_testsuite(testsuite) - -# DEFAULT PROTOCOL COMMANDS TESTS -testsuite = create_testsuite('spi commands tests') -commands = yaml["protocols"]["spi"]["commands"] -open_protocol('spi') -for command in commands: - expectation = yaml["protocols"]["spi"]["commands"][command] - testsuite.add_testcase(generic_input_test('a %s' % command, expectation, - 'spi command test: a %s' % command)) -add_testsuite(testsuite) - - -log('writing xml report to file...') -write_xml_report(testsuites) -# run all combinations of choices for each protocol -#for protocol in yaml["protocols"]: -protocol = 'spi' -inputs = yaml["protocols"][protocol]["combinations"]["inputs"] -expectations = yaml["protocols"][protocol]["combinations"]["expectations"] -permutations = list(itertools.product(*inputs)) -testsuite = create_testsuite('%s_configuration_combinations_tests' % protocol) +# GENERAL COMMANDS TESTS +if args.general_tests: + # test cases which lives in yaml + testsuite = create_testsuite('general commands tests') + for command in yaml["commands"]: + expectation = yaml["commands"][command]['expectation'] + for input in yaml["commands"][command]['input']: + testsuite.add_testcase( + generic_input_test(input, expectation, + # passing a testname for XML report readability + "{0} [{1}]".format(command, input))) + + # special test cases (too 'complex' for YAML config) + for testcase in selftest(): + testsuite.add_testcase(testcase) + + # special test cases which needs testboard + if args.testboard: + print 'tbc' + #for testcase in pinstest(): + #testsuite.add_testcase(testcase) + add_testsuite(testsuite) + +# DEFAULT PROTOCOLS TESTS +if args.default_protocols_test: + testsuite = create_testsuite('default protocols tests') + for protocol in get_protocols_based_on_hw_verison('all'): + testsuite.add_testcase(prot_default_settings_test(protocol)) + add_testsuite(testsuite) + +# DEFAULT PROTOCOLS COMMANDS TESTS +if args.protocol_command_tests: + for protocol in get_protocols_based_on_hw_verison(args.protocol_command_tests): + testsuite = create_testsuite('%s commands tests' % protocol) + open_protocol(protocol) + for command in yaml["protocols"][protocol]["commands"]: + expectation = yaml["protocols"][protocol]["commands"][command] + testsuite.add_testcase(generic_input_test('a %s' % command, expectation, + '{0} command test: a {1}'.format(protocol, command))) + add_testsuite(testsuite) + softreset_busvoodoo() + +# PROTOCOL COMBINATION TESTS +if args.protocol_combination_tests: + for protocol in get_protocols_based_on_hw_verison(args.protocol_combination_tests): + inputs = yaml["protocols"][protocol]["combinations"]["inputs"] + expectations = yaml["protocols"][protocol]["combinations"]["expectations"] + testsuite = create_testsuite('%s_configuration_combinations_tests' % protocol) + + for permutation in list(itertools.product(*inputs)): + name = '{0}_combination-test_{1}'.format(protocol, '-'.join(permutation)) + tc = TestCase(name) + result = generic_inputs(permutation, expectations) + if result[0] > 0: + tc.Result = Error(result[1], 'error') + failure(name) + else: + success(name) + testsuite.add_testcase(tc) + + add_testsuite(testsuite) -for permutation in permutations: - name = 'spi_combination-test_%s' % '-'.join(permutation) - tc = TestCase(name) - result = generic_inputs(permutation, expectations) - if result[0] > 0: - error = result[1] - tc.Result = Error(error, 'error') - failure(name) - else: - success(name) - testsuite.add_testcase(tc) -add_testsuite(testsuite) +log('writing xml report to file...') write_xml_report(testsuites) diff --git a/serial_e2e_tests/BusVoodoo.yml b/serial_e2e_tests/busvoodoo_e2e_tests.yml similarity index 87% rename from serial_e2e_tests/BusVoodoo.yml rename to serial_e2e_tests/busvoodoo_e2e_tests.yml index 61d3f84..2d04854 100644 --- a/serial_e2e_tests/BusVoodoo.yml +++ b/serial_e2e_tests/busvoodoo_e2e_tests.yml @@ -3,7 +3,7 @@ # quit, live in python script. commands: - # self-test and pins-test live in python script. + # 'self-test' and 'pins-test' and 'reset' live in python script. version: input: [ 'v', 'version' ] expectation: [ 'BusVoodoo flavor', 'firmware date', 'device ID' ] @@ -28,6 +28,7 @@ commands: protocols: uart: + hardware_version: '0,A' commands: combinations: inputs: @@ -46,8 +47,10 @@ protocols: - ['drive mode', 'push', 'external', 'embedded'] i2c: + hardware_version: '0,A' spi: + hardware_version: '0,A' commands: # "[/] select/deselect slave" is not tested 10100: [ 'write: 0x74' ] @@ -59,7 +62,6 @@ protocols: m: [ 'wait for 1 ms' ] m:42: [ 'wait for 42 ms' ] x: [ 'andre' ] - combinations: inputs: - ['m spi'] @@ -79,5 +81,15 @@ protocols: - [ 'SPI:' ] 1-wire: + hardware_version: '0,A' commands: - options: + combinations: + inputs: + expectations: + + can: + hardware_version: 'A' + commands: + combinations: + inputs: + expectations: