ST: Change CLI to be more fine grained in testsuite selection

serial_e2e_tests/BusVoodoo_e2e_tests.py

@@ -1,10 +1,11 @@
 #!/usr/bin/python
 # deps: pip install pyserial junit-xml termcolor yaml
 import argparse
+import datetime
 import itertools
 import re
 import serial
-import time
+import sys
 import yaml
 
 from junitparser import TestCase, TestSuite, JUnitXml, Skipped, Error, IntAttr, Attr
@@ -32,19 +33,22 @@ def create_testsuite(name):
     log('executing %s' % name)
     return TestSuite(name)
 
-def write_xml_report(testsuites):
+def write_xml_report(testsuites, name='busvoodoo_e2e_%s_testreport.xml' % datetime.datetime.now().strftime("%Y-%m-%d_%H:%m")):
 	xml = JUnitXml()
 	for testsuite in testsuites:
 		xml.add_testsuite(testsuite)
-	xml.write('busvoodoo_%s_testreport.xml' % 'date')
+	xml.write(name)
 
 # bv serial commands helper
 def bv_send(msg):
     bv_serial.write(b'%s\r' % msg)
 
-def reset_busvoodoo():
+def softreset_busvoodoo():
+    log('soft-resetting BusVoodoo via \'reset\' command')
+    # end potential protocol config sequence
     for i in range(10):
         bv_send('')
+    # quit to HiZ mode
     bv_send('q')
 
 def open_protocol(protocol):
@@ -54,29 +58,22 @@ def open_protocol(protocol):
 
 ### GENERIC TEST FUNCTIONNS###
 def generic_input(input, expectation):
-    #flush buffer befor sending command
+    #flush buffer before sending command
     bv_serial.readlines()
+
     bv_send(input)
-    output = (bv_serial.readlines())
-    output_stripped = ''.join(output)
+    output = ''.join(bv_serial.readlines())
 
     for e in expectation:
-        if e is not '' and e not in output_stripped:
-            return [ 1, output_stripped ]
-    return [ 0, output_stripped ]
+        if e is not '' and e not in output:
+            return [1, output]
+    return [0, output]
 
 def generic_input_test(input, expectation, testname):
     tc = TestCase(testname)
-    #flush buffer befor sending command
-    bv_serial.readlines()
-
-    bv_send(input)
-    output = (bv_serial.readlines())
-    output_stripped = ''.join(output)
-
-    for e in expectation:
-        if e not in output_stripped:
-            tc.result = Error(ansi_escape.sub('', output_stripped), 'error')
+    result = generic_input(input, expectation)
+    if result[0] > 0:
+            tc.result = Error(ansi_escape.sub('', result[1]), 'error')
             failure(testname)
             return tc
     success(testname)
@@ -88,7 +85,6 @@ def generic_inputs(inputs, expectations):
         error(err_msg)
         return [1, err_msg]
 
-    reset_busvoodoo()
     outputs = ''
 
     for i in range(len(inputs)):
@@ -99,6 +95,24 @@ def generic_inputs(inputs, expectations):
             return [1, outputs]
     return [0, outputs]
 
+def get_protocols_based_on_hw_verison(protocols):
+    # only one protocol specified
+    if protocols is not 'all' and ',' not in protocols:
+        return [protocols]
+    # used for default choices
+    if protocols is 'all':
+        protocols = yaml["protocols"]
+    # comma-separated list of protocols
+    elif ',' in protocols:
+        protocols = args.protocol_command_tests.split(',')
+
+    supported_protocols = []
+    for protocol in protocols:
+        supported_versions = yaml["protocols"][protocol]['hardware_version'].split(',')
+        if args.hardware_version in supported_versions:
+            supported_protocols.insert(len(supported_protocols),protocol)
+    return supported_protocols
+
 ### TEST SPECIIC FUNCTIONS
 def prot_default_settings_test(protocol):
     #
@@ -151,106 +165,110 @@ def pinstest():
     print "TBC..."
 
 ############### actual script #####################
-# argparse
-# so it could look like: BusVoodoo_e2e_tests.py --general_tests true --default_protocols_tests true --protocols_commands_test true
-description = 'This is a test program. It demonstrates how to use the argparse module with a program description.'
-parser = argparse.ArgumentParser()
-parser = argparse.ArgumentParser(description = description)
-parser.add_argument("-g", "--general_tests", action='store_true', help="Set to execute general tests")
-parser.add_argument("-p", "--protocol_command_tests", help="Execute commands test. Pass 'all' or comma separated string holding protocols 'spi,1-wire,uart'.")
-parser.add_argument("-c", "--protocol_combination_tests", help="Execute combination test. Pass 'all' or comma separated string holding protocols 'spi,1-wire,uart'.")
-parser.add_argument("-x", "--xml_report", help="Pass 'false' or path to disable report generation or specify absolute path (default is $PWD/busvoodoo_$(date)_test-report.xml)")
-parser.add_argument("-f", "--flavor", required=True, help="flavor of BusVoodoo under test (light,full)")
-parser.add_argument("-w", "--hardware_version", required=True, help="hardware version of BusVoodoo under test (v0,vA)")
-parser.add_argument("-s", "--serial_address", required=True, help="Path to BusVoodoo serial device e.g. '/dev/ttyACM0'")
-args = parser.parse_args()
-# ./BusVoodoo_e2e_tests.py -w v0 -f light -s /dev/ttyACM0 -g -p all -c all
-
 # bv output is ansi colored, so we need to replace
 # these colored and XML invalid chars in order to put
 # the output in the JUnit XML report for Jenkins.
 ansi_escape = re.compile(r'\x1B\[[0-?]*[ -/]*[@-~]')
 
-print args.general_tests
-print args.protocol_command_tests
-print args.flavor
-print args.hardware_version
-
-sys.exit(2)
-
 # ... so all testsuites can add themselves after executing.
 testsuites = []
 
+description = 'This is a test program. It demonstrates how to use the argparse module with a program description.'
+parser = argparse.ArgumentParser()
+parser = argparse.ArgumentParser(description = description)
+parser.add_argument("-W", "--hardware_version", required=True, help="hardware version of BusVoodoo under test (v0,vA)")
+# TODO: Ask whether the flavor is important for testing?
+#parser.add_argument("-F", "--flavor", required=True, help="flavor of BusVoodoo under test (light,full)")
+parser.add_argument("-S", "--serial_address", required=True, help="Path to BusVoodoo serial device e.g. '/dev/ttyACM0'")
+parser.add_argument("-t", "--testboard", help="Set serial address of BusVoodoo testboard to run all general tests.")
+parser.add_argument("-g", "--general_tests", action='store_true', help="Set to execute general tests")
+parser.add_argument("-d", "--default_protocols_test", action='store_true', help="Testsuite which opens every protocol mode supported by passed 'hardware_version'")
+parser.add_argument("-p", "--protocol_command_tests", help="Execute commands test. Pass 'all' or comma separated string holding protocols 'spi,1-wire,uart'.")
+parser.add_argument("-c", "--protocol_combination_tests", help="Execute combination test. Pass 'all' or comma separated string holding protocols 'spi,1-wire,uart'.")
+parser.add_argument("-x", "--xml_report", help="Pass 'false' or path to disable report generation or specify absolute path (default is $PWD/busvoodoo_$(date)_test-report.xml)")
+args = parser.parse_args()
 
-# TODO: write help
-
+# INIT & SANITY
 log('opening Busvoodoo configuration YAML file...')
-with open("BusVoodoo.yml", 'r') as stream:
+with open("busvoodoo_e2e_tests.yml", 'r') as stream:
     yaml = yaml.load(stream)
 
 log('init serial connection to BusVoodoo...')
 bv_serial = serial.Serial('/dev/ttyACM0', timeout=1)
+softreset_busvoodoo() # clean start
 
-#log('...init serial connection to testboard...')
-#tb_serial = serial.Serial('/dev/ttyACM1', timeout=1)
+log('verifying passed HW_VERSION with device HW_VERSION')
+if generic_input('v', ['hardware version: %s' % args.hardware_version])[0] > 0:
+    error("passed HW_VERSION does NOT match device HW_VERSION!")
+    sys.exit(1)
 
-reset_busvoodoo() # quit to HiZ mode
+if args.testboard:
+    log('...init serial connection to testboard...')
+    tb_serial = serial.Serial('/dev/ttyACM1', timeout=1)
 
-# GENERAL COMMANDS TESTS
-testsuite = create_testsuite('general commands tests')
-for command in yaml["commands"]:
-    expectation = yaml["commands"][command]['expectation']
-    for input in yaml["commands"][command]['input']:
-        testsuite.add_testcase(
-            generic_input_test(input, expectation,
-                # testname
-                "{0} [{1}]".format(command, input)))
-# special test cases of "general commands tests"
-for testcase in selftest():
-    testsuite.add_testcase(testcase)
-#for testcase in pinstest():it
-#   testsuite.add_testcase(testcase)
-add_testsuite(testsuite)
-
-# DEFAULT PROTOCOL TESTS
-testsuite = create_testsuite('default protocol tests')
-for protocol in yaml["protocols"]:
-    testsuite.add_testcase(prot_default_settings_test(protocol))
-add_testsuite(testsuite)
-
-# DEFAULT PROTOCOL COMMANDS TESTS
-testsuite = create_testsuite('spi commands tests')
-commands = yaml["protocols"]["spi"]["commands"]
-open_protocol('spi')
-for command in commands:
-    expectation = yaml["protocols"]["spi"]["commands"][command]
-    testsuite.add_testcase(generic_input_test('a %s' % command, expectation,
-        'spi command test: a %s' % command))
-add_testsuite(testsuite)
-
-
-log('writing xml report to file...')
-write_xml_report(testsuites)
 
-# run all combinations of choices for each protocol
-#for protocol in yaml["protocols"]:
-protocol = 'spi'
 
-inputs = yaml["protocols"][protocol]["combinations"]["inputs"]
-expectations = yaml["protocols"][protocol]["combinations"]["expectations"]
-permutations = list(itertools.product(*inputs))
-testsuite = create_testsuite('%s_configuration_combinations_tests' % protocol)
+# GENERAL COMMANDS TESTS
+if args.general_tests:
+    # test cases which lives in yaml
+    testsuite = create_testsuite('general commands tests')
+    for command in yaml["commands"]:
+        expectation = yaml["commands"][command]['expectation']
+        for input in yaml["commands"][command]['input']:
+            testsuite.add_testcase(
+                generic_input_test(input, expectation,
+                    # passing a testname for XML report readability
+                    "{0} [{1}]".format(command, input)))
+
+    # special test cases (too 'complex' for YAML config)
+    for testcase in selftest():
+        testsuite.add_testcase(testcase)
+
+    # special test cases which needs testboard
+    if args.testboard:
+        print 'tbc'
+        #for testcase in pinstest():
+        #testsuite.add_testcase(testcase)
+    add_testsuite(testsuite)
+
+# DEFAULT PROTOCOLS TESTS
+if args.default_protocols_test:
+    testsuite = create_testsuite('default protocols tests')
+    for protocol in get_protocols_based_on_hw_verison('all'):
+            testsuite.add_testcase(prot_default_settings_test(protocol))
+    add_testsuite(testsuite)
+
+# DEFAULT PROTOCOLS COMMANDS TESTS
+if args.protocol_command_tests:
+    for protocol in get_protocols_based_on_hw_verison(args.protocol_command_tests):
+        testsuite = create_testsuite('%s commands tests' % protocol)
+        open_protocol(protocol)
+        for command in yaml["protocols"][protocol]["commands"]:
+            expectation = yaml["protocols"][protocol]["commands"][command]
+            testsuite.add_testcase(generic_input_test('a %s' % command, expectation,
+                '{0} command test: a {1}'.format(protocol, command)))
+        add_testsuite(testsuite)
+    softreset_busvoodoo()
+
+# PROTOCOL COMBINATION TESTS
+if args.protocol_combination_tests:
+    for protocol in get_protocols_based_on_hw_verison(args.protocol_combination_tests):
+        inputs = yaml["protocols"][protocol]["combinations"]["inputs"]
+        expectations = yaml["protocols"][protocol]["combinations"]["expectations"]
+        testsuite = create_testsuite('%s_configuration_combinations_tests' % protocol)
+
+        for permutation in list(itertools.product(*inputs)):
+            name = '{0}_combination-test_{1}'.format(protocol, '-'.join(permutation))
+            tc = TestCase(name)
+            result = generic_inputs(permutation, expectations)
+            if result[0] > 0:
+                tc.Result = Error(result[1], 'error')
+                failure(name)
+            else:
+                success(name)
+            testsuite.add_testcase(tc)
+
+        add_testsuite(testsuite)
 
-for permutation in permutations:
-    name = 'spi_combination-test_%s' % '-'.join(permutation)
-    tc = TestCase(name)
-    result = generic_inputs(permutation, expectations)
-    if result[0] > 0:
-        error = result[1]
-        tc.Result = Error(error, 'error')
-        failure(name)
-    else:
-        success(name)
-    testsuite.add_testcase(tc)
-add_testsuite(testsuite)
+log('writing xml report to file...')
 write_xml_report(testsuites)

serial_e2e_tests/BusVoodoo.yml → serial_e2e_tests/busvoodoo_e2e_tests.yml

@@ -3,7 +3,7 @@
 #       quit, live in python script.
 
 commands:
-  # self-test and pins-test live in python script.
+  # 'self-test' and 'pins-test' and 'reset' live in python script.
   version:
     input: [ 'v', 'version' ]
     expectation: [ 'BusVoodoo flavor', 'firmware date', 'device ID' ]
@@ -28,6 +28,7 @@ commands:
 
 protocols:
   uart:
+    hardware_version: '0,A'
     commands:
     combinations:
       inputs:
@@ -46,8 +47,10 @@ protocols:
         - ['drive mode', 'push', 'external', 'embedded']
 
   i2c:
+    hardware_version: '0,A'
 
   spi:
+    hardware_version: '0,A'
     commands:
       # "[/]	select/deselect slave" is not tested
       10100: [ 'write: 0x74' ]
@@ -59,7 +62,6 @@ protocols:
       m: [ 'wait for 1 ms' ]
       m:42: [ 'wait for 42 ms' ]
       x: [ 'andre' ]
-
     combinations:
       inputs:
         - ['m spi']
@@ -79,5 +81,15 @@ protocols:
         - [ 'SPI:' ]
 
   1-wire:
+    hardware_version: '0,A'
     commands:
-    options:
+    combinations:
+      inputs:
+      expectations:
+
+  can:
+    hardware_version: 'A'
+    commands:
+    combinations:
+      inputs:
+      expectations: