From 5a68c382e15cd7fe8d721e9ffb7b20c08b2ff9e7 Mon Sep 17 00:00:00 2001 From: Stephen Lane-Walsh Date: Fri, 31 May 2024 15:26:34 -0400 Subject: [PATCH 1/2] Build: Multithread IDL Tests First attempt at multithreading the IDL tests TODO: The write tests still need a way to access mdsip, and now in parallel We'll probably use different shot numbers or something to that effect --- idl/testing/run_tests.py | 333 ++++++++++++++++++++++----------------- 1 file changed, 186 insertions(+), 147 deletions(-) diff --git a/idl/testing/run_tests.py b/idl/testing/run_tests.py index 244d6bdc4e..81453d283b 100755 --- a/idl/testing/run_tests.py +++ b/idl/testing/run_tests.py @@ -4,6 +4,7 @@ import subprocess import argparse import tempfile +import threading # The default values are intended to be used from within the PSFC network # If you want to run these tests on your own infrastructure, provide the @@ -231,7 +232,6 @@ args = parser.parse_args() - #--------------------------------------------------------------------------- # Each IDL write test should start with a clean tree. # Write tests use a local tree, but eventually will be upgraded to use mdsip. @@ -259,98 +259,126 @@ def build_write_tree(tree, shot): t.close() -# Temporary directory for transient test scripts and artifacts -TEST_DIR = tempfile.TemporaryDirectory(prefix='test_idl_', dir='/tmp') -os.environ['default_tree_path'] = TEST_DIR.name -os.environ['IDL_PATH'] = os.getenv('IDL_PATH') + ':' + TEST_DIR.name +class IDLTest(threading.Thread): -build_write_tree(args.write_tree, args.write_shot) + def __init__(self, name, code, expected_output): + super(IDLTest, self).__init__(name=name) -all_tests_passed = True -def idl_test(code, expected_output): - global all_tests_passed - - code_lines = [ line.strip() for line in code.splitlines() ] - code_lines = list(filter(None, code_lines)) - code = '\n'.join(code_lines) - - # Running IDL code at the IDL> prompt vs running it as a script causes - # weird differences in the evaluation, so we use a test.pro file - - code = 'pro test\n' + code + '\nend' - test_file = TEST_DIR.name + '/test.pro' - open(test_file, 'wt').write(code) - - expected_lines = [ line.strip() for line in expected_output.splitlines() ] - expected_lines = list(filter(None, expected_lines)) - - print('Running:') - for line in code_lines: - print(f'IDL> {line}') - print() - - print('Expected:') - for line in expected_lines: - print(line) - print() - - proc = subprocess.Popen( - ['idl'], - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT - ) - - # We call our test.pro module - - proc.stdin.write('test\nexit\n'.encode()) - proc.stdin.flush() - - hide_output = True - - print('Actual:') - lines = [] - while True: - line = proc.stdout.readline().decode() - if not line: - break - - line = line.strip() - - if line is not None and not hide_output: - print(line) - if line != '': - lines.append(line.strip()) - - # Skip the IDL header and licenseing information, which is everything - # above this line - if line == '% Compiled module: TEST.': - hide_output = False - print() - - this_test_passed = True - - for line, expected in zip(lines, expected_lines): - if line != expected: - print(f'`{line}` != `{expected}`') - this_test_passed = False - all_tests_passed = False - - if this_test_passed: - print('Success') - else: - print('Failure') - - print() - print('----------------------------------------------------------------\n') - print() + self.name = name + self.code = code + self.expected_output = expected_output + self.passed = False + + self.logfile = os.path.join(os.getcwd(), f'{self.name}.log') + + def run(self): + + log = open(self.logfile, 'wt') + + # Temporary directory for transient test scripts and artifacts + TEST_DIR = tempfile.TemporaryDirectory(prefix='test_idl_', dir='/tmp') + + code_lines = [ line.strip() for line in self.code.splitlines() ] + code_lines = list(filter(None, code_lines)) + code = '\n'.join(code_lines) + + # Running IDL code at the IDL> prompt vs running it as a script causes + # weird differences in the evaluation, so we use a test.pro file + + code = 'pro test\n' + code + '\nend' + test_file = TEST_DIR.name + '/test.pro' + open(test_file, 'wt').write(code) + + expected_lines = [ line.strip() for line in self.expected_output.splitlines() ] + expected_lines = list(filter(None, expected_lines)) + + log.write(f'Running {self.name}:\n') + for line in code_lines: + log.write(f'IDL> {line}\n') + log.write('\n') + + log.write('Expected:\n') + for line in expected_lines: + log.write(f'{line}\n') + log.write('\n') + + env = dict(os.environ) + env['IDL_PATH'] = env['IDL_PATH'] + ':' + TEST_DIR.name + + proc = subprocess.Popen( + ['idl'], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + env=env + ) + + # We call our test.pro module + + proc.stdin.write('test\nexit\n'.encode()) + proc.stdin.flush() + + hide_output = True + + log.write('Actual:\n') + lines = [] + while True: + line = proc.stdout.readline().decode() + if not line: + break + + line = line.strip() + + if line is not None and not hide_output: + log.write(f'{line}\n') + log.flush() + if line != '': + lines.append(line.strip()) + + # Skip the IDL header and licenseing information, which is everything + # above this line + if line == '% Compiled module: TEST.': + hide_output = False + log.write('\n') + + self.passed = True + + for line, expected in zip(lines, expected_lines): + if line != expected: + log.write(f'`{line}` != `{expected}`\n') + self.passed = False + + if self.passed: + log.write('Success\n') + else: + log.write(f'Failure, see {self.logfile}\n') + + log.write('\n') + log.write('----------------------------------------------------------------\n') + log.write('\n') + + log.close() + + with open(self.logfile, 'rt') as log: + print(log.read(), flush=True) + + +all_tests = [] +def idl_test(name, code, expected_output): + global all_tests + + print(f'Starting {name}') + + t = IDLTest(name, code, expected_output) + t.start() + all_tests.append(t) #------------------------------------------------------------------------------- # Tree open / read / close -idl_test(f''' - -testid = 'IDL-tree-read' +idl_test('IDL-tree-read', +f''' + mdsconnect, '{args.mdsip_server}' mdsopen, '{args.tree}', {args.shot} print, mdsvalue('{args.node1}') @@ -378,9 +406,9 @@ def idl_test(code, expected_output): # https://github.com/MDSplus/mdsplus/issues/2580 # Issue 2580: first connect should be on socket 0. # Very rare for returned status to be zero. -idl_test(f''' - -testid = 'IDL-2580-connect' +idl_test('IDL-2580-connect', +f''' + PASS = 1 FAIL = 0 BOGUS = -77 @@ -425,12 +453,12 @@ def idl_test(code, expected_output): # https://github.com/MDSplus/mdsplus/issues/2625 # Issue #2625: database on socket 0, subsequent connect doesn't break proxy. # If the queries work, the "val" variables will be changed to a text timestamp. - idl_test(f''' + idl_test('IDL-2625-simple', + f''' - testid = "IDL-2625-simple" PASS = 1 FAIL = 0 - BOGUS = -77 + BOGUS = -77 proxy = '{args.database_name}' mdsip_server = '{args.mdsip_server}' query = 'select getdate()' @@ -475,12 +503,12 @@ def idl_test(code, expected_output): #--------------------------------------------------------------------------- # https://github.com/MDSplus/mdsplus/issues/2625 # Issue #2625: usual pattern at GA is connect, then database. - idl_test(f''' + idl_test('IDL-2625-usual', + f''' - testid = "IDL-2625-usual" PASS = 1 FAIL = 0 - BOGUS = -77 + BOGUS = -77 proxy = '{args.database_name}' mdsip_server = '{args.mdsip_server}' query = 'select getdate()' @@ -523,12 +551,12 @@ def idl_test(code, expected_output): #--------------------------------------------------------------------------- # https://github.com/MDSplus/mdsplus/issues/2625 # Issue #2625: second database call not affected. - idl_test(f''' + idl_test('IDL-2625-two-database', + f''' - testid = "IDL-2625-two-database" PASS = 1 FAIL = 0 - BOGUS = -77 + BOGUS = -77 proxy = '{args.database_name}' mdsip_server = '{args.mdsip_server}' query = 'select getdate()' @@ -579,13 +607,13 @@ def idl_test(code, expected_output): # Issue #2625: stress test database proxy with many connects. # NLOOPS should be more than 64 (see Issue #2638). If the # disconnect works, will never exceed the concurrent limit. - idl_test(f''' + idl_test('IDL-2625-loop', + f''' - testid = "IDL-2625-loop" PASS = 1 FAIL = 0 BOGUS = -77 - NLOOPS = 100 + NLOOPS = 100 proxy = '{args.database_name}' mdsip_server = '{args.mdsip_server}' query = 'select getdate()' @@ -734,9 +762,9 @@ def idl_test(code, expected_output): # https://github.com/MDSplus/mdsplus/issues/2638 # Issue #2638: crashes with too many concurrent sockets. # NLOOPS should be more than 64. -idl_test(f''' +idl_test('IDL-2638-loop', +f''' -testid = "IDL-2638-loop" PASS = 1 FAIL = 0 NLOOPS = 100 @@ -869,9 +897,9 @@ def idl_test(code, expected_output): #------------------------------------------------------------------------------- # https://github.com/MDSplus/mdsplus/issues/2639 # Issue #2639: mdsvalue works without a socket -idl_test(f''' - -testid = 'IDL-2639-no-socket' +idl_test('IDL-2639-no-socket', +f''' + PASS = 1 FAIL = 0 DATA = '55' @@ -905,9 +933,9 @@ def idl_test(code, expected_output): #------------------------------------------------------------------------------- # https://github.com/MDSplus/mdsplus/issues/2639 # Issue #2639: mdsvalue and interaction with killed socket. -# idl_test(f''' - -# testid = 'IDL-2639-kill-last-socket' +# idl_test('IDL-2639-kill-last-socket', +# f''' + # PASS = 1 # FAIL = 0 # DATA = '55' @@ -952,9 +980,9 @@ def idl_test(code, expected_output): #------------------------------------------------------------------------------- # https://github.com/MDSplus/mdsplus/issues/2639 # Issue #2639: mdsvalue and interaction with killed socket 0. -idl_test(f''' - -testid = 'IDL-2639-kill-first-socket' +idl_test('IDL-2639-kill-first-socket', +f''' + PASS = 1 FAIL = 0 DATA = '55' @@ -999,9 +1027,9 @@ def idl_test(code, expected_output): #------------------------------------------------------------------------------- # https://github.com/MDSplus/mdsplus/issues/2639 # Issue #2639: mdsvalue and kill default socket. -idl_test(f''' - -testid = 'IDL-2639-kill-default-socket' +idl_test('IDL-2639-kill-default-socket', +f''' + PASS = 1 FAIL = 0 DATA = '55' @@ -1047,9 +1075,9 @@ def idl_test(code, expected_output): #------------------------------------------------------------------------------- # https://github.com/MDSplus/mdsplus/issues/2639 # Issue #2639: mdsvalue, one connect, and kill socket. -idl_test(f''' - -testid = 'IDL-2639-kill-single-socket' +idl_test('IDL-2639-kill-single-socket', +f''' + PASS = 1 FAIL = 0 DATA = '55' @@ -1094,9 +1122,9 @@ def idl_test(code, expected_output): # https://github.com/MDSplus/mdsplus/issues/2639 # Issue #2639: mdsvalue, one connect, and kill 0 socket. # Note different behavior from IDL-2639-kill-single-socket test. -# idl_test(f''' - -# testid = 'IDL-2639-kill-single-zero' +# idl_test('IDL-2639-kill-single-zero', +# f''' + # PASS = 1 # FAIL = 0 # DATA = '55' @@ -1142,9 +1170,9 @@ def idl_test(code, expected_output): # Issue #2640: disconnect returns correct status. # First disconnect should succeed and thus return True (1). # But disconnecting an already disconnected socket should return False (0). -# idl_test(f''' - -# testid = 'IDL-2640-status' +# idl_test('IDL-2640-status', +# f''' + # PASS = 1 # FAIL = 0 # BOGUS = -77 @@ -1189,9 +1217,9 @@ def idl_test(code, expected_output): #--------------------------------------------------------------------------- # Database: dbdisconnect kills database proxy - idl_test(f''' - - testid = 'IDL-db-dbdisconnect' + idl_test('IDL-db-dbdisconnect', + f''' + PASS = 1 FAIL = 0 BOGUS = -77 @@ -1245,9 +1273,9 @@ def idl_test(code, expected_output): # Database: regular disconnect kills database proxy. # GA usually has mdsconnect followed by set_database. # Note different behavior compared to IDL-db-dbdisconnect test. - idl_test(f''' - - testid = 'IDL-db-disconnect' + idl_test('IDL-db-disconnect', + f''' + PASS = 1 FAIL = 0 BOGUS = -77 @@ -1311,9 +1339,9 @@ def idl_test(code, expected_output): #--------------------------------------------------------------------------- # Sockets: a sequence of socket operations. - idl_test(f''' - - testid = 'IDL-socket-sequence' + idl_test('IDL-socket-sequence', + f''' + PASS = 1 FAIL = 0 BOGUS = -77 @@ -1390,9 +1418,9 @@ def idl_test(code, expected_output): #--------------------------------------------------------------------------- # Sockets: reset of the !MDS* system variables - idl_test(f''' - - testid = 'IDL-socket-reset' + idl_test('IDL-socket-reset', + f''' + PASS = 1 FAIL = 0 BOGUS = -77 @@ -1446,9 +1474,9 @@ def idl_test(code, expected_output): #--------------------------------------------------------------------------- # Sockets: explicitly reset socket 0. # Note different behavior than IDL-socket-reset test. - idl_test(f''' - - testid = 'IDL-socket-reset-zero' + idl_test('IDL-socket-reset-zero', + f''' + PASS = 1 FAIL = 0 BOGUS = -77 @@ -1501,9 +1529,8 @@ def idl_test(code, expected_output): #--------------------------------------------------------------------------- # Read: various permutations of reading data from a tree -idl_test(f''' - -testid = 'IDL-read-various' +idl_test('IDL-read-various', +f''' mdsconnect, '{args.mdsip_server}' mdsopen, '{args.tree}', {args.shot} @@ -1552,14 +1579,18 @@ def idl_test(code, expected_output): ''') +# TODO: Allow for threading, use mdsip +WRITE_TREE_DIR = tempfile.TemporaryDirectory(prefix='test_idl_', dir='/tmp') +os.environ['default_tree_path'] = WRITE_TREE_DIR.name +build_write_tree(args.write_tree, args.write_shot) + #--------------------------------------------------------------------------- # Write: various permutations of writing data to a tree # # Note: Uses local tree on the build server (i.e., not using mdsip). # The $default_tree_path must point to the "idl/testing" directory. -idl_test(f''' - -testid = 'IDL-write-various' +idl_test('IDL-write-various', +f''' mdsopen, '{args.write_tree}', '{args.write_shot}' mdsput, 'A_TEXT', ' "string_a" ' @@ -1571,7 +1602,7 @@ def idl_test(code, expected_output): mdsput, '.-.SUBTREE_2:F_NUM', '66' mdsput, '\TAG_G', '77' mdsclose, '{args.write_tree}', '{args.write_shot}' - + mdsopen, '{args.write_tree}', '{args.write_shot}' print, mdsvalue('A_TEXT') print, mdsvalue('B_NUM') @@ -1611,6 +1642,14 @@ def idl_test(code, expected_output): ''') +all_tests_passed = True +for test in all_tests: + test.join() + + if not test.passed: + print(f'Test {test.name} failed, see {test.logfile}') + all_tests_passed = False + if not all_tests_passed: exit(1) From cf5c57db70328c7173e508e7ea36b70bd6efa303 Mon Sep 17 00:00:00 2001 From: Timothy Heidcamp Date: Tue, 15 Apr 2025 15:10:50 -0400 Subject: [PATCH 2/2] IDL tests now output jUnit XML file --- idl/testing/run_tests.py | 43 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/idl/testing/run_tests.py b/idl/testing/run_tests.py index 81453d283b..6df1edcf73 100755 --- a/idl/testing/run_tests.py +++ b/idl/testing/run_tests.py @@ -5,6 +5,7 @@ import argparse import tempfile import threading +from datetime import datetime # The default values are intended to be used from within the PSFC network # If you want to run these tests on your own infrastructure, provide the @@ -258,6 +259,7 @@ def build_write_tree(tree, shot): t.write() t.close() +tests_failed_count = 0 class IDLTest(threading.Thread): @@ -269,9 +271,12 @@ def __init__(self, name, code, expected_output): self.expected_output = expected_output self.passed = False + self.time = 0 + self.logfile = os.path.join(os.getcwd(), f'{self.name}.log') def run(self): + start_time = datetime.now() log = open(self.logfile, 'wt') @@ -351,6 +356,7 @@ def run(self): if self.passed: log.write('Success\n') else: + tests_failed_count += 1 log.write(f'Failure, see {self.logfile}\n') log.write('\n') @@ -359,9 +365,12 @@ def run(self): log.close() + self.time = datetime.now() - start_time with open(self.logfile, 'rt') as log: print(log.read(), flush=True) +total_time_test = 0 +g_start_time = datetime.now() all_tests = [] def idl_test(name, code, expected_output): @@ -1650,6 +1659,40 @@ def idl_test(name, code, expected_output): print(f'Test {test.name} failed, see {test.logfile}') all_tests_passed = False +total_time_test = datetime.now() - g_start_time + +import xml.etree.ElementTree as xml + +root = xml.Element('testsuites') +root.attrib['time'] = str(total_time_test) +root.attrib['tests'] = str(len(all_tests)) +root.attrib['failures'] = str(tests_failed_count) + +testsuite = xml.SubElement(root, 'testsuite') +# testsuite.attrib['time'] = str(total_time_test) +testsuite.attrib['name'] = args.junit_suite_name + +for test in all_tests: + testcase = xml.SubElement(testsuite, 'testcase') + testcase.attrib['name'] = test.name + testcase.attrib['time'] = str(test.time) + + system_out = xml.SubElement(testcase, 'system-out') + system_out.text = open(test['log'], 'rt').read() + + # The BEL character causes issues when loaded into Jenkins + system_out.text = system_out.text.replace('\x07', '') + + if not test['passed']: + failure = xml.SubElement(testcase, 'failure') + failure.attrib['message'] = 'Failed' + + +junit_filename = os.path.join(args.workspace, 'mdsplus-junit.xml') +print(f'Writing jUnit XML to {junit_filename}') +with open(junit_filename, 'wb') as file: + file.write(xml.tostring(root)) + if not all_tests_passed: exit(1)