twister: Improve error handling for user's errors

Exit from twiser in case of user's error e.g. empty list
of tests provided by user instead of throwing unhandled
exception.

Signed-off-by: Lukasz Fundakowski <[email protected]>

Author: fundakol

scripts/pylib/twister/twisterlib/testplan.py

@@ -217,9 +217,10 @@ def discover(self):
                                 testsuite_pattern=self.options.test_pattern)
 
         if num == 0:
-            raise TwisterRuntimeError("No testsuites found at the specified location...")
+            print("No testsuites found at the specified location...", file=sys.stderr)
+            raise SystemExit(1)
         if self.load_errors:
-            raise TwisterRuntimeError(
+            raise SystemExit(
                 f"Found {self.load_errors} errors loading {num} test configurations."
             )
 
@@ -236,7 +237,7 @@ def discover(self):
         qv = self.options.quarantine_verify
         if qv and not ql:
             logger.error("No quarantine list given to be verified")
-            raise TwisterRuntimeError("No quarantine list given to be verified")
+            raise SystemExit("No quarantine list given to be verified")
         if ql:
             for quarantine_file in ql:
                 try:

scripts/tests/twister/test_testplan.py

@@ -629,9 +629,9 @@ def test_testplan_find_subtests(
 
 
 TESTDATA_3 = [
-    (0, 0, [], False, [], TwisterRuntimeError, []),
-    (1, 1, [], False, [], TwisterRuntimeError, []),
-    (1, 0, [], True, [], TwisterRuntimeError, ['No quarantine list given to be verified']),
+    (0, 0, [], False, [], SystemExit, []),
+    (1, 1, [], False, [], SystemExit, []),
+    (1, 0, [], True, [], SystemExit, ['No quarantine list given to be verified']),
     (1, 0, ['qfile.yaml'], False, ['- platforms:\n  - demo_board_3\n  comment: "board_3"'], None, []),
 ]
 
@@ -702,10 +702,10 @@ def test_testplan_discover(
 ]
 
 @pytest.mark.parametrize(
-    'report_suffix, only_failed, load_tests, test_only, subset,' \
-    ' exception, expected_selected_platforms, expected_generate_subset_args',
+    'report_suffix, only_failed, load_tests, test_only, subset, ' \
+    'exception, expected_selected_platforms, expected_generate_subset_args',
     TESTDATA_4,
-    ids=['apply_filters only', 'only failed', 'load tests', 'test only']
+    ids=['apply-filters-only', 'only-failed', 'load-tests', 'test-only']
 )
 def test_testplan_load(
     tmp_path,

scripts/tests/twister_blackbox/test_error.py

@@ -6,17 +6,16 @@
 Blackbox tests for twister's command line functions - simple does-error-out or not tests
 """
 
-import importlib
-from unittest import mock
 import os
 import pytest
-import sys
 import re
+import sys
+from unittest import mock
 
 # pylint: disable=no-name-in-module
-from conftest import ZEPHYR_BASE, TEST_DATA, suite_filename_mock
+from conftest import TEST_DATA, suite_filename_mock
 from twisterlib.testplan import TestPlan
-from twisterlib.error import TwisterRuntimeError
+from twisterlib.twister_main import main as twister_main
 
 
 class TestError:
@@ -26,17 +25,17 @@ class TestError:
             os.path.join('scripts', 'tests', 'twister_blackbox', 'test_data', 'tests',
                          'dummy', 'agnostic', 'group1', 'subgroup1',
                          'dummy.agnostic.group1.subgroup1'),
-            SystemExit
+            (0, '')
         ),
         (
             None,
             'dummy.agnostic.group1.subgroup1',
-            TwisterRuntimeError
+            (1, 'No testsuites found at the specified location...')
         ),
         (
             os.path.join(TEST_DATA, 'tests', 'dummy'),
             'dummy.agnostic.group1.subgroup1',
-            SystemExit
+            (0, '')
         )
     ]
     TESTDATA_2 = [
@@ -50,24 +49,13 @@ class TestError:
         )
     ]
 
-    @classmethod
-    def setup_class(cls):
-        apath = os.path.join(ZEPHYR_BASE, 'scripts', 'twister')
-        cls.loader = importlib.machinery.SourceFileLoader('__main__', apath)
-        cls.spec = importlib.util.spec_from_loader(cls.loader.name, cls.loader)
-        cls.twister_module = importlib.util.module_from_spec(cls.spec)
-
-    @classmethod
-    def teardown_class(cls):
-        pass
-
     @pytest.mark.parametrize(
-        'testroot, test, expected_exception',
+        'testroot, test, expected_return',
         TESTDATA_1,
         ids=['valid', 'invalid', 'valid']
     )
     @mock.patch.object(TestPlan, 'TESTSUITE_FILENAME', suite_filename_mock)
-    def test_test(self, out_path, testroot, test, expected_exception):
+    def test_test(self, out_path, testroot, test, expected_return, capsys):
         test_platforms = ['qemu_x86', 'intel_adl_crb']
         args = []
         if testroot:
@@ -77,13 +65,14 @@ def test_test(self, out_path, testroot, test, expected_exception):
                    ['-p'] * len(test_platforms), test_platforms
                ) for val in pair]
 
-        with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
-                pytest.raises(expected_exception) as exc:
-            self.loader.exec_module(self.twister_module)
+        expected_return_code, expected_message = expected_return
+
+        return_code = twister_main(args)
+        captured = capsys.readouterr()
 
-        if expected_exception == SystemExit:
-            assert str(exc.value) == '0'
-        assert True
+        assert return_code == expected_return_code
+        if expected_message:
+            assert expected_message in captured.err
 
     @pytest.mark.parametrize(
         'switch, expected',
@@ -106,18 +95,16 @@ def test_overflow_as_errors(self, capfd, out_path, switch, expected):
         if switch:
             args += [switch]
 
-        with mock.patch.object(sys, 'argv', [sys.argv[0]] + args), \
-            pytest.raises(SystemExit) as sys_exit:
-            self.loader.exec_module(self.twister_module)
+        return_code = twister_main(args)
 
         out, err = capfd.readouterr()
         sys.stdout.write(out)
         sys.stderr.write(err)
 
         print(args)
         if switch:
-            assert str(sys_exit.value) == '1'
+            assert return_code == 1
         else:
-            assert str(sys_exit.value) == '0'
+            assert return_code == 0
 
         assert re.search(expected, err)