tests: make integV2 locally runnable (#5029)

tests/integrationv2/.gitignore

@@ -0,0 +1,2 @@
+uv.lock
+.python-version

tests/integrationv2/README.md

@@ -10,6 +10,16 @@ have all the dependencies installed correctly. The integration test dependencies
  * Compiled Java SSLSocketClient for the Java provider
  * Compiled an s2nc executable named s2nc_head in the bin directory for the cross compatibility test
 
+Alternately, you can use the "best effort" mode with `uv`. This will only run the integration tests with the currently available binaries.
+```
+# install uv
+curl -LsSf https://astral.sh/uv/install.sh | sh
+# run pytest
+# -x: exit on the first failure
+# -rpfs: print a (r)eport with (p)assing, (f)ailed, and (s)kipped tests.
+uv run pytest --provider-version <LINKED_LIBCRYPTO> --best-effort-NOT-FOR-CI -x -rpfs -n auto
+```
+
 ## Run all tests
 
 The fastest way to run the integrationv2 tests is to run `make` from the S2N root directory.

tests/integrationv2/common.py

@@ -51,7 +51,12 @@ class AvailablePorts(object):
     """
 
     def __init__(self, low=8000, high=30000):
-        worker_count = int(os.getenv('PYTEST_XDIST_WORKER_COUNT'))
+        worker_count = 1
+        # If pytest is being run in parallel, worker processes will have
+        # the WORKER_COUNT variable set.
+        parallel_workers = os.getenv('PYTEST_XDIST_WORKER_COUNT')
+        if parallel_workers is not None:
+            worker_count = int(parallel_workers)
         chunk_size = int((high - low) / worker_count)
 
         # If xdist is being used, parse the workerid from the envvar. This can

tests/integrationv2/conftest.py

@@ -1,21 +1,65 @@
 # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0
+import os
+import pytest
 from global_flags import set_flag, S2N_PROVIDER_VERSION, S2N_FIPS_MODE
+from providers import S2N, JavaSSL
 
+PATH_CONFIGURATION_KEY = pytest.StashKey()
 
-def pytest_addoption(parser):
+
+def available_providers():
+    """
+    1. determine available providers
+    2. modify PATH to make the providers available
+
+    Currently only supports s2nc/s2nd and the Java SSL client.
+    """
+    providers = set()
+
+    # s2n-tls MUST be available, and we expect it to be in
+    # <git_root>/build/bin
+    expected_location = os.path.abspath("../../build/bin")
+    for binary in ["s2nd", "s2nc"]:
+        bin_path = f"{expected_location}/{binary}"
+        if not os.path.exists(bin_path):
+            pytest.fail(f"unable to locate {binary}")
+    os.environ['PATH'] += os.pathsep + expected_location
+    providers.add(S2N)
+
+    if os.path.exists("./bin/SSLSocketClient.class"):
+        providers.add(JavaSSL)
+
+    return providers
+
+
+def pytest_addoption(parser: pytest.Parser):
     parser.addoption("--provider-version", action="store", dest="provider-version",
                      default=None, type=str, help="Set the version of the TLS provider")
+    parser.addoption(
+        "--best-effort-NOT-FOR-CI",
+        action="store_true",
+        default=False,
+        help="""If enabled, run as many tests are possible 
+        for the discovered providers, and skip any providers 
+        that aren't available""",
+    )
 
 
-def pytest_configure(config):
+def pytest_configure(config: pytest.Config):
     """
     pytest hook that adds the function to deselect tests if the parameters
     don't makes sense.
+
+    This is executed once per pytest session on process startup.
     """
     config.addinivalue_line(
         "markers", "uncollect_if(*, func): function to unselect tests from parametrization"
     )
+
+    if config.getoption("--best-effort-NOT-FOR-CI"):
+        config.stash[PATH_CONFIGURATION_KEY] = available_providers()
+
     provider_version = config.getoption('provider-version', None)
     if "fips" in provider_version:
         set_flag(S2N_FIPS_MODE, True)

tests/integrationv2/fixtures.py

@@ -9,10 +9,11 @@
 from providers import Provider, S2N
 
 from common import ProviderOptions
+from conftest import PATH_CONFIGURATION_KEY
 
 
 @pytest.fixture
-def managed_process():
+def managed_process(request: pytest.FixtureRequest):
     """
     Generic process manager. This could be used to launch any process as a background
     task and cleanup when finished.
@@ -22,11 +23,27 @@ def managed_process():
     allows cleanup after a test, even if a failure occurred.
     """
     processes = []
+    # Indicates whether a launch was aborted. If so, non-graceful shutdown is allowed
+    aborted = False
 
     def _fn(provider_class: Provider, options: ProviderOptions, timeout=5, send_marker=None, close_marker=None,
             expect_stderr=None, kill_marker=None, send_with_newline=None):
+        best_effort_mode = request.config.getoption("--best-effort-NOT-FOR-CI")
+        if best_effort_mode:
+            # modify the `aborted` field in the generator object
+            nonlocal aborted
+            available_providers = request.config.stash[PATH_CONFIGURATION_KEY]
+            if provider_class not in available_providers:
+                aborted = True
+                pytest.skip(f"{provider_class} is not available")
+
         provider = provider_class(options)
         cmd_line = provider.get_cmd_line()
+
+        if best_effort_mode and provider_class is S2N and not (cmd_line[0] == "s2nc" or cmd_line[0] == "s2nd"):
+            aborted = True
+            pytest.skip("s2nc_head or s2nd_head not supported for best-effort")
+
         # The process will default to send markers in the providers.py file
         # if not specified by a test.
         if send_marker is not None:
@@ -67,7 +84,10 @@ def _fn(provider_class: Provider, options: ProviderOptions, timeout=5, send_mark
     finally:
         # Whether the processes succeeded or not, clean then up.
         for p in processes:
-            p.join()
+            if aborted:
+                p.kill()
+            else:
+                p.join()
 
 
 def _swap_mtu(device, new_mtu):

tests/integrationv2/providers.py

@@ -334,8 +334,6 @@ def setup_server(self):
 
 
 class OpenSSL(Provider):
-    _version = get_flag(S2N_PROVIDER_VERSION)
-
     def __init__(self, options: ProviderOptions):
         Provider.__init__(self, options)
         # We print some OpenSSL logging that includes stderr
@@ -390,7 +388,7 @@ def _cipher_to_cmdline(self, cipher):
 
     @classmethod
     def get_version(cls):
-        return cls._version
+        return get_flag(S2N_PROVIDER_VERSION)
 
     @classmethod
     def supports_protocol(cls, protocol, with_cert=None):

tests/integrationv2/pyproject.toml

@@ -0,0 +1,13 @@
+[project]
+name = "integrationv2"
+version = "0.0.1"
+description = "Integration tests for s2n-tls"
+readme = "README.md"
+requires-python = ">=3.12"
+
+dependencies = [
+    "pytest>=8.3.4",
+    "pytest-rerunfailures>=15.0",
+    "pytest-xdist>=3.6.1",
+    "sslyze>=6.1.0",
+]

tests/integrationv2/test_sslyze.py

@@ -22,7 +22,7 @@
     Protocols.TLS13
 ]
 
-SSLYZE_SCANS_TO_TEST = {
+SSLYZE_SCANS_TO_TEST = [
     sslyze.ScanCommand.ROBOT,
     sslyze.ScanCommand.SESSION_RESUMPTION,
     sslyze.ScanCommand.TLS_COMPRESSION,
@@ -31,7 +31,7 @@
     sslyze.ScanCommand.HEARTBLEED,
     sslyze.ScanCommand.OPENSSL_CCS_INJECTION,
     sslyze.ScanCommand.SESSION_RENEGOTIATION
-}
+]
 
 CERTS_TO_TEST = [
     cert for cert in ALL_TEST_CERTS if cert.name not in {