diff --git a/.buildkite/pipelines/build_linux.json.py b/.buildkite/pipelines/build_linux.json.py index 56c6002aaf..efe313b155 100755 --- a/.buildkite/pipelines/build_linux.json.py +++ b/.buildkite/pipelines/build_linux.json.py @@ -72,7 +72,6 @@ def main(args): "RUN_TESTS": "true", "BOOST_TEST_OUTPUT_FORMAT_FLAGS": "--logger=JUNIT,error,boost_test_results.junit", }, - "artifact_paths": "*/**/unittest/boost_test_results.junit", "plugins": { "test-collector#v1.2.0": { "files": "*/*/unittest/boost_test_results.junit", diff --git a/.buildkite/scripts/steps/build_and_test.sh b/.buildkite/scripts/steps/build_and_test.sh index 5fc133874c..023d4c7488 100755 --- a/.buildkite/scripts/steps/build_and_test.sh +++ b/.buildkite/scripts/steps/build_and_test.sh @@ -106,7 +106,7 @@ fi if [[ -z "$CPP_CROSS_COMPILE" ]] ; then OS=$(uname -s | tr "A-Z" "a-z") TEST_RESULTS_ARCHIVE=${OS}-${HARDWARE_ARCH}-unit_test_results.tgz - find . -path "*/**/ml_test_*.out" -o -path "*/**/*.junit" | xargs tar cvzf ${TEST_RESULTS_ARCHIVE} + find . \( -path "*/**/ml_test_*.out" -o -path "*/**/*.junit" \) -print0 | tar czf ${TEST_RESULTS_ARCHIVE} --null -T - buildkite-agent artifact upload "${TEST_RESULTS_ARCHIVE}" fi diff --git a/3rd_party/3rd_party.cmake b/3rd_party/3rd_party.cmake index 2c9d796221..5878d7f86b 100644 --- a/3rd_party/3rd_party.cmake +++ b/3rd_party/3rd_party.cmake @@ -25,6 +25,10 @@ if(NOT INSTALL_DIR) message(FATAL_ERROR "INSTALL_DIR not specified") endif() +STRING(REPLACE "//" "/" INSTALL_DIR ${INSTALL_DIR}) + +message(STATUS "3rd_party: CMAKE_CXX_COMPILER_VERSION_MAJOR=${CMAKE_CXX_COMPILER_VERSION_MAJOR}") + string(TOLOWER ${CMAKE_HOST_SYSTEM_NAME} HOST_SYSTEM_NAME) message(STATUS "3rd_party: HOST_SYSTEM_NAME=${HOST_SYSTEM_NAME}") @@ -43,7 +47,9 @@ set(ARCH ${HOST_SYSTEM_PROCESSOR}) if ("${HOST_SYSTEM_NAME}" STREQUAL "darwin") message(STATUS "3rd_party: Copying macOS 3rd party libraries") set(BOOST_LOCATION "/usr/local/lib") - set(BOOST_COMPILER "clang") + set(BOOST_COMPILER "clang-darwin${CMAKE_CXX_COMPILER_VERSION_MAJOR}") + message(STATUS "3rd_party: BOOST_COMPILER=${BOOST_COMPILER}") + if( "${ARCH}" STREQUAL "x86_64" ) set(BOOST_ARCH "x64") else() @@ -63,7 +69,7 @@ elseif ("${HOST_SYSTEM_NAME}" STREQUAL "linux") if(NOT DEFINED ENV{CPP_CROSS_COMPILE} OR "$ENV{CPP_CROSS_COMPILE}" STREQUAL "") message(STATUS "3rd_party: NOT cross compiling. Copying Linux 3rd party libraries") set(BOOST_LOCATION "/usr/local/gcc133/lib") - set(BOOST_COMPILER "gcc") + set(BOOST_COMPILER "gcc${CMAKE_CXX_COMPILER_VERSION_MAJOR}") if( "${ARCH}" STREQUAL "aarch64" ) set(BOOST_ARCH "a64") else() @@ -93,7 +99,7 @@ elseif ("${HOST_SYSTEM_NAME}" STREQUAL "linux") message(STATUS "3rd_party: Cross compile for macosx: Copying macOS 3rd party libraries") set(SYSROOT "/usr/local/sysroot-x86_64-apple-macosx10.14") set(BOOST_LOCATION "${SYSROOT}/usr/local/lib") - set(BOOST_COMPILER "clang") + set(BOOST_COMPILER "clang-darwin${CMAKE_CXX_COMPILER_VERSION_MAJOR}") set(BOOST_EXTENSION "mt-x64-1_86.dylib") set(BOOST_LIBRARIES "atomic" "chrono" "date_time" "filesystem" "iostreams" "log" "log_setup" "program_options" "regex" "system" "thread" "unit_test_framework") set(XML_LOCATION) @@ -108,7 +114,7 @@ elseif ("${HOST_SYSTEM_NAME}" STREQUAL "linux") message(STATUS "3rd_party: Cross compile for linux-aarch64: Copying Linux 3rd party libraries") set(SYSROOT "/usr/local/sysroot-$ENV{CPP_CROSS_COMPILE}-linux-gnu") set(BOOST_LOCATION "${SYSROOT}/usr/local/gcc133/lib") - set(BOOST_COMPILER "gcc") + set(BOOST_COMPILER "gcc${CMAKE_CXX_COMPILER_VERSION_MAJOR}") if("$ENV{CPP_CROSS_COMPILE}" STREQUAL "aarch64") set(BOOST_ARCH "a64") else() @@ -188,6 +194,9 @@ function(install_libs _target _source_dir _prefix _postfix) set(LIBRARIES ${ARGN}) + message(STATUS "_target=${_target} _source_dir=${_source_dir} _prefix=${_prefix} _postfix=${_postfix} LIBRARIES=${LIBRARIES}") + + file(GLOB _LIBS ${_source_dir}/*${_prefix}*${_postfix}) if(_LIBS) @@ -219,7 +228,7 @@ function(install_libs _target _source_dir _prefix _postfix) endif() file(CHMOD ${INSTALL_DIR}/${_LIB} PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE) else() - file(COPY ${_RESOLVED_PATH} DESTINATION ${INSTALL_DIR}) + file(COPY ${_RESOLVED_PATH} DESTINATION "${INSTALL_DIR}") file(CHMOD ${INSTALL_DIR}/${_RESOLVED_LIB} PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE) endif() endforeach() diff --git a/3rd_party/CMakeLists.txt b/3rd_party/CMakeLists.txt index ffdb8e093e..f2b092f913 100644 --- a/3rd_party/CMakeLists.txt +++ b/3rd_party/CMakeLists.txt @@ -22,7 +22,7 @@ add_custom_target(licenses ALL # as part of the CMake configuration step - avoiding # the need for it to be done on every build execute_process( - COMMAND ${CMAKE_COMMAND} -DINSTALL_DIR=${INSTALL_DIR} -P ./3rd_party.cmake + COMMAND ${CMAKE_COMMAND} -DINSTALL_DIR=${INSTALL_DIR} -DCMAKE_CXX_COMPILER_VERSION_MAJOR=${CMAKE_CXX_COMPILER_VERSION_MAJOR} -P ./3rd_party.cmake WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} ) diff --git a/cmake/compiler/clang.cmake b/cmake/compiler/clang.cmake index 1749ad0a89..cc4042dbc7 100644 --- a/cmake/compiler/clang.cmake +++ b/cmake/compiler/clang.cmake @@ -16,7 +16,7 @@ set(CMAKE_RANLIB "ranlib") set(CMAKE_STRIP "strip") -list(APPEND ML_C_FLAGS +list(APPEND ML_C_FLAGS ${CROSS_FLAGS} ${ARCHCFLAGS} "-fstack-protector" diff --git a/cmake/functions.cmake b/cmake/functions.cmake index c39a86089a..ea8070a712 100644 --- a/cmake/functions.cmake +++ b/cmake/functions.cmake @@ -392,6 +392,13 @@ function(ml_add_test_executable _target) COMMENT "Running test: ml_test_${_target}" WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} ) + + add_custom_target(test_${_target}_individually + DEPENDS ml_test_${_target} + COMMAND ${CMAKE_SOURCE_DIR}/run_tests_as_seperate_processes.sh ${CMAKE_BINARY_DIR} ${CMAKE_CURRENT_BINARY_DIR} test_${_target} + COMMENT "Running test: ml_test_${_target}_individually" + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + ) endif() endfunction() @@ -420,8 +427,10 @@ function(ml_add_test _directory _target) add_subdirectory(../${_directory} ${_directory}) list(APPEND ML_BUILD_TEST_DEPENDS ml_test_${_target}) list(APPEND ML_TEST_DEPENDS test_${_target}) + list(APPEND ML_TEST_INDIVIDUALLY_DEPENDS test_${_target}_individually) set(ML_BUILD_TEST_DEPENDS ${ML_BUILD_TEST_DEPENDS} PARENT_SCOPE) set(ML_TEST_DEPENDS ${ML_TEST_DEPENDS} PARENT_SCOPE) + set(ML_TEST_INDIVIDUALLY_DEPENDS ${ML_TEST_INDIVIDUALLY_DEPENDS} PARENT_SCOPE) endfunction() diff --git a/cmake/test-runner.cmake b/cmake/test-runner.cmake index 2bba0cb5c4..b17d85ba6f 100644 --- a/cmake/test-runner.cmake +++ b/cmake/test-runner.cmake @@ -9,30 +9,44 @@ # limitation. # -if(TEST_NAME STREQUAL "ml_test_seccomp") - execute_process(COMMAND ${TEST_DIR}/${TEST_NAME} $ENV{BOOST_TEST_OUTPUT_FORMAT_FLAGS} --logger=HRF,all --report_format=HRF --show_progress=no --no_color_output OUTPUT_FILE ${TEST_DIR}/${TEST_NAME}.out ERROR_FILE ${TEST_DIR}/${TEST_NAME}.out RESULT_VARIABLE TEST_SUCCESS) -else() - # Turn the TEST_FLAGS environment variable into a CMake list variable - if (DEFINED ENV{TEST_FLAGS} AND NOT "$ENV{TEST_FLAGS}" STREQUAL "") - string(REPLACE " " ";" TEST_FLAGS $ENV{TEST_FLAGS}) - endif() +execute_process(COMMAND ${CMAKE_COMMAND} -E rm -f ${TEST_DIR}/*.out) +execute_process(COMMAND ${CMAKE_COMMAND} -E rm -f ${TEST_DIR}/*.failed) +execute_process(COMMAND ${CMAKE_COMMAND} -E rm -f boost_test_results*.xml) +execute_process(COMMAND ${CMAKE_COMMAND} -E rm -f boost_test_results*.junit) - # Special case for specifying a subset of tests to run (can be regex) - if (DEFINED ENV{TESTS} AND NOT "$ENV{TESTS}" STREQUAL "") - set(TESTS "--run_test=$ENV{TESTS}") - endif() +# Turn the TEST_FLAGS environment variable into a CMake list variable +if (DEFINED ENV{TEST_FLAGS} AND NOT "$ENV{TEST_FLAGS}" STREQUAL "") + string(REPLACE " " ";" TEST_FLAGS $ENV{TEST_FLAGS}) +endif() + +set(SAFE_TEST_NAME "") +set(TESTS "") +# Special case for specifying a subset of tests to run (can be regex) +if (DEFINED ENV{TESTS} AND NOT "$ENV{TESTS}" STREQUAL "") + set(TESTS "--run_test=$ENV{TESTS}") + string(REGEX REPLACE "[^a-zA-Z0-9_]" "_" SAFE_TEST_NAME "$ENV{TESTS}") + set(SAFE_TEST_NAME "_${SAFE_TEST_NAME}") +endif() + +string(REPLACE "boost_test_results" "boost_test_results${SAFE_TEST_NAME}" BOOST_TEST_OUTPUT_FORMAT_FLAGS "$ENV{BOOST_TEST_OUTPUT_FORMAT_FLAGS}") +set(OUTPUT_FILE "${TEST_DIR}/${TEST_NAME}${SAFE_TEST_NAME}.out") +set(FAILED_FILE "${TEST_DIR}/${TEST_NAME}${SAFE_TEST_NAME}.failed") - # If any special command line args are present run the tests in the foreground - if (DEFINED TEST_FLAGS OR DEFINED TESTS) - message(STATUS "executing process ${TEST_DIR}/${TEST_NAME} ${TEST_FLAGS} ${TESTS} $ENV{BOOST_TEST_OUTPUT_FORMAT_FLAGS}") - execute_process(COMMAND ${TEST_DIR}/${TEST_NAME} ${TEST_FLAGS} ${TESTS} $ENV{BOOST_TEST_OUTPUT_FORMAT_FLAGS} RESULT_VARIABLE TEST_SUCCESS) +# If env var RUN_BOOST_TESTS_IN_FOREGROUND is defined run the tests in the foreground +if(TEST_NAME STREQUAL "ml_test_seccomp") + execute_process(COMMAND ${TEST_DIR}/${TEST_NAME} ${TEST_FLAGS} ${TESTS} ${BOOST_TEST_OUTPUT_FORMAT_FLAGS} --logger=HRF,all --report_format=HRF --show_progress=no --no_color_output OUTPUT_FILE ${OUTPUT_FILE} ERROR_FILE ${OUTPUT_FILE} RESULT_VARIABLE TEST_SUCCESS) +else() + if(NOT DEFINED ENV{RUN_BOOST_TESTS_IN_FOREGROUND}) + execute_process(COMMAND ${TEST_DIR}/${TEST_NAME} ${TEST_FLAGS} ${TESTS} ${BOOST_TEST_OUTPUT_FORMAT_FLAGS} --no_color_output OUTPUT_FILE ${OUTPUT_FILE} ERROR_FILE ${OUTPUT_FILE} RESULT_VARIABLE TEST_SUCCESS) else() - execute_process(COMMAND ${TEST_DIR}/${TEST_NAME} $ENV{TEST_FLAGS} $ENV{BOOST_TEST_OUTPUT_FORMAT_FLAGS} - --no_color_output OUTPUT_FILE ${TEST_DIR}/${TEST_NAME}.out ERROR_FILE ${TEST_DIR}/${TEST_NAME}.out RESULT_VARIABLE TEST_SUCCESS) + execute_process(COMMAND ${TEST_DIR}/${TEST_NAME} ${TEST_FLAGS} ${TESTS} ${BOOST_TEST_OUTPUT_FORMAT_FLAGS} RESULT_VARIABLE TEST_SUCCESS) endif() endif() if (NOT TEST_SUCCESS EQUAL 0) - execute_process(COMMAND ${CMAKE_COMMAND} -E cat ${TEST_DIR}/${TEST_NAME}.out) - file(WRITE "${TEST_DIR}/${TEST_NAME}.failed" "") + if (EXISTS ${TEST_DIR}/${TEST_NAME}) + execute_process(COMMAND ${CMAKE_COMMAND} -E cat ${OUTPUT_FILE}) + file(WRITE "${FAILED_FILE}" "") + endif() endif() + diff --git a/dev-tools/docker/docker_entrypoint.sh b/dev-tools/docker/docker_entrypoint.sh index 475c05344f..8311efffe4 100755 --- a/dev-tools/docker/docker_entrypoint.sh +++ b/dev-tools/docker/docker_entrypoint.sh @@ -66,6 +66,6 @@ if [ "x$1" = "x--test" ] ; then # failure is the unit tests, and then the detailed test results can be # copied from the image echo passed > build/test_status.txt - cmake --build cmake-build-docker ${CMAKE_VERBOSE} -j`nproc` -t test || echo failed > build/test_status.txt + cmake --build cmake-build-docker ${CMAKE_VERBOSE} -j $(nproc) -t test_individually || echo failed > build/test_status.txt fi diff --git a/dev-tools/docker_test.sh b/dev-tools/docker_test.sh index aed18fa609..22a46f4c9f 100755 --- a/dev-tools/docker_test.sh +++ b/dev-tools/docker_test.sh @@ -92,7 +92,10 @@ do # Using tar to copy the build and test artifacts out of the container seems # more reliable than docker cp, and also means the files end up with the # correct uid/gid - docker run --rm --workdir=/ml-cpp $TEMP_TAG bash -c "find . $EXTRACT_FIND | xargs tar cf - $EXTRACT_EXPLICIT && sleep 30" | tar xvf - + docker run --rm --workdir=/ml-cpp $TEMP_TAG bash -c "find . \( $EXTRACT_FIND \) -print0 | tar cf - $EXTRACT_EXPLICIT --null -T -" | tar xvf - + if [ $? != 0 ]; then + echo "Copying build and test artifacts from docker container failed" + fi docker rmi --force $TEMP_TAG # The image build is set to return zero (i.e. succeed as far as Docker is # concerned) when the only problem is that the unit tests fail, as this diff --git a/lib/api/unittest/CMultiFileDataAdderTest.cc b/lib/api/unittest/CMultiFileDataAdderTest.cc index 96231c5679..f03dd81fd6 100644 --- a/lib/api/unittest/CMultiFileDataAdderTest.cc +++ b/lib/api/unittest/CMultiFileDataAdderTest.cc @@ -37,6 +37,8 @@ #include #include #include +#include // For random number generation facilities +#include #include #include @@ -100,7 +102,16 @@ void detectorPersistHelper(const std::string& configFileName, // Persist the detector state to file(s) - std::string baseOrigOutputFilename(ml::test::CTestTmpDir::tmpDir() + "/orig"); + // Create a random number to use to generate a unique file name for each test + // this allows tests to be run successfully in parallel + std::random_device rd; + std::mt19937 gen(rd()); + std::uniform_int_distribution<> distrib(1, 100); + std::ostringstream oss; + oss << distrib(gen); + + std::string baseOrigOutputFilename(ml::test::CTestTmpDir::tmpDir() + + "/orig_" + oss.str()); { // Clean up any leftovers of previous failures boost::filesystem::path origDir(baseOrigOutputFilename); diff --git a/lib/core/unittest/CLoggerTest.cc b/lib/core/unittest/CLoggerTest.cc index 1abec95da0..e626fb8fcd 100644 --- a/lib/core/unittest/CLoggerTest.cc +++ b/lib/core/unittest/CLoggerTest.cc @@ -49,12 +49,13 @@ class CTestFixture { } }; -std::function makeReader(std::ostringstream& loggedData) { - return [&loggedData] { +std::function makeReader(std::ostringstream& loggedData, const std::string& pipeName) { + return [&loggedData, &pipeName]() { + for (std::size_t attempt = 1; attempt <= 100; ++attempt) { // wait a bit so that pipe has been created std::this_thread::sleep_for(std::chrono::milliseconds(50)); - std::ifstream strm(TEST_PIPE_NAME); + std::ifstream strm(pipeName); if (strm.is_open()) { std::copy(std::istreambuf_iterator(strm), std::istreambuf_iterator(), @@ -62,7 +63,7 @@ std::function makeReader(std::ostringstream& loggedData) { return; } } - BOOST_FAIL("Failed to connect to logging pipe within a reasonable time"); + BOOST_FAIL("Failed to connect to logging pipe " + pipeName + " within a reasonable time"); }; } @@ -204,12 +205,13 @@ BOOST_FIXTURE_TEST_CASE(testNonAsciiJsonLogging, CTestFixture) { "Non-iso8859-15: 编码 test", "surrogate pair: 𐐷 test"}; std::ostringstream loggedData; - std::thread reader(makeReader(loggedData)); + const std::string& pipeName = std::string{TEST_PIPE_NAME} + "_testNonAsciiJsonLogging"; + std::thread reader(makeReader(loggedData, pipeName)); ml::core::CLogger& logger = ml::core::CLogger::instance(); // logger might have been reconfigured in previous tests, so reset and reconfigure it logger.reset(); - logger.reconfigure(TEST_PIPE_NAME, ""); + logger.reconfigure(pipeName, ""); for (const auto& m : messages) { LOG_INFO(<< m); @@ -225,14 +227,16 @@ BOOST_FIXTURE_TEST_CASE(testNonAsciiJsonLogging, CTestFixture) { BOOST_FIXTURE_TEST_CASE(testWarnAndErrorThrottling, CTestFixture) { std::ostringstream loggedData; - std::thread reader{makeReader(loggedData)}; + const std::string& pipeName = std::string{TEST_PIPE_NAME} + "_testWarnAndErrorThrottling"; + + std::thread reader{makeReader(loggedData, pipeName)}; TStrVec messages{"Warn should only be seen once", "Error should only be seen once"}; ml::core::CLogger& logger = ml::core::CLogger::instance(); // logger might have been reconfigured in previous tests, so reset and reconfigure it logger.reset(); - logger.reconfigure(TEST_PIPE_NAME, ""); + logger.reconfigure(pipeName, ""); for (std::size_t i = 0; i < 10; ++i) { LOG_WARN(<< messages[0]); diff --git a/lib/core/unittest/CNamedPipeFactoryTest.cc b/lib/core/unittest/CNamedPipeFactoryTest.cc index 39aef5e07e..19dc01323b 100644 --- a/lib/core/unittest/CNamedPipeFactoryTest.cc +++ b/lib/core/unittest/CNamedPipeFactoryTest.cc @@ -38,9 +38,9 @@ const std::size_t MAX_ATTEMPTS{100}; const std::size_t TEST_SIZE{10000}; const char TEST_CHAR{'a'}; #ifdef Windows -const char* const TEST_PIPE_NAME{"\\\\.\\pipe\\testpipe"}; +const std::string TEST_PIPE_NAME{"\\\\.\\pipe\\testpipe"}; #else -const char* const TEST_PIPE_NAME{"testfiles/testpipe"}; +const std::string TEST_PIPE_NAME{"testfiles/testpipe"}; #endif class CThreadBlockCanceller : public ml::core::CThread { @@ -71,13 +71,13 @@ class CThreadBlockCanceller : public ml::core::CThread { } BOOST_AUTO_TEST_CASE(testServerIsCppReader) { - ml::test::CThreadDataWriter threadWriter{SLEEP_TIME_MS, TEST_PIPE_NAME, - TEST_CHAR, TEST_SIZE}; + const std::string pipeName = TEST_PIPE_NAME + "_testServerIsCppReader"; + ml::test::CThreadDataWriter threadWriter{SLEEP_TIME_MS, pipeName, TEST_CHAR, TEST_SIZE}; BOOST_TEST_REQUIRE(threadWriter.start()); std::atomic_bool dummy{false}; ml::core::CNamedPipeFactory::TIStreamP strm{ - ml::core::CNamedPipeFactory::openPipeStreamRead(TEST_PIPE_NAME, dummy)}; + ml::core::CNamedPipeFactory::openPipeStreamRead(pipeName, dummy)}; BOOST_TEST_REQUIRE(strm); static const std::streamsize BUF_SIZE{512}; @@ -100,13 +100,14 @@ BOOST_AUTO_TEST_CASE(testServerIsCppReader) { } BOOST_AUTO_TEST_CASE(testServerIsCReader) { - ml::test::CThreadDataWriter threadWriter{SLEEP_TIME_MS, TEST_PIPE_NAME, - TEST_CHAR, TEST_SIZE}; + const std::string pipeName = TEST_PIPE_NAME + "_testServerIsCReader"; + + ml::test::CThreadDataWriter threadWriter{SLEEP_TIME_MS, pipeName, TEST_CHAR, TEST_SIZE}; BOOST_TEST_REQUIRE(threadWriter.start()); std::atomic_bool dummy{false}; ml::core::CNamedPipeFactory::TFileP file{ - ml::core::CNamedPipeFactory::openPipeFileRead(TEST_PIPE_NAME, dummy)}; + ml::core::CNamedPipeFactory::openPipeFileRead(pipeName, dummy)}; BOOST_TEST_REQUIRE(file); static const std::size_t BUF_SIZE{512}; @@ -129,12 +130,14 @@ BOOST_AUTO_TEST_CASE(testServerIsCReader) { } BOOST_AUTO_TEST_CASE(testServerIsCppWriter) { - ml::test::CThreadDataReader threadReader{PAUSE_TIME_MS, MAX_ATTEMPTS, TEST_PIPE_NAME}; + const std::string pipeName = TEST_PIPE_NAME + "_testServerIsCppWriter"; + + ml::test::CThreadDataReader threadReader{PAUSE_TIME_MS, MAX_ATTEMPTS, pipeName}; BOOST_TEST_REQUIRE(threadReader.start()); std::atomic_bool dummy{false}; ml::core::CNamedPipeFactory::TOStreamP strm{ - ml::core::CNamedPipeFactory::openPipeStreamWrite(TEST_PIPE_NAME, dummy)}; + ml::core::CNamedPipeFactory::openPipeStreamWrite(pipeName, dummy)}; BOOST_TEST_REQUIRE(strm); std::size_t charsLeft{TEST_SIZE}; @@ -159,12 +162,14 @@ BOOST_AUTO_TEST_CASE(testServerIsCppWriter) { } BOOST_AUTO_TEST_CASE(testServerIsCWriter) { - ml::test::CThreadDataReader threadReader{PAUSE_TIME_MS, MAX_ATTEMPTS, TEST_PIPE_NAME}; + const std::string pipeName = TEST_PIPE_NAME + "_testServerIsCWriter"; + + ml::test::CThreadDataReader threadReader{PAUSE_TIME_MS, MAX_ATTEMPTS, pipeName}; BOOST_TEST_REQUIRE(threadReader.start()); std::atomic_bool dummy{false}; ml::core::CNamedPipeFactory::TFileP file{ - ml::core::CNamedPipeFactory::openPipeFileWrite(TEST_PIPE_NAME, dummy)}; + ml::core::CNamedPipeFactory::openPipeFileWrite(pipeName, dummy)}; BOOST_TEST_REQUIRE(file); std::size_t charsLeft{TEST_SIZE}; @@ -193,14 +198,14 @@ BOOST_AUTO_TEST_CASE(testCancelBlock) { BOOST_TEST_REQUIRE(cancellerThread.start()); ml::core::CNamedPipeFactory::TOStreamP strm{ml::core::CNamedPipeFactory::openPipeStreamWrite( - TEST_PIPE_NAME, cancellerThread.hasCancelledBlockingCall())}; + TEST_PIPE_NAME + "_testCancelBlock", cancellerThread.hasCancelledBlockingCall())}; BOOST_TEST_REQUIRE(strm == nullptr); BOOST_TEST_REQUIRE(cancellerThread.stop()); } BOOST_AUTO_TEST_CASE(testErrorIfRegularFile) { - std::atomic_bool dummy{false}; + const std::atomic_bool dummy{false}; ml::core::CNamedPipeFactory::TIStreamP strm{ ml::core::CNamedPipeFactory::openPipeStreamRead("Main.cc", dummy)}; BOOST_TEST_REQUIRE(strm == nullptr); @@ -215,23 +220,24 @@ BOOST_AUTO_TEST_CASE(testErrorIfSymlink) { // Suppress the error about no assertions in this case BOOST_REQUIRE(BOOST_IS_DEFINED(Windows)); #else - static const char* const TEST_SYMLINK_NAME{"test_symlink"}; + const std::string TEST_SYMLINK_NAME{"test_symlink_testErrorIfSymlink"}; + const std::string testPipeName{TEST_PIPE_NAME + "_test_symlink_testErrorIfSymlink"}; // Remove any files left behind by a previous failed test, but don't check // the return codes as these calls will usually fail - ::unlink(TEST_SYMLINK_NAME); - ::unlink(TEST_PIPE_NAME); + ::unlink(TEST_SYMLINK_NAME.c_str()); + ::unlink(testPipeName.c_str()); - BOOST_REQUIRE_EQUAL(0, ::mkfifo(TEST_PIPE_NAME, S_IRUSR | S_IWUSR)); - BOOST_REQUIRE_EQUAL(0, ::symlink(TEST_PIPE_NAME, TEST_SYMLINK_NAME)); + BOOST_REQUIRE_EQUAL(0, ::mkfifo(testPipeName.c_str(), S_IRUSR | S_IWUSR)); + BOOST_REQUIRE_EQUAL(0, ::symlink(testPipeName.c_str(), TEST_SYMLINK_NAME.c_str())); std::atomic_bool dummy{false}; ml::core::CNamedPipeFactory::TIStreamP strm{ ml::core::CNamedPipeFactory::openPipeStreamRead(TEST_SYMLINK_NAME, dummy)}; BOOST_TEST_REQUIRE(strm == nullptr); - BOOST_REQUIRE_EQUAL(0, ::unlink(TEST_SYMLINK_NAME)); - BOOST_REQUIRE_EQUAL(0, ::unlink(TEST_PIPE_NAME)); + BOOST_REQUIRE_EQUAL(0, ::unlink(TEST_SYMLINK_NAME.c_str())); + BOOST_REQUIRE_EQUAL(0, ::unlink(testPipeName.c_str())); #endif } diff --git a/run_tests_as_seperate_processes.sh b/run_tests_as_seperate_processes.sh new file mode 100755 index 0000000000..deb60a4298 --- /dev/null +++ b/run_tests_as_seperate_processes.sh @@ -0,0 +1,179 @@ +#!/bin/bash +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License +# 2.0 and the following additional limitation. Functionality enabled by the +# files subject to the Elastic License 2.0 may only be used in production when +# invoked by an Elasticsearch process with a license key installed that permits +# use of machine learning features. You may not use this file except in +# compliance with the Elastic License 2.0 and the foregoing additional +# limitation. +# + +# This script ultimately gets called from within the docker entry point script. +# It provides a wrapper around the call to "cmake" that runs the test cases +# and provides some flexibility as to how the tests should be run in terms of how they +# are spread across processes. This is necessary when trying to isolate the impact memory +# usage of tests have upon one another. +# +# It is intended to be called as part of the CI build/test process but should be able to be run manually. +# +# It should be called with 3 parameters +# cmake_build_dir: The directory that cmake is using for build outputs, i.e. that passed to cmake's --build argument +# cmake_current_binary_dir: The directory containing the current test suite executable e.g. /test/lib/api/unittest +# test_suite: The name of the test suite to run, minus any leading "ml_", e.g. "test_api" +# +# In addition to the required parameters there are several environment variables that control the script's behaviour +# BOOST_TEST_MAX_ARGS: The maximum number of test cases to be passed off to a sub shell +# BOOST_TEST_MAX_PROCS: The maximum number of sub shells to use +# BOOST_TEST_MIXED_MODE: If set to "true" then rather than iterating over each individual test passed to a sub-shell +# run them all in the same BOOST test executable process. +# +# Design decisions: The script relies upon the simplest tools available on most unix like platforms - bash, sed and +# awk (the awk script does not use any GNU extensions for maximum portability). This is to keep the number of dependencies +# required by CI build images to a minimum (so e.g. no python etc.) + +if [ $# -lt 3 ]; then + echo "Usage: $0 " + echo "e.g.: $0 ${CPP_SRC_HOME}/cmake-build-relwithdebinfo-local ${CPP_SRC_HOME}/cmake-build-relwithdebinfo-local/test/lib/api/unittest test_api" + exit +fi + +export BUILD_DIR=$( echo $1 | sed 's|/$||' ) +export BINARY_DIR=$( echo $2 | sed 's|/$||' ) +export TEST_SUITE=$3 + +TEST_DIR=${CPP_SRC_HOME}/$(echo $BINARY_DIR | sed -e "s|$BUILD_DIR/test/||" -e 's|unittest.*|unittest|') + +export TEST_EXECUTABLE="$2/ml_$3" +export LOG_DIR="$2/test_logs" + +function num_procs() { + if [ `uname` = "Darwin" ]; then + sysctl -n hw.logicalcpu + else + nproc + fi +} + +MAX_ARGS=1 +MAX_PROCS=$(num_procs) + +if [[ -n "$BOOST_TEST_MAX_ARGS" ]]; then + MAX_ARGS=$BOOST_TEST_MAX_ARGS +fi + +if [[ -n "$BOOST_TEST_MAX_PROCS" ]]; then + MAX_PROCS=$BOOST_TEST_MAX_PROCS +fi + +rm -rf "$LOG_DIR" +mkdir -p "$LOG_DIR" + +function get_qualified_test_names() { + executable_path=$1 + + output_lines=$($executable_path --list_content 2>&1) + + while IFS= read -r line; do + match=$(grep -w '^[ ]*C.*Test' <<< "$line"); + if [ $? -eq 0 ]; then + suite=$match + continue + fi + match=$(grep -w 'test.*\*$' <<< "$line"); + if [ $? -eq 0 ]; then + case=$(sed 's/[ \*]//g' <<< "$suite/$match") + echo "$case" + fi + done <<< "$output_lines" +} + +# get the fully qualified test names +echo "Discovering tests..." +ALL_TEST_NAMES=$(get_qualified_test_names "$TEST_EXECUTABLE") + +if [ -z "$ALL_TEST_NAMES" ]; then + echo "No tests found to run or error in test discovery." + exit 1 +fi + +function execute_tests() { + + if [[ "$BOOST_TEST_MIXED_MODE" == "true" ]]; then + TEST_CASES=$(sed 's/ /:/g' <<< $@) + else + TEST_CASES=$@ + fi + + # Loop through each test + for TEST_NAME in $TEST_CASES; do + echo "--------------------------------------------------" + echo "Running test: $TEST_NAME" + + # Replace slashes and potentially other special chars for a safe filename + SAFE_TEST_LOG_FILENAME=$(echo "$TEST_NAME" | sed 's/[^a-zA-Z0-9_]/_/g' | cut -c-100) + LOG_FILE="$LOG_DIR/${SAFE_TEST_LOG_FILENAME}.log" + + # Execute the test in a separate process + TESTS=$TEST_NAME cmake --build $BUILD_DIR -t $TEST_SUITE > "$LOG_FILE" 2>&1 + TEST_STATUS=$? + + if [ $TEST_STATUS -eq 0 ]; then + echo "Test '$TEST_NAME' PASSED." + else + echo "Test '$TEST_NAME' FAILED with exit code $TEST_STATUS. Check '$LOG_FILE' for details." + fi + done +} + +export -f execute_tests + +RESULTS=$(echo $ALL_TEST_NAMES | xargs -n $MAX_ARGS -P $MAX_PROCS bash -c 'execute_tests "$@"' _) + +echo "--------------------------------------------------" + +grep 'FAILED with exit code' <<< $RESULT +if [ $? -eq 0 ] +then + echo "$TEST_SUITE: Some individual tests FAILED. Check logs in '$LOG_DIR'." + echo found +else + echo "$TEST_SUITE: All individual tests PASSED." +fi + +function merge_junit_results() { + JUNIT_FILES="$@" + echo "" + cat $JUNIT_FILES | \ + awk ' + BEGIN{tests=0; skipped=0; errors=0; failures=0; id=""; time=0.0; name=""} + $0 ~ /"}' + + cat $JUNIT_FILES | sed -e '/xml/d' -e '/testsuite/d' -e '//{H;d;};x;/skipped/d' | grep '.' +echo "" +echo +} + +if [[ $BOOST_TEST_OUTPUT_FORMAT_FLAGS =~ junit ]]; then + merge_junit_results $TEST_DIR/boost_test_results_C*.junit > $TEST_DIR/boost_test_results.junit +fi + diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 1877e64b5e..b96518a8a9 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -31,8 +31,17 @@ add_custom_target(run_tests DEPENDS clean_test_results ${ML_TEST_DEPENDS} WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} ) +add_custom_target(run_tests_individually + DEPENDS clean_test_results ${ML_TEST_INDIVIDUALLY_DEPENDS} + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} +) add_custom_target(test DEPENDS run_tests COMMAND ${CMAKE_COMMAND} -DTEST_DIR=${CMAKE_BINARY_DIR} -P ${CMAKE_SOURCE_DIR}/cmake/test-check-success.cmake WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} ) +add_custom_target(test_individually + DEPENDS run_tests_individually + COMMAND ${CMAKE_COMMAND} -DTEST_DIR=${CMAKE_BINARY_DIR} -P ${CMAKE_SOURCE_DIR}/cmake/test-check-success.cmake + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} +) \ No newline at end of file