From 78229f53d4869731210bebe47d8963c2c890cb3b Mon Sep 17 00:00:00 2001 From: vvbae Date: Mon, 13 May 2024 16:08:04 +0000 Subject: [PATCH 1/2] add testing to CI --- targets/bmv2/CMakeLists.txt | 6 ++++ targets/bmv2/test/P4Tests.cmake | 43 +++++++++++++++++++++++ targets/bmv2/test/TestTemplate.cmake | 51 ++++++++++++++++++++++++++++ targets/bmv2/test/run_test_batch.py | 35 +++++++++++++++---- 4 files changed, 128 insertions(+), 7 deletions(-) create mode 100644 targets/bmv2/test/P4Tests.cmake create mode 100644 targets/bmv2/test/TestTemplate.cmake mode change 100644 => 100755 targets/bmv2/test/run_test_batch.py diff --git a/targets/bmv2/CMakeLists.txt b/targets/bmv2/CMakeLists.txt index c2547ab..23d2fab 100644 --- a/targets/bmv2/CMakeLists.txt +++ b/targets/bmv2/CMakeLists.txt @@ -1,3 +1,9 @@ +if(ENABLE_TESTING) + # Include the test subdirectory. + message("-- Adding Rtsmith v1model test suite.") + include(test/P4Tests.cmake) +endif() + # Source files for the main P4RtSmith. set(RTSMITH_SOURCES ${RTSMITH_SOURCES} diff --git a/targets/bmv2/test/P4Tests.cmake b/targets/bmv2/test/P4Tests.cmake new file mode 100644 index 0000000..2d914f5 --- /dev/null +++ b/targets/bmv2/test/P4Tests.cmake @@ -0,0 +1,43 @@ +include(${CMAKE_CURRENT_LIST_DIR}/TestTemplate.cmake) +# ################################################################################################## +# TEST PROGRAMS +# ################################################################################################## +set(PYTHON_TEST_SCRIPT "${CMAKE_CURRENT_LIST_DIR}/run_test_batch.py") + +set(V1_SEARCH_PATTERNS "include.*v1model.p4" "main|common_v1_test") +# General BMv2 tests supplied by the compiler. +set(P4TESTS_FOR_BMV2 + "${P4C_SOURCE_DIR}/testdata/p4_16_samples/*.p4" + "${P4C_SOURCE_DIR}/testdata/p4_16_samples/dash/*.p4" + "${P4C_SOURCE_DIR}/testdata/p4_16_samples/fabric_*/fabric.p4" + "${P4C_SOURCE_DIR}/testdata/p4_16_samples/omec/*.p4" + "${P4C_SOURCE_DIR}/testdata/p4_16_samples/pins/*.p4" + # Custom tests + "${CMAKE_CURRENT_LIST_DIR}/programs/*.p4" +) + +p4c_find_tests("${P4TESTS_FOR_BMV2}" P4_16_V1_TESTS INCLUDE "${V1_SEARCH_PATTERNS}" EXCLUDE "") + +# Filter some programs because they have issues that are not captured with Xfails. +list( + REMOVE_ITEM + P4_16_V1_TESTS + # These tests time out and require fixing. +) + +set (EXTRA_OPTS "") + +p4tools_add_tests( + TESTS + "${P4_16_V1_TESTS}" + TAG + "rtsmith-bmv2-v1model" + DRIVER + ${PYTHON_TEST_SCRIPT} + TARGET + "bmv2" + ARCH + "v1model" + TEST_ARGS + "${EXTRA_OPTS}" +) \ No newline at end of file diff --git a/targets/bmv2/test/TestTemplate.cmake b/targets/bmv2/test/TestTemplate.cmake new file mode 100644 index 0000000..4deee14 --- /dev/null +++ b/targets/bmv2/test/TestTemplate.cmake @@ -0,0 +1,51 @@ +# This file defines how a test should be written for a particular target. This is used by testutils + +# Add a single test to the testsuite. Arguments: - TAG is a label for the set of test suite where +# this test belongs (for example, p4ctest) - DRIVER is the script that is used to run the test and +# compare the results - ALIAS is a possibly different name for the test such that the same p4 +# program can be used in different test configurations. Must be unique across the test suite. - +# P4TEST is the name of the p4 program to test (path relative to the p4c directory) - TARGET is the +# target to test against - ARCH is the p4 architecture - TEST_ARGS is a list of arguments to pass to +# the test - CMAKE_ARGS are additional arguments to pass to the test +# +# It generates a ${p4test}.test file invoking ${driver} on the p4 program with command line +# arguments ${args} Sets the timeout on tests at 300s. For the slow CI machines. +function(p4tools_add_test_with_args) + # Parse arguments. + set(options) + set(oneValueArgs TAG DRIVER ALIAS P4TEST TARGET ARCH) + set(multiValueArgs TEST_ARGS CMAKE_ARGS) + cmake_parse_arguments( + TOOLS_RTSMITH_V1MODEL_TESTS "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN} + ) + # Set some lowercase variables for convenience. + set(tag ${TOOLS_RTSMITH_V1MODEL_TESTS_TAG}) + set(driver ${TOOLS_RTSMITH_V1MODEL_TESTS_DRIVER}) + set(alias ${TOOLS_RTSMITH_V1MODEL_TESTS_ALIAS}) + set(p4test ${TOOLS_RTSMITH_V1MODEL_TESTS_P4TEST}) + set(target ${TOOLS_RTSMITH_V1MODEL_TESTS_TARGET}) + set(arch ${TOOLS_RTSMITH_V1MODEL_TESTS_ARCH}) + set(test_args ${TOOLS_RTSMITH_V1MODEL_TESTS_TEST_ARGS}) + set(cmake_args ${TOOLS_RTSMITH_V1MODEL_TESTS_CMAKE_ARGS}) + + # This is the actual test processing. + p4c_test_set_name(__testname ${tag} ${alias}) + string(REGEX REPLACE ".p4" "" aliasname ${alias}) + set(__testfile "${RTSMITH_DIR}/${tag}/${alias}.test") + get_filename_component(__testdir ${p4test} DIRECTORY) + file(WRITE ${__testfile} "#! /usr/bin/env bash\n") + file(APPEND ${__testfile} "# Generated file, modify with care\n\n") + file(APPEND ${__testfile} "set -e\n") + file(APPEND ${__testfile} "cd ${P4C_BINARY_DIR}\n") + + file(APPEND ${__testfile} "${driver} ${p4test} ${RTSMITH_DRIVER} \n" + ) + + execute_process(COMMAND chmod +x ${__testfile}) + separate_arguments(__args UNIX_COMMAND ${cmake_args}) + add_test(NAME ${__testname} COMMAND ${tag}/${alias}.test ${__args} WORKING_DIRECTORY ${RTSMITH_DIR}) + if(NOT DEFINED ${tag}_timeout) + set(${tag}_timeout 420) + endif() + set_tests_properties(${__testname} PROPERTIES LABELS ${tag} TIMEOUT ${${tag}_timeout}) +endfunction(p4tools_add_test_with_args) diff --git a/targets/bmv2/test/run_test_batch.py b/targets/bmv2/test/run_test_batch.py old mode 100644 new mode 100755 index 5d7bc7d..0366dcd --- a/targets/bmv2/test/run_test_batch.py +++ b/targets/bmv2/test/run_test_batch.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python3 import argparse import logging import os @@ -26,6 +27,12 @@ dest="seed", help="Random seed for generating configs.", ) +PARSER.add_argument( + "--num_init_config", + default=1, + dest="num_init_config", + help="Number of initial configs to generate for the given P4 program.", +) # Parse options and process argv @@ -46,16 +53,20 @@ class Options: testdir: Path = Path(".") # Random seed for generating configs. seed: int = 1 + # Number of initial configs to generate for the given P4 program. + num_init_config: int = 1 def generate_config(p4rtsmith_path, seed, p4_program_path, testdir, config_file_path): command = f"{p4rtsmith_path} --target bmv2 --arch v1model --seed {seed} --output-dir {testdir} --generate-config {config_file_path} {p4_program_path}" - subprocess.run(command, shell=True) + returncode = subprocess.run(command, shell=True) + return returncode.returncode def run_test(run_test_script, p4_program_path, config_file_path): command = f"sudo -E {run_test_script} .. {p4_program_path} -tf {config_file_path}" - subprocess.run(command, shell=True) + returncode = subprocess.run(command, shell=True) + return returncode.returncode def find_p4c_dir(): @@ -73,16 +84,24 @@ def find_p4c_dir(): def run_tests(options: Options) -> int: - config_file_path = "initial_config.txtpb" - seed = options.seed testdir = options.testdir p4rtsmith_path = options.p4rtsmith run_test_script = FILE_DIR / "run-bmv2-proto-test.py" p4_program_path = options.p4_file + filename = os.path.splitext(os.path.basename(p4_program_path))[0] - generate_config(p4rtsmith_path, seed, p4_program_path, testdir, config_file_path) - run_test(run_test_script, p4_program_path, testdir / config_file_path) + for i in range(options.num_init_config): + config_file_path = f"initial_config_{filename}_{i}.txtpb" + result = generate_config( + p4rtsmith_path, seed, p4_program_path, testdir, config_file_path + ) + if result != 0: + return result + result = run_test(run_test_script, p4_program_path, testdir / config_file_path) + if result != 0: + return result + return 0 def create_options(test_args: Any) -> Optional[Options]: @@ -101,6 +120,7 @@ def create_options(test_args: Any) -> Optional[Options]: os.chmod(testdir, 0o755) options.testdir = Path(testdir) options.seed = test_args.seed + options.num_init_config = test_args.num_init_config # Configure logging. logging.basicConfig( @@ -120,4 +140,5 @@ def create_options(test_args: Any) -> Optional[Options]: if not test_options: sys.exit() - run_tests(test_options) + test_result = run_tests(test_options) + sys.exit(test_result) From 51b945539f9187c06b6fc2d8668ea41d4e6647f3 Mon Sep 17 00:00:00 2001 From: vvbae Date: Sat, 25 May 2024 05:29:59 +0000 Subject: [PATCH 2/2] fix imports --- .gitignore | 1 + targets/bmv2/test/TestTemplate.cmake | 2 ++ 2 files changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index ff9f0ab..e9b4e93 100644 --- a/.gitignore +++ b/.gitignore @@ -88,6 +88,7 @@ coverage.xml *.py,cover .hypothesis/ .pytest_cache/ +Testing/ # Translations *.mo diff --git a/targets/bmv2/test/TestTemplate.cmake b/targets/bmv2/test/TestTemplate.cmake index 4deee14..b5cd034 100644 --- a/targets/bmv2/test/TestTemplate.cmake +++ b/targets/bmv2/test/TestTemplate.cmake @@ -1,3 +1,5 @@ +# General test utilities. +include(${P4TOOLS_SOURCE_DIR}/cmake/TestUtils.cmake) # This file defines how a test should be written for a particular target. This is used by testutils # Add a single test to the testsuite. Arguments: - TAG is a label for the set of test suite where