add testing to CI

.gitignore

@@ -88,6 +88,7 @@ coverage.xml
 *.py,cover
 .hypothesis/
 .pytest_cache/
+Testing/
 
 # Translations
 *.mo

targets/bmv2/CMakeLists.txt

@@ -1,3 +1,9 @@
+if(ENABLE_TESTING)
+  # Include the test subdirectory.
+  message("-- Adding Rtsmith v1model test suite.")
+  include(test/P4Tests.cmake)
+endif()
+
 # Source files for the main P4RtSmith.
 set(RTSMITH_SOURCES
     ${RTSMITH_SOURCES}

targets/bmv2/test/P4Tests.cmake

@@ -0,0 +1,43 @@
+include(${CMAKE_CURRENT_LIST_DIR}/TestTemplate.cmake)
+# ##################################################################################################
+# TEST PROGRAMS
+# ##################################################################################################
+set(PYTHON_TEST_SCRIPT "${CMAKE_CURRENT_LIST_DIR}/run_test_batch.py")
+
+set(V1_SEARCH_PATTERNS "include.*v1model.p4" "main|common_v1_test")
+# General BMv2 tests supplied by the compiler.
+set(P4TESTS_FOR_BMV2
+    "${P4C_SOURCE_DIR}/testdata/p4_16_samples/*.p4"
+    "${P4C_SOURCE_DIR}/testdata/p4_16_samples/dash/*.p4"
+    "${P4C_SOURCE_DIR}/testdata/p4_16_samples/fabric_*/fabric.p4"
+    "${P4C_SOURCE_DIR}/testdata/p4_16_samples/omec/*.p4"
+    "${P4C_SOURCE_DIR}/testdata/p4_16_samples/pins/*.p4"
+    # Custom tests
+    "${CMAKE_CURRENT_LIST_DIR}/programs/*.p4"
+)
+
+p4c_find_tests("${P4TESTS_FOR_BMV2}" P4_16_V1_TESTS INCLUDE "${V1_SEARCH_PATTERNS}" EXCLUDE "")
+
+# Filter some programs  because they have issues that are not captured with Xfails.
+list(
+  REMOVE_ITEM
+  P4_16_V1_TESTS
+  # These tests time out and require fixing.
+)
+
+set (EXTRA_OPTS "")
+
+p4tools_add_tests(
+  TESTS
+  "${P4_16_V1_TESTS}"
+  TAG
+  "rtsmith-bmv2-v1model"
+  DRIVER
+  ${PYTHON_TEST_SCRIPT}
+  TARGET
+  "bmv2"
+  ARCH
+  "v1model"
+  TEST_ARGS
+  "${EXTRA_OPTS}"
+)

targets/bmv2/test/TestTemplate.cmake

@@ -0,0 +1,53 @@
+# General test utilities.
+include(${P4TOOLS_SOURCE_DIR}/cmake/TestUtils.cmake)
+# This file defines how a test should be written for a particular target. This is used by testutils
+
+# Add a single test to the testsuite. Arguments: - TAG is a label for the set of test suite where
+# this test belongs (for example, p4ctest) - DRIVER is the script that is used to run the test and
+# compare the results - ALIAS is a possibly different name for the test such that the same p4
+# program can be used in different test configurations. Must be unique across the test suite. -
+# P4TEST is the name of the p4 program to test (path relative to the p4c directory) - TARGET is the
+# target to test against - ARCH is the p4 architecture - TEST_ARGS is a list of arguments to pass to
+# the test - CMAKE_ARGS are additional arguments to pass to the test
+#
+# It generates a ${p4test}.test file invoking ${driver} on the p4 program with command line
+# arguments ${args} Sets the timeout on tests at 300s. For the slow CI machines.
+function(p4tools_add_test_with_args)
+  # Parse arguments.
+  set(options)
+  set(oneValueArgs TAG DRIVER ALIAS P4TEST TARGET ARCH)
+  set(multiValueArgs TEST_ARGS CMAKE_ARGS)
+  cmake_parse_arguments(
+    TOOLS_RTSMITH_V1MODEL_TESTS "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}
+  )
+  # Set some lowercase variables for convenience.
+  set(tag ${TOOLS_RTSMITH_V1MODEL_TESTS_TAG})
+  set(driver ${TOOLS_RTSMITH_V1MODEL_TESTS_DRIVER})
+  set(alias ${TOOLS_RTSMITH_V1MODEL_TESTS_ALIAS})
+  set(p4test ${TOOLS_RTSMITH_V1MODEL_TESTS_P4TEST})
+  set(target ${TOOLS_RTSMITH_V1MODEL_TESTS_TARGET})
+  set(arch ${TOOLS_RTSMITH_V1MODEL_TESTS_ARCH})
+  set(test_args ${TOOLS_RTSMITH_V1MODEL_TESTS_TEST_ARGS})
+  set(cmake_args ${TOOLS_RTSMITH_V1MODEL_TESTS_CMAKE_ARGS})
+
+  # This is the actual test processing.
+  p4c_test_set_name(__testname ${tag} ${alias})
+  string(REGEX REPLACE ".p4" "" aliasname ${alias})
+  set(__testfile "${RTSMITH_DIR}/${tag}/${alias}.test")
+  get_filename_component(__testdir ${p4test} DIRECTORY)
+  file(WRITE ${__testfile} "#! /usr/bin/env bash\n")
+  file(APPEND ${__testfile} "# Generated file, modify with care\n\n")
+  file(APPEND ${__testfile} "set -e\n")
+  file(APPEND ${__testfile} "cd ${P4C_BINARY_DIR}\n")
+
+  file(APPEND ${__testfile} "${driver} ${p4test} ${RTSMITH_DRIVER} \n"
+  )
+
+  execute_process(COMMAND chmod +x ${__testfile})
+  separate_arguments(__args UNIX_COMMAND ${cmake_args})
+  add_test(NAME ${__testname} COMMAND ${tag}/${alias}.test ${__args} WORKING_DIRECTORY ${RTSMITH_DIR})
+  if(NOT DEFINED ${tag}_timeout)
+    set(${tag}_timeout 420)
+  endif()
+  set_tests_properties(${__testname} PROPERTIES LABELS ${tag} TIMEOUT ${${tag}_timeout})
+endfunction(p4tools_add_test_with_args)

targets/bmv2/test/run_test_batch.py

@@ -1,3 +1,4 @@
+#!/usr/bin/env python3
 import argparse
 import logging
 import os
@@ -26,6 +27,12 @@
     dest="seed",
     help="Random seed for generating configs.",
 )
+PARSER.add_argument(
+    "--num_init_config",
+    default=1,
+    dest="num_init_config",
+    help="Number of initial configs to generate for the given P4 program.",
+)
 
 
 # Parse options and process argv
@@ -46,16 +53,20 @@ class Options:
     testdir: Path = Path(".")
     # Random seed for generating configs.
     seed: int = 1
+    # Number of initial configs to generate for the given P4 program.
+    num_init_config: int = 1
 
 
 def generate_config(p4rtsmith_path, seed, p4_program_path, testdir, config_file_path):
     command = f"{p4rtsmith_path} --target bmv2 --arch v1model --seed {seed} --output-dir {testdir} --generate-config {config_file_path} {p4_program_path}"
-    subprocess.run(command, shell=True)
+    returncode = subprocess.run(command, shell=True)
+    return returncode.returncode
 
 
 def run_test(run_test_script, p4_program_path, config_file_path):
     command = f"sudo -E {run_test_script} .. {p4_program_path} -tf {config_file_path}"
-    subprocess.run(command, shell=True)
+    returncode = subprocess.run(command, shell=True)
+    return returncode.returncode
 
 
 def find_p4c_dir():
@@ -73,16 +84,24 @@ def find_p4c_dir():
 
 
 def run_tests(options: Options) -> int:
-    config_file_path = "initial_config.txtpb"
-
     seed = options.seed
     testdir = options.testdir
     p4rtsmith_path = options.p4rtsmith
     run_test_script = FILE_DIR / "run-bmv2-proto-test.py"
     p4_program_path = options.p4_file
+    filename = os.path.splitext(os.path.basename(p4_program_path))[0]
 
-    generate_config(p4rtsmith_path, seed, p4_program_path, testdir, config_file_path)
-    run_test(run_test_script, p4_program_path, testdir / config_file_path)
+    for i in range(options.num_init_config):
+        config_file_path = f"initial_config_{filename}_{i}.txtpb"
+        result = generate_config(
+            p4rtsmith_path, seed, p4_program_path, testdir, config_file_path
+        )
+        if result != 0:
+            return result
+        result = run_test(run_test_script, p4_program_path, testdir / config_file_path)
+        if result != 0:
+            return result
+    return 0
 
 
 def create_options(test_args: Any) -> Optional[Options]:
@@ -101,6 +120,7 @@ def create_options(test_args: Any) -> Optional[Options]:
         os.chmod(testdir, 0o755)
     options.testdir = Path(testdir)
     options.seed = test_args.seed
+    options.num_init_config = test_args.num_init_config
 
     # Configure logging.
     logging.basicConfig(
@@ -120,4 +140,5 @@ def create_options(test_args: Any) -> Optional[Options]:
     if not test_options:
         sys.exit()
 
-    run_tests(test_options)
+    test_result = run_tests(test_options)
+    sys.exit(test_result)