Merge pull request #4340 from NREL/62470514_RegressionTests

62470514 regression tests

.decent_ci-Linux.yaml

@@ -1,6 +1,7 @@
 compilers:
   - name: "gcc"
     version: "4.8"
-    cmake_extra_flags: -DBUILD_TESTING:BOOL=ON -DBUILD_PACKAGE:BOOL=ON
+    cmake_extra_flags: -DBUILD_PACKAGE:BOOL=ON -DBUILD_TESTING:BOOL=ON -DENABLE_REGRESSION_TESTING:BOOL=ON -DREGRESSION_BASELINE_PATH:PATH=$REGRESSION_BASELINE -DREGRESSION_SCRIPT_PATH:PATH=$REGRESSION_DIR -DREGRESSION_BASELINE_SHA:STRING=$REGRESSION_BASELINE_SHA -DCOMMIT_SHA=$COMMIT_SHA
   - name: cppcheck
-    compiler_extra_flags: --enable=warning  --suppress="*:*gtest*" --suppress="constStatement:*Objex*" --suppress="cppcheckError:*" --suppres="uninitvar:*"
+    compiler_extra_flags: --enable=warning  --suppress="*:*gtest*" --suppress="constStatement:*Objex*" --suppress="cppcheckError:*" --suppress="uninitvar:*" --suppress="syntaxError:*" --suppress="*:*sqlite*"
+

.decent_ci-MacOS.yaml

@@ -1,7 +1,7 @@
 compilers:
   - name: clang
-    cmake_extra_flags: -DBUILD_TESTING:BOOL=ON -DBUILD_PACKAGE:BOOL=ON
+    cmake_extra_flags: -DBUILD_PACKAGE:BOOL=ON -DBUILD_TESTING:BOOL=ON -DENABLE_REGRESSION_TESTING:BOOL=ON -DREGRESSION_BASELINE_PATH:PATH=$REGRESSION_BASELINE -DREGRESSION_SCRIPT_PATH:PATH=$REGRESSION_DIR -DREGRESSION_BASELINE_SHA:STRING=$REGRESSION_BASELINE_SHA -DCOMMIT_SHA=$COMMIT_SHA
   - name: clang
     build_type: Debug
-    cmake_extra_flags: -DBUILD_TESTING:BOOL=ON -DBUILD_PACKAGE:BOOL=ON
+    cmake_extra_flags: -DBUILD_PACKAGE:BOOL=ON -DBUILD_TESTING:BOOL=ON -DENABLE_REGRESSION_TESTING:BOOL=ON -DREGRESSION_BASELINE_PATH:PATH=$REGRESSION_BASELINE -DREGRESSION_SCRIPT_PATH:PATH=$REGRESSION_DIR -DREGRESSION_BASELINE_SHA:STRING=$REGRESSION_BASELINE_SHA -DCOMMIT_SHA=$COMMIT_SHA
 

.decent_ci-Windows.yaml

@@ -1,10 +1,10 @@
 compilers:
   - name: Visual Studio
     version: 12
-    cmake_extra_flags: -DBUILD_TESTING:BOOL=ON -DBUILD_PACKAGE:BOOL=ON
+    cmake_extra_flags: -DBUILD_PACKAGE:BOOL=ON -DBUILD_TESTING:BOOL=ON -DENABLE_REGRESSION_TESTING:BOOL=ON -DREGRESSION_BASELINE_PATH:PATH=%REGRESSION_BASELINE% -DREGRESSION_SCRIPT_PATH:PATH=%REGRESSION_DIR% -DREGRESSION_BASELINE_SHA:STRING=%REGRESSION_BASELINE_SHA% -DCOMMIT_SHA=%COMMIT_SHA%
   - name: Visual Studio
     version: 12
     architecture: Win64
-    cmake_extra_flags: -DBUILD_TESTING:BOOL=ON -DBUILD_PACKAGE:BOOL=ON
+    cmake_extra_flags: -DBUILD_PACKAGE:BOOL=ON -DBUILD_TESTING:BOOL=ON -DENABLE_REGRESSION_TESTING:BOOL=ON -DREGRESSION_BASELINE_PATH:PATH=%REGRESSION_BASELINE% -DREGRESSION_SCRIPT_PATH:PATH=%REGRESSION_DIR% -DREGRESSION_BASELINE_SHA:STRING=%REGRESSION_BASELINE_SHA% -DCOMMIT_SHA=%COMMIT_SHA%
 
 

.decent_ci.yaml

@@ -1,3 +1,8 @@
 results_repository : NREL/EnergyPlusBuildResults
 results_path : _posts
 results_base_url : https://nrel.github.io/EnergyPlusBuildResults
+regression_repository : NREL/EnergyPlusDevSupport
+regression_branch : master
+notification_recipients:
+  - lgentile
+aging_pull_requests_notification: true

CMakeLists.txt

@@ -32,6 +32,28 @@ option( BUILD_PACKAGE "Build package" OFF )
 option( BUILD_TESTING "Build testing targets" OFF )
 option( BUILD_FORTRAN "Build Fortran stuff" OFF )
 
+if( BUILD_TESTING )
+  option( ENABLE_REGRESSION_TESTING "Enable Regression Tests" OFF )
+  if( ENABLE_REGRESSION_TESTING )
+    if ( CMAKE_CL_64 )
+      set( ARCH_FLAG "-64bit" )
+    else()
+      set( ARCH_FLAG "" )
+    endif()
+
+    find_package(PythonInterp REQUIRED)
+    set( REGRESSION_BASELINE_SHA "" CACHE STRING "SHA of baseline comparison" )
+    set( COMMIT_SHA "" CACHE STRING "Commit sha of this build" )
+    set( REGRESSION_SCRIPT_PATH "" CACHE PATH "Path to regression scripts" )
+    set( REGRESSION_BASELINE_PATH "" CACHE PATH "Path to energyplus baseline folder to compare against" )
+    set( DEVICE_ID "${CMAKE_SYSTEM_NAME}-${CMAKE_CXX_COMPILER_ID}${ARCH_FLAG}" CACHE STRING "Identifier for this device configuration" )
+  endif()
+endif()
+
+# Not yet released by CMake, but is being developed
+# This will help us set the proper options for c++11 across various configurations
+#FIND_PACKAGE( CXXFeatures ) 
+
 if (MSVC)
   set(gtest_force_shared_crt ON)
 endif()

cmake/ProjectMacros.cmake

@@ -93,8 +93,30 @@ function( ADD_SIMULATION_TEST )
       -P ${CMAKE_SOURCE_DIR}/cmake/RunSimulation.cmake
     )  
   endif()
+
   SET_TESTS_PROPERTIES("integration.${IDF_NAME}" PROPERTIES PASS_REGULAR_EXPRESSION "Test Passed")
   SET_TESTS_PROPERTIES("integration.${IDF_NAME}" PROPERTIES FAIL_REGULAR_EXPRESSION "ERROR;FAIL;Test Failed")
+
+  if( ENABLE_REGRESSION_TESTING )
+    add_test(NAME "regression.${IDF_NAME}" COMMAND ${CMAKE_COMMAND}
+      -DBINARY_DIR=${CMAKE_BINARY_DIR}
+      -DPYTHON_EXECUTABLE=${PYTHON_EXECUTABLE}
+      -DIDF_FILE=${ADD_SIM_TEST_IDF_FILE}
+      -DREGRESSION_SCRIPT_PATH=${REGRESSION_SCRIPT_PATH}
+      -DREGRESSION_BASELINE_PATH=${REGRESSION_BASELINE_PATH}
+      -DREGRESSION_BASELINE_SHA=${REGRESSION_BASELINE_SHA}
+      -DCOMMIT_SHA=${COMMIT_SHA}
+      -DDEVICE_ID=${DEVICE_ID}
+      -P ${CMAKE_SOURCE_DIR}/cmake/RunRegression.cmake
+      )
+    # Note, CMake / CTest doesn't seem to validate if this dependent name actually exists,
+    # but it does seem to honor the requirement
+    set_tests_properties("regression.${IDF_NAME}" PROPERTIES DEPENDS "integration.${IDF_NAME}")
+    set_tests_properties("regression.${IDF_NAME}" PROPERTIES PASS_REGULAR_EXPRESSION "Success")
+    set_tests_properties("regression.${IDF_NAME}" PROPERTIES FAIL_REGULAR_EXPRESSION "ERROR;FAIL;Test Failed")
+  endif()
+
+
 endfunction()
 
 macro( ADD_CXX_DEFINITIONS NEWFLAGS )

cmake/RunRegression.cmake

@@ -0,0 +1,15 @@
+
+# These need to be defined by the caller
+# IDF_FILE
+# BINARY_DIR
+# PYTHON_EXECUTABLE
+# REGRESSION_SCRIPT_PATH
+# REGRESSION_BASELINE_PATH
+# REGRESSION_BASELINE_SHA
+# COMMIT_SHA
+# DEVICE_ID
+
+get_filename_component(IDF_NAME "${IDF_FILE}" NAME_WE)
+
+execute_process(COMMAND ${PYTHON_EXECUTABLE} "${REGRESSION_SCRIPT_PATH}/Testing/RegressionSuite/Scripts/RunComparison.py" "${IDF_NAME}" "${REGRESSION_BASELINE_PATH}/testfiles/${IDF_NAME}" "${BINARY_DIR}/testfiles/${IDF_NAME}" ${REGRESSION_BASELINE_SHA} ${COMMIT_SHA} true "${DEVICE_ID}")
+

src/EnergyPlus/PlantPipingSystemsManager.cc

@@ -5702,6 +5702,7 @@ namespace PlantPipingSystemsManager {
 
 			//we have it quite simple
 
+			assert(ThisMesh.RegionMeshCount > 0);
 			CellWidth = GridWidth / ThisMesh.RegionMeshCount;
 
 			for ( I = 0; I <= ThisMesh.RegionMeshCount - 1; ++I ) {