Tests: Added command line utility to run all tests on multiple thread…

…s at once (No GUI) to speed up test runs
SourMesen · Dec 29, 2015 · 10fda05 · 10fda05
1 parent b6fc347
commit 10fda05
Show file tree

Hide file tree

Showing 4 changed files with 357 additions and 0 deletions.
diff --git a/NES.sln b/NES.sln
@@ -32,6 +32,11 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "PGOHelper", "PGOHelper\PGOH
 EndProject
 Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "BlipBuffer", "BlipBuffer\BlipBuffer.vcxproj", "{CF35D78C-F710-41D2-968F-C46ACCFF6F07}"
 EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "TestHelper", "TestHelper\TestHelper.vcxproj", "{2A607369-8B5D-494A-9E40-C5DC8D821AA3}"
+	ProjectSection(ProjectDependencies) = postProject
+		{37749BB2-FA78-4EC9-8990-5628FC0BBA19} = {37749BB2-FA78-4EC9-8990-5628FC0BBA19}
+	EndProjectSection
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|x64 = Debug|x64
@@ -95,6 +100,14 @@ Global
 		{CF35D78C-F710-41D2-968F-C46ACCFF6F07}.Release|x64.ActiveCfg = Release|x64
 		{CF35D78C-F710-41D2-968F-C46ACCFF6F07}.Release|x86.ActiveCfg = Release|Win32
 		{CF35D78C-F710-41D2-968F-C46ACCFF6F07}.Release|x86.Build.0 = Release|Win32
+		{2A607369-8B5D-494A-9E40-C5DC8D821AA3}.Debug|x64.ActiveCfg = Debug|x64
+		{2A607369-8B5D-494A-9E40-C5DC8D821AA3}.Debug|x64.Build.0 = Debug|x64
+		{2A607369-8B5D-494A-9E40-C5DC8D821AA3}.Debug|x86.ActiveCfg = Debug|Win32
+		{2A607369-8B5D-494A-9E40-C5DC8D821AA3}.Debug|x86.Build.0 = Debug|Win32
+		{2A607369-8B5D-494A-9E40-C5DC8D821AA3}.Release|x64.ActiveCfg = Release|x64
+		{2A607369-8B5D-494A-9E40-C5DC8D821AA3}.Release|x64.Build.0 = Release|x64
+		{2A607369-8B5D-494A-9E40-C5DC8D821AA3}.Release|x86.ActiveCfg = Release|Win32
+		{2A607369-8B5D-494A-9E40-C5DC8D821AA3}.Release|x86.Build.0 = Release|Win32
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE

diff --git a/TestHelper/TestHelper.cpp b/TestHelper/TestHelper.cpp
@@ -0,0 +1,161 @@
+#if _WIN64 || __x86_64__ || __ppc64__
+	#define ENVIRONMENT64
+#else
+	#define ENVIRONMENT32
+#endif
+
+#ifdef ENVIRONMENT32
+	#ifdef _DEBUG
+		#define MESEN_LIBRARY_PATH "../bin/x86/Debug/"
+	#else 
+		#define MESEN_LIBRARY_PATH "../bin/x86/Release/"
+	#endif
+#else
+	#ifdef _DEBUG
+		#define MESEN_LIBRARY_PATH "../bin/x64/Debug/"
+	#else 
+		#define MESEN_LIBRARY_PATH "../bin/x64/Release/"
+	#endif
+#endif
+
+#pragma comment(lib, MESEN_LIBRARY_PATH"Utilities.lib")
+
+#include <Windows.h>
+#include <Shlobj.h>
+#include <iostream>
+#include <thread>
+#include <vector>
+#include <string>
+#include <thread>
+#include "../Utilities/FolderUtilities.h"
+#include "../Utilities/SimpleLock.h"
+#include "../Utilities/Timer.h"
+#include "../Core/MessageManager.h"
+
+typedef void (__stdcall *NotificationListenerCallback)(ConsoleNotificationType);
+
+class InteropNotificationListener : public INotificationListener
+{
+	NotificationListenerCallback _callback;
+public:
+	InteropNotificationListener(NotificationListenerCallback callback)
+	{
+		_callback = callback;
+	}
+
+	void ProcessNotification(ConsoleNotificationType type, void* parameter)
+	{
+		_callback((ConsoleNotificationType)type);
+	}
+};
+
+extern "C" {
+	void __stdcall InitializeEmu(char* homeFolder, void*, void*);
+	int __stdcall RomTestRun(char* filename);
+	void __stdcall LoadROM(char* filename);
+	void __stdcall Run();
+	void __stdcall Stop();
+	INotificationListener* __stdcall RegisterNotificationCallback(NotificationListenerCallback callback);
+}
+
+std::thread *runThread = nullptr;
+std::atomic<int> testIndex;
+vector<string> testFilenames;
+vector<string> failedTests;
+SimpleLock lock;
+Timer timer;
+
+void OnNotificationReceived(ConsoleNotificationType type)
+{
+	if(type == ConsoleNotificationType::GameLoaded) {
+		runThread = new std::thread(Run);
+	}
+}
+
+void RunEmu()
+{
+	Run();
+}
+
+void RunTest()
+{
+	while(true) {
+		lock.Acquire();
+		int index = testIndex++;
+		lock.Release();
+
+		if(index < testFilenames.size()) {
+			string filepath = testFilenames[index];
+			string filename = FolderUtilities::GetFilename(filepath, false);
+			string command = "TestHelper.exe /testrom \"" + filepath + "\"";
+
+			lock.Acquire();
+			std::cout << std::to_string(index) << ") " << filename << std::endl;
+			lock.Release();
+
+			if(std::system(command.c_str()) != 1) {
+				//Test failed
+				lock.Acquire();
+				failedTests.push_back(filename);
+				std::cout << "  ****  " << std::to_string(index) << ") " << filename << " failed" << std::endl;
+				lock.Release();
+			}
+		} else {
+			break;
+		}
+	}
+}
+
+int main(int argc, char* argv[])
+{
+	using namespace std;
+	if(argc <= 2) {
+		string testFolder;
+		if(argc == 1) {
+			wchar_t path[MAX_PATH];
+			SHGetFolderPath(NULL, CSIDL_MYDOCUMENTS, NULL, SHGFP_TYPE_CURRENT, path);
+			testFolder = utf8::utf8::encode(path) + "\\Mesen\\Tests";
+		} else {
+			testFolder = argv[1];
+		}
+
+		vector<std::thread*> testThreads;
+		testFilenames = FolderUtilities::GetFilesInFolder(testFolder, "*.mtp", true);
+		testIndex = 0;
+
+		timer.Reset();
+
+		for(int i = 0; i < 4; i++) {
+			std::thread *testThread = new std::thread(RunTest);
+			testThreads.push_back(testThread);
+		}
+
+		for(int i = 0; i < 4; i++) {
+			testThreads[i]->join();
+			delete testThreads[i];
+		}
+
+		std::cout << std::endl << std::endl;
+		std::cout << "------------" << std::endl;
+		std::cout << "Failed tests" << std::endl;
+		std::cout << "------------" << std::endl;
+		for(string failedTest : failedTests) {
+			std::cout << failedTest << std::endl;
+		}
+		std::cout << std::endl << std::endl << "Elapsed time: " << (timer.GetElapsedMS() / 1000) << " seconds";
+
+		std::getchar();
+	} else if(argc == 3) {
+		char* testFilename = argv[2];
+		RegisterNotificationCallback((NotificationListenerCallback)OnNotificationReceived);
+		InitializeEmu("C:\\Windows\\Temp\\Mesen", nullptr, nullptr);
+		int result = RomTestRun(testFilename);
+		if(runThread != nullptr) {
+			runThread->join();
+			delete runThread;
+		}
+		return result;
+	}
+	return 0;
+}
+
diff --git a/TestHelper/TestHelper.vcxproj b/TestHelper/TestHelper.vcxproj
@@ -0,0 +1,169 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="12.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <ItemGroup Label="ProjectConfigurations">
+    <ProjectConfiguration Include="Debug|Win32">
+      <Configuration>Debug</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Debug|x64">
+      <Configuration>Debug</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|Win32">
+      <Configuration>Release</Configuration>
+      <Platform>Win32</Platform>
+    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release|x64">
+      <Configuration>Release</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
+  </ItemGroup>
+  <PropertyGroup Label="Globals">
+    <ProjectGuid>{2A607369-8B5D-494A-9E40-C5DC8D821AA3}</ProjectGuid>
+    <Keyword>Win32Proj</Keyword>
+    <RootNamespace>TestHelper</RootNamespace>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <UseDebugLibraries>true</UseDebugLibraries>
+    <PlatformToolset>v120</PlatformToolset>
+    <CharacterSet>Unicode</CharacterSet>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <UseDebugLibraries>true</UseDebugLibraries>
+    <PlatformToolset>v120</PlatformToolset>
+    <CharacterSet>Unicode</CharacterSet>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <UseDebugLibraries>false</UseDebugLibraries>
+    <PlatformToolset>v120</PlatformToolset>
+    <WholeProgramOptimization>true</WholeProgramOptimization>
+    <CharacterSet>Unicode</CharacterSet>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
+    <ConfigurationType>Application</ConfigurationType>
+    <UseDebugLibraries>false</UseDebugLibraries>
+    <PlatformToolset>v120</PlatformToolset>
+    <WholeProgramOptimization>true</WholeProgramOptimization>
+    <CharacterSet>Unicode</CharacterSet>
+  </PropertyGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+  <ImportGroup Label="ExtensionSettings">
+  </ImportGroup>
+  <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <ImportGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="PropertySheets">
+    <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+  </ImportGroup>
+  <PropertyGroup Label="UserMacros" />
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+    <LinkIncremental>true</LinkIncremental>
+    <OutDir>$(SolutionDir)\bin\$(PlatformTarget)\$(Configuration)\</OutDir>
+    <IntDir>obj\$(Platform)\$(Configuration)\</IntDir>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+    <IntDir>obj\$(Platform)\$(Configuration)\</IntDir>
+    <LinkIncremental>true</LinkIncremental>
+    <OutDir>$(SolutionDir)\bin\$(PlatformTarget)\$(Configuration)\</OutDir>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+    <LinkIncremental>false</LinkIncremental>
+    <OutDir>$(SolutionDir)\bin\$(PlatformTarget)\$(Configuration)\</OutDir>
+    <IntDir>obj\$(Platform)\$(Configuration)\</IntDir>
+  </PropertyGroup>
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+    <IntDir>obj\$(Platform)\$(Configuration)\</IntDir>
+    <LinkIncremental>false</LinkIncremental>
+    <OutDir>$(SolutionDir)\bin\$(PlatformTarget)\$(Configuration)\</OutDir>
+  </PropertyGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
+    <ClCompile>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <WarningLevel>Level3</WarningLevel>
+      <Optimization>Disabled</Optimization>
+      <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <SDLCheck>true</SDLCheck>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
+    <ClCompile>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <WarningLevel>Level3</WarningLevel>
+      <Optimization>Disabled</Optimization>
+      <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <SDLCheck>true</SDLCheck>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
+    <ClCompile>
+      <WarningLevel>Level3</WarningLevel>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <Optimization>MaxSpeed</Optimization>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <IntrinsicFunctions>true</IntrinsicFunctions>
+      <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <SDLCheck>true</SDLCheck>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <EnableCOMDATFolding>true</EnableCOMDATFolding>
+      <OptimizeReferences>true</OptimizeReferences>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
+    <ClCompile>
+      <WarningLevel>Level3</WarningLevel>
+      <PrecompiledHeader>
+      </PrecompiledHeader>
+      <Optimization>MaxSpeed</Optimization>
+      <FunctionLevelLinking>true</FunctionLevelLinking>
+      <IntrinsicFunctions>true</IntrinsicFunctions>
+      <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <SDLCheck>true</SDLCheck>
+    </ClCompile>
+    <Link>
+      <SubSystem>Console</SubSystem>
+      <GenerateDebugInformation>true</GenerateDebugInformation>
+      <EnableCOMDATFolding>true</EnableCOMDATFolding>
+      <OptimizeReferences>true</OptimizeReferences>
+    </Link>
+  </ItemDefinitionGroup>
+  <ItemGroup>
+    <ClCompile Include="TestHelper.cpp" />
+  </ItemGroup>
+  <ItemGroup>
+    <ProjectReference Include="..\InteropDLL\InteropDLL.vcxproj">
+      <Project>{37749bb2-fa78-4ec9-8990-5628fc0bba19}</Project>
+      <Private>false</Private>
+      <ReferenceOutputAssembly>true</ReferenceOutputAssembly>
+      <CopyLocalSatelliteAssemblies>false</CopyLocalSatelliteAssemblies>
+      <LinkLibraryDependencies>true</LinkLibraryDependencies>
+      <UseLibraryDependencyInputs>true</UseLibraryDependencyInputs>
+    </ProjectReference>
+  </ItemGroup>
+  <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+  <ImportGroup Label="ExtensionTargets">
+  </ImportGroup>
+</Project>
diff --git a/TestHelper/TestHelper.vcxproj.filters b/TestHelper/TestHelper.vcxproj.filters
@@ -0,0 +1,14 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <ItemGroup>
+    <Filter Include="Source Files">
+      <UniqueIdentifier>{4FC737F1-C7A5-4376-A066-2A32D752A2FF}</UniqueIdentifier>
+      <Extensions>cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx</Extensions>
+    </Filter>
+  </ItemGroup>
+  <ItemGroup>
+    <ClCompile Include="TestHelper.cpp">
+      <Filter>Source Files</Filter>
+    </ClCompile>
+  </ItemGroup>
+</Project>