Merge pull request #16 from Deamon87/development

1. Custom SSE commands for culling
2. Classic database is usable. Note: you can use database from retail on classic, but not vice versa
3. Several optimizations in culling flow
4. The WMO and M2 are not loaded, when they are outside of camera

.github/workflows/main.yml

@@ -74,17 +74,6 @@ jobs:
             archiver: "7z a",
             generators: "Ninja"
           }
-          - {
-            name: "Ubuntu_GCC_9",
-            os: ubuntu-latest,
-            artifact: "ubuntu_gcc9.7z",
-            build_type: "Release",
-            cmake_defines: "",
-            cc: "gcc",
-            cxx: "g++",
-            archiver: "7z a",
-            generators: "Ninja"
-          }
           - {
             name: "macOS Latest Clang",
             os: macos-latest,

.gitmodules

@@ -29,9 +29,6 @@
 [submodule "3rdparty/SQLiteCpp"]
 	path = 3rdparty/SQLiteCpp
 	url = https://github.com/SRombauts/SQLiteCpp.git
-[submodule "wowViewerLib/3rdparty/tbb"]
-	path = wowViewerLib/3rdparty/tbb
-	url = https://github.com/Deamon87/tbb.git
 [submodule "3rdparty/filesystem_impl"]
 	path = 3rdparty/filesystem_impl
 	url = https://github.com/gulrak/filesystem.git
@@ -41,7 +38,9 @@
 [submodule "emscripten_port/3rdparty/zipper"]
 	path = emscripten_port/3rdparty/zipper
 	url = https://github.com/sebastiandev/zipper.git
-
 [submodule "3rdparty/DBImporter"]
 	path = 3rdparty/DBImporter
 	url = https://github.com/Deamon87/DBImporter.git
+[submodule "wowViewerLib/3rdparty/oneTbb"]
+	path = wowViewerLib/3rdparty/oneTbb
+	url = https://github.com/Deamon87/oneTBBWasm.git

CMakeLists.txt

@@ -90,7 +90,6 @@ if (TRUE)
 	message("CMAKE_BINARY_DIR = ${CMAKE_BINARY_DIR}")
 	message("CMAKE_BUILD_TYPE = ${CMAKE_BUILD_TYPE}")
 
-	set(ZLIB_LIBRARY "$<TARGET_FILE:zlibstatic>" CACHE BOOL "")
 endif()
 
 
@@ -327,7 +326,7 @@ endif()
 target_link_libraries(AWebWoWViewerCpp ${GLEW_LIBRARIES})
 target_link_libraries(AWebWoWViewerCpp ${OPENGL_LIBRARIES})
 target_link_libraries(AWebWoWViewerCpp ${OPENGL_LIB})
-target_link_libraries(AWebWoWViewerCpp zlibstatic)
+#target_link_libraries(AWebWoWViewerCpp zlibstatic)
 target_link_libraries(AWebWoWViewerCpp libprotobuf-lite)
 
 

emscripten_port/CMakeLists.txt

@@ -18,10 +18,13 @@ message( EMSCRIPTEN_ROOT_PATH = ${EMSCRIPTEN_ROOT_PATH})
 option(COMPILE_WITH_SIMD "Compile with experimental WASM SIMD support" OFF)
 message(COMPILE_WITH_SIMD = ${COMPILE_WITH_SIMD})
 
-set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} --emit-symbol-map" )
-set(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} --emit-symbol-map" )
-set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} --emit-symbol-map")
-set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} --emit-symbol-map")
+set(ADDITIONAL_COMPILATION_FLAGS "--emit-symbol-map")
+#set(ADDITIONAL_COMPILATION_FLAGS "${ADDITIONAL_COMPILATION_FLAGS} -gsource-map ")
+
+set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} ${ADDITIONAL_COMPILATION_FLAGS}" )
+set(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} ${ADDITIONAL_COMPILATION_FLAGS}" )
+set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} ${ADDITIONAL_COMPILATION_FLAGS}")
+set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} ${ADDITIONAL_COMPILATION_FLAGS}")
 
 check_cxx_compiler_flag(-std=c++17 HAVE_FLAG_STD_CXX17)
 if(HAVE_FLAG_STD_CXX17)

src/database/CSqliteDB.cpp

@@ -135,19 +135,19 @@ CSqliteDB::CSqliteDB(std::string dbFileName) :
 
     getLightData(m_sqliteDatabase, lightDataSQL),
     getLiquidObjectInfo(m_sqliteDatabase,
-  getIsClassic() ? liquidObjectInfoSQL_classic : liquidObjectInfoSQL
+        getHasLiquidTypeXTexture(m_sqliteDatabase) ? liquidObjectInfoSQL : liquidObjectInfoSQL_classic
     ),
     getLiquidTypeInfo(m_sqliteDatabase,
-        getIsClassic() ? liquidTypeSQL_classic : liquidTypeSQL
+        getHasLiquidTypeXTexture(m_sqliteDatabase) ? liquidTypeSQL : liquidTypeSQL_classic
     ),
     getZoneLightInfo(m_sqliteDatabase,
         "select ID, Name, LightID, Zmin, Zmax from ZoneLight where MapID = ?"
     ),
     getZoneLightPointsInfo(m_sqliteDatabase,
-        "select Pos_0, Pos_1 from ZoneLightPoint where ZoneLightID = ? order by PointOrder;"
+        "select Pos_0, Pos_1 from ZoneLightPoint where ZoneLightID = ? order by PointOrder DESC;"
     ),
-    getMapList(m_sqliteDatabase, !getIsClassic() ? getMapListSQL : getMapListSQL_classic),
-    getMapByIdStatement(m_sqliteDatabase, !getIsClassic() ? getMapByIDSQL : getMapByIDSQL_classic)
+    getMapList(m_sqliteDatabase, getHasWDTId(m_sqliteDatabase) ? getMapListSQL : getMapListSQL_classic),
+    getMapByIdStatement(m_sqliteDatabase, getHasWDTId(m_sqliteDatabase) ? getMapByIDSQL : getMapByIDSQL_classic)
 {
     char *sErrMsg = "";
     sqlite3_exec(m_sqliteDatabase.getHandle(), "PRAGMA synchronous = OFF", NULL, NULL, &sErrMsg);
@@ -245,13 +245,13 @@ AreaRecord CSqliteDB::getWmoArea(int wmoId, int nameId, int groupId) {
 
 template <int T>
 float getFloatFromInt(int value) {
-    if (T == 0) {
+    if constexpr (T == 0) {
         return (value & 0xFF) / 255.0f;
     }
-    if (T == 1) {
+    if constexpr (T == 1) {
         return ((value >> 8) & 0xFF) / 255.0f;
     }
-    if (T == 2) {
+    if constexpr (T == 2) {
         return ((value >> 16) & 0xFF) / 255.0f;
     }
 }
@@ -276,7 +276,7 @@ void blendTwoAndAdd(float *colorF, int currLdRes, int lastLdRes, float timeAlpha
                                     getFloatFromInt<2>(lastLdRes) *
                                     (1.0f - timeAlphaBlend)) * innerAlpha;
 }
-void blendTwoAndAdd(std::array<float, 3> &colorF, int currLdRes, int lastLdRes, float timeAlphaBlend, float innerAlpha) {
+inline void blendTwoAndAdd(std::array<float, 3> &colorF, int currLdRes, int lastLdRes, float timeAlphaBlend, float innerAlpha) {
     colorF[0] += (getFloatFromInt<0>(currLdRes) * timeAlphaBlend +
                   getFloatFromInt<0>(lastLdRes) *
                   (1.0f - timeAlphaBlend)) * innerAlpha;
@@ -288,23 +288,23 @@ void blendTwoAndAdd(std::array<float, 3> &colorF, int currLdRes, int lastLdRes,
                   (1.0f - timeAlphaBlend)) * innerAlpha;
 }
 
-void blendTwoAndAdd(float &colorF, float currLdRes, float lastLdRes, float timeAlphaBlend, float innerAlpha) {
+inline void blendTwoAndAdd(float &colorF, float currLdRes, float lastLdRes, float timeAlphaBlend, float innerAlpha) {
     colorF += (currLdRes * timeAlphaBlend + lastLdRes * (1.0f - timeAlphaBlend)) * innerAlpha;
 }
 
-void addOnlyOne(float *colorF, int currLdRes, float innerAlpha) {
+inline void addOnlyOne(float *colorF, int currLdRes, float innerAlpha) {
     colorF[0] += getFloatFromInt<0>(currLdRes) * innerAlpha;
     colorF[1] += getFloatFromInt<1>(currLdRes) * innerAlpha;
     colorF[2] += getFloatFromInt<2>(currLdRes) * innerAlpha;
 }
 
-void addOnlyOne(std::array<float, 3> &colorF, int currLdRes, float innerAlpha) {
+inline void addOnlyOne(std::array<float, 3> &colorF, int currLdRes, float innerAlpha) {
     colorF[0] += getFloatFromInt<0>(currLdRes) * innerAlpha;
     colorF[1] += getFloatFromInt<1>(currLdRes) * innerAlpha;
     colorF[2] += getFloatFromInt<2>(currLdRes) * innerAlpha;
 }
 
-void addOnlyOne(float &colorF, float currLdRes, float innerAlpha) {
+inline void addOnlyOne(float &colorF, float currLdRes, float innerAlpha) {
     colorF += currLdRes * innerAlpha;
 }
 void CSqliteDB::getLightById(int lightId, int time, LightResult &lightResult) {

src/database/CSqliteDB.h

@@ -65,17 +65,22 @@ class CSqliteDB : public IClientDatabase {
     StatementFieldHolder getMapList;
     StatementFieldHolder getMapByIdStatement;
 
+    bool getHasLiquidTypeXTexture(SQLite::Database &sqliteDatabase) {
+        bool m_hasLiquidTypeXTexture = sqliteDatabase.tableExists("LiquidTypeXTexture") ? 1 : 0;
 
-    bool m_isClassic = false;
-    bool m_isClassicChecked = false;
+        return m_hasLiquidTypeXTexture;
+    }
 
-    bool getIsClassic() {
-        if (!m_isClassicChecked) {
-            m_isClassic = !m_sqliteDatabase.tableExists("LiquidTypeXTexture");
-            m_isClassicChecked = true;
+    bool getHasWDTId(SQLite::Database &sqliteDatabase) {
+        bool m_hasWdtId;
+        try {
+            sqliteDatabase.execAndGet("select WdtFileDataID from Map");
+            m_hasWdtId = true;
+        } catch (...) {
+            m_hasWdtId = false;
         }
 
-        return m_isClassic;
+        return m_hasWdtId;
     }
 
     struct InnerLightResult {

src/minimapGenerator/minimapGenerator.cpp

@@ -578,7 +578,7 @@ void MinimapGenerator::process() {
         mathfu::vec3 minCoord = mathfu::vec3(20000, 20000, 20000);
         mathfu::vec3 maxCoord = mathfu::vec3(-20000, -20000, -20000);
 
-        for (auto &m2Object: lastFrameCull->m2Array) {
+        for (auto &m2Object: lastFrameCull->m2Array.getCandidates()) {
             auto objBB = m2Object->getAABB();
 
             if (!MathHelper::isAabbIntersect2d(objBB, adtBox2d)) continue;
@@ -595,7 +595,7 @@ void MinimapGenerator::process() {
             );
         }
 
-        for (auto &wmoObject: lastFrameCull->wmoArray) {
+        for (auto &wmoObject: lastFrameCull->wmoArray.getCandidates()) {
             auto objBB = wmoObject->getAABB();
             if (!MathHelper::isAabbIntersect2d(objBB, adtBox2d)) continue;
 

src/persistance/CascRequestProcessor.cpp

@@ -46,7 +46,7 @@ CascRequestProcessor::CascRequestProcessor(std::string &path, BuildDefinition &b
 
     m_cascDir = path;
     if (!buildDef.productName.empty()) {
-        path = path + ":"+buildDef.productName+":"+buildDef.region+":"+buildDef.buildConfig;
+        path = path + "*"+buildDef.productName+"**"+buildDef.region;
     }
 
     uint32_t localMask = 0xFFFFFFFF;

src/ui/FrontendUI.cpp

@@ -146,6 +146,7 @@ void FrontendUI::showCurrentStatsDialog() {
                 ImGui::Text("Current debug camera position: (%.1f,%.1f,%.1f)",
                             debugCameraPosition[0], debugCameraPosition[1], debugCameraPosition[2]);
             }
+            ImGui::Separator();
         }
         ImGui::Text("Application average %.3f ms/frame (%.1f FPS)", 1000.0f / ImGui::GetIO().Framerate,
                     ImGui::GetIO().Framerate);
@@ -171,17 +172,46 @@ void FrontendUI::showCurrentStatsDialog() {
             ImGui::Text("Elapsed time on textureUploadCNT: %.3f ms", m_api->getConfig()->textureUploadCNT);
             ImGui::Text("Elapsed time on drawStageAndDepsCNT: %.3f ms", m_api->getConfig()->drawStageAndDepsCNT);
             ImGui::Text("Elapsed time on endUpdateCNT: %.3f ms", m_api->getConfig()->endUpdateCNT);
+
+            ImGui::Text("Elapsed time on cullCreateVarsCounter: %.3f ms", m_api->getConfig()->cullCreateVarsCounter);
+            ImGui::Text("Elapsed time on cullGetCurrentWMOCounter: %.3f ms", m_api->getConfig()->cullGetCurrentWMOCounter);
+            ImGui::Text("Elapsed time on cullGetCurrentZoneCounter: %.3f ms", m_api->getConfig()->cullGetCurrentZoneCounter);
+            ImGui::Text("Elapsed time on cullUpdateLightsFromDBCounter: %.3f ms", m_api->getConfig()->cullUpdateLightsFromDBCounter);
+            ImGui::Text("Elapsed time on cullExterior: %.3f ms", m_api->getConfig()->cullExterior);
+            ImGui::Text("Elapsed time on cullExteriorWDLCull: %.3f ms", m_api->getConfig()->cullExteriorWDLCull);
+            ImGui::Text("Elapsed time on cullExteriorSetDecl: %.3f ms", m_api->getConfig()->cullExteriorSetDecl);
+            ImGui::Text("Elapsed time on cullExteriorGetCands: %.3f ms", m_api->getConfig()->cullExteriorGetCands);
+            ImGui::Text("Elapsed time on cullExterioFrustumWMO: %.3f ms", m_api->getConfig()->cullExterioFrustumWMO);
+            ImGui::Text("Elapsed time on cullExterioFrustumM2: %.3f ms", m_api->getConfig()->cullExterioFrustumM2);
+            ImGui::Text("Elapsed time on cullSkyDoms: %.3f ms", m_api->getConfig()->cullSkyDoms);
+            ImGui::Text("Elapsed time on cullCombineAllObjects: %.3f ms", m_api->getConfig()->cullCombineAllObjects);
+            ImGui::Separator();
         }
 
         int currentFrame = m_api->hDevice->getDrawFrameNumber();
         auto &cullStageData = m_cullstages[currentFrame];
 
-        if (ImGui::CollapsingHeader("Objects Drawn")) {
-            int m2ObjectsDrawn = cullStageData!= nullptr ? cullStageData->m2Array.size() : 0;
-            int wmoObjectsDrawn = cullStageData!= nullptr ? cullStageData->wmoArray.size() : 0;
+        if (ImGui::CollapsingHeader("Objects Drawn/Culled")) {
+            int m2ObjectsBeforeCullingExterior = 0;
+            if (cullStageData->viewsHolder.getExterior() != nullptr) {
+                m2ObjectsBeforeCullingExterior = cullStageData->viewsHolder.getExterior()->m2List.getCandidates().size();
+            }
+
+            int wmoGroupsInExterior = 0;
+            if (cullStageData->viewsHolder.getExterior() != nullptr) {
+                wmoGroupsInExterior = cullStageData->viewsHolder.getExterior()->wmoGroupArray.getToDraw().size();
+            }
+
+            int m2ObjectsDrawn = cullStageData!= nullptr ? cullStageData->m2Array.getDrawn().size() : 0;
+            int wmoObjectsBeforeCull = cullStageData!= nullptr ? cullStageData->wmoArray.getCandidates().size() : 0;
 
             ImGui::Text("M2 objects drawn: %s", std::to_string(m2ObjectsDrawn).c_str());
-            ImGui::Text("WMO objects drawn: %s", std::to_string(wmoObjectsDrawn).c_str());
+            ImGui::Text("WMO Groups in Exterior: %s", std::to_string(wmoGroupsInExterior).c_str());
+            ImGui::Text("Interiors (aka group WMOs): %s", std::to_string(cullStageData->viewsHolder.getInteriorViews().size()).c_str());
+            ImGui::Text("M2 Objects Before Culling in Exterior: %s", std::to_string(m2ObjectsBeforeCullingExterior).c_str());
+            ImGui::Text("WMO objects before culling: %s", std::to_string(wmoObjectsBeforeCull).c_str());
+
+            ImGui::Separator();
         }
 
         if (ImGui::CollapsingHeader("Current fog params")) {
@@ -215,6 +245,7 @@ void FrontendUI::showCurrentStatsDialog() {
                             cullStageData->frameDepedantData->FogHeightCoefficients.x,
                             cullStageData->frameDepedantData->FogHeightCoefficients.y,
                             cullStageData->frameDepedantData->FogHeightCoefficients.z);
+                ImGui::Separator();
             }
         }
 
@@ -484,7 +515,7 @@ void FrontendUI::showMainMenu() {
             ImGui::Separator();
             if (ImGui::MenuItem("Update database", "", false, cascOpened)) {
                 m_databaseUpdateWorkflow = std::make_shared<DatabaseUpdateWorkflow>(
-                        m_api->cacheStorage,
+                        m_api,
                         contains(fileDialog.getProductBuild().productName, "classic")
                     );
             }
@@ -1213,11 +1244,9 @@ void FrontendUI::showSettingsDialog() {
         }
 
 
-//        if (ImGui::SliderInt("Thread Count", &threadCount, 2, 16)) {
-//            if (setThreadCount){
-//                setThreadCount(threadCount);
-//            }
-//        }
+        if (ImGui::SliderInt("Thread Count", &threadCount, 2, 16)) {
+            m_api->getConfig()->threadCount = threadCount;
+        }
 //        if (ImGui::SliderInt("QuickSort cutoff", &quickSortCutoff, 1, 1000)) {
 //            if (setQuicksortCutoff){
 //                setQuicksortCutoff(quickSortCutoff);

src/ui/childWindow/databaseUpdateWorkflow/DatabaseUpdateWorkflow.cpp

@@ -42,8 +42,8 @@ static const std::string POPUP_UPDATING_DBD = "Updating DBD files";
 static const std::string POPUP_UPDATING_DATABASE_DB2 = "Updating database from DB2";
 static const std::string POPUP_UPDATED_DATABASE_DB2 = "Database updated succesfully";
 
-DatabaseUpdateWorkflow::DatabaseUpdateWorkflow(std::shared_ptr<WoWFilesCacheStorage> storage, bool isClassic) {
-    m_storage = storage;
+DatabaseUpdateWorkflow::DatabaseUpdateWorkflow(HApiContainer &api, bool isClassic) {
+    m_api = api;
     m_isClassic = isClassic;
 }
 
@@ -239,7 +239,7 @@ void DatabaseUpdateWorkflow::db2UpdateLogic() {
         auto &l_addTableLambda = addTableLambda;
         auto &l_checkDB2Lambda = checkDB2Lambda;
 
-        auto dbFile = m_storage->getDb2Cache()->getFileId(requiredTables[m_currentDB2File].fileDataId);
+        auto dbFile = m_api->cacheStorage->getDb2Cache()->getFileId(requiredTables[m_currentDB2File].fileDataId);
         checkDB2Lambda = [dbFile, &l_addTableLambda, &l_currentDB2File, &l_db2FailedMessage, &l_checkDB2Lambda]() -> bool {
             if (dbFile->getStatus() == FileStatus::FSNotLoaded)
                 return false;

src/ui/childWindow/databaseUpdateWorkflow/DatabaseUpdateWorkflow.h

@@ -11,23 +11,20 @@
 
 class DatabaseUpdateWorkflow {
 public:
-    DatabaseUpdateWorkflow(std::shared_ptr<WoWFilesCacheStorage> storage, bool isClassic);
+    DatabaseUpdateWorkflow(HApiContainer &api, bool isClassic);
 
     void render();
     bool isDatabaseUpdated() { return m_databaseUpdated;};
 private:
     //Showing elements state.
     bool m_showDBDPrompt = true;
-
-
-
     bool m_needToUpdateDBD = false;
     bool m_needToUpdateDatabase = false;
     bool m_showOkModal = false;
     bool m_databaseUpdated = false;
 
 private:
-    std::shared_ptr<WoWFilesCacheStorage> m_storage;
+    HApiContainer m_api;
     bool m_isClassic = false;
 
     void defineDialogs();
@@ -48,7 +45,6 @@ class DatabaseUpdateWorkflow {
     std::function<bool(std::string, std::shared_ptr<Db2File>)> addTableLambda = nullptr;
     bool m_db2ThreadFinished = true;
     std::string m_db2FailedMessage = "";
-
 };
 
 

src/ui/imguiLib/fileBrowser/buildDefinition.h

@@ -19,7 +19,7 @@ struct BuildDefinition {
         return
             (buildVersion == value.buildVersion) &&
             (buildConfig == value.buildConfig) &&
-            (region == value.region);
+            (region == value.region) &&
             (productName == value.productName);
     }
 };