diff --git a/.gitmodules b/.gitmodules index 64995f1c3188..54b352717be2 100644 --- a/.gitmodules +++ b/.gitmodules @@ -84,3 +84,6 @@ [submodule "Externals/Vulkan-Headers"] path = Externals/Vulkan-Headers url = https://github.com/KhronosGroup/Vulkan-Headers.git +[submodule "Externals/watcher/watcher"] + path = Externals/watcher/watcher + url = https://github.com/e-dant/watcher.git diff --git a/CMakeLists.txt b/CMakeLists.txt index 5b246b193b09..037242abb58b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -775,6 +775,8 @@ if (USE_RETRO_ACHIEVEMENTS) add_subdirectory(Externals/rcheevos) endif() +add_subdirectory(Externals/watcher) + ######################################## # Pre-build events: Define configuration variables and write SCM info header # diff --git a/Externals/watcher/CMakeLists.txt b/Externals/watcher/CMakeLists.txt new file mode 100644 index 000000000000..097f16d64753 --- /dev/null +++ b/Externals/watcher/CMakeLists.txt @@ -0,0 +1,4 @@ +add_library(watcher INTERFACE IMPORTED GLOBAL) +set_target_properties(watcher PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES ${CMAKE_CURRENT_LIST_DIR}/watcher/include +) diff --git a/Externals/watcher/watcher b/Externals/watcher/watcher new file mode 160000 index 000000000000..0d6b9b409cca --- /dev/null +++ b/Externals/watcher/watcher @@ -0,0 +1 @@ +Subproject commit 0d6b9b409ccaed6313437ea3dc8b2fc078f3d25b diff --git a/Source/Core/DolphinLib.props b/Source/Core/DolphinLib.props index e236276ce9c2..16062d9164a8 100644 --- a/Source/Core/DolphinLib.props +++ b/Source/Core/DolphinLib.props @@ -663,6 +663,8 @@ + + @@ -1308,6 +1310,7 @@ + diff --git a/Source/Core/VideoCommon/Assets/CustomAsset.cpp b/Source/Core/VideoCommon/Assets/CustomAsset.cpp index 1591f93f9138..96205b084c3c 100644 --- a/Source/Core/VideoCommon/Assets/CustomAsset.cpp +++ b/Source/Core/VideoCommon/Assets/CustomAsset.cpp @@ -6,8 +6,8 @@ namespace VideoCommon { CustomAsset::CustomAsset(std::shared_ptr library, - const CustomAssetLibrary::AssetID& asset_id) - : m_owning_library(std::move(library)), m_asset_id(asset_id) + const CustomAssetLibrary::AssetID& asset_id, u64 session_id) + : m_owning_library(std::move(library)), m_asset_id(asset_id), m_session_id(session_id) { } @@ -34,6 +34,11 @@ const CustomAssetLibrary::TimeType& CustomAsset::GetLastLoadedTime() const return m_last_loaded_time; } +std::size_t CustomAsset::GetSessionId() const +{ + return m_session_id; +} + const CustomAssetLibrary::AssetID& CustomAsset::GetAssetId() const { return m_asset_id; diff --git a/Source/Core/VideoCommon/Assets/CustomAsset.h b/Source/Core/VideoCommon/Assets/CustomAsset.h index ea4a182932aa..85f0471ff7b8 100644 --- a/Source/Core/VideoCommon/Assets/CustomAsset.h +++ b/Source/Core/VideoCommon/Assets/CustomAsset.h @@ -18,7 +18,7 @@ class CustomAsset { public: CustomAsset(std::shared_ptr library, - const CustomAssetLibrary::AssetID& asset_id); + const CustomAssetLibrary::AssetID& asset_id, u64 session_id); virtual ~CustomAsset() = default; CustomAsset(const CustomAsset&) = delete; CustomAsset(CustomAsset&&) = delete; @@ -39,6 +39,9 @@ class CustomAsset // Returns an id that uniquely identifies this asset const CustomAssetLibrary::AssetID& GetAssetId() const; + // Returns an id that is unique to this session + std::size_t GetSessionId() const; + // A rough estimate of how much space this asset // will take in memroy std::size_t GetByteSizeInMemory() const; @@ -49,6 +52,7 @@ class CustomAsset private: virtual CustomAssetLibrary::LoadInfo LoadImpl(const CustomAssetLibrary::AssetID& asset_id) = 0; CustomAssetLibrary::AssetID m_asset_id; + std::size_t m_session_id; mutable std::mutex m_info_lock; std::size_t m_bytes_loaded = 0; diff --git a/Source/Core/VideoCommon/Assets/CustomAssetLoader.cpp b/Source/Core/VideoCommon/Assets/CustomAssetLoader.cpp index 119d7444eae1..e33dff46ab0f 100644 --- a/Source/Core/VideoCommon/Assets/CustomAssetLoader.cpp +++ b/Source/Core/VideoCommon/Assets/CustomAssetLoader.cpp @@ -4,13 +4,17 @@ #include "VideoCommon/Assets/CustomAssetLoader.h" #include "Common/MemoryUtil.h" +#include "Common/Thread.h" + #include "VideoCommon/Assets/CustomAssetLibrary.h" +#include "VideoCommon/VideoEvents.h" namespace VideoCommon { void CustomAssetLoader::Init() { - m_asset_monitor_thread_shutdown.Clear(); + m_frame_event = + AfterFrameEvent::Register([this](Core::System&) { OnFrameEnd(); }, "CustomAssetLoader"); const size_t sys_mem = Common::MemPhysical(); const size_t recommended_min_mem = 2 * size_t(1024 * 1024 * 1024); @@ -18,65 +22,12 @@ void CustomAssetLoader::Init() m_max_memory_available = (sys_mem / 2 < recommended_min_mem) ? (sys_mem / 2) : (sys_mem - recommended_min_mem); - m_asset_monitor_thread = std::thread([this]() { - Common::SetCurrentThreadName("Asset monitor"); - while (true) - { - if (m_asset_monitor_thread_shutdown.IsSet()) - { - break; - } - - std::this_thread::sleep_for(TIME_BETWEEN_ASSET_MONITOR_CHECKS); - - std::lock_guard lk(m_asset_load_lock); - for (auto& [asset_id, asset_to_monitor] : m_assets_to_monitor) - { - if (auto ptr = asset_to_monitor.lock()) - { - const auto write_time = ptr->GetLastWriteTime(); - if (write_time > ptr->GetLastLoadedTime()) - { - (void)ptr->Load(); - } - } - } - } - }); - - m_asset_load_thread.Reset("Custom Asset Loader", [this](std::weak_ptr asset) { - if (auto ptr = asset.lock()) - { - if (m_memory_exceeded) - return; - - if (ptr->Load()) - { - std::lock_guard lk(m_asset_load_lock); - const std::size_t asset_memory_size = ptr->GetByteSizeInMemory(); - m_total_bytes_loaded += asset_memory_size; - m_assets_to_monitor.try_emplace(ptr->GetAssetId(), ptr); - if (m_total_bytes_loaded > m_max_memory_available) - { - ERROR_LOG_FMT(VIDEO, - "Asset memory exceeded with asset '{}', future assets won't load until " - "memory is available.", - ptr->GetAssetId()); - m_memory_exceeded = true; - } - } - } - }); + ResizeWorkerThreads(2); } void CustomAssetLoader ::Shutdown() { - m_asset_load_thread.Shutdown(true); - - m_asset_monitor_thread_shutdown.Set(); - m_asset_monitor_thread.join(); - m_assets_to_monitor.clear(); - m_total_bytes_loaded = 0; + Reset(false); } std::shared_ptr @@ -105,4 +56,242 @@ std::shared_ptr CustomAssetLoader::LoadMesh(const CustomAssetLibrary: { return LoadOrCreateAsset(asset_id, m_meshes, std::move(library)); } + +void CustomAssetLoader::AssetReferenced(u64 asset_session_id) +{ + auto& asset_load_info = m_asset_load_info[asset_session_id]; + asset_load_info.last_xfb_seen = m_xfbs_seen; +} + +void CustomAssetLoader::Reset(bool restart_worker_threads) +{ + const std::size_t worker_thread_count = m_worker_threads.size(); + StopWorkerThreads(); + + m_game_textures.clear(); + m_pixel_shaders.clear(); + m_materials.clear(); + m_meshes.clear(); + + m_assetid_to_asset_index.clear(); + m_asset_load_info.clear(); + + { + std::lock_guard guard(m_reload_work_lock); + m_assetids_to_reload.clear(); + } + + { + std::lock_guard guard(m_pending_work_lock); + m_pending_work_per_frame.clear(); + } + + { + std::lock_guard guard(m_completed_work_lock); + m_completed_work.clear(); + } + + if (restart_worker_threads) + { + StartWorkerThreads(static_cast(worker_thread_count)); + } +} + +void CustomAssetLoader::ReloadAsset(const CustomAssetLibrary::AssetID& asset_id) +{ + std::lock_guard guard(m_reload_work_lock); + m_assetids_to_reload.push_back(asset_id); +} + +bool CustomAssetLoader::StartWorkerThreads(u32 num_worker_threads) +{ + if (num_worker_threads == 0) + return true; + + for (u32 i = 0; i < num_worker_threads; i++) + { + m_worker_thread_start_result.store(false); + + void* thread_param = nullptr; + std::thread thr(&CustomAssetLoader::WorkerThreadEntryPoint, this, thread_param); + m_init_event.Wait(); + + if (!m_worker_thread_start_result.load()) + { + WARN_LOG_FMT(VIDEO, "Failed to start asset load worker thread."); + thr.join(); + break; + } + + m_worker_threads.push_back(std::move(thr)); + } + + return HasWorkerThreads(); +} + +bool CustomAssetLoader::ResizeWorkerThreads(u32 num_worker_threads) +{ + if (m_worker_threads.size() == num_worker_threads) + return true; + + StopWorkerThreads(); + return StartWorkerThreads(num_worker_threads); +} + +bool CustomAssetLoader::HasWorkerThreads() const +{ + return !m_worker_threads.empty(); +} + +void CustomAssetLoader::StopWorkerThreads() +{ + if (!HasWorkerThreads()) + return; + + // Signal worker threads to stop, and wake all of them. + { + std::lock_guard guard(m_pending_work_lock); + m_exit_flag.Set(); + m_worker_thread_wake.notify_all(); + } + + // Wait for worker threads to exit. + for (std::thread& thr : m_worker_threads) + thr.join(); + m_worker_threads.clear(); + m_exit_flag.Clear(); +} + +void CustomAssetLoader::WorkerThreadEntryPoint(void* param) +{ + Common::SetCurrentThreadName("Asset Loader Worker"); + + m_worker_thread_start_result.store(true); + m_init_event.Set(); + + WorkerThreadRun(); +} + +void CustomAssetLoader::WorkerThreadRun() +{ + std::unique_lock pending_lock(m_pending_work_lock); + while (!m_exit_flag.IsSet()) + { + m_worker_thread_wake.wait(pending_lock); + + while (!m_pending_work_per_frame.empty() && !m_exit_flag.IsSet()) + { + m_busy_workers++; + + auto pending_iter = m_pending_work_per_frame.begin(); + auto item(std::move(pending_iter->second)); + m_pending_work_per_frame.erase(pending_iter); + + const auto item_shared = item.lock(); + pending_lock.unlock(); + + if (item_shared && m_last_frame_total_loaded_memory < m_max_memory_available) + { + if (item_shared->Load()) + { + // This asset could be double counted, but will be corected on the next frame + m_last_frame_total_loaded_memory += item_shared->GetByteSizeInMemory(); + } + + // Regardless of whether the load was successful or not + // mark it as complete + { + std::lock_guard completed_guard(m_completed_work_lock); + m_completed_work.push_back(item_shared->GetSessionId()); + } + } + + pending_lock.lock(); + m_busy_workers--; + } + } +} + +void CustomAssetLoader::OnFrameEnd() +{ + std::vector assetids_to_reload; + { + std::lock_guard guard(m_reload_work_lock); + m_assetids_to_reload.swap(assetids_to_reload); + } + for (const CustomAssetLibrary::AssetID& asset_id : assetids_to_reload) + { + if (const auto asset_session_id_iter = m_assetid_to_asset_index.find(asset_id); + asset_session_id_iter != m_assetid_to_asset_index.end()) + { + auto& asset_load_info = m_asset_load_info[asset_session_id_iter->second]; + asset_load_info.xfb_load_request = m_xfbs_seen; + } + } + + std::vector completed_work; + { + std::lock_guard guard(m_completed_work_lock); + m_completed_work.swap(completed_work); + } + for (std::size_t completed_index : completed_work) + { + auto& asset_load_info = m_asset_load_info[completed_index]; + + // If we had a load request and it wasn't from this frame, clear it + if (asset_load_info.xfb_load_request && asset_load_info.xfb_load_request != m_xfbs_seen) + { + asset_load_info.xfb_load_request = std::nullopt; + } + } + + m_xfbs_seen++; + + std::size_t total_bytes_loaded = 0; + + // Build up the work prioritizing newest requested assets first + PendingWorkContainer new_pending_work; + for (const auto& asset_load_info : m_asset_load_info) + { + if (const auto asset = asset_load_info.asset.lock()) + { + total_bytes_loaded += asset->GetByteSizeInMemory(); + if (total_bytes_loaded > m_max_memory_available) + { + if (!m_memory_exceeded) + { + m_memory_exceeded = true; + ERROR_LOG_FMT(VIDEO, + "Asset memory exceeded with asset '{}', future assets won't load until " + "memory is available.", + asset->GetAssetId()); + } + break; + } + if (asset_load_info.xfb_load_request) + { + new_pending_work.emplace(asset_load_info.last_xfb_seen, asset_load_info.asset); + } + } + } + + if (m_memory_exceeded && total_bytes_loaded < m_max_memory_available) + { + INFO_LOG_FMT(VIDEO, "Asset memory went below limit, new assets can begin loading."); + m_memory_exceeded = false; + } + + m_last_frame_total_loaded_memory = total_bytes_loaded; + + if (new_pending_work.empty()) + return; + + // Now notify our workers + { + std::lock_guard guard(m_pending_work_lock); + std::swap(m_pending_work_per_frame, new_pending_work); + m_worker_thread_wake.notify_all(); + } +} + } // namespace VideoCommon diff --git a/Source/Core/VideoCommon/Assets/CustomAssetLoader.h b/Source/Core/VideoCommon/Assets/CustomAssetLoader.h index 90d4f81a0ea2..e376b6b71f8e 100644 --- a/Source/Core/VideoCommon/Assets/CustomAssetLoader.h +++ b/Source/Core/VideoCommon/Assets/CustomAssetLoader.h @@ -4,19 +4,24 @@ #pragma once #include +#include #include #include #include +#include #include +#include +#include "Common/Event.h" #include "Common/Flag.h" +#include "Common/HookableEvent.h" #include "Common/Logging/Log.h" -#include "Common/WorkQueueThread.h" #include "VideoCommon/Assets/CustomAsset.h" #include "VideoCommon/Assets/MaterialAsset.h" #include "VideoCommon/Assets/MeshAsset.h" #include "VideoCommon/Assets/ShaderAsset.h" #include "VideoCommon/Assets/TextureAsset.h" +#include "VideoCommon/Present.h" namespace VideoCommon { @@ -52,6 +57,14 @@ class CustomAssetLoader std::shared_ptr LoadMesh(const CustomAssetLibrary::AssetID& asset_id, std::shared_ptr library); + // Notifies the asset system that an asset has been used + void AssetReferenced(u64 asset_session_id); + + // Requests that an asset that exists be reloaded + void ReloadAsset(const CustomAssetLibrary::AssetID& asset_id); + + void Reset(bool restart_worker_threads = true); + private: // TODO C++20: use a 'derived_from' concept against 'CustomAsset' when available template @@ -65,44 +78,88 @@ class CustomAssetLoader { auto shared = it->second.lock(); if (shared) - return shared; - } - std::shared_ptr ptr(new AssetType(std::move(library), asset_id), [&](AssetType* a) { { - std::lock_guard lk(m_asset_load_lock); - m_total_bytes_loaded -= a->GetByteSizeInMemory(); - m_assets_to_monitor.erase(a->GetAssetId()); - if (m_max_memory_available >= m_total_bytes_loaded && m_memory_exceeded) - { - INFO_LOG_FMT(VIDEO, "Asset memory went below limit, new assets can begin loading."); - m_memory_exceeded = false; - } + auto& asset_load_info = m_asset_load_info[shared->GetSessionId()]; + asset_load_info.last_xfb_seen = m_xfbs_seen; + asset_load_info.xfb_load_request = m_xfbs_seen; + return shared; } - delete a; - }); + } + + const auto [index_it, index_inserted] = m_assetid_to_asset_index.try_emplace(asset_id, 0); + if (index_inserted) + { + index_it->second = m_asset_load_info.size(); + } + + auto ptr = std::make_shared(std::move(library), asset_id, index_it->second); it->second = ptr; - m_asset_load_thread.Push(it->second); + + AssetLoadInfo* asset_load_info; + if (index_inserted) + { + m_asset_load_info.emplace_back(); + asset_load_info = &m_asset_load_info.back(); + } + else + { + asset_load_info = &m_asset_load_info[index_it->second]; + } + + asset_load_info->asset = ptr; + asset_load_info->last_xfb_seen = m_xfbs_seen; + asset_load_info->xfb_load_request = m_xfbs_seen; return ptr; } - static constexpr auto TIME_BETWEEN_ASSET_MONITOR_CHECKS = std::chrono::milliseconds{500}; + bool StartWorkerThreads(u32 num_worker_threads); + bool ResizeWorkerThreads(u32 num_worker_threads); + bool HasWorkerThreads() const; + void StopWorkerThreads(); + + void WorkerThreadEntryPoint(void* param); + void WorkerThreadRun(); + + void OnFrameEnd(); + + Common::EventHook m_frame_event; std::map> m_game_textures; std::map> m_pixel_shaders; std::map> m_materials; std::map> m_meshes; - std::thread m_asset_monitor_thread; - Common::Flag m_asset_monitor_thread_shutdown; - std::size_t m_total_bytes_loaded = 0; + std::map m_assetid_to_asset_index; + + struct AssetLoadInfo + { + std::weak_ptr asset; + u64 last_xfb_seen = 0; + std::optional xfb_load_request; + }; + std::vector m_asset_load_info; + u64 m_xfbs_seen = 0; + std::size_t m_max_memory_available = 0; - std::atomic_bool m_memory_exceeded = false; + bool m_memory_exceeded = false; + std::atomic_size_t m_last_frame_total_loaded_memory = 0; + + Common::Flag m_exit_flag; + Common::Event m_init_event; + + std::vector m_worker_threads; + std::atomic_bool m_worker_thread_start_result{false}; + + using PendingWorkContainer = std::multimap, std::greater<>>; + PendingWorkContainer m_pending_work_per_frame; + std::mutex m_pending_work_lock; + std::condition_variable m_worker_thread_wake; + std::atomic_size_t m_busy_workers{0}; - std::map> m_assets_to_monitor; + std::vector m_completed_work; + std::mutex m_completed_work_lock; - // Use a recursive mutex to handle the scenario where an asset goes out of scope while - // iterating over the assets to monitor which calls the lock above in 'LoadOrCreateAsset' - std::recursive_mutex m_asset_load_lock; - Common::WorkQueueThread> m_asset_load_thread; + std::vector m_assetids_to_reload; + std::mutex m_reload_work_lock; }; } // namespace VideoCommon diff --git a/Source/Core/VideoCommon/Assets/DirectFilesystemAssetLibrary.cpp b/Source/Core/VideoCommon/Assets/DirectFilesystemAssetLibrary.cpp index 90d77d8ec96b..73f275f3d546 100644 --- a/Source/Core/VideoCommon/Assets/DirectFilesystemAssetLibrary.cpp +++ b/Source/Core/VideoCommon/Assets/DirectFilesystemAssetLibrary.cpp @@ -13,6 +13,8 @@ #include "Common/JsonUtil.h" #include "Common/Logging/Log.h" #include "Common/StringUtil.h" +#include "Core/System.h" +#include "VideoCommon/Assets/CustomAssetLoader.h" #include "VideoCommon/Assets/MaterialAsset.h" #include "VideoCommon/Assets/MeshAsset.h" #include "VideoCommon/Assets/ShaderAsset.h" @@ -53,7 +55,7 @@ std::size_t GetAssetSize(const CustomTextureData& data) CustomAssetLibrary::TimeType DirectFilesystemAssetLibrary::GetLastAssetWriteTime(const AssetID& asset_id) const { - std::lock_guard lk(m_lock); + std::lock_guard lk(m_asset_map_lock); if (auto iter = m_assetid_to_asset_map_path.find(asset_id); iter != m_assetid_to_asset_map_path.end()) { @@ -434,10 +436,42 @@ CustomAssetLibrary::LoadInfo DirectFilesystemAssetLibrary::LoadTexture(const Ass } void DirectFilesystemAssetLibrary::SetAssetIDMapData(const AssetID& asset_id, - AssetMap asset_path_map) + VideoCommon::Assets::AssetMap asset_path_map) { - std::lock_guard lk(m_lock); - m_assetid_to_asset_map_path[asset_id] = std::move(asset_path_map); + VideoCommon::Assets::AssetMap previous_asset_map; + { + std::lock_guard lk(m_asset_map_lock); + previous_asset_map = m_assetid_to_asset_map_path[asset_id]; + } + + { + std::lock_guard lk(m_path_map_lock); + for (const auto& [name, path] : previous_asset_map) + { + m_path_to_asset_id.erase(PathToString(path)); + } + + for (const auto& [name, path] : asset_path_map) + { + m_path_to_asset_id[PathToString(path)] = asset_id; + } + } + + { + std::lock_guard lk(m_asset_map_lock); + m_assetid_to_asset_map_path[asset_id] = std::move(asset_path_map); + } +} + +void DirectFilesystemAssetLibrary::PathModified(std::string_view path) +{ + std::lock_guard lk(m_path_map_lock); + if (const auto iter = m_path_to_asset_id.find(path); iter != m_path_to_asset_id.end()) + { + auto& system = Core::System::GetInstance(); + auto& loader = system.GetCustomAssetLoader(); + loader.ReloadAsset(iter->second); + } } bool DirectFilesystemAssetLibrary::LoadMips(const std::filesystem::path& asset_path, @@ -492,10 +526,10 @@ bool DirectFilesystemAssetLibrary::LoadMips(const std::filesystem::path& asset_p return true; } -DirectFilesystemAssetLibrary::AssetMap +VideoCommon::Assets::AssetMap DirectFilesystemAssetLibrary::GetAssetMapForID(const AssetID& asset_id) const { - std::lock_guard lk(m_lock); + std::lock_guard lk(m_asset_map_lock); if (auto iter = m_assetid_to_asset_map_path.find(asset_id); iter != m_assetid_to_asset_map_path.end()) { diff --git a/Source/Core/VideoCommon/Assets/DirectFilesystemAssetLibrary.h b/Source/Core/VideoCommon/Assets/DirectFilesystemAssetLibrary.h index c4d99baf82c1..cbc72a0e9ade 100644 --- a/Source/Core/VideoCommon/Assets/DirectFilesystemAssetLibrary.h +++ b/Source/Core/VideoCommon/Assets/DirectFilesystemAssetLibrary.h @@ -8,18 +8,17 @@ #include #include -#include "VideoCommon/Assets/CustomAssetLibrary.h" #include "VideoCommon/Assets/CustomTextureData.h" +#include "VideoCommon/Assets/Types.h" +#include "VideoCommon/Assets/WatchableFilesystemAssetLibrary.h" namespace VideoCommon { // This class implements 'CustomAssetLibrary' and loads any assets // directly from the filesystem -class DirectFilesystemAssetLibrary final : public CustomAssetLibrary +class DirectFilesystemAssetLibrary final : public WatchableFilesystemAssetLibrary { public: - using AssetMap = std::map; - LoadInfo LoadTexture(const AssetID& asset_id, TextureData* data) override; LoadInfo LoadPixelShader(const AssetID& asset_id, PixelShaderData* data) override; LoadInfo LoadMaterial(const AssetID& asset_id, MaterialData* data) override; @@ -31,16 +30,21 @@ class DirectFilesystemAssetLibrary final : public CustomAssetLibrary // Assigns the asset id to a map of files, how this map is read is dependent on the data // For instance, a raw texture would expect the map to have a single entry and load that // file as the asset. But a model file data might have its data spread across multiple files - void SetAssetIDMapData(const AssetID& asset_id, AssetMap asset_path_map); + void SetAssetIDMapData(const AssetID& asset_id, Assets::AssetMap asset_path_map); private: + void PathModified(std::string_view path) override; + // Loads additional mip levels into the texture structure until _mip texture is not found bool LoadMips(const std::filesystem::path& asset_path, CustomTextureData::ArraySlice* data); // Gets the asset map given an asset id - AssetMap GetAssetMapForID(const AssetID& asset_id) const; + Assets::AssetMap GetAssetMapForID(const AssetID& asset_id) const; + + mutable std::mutex m_asset_map_lock; + std::map m_assetid_to_asset_map_path; - mutable std::mutex m_lock; - std::map> m_assetid_to_asset_map_path; + mutable std::mutex m_path_map_lock; + std::map> m_path_to_asset_id; }; } // namespace VideoCommon diff --git a/Source/Core/VideoCommon/Assets/Types.h b/Source/Core/VideoCommon/Assets/Types.h new file mode 100644 index 000000000000..c10d45587cbe --- /dev/null +++ b/Source/Core/VideoCommon/Assets/Types.h @@ -0,0 +1,13 @@ +// Copyright 2024 Dolphin Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +#pragma once + +#include +#include +#include + +namespace VideoCommon::Assets +{ +using AssetMap = std::map; +} diff --git a/Source/Core/VideoCommon/Assets/WatchableFilesystemAssetLibrary.cpp b/Source/Core/VideoCommon/Assets/WatchableFilesystemAssetLibrary.cpp new file mode 100644 index 000000000000..4808463b4a0f --- /dev/null +++ b/Source/Core/VideoCommon/Assets/WatchableFilesystemAssetLibrary.cpp @@ -0,0 +1,57 @@ +// Copyright 2024 Dolphin Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +#include "VideoCommon/Assets/WatchableFilesystemAssetLibrary.h" + +#include "Common/Logging/Log.h" +#include "Common/StringUtil.h" + +namespace VideoCommon +{ +void WatchableFilesystemAssetLibrary::Watch(const std::string& path) +{ + const auto [iter, inserted] = m_watched_paths.try_emplace(path, nullptr); + if (inserted) + { + iter->second = std::make_unique(path, [this](wtr::event e) { + if (e.path_type == wtr::event::path_type::watcher) + { + return; + } + + if (e.effect_type == wtr::event::effect_type::create) + { + const auto path = WithUnifiedPathSeparators(PathToString(e.path_name)); + PathAdded(path); + } + else if (e.effect_type == wtr::event::effect_type::modify) + { + const auto path = WithUnifiedPathSeparators(PathToString(e.path_name)); + PathModified(path); + } + else if (e.effect_type == wtr::event::effect_type::rename) + { + if (!e.associated) + { + WARN_LOG_FMT(VIDEO, "Rename on path seen without association!"); + return; + } + + const auto old_path = WithUnifiedPathSeparators(PathToString(e.path_name)); + const auto new_path = WithUnifiedPathSeparators(PathToString(e.associated->path_name)); + PathRenamed(old_path, new_path); + } + else if (e.effect_type == wtr::event::effect_type::destroy) + { + const auto path = WithUnifiedPathSeparators(PathToString(e.path_name)); + PathDeleted(path); + } + }); + } +} + +void WatchableFilesystemAssetLibrary::Unwatch(const std::string& path) +{ + m_watched_paths.erase(path); +} +} // namespace VideoCommon diff --git a/Source/Core/VideoCommon/Assets/WatchableFilesystemAssetLibrary.h b/Source/Core/VideoCommon/Assets/WatchableFilesystemAssetLibrary.h new file mode 100644 index 000000000000..947127a469fa --- /dev/null +++ b/Source/Core/VideoCommon/Assets/WatchableFilesystemAssetLibrary.h @@ -0,0 +1,38 @@ +// Copyright 2024 Dolphin Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +#pragma once + +#include +#include +#include +#include + +#include + +#include "VideoCommon/Assets/CustomAssetLibrary.h" + +namespace VideoCommon +{ +class WatchableFilesystemAssetLibrary : public CustomAssetLibrary +{ +public: + void Watch(const std::string& path); + void Unwatch(const std::string& path); + +private: + // A new file or folder was added to one of the watched paths + virtual void PathAdded(std::string_view path) {} + + // A file or folder was modified in one of the watched paths + virtual void PathModified(std::string_view path) {} + + // A file or folder was renamed in one of the watched paths + virtual void PathRenamed(std::string_view old_path, std::string_view new_path) {} + + // A file or folder was deleted in one of the watched paths + virtual void PathDeleted(std::string_view path) {} + + std::map> m_watched_paths; +}; +} // namespace VideoCommon diff --git a/Source/Core/VideoCommon/CMakeLists.txt b/Source/Core/VideoCommon/CMakeLists.txt index e24680185e45..f368ce657c45 100644 --- a/Source/Core/VideoCommon/CMakeLists.txt +++ b/Source/Core/VideoCommon/CMakeLists.txt @@ -26,6 +26,9 @@ add_library(videocommon Assets/ShaderAsset.h Assets/TextureAsset.cpp Assets/TextureAsset.h + Assets/Types.h + Assets/WatchableFilesystemAssetLibrary.cpp + Assets/WatchableFilesystemAssetLibrary.h AsyncRequests.cpp AsyncRequests.h AsyncShaderCompiler.cpp @@ -219,6 +222,7 @@ PRIVATE implot glslang tinygltf + watcher ) if(_M_X86_64) diff --git a/Source/Core/VideoCommon/GraphicsModSystem/Config/GraphicsModAsset.h b/Source/Core/VideoCommon/GraphicsModSystem/Config/GraphicsModAsset.h index 53d8d718926f..d623a00e2ec1 100644 --- a/Source/Core/VideoCommon/GraphicsModSystem/Config/GraphicsModAsset.h +++ b/Source/Core/VideoCommon/GraphicsModSystem/Config/GraphicsModAsset.h @@ -7,12 +7,13 @@ #include -#include "VideoCommon/Assets/DirectFilesystemAssetLibrary.h" +#include "VideoCommon/Assets/CustomAssetLibrary.h" +#include "VideoCommon/Assets/Types.h" struct GraphicsModAssetConfig { VideoCommon::CustomAssetLibrary::AssetID m_asset_id; - VideoCommon::DirectFilesystemAssetLibrary::AssetMap m_map; + VideoCommon::Assets::AssetMap m_map; void SerializeToConfig(picojson::object& json_obj) const; bool DeserializeFromConfig(const picojson::object& obj); diff --git a/Source/Core/VideoCommon/GraphicsModSystem/Runtime/CustomPipeline.cpp b/Source/Core/VideoCommon/GraphicsModSystem/Runtime/CustomPipeline.cpp index 473cc066906e..99bac7a30bba 100644 --- a/Source/Core/VideoCommon/GraphicsModSystem/Runtime/CustomPipeline.cpp +++ b/Source/Core/VideoCommon/GraphicsModSystem/Runtime/CustomPipeline.cpp @@ -182,6 +182,10 @@ void CustomPipeline::UpdatePixelData( { m_pixel_material.m_asset = loader.LoadMaterial(material_to_load, library); } + else + { + loader.AssetReferenced(m_pixel_material.m_asset->GetSessionId()); + } const auto material_data = m_pixel_material.m_asset->GetData(); if (!material_data) @@ -217,6 +221,10 @@ void CustomPipeline::UpdatePixelData( m_last_generated_shader_code = ShaderCode{}; } + else + { + loader.AssetReferenced(m_pixel_shader.m_asset->GetSessionId()); + } const auto shader_data = m_pixel_shader.m_asset->GetData(); if (!shader_data) diff --git a/Source/Core/VideoCommon/HiresTextures.cpp b/Source/Core/VideoCommon/HiresTextures.cpp index 2f5fd93709f5..aa8dc53a4b92 100644 --- a/Source/Core/VideoCommon/HiresTextures.cpp +++ b/Source/Core/VideoCommon/HiresTextures.cpp @@ -104,6 +104,9 @@ void HiresTexture::Update() for (const auto& texture_directory : texture_directories) { + // Watch this directory for any texture reloads + s_file_library->Watch(texture_directory); + const auto texture_paths = Common::DoFileSearch({texture_directory}, extensions, /*recursive*/ true); diff --git a/Source/Core/VideoCommon/TextureCacheBase.cpp b/Source/Core/VideoCommon/TextureCacheBase.cpp index 5ef7d3fc985a..c8942e449f13 100644 --- a/Source/Core/VideoCommon/TextureCacheBase.cpp +++ b/Source/Core/VideoCommon/TextureCacheBase.cpp @@ -36,6 +36,7 @@ #include "VideoCommon/AbstractFramebuffer.h" #include "VideoCommon/AbstractGfx.h" #include "VideoCommon/AbstractStagingTexture.h" +#include "VideoCommon/Assets/CustomAssetLoader.h" #include "VideoCommon/Assets/CustomTextureData.h" #include "VideoCommon/BPMemory.h" #include "VideoCommon/FramebufferManager.h" @@ -263,10 +264,13 @@ void TextureCacheBase::SetBackupConfig(const VideoConfig& config) bool TextureCacheBase::DidLinkedAssetsChange(const TCacheEntry& entry) { + auto& system = Core::System::GetInstance(); + auto& loader = system.GetCustomAssetLoader(); for (const auto& cached_asset : entry.linked_game_texture_assets) { if (cached_asset.m_asset) { + loader.AssetReferenced(cached_asset.m_asset->GetSessionId()); if (cached_asset.m_asset->GetLastLoadedTime() > cached_asset.m_cached_write_time) return true; } diff --git a/Source/VSProps/Base.Dolphin.props b/Source/VSProps/Base.Dolphin.props index d3ef409c9eb2..53e5ec264c54 100644 --- a/Source/VSProps/Base.Dolphin.props +++ b/Source/VSProps/Base.Dolphin.props @@ -16,6 +16,7 @@ $(ExternalsDir)rangeset\include;%(AdditionalIncludeDirectories) $(ExternalsDir)Vulkan-Headers\include;%(AdditionalIncludeDirectories) $(ExternalsDir)VulkanMemoryAllocator\include;%(AdditionalIncludeDirectories) + $(ExternalsDir)watcher\watcher\include;%(AdditionalIncludeDirectories) $(ExternalsDir)WIL\include;%(AdditionalIncludeDirectories) WIL_SUPPRESS_EXCEPTIONS;%(PreprocessorDefinitions)