From 5f75c371f1a7626cefe04373ba080a223fe94af9 Mon Sep 17 00:00:00 2001 From: allen hux Date: Wed, 31 Aug 2022 15:47:29 -0700 Subject: [PATCH] Update tiles oldest-first, improves visual quality. convert.bat includes .xet files e.g. for updating the older hubble files. --- TileUpdateManager/TileUpdateManagerBase.cpp | 205 +++++++++++--------- TileUpdateManager/TileUpdateManagerBase.h | 1 + scripts/convert.bat | 10 +- 3 files changed, 118 insertions(+), 98 deletions(-) diff --git a/TileUpdateManager/TileUpdateManagerBase.cpp b/TileUpdateManager/TileUpdateManagerBase.cpp index a77b66d..4685b4b 100644 --- a/TileUpdateManager/TileUpdateManagerBase.cpp +++ b/TileUpdateManager/TileUpdateManagerBase.cpp @@ -99,7 +99,7 @@ Streaming::TileUpdateManagerBase::~TileUpdateManagerBase() //----------------------------------------------------------------------------- -// kick off thread that continuously streams tiles +// kick off threads that continuously streams tiles // gives StreamingResources opportunities to update feedback //----------------------------------------------------------------------------- void Streaming::TileUpdateManagerBase::StartThreads() @@ -115,127 +115,138 @@ void Streaming::TileUpdateManagerBase::StartThreads() { DebugPrint(L"Created Feedback Thread\n"); - // NOTE: expects the streaming resource array size to be unchanged during thread lifetime + ProcessFeedbackThread(); - // array of indices to resources that need tiles loaded/evicted - std::vector staleResources; - staleResources.reserve(m_streamingResources.size()); + DebugPrint(L"Destroyed ProcessFeedback Thread\n"); + }); - // flags to prevent duplicates in the staleResources array - std::vector pending(m_streamingResources.size(), 0); + m_updateResidencyThread = std::thread([&] + { + DebugPrint(L"Created UpdateResidency Thread\n"); - UINT64 previousFrameFenceValue = m_frameFenceValue; + // continuously modify residency maps as a result of gpu completion events + // FIXME? probably not enough work to deserve it's own thread + // Note that UpdateMinMipMap() exits quickly if nothing to do while (m_threadsRunning) { - // prioritize loading packed mips, as objects shouldn't be displayed until packed mips load - bool expected = true; - if (m_havePackedMipsToLoad.compare_exchange_weak(expected, false)) + m_residencyChangedFlag.Wait(); + + for (auto p : m_streamingResources) { - for (auto p : m_streamingResources) - { - if (!p->InitPackedMips()) - { - m_havePackedMipsToLoad = true; - } - } - if (m_havePackedMipsToLoad) - { - continue; // still working on loading packed mips. don't move on to other streaming tasks yet. - } + p->UpdateMinMipMap(); } + } + DebugPrint(L"Destroyed UpdateResidency Thread\n"); + }); +} - // DEBUG: verify that no streaming resources have been added/removed during thread lifetime - ASSERT(m_streamingResources.size() == pending.size()); +//----------------------------------------------------------------------------- +// per frame, call StreamingResource::ProcessFeedback() +// expects the no change in # of streaming resources during thread lifetime +//----------------------------------------------------------------------------- +void Streaming::TileUpdateManagerBase::ProcessFeedbackThread() +{ + // array of indices to resources that need tiles loaded/evicted + std::vector staleResources; + staleResources.reserve(m_streamingResources.size()); - UINT64 frameFenceValue = m_frameFence->GetCompletedValue(); + // flags to prevent duplicates in the staleResources array + std::vector pending(m_streamingResources.size(), 0); - // Only process feedback buffers once per frame - if (previousFrameFenceValue != frameFenceValue) + UINT64 previousFrameFenceValue = m_frameFenceValue; + while (m_threadsRunning) + { + // prioritize loading packed mips, as objects shouldn't be displayed until packed mips load + bool expected = true; + if (m_havePackedMipsToLoad.compare_exchange_weak(expected, false)) + { + for (auto p : m_streamingResources) + { + if (!p->InitPackedMips()) { - previousFrameFenceValue = frameFenceValue; - - auto startTime = m_cpuTimer.GetTime(); - UINT j = 0; - for (auto p : m_streamingResources) - { - // early exit, important for application exit or TUM::Finish() when adding/deleting objects - if (!m_threadsRunning) - { - break; - } - - p->ProcessFeedback(frameFenceValue); - if (p->IsStale() && !pending[j]) - { - staleResources.push_back(j); - pending[j] = 1; - } - j++; - } - m_processFeedbackTime += UINT64(m_cpuTimer.GetTime() - startTime); + m_havePackedMipsToLoad = true; } + } + if (m_havePackedMipsToLoad) + { + Sleep(2); + continue; // still working on loading packed mips. don't move on to other streaming tasks yet. + } + } - // continuously push uploads and evictions - bool uploadRequested = false; - for (UINT i = 0; i < staleResources.size(); ) - { - // exit loop if we ran out of UpdateLists or application exiting - if ((!m_pDataUploader->UpdateListAvailable()) || (!m_threadsRunning)) - { - break; - } - - UINT resourceIndex = staleResources[i]; - auto p = m_streamingResources[resourceIndex]; - bool tilesQueued = p->QueueTiles(); - uploadRequested = uploadRequested || tilesQueued; - - // if all loads/evictions handled, remove from staleResource list - if (!p->IsStale()) - { - pending[resourceIndex] = 0; // clear the flag that prevents duplicates - // compact the array by swapping this entry with the last - staleResources[i] = staleResources.back(); - staleResources.resize(staleResources.size() - 1); - } - else - { - i++; - } - } + // DEBUG: verify that no streaming resources have been added/removed during thread lifetime + ASSERT(m_streamingResources.size() == pending.size()); - // if uploads were queued, tell the file streamer to signal the corresponding fence - if (uploadRequested) + UINT64 frameFenceValue = m_frameFence->GetCompletedValue(); + + // Only process feedback buffers once per frame + if (previousFrameFenceValue != frameFenceValue) + { + previousFrameFenceValue = frameFenceValue; + + auto startTime = m_cpuTimer.GetTime(); + UINT j = 0; + for (auto p : m_streamingResources) + { + // early exit, important for application exit or TUM::Finish() when adding/deleting objects + if (!m_threadsRunning) { - m_pDataUploader->SignalFileStreamer(); + break; } - // nothing to do? wait for next frame - if ((0 == staleResources.size()) && m_threadsRunning) + p->ProcessFeedback(frameFenceValue); + if (p->IsStale() && !pending[j]) { - m_processFeedbackFlag.Wait(); + staleResources.push_back(j); + pending[j] = 1; } + j++; } - DebugPrint(L"Destroyed ProcessFeedback Thread\n"); - }); + // add the amount of time we just spent processing feedback for a single frame + m_processFeedbackTime += UINT64(m_cpuTimer.GetTime() - startTime); + } - m_updateResidencyThread = std::thread([&] + // push uploads and evictions for stale resources + bool uploadRequested = false; // remember if any work was queued so we can signal afterwards + UINT newStaleSize = 0; // track number of stale resources, then resize the array to the updated number + for (UINT i = 0; i < staleResources.size(); i++) { - DebugPrint(L"Created UpdateResidency Thread\n"); - // continuously modify residency maps as a result of gpu completion events - // FIXME? probably not enough work to deserve it's own thread - // Note that UpdateMinMipMap() exits quickly if nothing to do - while (m_threadsRunning) + + UINT resourceIndex = staleResources[i]; + auto p = m_streamingResources[resourceIndex]; + + if (m_pDataUploader->UpdateListAvailable() && m_threadsRunning) { - m_residencyChangedFlag.Wait(); + uploadRequested = p->QueueTiles() || uploadRequested; + } - for (auto p : m_streamingResources) - { - p->UpdateMinMipMap(); - } + // if all loads/evictions handled, remove from staleResource list + if (p->IsStale()) + { + // compact, removing non-stale resource indices while retaining oldest-first ordering + staleResources[newStaleSize] = resourceIndex; + newStaleSize++; } - DebugPrint(L"Destroyed UpdateResidency Thread\n"); - }); + else + { + pending[resourceIndex] = 0; // clear the flag that prevents duplicates + } + } + + staleResources.resize(newStaleSize); + + // if uploads were queued, tell the file streamer to signal the corresponding fence + if (uploadRequested) + { + m_pDataUploader->SignalFileStreamer(); + } + + // nothing to do? wait for next frame + if ((0 == staleResources.size()) && m_threadsRunning) + { + m_processFeedbackFlag.Wait(); + } + } } //----------------------------------------------------------------------------- diff --git a/TileUpdateManager/TileUpdateManagerBase.h b/TileUpdateManager/TileUpdateManagerBase.h index e55803d..d0fe865 100644 --- a/TileUpdateManager/TileUpdateManagerBase.h +++ b/TileUpdateManager/TileUpdateManagerBase.h @@ -134,6 +134,7 @@ namespace Streaming std::atomic m_havePackedMipsToLoad{ false }; void StartThreads(); + void ProcessFeedbackThread(); //--------------------------------------------------------------------------- // TUM creates 2 command lists to be executed Before & After application draw diff --git a/scripts/convert.bat b/scripts/convert.bat index c4ab89d..647071f 100644 --- a/scripts/convert.bat +++ b/scripts/convert.bat @@ -9,8 +9,16 @@ set outdir=%cd% popd pushd %1 +SHIFT +SHIFT + for /R %%f in (*.dds) do ( echo %%~nf.dds - %exedir%\DdsToXet.exe -in %%~nf.dds -out %outdir%\%%~nf.xet %3 + %exedir%\DdsToXet.exe -in %%~nf.dds -out %outdir%\%%~nf.xet %* +) + +for /R %%f in (*.xet) do ( + echo %%~nf.xet + %exedir%\DdsToXet.exe -in %%~nf.xet -out %outdir%\%%~nf.xet %* ) popd