From 580a2636cdd6903795f8c70a000d976b357e46fa Mon Sep 17 00:00:00 2001 From: allen hux Date: Wed, 12 Oct 2022 14:49:49 -0700 Subject: [PATCH] Fix to eliminate submit() calls when there are no requests. Added -captureTrace capability (which helped identify the issue). --- TileUpdateManager/DataUploader.h | 1 + TileUpdateManager/FileStreamer.cpp | 49 ++++++++++++++++++++ TileUpdateManager/FileStreamer.h | 17 ++++++- TileUpdateManager/FileStreamerDS.cpp | 13 ++++-- TileUpdateManager/FileStreamerDS.h | 4 +- TileUpdateManager/FileStreamerReference.h | 4 +- TileUpdateManager/SamplerFeedbackStreaming.h | 2 +- TileUpdateManager/StreamingResourceBase.cpp | 11 ++--- TileUpdateManager/StreamingResourceBase.h | 2 +- TileUpdateManager/TileUpdateManager.cpp | 9 ++-- TileUpdateManager/TileUpdateManagerBase.cpp | 22 +++++---- TileUpdateManager/TileUpdateManagerBase.h | 1 + src/CommandLineArgs.h | 1 + src/FrameEventTracing.h | 2 +- src/Scene.cpp | 23 +++++---- src/winMain.cpp | 3 ++ 16 files changed, 127 insertions(+), 37 deletions(-) diff --git a/TileUpdateManager/DataUploader.h b/TileUpdateManager/DataUploader.h index 2d01e7b..842b4fc 100644 --- a/TileUpdateManager/DataUploader.h +++ b/TileUpdateManager/DataUploader.h @@ -93,6 +93,7 @@ namespace Streaming float GetApproximateTileCopyLatency() const { return m_pFenceThreadTimer->GetSecondsFromDelta(m_totalTileCopyLatency); } // sum of per-tile latencies so far void SetVisualizationMode(UINT in_mode) { m_pFileStreamer->SetVisualizationMode(in_mode); } + void CaptureTraceFile(bool in_captureTrace) { m_pFileStreamer->CaptureTraceFile(in_captureTrace); } private: // upload buffer size const UINT m_stagingBufferSizeMB{ 0 }; diff --git a/TileUpdateManager/FileStreamer.cpp b/TileUpdateManager/FileStreamer.cpp index 096cb89..1f41663 100644 --- a/TileUpdateManager/FileStreamer.cpp +++ b/TileUpdateManager/FileStreamer.cpp @@ -96,6 +96,55 @@ Streaming::FileStreamer::FileStreamer(ID3D12Device* in_pDevice) } } +Streaming::FileStreamer::~FileStreamer() +{ + // write trace file + if (m_captureTrace) + { + int index = 0; + while (1) + { + std::wstring unique = L"uploadTraceFile_" + std::to_wstring(++index) + L".json"; + if (!std::filesystem::exists(unique)) + { + m_trace.Write(unique); + break; + } + } + } +} + +//----------------------------------------------------------------------------- +// append an upload request to the trace file +//----------------------------------------------------------------------------- +void Streaming::FileStreamer::TraceRequest(const D3D12_TILED_RESOURCE_COORDINATE& in_coord, + UINT64 in_offset, UINT64 in_fileHandle, UINT32 in_numBytes) +{ + auto& r = m_trace.GetRoot()["submits"][m_traceSubmitIndex][m_traceRequestIndex++]; + r["coord"][0] = in_coord.X; + r["coord"][1] = in_coord.Y; + r["coord"][2] = in_coord.Subresource; + r["off"] = in_offset; + r["file"] = in_fileHandle; + r["size"] = in_numBytes; +} + +//----------------------------------------------------------------------------- +// capture submit() to the trace file +// increments index to next array of requests +//----------------------------------------------------------------------------- +void Streaming::FileStreamer::TraceSubmit() +{ + if (m_firstSubmit) // ignore first submit while tracing in case enabled mid-frame. + { + m_firstSubmit = false; + return; + } + ASSERT(0 != m_traceRequestIndex); // should never call Submit() without any requests + m_traceRequestIndex = 0; + m_traceSubmitIndex++; +} + //----------------------------------------------------------------------------- //----------------------------------------------------------------------------- void Streaming::FileStreamer::InitializeBC7() diff --git a/TileUpdateManager/FileStreamer.h b/TileUpdateManager/FileStreamer.h index c5e1796..5aec983 100644 --- a/TileUpdateManager/FileStreamer.h +++ b/TileUpdateManager/FileStreamer.h @@ -27,6 +27,7 @@ #pragma once #include "Streaming.h" +#include "ConfigurationParser.h" namespace Streaming { @@ -43,7 +44,7 @@ namespace Streaming { public: FileStreamer(ID3D12Device* in_pDevice); - virtual ~FileStreamer() {} + virtual ~FileStreamer(); virtual FileHandle* OpenFile(const std::wstring& in_path) = 0; @@ -60,6 +61,8 @@ namespace Streaming void SetVisualizationMode(UINT in_mode) { m_visualizationMode = (VisualizationMode)in_mode; } bool GetCompleted(const UpdateList& in_updateList) const; + + void CaptureTraceFile(bool in_captureTrace) { m_captureTrace = in_captureTrace; } // enable/disable writing requests/submits to a trace file protected: // copy queue fence ComPtr m_copyFence; @@ -79,5 +82,17 @@ namespace Streaming static BYTE m_BC1[m_lutSize][D3D12_TILED_RESOURCE_TILE_SIZE_IN_BYTES]; void InitializeBC1(); + + // trace file + bool m_captureTrace{ false }; + + void TraceRequest(const D3D12_TILED_RESOURCE_COORDINATE& in_coord, + UINT64 in_offset, UINT64 in_fileHandle, UINT32 in_numBytes); + void TraceSubmit(); + private: + bool m_firstSubmit{ true }; + ConfigurationParser m_trace; // array of submits, each submit is an array of requests + UINT m_traceSubmitIndex{ 0 }; + UINT m_traceRequestIndex{ 0 }; }; } diff --git a/TileUpdateManager/FileStreamerDS.cpp b/TileUpdateManager/FileStreamerDS.cpp index 225d886..c91f641 100644 --- a/TileUpdateManager/FileStreamerDS.cpp +++ b/TileUpdateManager/FileStreamerDS.cpp @@ -62,7 +62,7 @@ void Streaming::FileStreamerDS::StreamTexture(Streaming::UpdateList& in_updateLi DXGI_FORMAT textureFormat = pTextureFileInfo->GetFormat(); auto pDstHeap = in_updateList.m_pStreamingResource->GetHeap(); - DSTORAGE_REQUEST request = {}; + DSTORAGE_REQUEST request{}; request.Options.DestinationType = DSTORAGE_REQUEST_DESTINATION_TILES; request.Destination.Tiles.TileRegionSize = D3D12_TILE_REGION_SIZE{ 1, FALSE, 0, 0, 0 }; @@ -77,7 +77,7 @@ void Streaming::FileStreamerDS::StreamTexture(Streaming::UpdateList& in_updateLi { request.Source.File.Offset = pTextureFileInfo->GetFileOffset(in_updateList.m_coords[i], request.Source.File.Size); - D3D12_TILED_RESOURCE_COORDINATE coord; + D3D12_TILED_RESOURCE_COORDINATE coord{}; ID3D12Resource* pAtlas = pDstHeap->ComputeCoordFromTileIndex(coord, in_updateList.m_heapIndices[i], textureFormat); request.Destination.Tiles.Resource = pAtlas; @@ -85,6 +85,11 @@ void Streaming::FileStreamerDS::StreamTexture(Streaming::UpdateList& in_updateLi request.Options.CompressionFormat = (DSTORAGE_COMPRESSION_FORMAT)pTextureFileInfo->GetCompressionFormat(); m_fileQueue->EnqueueRequest(&request); + + if (m_captureTrace) + { + TraceRequest(coord, request.Source.File.Offset, (UINT64)request.Source.File.Source, (UINT32)request.Source.File.Size); + } } } else // visualization color is loaded from memory @@ -98,7 +103,7 @@ void Streaming::FileStreamerDS::StreamTexture(Streaming::UpdateList& in_updateLi { request.Source.Memory.Source = GetVisualizationData(in_updateList.m_coords[i], textureFormat); - D3D12_TILED_RESOURCE_COORDINATE coord; + D3D12_TILED_RESOURCE_COORDINATE coord{}; ID3D12Resource* pAtlas = pDstHeap->ComputeCoordFromTileIndex(coord, in_updateList.m_heapIndices[i], textureFormat); request.Destination.Tiles.Resource = pAtlas; @@ -122,6 +127,8 @@ void Streaming::FileStreamerDS::Signal() { m_fileQueue->EnqueueSignal(m_copyFence.Get(), m_copyFenceValue); m_fileQueue->Submit(); + + if (m_captureTrace) { TraceSubmit(); } } else { diff --git a/TileUpdateManager/FileStreamerDS.h b/TileUpdateManager/FileStreamerDS.h index e581116..69c81bd 100644 --- a/TileUpdateManager/FileStreamerDS.h +++ b/TileUpdateManager/FileStreamerDS.h @@ -16,8 +16,6 @@ namespace Streaming virtual FileHandle* OpenFile(const std::wstring& in_path) override; virtual void StreamTexture(Streaming::UpdateList& in_updateList) override; - static IDStorageFile* GetFileHandle(const FileHandle* in_pHandle); - // for DS, we don't have a way to batch batches // this allows the calling thread to periodically request Submit() vs. every enqueue virtual void Signal() override; @@ -39,5 +37,7 @@ namespace Streaming // memory queue when for visualization modes, which copy from cpu memory ComPtr m_memoryQueue; + + static IDStorageFile* GetFileHandle(const FileHandle* in_pHandle); }; }; diff --git a/TileUpdateManager/FileStreamerReference.h b/TileUpdateManager/FileStreamerReference.h index 53c4880..2a0646e 100644 --- a/TileUpdateManager/FileStreamerReference.h +++ b/TileUpdateManager/FileStreamerReference.h @@ -48,8 +48,6 @@ namespace Streaming virtual void Signal() override {} // reference auto-submits - static HANDLE GetFileHandle(const FileHandle* in_pHandle) { return dynamic_cast(in_pHandle)->GetHandle(); } - static const UINT MEDIA_SECTOR_SIZE = 4096; // see https://docs.microsoft.com/en-us/windows/win32/fileio/file-buffering private: class FileHandleReference : public FileHandle @@ -128,5 +126,7 @@ namespace Streaming void LoadTexture(CopyBatch& in_copyBatch, UINT in_numtilesToLoad); void CopyTiles(ID3D12GraphicsCommandList* out_pCopyCmdList, ID3D12Resource* in_pSrcResource, const UpdateList* in_pUpdateList, const std::vector& in_indices); + + static HANDLE GetFileHandle(const FileHandle* in_pHandle) { return dynamic_cast(in_pHandle)->GetHandle(); } }; } diff --git a/TileUpdateManager/SamplerFeedbackStreaming.h b/TileUpdateManager/SamplerFeedbackStreaming.h index 183bebe..0ba6c27 100644 --- a/TileUpdateManager/SamplerFeedbackStreaming.h +++ b/TileUpdateManager/SamplerFeedbackStreaming.h @@ -222,7 +222,7 @@ struct TileUpdateManager // for visualization //-------------------------------------------- virtual void SetVisualizationMode(UINT in_mode) = 0; - + virtual void CaptureTraceFile(bool in_captureTrace) = 0; // capture a trace file of tile uploads virtual float GetGpuStreamingTime() const = 0; virtual float GetCpuProcessFeedbackTime() = 0; // approx. cpu time spent processing feedback last frame. expected usage is to average over many frames virtual UINT GetTotalNumUploads() const = 0; // number of tiles uploaded so far diff --git a/TileUpdateManager/StreamingResourceBase.cpp b/TileUpdateManager/StreamingResourceBase.cpp index 68e42c4..07f2d9d 100644 --- a/TileUpdateManager/StreamingResourceBase.cpp +++ b/TileUpdateManager/StreamingResourceBase.cpp @@ -495,7 +495,7 @@ void Streaming::StreamingResourceBase::AbandonPendingLoads() //----------------------------------------------------------------------------- UINT Streaming::StreamingResourceBase::QueueTiles() { - UINT uploadRequested = 0; + UINT uploadsRequested = 0; UINT numEvictions = (UINT)m_pendingEvictions.GetReadyToEvict().size(); UINT numLoads = (UINT)m_pendingTileLoads.size(); @@ -513,7 +513,8 @@ UINT Streaming::StreamingResourceBase::QueueTiles() // queue as many new tiles as possible if (numLoads && m_pHeap->GetAllocator().GetAvailable()) { - uploadRequested = QueuePendingTileLoads(&scratchUL); + QueuePendingTileLoads(&scratchUL); + uploadsRequested = (UINT)scratchUL.m_coords.size(); // number of uploads in UpdateList numLoads = (UINT)m_pendingTileLoads.size(); } @@ -531,7 +532,7 @@ UINT Streaming::StreamingResourceBase::QueueTiles() m_pTileUpdateManager->SubmitUpdateList(*pUpdateList); } } - return uploadRequested; + return uploadsRequested; } /*----------------------------------------------------------------------------- @@ -623,7 +624,7 @@ void Streaming::StreamingResourceBase::QueuePendingTileEvictions(Streaming::Upda // FIFO order: work from the front of the array // NOTE: greedy, takes every available UpdateList if it can //----------------------------------------------------------------------------- -UINT Streaming::StreamingResourceBase::QueuePendingTileLoads(Streaming::UpdateList* out_pUpdateList) +void Streaming::StreamingResourceBase::QueuePendingTileLoads(Streaming::UpdateList* out_pUpdateList) { ASSERT(out_pUpdateList); ASSERT(m_pHeap->GetAllocator().GetAvailable()); @@ -682,8 +683,6 @@ UINT Streaming::StreamingResourceBase::QueuePendingTileLoads(Streaming::UpdateLi { m_pendingTileLoads.erase(m_pendingTileLoads.begin() + skippedIndex, m_pendingTileLoads.begin() + numConsumed); } - - return numConsumed; } //----------------------------------------------------------------------------- diff --git a/TileUpdateManager/StreamingResourceBase.h b/TileUpdateManager/StreamingResourceBase.h index 5829e01..69abd44 100644 --- a/TileUpdateManager/StreamingResourceBase.h +++ b/TileUpdateManager/StreamingResourceBase.h @@ -326,7 +326,7 @@ namespace Streaming // only push evictions to DataUploader once per rendered frame (i.e. "on the next frame") void QueuePendingTileEvictions(Streaming::UpdateList* out_pUpdateList); - UINT QueuePendingTileLoads(Streaming::UpdateList* out_pUpdateList); // returns # tiles queued + void QueuePendingTileLoads(Streaming::UpdateList* out_pUpdateList); // returns # tiles queued void LoadPackedMips(); diff --git a/TileUpdateManager/TileUpdateManager.cpp b/TileUpdateManager/TileUpdateManager.cpp index 0df61bf..74c326f 100644 --- a/TileUpdateManager/TileUpdateManager.cpp +++ b/TileUpdateManager/TileUpdateManager.cpp @@ -48,9 +48,7 @@ TileUpdateManager* TileUpdateManager::Create( const TileUpdateManagerDesc& in_desc) { - auto p = new Streaming::TileUpdateManagerBase(in_pDevice, in_pDirectCommandQueue, in_desc); - p->UseDirectStorage(in_desc.m_useDirectStorage); - return p; + return new Streaming::TileUpdateManagerBase(in_pDevice, in_pDirectCommandQueue, in_desc); } void Streaming::TileUpdateManagerBase::Destroy() @@ -170,6 +168,11 @@ void Streaming::TileUpdateManagerBase::SetVisualizationMode(UINT in_mode) m_pDataUploader->SetVisualizationMode(in_mode); } +void Streaming::TileUpdateManagerBase::CaptureTraceFile(bool in_captureTrace) +{ + m_pDataUploader->CaptureTraceFile(in_captureTrace); +} + //----------------------------------------------------------------------------- // Call this method once for each TileUpdateManager that shares heap/upload buffers // expected to be called once per frame, before anything is drawn. diff --git a/TileUpdateManager/TileUpdateManagerBase.cpp b/TileUpdateManager/TileUpdateManagerBase.cpp index eaee496..786ec5f 100644 --- a/TileUpdateManager/TileUpdateManagerBase.cpp +++ b/TileUpdateManager/TileUpdateManagerBase.cpp @@ -92,6 +92,8 @@ m_numSwapBuffers(in_desc.m_swapChainBufferCount) // advance frame number to the first frame... m_frameFenceValue++; + + UseDirectStorage(in_desc.m_useDirectStorage); } Streaming::TileUpdateManagerBase::~TileUpdateManagerBase() @@ -247,15 +249,19 @@ void Streaming::TileUpdateManagerBase::ProcessFeedbackThread() staleResources.resize(newStaleSize); // compact array } - // tell the file streamer to signal the corresponding fence - if ((flushPendingUploadRequests) || // flush requests from previous frame - (0 == staleResources.size()) || // flush because there's no more work to be done (no stale resources, all feedback has been processed) - // if we need updatelists and there is a minimum amount of pending work, go ahead and submit - // this minimum heuristic prevents "storms" of submits with too few tiles to sustain good throughput - ((0 == m_pDataUploader->GetNumUpdateListsAvailable()) && (uploadsRequested > m_minNumUploadRequests))) + // if there are uploads, maybe signal depending on heuristic to minimize # signals + if (uploadsRequested) { - SignalFileStreamer(); - uploadsRequested = 0; + // tell the file streamer to signal the corresponding fence + if ((flushPendingUploadRequests) || // flush requests from previous frame + (0 == staleResources.size()) || // flush because there's no more work to be done (no stale resources, all feedback has been processed) + // if we need updatelists and there is a minimum amount of pending work, go ahead and submit + // this minimum heuristic prevents "storms" of submits with too few tiles to sustain good throughput + ((0 == m_pDataUploader->GetNumUpdateListsAvailable()) && (uploadsRequested > m_minNumUploadRequests))) + { + SignalFileStreamer(); + uploadsRequested = 0; + } } // nothing to do? wait for next frame diff --git a/TileUpdateManager/TileUpdateManagerBase.h b/TileUpdateManager/TileUpdateManagerBase.h index 8477e2b..a74ff4d 100644 --- a/TileUpdateManager/TileUpdateManagerBase.h +++ b/TileUpdateManager/TileUpdateManagerBase.h @@ -84,6 +84,7 @@ namespace Streaming virtual bool GetWithinFrame() const override { return m_withinFrame; } virtual float GetGpuTime() const override; virtual void SetVisualizationMode(UINT in_mode) override; + virtual void CaptureTraceFile(bool in_captureTrace) override; virtual float GetGpuStreamingTime() const override; virtual float GetCpuProcessFeedbackTime() override; virtual UINT GetTotalNumUploads() const override; diff --git a/src/CommandLineArgs.h b/src/CommandLineArgs.h index 88d24ca..c856f82 100644 --- a/src/CommandLineArgs.h +++ b/src/CommandLineArgs.h @@ -113,5 +113,6 @@ struct CommandLineArgs TerrainGenerator::Params m_terrainParams; + bool m_captureTrace{ false }; // capture a trace file of tile uploads int m_threadPriority{ 0 }; // applies to internal threads }; diff --git a/src/FrameEventTracing.h b/src/FrameEventTracing.h index 5f02664..6bf82d3 100644 --- a/src/FrameEventTracing.h +++ b/src/FrameEventTracing.h @@ -133,7 +133,7 @@ inline void FrameEventTracing::WriteEvents(HWND in_hWnd, const CommandLineArgs& *this << "\nTimers (ms)\n" << "-----------------------------------------------------------------------------------------------------------\n" - << "cpu_draw TUM::EndFrame exec_cmd_list wait_present total_frame_time evictions copies cpu_feedback feedback_resolve num_resolves num_submits\n" + << "cpu_draw TUM::EndFrame exec_cmd_list wait_present total_frame_time evictions_completed copies_completed cpu_feedback feedback_resolve num_resolves num_submits\n" << "-----------------------------------------------------------------------------------------------------------\n"; for (auto& e : m_events) diff --git a/src/Scene.cpp b/src/Scene.cpp index 3faa0b1..fc277b4 100644 --- a/src/Scene.cpp +++ b/src/Scene.cpp @@ -1272,17 +1272,22 @@ void Scene::GatherStatistics() PostQuitMessage(0); } - m_frameNumber++; - - // start timing and gathering uploads from the very beginning of the timed region - if (m_args.m_timingFrameFileName.size() && (m_frameNumber == m_args.m_timingStartFrame)) + if (m_frameNumber == m_args.m_timingStartFrame) { - numSubmits = m_pTileUpdateManager->GetTotalNumSubmits(); - m_startUploadCount = m_pTileUpdateManager->GetTotalNumUploads(); - m_startSubmitCount = m_pTileUpdateManager->GetTotalNumSubmits(); - m_totalTileLatency = m_pTileUpdateManager->GetTotalTileCopyLatency(); - m_cpuTimer.Start(); + m_pTileUpdateManager->CaptureTraceFile(m_args.m_captureTrace); + + // start timing and gathering uploads from the very beginning of the timed region + if (m_args.m_timingFrameFileName.size()) + { + numSubmits = m_pTileUpdateManager->GetTotalNumSubmits(); + m_startUploadCount = m_pTileUpdateManager->GetTotalNumUploads(); + m_startSubmitCount = m_pTileUpdateManager->GetTotalNumSubmits(); + m_totalTileLatency = m_pTileUpdateManager->GetTotalTileCopyLatency(); + m_cpuTimer.Start(); + } } + + m_frameNumber++; } //------------------------------------------------------------------------- diff --git a/src/winMain.cpp b/src/winMain.cpp index 7a31001..b75e8c0 100644 --- a/src/winMain.cpp +++ b/src/winMain.cpp @@ -219,6 +219,9 @@ void ParseCommandLine(CommandLineArgs& out_args) argParser.AddArg(L"-directStorage", [&]() { out_args.m_useDirectStorage = true; }, L"force enable DirectStorage"); argParser.AddArg(L"-directStorageOff", [&]() { out_args.m_useDirectStorage = false; }, L"force disable DirectStorage"); argParser.AddArg(L"-stagingSizeMB", out_args.m_stagingSizeMB, L"DirectStorage staging buffer size"); + + argParser.AddArg(L"-captureTrace", [&]() { out_args.m_captureTrace = true; }, false, L"capture a trace of tile requests and submits (DS only)"); + argParser.Parse(); }