Fix to eliminate submit() calls when there are no requests. Added -ca…

…ptureTrace capability (which helped identify the issue).
GameTechDev · Oct 12, 2022 · 580a263 · 580a263
1 parent 74275a2
commit 580a263
Show file tree

Hide file tree

Showing 16 changed files with 127 additions and 37 deletions.
diff --git a/TileUpdateManager/DataUploader.h b/TileUpdateManager/DataUploader.h
@@ -93,6 +93,7 @@ namespace Streaming
         float GetApproximateTileCopyLatency() const { return m_pFenceThreadTimer->GetSecondsFromDelta(m_totalTileCopyLatency); } // sum of per-tile latencies so far
 
         void SetVisualizationMode(UINT in_mode) { m_pFileStreamer->SetVisualizationMode(in_mode); }
+        void CaptureTraceFile(bool in_captureTrace) { m_pFileStreamer->CaptureTraceFile(in_captureTrace); }
     private:
         // upload buffer size
         const UINT m_stagingBufferSizeMB{ 0 };

diff --git a/TileUpdateManager/FileStreamer.cpp b/TileUpdateManager/FileStreamer.cpp
@@ -96,6 +96,55 @@ Streaming::FileStreamer::FileStreamer(ID3D12Device* in_pDevice)
     }
 }
 
+Streaming::FileStreamer::~FileStreamer()
+{
+    // write trace file
+    if (m_captureTrace)
+    {
+        int index = 0;
+        while (1)
+        {
+            std::wstring unique = L"uploadTraceFile_" + std::to_wstring(++index) + L".json";
+            if (!std::filesystem::exists(unique))
+            {
+                m_trace.Write(unique);
+                break;
+            }
+        }
+    }
+}
+
+//-----------------------------------------------------------------------------
+// append an upload request to the trace file
+//-----------------------------------------------------------------------------
+void Streaming::FileStreamer::TraceRequest(const D3D12_TILED_RESOURCE_COORDINATE& in_coord,
+    UINT64 in_offset, UINT64 in_fileHandle, UINT32 in_numBytes)
+{
+    auto& r = m_trace.GetRoot()["submits"][m_traceSubmitIndex][m_traceRequestIndex++];
+    r["coord"][0] = in_coord.X;
+    r["coord"][1] = in_coord.Y;
+    r["coord"][2] = in_coord.Subresource;
+    r["off"] = in_offset;
+    r["file"] = in_fileHandle;
+    r["size"] = in_numBytes;
+}
+
+//-----------------------------------------------------------------------------
+// capture submit() to the trace file
+// increments index to next array of requests
+//-----------------------------------------------------------------------------
+void Streaming::FileStreamer::TraceSubmit()
+{
+    if (m_firstSubmit) // ignore first submit while tracing in case enabled mid-frame.
+    {
+        m_firstSubmit = false;
+        return;
+    }
+    ASSERT(0 != m_traceRequestIndex); // should never call Submit() without any requests
+    m_traceRequestIndex = 0;
+    m_traceSubmitIndex++;
+}
+
 //-----------------------------------------------------------------------------
 //-----------------------------------------------------------------------------
 void Streaming::FileStreamer::InitializeBC7()

diff --git a/TileUpdateManager/FileStreamer.h b/TileUpdateManager/FileStreamer.h
@@ -27,6 +27,7 @@
 #pragma once
 
 #include "Streaming.h"
+#include "ConfigurationParser.h"
 
 namespace Streaming
 {
@@ -43,7 +44,7 @@ namespace Streaming
     {
     public:
         FileStreamer(ID3D12Device* in_pDevice);
-        virtual ~FileStreamer() {}
+        virtual ~FileStreamer();
 
         virtual FileHandle* OpenFile(const std::wstring& in_path) = 0;
 
@@ -60,6 +61,8 @@ namespace Streaming
         void SetVisualizationMode(UINT in_mode) { m_visualizationMode = (VisualizationMode)in_mode; }
 
         bool GetCompleted(const UpdateList& in_updateList) const;
+
+        void CaptureTraceFile(bool in_captureTrace) { m_captureTrace = in_captureTrace; } // enable/disable writing requests/submits to a trace file
     protected:
         // copy queue fence
         ComPtr<ID3D12Fence> m_copyFence;
@@ -79,5 +82,17 @@ namespace Streaming
 
         static BYTE m_BC1[m_lutSize][D3D12_TILED_RESOURCE_TILE_SIZE_IN_BYTES];
         void InitializeBC1();
+
+        // trace file
+        bool m_captureTrace{ false };
+
+        void TraceRequest(const D3D12_TILED_RESOURCE_COORDINATE& in_coord,
+            UINT64 in_offset, UINT64 in_fileHandle, UINT32 in_numBytes);
+        void TraceSubmit();
+    private:
+        bool m_firstSubmit{ true };
+        ConfigurationParser m_trace; // array of submits, each submit is an array of requests
+        UINT m_traceSubmitIndex{ 0 };
+        UINT m_traceRequestIndex{ 0 };
     };
 }
diff --git a/TileUpdateManager/FileStreamerDS.cpp b/TileUpdateManager/FileStreamerDS.cpp
@@ -62,7 +62,7 @@ void Streaming::FileStreamerDS::StreamTexture(Streaming::UpdateList& in_updateLi
     DXGI_FORMAT textureFormat = pTextureFileInfo->GetFormat();
     auto pDstHeap = in_updateList.m_pStreamingResource->GetHeap();
 
-    DSTORAGE_REQUEST request = {};
+    DSTORAGE_REQUEST request{};
     request.Options.DestinationType = DSTORAGE_REQUEST_DESTINATION_TILES;
     request.Destination.Tiles.TileRegionSize = D3D12_TILE_REGION_SIZE{ 1, FALSE, 0, 0, 0 };
 
@@ -77,14 +77,19 @@ void Streaming::FileStreamerDS::StreamTexture(Streaming::UpdateList& in_updateLi
         {
             request.Source.File.Offset = pTextureFileInfo->GetFileOffset(in_updateList.m_coords[i], request.Source.File.Size);
 
-            D3D12_TILED_RESOURCE_COORDINATE coord;
+            D3D12_TILED_RESOURCE_COORDINATE coord{};
             ID3D12Resource* pAtlas = pDstHeap->ComputeCoordFromTileIndex(coord, in_updateList.m_heapIndices[i], textureFormat);
 
             request.Destination.Tiles.Resource = pAtlas;
             request.Destination.Tiles.TiledRegionStartCoordinate = coord;
             request.Options.CompressionFormat = (DSTORAGE_COMPRESSION_FORMAT)pTextureFileInfo->GetCompressionFormat();
 
             m_fileQueue->EnqueueRequest(&request);
+
+            if (m_captureTrace)
+            {
+                TraceRequest(coord, request.Source.File.Offset, (UINT64)request.Source.File.Source, (UINT32)request.Source.File.Size);
+            }
         }
     }
     else // visualization color is loaded from memory
@@ -98,7 +103,7 @@ void Streaming::FileStreamerDS::StreamTexture(Streaming::UpdateList& in_updateLi
         {
             request.Source.Memory.Source = GetVisualizationData(in_updateList.m_coords[i], textureFormat);
 
-            D3D12_TILED_RESOURCE_COORDINATE coord;
+            D3D12_TILED_RESOURCE_COORDINATE coord{};
             ID3D12Resource* pAtlas = pDstHeap->ComputeCoordFromTileIndex(coord, in_updateList.m_heapIndices[i], textureFormat);
 
             request.Destination.Tiles.Resource = pAtlas;
@@ -122,6 +127,8 @@ void Streaming::FileStreamerDS::Signal()
     {
         m_fileQueue->EnqueueSignal(m_copyFence.Get(), m_copyFenceValue);
         m_fileQueue->Submit();
+
+        if (m_captureTrace) { TraceSubmit(); }
     }
     else
     {

diff --git a/TileUpdateManager/FileStreamerDS.h b/TileUpdateManager/FileStreamerDS.h
@@ -16,8 +16,6 @@ namespace Streaming
         virtual FileHandle* OpenFile(const std::wstring& in_path) override;
         virtual void StreamTexture(Streaming::UpdateList& in_updateList) override;
 
-        static IDStorageFile* GetFileHandle(const FileHandle* in_pHandle);
-
         // for DS, we don't have a way to batch batches
         // this allows the calling thread to periodically request Submit() vs. every enqueue
         virtual void Signal() override;
@@ -39,5 +37,7 @@ namespace Streaming
 
         // memory queue when for visualization modes, which copy from cpu memory
         ComPtr<IDStorageQueue> m_memoryQueue;
+
+        static IDStorageFile* GetFileHandle(const FileHandle* in_pHandle);
     };
 };
diff --git a/TileUpdateManager/FileStreamerReference.h b/TileUpdateManager/FileStreamerReference.h
@@ -48,8 +48,6 @@ namespace Streaming
 
         virtual void Signal() override {} // reference auto-submits
 
-        static HANDLE GetFileHandle(const FileHandle* in_pHandle) { return dynamic_cast<const FileHandleReference*>(in_pHandle)->GetHandle(); }
-
         static const UINT MEDIA_SECTOR_SIZE = 4096; // see https://docs.microsoft.com/en-us/windows/win32/fileio/file-buffering
     private:
         class FileHandleReference : public FileHandle
@@ -128,5 +126,7 @@ namespace Streaming
         void LoadTexture(CopyBatch& in_copyBatch, UINT in_numtilesToLoad);
         void CopyTiles(ID3D12GraphicsCommandList* out_pCopyCmdList, ID3D12Resource* in_pSrcResource,
             const UpdateList* in_pUpdateList, const std::vector<UINT>& in_indices);
+
+        static HANDLE GetFileHandle(const FileHandle* in_pHandle) { return dynamic_cast<const FileHandleReference*>(in_pHandle)->GetHandle(); }
     };
 }
diff --git a/TileUpdateManager/SamplerFeedbackStreaming.h b/TileUpdateManager/SamplerFeedbackStreaming.h
@@ -222,7 +222,7 @@ struct TileUpdateManager
     // for visualization
     //--------------------------------------------
     virtual void SetVisualizationMode(UINT in_mode) = 0;
-
+    virtual void CaptureTraceFile(bool in_captureTrace) = 0; // capture a trace file of tile uploads
     virtual float GetGpuStreamingTime() const = 0;
     virtual float GetCpuProcessFeedbackTime() = 0; // approx. cpu time spent processing feedback last frame. expected usage is to average over many frames
     virtual UINT GetTotalNumUploads() const = 0;   // number of tiles uploaded so far

diff --git a/TileUpdateManager/StreamingResourceBase.cpp b/TileUpdateManager/StreamingResourceBase.cpp
@@ -495,7 +495,7 @@ void Streaming::StreamingResourceBase::AbandonPendingLoads()
 //-----------------------------------------------------------------------------
 UINT Streaming::StreamingResourceBase::QueueTiles()
 {
-    UINT uploadRequested = 0;
+    UINT uploadsRequested = 0;
 
     UINT numEvictions = (UINT)m_pendingEvictions.GetReadyToEvict().size();
     UINT numLoads = (UINT)m_pendingTileLoads.size();
@@ -513,7 +513,8 @@ UINT Streaming::StreamingResourceBase::QueueTiles()
         // queue as many new tiles as possible
         if (numLoads && m_pHeap->GetAllocator().GetAvailable())
         {
-            uploadRequested = QueuePendingTileLoads(&scratchUL);
+            QueuePendingTileLoads(&scratchUL);
+            uploadsRequested = (UINT)scratchUL.m_coords.size(); // number of uploads in UpdateList
             numLoads = (UINT)m_pendingTileLoads.size();
         }
 
@@ -531,7 +532,7 @@ UINT Streaming::StreamingResourceBase::QueueTiles()
             m_pTileUpdateManager->SubmitUpdateList(*pUpdateList);
         }
     }
-    return uploadRequested;
+    return uploadsRequested;
 }
 
 /*-----------------------------------------------------------------------------
@@ -623,7 +624,7 @@ void Streaming::StreamingResourceBase::QueuePendingTileEvictions(Streaming::Upda
 // FIFO order: work from the front of the array
 // NOTE: greedy, takes every available UpdateList if it can
 //-----------------------------------------------------------------------------
-UINT Streaming::StreamingResourceBase::QueuePendingTileLoads(Streaming::UpdateList* out_pUpdateList)
+void Streaming::StreamingResourceBase::QueuePendingTileLoads(Streaming::UpdateList* out_pUpdateList)
 {
     ASSERT(out_pUpdateList);
     ASSERT(m_pHeap->GetAllocator().GetAvailable());
@@ -682,8 +683,6 @@ UINT Streaming::StreamingResourceBase::QueuePendingTileLoads(Streaming::UpdateLi
     {
         m_pendingTileLoads.erase(m_pendingTileLoads.begin() + skippedIndex, m_pendingTileLoads.begin() + numConsumed);
     }
-
-    return numConsumed;
 }
 
 //-----------------------------------------------------------------------------

diff --git a/TileUpdateManager/StreamingResourceBase.h b/TileUpdateManager/StreamingResourceBase.h
@@ -326,7 +326,7 @@ namespace Streaming
         // only push evictions to DataUploader once per rendered frame (i.e. "on the next frame")
         void QueuePendingTileEvictions(Streaming::UpdateList* out_pUpdateList);
 
-        UINT QueuePendingTileLoads(Streaming::UpdateList* out_pUpdateList); // returns # tiles queued
+        void QueuePendingTileLoads(Streaming::UpdateList* out_pUpdateList); // returns # tiles queued
 
         void LoadPackedMips();
 

diff --git a/TileUpdateManager/TileUpdateManager.cpp b/TileUpdateManager/TileUpdateManager.cpp
@@ -48,9 +48,7 @@ TileUpdateManager* TileUpdateManager::Create(
 
     const TileUpdateManagerDesc& in_desc)
 {
-    auto p = new Streaming::TileUpdateManagerBase(in_pDevice, in_pDirectCommandQueue, in_desc);
-    p->UseDirectStorage(in_desc.m_useDirectStorage);
-    return p;
+    return new Streaming::TileUpdateManagerBase(in_pDevice, in_pDirectCommandQueue, in_desc);
 }
 
 void Streaming::TileUpdateManagerBase::Destroy()
@@ -170,6 +168,11 @@ void Streaming::TileUpdateManagerBase::SetVisualizationMode(UINT in_mode)
     m_pDataUploader->SetVisualizationMode(in_mode);
 }
 
+void Streaming::TileUpdateManagerBase::CaptureTraceFile(bool in_captureTrace)
+{
+    m_pDataUploader->CaptureTraceFile(in_captureTrace);
+}
+
 //-----------------------------------------------------------------------------
 // Call this method once for each TileUpdateManager that shares heap/upload buffers
 // expected to be called once per frame, before anything is drawn.

diff --git a/TileUpdateManager/TileUpdateManagerBase.cpp b/TileUpdateManager/TileUpdateManagerBase.cpp
@@ -92,6 +92,8 @@ m_numSwapBuffers(in_desc.m_swapChainBufferCount)
 
     // advance frame number to the first frame...
     m_frameFenceValue++;
+
+    UseDirectStorage(in_desc.m_useDirectStorage);
 }
 
 Streaming::TileUpdateManagerBase::~TileUpdateManagerBase()
@@ -247,15 +249,19 @@ void Streaming::TileUpdateManagerBase::ProcessFeedbackThread()
             staleResources.resize(newStaleSize); // compact array
         }
 
-        // tell the file streamer to signal the corresponding fence
-        if ((flushPendingUploadRequests) || // flush requests from previous frame
-            (0 == staleResources.size()) || // flush because there's no more work to be done (no stale resources, all feedback has been processed)
-            // if we need updatelists and there is a minimum amount of pending work, go ahead and submit
-            // this minimum heuristic prevents "storms" of submits with too few tiles to sustain good throughput
-            ((0 == m_pDataUploader->GetNumUpdateListsAvailable()) && (uploadsRequested > m_minNumUploadRequests)))
+        // if there are uploads, maybe signal depending on heuristic to minimize # signals
+        if (uploadsRequested)
         {
-            SignalFileStreamer();
-            uploadsRequested = 0;
+            // tell the file streamer to signal the corresponding fence
+            if ((flushPendingUploadRequests) || // flush requests from previous frame
+                (0 == staleResources.size()) || // flush because there's no more work to be done (no stale resources, all feedback has been processed)
+                // if we need updatelists and there is a minimum amount of pending work, go ahead and submit
+                // this minimum heuristic prevents "storms" of submits with too few tiles to sustain good throughput
+                ((0 == m_pDataUploader->GetNumUpdateListsAvailable()) && (uploadsRequested > m_minNumUploadRequests)))
+            {
+                SignalFileStreamer();
+                uploadsRequested = 0;
+            }
         }
 
         // nothing to do? wait for next frame

diff --git a/TileUpdateManager/TileUpdateManagerBase.h b/TileUpdateManager/TileUpdateManagerBase.h
@@ -84,6 +84,7 @@ namespace Streaming
         virtual bool GetWithinFrame() const  override { return m_withinFrame; }
         virtual float GetGpuTime() const override;
         virtual void SetVisualizationMode(UINT in_mode) override;
+        virtual void CaptureTraceFile(bool in_captureTrace) override;
         virtual float GetGpuStreamingTime() const override;
         virtual float GetCpuProcessFeedbackTime() override;
         virtual UINT GetTotalNumUploads() const override;

diff --git a/src/CommandLineArgs.h b/src/CommandLineArgs.h
@@ -113,5 +113,6 @@ struct CommandLineArgs
 
     TerrainGenerator::Params m_terrainParams;
 
+    bool m_captureTrace{ false }; // capture a trace file of tile uploads
     int m_threadPriority{ 0 }; // applies to internal threads
 };
diff --git a/src/FrameEventTracing.h b/src/FrameEventTracing.h
@@ -133,7 +133,7 @@ inline void FrameEventTracing::WriteEvents(HWND in_hWnd, const CommandLineArgs&
 
     *this << "\nTimers (ms)\n"
         << "-----------------------------------------------------------------------------------------------------------\n"
-        << "cpu_draw TUM::EndFrame exec_cmd_list wait_present total_frame_time evictions copies cpu_feedback feedback_resolve num_resolves num_submits\n"
+        << "cpu_draw TUM::EndFrame exec_cmd_list wait_present total_frame_time evictions_completed copies_completed cpu_feedback feedback_resolve num_resolves num_submits\n"
         << "-----------------------------------------------------------------------------------------------------------\n";
 
     for (auto& e : m_events)

diff --git a/src/Scene.cpp b/src/Scene.cpp
@@ -1272,17 +1272,22 @@ void Scene::GatherStatistics()
         PostQuitMessage(0);
     }
 
-    m_frameNumber++;
-
-    // start timing and gathering uploads from the very beginning of the timed region
-    if (m_args.m_timingFrameFileName.size() && (m_frameNumber == m_args.m_timingStartFrame))
+    if (m_frameNumber == m_args.m_timingStartFrame)
     {
-        numSubmits = m_pTileUpdateManager->GetTotalNumSubmits();
-        m_startUploadCount = m_pTileUpdateManager->GetTotalNumUploads();
-        m_startSubmitCount = m_pTileUpdateManager->GetTotalNumSubmits();
-        m_totalTileLatency = m_pTileUpdateManager->GetTotalTileCopyLatency();
-        m_cpuTimer.Start();
+        m_pTileUpdateManager->CaptureTraceFile(m_args.m_captureTrace);
+
+        // start timing and gathering uploads from the very beginning of the timed region
+        if (m_args.m_timingFrameFileName.size())
+        {
+            numSubmits = m_pTileUpdateManager->GetTotalNumSubmits();
+            m_startUploadCount = m_pTileUpdateManager->GetTotalNumUploads();
+            m_startSubmitCount = m_pTileUpdateManager->GetTotalNumSubmits();
+            m_totalTileLatency = m_pTileUpdateManager->GetTotalTileCopyLatency();
+            m_cpuTimer.Start();
+        }
     }
+
+    m_frameNumber++;
 }
 
 //-------------------------------------------------------------------------

diff --git a/src/winMain.cpp b/src/winMain.cpp
@@ -219,6 +219,9 @@ void ParseCommandLine(CommandLineArgs& out_args)
     argParser.AddArg(L"-directStorage", [&]() { out_args.m_useDirectStorage = true; }, L"force enable DirectStorage");
     argParser.AddArg(L"-directStorageOff", [&]() { out_args.m_useDirectStorage = false; }, L"force disable DirectStorage");
     argParser.AddArg(L"-stagingSizeMB", out_args.m_stagingSizeMB, L"DirectStorage staging buffer size");
+
+    argParser.AddArg(L"-captureTrace", [&]() { out_args.m_captureTrace = true; }, false, L"capture a trace of tile requests and submits (DS only)");
+
     argParser.Parse();
 }