Skip to content
This repository was archived by the owner on Dec 25, 2023. It is now read-only.

Commit 580a263

Browse files
allenhux-intelallenhux-intel
committed
Fix to eliminate submit() calls when there are no requests. Added -captureTrace capability (which helped identify the issue).
1 parent 74275a2 commit 580a263

16 files changed

+127
-37
lines changed

‎TileUpdateManager/DataUploader.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,7 @@ namespace Streaming
9393
float GetApproximateTileCopyLatency() const { return m_pFenceThreadTimer->GetSecondsFromDelta(m_totalTileCopyLatency); } // sum of per-tile latencies so far
9494

9595
void SetVisualizationMode(UINT in_mode) { m_pFileStreamer->SetVisualizationMode(in_mode); }
96+
void CaptureTraceFile(bool in_captureTrace) { m_pFileStreamer->CaptureTraceFile(in_captureTrace); }
9697
private:
9798
// upload buffer size
9899
const UINT m_stagingBufferSizeMB{ 0 };

‎TileUpdateManager/FileStreamer.cpp

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,55 @@ Streaming::FileStreamer::FileStreamer(ID3D12Device* in_pDevice)
9696
}
9797
}
9898

99+
Streaming::FileStreamer::~FileStreamer()
100+
{
101+
// write trace file
102+
if (m_captureTrace)
103+
{
104+
int index = 0;
105+
while (1)
106+
{
107+
std::wstring unique = L"uploadTraceFile_" + std::to_wstring(++index) + L".json";
108+
if (!std::filesystem::exists(unique))
109+
{
110+
m_trace.Write(unique);
111+
break;
112+
}
113+
}
114+
}
115+
}
116+
117+
//-----------------------------------------------------------------------------
118+
// append an upload request to the trace file
119+
//-----------------------------------------------------------------------------
120+
void Streaming::FileStreamer::TraceRequest(const D3D12_TILED_RESOURCE_COORDINATE& in_coord,
121+
UINT64 in_offset, UINT64 in_fileHandle, UINT32 in_numBytes)
122+
{
123+
auto& r = m_trace.GetRoot()["submits"][m_traceSubmitIndex][m_traceRequestIndex++];
124+
r["coord"][0] = in_coord.X;
125+
r["coord"][1] = in_coord.Y;
126+
r["coord"][2] = in_coord.Subresource;
127+
r["off"] = in_offset;
128+
r["file"] = in_fileHandle;
129+
r["size"] = in_numBytes;
130+
}
131+
132+
//-----------------------------------------------------------------------------
133+
// capture submit() to the trace file
134+
// increments index to next array of requests
135+
//-----------------------------------------------------------------------------
136+
void Streaming::FileStreamer::TraceSubmit()
137+
{
138+
if (m_firstSubmit) // ignore first submit while tracing in case enabled mid-frame.
139+
{
140+
m_firstSubmit = false;
141+
return;
142+
}
143+
ASSERT(0 != m_traceRequestIndex); // should never call Submit() without any requests
144+
m_traceRequestIndex = 0;
145+
m_traceSubmitIndex++;
146+
}
147+
99148
//-----------------------------------------------------------------------------
100149
//-----------------------------------------------------------------------------
101150
void Streaming::FileStreamer::InitializeBC7()

‎TileUpdateManager/FileStreamer.h

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
#pragma once
2828

2929
#include "Streaming.h"
30+
#include "ConfigurationParser.h"
3031

3132
namespace Streaming
3233
{
@@ -43,7 +44,7 @@ namespace Streaming
4344
{
4445
public:
4546
FileStreamer(ID3D12Device* in_pDevice);
46-
virtual ~FileStreamer() {}
47+
virtual ~FileStreamer();
4748

4849
virtual FileHandle* OpenFile(const std::wstring& in_path) = 0;
4950

@@ -60,6 +61,8 @@ namespace Streaming
6061
void SetVisualizationMode(UINT in_mode) { m_visualizationMode = (VisualizationMode)in_mode; }
6162

6263
bool GetCompleted(const UpdateList& in_updateList) const;
64+
65+
void CaptureTraceFile(bool in_captureTrace) { m_captureTrace = in_captureTrace; } // enable/disable writing requests/submits to a trace file
6366
protected:
6467
// copy queue fence
6568
ComPtr<ID3D12Fence> m_copyFence;
@@ -79,5 +82,17 @@ namespace Streaming
7982

8083
static BYTE m_BC1[m_lutSize][D3D12_TILED_RESOURCE_TILE_SIZE_IN_BYTES];
8184
void InitializeBC1();
85+
86+
// trace file
87+
bool m_captureTrace{ false };
88+
89+
void TraceRequest(const D3D12_TILED_RESOURCE_COORDINATE& in_coord,
90+
UINT64 in_offset, UINT64 in_fileHandle, UINT32 in_numBytes);
91+
void TraceSubmit();
92+
private:
93+
bool m_firstSubmit{ true };
94+
ConfigurationParser m_trace; // array of submits, each submit is an array of requests
95+
UINT m_traceSubmitIndex{ 0 };
96+
UINT m_traceRequestIndex{ 0 };
8297
};
8398
}

‎TileUpdateManager/FileStreamerDS.cpp

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,7 @@ void Streaming::FileStreamerDS::StreamTexture(Streaming::UpdateList& in_updateLi
6262
DXGI_FORMAT textureFormat = pTextureFileInfo->GetFormat();
6363
auto pDstHeap = in_updateList.m_pStreamingResource->GetHeap();
6464

65-
DSTORAGE_REQUEST request = {};
65+
DSTORAGE_REQUEST request{};
6666
request.Options.DestinationType = DSTORAGE_REQUEST_DESTINATION_TILES;
6767
request.Destination.Tiles.TileRegionSize = D3D12_TILE_REGION_SIZE{ 1, FALSE, 0, 0, 0 };
6868

@@ -77,14 +77,19 @@ void Streaming::FileStreamerDS::StreamTexture(Streaming::UpdateList& in_updateLi
7777
{
7878
request.Source.File.Offset = pTextureFileInfo->GetFileOffset(in_updateList.m_coords[i], request.Source.File.Size);
7979

80-
D3D12_TILED_RESOURCE_COORDINATE coord;
80+
D3D12_TILED_RESOURCE_COORDINATE coord{};
8181
ID3D12Resource* pAtlas = pDstHeap->ComputeCoordFromTileIndex(coord, in_updateList.m_heapIndices[i], textureFormat);
8282

8383
request.Destination.Tiles.Resource = pAtlas;
8484
request.Destination.Tiles.TiledRegionStartCoordinate = coord;
8585
request.Options.CompressionFormat = (DSTORAGE_COMPRESSION_FORMAT)pTextureFileInfo->GetCompressionFormat();
8686

8787
m_fileQueue->EnqueueRequest(&request);
88+
89+
if (m_captureTrace)
90+
{
91+
TraceRequest(coord, request.Source.File.Offset, (UINT64)request.Source.File.Source, (UINT32)request.Source.File.Size);
92+
}
8893
}
8994
}
9095
else // visualization color is loaded from memory
@@ -98,7 +103,7 @@ void Streaming::FileStreamerDS::StreamTexture(Streaming::UpdateList& in_updateLi
98103
{
99104
request.Source.Memory.Source = GetVisualizationData(in_updateList.m_coords[i], textureFormat);
100105

101-
D3D12_TILED_RESOURCE_COORDINATE coord;
106+
D3D12_TILED_RESOURCE_COORDINATE coord{};
102107
ID3D12Resource* pAtlas = pDstHeap->ComputeCoordFromTileIndex(coord, in_updateList.m_heapIndices[i], textureFormat);
103108

104109
request.Destination.Tiles.Resource = pAtlas;
@@ -122,6 +127,8 @@ void Streaming::FileStreamerDS::Signal()
122127
{
123128
m_fileQueue->EnqueueSignal(m_copyFence.Get(), m_copyFenceValue);
124129
m_fileQueue->Submit();
130+
131+
if (m_captureTrace) { TraceSubmit(); }
125132
}
126133
else
127134
{

‎TileUpdateManager/FileStreamerDS.h

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,6 @@ namespace Streaming
1616
virtual FileHandle* OpenFile(const std::wstring& in_path) override;
1717
virtual void StreamTexture(Streaming::UpdateList& in_updateList) override;
1818

19-
static IDStorageFile* GetFileHandle(const FileHandle* in_pHandle);
20-
2119
// for DS, we don't have a way to batch batches
2220
// this allows the calling thread to periodically request Submit() vs. every enqueue
2321
virtual void Signal() override;
@@ -39,5 +37,7 @@ namespace Streaming
3937

4038
// memory queue when for visualization modes, which copy from cpu memory
4139
ComPtr<IDStorageQueue> m_memoryQueue;
40+
41+
static IDStorageFile* GetFileHandle(const FileHandle* in_pHandle);
4242
};
4343
};

‎TileUpdateManager/FileStreamerReference.h

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -48,8 +48,6 @@ namespace Streaming
4848

4949
virtual void Signal() override {} // reference auto-submits
5050

51-
static HANDLE GetFileHandle(const FileHandle* in_pHandle) { return dynamic_cast<const FileHandleReference*>(in_pHandle)->GetHandle(); }
52-
5351
static const UINT MEDIA_SECTOR_SIZE = 4096; // see https://docs.microsoft.com/en-us/windows/win32/fileio/file-buffering
5452
private:
5553
class FileHandleReference : public FileHandle
@@ -128,5 +126,7 @@ namespace Streaming
128126
void LoadTexture(CopyBatch& in_copyBatch, UINT in_numtilesToLoad);
129127
void CopyTiles(ID3D12GraphicsCommandList* out_pCopyCmdList, ID3D12Resource* in_pSrcResource,
130128
const UpdateList* in_pUpdateList, const std::vector<UINT>& in_indices);
129+
130+
static HANDLE GetFileHandle(const FileHandle* in_pHandle) { return dynamic_cast<const FileHandleReference*>(in_pHandle)->GetHandle(); }
131131
};
132132
}

‎TileUpdateManager/SamplerFeedbackStreaming.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -222,7 +222,7 @@ struct TileUpdateManager
222222
// for visualization
223223
//--------------------------------------------
224224
virtual void SetVisualizationMode(UINT in_mode) = 0;
225-
225+
virtual void CaptureTraceFile(bool in_captureTrace) = 0; // capture a trace file of tile uploads
226226
virtual float GetGpuStreamingTime() const = 0;
227227
virtual float GetCpuProcessFeedbackTime() = 0; // approx. cpu time spent processing feedback last frame. expected usage is to average over many frames
228228
virtual UINT GetTotalNumUploads() const = 0; // number of tiles uploaded so far

‎TileUpdateManager/StreamingResourceBase.cpp

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -495,7 +495,7 @@ void Streaming::StreamingResourceBase::AbandonPendingLoads()
495495
//-----------------------------------------------------------------------------
496496
UINT Streaming::StreamingResourceBase::QueueTiles()
497497
{
498-
UINT uploadRequested = 0;
498+
UINT uploadsRequested = 0;
499499

500500
UINT numEvictions = (UINT)m_pendingEvictions.GetReadyToEvict().size();
501501
UINT numLoads = (UINT)m_pendingTileLoads.size();
@@ -513,7 +513,8 @@ UINT Streaming::StreamingResourceBase::QueueTiles()
513513
// queue as many new tiles as possible
514514
if (numLoads && m_pHeap->GetAllocator().GetAvailable())
515515
{
516-
uploadRequested = QueuePendingTileLoads(&scratchUL);
516+
QueuePendingTileLoads(&scratchUL);
517+
uploadsRequested = (UINT)scratchUL.m_coords.size(); // number of uploads in UpdateList
517518
numLoads = (UINT)m_pendingTileLoads.size();
518519
}
519520

@@ -531,7 +532,7 @@ UINT Streaming::StreamingResourceBase::QueueTiles()
531532
m_pTileUpdateManager->SubmitUpdateList(*pUpdateList);
532533
}
533534
}
534-
return uploadRequested;
535+
return uploadsRequested;
535536
}
536537

537538
/*-----------------------------------------------------------------------------
@@ -623,7 +624,7 @@ void Streaming::StreamingResourceBase::QueuePendingTileEvictions(Streaming::Upda
623624
// FIFO order: work from the front of the array
624625
// NOTE: greedy, takes every available UpdateList if it can
625626
//-----------------------------------------------------------------------------
626-
UINT Streaming::StreamingResourceBase::QueuePendingTileLoads(Streaming::UpdateList* out_pUpdateList)
627+
void Streaming::StreamingResourceBase::QueuePendingTileLoads(Streaming::UpdateList* out_pUpdateList)
627628
{
628629
ASSERT(out_pUpdateList);
629630
ASSERT(m_pHeap->GetAllocator().GetAvailable());
@@ -682,8 +683,6 @@ UINT Streaming::StreamingResourceBase::QueuePendingTileLoads(Streaming::UpdateLi
682683
{
683684
m_pendingTileLoads.erase(m_pendingTileLoads.begin() + skippedIndex, m_pendingTileLoads.begin() + numConsumed);
684685
}
685-
686-
return numConsumed;
687686
}
688687

689688
//-----------------------------------------------------------------------------

‎TileUpdateManager/StreamingResourceBase.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -326,7 +326,7 @@ namespace Streaming
326326
// only push evictions to DataUploader once per rendered frame (i.e. "on the next frame")
327327
void QueuePendingTileEvictions(Streaming::UpdateList* out_pUpdateList);
328328

329-
UINT QueuePendingTileLoads(Streaming::UpdateList* out_pUpdateList); // returns # tiles queued
329+
void QueuePendingTileLoads(Streaming::UpdateList* out_pUpdateList); // returns # tiles queued
330330

331331
void LoadPackedMips();
332332

‎TileUpdateManager/TileUpdateManager.cpp

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -48,9 +48,7 @@ TileUpdateManager* TileUpdateManager::Create(
4848

4949
const TileUpdateManagerDesc& in_desc)
5050
{
51-
auto p = new Streaming::TileUpdateManagerBase(in_pDevice, in_pDirectCommandQueue, in_desc);
52-
p->UseDirectStorage(in_desc.m_useDirectStorage);
53-
return p;
51+
return new Streaming::TileUpdateManagerBase(in_pDevice, in_pDirectCommandQueue, in_desc);
5452
}
5553

5654
void Streaming::TileUpdateManagerBase::Destroy()
@@ -170,6 +168,11 @@ void Streaming::TileUpdateManagerBase::SetVisualizationMode(UINT in_mode)
170168
m_pDataUploader->SetVisualizationMode(in_mode);
171169
}
172170

171+
void Streaming::TileUpdateManagerBase::CaptureTraceFile(bool in_captureTrace)
172+
{
173+
m_pDataUploader->CaptureTraceFile(in_captureTrace);
174+
}
175+
173176
//-----------------------------------------------------------------------------
174177
// Call this method once for each TileUpdateManager that shares heap/upload buffers
175178
// expected to be called once per frame, before anything is drawn.

‎TileUpdateManager/TileUpdateManagerBase.cpp

Lines changed: 14 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,8 @@ m_numSwapBuffers(in_desc.m_swapChainBufferCount)
9292

9393
// advance frame number to the first frame...
9494
m_frameFenceValue++;
95+
96+
UseDirectStorage(in_desc.m_useDirectStorage);
9597
}
9698

9799
Streaming::TileUpdateManagerBase::~TileUpdateManagerBase()
@@ -247,15 +249,19 @@ void Streaming::TileUpdateManagerBase::ProcessFeedbackThread()
247249
staleResources.resize(newStaleSize); // compact array
248250
}
249251

250-
// tell the file streamer to signal the corresponding fence
251-
if ((flushPendingUploadRequests) || // flush requests from previous frame
252-
(0 == staleResources.size()) || // flush because there's no more work to be done (no stale resources, all feedback has been processed)
253-
// if we need updatelists and there is a minimum amount of pending work, go ahead and submit
254-
// this minimum heuristic prevents "storms" of submits with too few tiles to sustain good throughput
255-
((0 == m_pDataUploader->GetNumUpdateListsAvailable()) && (uploadsRequested > m_minNumUploadRequests)))
252+
// if there are uploads, maybe signal depending on heuristic to minimize # signals
253+
if (uploadsRequested)
256254
{
257-
SignalFileStreamer();
258-
uploadsRequested = 0;
255+
// tell the file streamer to signal the corresponding fence
256+
if ((flushPendingUploadRequests) || // flush requests from previous frame
257+
(0 == staleResources.size()) || // flush because there's no more work to be done (no stale resources, all feedback has been processed)
258+
// if we need updatelists and there is a minimum amount of pending work, go ahead and submit
259+
// this minimum heuristic prevents "storms" of submits with too few tiles to sustain good throughput
260+
((0 == m_pDataUploader->GetNumUpdateListsAvailable()) && (uploadsRequested > m_minNumUploadRequests)))
261+
{
262+
SignalFileStreamer();
263+
uploadsRequested = 0;
264+
}
259265
}
260266

261267
// nothing to do? wait for next frame

‎TileUpdateManager/TileUpdateManagerBase.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,7 @@ namespace Streaming
8484
virtual bool GetWithinFrame() const override { return m_withinFrame; }
8585
virtual float GetGpuTime() const override;
8686
virtual void SetVisualizationMode(UINT in_mode) override;
87+
virtual void CaptureTraceFile(bool in_captureTrace) override;
8788
virtual float GetGpuStreamingTime() const override;
8889
virtual float GetCpuProcessFeedbackTime() override;
8990
virtual UINT GetTotalNumUploads() const override;

‎src/CommandLineArgs.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -113,5 +113,6 @@ struct CommandLineArgs
113113

114114
TerrainGenerator::Params m_terrainParams;
115115

116+
bool m_captureTrace{ false }; // capture a trace file of tile uploads
116117
int m_threadPriority{ 0 }; // applies to internal threads
117118
};

‎src/FrameEventTracing.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -133,7 +133,7 @@ inline void FrameEventTracing::WriteEvents(HWND in_hWnd, const CommandLineArgs&
133133

134134
*this << "\nTimers (ms)\n"
135135
<< "-----------------------------------------------------------------------------------------------------------\n"
136-
<< "cpu_draw TUM::EndFrame exec_cmd_list wait_present total_frame_time evictions copies cpu_feedback feedback_resolve num_resolves num_submits\n"
136+
<< "cpu_draw TUM::EndFrame exec_cmd_list wait_present total_frame_time evictions_completed copies_completed cpu_feedback feedback_resolve num_resolves num_submits\n"
137137
<< "-----------------------------------------------------------------------------------------------------------\n";
138138

139139
for (auto& e : m_events)

‎src/Scene.cpp

Lines changed: 14 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1272,17 +1272,22 @@ void Scene::GatherStatistics()
12721272
PostQuitMessage(0);
12731273
}
12741274

1275-
m_frameNumber++;
1276-
1277-
// start timing and gathering uploads from the very beginning of the timed region
1278-
if (m_args.m_timingFrameFileName.size() && (m_frameNumber == m_args.m_timingStartFrame))
1275+
if (m_frameNumber == m_args.m_timingStartFrame)
12791276
{
1280-
numSubmits = m_pTileUpdateManager->GetTotalNumSubmits();
1281-
m_startUploadCount = m_pTileUpdateManager->GetTotalNumUploads();
1282-
m_startSubmitCount = m_pTileUpdateManager->GetTotalNumSubmits();
1283-
m_totalTileLatency = m_pTileUpdateManager->GetTotalTileCopyLatency();
1284-
m_cpuTimer.Start();
1277+
m_pTileUpdateManager->CaptureTraceFile(m_args.m_captureTrace);
1278+
1279+
// start timing and gathering uploads from the very beginning of the timed region
1280+
if (m_args.m_timingFrameFileName.size())
1281+
{
1282+
numSubmits = m_pTileUpdateManager->GetTotalNumSubmits();
1283+
m_startUploadCount = m_pTileUpdateManager->GetTotalNumUploads();
1284+
m_startSubmitCount = m_pTileUpdateManager->GetTotalNumSubmits();
1285+
m_totalTileLatency = m_pTileUpdateManager->GetTotalTileCopyLatency();
1286+
m_cpuTimer.Start();
1287+
}
12851288
}
1289+
1290+
m_frameNumber++;
12861291
}
12871292

12881293
//-------------------------------------------------------------------------

‎src/winMain.cpp

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -219,6 +219,9 @@ void ParseCommandLine(CommandLineArgs& out_args)
219219
argParser.AddArg(L"-directStorage", [&]() { out_args.m_useDirectStorage = true; }, L"force enable DirectStorage");
220220
argParser.AddArg(L"-directStorageOff", [&]() { out_args.m_useDirectStorage = false; }, L"force disable DirectStorage");
221221
argParser.AddArg(L"-stagingSizeMB", out_args.m_stagingSizeMB, L"DirectStorage staging buffer size");
222+
223+
argParser.AddArg(L"-captureTrace", [&]() { out_args.m_captureTrace = true; }, false, L"capture a trace of tile requests and submits (DS only)");
224+
222225
argParser.Parse();
223226
}
224227

0 commit comments

Comments
 (0)