DXIL Shader Debugger fixes for GPU batching of SampleGather operations

Author: Zorro666

renderdoc/driver/d3d12/d3d12_debug.cpp

@@ -605,7 +605,7 @@ bool D3D12DebugManager::CreateShaderDebugResources()
 {
   // MathOp is 2, SampleGatherOp is 6
   const uint64_t resultMaxElementSize = sizeof(Vec4f) * (2 + 6);
-  const uint32_t maxQueuedResults = D3D12DebugManager::MAX_SHADER_DEBUG_QUEUED_OPS;
+  const uint32_t maxQueuedResults = ShaderDebugConstants::MAX_SHADER_DEBUG_QUEUED_OPS;
   const uint64_t shaderDebugReadbackSize = resultMaxElementSize * maxQueuedResults;
   RDCCOMPILE_ASSERT(shaderDebugReadbackSize < m_ReadbackSize, "Readback buffer is not big enough");
 

renderdoc/driver/d3d12/d3d12_debug.h

@@ -42,6 +42,13 @@ class DXBCContainer;
 
 #define D3D12_MSAA_SAMPLECOUNT 4
 
+enum ShaderDebugConstants
+{
+  MAX_SHADER_DEBUG_QUEUED_OPS = 128,
+  COUNT_SRVS_PER_DEBUG = 25,
+  COUNT_SAMPLERS_PER_DEBUG = 2,
+};
+
 // baked indices in descriptor heaps
 enum CBVUAVSRVSlot
 {
@@ -81,7 +88,8 @@ enum CBVUAVSRVSlot
   STENCIL_MSAA_SRV32x,
 
   FIRST_SHADDEBUG_SRV,
-  LAST_SHADDEBUG_SRV = FIRST_SHADDEBUG_SRV + 25,
+  LAST_SHADDEBUG_SRV = FIRST_SHADDEBUG_SRV + ShaderDebugConstants::COUNT_SRVS_PER_DEBUG *
+                                                 ShaderDebugConstants::MAX_SHADER_DEBUG_QUEUED_OPS,
 
   FIRST_PIXELHISTORY_SRV,
   LAST_PIXELHISTORY_SRV = FIRST_PIXELHISTORY_SRV + 10,
@@ -116,7 +124,8 @@ enum SamplerSlot
   FIRST_SAMP = POINT_SAMP,
   LINEAR_SAMP,
   SHADDEBUG_SAMPLER0,
-  SHADDEBUG_SAMPLER1,
+  LAST_SHADDEBUG_SAMPLER = SHADDEBUG_SAMPLER0 + ShaderDebugConstants::COUNT_SAMPLERS_PER_DEBUG *
+                                                    ShaderDebugConstants::MAX_SHADER_DEBUG_QUEUED_OPS,
 };
 
 enum DSVSlot
@@ -152,11 +161,6 @@ class D3D12DebugManager
   D3D12DebugManager(WrappedID3D12Device *wrapper);
   ~D3D12DebugManager();
 
-  enum
-  {
-    MAX_SHADER_DEBUG_QUEUED_OPS = 128
-  };
-
   void GetBufferData(ID3D12Resource *buff, uint64_t offset, uint64_t length, bytebuf &retData);
 
   ID3D12Resource *MakeCBuffer(UINT64 size);

renderdoc/driver/d3d12/d3d12_dxil_debug.cpp

@@ -575,8 +575,8 @@ D3D12APIWrapper::D3D12APIWrapper(WrappedID3D12Device *device, const DXIL::Progra
       m_QueuedMathOpIndex(0),
       m_QueuedSampleGatherOpIndex(0),
       m_MathOpResultOffset(0),
-      m_MaxQueuedOps(D3D12DebugManager::MAX_SHADER_DEBUG_QUEUED_OPS),
-      m_SampleGatherOpResultsStart(D3D12DebugManager::MAX_SHADER_DEBUG_QUEUED_OPS *
+      m_MaxQueuedOps(ShaderDebugConstants::MAX_SHADER_DEBUG_QUEUED_OPS),
+      m_SampleGatherOpResultsStart(ShaderDebugConstants::MAX_SHADER_DEBUG_QUEUED_OPS *
                                    m_MathOpResultByteSize)
 {
   // Create the storage layout for the constant buffers
@@ -1626,8 +1626,13 @@ bool D3D12APIWrapper::QueueMathIntrinsic(DXIL::DXOp dxOp, const ShaderVariable &
       return false;
   }
 
-  return D3D12ShaderDebug::QueueMathIntrinsic(false, m_Device, cmdList, mathOp, input,
-                                              m_QueuedMathOpIndex++);
+  if(D3D12ShaderDebug::QueueMathIntrinsic(false, m_Device, cmdList, mathOp, input,
+                                          m_QueuedMathOpIndex))
+  {
+    m_QueuedMathOpIndex++;
+    return true;
+  }
+  return false;
 }
 
 // Must be called from the replay manager thread (the debugger thread)
@@ -1682,10 +1687,15 @@ bool D3D12APIWrapper::QueueSampleGather(DXIL::DXOp dxOp, SampleGatherResourceDat
 
   const char *opString = ToStr(dxOp).c_str();
   uint8_t swizzle[4] = {0, 1, 2, 3};
-  return D3D12ShaderDebug::QueueSampleGather(
-      true, m_Device, m_QueuedOpCmdList, sampleOp, resourceData, samplerData, uv, ddxCalc, ddyCalc,
-      texelOffsets, multisampleIndex, lodValue, compareValue, swizzle, gatherChannel, m_ShaderType,
-      instructionIdx, opString, m_QueuedSampleGatherOpIndex++, sampleRetType);
+  if(D3D12ShaderDebug::QueueSampleGather(
+         true, m_Device, m_QueuedOpCmdList, sampleOp, resourceData, samplerData, uv, ddxCalc,
+         ddyCalc, texelOffsets, multisampleIndex, lodValue, compareValue, swizzle, gatherChannel,
+         m_ShaderType, instructionIdx, opString, m_QueuedSampleGatherOpIndex, sampleRetType))
+  {
+    m_QueuedSampleGatherOpIndex++;
+    return true;
+  }
+  return false;
 }
 
 // Must be called from the replay manager thread (the debugger thread)