Update TensorRT-LLM backend (#362)

kaiyux · web-flow · commit 49def341ca37 · 2024-03-05T18:35:42.000+08:00
* Update TensorRT-LLM backend
diff --git a/inflight_batcher_llm/src/model_instance_state.cc b/inflight_batcher_llm/src/model_instance_state.cc
@@ -405,7 +405,7 @@ std::list<std::shared_ptr<InferenceRequest>> ModelInstanceState::get_inference_r
             mHasActiveRequests = (num_new_work_items > 0 || mBatchManager->getNumActiveRequests() > 0);
             if (mHasActiveRequests)
             {
-                commSession.bcast(num_new_work_items, 0);
+                commSession.bcastValue(num_new_work_items, 0);
             }
 
             if (num_new_work_items > 0)
@@ -425,7 +425,7 @@ std::list<std::shared_ptr<InferenceRequest>> ModelInstanceState::get_inference_r
     {
         // subordinate ranks hang until master rank sends work
         int64_t num_new_work_items;
-        commSession.bcast(num_new_work_items, 0);
+        commSession.bcastValue(num_new_work_items, 0);
         mHasActiveRequests = (num_new_work_items > 0 || mBatchManager->getNumActiveRequests() > 0);
         if (num_new_work_items > 0)
         {
@@ -483,7 +483,7 @@ std::unordered_set<uint64_t> ModelInstanceState::pollStopSignals()
     if (commSession.getSize() > 1 && mHasActiveRequests)
     {
         // Broadcast number of stopped requests
-        commSession.bcast(nStoppedReqIds, 0);
+        commSession.bcastValue(nStoppedReqIds, 0);
 
         if (nStoppedReqIds > 0)
         {
diff --git a/inflight_batcher_llm/src/model_instance_state.h b/inflight_batcher_llm/src/model_instance_state.h
@@ -40,6 +40,7 @@
 #include "tensorrt_llm/batch_manager/namedTensor.h"
 #include "tensorrt_llm/batch_manager/schedulerPolicy.h"
 #include "tensorrt_llm/batch_manager/trtGptModelOptionalParams.h"
+#include "tensorrt_llm/runtime/decodingMode.h"
 
 #include "model_state.h"
 #include "work_item.h"
diff --git a/inflight_batcher_llm/src/work_item.cc b/inflight_batcher_llm/src/work_item.cc
@@ -25,6 +25,7 @@
 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include "work_item.h"
+#include <map>
 
 namespace triton::backend::inflight_batcher_llm
 {
diff --git a/tensorrt_llm b/tensorrt_llm
@@ -1 +1 @@
-Subproject commit 655524dd82648edea35591ff146da8be4dcf2e02
+Subproject commit 728cc0044bb76d1fafbcaa720c403e8de4f81906
diff --git a/tools/version.txt b/tools/version.txt
@@ -1 +1 @@
-721a579afde43dd2e2037153da244baac6eedd29
+6ffd999ccac3393b80e94753a8d2bc302fb70005

Original file line number	Diff line number	Diff line change
`@@ -405,7 +405,7 @@ std::list<std::shared_ptr<InferenceRequest>> ModelInstanceState::get_inference_r`
`405`	`405`	`mHasActiveRequests = (num_new_work_items > 0 \|\| mBatchManager->getNumActiveRequests() > 0);`
`406`	`406`	`if (mHasActiveRequests)`
`407`	`407`	`{`
`408`		`- commSession.bcast(num_new_work_items, 0);`
	`408`	`+ commSession.bcastValue(num_new_work_items, 0);`
`409`	`409`	`}`
`410`	`410`
`411`	`411`	`if (num_new_work_items > 0)`
`@@ -425,7 +425,7 @@ std::list<std::shared_ptr<InferenceRequest>> ModelInstanceState::get_inference_r`
`425`	`425`	`{`
`426`	`426`	`// subordinate ranks hang until master rank sends work`
`427`	`427`	`int64_t num_new_work_items;`
`428`		`- commSession.bcast(num_new_work_items, 0);`
	`428`	`+ commSession.bcastValue(num_new_work_items, 0);`
`429`	`429`	`mHasActiveRequests = (num_new_work_items > 0 \|\| mBatchManager->getNumActiveRequests() > 0);`
`430`	`430`	`if (num_new_work_items > 0)`
`431`	`431`	`{`
`@@ -483,7 +483,7 @@ std::unordered_set<uint64_t> ModelInstanceState::pollStopSignals()`
`483`	`483`	`if (commSession.getSize() > 1 && mHasActiveRequests)`
`484`	`484`	`{`
`485`	`485`	`// Broadcast number of stopped requests`
`486`		`- commSession.bcast(nStoppedReqIds, 0);`
	`486`	`+ commSession.bcastValue(nStoppedReqIds, 0);`
`487`	`487`
`488`	`488`	`if (nStoppedReqIds > 0)`
`489`	`489`	`{`
Original file line number	Diff line number	Diff line change
`@@ -25,6 +25,7 @@`
`25`	`25`	`// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.`
`26`	`26`
`27`	`27`	`#include "work_item.h"`
	`28`	`+#include <map>`
`28`	`29`
`29`	`30`	`namespace triton::backend::inflight_batcher_llm`
`30`	`31`	`{`
Original file line number	Diff line number	Diff line change
`@@ -1 +1 @@`
`1`		`-721a579afde43dd2e2037153da244baac6eedd29`
	`1`	`+6ffd999ccac3393b80e94753a8d2bc302fb70005`