Commitc1c53b6

committed

fix kernel launch parameter under large num_tokens

Signed-off-by: Pengbo Wang <221450789+pengbowang-nv@users.noreply.github.com>

1 parent5b1e7c9 commitc1c53b6Copy full SHA for c1c53b6

File tree

1 file changed

-4

lines changed

cpp/tensorrt_llm/kernels/trtllmGenKernels/blockScaleMoe
- DevKernel.cu

1 file changed

-4

lines changed

`‎cpp/tensorrt_llm/kernels/trtllmGenKernels/blockScaleMoe/DevKernel.cu‎`

Lines changed: 4 additions & 4 deletions

Original file line number	Diff line number	Diff line change
`@@ -186,14 +186,14 @@ void run(Data const& data, void* stream)`
`186`	`186`	`if (data.mUseDeepSeekFp8)`
`187`	`187`	`{`
`188`	`188`	`intconst numThreads =128;`
`189`		`-constdim3grid(data.innerDim /128, data.topK, data.numTokens);`
	`189`	`+constdim3grid(data.innerDim /128, data.topK,std::min(8192,data.numTokens));`
`190`	`190`
`191`	`191`	`LAUNCH(data, activationDeepSeekKernel, grid, numThreads,0, stream);`
`192`	`192`	`}`
`193`	`193`	`else`
`194`	`194`	`{`
`195`	`195`	`intconst numThreads =256;`
`196`		`-constdim3grid(data.innerDim /128, data.topK, data.numTokens);`
	`196`	`+constdim3grid(data.innerDim /128, data.topK,std::min(8192,data.numTokens));`
`197`	`197`
`198`	`198`	`LAUNCH(data, activationKernel, grid, numThreads,0, stream);`
`199`	`199`	`}`
`@@ -371,7 +371,7 @@ void run(Data const& data, void* stream)`
`371`	`371`	`constexprint VecSize =4;`
`372`	`372`	`intconst numThreads =128;`
`373`	`373`	`intconst numBlocksX = (data.hiddenDimSf / VecSize -1 + numThreads) / numThreads;`
`374`		`-intconst numBlocksY = data.numTokens;`
	`374`	`+intconst numBlocksY =std::min(8192,data.numTokens);`
`375`	`375`	`dim3numBlocks(numBlocksX, numBlocksY);`
`376`	`376`	`#defineCONVERT_FP4_SF_LAUNCH(LayoutSrc, LayoutDst) \`
`377`	`377`	`if (data.sfLayoutSrc == tg::SfLayout::LayoutSrc && data.sfLayoutDst == tg::SfLayout::LayoutDst) \`
`@@ -457,7 +457,7 @@ void run(Data const& data, void* stream)`
`457`	`457`	`{`
`458`	`458`	`intconst numThreads =256;`
`459`	`459`	`intconst numBlocksX = (data.hiddenDim -1 + numThreads) / numThreads;`
`460`		`-intconst numBlocksY = data.numTokens;`
	`460`	`+intconst numBlocksY =std::min(8192,data.numTokens);`
`461`	`461`	`dim3numBlocks(numBlocksX, numBlocksY);`
`462`	`462`
`463`	`463`	`LAUNCH(data, permuteKernel, numBlocks, numThreads,0, stream);`

0 commit comments

Comments

(0)

Movatterモバイル変換

Navigation Menu

Search code, repositories, users, issues, pull requests...

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Uh oh!

Commitc1c53b6

File tree

1 file changed

1 file changed

`‎cpp/tensorrt_llm/kernels/trtllmGenKernels/blockScaleMoe/DevKernel.cu‎`

0 commit comments