v4.3 update. (#2709)
* v4.3 update. * Update the cute_dsl_api changelog's doc link * Update version to 4.3.0 * Update the example link * Update doc to encourage user to install DSL from requirements.txt --------- Co-authored-by: Larry Wu <larwu@nvidia.com>
This commit is contained in:
@@ -329,6 +329,20 @@ void fmha_bwd_reference_dQ(
|
||||
dim3 grid(size<0>(mDQ), size<2>(mDQ), 1);
|
||||
dim3 block(256);
|
||||
int shared_mem = size<0>(mK) * sizeof(typename TensorDQ::value_type);
|
||||
cudaError_t result;
|
||||
if (shared_mem >= (48 << 10)) {
|
||||
result = cudaFuncSetAttribute(
|
||||
&fmha_bwd_reference_dQ_kernel<ProblemShape, TensorQ, TensorK, TensorV,
|
||||
TensorO, TensorLSE, TensorDO, TensorDQ, Fusion>,
|
||||
cudaFuncAttributeMaxDynamicSharedMemorySize,
|
||||
shared_mem);
|
||||
if (cudaSuccess != result) {
|
||||
cudaGetLastError(); // Clear the error state
|
||||
throw std::runtime_error("Failed to allocate " +
|
||||
std::to_string(shared_mem >> 10) + " KB dynamic smem for dQ tensor in ref. check - " +
|
||||
"please try reducing seq_len or skipping ref. check");
|
||||
}
|
||||
}
|
||||
fmha_bwd_reference_dQ_kernel<<<grid, block, shared_mem>>>(problem_shape, mQ, mK, mV, mO, mLSE, mDO, mDQ, fusion);
|
||||
}
|
||||
|
||||
@@ -356,6 +370,20 @@ void fmha_bwd_reference_dK(
|
||||
dim3 grid(K, H_K * B, 1);
|
||||
dim3 block(std::max(D, 256));
|
||||
int shared_mem = size<0>(mDO) * sizeof(typename TensorDK::value_type);
|
||||
cudaError_t result;
|
||||
if (shared_mem >= (48 << 10)) {
|
||||
result = cudaFuncSetAttribute(
|
||||
&fmha_bwd_reference_dK_kernel<ProblemShape, TensorQ, TensorK, TensorV,
|
||||
TensorO, TensorLSE, TensorDO, TensorDK, Fusion>,
|
||||
cudaFuncAttributeMaxDynamicSharedMemorySize,
|
||||
shared_mem);
|
||||
if (cudaSuccess != result) {
|
||||
cudaGetLastError(); // Clear the error state
|
||||
throw std::runtime_error("Failed to allocate " +
|
||||
std::to_string(shared_mem >> 10) + " KB dynamic smem for dO tensor in ref. check - " +
|
||||
"please try reducing seq_len or skipping ref. check");
|
||||
}
|
||||
}
|
||||
fmha_bwd_reference_dK_kernel<<<grid, block, shared_mem>>>(problem_shape, mQ, mK, mV, mO, mLSE, mDO, mDK, fusion);
|
||||
}
|
||||
|
||||
@@ -383,6 +411,20 @@ void fmha_bwd_reference_dV(
|
||||
dim3 grid(K, H_K * B, 1);
|
||||
dim3 block(std::max(D_VO, 256));
|
||||
int shared_mem = size<0>(mDO) * sizeof(typename TensorDV::value_type);
|
||||
cudaError_t result;
|
||||
if (shared_mem >= (48 << 10)) {
|
||||
result = cudaFuncSetAttribute(
|
||||
&fmha_bwd_reference_dV_kernel<ProblemShape, TensorQ, TensorK, TensorV,
|
||||
TensorO, TensorLSE, TensorDO, TensorDV, Fusion>,
|
||||
cudaFuncAttributeMaxDynamicSharedMemorySize,
|
||||
shared_mem);
|
||||
if (cudaSuccess != result) {
|
||||
cudaGetLastError(); // Clear the error state
|
||||
throw std::runtime_error("Failed to allocate " +
|
||||
std::to_string(shared_mem >> 10) + " KB dynamic smem for dO tensor in ref. check - " +
|
||||
"please try reducing seq_len or skipping ref. check");
|
||||
}
|
||||
}
|
||||
fmha_bwd_reference_dV_kernel<<<grid, block, shared_mem>>>(problem_shape, mQ, mK, mV, mO, mLSE, mDO, mDV, fusion);
|
||||
}
|
||||
|
||||
|
||||
@@ -189,6 +189,19 @@ void fmha_reference(
|
||||
dim3 grid(size<0>(mO), size<2>(mO), 1);
|
||||
dim3 block(256);
|
||||
int shared_mem = size<0>(mK) * int(sizeof(typename TensorLSE::value_type));
|
||||
cudaError_t result;
|
||||
if (shared_mem >= (48 << 10)) {
|
||||
result = cudaFuncSetAttribute(
|
||||
&fmha_reference_kernel<ProblemShapeIn, TensorQ, TensorK, TensorV, TensorO, TensorLSE, Mask>,
|
||||
cudaFuncAttributeMaxDynamicSharedMemorySize,
|
||||
shared_mem);
|
||||
if (cudaSuccess != result) {
|
||||
cudaGetLastError(); // Clear the error state
|
||||
throw std::runtime_error("Failed to allocate " +
|
||||
std::to_string(shared_mem >> 10) + " KB dynamic smem for S/P tensor in ref. check - " +
|
||||
"please try reducing seq_len or skipping ref. check");
|
||||
}
|
||||
}
|
||||
fmha_reference_kernel<<<grid, block, shared_mem>>>(problem_shape_in, mQ, mK, mV, mO, mLSE, mask);
|
||||
}
|
||||
|
||||
|
||||
@@ -185,9 +185,11 @@ void fmha_mla_reference(
|
||||
cudaFuncAttributeMaxDynamicSharedMemorySize,
|
||||
shared_mem);
|
||||
if (cudaSuccess != result) {
|
||||
result = cudaGetLastError(); // to clear the error bit
|
||||
throw std::runtime_error("couldn't perform smem optin");
|
||||
}
|
||||
cudaGetLastError(); // Clear the error state
|
||||
throw std::runtime_error("Failed to allocate " +
|
||||
std::to_string(shared_mem >> 10) + " KB dynamic smem for S/P tensor in ref. check - " +
|
||||
"please try reducing seq_len or skipping ref. check");
|
||||
}
|
||||
}
|
||||
fmha_mla_reference_kernel<<<grid, block, shared_mem>>>(
|
||||
problem_shape, mSeq, mPT, mQL, mQR, mCL, mKR, mO, mLSE, scale);
|
||||
|
||||
Reference in New Issue
Block a user