v3.8.0 update (#2082)

* 3.8 update

* fix Markus' name

---------

Co-authored-by: yuzhai <yuzhai@nvidia.com>
This commit is contained in:
Yujia Zhai
2025-02-06 21:33:40 -05:00
committed by GitHub
co-authored by yuzhai
parent affd1b693d
commit 833f6990e0
168 changed files with 24945 additions and 3436 deletions
+48 -28
View File
@@ -893,7 +893,6 @@ struct HostCollectiveMainloop<ScheduleType_, Gemm, ElementA_, ElementB_,
};
//
// Block Scaled Gemm Input Operands : A , B, scalefactorA, scalefactorB
//
@@ -1412,8 +1411,12 @@ struct HostCollectiveEpilogue {
cutlass::epilogue::thread::Identity<ElementCompute>>;
static constexpr bool IsRowBiasEnabled = FusionOp::IsPerRowBiasSupported;
static constexpr bool IsColBiasEnabled = FusionOp::IsPerColBiasSupported;
static_assert(not (IsColBiasEnabled && IsRowBiasEnabled));
static constexpr bool IsDeBiasEnabled = FusionOp::IsDePerRowBiasSupported;
static constexpr bool IsPerRowScaleEnabled = FusionOp::IsPerRowScaleSupported;
static constexpr bool IsPerColScaleEnabled = FusionOp::IsPerColScaleSupported;
static constexpr bool IsScaleFactorEnabled = FusionOp::IsScaleFactorSupported;
static constexpr bool IsAuxInEnabled = FusionOp::IsAuxInSupported;
static constexpr bool IsAuxOutEnabled = FusionOp::IsAuxOutSupported;
@@ -1462,7 +1465,7 @@ struct HostCollectiveEpilogue {
CheckEquality check_relative_equality = CheckEquality::EXACT;
// Are scalars copied to device memory before kernel launch
ScalarLoc use_device_scalars = ScalarLoc::ON_HOST;
// If per-row scale is enabled and this is disabled, alpha/beta are passed as a host or device scalar instead of device vector
// If vector scale is supported and this is disabled, alpha/beta are passed as a host or device scalar instead of device vector
VectorScale vector_scale_mode = VectorScale::DISABLED;
// Random distribution with which to initialize the A/B/C/D/Aux scaling factors
@@ -1555,8 +1558,7 @@ struct HostCollectiveEpilogue {
auto col_vector_coord = cutlass::make_Coord(M);
auto row_vector_coord = cutlass::make_Coord(N);
auto batch_vector_coord = cutlass::make_Coord(L);
auto ML_coord = cutlass::make_Coord(M * L);
if constexpr (IsPerRowScaleEnabled) {
if constexpr (IsPerRowScaleEnabled or IsPerColScaleEnabled) {
// scalars
if (vector_scale_mode == VectorScale::DISABLED) {
// batched scalars
@@ -1581,8 +1583,9 @@ struct HostCollectiveEpilogue {
}
// batched vectors
else {
alpha.resize(ML_coord, true);
beta.resize(ML_coord, true);
auto batched_vector_coord = cutlass::make_Coord((IsPerRowScaleEnabled ? M : N) * L);
alpha.resize(batched_vector_coord, true);
beta.resize(batched_vector_coord, true);
EXPECT_TRUE(initialize_tensor(alpha.host_view(), init_scale, seed + 2023));
if (beta_ != ElementScalar(0)) {
EXPECT_TRUE(initialize_tensor(beta.host_view(), init_scale, seed + 2024));
@@ -1627,9 +1630,7 @@ struct HostCollectiveEpilogue {
scale_D.sync_device();
}
if constexpr (
IsRowBiasEnabled
) {
if constexpr (IsRowBiasEnabled or IsColBiasEnabled) {
bias.resize(IsRowBiasEnabled ? col_vector_coord : row_vector_coord);
EXPECT_TRUE(initialize_tensor(bias.host_view(), init_bias, seed + 2023));
bias.sync_device();
@@ -1810,7 +1811,6 @@ struct HostCollectiveEpilogue {
}
passed &= passed_sf;
}
return passed;
}
@@ -1823,7 +1823,7 @@ struct HostCollectiveEpilogue {
<< ", scale_b: " << scale_B.at(coord_0)
<< ", scale_c: " << scale_C.at(coord_0);
}
if constexpr (IsPerRowScaleEnabled) {
if constexpr (IsPerRowScaleEnabled or IsPerColScaleEnabled) {
file << "\n\nvalpha = \n" << alpha.host_view();
file << "\n\nvbeta = \n" << beta.host_view();
} else {
@@ -1853,9 +1853,10 @@ struct HostCollectiveEpilogue {
file << "\n\n";
}
if constexpr (IsRowBiasEnabled) {
if constexpr (IsRowBiasEnabled or IsColBiasEnabled) {
file << "\n\nBias = \n" << bias.host_view();
}
if constexpr (IsAuxInEnabled) {
file << "\n\nAux Input = \n" << tensor_Aux.host_view();
}
@@ -1876,7 +1877,6 @@ struct HostCollectiveEpilogue {
<< "\n\nSFD Reference =\n" << reference_SFD.host_view()
<< "\n\nSFD Computed =\n" << tensor_SFD.host_view();
}
file
<< "\nC =\n" << tensor_C.host_view()
@@ -1921,6 +1921,12 @@ struct HostCollectiveEpilogue {
fusion_args.dAlpha = cute::make_stride(bool(m_stride),cute::_0{}, l_stride);
fusion_args.dBeta = cute::make_stride(bool(m_stride),cute::_0{}, l_stride);
}
else if constexpr (IsPerColScaleEnabled) {
int32_t n_stride = vector_scale_mode == VectorScale::ENABLED ? 1 : 0;
int64_t l_stride = vector_scale_mode == VectorScale::ENABLED ? N : (use_device_scalars == ScalarLoc::ON_DEVICE ? 1 : 0);
fusion_args.dAlpha = cute::make_stride(cute::_0{}, bool(n_stride), l_stride);
fusion_args.dBeta = cute::make_stride(cute::_0{}, bool(n_stride), l_stride);
}
else {
if constexpr (not IsFfma2Kernel) {
if (use_device_scalars == ScalarLoc::ON_DEVICE) {
@@ -1943,9 +1949,7 @@ struct HostCollectiveEpilogue {
fusion_args.scale_d_ptr = scale_D.device_data();
}
if constexpr (
IsRowBiasEnabled
) {
if constexpr (IsRowBiasEnabled or IsColBiasEnabled) {
fusion_args.bias_ptr = bias.device_data();
}
@@ -1993,7 +1997,6 @@ struct HostCollectiveEpilogue {
arguments.thread.block_scale_factor_ptr = tensor_SFD.device_data();
arguments.thread.norm_constant_ptr = norm_constant.device_data();
}
}
return arguments;
@@ -2025,6 +2028,12 @@ struct HostCollectiveEpilogue {
return cute::make_tensor(detail::make_iterator(alpha.host_data()),
cute::make_layout(cute::make_shape(M, N, L), make_stride(m_stride, cute::_0{}, l_stride)));
}
else if constexpr (IsPerColScaleEnabled) {
int n_stride = vector_scale_mode == VectorScale::ENABLED ? 1 : 0;
int l_stride = vector_scale_mode == VectorScale::ENABLED ? N : (use_device_scalars == ScalarLoc::ON_DEVICE ? 1 : 0);
return cute::make_tensor(detail::make_iterator(alpha.host_data()),
cute::make_layout(cute::make_shape(M, N, L), make_stride(cute::_0{}, n_stride, l_stride)));
}
else {
return cute::make_tensor(detail::make_iterator(alpha.host_data()),
cute::make_layout(cute::make_shape(M, N, L), make_stride(cute::_0{}, cute::_0{}, cute::_1{})));
@@ -2038,6 +2047,12 @@ struct HostCollectiveEpilogue {
return cute::make_tensor(detail::make_iterator(beta.host_data()),
cute::make_layout(cute::make_shape(M, N, L), make_stride(m_stride, cute::_0{}, l_stride)));
}
else if constexpr (IsPerColScaleEnabled) {
int n_stride = vector_scale_mode == VectorScale::ENABLED ? 1 : 0;
int l_stride = vector_scale_mode == VectorScale::ENABLED ? N : (use_device_scalars == ScalarLoc::ON_DEVICE ? 1 : 0);
return cute::make_tensor(detail::make_iterator(beta.host_data()),
cute::make_layout(cute::make_shape(M, N, L), make_stride(cute::_0{}, n_stride, l_stride)));
}
else {
return cute::make_tensor(detail::make_iterator(beta.host_data()),
cute::make_layout(cute::make_shape(M, N, L), make_stride(cute::_0{}, cute::_0{}, cute::_1{})));
@@ -2069,8 +2084,8 @@ struct HostCollectiveEpilogue {
ActivationFunctor,
decltype(SfD),
Int<SFD_VectorSize>,
cutlass::plus<ElementCompute>
, false /*PerColumnBias_*/
cutlass::plus<ElementCompute>,
IsColBiasEnabled
, SfGenStrategy
> epilogue_params{};
@@ -2086,8 +2101,7 @@ struct HostCollectiveEpilogue {
epilogue_params.scale_d = scale_D.at(coord_0);
}
if constexpr (IsRowBiasEnabled
or IsDeBiasEnabled)
if constexpr (IsRowBiasEnabled or IsColBiasEnabled or IsDeBiasEnabled)
{
epilogue_params.Bias = Bias;
}
@@ -2110,7 +2124,7 @@ struct HostCollectiveEpilogue {
}
}
if constexpr (IsPerRowScaleEnabled) {
if constexpr (IsPerRowScaleEnabled or IsPerColScaleEnabled) {
epilogue_params.Valpha = Valpha;
if (vector_scale_mode == VectorScale::ENABLED) {
epilogue_params.Vbeta = Vbeta;
@@ -3294,9 +3308,14 @@ bool TestAll(double alpha = 1.0, double beta = cute::is_same_v<typename Gemm::Ge
Testbed3x<Gemm, ActivationFunctor> testbed(check_relative_equality, ScalarLoc::ON_HOST, VectorScale::DISABLED);
int max_alignment = std::max(Gemm::kAlignmentA, Gemm::kAlignmentB);
std::vector<int> problem_size_m = {max_alignment, 512 - 3 * max_alignment};
std::vector<int> problem_size_n = {max_alignment, 512 - 2 * max_alignment};
int max_alignment_m = std::max({Gemm::kAlignmentA, Gemm::kAlignmentC, Gemm::kAlignmentD});
int max_alignment_n = std::max({Gemm::kAlignmentB, Gemm::kAlignmentC, Gemm::kAlignmentD});
if constexpr (std::is_base_of_v<cutlass::epilogue::fusion::FusionOperation, typename Gemm::EpilogueOutputOp>) {
max_alignment_m = std::max(max_alignment_m, Gemm::EpilogueOutputOp::AlignmentAux);
max_alignment_n = std::max(max_alignment_n, Gemm::EpilogueOutputOp::AlignmentAux);
}
std::vector<int> problem_size_m = {max_alignment_m, 512 - 3 * max_alignment_m};
std::vector<int> problem_size_n = {max_alignment_n, 512 - 2 * max_alignment_n};
if constexpr (cute::is_same_v<typename Gemm::GemmKernel::DispatchPolicy::Schedule,
cutlass::gemm::KernelTmaWarpSpecializedPingpong>) {
@@ -3307,7 +3326,8 @@ bool TestAll(double alpha = 1.0, double beta = cute::is_same_v<typename Gemm::Ge
constexpr int Stages = Gemm::GemmKernel::DispatchPolicy::Stages;
constexpr int TileShapeK = cute::size<2>(typename Gemm::GemmKernel::TileShape{});
std::vector<int> problem_size_k = {max_alignment, TileShapeK * (Stages + 1) - max_alignment};
int max_alignment_k = std::max(Gemm::kAlignmentA, Gemm::kAlignmentB);
std::vector<int> problem_size_k = {max_alignment_k, TileShapeK * (Stages + 1) - max_alignment_k};
using DecompositionMode = typename cutlass::gemm::kernel::detail::PersistentTileSchedulerSm90StreamKParams::DecompositionMode;
std::vector<DecompositionMode> decomposition_modes = {DecompositionMode::Heuristic};
@@ -3323,7 +3343,7 @@ bool TestAll(double alpha = 1.0, double beta = cute::is_same_v<typename Gemm::Ge
// Use larger K sizes for stream-K tests
static constexpr int min_tiles_per_sk_unit = cutlass::gemm::kernel::detail::PersistentTileSchedulerSm90StreamKParams::min_iters_per_sk_unit_;
problem_size_k = {TileShapeK * min_tiles_per_sk_unit, TileShapeK * 3 * min_tiles_per_sk_unit - max_alignment};
problem_size_k = {TileShapeK * min_tiles_per_sk_unit, TileShapeK * 3 * min_tiles_per_sk_unit - max_alignment_k};
}
using RasterOrderOptions = typename cutlass::gemm::kernel::detail::PersistentTileSchedulerSm90::RasterOrderOptions;
@@ -3421,7 +3441,7 @@ bool TestAll(double alpha = 1.0, double beta = cute::is_same_v<typename Gemm::Ge
// if we do support batched GEMM, just run one test on it to save on test time
if constexpr (cute::rank(ProblemShapeType{}) == 4) {
auto problem_size = ProblemShapeType{256 + max_alignment, 256 + max_alignment, 160 + max_alignment, /* l */ 3};
auto problem_size = ProblemShapeType{256 + max_alignment_m, 256 + max_alignment_n, 160 + max_alignment_k, /* l */ 3};
passed = testbed.run(
problem_size,
cutlass::from_real<ElementScalar>(alpha),