CUTLASS 3.7 (#2045)

* CUTLASS 3.7

* clean up changelog

---------

Co-authored-by: yuzhai <yuzhai@nvidia.com>
Co-authored-by: Haicheng Wu <haichengw@nvidia.com>
This commit is contained in:
Yujia Zhai
2025-01-18 09:53:07 -05:00
committed by GitHub
co-authored by yuzhai Haicheng Wu
parent 902dff3663
commit b78588d163
2030 changed files with 8947 additions and 3475 deletions
+1 -1
View File
@@ -1,4 +1,4 @@
# Copyright (c) 2023 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# Copyright (c) 2023 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: BSD-3-Clause
#
# Redistribution and use in source and binary forms, with or without
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2023 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2023 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
@@ -821,6 +821,78 @@ get_conv_problem_vector<3, cutlass::conv::Operator::kWgrad>() {
return problem_shapes;
}
/////////////////////////////////////////////////////////////////////////////////////////////////
// Grouped Wgrad
/////////////////////////////////////////////////////////////////////////////////////////////////
// Get problem size vectors for group conv problems
template<int SpatialDim, cutlass::conv::Operator ConvOp>
std::vector<cutlass::conv::ConvProblemShape<ConvOp, SpatialDim>>
inline
get_grouped_conv_problem_vector(int GroupsPerTile);
// Specialization for 3D wgrad problems
template<>
std::vector<cutlass::conv::ConvProblemShape<cutlass::conv::Operator::kWgrad, 3>> inline
get_grouped_conv_problem_vector<3, cutlass::conv::Operator::kWgrad>(int GroupsPerTile) {
using ProblemShape = cutlass::conv::ConvProblemShape<cutlass::conv::Operator::kWgrad, 3>;
std::vector<ProblemShape> problem_shapes;
if (GroupsPerTile == 1) {
// channel_per_group == 64
problem_shapes.push_back({
cutlass::conv::Mode::kCrossCorrelation,
{1, 1, 16, 16, 2048}, // ndhwc
{2048, 1, 3, 3, 64}, // ktrsc
{0, 1, 1}, // padding lower (pad_d, pad_h, pad_w)
{0, 1, 1}, // padding upper (pad_d, pad_h, pad_w)
{1, 1, 1}, // stride (stride_d, stride_h, stride_w)
{1, 1, 1}, // dilation (dilation_d, dilation_h, dilation_w)
32 // groups
});
}
else if (GroupsPerTile == 2) {
// channel_per_group == 32
problem_shapes.push_back({
cutlass::conv::Mode::kCrossCorrelation,
{1, 1, 16, 16, 1024}, // ndhwc
{1024, 1, 3, 3, 32}, // ktrsc
{0, 1, 1}, // padding lower (pad_d, pad_h, pad_w)
{0, 1, 1}, // padding upper (pad_d, pad_h, pad_w)
{1, 1, 1}, // stride (stride_d, stride_h, stride_w)
{1, 1, 1}, // dilation (dilation_d, dilation_h, dilation_w)
32 // groups
});
}
else if (GroupsPerTile == 4) {
// channel_per_group == 16
problem_shapes.push_back({
cutlass::conv::Mode::kCrossCorrelation,
{1, 1, 16, 16, 512}, // ndhwc
{512, 1, 3, 3, 16}, // ktrsc
{0, 1, 1}, // padding lower (pad_d, pad_h, pad_w)
{0, 1, 1}, // padding upper (pad_d, pad_h, pad_w)
{1, 1, 1}, // stride (stride_d, stride_h, stride_w)
{1, 1, 1}, // dilation (dilation_d, dilation_h, dilation_w)
32 // groups
});
}
else if (GroupsPerTile == 8) {
// channel_per_group == 8
problem_shapes.push_back({
cutlass::conv::Mode::kCrossCorrelation,
{1, 1, 16, 16, 256}, // ndhwc
{256, 1, 3, 3, 8}, // ktrsc
{0, 1, 1}, // padding lower (pad_d, pad_h, pad_w)
{0, 1, 1}, // padding upper (pad_d, pad_h, pad_w)
{1, 1, 1}, // stride (stride_d, stride_h, stride_w)
{1, 1, 1}, // dilation (dilation_d, dilation_h, dilation_w)
32 // groups
});
}
return problem_shapes;
}
/////////////////////////////////////////////////////////////////////////////////////////////////
// Unit Stride Dgrad
/////////////////////////////////////////////////////////////////////////////////////////////////
@@ -1,4 +1,4 @@
# Copyright (c) 2023 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# Copyright (c) 2023 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: BSD-3-Clause
#
# Redistribution and use in source and binary forms, with or without
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2023 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2023 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2023 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2023 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2023 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2023 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2023 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2023 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2023 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2023 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2023 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2023 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
@@ -1,4 +1,4 @@
# Copyright (c) 2013 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# Copyright (c) 2013 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: BSD-3-Clause
#
# Redistribution and use in source and binary forms, with or without
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2023 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2023 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2023 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2023 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2023 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2023 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2023 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2023 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2023 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2023 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2023 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2023 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2023 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2023 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2023 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2023 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2023 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2023 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2023 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2023 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2023 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2023 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2023 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2023 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
+74 -11
View File
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2023 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2023 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
@@ -113,14 +113,14 @@ struct DenseConvParams {
// Default Kernel data types
using ElementA = typename Conv::ConvKernel::ElementA;
using ElementB = typename Conv::ConvKernel::ElementB;
static constexpr cutlass::conv::Operator ConvOp = Conv::DispatchPolicy::ConvOp;
static constexpr int NumSpatialDimensions = Conv::NumSpatialDimensions;
using ProblemShape = cutlass::conv::ConvProblemShape<ConvOp, NumSpatialDimensions>;
// get the default arguments without sparse data
auto get_mainloop_arguments(
[[maybe_unused]] ProblemShape const& problem_shape,
[[maybe_unused]] ProblemShape const& problem_shape,
thrust::universal_vector<ElementA>& tensor_A,
thrust::universal_vector<ElementB>& tensor_B
) {
@@ -242,6 +242,74 @@ struct ConvTestbed {
return max_smem_size >= Conv::ConvKernel::SharedStorageSize;
}
auto transform_shape_and_stride_with_groups(ProblemShape const& problem_shape) {
using TensorExtent = cute::array<int32_t, NumSpatialDimensions + 3>;
using TensorStride = cute::array<int64_t, NumSpatialDimensions + 3>;
TensorExtent shape_a_g{};
TensorExtent shape_b_g{};
TensorExtent shape_c_g{};
TensorStride stride_a_g{};
TensorStride stride_b_g{};
TensorStride stride_c_g{};
auto shape_a = cute::reverse(problem_shape.shape_A);
auto shape_b = cute::reverse(problem_shape.shape_B);
auto shape_c = cute::reverse(problem_shape.shape_C);
auto stride_a = cute::reverse(problem_shape.stride_A);
auto stride_b = cute::reverse(problem_shape.stride_B);
auto stride_c = cute::reverse(problem_shape.stride_C);
int32_t G = problem_shape.groups;
if constexpr (ConvOp == cutlass::conv::Operator::kFprop ||
ConvOp == cutlass::conv::Operator::kDgrad) {
// shape_a_g = (c,w,h,d,n,g) or (k,q,p,z,n,g)
// shape_b_g = (c,s,r,k,t,g)
// shape_c_g = (k,q,p,z,n,g) or (c,w,h,d,n,g)
shape_a_g = cute::to_array<int32_t>(tuple_cat(
cute::make_shape(cute::size<0>(shape_a) / G),
cute::take<1,NumSpatialDimensions + 2>(shape_a),
cute::make_shape(G)));
shape_b_g = cute::to_array<int32_t>(tuple_cat(
cute::take<0,NumSpatialDimensions + 1>(shape_b),
cute::make_shape(cute::size<NumSpatialDimensions + 1>(shape_b) / G, G)));
shape_c_g = cute::to_array<int32_t>(tuple_cat(
cute::make_shape(cute::size<0>(shape_c) / G),
cute::take<1,NumSpatialDimensions + 2>(shape_c),
cute::make_shape(G)));
stride_a_g = cute::to_array<int64_t>(append(stride_a, cute::size<0>(shape_a) / G));
stride_b_g = cute::to_array<int64_t>(append(stride_b,
cute::size<NumSpatialDimensions + 1>(stride_b) * cute::size<NumSpatialDimensions + 1>(shape_b) / G));
stride_c_g = cute::to_array<int64_t>(append(stride_c, cute::size<0>(shape_c) / G));
}
else if constexpr (ConvOp == cutlass::conv::Operator::kWgrad) {
// shape_a_g = (k,q,p,z,n,g)
// shape_b_g = (c,w,h,d,n,g)
// shape_c_g = (c,s,r,k,t,g)
shape_a_g = cute::to_array<int32_t>(tuple_cat(
cute::make_shape(cute::size<0>(shape_a) / G),
cute::take<1,NumSpatialDimensions + 2>(shape_a),
cute::make_shape(G)));
shape_b_g = cute::to_array<int32_t>(tuple_cat(
cute::make_shape(cute::size<0>(shape_b) / G),
cute::take<1,NumSpatialDimensions + 2>(shape_b),
cute::make_shape(G)));
shape_c_g = cute::to_array<int32_t>(tuple_cat(
cute::take<0,NumSpatialDimensions + 1>(shape_c),
cute::make_shape(cute::size<NumSpatialDimensions + 1>(shape_c) / G, G)));
stride_a_g = cute::to_array<int64_t>(append(stride_a, cute::size<0>(shape_a) / G));
stride_b_g = cute::to_array<int64_t>(append(stride_b, cute::size<0>(shape_b) / G));
stride_c_g = cute::to_array<int64_t>(append(stride_c,
cute::size<NumSpatialDimensions + 1>(stride_c) * cute::size<NumSpatialDimensions + 1>(shape_c) / G));
}
return make_tuple(shape_a_g, shape_b_g, shape_c_g,
stride_a_g, stride_b_g, stride_c_g);
}
// Executes one test
bool run(
ProblemShape const& problem_shape,
@@ -263,7 +331,7 @@ struct ConvTestbed {
}
bool ret = initialize(problem_shape);
if (!ret) {
std::cerr << "initialize failed for the given problem_shape: \n";
return false;
@@ -382,15 +450,10 @@ struct ConvTestbed {
<< cudaGetErrorString(result);
// Create cute::Tensors using the logical rank-3 MNK multi-mode shapes the mainloop gives us
auto shape_mA = cute::reverse(problem_shape.shape_A);
auto shape_mB = cute::reverse(problem_shape.shape_B);
auto shape_mC = cute::reverse(problem_shape.shape_C);
auto [shape_mA, shape_mB, shape_mC, stride_mA, stride_mB, stride_mC] =
transform_shape_and_stride_with_groups(problem_shape);
auto shape_mBias = cute::make_shape(cute::size(cute::get<0>(problem_shape.get_shape_B())));
auto stride_mA = cute::reverse(problem_shape.stride_A);
auto stride_mB = cute::reverse(problem_shape.stride_B);
auto stride_mC = cute::reverse(problem_shape.stride_C);
auto mA = make_tensor(tensor_A.data().get(), make_layout(shape_mA, stride_mA));
auto mB = make_tensor(tensor_B.data().get(), make_layout(shape_mB, stride_mB));
auto mC = make_tensor(tensor_C.data().get(), make_layout(shape_mC, stride_mC));
@@ -1,4 +1,4 @@
# Copyright (c) 2013 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# Copyright (c) 2013 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: BSD-3-Clause
#
# Redistribution and use in source and binary forms, with or without
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2023 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2023 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2023 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2023 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2023 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2023 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2023 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2023 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2023 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2023 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
@@ -1,5 +1,5 @@
/***************************************************************************************************
* Copyright (c) 2023 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* Copyright (c) 2023 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without