leavelet/cutlass
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

v4.3 update. (#2709)

Browse Source 
* v4.3 update.

* Update the cute_dsl_api changelog's doc link

* Update version to 4.3.0

* Update the example link

* Update doc to encourage user to install DSL from requirements.txt

---------

Co-authored-by: Larry Wu <larwu@nvidia.com>
This commit is contained in:
Junkai-Wu
2025-10-22 02:26:30 +08:00
committed by GitHub
parent e6e2cc29f5
commit b1d6e2c9b3
244 changed files with 59272 additions and 10455 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
test/unit/conv/device_3x/dgrad/CMakeLists.txt
View File

@@ -32,6 +32,7 @@ add_custom_target(
cutlass_test_unit_conv_dgrad_device_tensorop_sm90
cutlass_test_unit_conv_dgrad_device_tensorop_sm100
cutlass_test_unit_conv_dgrad_device_tensorop_sm100_fusion
cutlass_test_unit_conv_dgrad_device_tensorop_sm100_streamk
)
cutlass_test_unit_add_executable(
@@ -88,4 +89,10 @@ cutlass_test_unit_add_executable(
sm100_conv3d_dgrad_implicit_gemm_f16_f16_f16_tensorop_f16_with_fusion.cu
)
cutlass_test_unit_add_executable_split_file(
cutlass_test_unit_conv_dgrad_device_tensorop_sm100_streamk
sm100_conv3d_dgrad_implicit_gemm_f16_f16_f16_tensorop_f16_streamk.cu
)
endif()

344
test/unit/conv/device_3x/dgrad/sm100_conv3d_dgrad_implicit_gemm_f16_f16_f16_tensorop_f16_streamk.cu Normal file
View File

@@ -0,0 +1,344 @@
/***************************************************************************************************
* Copyright (c) 2023 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/*! \file
\brief Tests for device-wide CONV interface
*/
#include "cutlass_unit_test.h"
#include "cutlass/cutlass.h"
#include "cute/tensor.hpp"
#include "cute/atom/mma_atom.hpp"
#include "cutlass/numeric_types.h"
#include "cutlass/conv/device/conv_universal_adapter.hpp"
#include "cutlass/conv/kernel/conv_universal.hpp"
#include "cutlass/conv/collective/collective_builder.hpp"
#include "cutlass/epilogue/collective/collective_builder.hpp"
#include "../testbed_conv.hpp"
using namespace cute;
#if defined(CUTLASS_ARCH_MMA_SM100_SUPPORTED)
//////////////////////////////////////////////////////////////////////////////////////////////////
// Static cluster
//////////////////////////////////////////////////////////////////////////////////////////////////
//
// Cluster tile shape 64x64x64
// Cluster shape 1x1x1
//
TEST(SM100_device_conv3d_dgrad_implicitgemm_f16ndhwc_f16ndhwc_f16ndhwc_tensor_op_f16, 64x64x64_1x1x1) {
using ElementAct = cutlass::half_t;
using ElementFlt = cutlass::half_t;
using ElementOut = cutlass::half_t;
using ElementAcc = cutlass::half_t;
using ElementCompute = float;
using MmaTileShape = Shape<_64, _64, Shape<_64>>;
using ClusterShape = Shape<_1,_1,_1>;
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
MmaTileShape, ClusterShape,
cutlass::epilogue::collective::EpilogueTileAuto,
ElementAcc, ElementCompute,
ElementAct, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
ElementOut, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
cutlass::epilogue::collective::EpilogueScheduleAuto
>::CollectiveOp;
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
cutlass::conv::Operator::kDgrad,
ElementAct, cutlass::layout::TensorNDHWC, 8,
ElementFlt, cutlass::layout::TensorNDHWC, 8,
ElementAcc,
MmaTileShape, ClusterShape,
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape = cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue,
cutlass::gemm::StreamKScheduler
>;
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
}
//
// Cluster tile shape 128x64x64
// Cluster shape 1x1x1
//
TEST(SM100_device_conv3d_dgrad_implicitgemm_f16ndhwc_f16ndhwc_f16ndhwc_tensor_op_f16, 128x64x64_1x1x1) {
using ElementAct = cutlass::half_t;
using ElementFlt = cutlass::half_t;
using ElementOut = cutlass::half_t;
using ElementAcc = cutlass::half_t;
using ElementCompute = float;
using MmaTileShape = Shape<_128, _64, Shape<_64>>;
using ClusterShape = Shape<_1,_1,_1>;
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
MmaTileShape, ClusterShape,
cutlass::epilogue::collective::EpilogueTileAuto,
ElementAcc, ElementCompute,
ElementAct, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
ElementOut, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
cutlass::epilogue::collective::EpilogueScheduleAuto
>::CollectiveOp;
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
cutlass::conv::Operator::kDgrad,
ElementAct, cutlass::layout::TensorNDHWC, 8,
ElementFlt, cutlass::layout::TensorNDHWC, 8,
ElementAcc,
MmaTileShape, ClusterShape,
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape = cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue,
cutlass::gemm::StreamKScheduler
>;
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
}
//
// Cluster tile shape 128x128x64
// Cluster shape 1x2x1
//
TEST(SM100_device_conv3d_dgrad_implicitgemm_f16ndhwc_f16ndhwc_f16ndhwc_tensor_op_f16, 128x128x64_1x2x1) {
using ElementAct = cutlass::half_t;
using ElementFlt = cutlass::half_t;
using ElementOut = cutlass::half_t;
using ElementAcc = cutlass::half_t;
using ElementCompute = float;
using MmaTileShape = Shape<_128, _64, Shape<_64>>;
using ClusterShape = Shape<_1,_2,_1>;
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
MmaTileShape, ClusterShape,
cutlass::epilogue::collective::EpilogueTileAuto,
ElementAcc, ElementCompute,
ElementAct, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
ElementOut, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
cutlass::epilogue::collective::EpilogueScheduleAuto
>::CollectiveOp;
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
cutlass::conv::Operator::kDgrad,
ElementAct, cutlass::layout::TensorNDHWC, 8,
ElementFlt, cutlass::layout::TensorNDHWC, 8,
ElementAcc,
MmaTileShape, ClusterShape,
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape = cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue,
cutlass::gemm::StreamKScheduler
>;
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
}
//
// Cluster tile shape 256x64x64
// Cluster shape 2x1x1
//
TEST(SM100_device_conv3d_dgrad_implicitgemm_f16ndhwc_f16ndhwc_f16ndhwc_tensor_op_f16, 256x64x64_2x1x1) {
using ElementAct = cutlass::half_t;
using ElementFlt = cutlass::half_t;
using ElementOut = cutlass::half_t;
using ElementAcc = cutlass::half_t;
using ElementCompute = float;
using MmaTileShape = Shape<_256, _64, Shape<_64>>;
using ClusterShape = Shape<_2,_1,_1>;
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
MmaTileShape, ClusterShape,
cutlass::epilogue::collective::EpilogueTileAuto,
ElementAcc, ElementCompute,
ElementAct, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
ElementOut, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
cutlass::epilogue::collective::EpilogueScheduleAuto
>::CollectiveOp;
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
cutlass::conv::Operator::kDgrad,
ElementAct, cutlass::layout::TensorNDHWC, 8,
ElementFlt, cutlass::layout::TensorNDHWC, 8,
ElementAcc,
MmaTileShape, ClusterShape,
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape = cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue,
cutlass::gemm::StreamKScheduler
>;
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
}
//
// Cluster tile shape 256x128x64
// Cluster shape 2x2x1
//
TEST(SM100_device_conv3d_dgrad_implicitgemm_f16ndhwc_f16ndhwc_f16ndhwc_tensor_op_f16, 256x128x64_2x2x1) {
using ElementAct = cutlass::half_t;
using ElementFlt = cutlass::half_t;
using ElementOut = cutlass::half_t;
using ElementAcc = cutlass::half_t;
using ElementCompute = float;
using MmaTileShape = Shape<_256, _64, Shape<_64>>;
using ClusterShape = Shape<_2,_2,_1>;
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
MmaTileShape, ClusterShape,
cutlass::epilogue::collective::EpilogueTileAuto,
ElementAcc, ElementCompute,
ElementAct, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
ElementOut, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
cutlass::epilogue::collective::EpilogueScheduleAuto
>::CollectiveOp;
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
cutlass::conv::Operator::kDgrad,
ElementAct, cutlass::layout::TensorNDHWC, 8,
ElementFlt, cutlass::layout::TensorNDHWC, 8,
ElementAcc,
MmaTileShape, ClusterShape,
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape = cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue,
cutlass::gemm::StreamKScheduler
>;
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
}
//////////////////////////////////////////////////////////////////////////////////////////////////
// Dynamic cluster
//////////////////////////////////////////////////////////////////////////////////////////////////
//
// CTA tile shape 64x64x64
// preferred cluster shape 2x4x1
// fallback cluster shape 2x2x1
//
TEST(SM100_device_conv3d_dgrad_implicitgemm_f16ndhwc_f16ndhwc_f16ndhwc_tensor_op_f16, 64x64x64_preferred_2x4x1_fallback_2x2x1) {
using ElementAct = cutlass::half_t;
using ElementFlt = cutlass::half_t;
using ElementOut = cutlass::half_t;
using ElementAcc = cutlass::half_t;
using ElementCompute = float;
using MmaTileShape = Shape<_64, _64, Shape<_64>>;
using ClusterShape = decltype(make_shape(int(0), int(0), Int<1>{}));
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
MmaTileShape, ClusterShape,
cutlass::epilogue::collective::EpilogueTileAuto,
ElementAcc, ElementCompute,
ElementAct, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
ElementOut, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
cutlass::epilogue::collective::EpilogueScheduleAuto
>::CollectiveOp;
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
cutlass::conv::Operator::kDgrad,
ElementAct, cutlass::layout::TensorNDHWC, 8,
ElementFlt, cutlass::layout::TensorNDHWC, 8,
ElementAcc,
MmaTileShape, ClusterShape,
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape = cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue,
cutlass::gemm::StreamKScheduler
>;
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>(1.0, 0.0, 0.0f, dim3(2,4,1), dim3(2,2,1)));
}
#endif // defined(CUTLASS_ARCH_MMA_SM100_SUPPORTED)

11
test/unit/conv/device_3x/fprop/CMakeLists.txt
View File

@@ -34,6 +34,7 @@ add_custom_target(
cutlass_test_unit_conv3d_fprop_device_tensorop_sm90
cutlass_test_unit_conv_fprop_device_tensorop_sm100
cutlass_test_unit_conv_fprop_device_tensorop_sm100_fusion
cutlass_test_unit_conv_fprop_device_tensorop_sm100_streamk
)
cutlass_test_unit_add_executable(
@@ -121,4 +122,14 @@ cutlass_test_unit_add_executable(
sm100_conv3d_fprop_implicit_gemm_tf32_tf32_f32_tensorop_f32_with_fusion.cu
)
cutlass_test_unit_add_executable_split_file(
cutlass_test_unit_conv_fprop_device_tensorop_sm100_streamk
# No batching of source to control compiler memory usage
BATCH_SOURCES ON
BATCH_SIZE 1
sm100_conv3d_fprop_implicit_gemm_f16_f16_f16_tensorop_f16_streamk.cu
)
endif()

350
test/unit/conv/device_3x/fprop/sm100_conv3d_fprop_implicit_gemm_f16_f16_f16_tensorop_f16_streamk.cu Normal file
View File

@@ -0,0 +1,350 @@
/***************************************************************************************************
* Copyright (c) 2023 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/*! \file
\brief Tests for device-wide CONV interface
*/
#include "cutlass_unit_test.h"
#include "cutlass/cutlass.h"
#include "cute/tensor.hpp"
#include "cute/atom/mma_atom.hpp"
#include "cutlass/numeric_types.h"
#include "cutlass/conv/device/conv_universal_adapter.hpp"
#include "cutlass/conv/kernel/conv_universal.hpp"
#include "cutlass/conv/collective/collective_builder.hpp"
#include "cutlass/epilogue/collective/collective_builder.hpp"
#include "../testbed_conv.hpp"
using namespace cute;
#if defined(CUTLASS_ARCH_MMA_SM100_SUPPORTED)
//////////////////////////////////////////////////////////////////////////////////////////////////
// Static cluster
//////////////////////////////////////////////////////////////////////////////////////////////////
//
// Cluster tile shape 64x64x64
// Cluster shape 1x1x1
//
TEST(SM100_device_conv3d_fprop_implicitgemm_f16ndhwc_f16ndhwc_f16ndhwc_tensor_op_f16, 64x64x64_1x1x1) {
using ElementAct = cutlass::half_t;
using ElementFlt = cutlass::half_t;
using ElementOut = cutlass::half_t;
using ElementAcc = cutlass::half_t;
using ElementCompute = float;
using MmaTileShape = Shape<_64, _64, Shape<_64>>;
using ClusterShape = Shape<_1,_1,_1>;
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
MmaTileShape, ClusterShape,
cutlass::epilogue::collective::EpilogueTileAuto,
ElementAcc, ElementCompute,
ElementAct, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
ElementOut, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
cutlass::epilogue::collective::EpilogueScheduleAuto
>::CollectiveOp;
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
cutlass::conv::Operator::kFprop,
ElementAct, cutlass::layout::TensorNDHWC, 8,
ElementFlt, cutlass::layout::TensorNDHWC, 8,
ElementAcc,
MmaTileShape, ClusterShape,
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape = cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue,
cutlass::gemm::StreamKScheduler
>;
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
}
//
// Cluster tile shape 128x64x64
// Cluster shape 1x1x1
//
TEST(SM100_device_conv3d_fprop_implicitgemm_f16ndhwc_f16ndhwc_f16ndhwc_tensor_op_f16, 128x64x64_1x1x1) {
using ElementAct = cutlass::half_t;
using ElementFlt = cutlass::half_t;
using ElementOut = cutlass::half_t;
using ElementAcc = cutlass::half_t;
using ElementCompute = float;
using MmaTileShape = Shape<_128, _64, Shape<_64>>;
using ClusterShape = Shape<_1,_1,_1>;
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
MmaTileShape, ClusterShape,
cutlass::epilogue::collective::EpilogueTileAuto,
ElementAcc, ElementCompute,
ElementAct, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
ElementOut, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
cutlass::epilogue::collective::EpilogueScheduleAuto
>::CollectiveOp;
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
cutlass::conv::Operator::kFprop,
ElementAct, cutlass::layout::TensorNDHWC, 8,
ElementFlt, cutlass::layout::TensorNDHWC, 8,
ElementAcc,
MmaTileShape, ClusterShape,
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape = cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue,
cutlass::gemm::StreamKScheduler
>;
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
}
//
// Cluster tile shape 128x128x64
// Cluster shape 1x2x1
//
TEST(SM100_device_conv3d_fprop_implicitgemm_f16ndhwc_f16ndhwc_f16ndhwc_tensor_op_f16, 128x128x64_1x2x1) {
using ElementAct = cutlass::half_t;
using ElementFlt = cutlass::half_t;
using ElementOut = cutlass::half_t;
using ElementAcc = cutlass::half_t;
using ElementCompute = float;
using MmaTileShape = Shape<_128, _64, Shape<_64>>;
using ClusterShape = Shape<_1,_2,_1>;
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
MmaTileShape, ClusterShape,
cutlass::epilogue::collective::EpilogueTileAuto,
ElementAcc, ElementCompute,
ElementAct, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
ElementOut, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
cutlass::epilogue::collective::EpilogueScheduleAuto
>::CollectiveOp;
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
cutlass::conv::Operator::kFprop,
ElementAct, cutlass::layout::TensorNDHWC, 8,
ElementFlt, cutlass::layout::TensorNDHWC, 8,
ElementAcc,
MmaTileShape, ClusterShape,
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape = cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue,
cutlass::gemm::StreamKScheduler
>;
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
}
//
// Cluster tile shape 256x64x64
// Cluster shape 2x1x1
//
TEST(SM100_device_conv3d_fprop_implicitgemm_f16ndhwc_f16ndhwc_f16ndhwc_tensor_op_f16, 256x64x64_2x1x1) {
using ElementAct = cutlass::half_t;
using ElementFlt = cutlass::half_t;
using ElementOut = cutlass::half_t;
using ElementAcc = cutlass::half_t;
using ElementCompute = float;
using MmaTileShape = Shape<_256, _64, Shape<_64>>;
using ClusterShape = Shape<_2,_1,_1>;
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
MmaTileShape, ClusterShape,
cutlass::epilogue::collective::EpilogueTileAuto,
ElementAcc, ElementCompute,
ElementAct, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
ElementOut, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
cutlass::epilogue::collective::EpilogueScheduleAuto
>::CollectiveOp;
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
cutlass::conv::Operator::kFprop,
ElementAct, cutlass::layout::TensorNDHWC, 8,
ElementFlt, cutlass::layout::TensorNDHWC, 8,
ElementAcc,
MmaTileShape, ClusterShape,
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape = cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue,
cutlass::gemm::StreamKScheduler
>;
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
}
//
// Cluster tile shape 256x128x64
// Cluster shape 2x2x1
//
TEST(SM100_device_conv3d_fprop_implicitgemm_f16ndhwc_f16ndhwc_f16ndhwc_tensor_op_f16, 256x128x64_2x2x1) {
using ElementAct = cutlass::half_t;
using ElementFlt = cutlass::half_t;
using ElementOut = cutlass::half_t;
using ElementAcc = cutlass::half_t;
using ElementCompute = float;
using MmaTileShape = Shape<_256, _64, Shape<_64>>;
using ClusterShape = Shape<_2,_2,_1>;
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
MmaTileShape, ClusterShape,
cutlass::epilogue::collective::EpilogueTileAuto,
ElementAcc, ElementCompute,
ElementAct, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
ElementOut, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
cutlass::epilogue::collective::EpilogueScheduleAuto
>::CollectiveOp;
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
cutlass::conv::Operator::kFprop,
ElementAct, cutlass::layout::TensorNDHWC, 8,
ElementFlt, cutlass::layout::TensorNDHWC, 8,
ElementAcc,
MmaTileShape, ClusterShape,
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape = cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue,
cutlass::gemm::StreamKScheduler
>;
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
}
//////////////////////////////////////////////////////////////////////////////////////////////////
// Dynamic cluster
//////////////////////////////////////////////////////////////////////////////////////////////////
//
// CTA tile shape 64x64x64
// preferred cluster shape 2x4x1
// fallback cluster shape 2x2x1
//
TEST(SM100_device_conv3d_fprop_implicitgemm_f16ndhwc_f16ndhwc_f16ndhwc_tensor_op_f16, 64x64x64_preferred_2x4x1_fallback_2x2x1) {
using ElementAct = cutlass::half_t;
using ElementFlt = cutlass::half_t;
using ElementOut = cutlass::half_t;
using ElementAcc = cutlass::half_t;
using ElementCompute = float;
using MmaTileShape = Shape<_64, _64, Shape<_64>>;
using ClusterShape = decltype(make_shape(int(0), int(0), Int<1>{}));
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
MmaTileShape, ClusterShape,
cutlass::epilogue::collective::EpilogueTileAuto,
ElementAcc, ElementCompute,
ElementAct, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
ElementOut, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
cutlass::epilogue::collective::EpilogueScheduleAuto
>::CollectiveOp;
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
cutlass::conv::Operator::kFprop,
ElementAct, cutlass::layout::TensorNDHWC, 8,
ElementFlt, cutlass::layout::TensorNDHWC, 8,
ElementAcc,
MmaTileShape, ClusterShape,
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape = cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue,
cutlass::gemm::StreamKScheduler
>;
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>(1.0, 0.0, 0.0f, dim3(2,4,1), dim3(2,2,1)));
}
#endif // defined(CUTLASS_ARCH_MMA_SM100_SUPPORTED)

21
test/unit/conv/device_3x/wgrad/CMakeLists.txt
View File

@@ -32,6 +32,9 @@ add_custom_target(
cutlass_test_unit_conv_wgrad_device_tensorop_sm90
cutlass_test_unit_conv_wgrad_device_tensorop_sm100
cutlass_test_unit_conv_wgrad_device_tensorop_sm100_fusion
cutlass_test_unit_conv1d_wgrad_device_tensorop_sm100_streamk
cutlass_test_unit_conv2d_wgrad_device_tensorop_sm100_streamk
cutlass_test_unit_conv3d_wgrad_device_tensorop_sm100_streamk
)
cutlass_test_unit_add_executable(
@@ -68,4 +71,22 @@ cutlass_test_unit_add_executable_split_file(
sm100_conv3d_wgrad_implicit_gemm_f16_f16_f16_tensorop_f16_with_fusion.cu
)
cutlass_test_unit_add_executable_split_file(
cutlass_test_unit_conv1d_wgrad_device_tensorop_sm100_streamk
sm100_conv1d_wgrad_implicit_gemm_f16_f16_f16_tensorop_f16_streamk.cu
)
cutlass_test_unit_add_executable_split_file(
cutlass_test_unit_conv2d_wgrad_device_tensorop_sm100_streamk
sm100_conv2d_wgrad_implicit_gemm_f16_f16_f16_tensorop_f16_streamk.cu
)
cutlass_test_unit_add_executable_split_file(
cutlass_test_unit_conv3d_wgrad_device_tensorop_sm100_streamk
sm100_conv3d_wgrad_implicit_gemm_f16_f16_f16_tensorop_f16_streamk.cu
)
endif()

344
test/unit/conv/device_3x/wgrad/sm100_conv1d_wgrad_implicit_gemm_f16_f16_f16_tensorop_f16_streamk.cu Normal file
View File

@@ -0,0 +1,344 @@
/***************************************************************************************************
* Copyright (c) 2024 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/*! \file
\brief Tests for device-wide CONV interface
*/
#include "cutlass_unit_test.h"
#include "cutlass/cutlass.h"
#include "cute/tensor.hpp"
#include "cute/atom/mma_atom.hpp"
#include "cutlass/numeric_types.h"
#include "cutlass/conv/device/conv_universal_adapter.hpp"
#include "cutlass/conv/kernel/conv_universal.hpp"
#include "cutlass/conv/collective/collective_builder.hpp"
#include "cutlass/epilogue/collective/collective_builder.hpp"
#include "../testbed_conv.hpp"
using namespace cute;
#if defined(CUTLASS_ARCH_MMA_SM100_SUPPORTED)
//////////////////////////////////////////////////////////////////////////////////////////////////
// Static cluster
//////////////////////////////////////////////////////////////////////////////////////////////////
//
// Cluster tile shape 64x64x64
// Cluster shape 1x1x1
//
TEST(SM100_device_conv1d_wgrad_implicitgemm_f16nwc_f16nwc_f16nwc_tensor_op_f16, 64x64x64_1x1x1) {
using ElementAct = cutlass::half_t;
using ElementFlt = cutlass::half_t;
using ElementOut = cutlass::half_t;
using ElementAcc = cutlass::half_t;
using ElementCompute = float;
using MmaTileShape = Shape<_64, Shape<_64>, Shape<_64>>;
using ClusterShape = Shape<_1,_1,_1>;
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
MmaTileShape, ClusterShape,
cutlass::epilogue::collective::EpilogueTileAuto,
ElementAcc, ElementCompute,
ElementAct, cutlass::layout::TensorKCS, 128 / cutlass::sizeof_bits<ElementAct>::value,
ElementOut, cutlass::layout::TensorKCS, 128 / cutlass::sizeof_bits<ElementOut>::value,
cutlass::epilogue::TmaWarpSpecialized1Sm
>::CollectiveOp;
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
cutlass::conv::Operator::kWgrad,
ElementAct, cutlass::layout::TensorNWC, 8,
ElementFlt, cutlass::layout::TensorNWC, 8,
ElementAcc,
MmaTileShape, ClusterShape,
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue,
cutlass::gemm::StreamKScheduler
>;
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
}
//
// Cluster tile shape 128x64x64
// Cluster shape 1x1x1
//
TEST(SM100_device_conv1d_wgrad_implicitgemm_f16nwc_f16nwc_f16nwc_tensor_op_f16, 128x64x64_1x1x1) {
using ElementAct = cutlass::half_t;
using ElementFlt = cutlass::half_t;
using ElementOut = cutlass::half_t;
using ElementAcc = cutlass::half_t;
using ElementCompute = float;
using MmaTileShape = Shape<_128, Shape<_64>, Shape<_64>>;
using ClusterShape = Shape<_1,_1,_1>;
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
MmaTileShape, ClusterShape,
cutlass::epilogue::collective::EpilogueTileAuto,
ElementAcc, ElementCompute,
ElementAct, cutlass::layout::TensorKCS, 128 / cutlass::sizeof_bits<ElementAct>::value,
ElementOut, cutlass::layout::TensorKCS, 128 / cutlass::sizeof_bits<ElementOut>::value,
cutlass::epilogue::NoSmemWarpSpecialized1Sm
>::CollectiveOp;
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
cutlass::conv::Operator::kWgrad,
ElementAct, cutlass::layout::TensorNWC, 8,
ElementFlt, cutlass::layout::TensorNWC, 8,
ElementAcc,
MmaTileShape, ClusterShape,
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue,
cutlass::gemm::StreamKScheduler
>;
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
}
//
// Cluster tile shape 128x128x64
// Cluster shape 1x2x1
//
TEST(SM100_device_conv1d_wgrad_implicitgemm_f16nwc_f16nwc_f16nwc_tensor_op_f16, 128x128x64_1x2x1) {
using ElementAct = cutlass::half_t;
using ElementFlt = cutlass::half_t;
using ElementOut = cutlass::half_t;
using ElementAcc = cutlass::half_t;
using ElementCompute = float;
using MmaTileShape = Shape<_128, Shape<_64>, Shape<_64>>;
using ClusterShape = Shape<_1,_2,_1>;
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
MmaTileShape, ClusterShape,
cutlass::epilogue::collective::EpilogueTileAuto,
ElementAcc, ElementCompute,
ElementAct, cutlass::layout::TensorKCS, 128 / cutlass::sizeof_bits<ElementAct>::value,
ElementOut, cutlass::layout::TensorKCS, 128 / cutlass::sizeof_bits<ElementOut>::value,
cutlass::epilogue::NoSmemWarpSpecialized1Sm
>::CollectiveOp;
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
cutlass::conv::Operator::kWgrad,
ElementAct, cutlass::layout::TensorNWC, 8,
ElementFlt, cutlass::layout::TensorNWC, 8,
ElementAcc,
MmaTileShape, ClusterShape,
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue,
cutlass::gemm::StreamKScheduler
>;
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
}
//
// Cluster tile shape 256x64x64
// Cluster shape 2x1x1
//
TEST(SM100_device_conv1d_wgrad_implicitgemm_f16nwc_f16nwc_f16nwc_tensor_op_f16, 256x64x64_2x1x1) {
using ElementAct = cutlass::half_t;
using ElementFlt = cutlass::half_t;
using ElementOut = cutlass::half_t;
using ElementAcc = cutlass::half_t;
using ElementCompute = float;
using MmaTileShape = Shape<_256, Shape<_64>, Shape<_64>>;
using ClusterShape = Shape<_2,_1,_1>;
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
MmaTileShape, ClusterShape,
cutlass::epilogue::collective::EpilogueTileAuto,
ElementAcc, ElementCompute,
ElementAct, cutlass::layout::TensorKCS, 128 / cutlass::sizeof_bits<ElementAct>::value,
ElementOut, cutlass::layout::TensorKCS, 128 / cutlass::sizeof_bits<ElementOut>::value,
cutlass::epilogue::NoSmemWarpSpecialized2Sm
>::CollectiveOp;
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
cutlass::conv::Operator::kWgrad,
ElementAct, cutlass::layout::TensorNWC, 8,
ElementFlt, cutlass::layout::TensorNWC, 8,
ElementAcc,
MmaTileShape, ClusterShape,
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue,
cutlass::gemm::StreamKScheduler
>;
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
}
//
// Cluster tile shape 256x128x64
// Cluster shape 2x2x1
//
TEST(SM100_device_conv1d_wgrad_implicitgemm_f16nwc_f16nwc_f16nwc_tensor_op_f16, 256x128x64_2x2x1) {
using ElementAct = cutlass::half_t;
using ElementFlt = cutlass::half_t;
using ElementOut = cutlass::half_t;
using ElementAcc = cutlass::half_t;
using ElementCompute = float;
using MmaTileShape = Shape<_256, Shape<_64>, Shape<_64>>;
using ClusterShape = Shape<_2,_2,_1>;
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
MmaTileShape, ClusterShape,
cutlass::epilogue::collective::EpilogueTileAuto,
ElementAcc, ElementCompute,
ElementAct, cutlass::layout::TensorKCS, 128 / cutlass::sizeof_bits<ElementAct>::value,
ElementOut, cutlass::layout::TensorKCS, 128 / cutlass::sizeof_bits<ElementOut>::value,
cutlass::epilogue::NoSmemWarpSpecialized2Sm
>::CollectiveOp;
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
cutlass::conv::Operator::kWgrad,
ElementAct, cutlass::layout::TensorNWC, 8,
ElementFlt, cutlass::layout::TensorNWC, 8,
ElementAcc,
MmaTileShape, ClusterShape,
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue,
cutlass::gemm::StreamKScheduler
>;
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
}
//////////////////////////////////////////////////////////////////////////////////////////////////
// Dynamic cluster
//////////////////////////////////////////////////////////////////////////////////////////////////
//
// CTA tile shape 64x64x64
// preferred cluster shape 2x4x1
// fallback cluster shape 2x2x1
//
TEST(SM100_device_conv1d_wgrad_implicitgemm_f16nwc_f16nwc_f16nwc_tensor_op_f16, 64x64x64_preferred_2x4x1_fallback_2x2x1) {
using ElementAct = cutlass::half_t;
using ElementFlt = cutlass::half_t;
using ElementOut = cutlass::half_t;
using ElementAcc = cutlass::half_t;
using ElementCompute = float;
using MmaTileShape = Shape<_64, Shape<_64>, Shape<_64>>;
using ClusterShape = decltype(make_shape(int(0), int(0), Int<1>{}));
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
MmaTileShape, ClusterShape,
cutlass::epilogue::collective::EpilogueTileAuto,
ElementAcc, ElementCompute,
ElementAct, cutlass::layout::TensorKCS, 128 / cutlass::sizeof_bits<ElementAct>::value,
ElementOut, cutlass::layout::TensorKCS, 128 / cutlass::sizeof_bits<ElementOut>::value,
cutlass::epilogue::collective::EpilogueScheduleAuto
>::CollectiveOp;
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
cutlass::conv::Operator::kWgrad,
ElementAct, cutlass::layout::TensorNWC, 8,
ElementFlt, cutlass::layout::TensorNWC, 8,
ElementAcc,
MmaTileShape, ClusterShape,
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue,
cutlass::gemm::StreamKScheduler
>;
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>(1.0, 0.0, 0.0f, dim3(2,4,1), dim3(2,2,1)));
}
#endif // defined(CUTLASS_ARCH_MMA_SM100_SUPPORTED)

344
test/unit/conv/device_3x/wgrad/sm100_conv2d_wgrad_implicit_gemm_f16_f16_f16_tensorop_f16_streamk.cu Normal file
View File

@@ -0,0 +1,344 @@
/***************************************************************************************************
* Copyright (c) 2024 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/*! \file
\brief Tests for device-wide CONV interface
*/
#include "cutlass_unit_test.h"
#include "cutlass/cutlass.h"
#include "cute/tensor.hpp"
#include "cute/atom/mma_atom.hpp"
#include "cutlass/numeric_types.h"
#include "cutlass/conv/device/conv_universal_adapter.hpp"
#include "cutlass/conv/kernel/conv_universal.hpp"
#include "cutlass/conv/collective/collective_builder.hpp"
#include "cutlass/epilogue/collective/collective_builder.hpp"
#include "../testbed_conv.hpp"
using namespace cute;
#if defined(CUTLASS_ARCH_MMA_SM100_SUPPORTED)
//////////////////////////////////////////////////////////////////////////////////////////////////
// Static cluster
//////////////////////////////////////////////////////////////////////////////////////////////////
//
// Cluster tile shape 64x64x64
// Cluster shape 1x1x1
//
TEST(SM100_device_conv2d_wgrad_implicitgemm_f16nhwc_f16nhwc_f16nhwc_tensor_op_f16, 64x64x64_1x1x1) {
using ElementAct = cutlass::half_t;
using ElementFlt = cutlass::half_t;
using ElementOut = cutlass::half_t;
using ElementAcc = cutlass::half_t;
using ElementCompute = float;
using MmaTileShape = Shape<_64, Shape<_64>, Shape<_64>>;
using ClusterShape = Shape<_1,_1,_1>;
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
MmaTileShape, ClusterShape,
cutlass::epilogue::collective::EpilogueTileAuto,
ElementAcc, ElementCompute,
ElementAct, cutlass::layout::TensorKCSR, 128 / cutlass::sizeof_bits<ElementAct>::value,
ElementOut, cutlass::layout::TensorKCSR, 128 / cutlass::sizeof_bits<ElementOut>::value,
cutlass::epilogue::TmaWarpSpecialized1Sm
>::CollectiveOp;
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
cutlass::conv::Operator::kWgrad,
ElementAct, cutlass::layout::TensorNHWC, 8,
ElementFlt, cutlass::layout::TensorNHWC, 8,
ElementAcc,
MmaTileShape, ClusterShape,
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue,
cutlass::gemm::StreamKScheduler
>;
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
}
//
// Cluster tile shape 128x64x64
// Cluster shape 1x1x1
//
TEST(SM100_device_conv2d_wgrad_implicitgemm_f16nhwc_f16nhwc_f16nhwc_tensor_op_f16, 128x64x64_1x1x1) {
using ElementAct = cutlass::half_t;
using ElementFlt = cutlass::half_t;
using ElementOut = cutlass::half_t;
using ElementAcc = cutlass::half_t;
using ElementCompute = float;
using MmaTileShape = Shape<_128, Shape<_64>, Shape<_64>>;
using ClusterShape = Shape<_1,_1,_1>;
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
MmaTileShape, ClusterShape,
cutlass::epilogue::collective::EpilogueTileAuto,
ElementAcc, ElementCompute,
ElementAct, cutlass::layout::TensorKCSR, 128 / cutlass::sizeof_bits<ElementAct>::value,
ElementOut, cutlass::layout::TensorKCSR, 128 / cutlass::sizeof_bits<ElementOut>::value,
cutlass::epilogue::NoSmemWarpSpecialized1Sm
>::CollectiveOp;
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
cutlass::conv::Operator::kWgrad,
ElementAct, cutlass::layout::TensorNHWC, 8,
ElementFlt, cutlass::layout::TensorNHWC, 8,
ElementAcc,
MmaTileShape, ClusterShape,
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue,
cutlass::gemm::StreamKScheduler
>;
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
}
//
// Cluster tile shape 128x128x64
// Cluster shape 1x2x1
//
TEST(SM100_device_conv2d_wgrad_implicitgemm_f16nhwc_f16nhwc_f16nhwc_tensor_op_f16, 128x128x64_1x2x1) {
using ElementAct = cutlass::half_t;
using ElementFlt = cutlass::half_t;
using ElementOut = cutlass::half_t;
using ElementAcc = cutlass::half_t;
using ElementCompute = float;
using MmaTileShape = Shape<_128, Shape<_64>, Shape<_64>>;
using ClusterShape = Shape<_1,_2,_1>;
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
MmaTileShape, ClusterShape,
cutlass::epilogue::collective::EpilogueTileAuto,
ElementAcc, ElementCompute,
ElementAct, cutlass::layout::TensorKCSR, 128 / cutlass::sizeof_bits<ElementAct>::value,
ElementOut, cutlass::layout::TensorKCSR, 128 / cutlass::sizeof_bits<ElementOut>::value,
cutlass::epilogue::NoSmemWarpSpecialized1Sm
>::CollectiveOp;
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
cutlass::conv::Operator::kWgrad,
ElementAct, cutlass::layout::TensorNHWC, 8,
ElementFlt, cutlass::layout::TensorNHWC, 8,
ElementAcc,
MmaTileShape, ClusterShape,
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue,
cutlass::gemm::StreamKScheduler
>;
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
}
//
// Cluster tile shape 256x64x64
// Cluster shape 2x1x1
//
TEST(SM100_device_conv2d_wgrad_implicitgemm_f16nhwc_f16nhwc_f16nhwc_tensor_op_f16, 256x64x64_2x1x1) {
using ElementAct = cutlass::half_t;
using ElementFlt = cutlass::half_t;
using ElementOut = cutlass::half_t;
using ElementAcc = cutlass::half_t;
using ElementCompute = float;
using MmaTileShape = Shape<_256, Shape<_64>, Shape<_64>>;
using ClusterShape = Shape<_2,_1,_1>;
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
MmaTileShape, ClusterShape,
cutlass::epilogue::collective::EpilogueTileAuto,
ElementAcc, ElementCompute,
ElementAct, cutlass::layout::TensorKCSR, 128 / cutlass::sizeof_bits<ElementAct>::value,
ElementOut, cutlass::layout::TensorKCSR, 128 / cutlass::sizeof_bits<ElementOut>::value,
cutlass::epilogue::NoSmemWarpSpecialized2Sm
>::CollectiveOp;
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
cutlass::conv::Operator::kWgrad,
ElementAct, cutlass::layout::TensorNHWC, 8,
ElementFlt, cutlass::layout::TensorNHWC, 8,
ElementAcc,
MmaTileShape, ClusterShape,
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue,
cutlass::gemm::StreamKScheduler
>;
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
}
//
// Cluster tile shape 256x128x64
// Cluster shape 2x2x1
//
TEST(SM100_device_conv2d_wgrad_implicitgemm_f16nhwc_f16nhwc_f16nhwc_tensor_op_f16, 256x128x64_2x2x1) {
using ElementAct = cutlass::half_t;
using ElementFlt = cutlass::half_t;
using ElementOut = cutlass::half_t;
using ElementAcc = cutlass::half_t;
using ElementCompute = float;
using MmaTileShape = Shape<_256, Shape<_64>, Shape<_64>>;
using ClusterShape = Shape<_2,_2,_1>;
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
MmaTileShape, ClusterShape,
cutlass::epilogue::collective::EpilogueTileAuto,
ElementAcc, ElementCompute,
ElementAct, cutlass::layout::TensorKCSR, 128 / cutlass::sizeof_bits<ElementAct>::value,
ElementOut, cutlass::layout::TensorKCSR, 128 / cutlass::sizeof_bits<ElementOut>::value,
cutlass::epilogue::NoSmemWarpSpecialized2Sm
>::CollectiveOp;
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
cutlass::conv::Operator::kWgrad,
ElementAct, cutlass::layout::TensorNHWC, 8,
ElementFlt, cutlass::layout::TensorNHWC, 8,
ElementAcc,
MmaTileShape, ClusterShape,
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue,
cutlass::gemm::StreamKScheduler
>;
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
}
//////////////////////////////////////////////////////////////////////////////////////////////////
// Dynamic cluster
//////////////////////////////////////////////////////////////////////////////////////////////////
//
// CTA tile shape 64x64x64
// preferred cluster shape 2x4x1
// fallback cluster shape 2x2x1
//
TEST(SM100_device_conv2d_wgrad_implicitgemm_f16nhwc_f16nhwc_f16nhwc_tensor_op_f16, 64x64x64_preferred_2x4x1_fallback_2x2x1) {
using ElementAct = cutlass::half_t;
using ElementFlt = cutlass::half_t;
using ElementOut = cutlass::half_t;
using ElementAcc = cutlass::half_t;
using ElementCompute = float;
using MmaTileShape = Shape<_64, Shape<_64>, Shape<_64>>;
using ClusterShape = decltype(make_shape(int(0), int(0), Int<1>{}));
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
MmaTileShape, ClusterShape,
cutlass::epilogue::collective::EpilogueTileAuto,
ElementAcc, ElementCompute,
ElementAct, cutlass::layout::TensorKCSR, 128 / cutlass::sizeof_bits<ElementAct>::value,
ElementOut, cutlass::layout::TensorKCSR, 128 / cutlass::sizeof_bits<ElementOut>::value,
cutlass::epilogue::collective::EpilogueScheduleAuto
>::CollectiveOp;
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
cutlass::conv::Operator::kWgrad,
ElementAct, cutlass::layout::TensorNHWC, 8,
ElementFlt, cutlass::layout::TensorNHWC, 8,
ElementAcc,
MmaTileShape, ClusterShape,
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue,
cutlass::gemm::StreamKScheduler
>;
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>(1.0, 0.0, 0.0f, dim3(2,4,1), dim3(2,2,1)));
}
#endif // defined(CUTLASS_ARCH_MMA_SM100_SUPPORTED)

250
test/unit/conv/device_3x/wgrad/sm100_conv3d_wgrad_implicit_gemm_f16_f16_f16_tensorop_f16_streamk.cu Normal file
View File

@@ -0,0 +1,250 @@
/***************************************************************************************************
* Copyright (c) 2024 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: BSD-3-Clause
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* 1. Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
*
* 3. Neither the name of the copyright holder nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/*! \file
\brief Tests for device-wide CONV interface
*/
#include "cutlass_unit_test.h"
#include "cutlass/cutlass.h"
#include "cute/tensor.hpp"
#include "cute/atom/mma_atom.hpp"
#include "cutlass/numeric_types.h"
#include "cutlass/conv/device/conv_universal_adapter.hpp"
#include "cutlass/conv/kernel/conv_universal.hpp"
#include "cutlass/conv/collective/collective_builder.hpp"
#include "cutlass/epilogue/collective/collective_builder.hpp"
#include "../testbed_conv.hpp"
using namespace cute;
#if defined(CUTLASS_ARCH_MMA_SM100_SUPPORTED)
//////////////////////////////////////////////////////////////////////////////////////////////////
// Static cluster
//////////////////////////////////////////////////////////////////////////////////////////////////
//
// Cluster tile shape 64x64x64
// Cluster shape 1x1x1
//
TEST(SM100_device_conv3d_wgrad_implicitgemm_f16ndhwc_f16ndhwc_f16ndhwc_tensor_op_f16, 64x64x64_1x1x1) {
using ElementAct = cutlass::half_t;
using ElementFlt = cutlass::half_t;
using ElementOut = cutlass::half_t;
using ElementAcc = cutlass::half_t;
using ElementCompute = float;
using MmaTileShape = Shape<_64, Shape<_64>, Shape<_64>>;
using ClusterShape = Shape<_1,_1,_1>;
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
MmaTileShape, ClusterShape,
cutlass::epilogue::collective::EpilogueTileAuto,
ElementAcc, ElementCompute,
ElementAct, cutlass::layout::TensorKCSRT, 128 / cutlass::sizeof_bits<ElementAct>::value,
ElementOut, cutlass::layout::TensorKCSRT, 128 / cutlass::sizeof_bits<ElementOut>::value,
cutlass::epilogue::TmaWarpSpecialized1Sm
>::CollectiveOp;
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
cutlass::conv::Operator::kWgrad,
ElementAct, cutlass::layout::TensorNDHWC, 8,
ElementFlt, cutlass::layout::TensorNDHWC, 8,
ElementAcc,
MmaTileShape, ClusterShape,
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue,
cutlass::gemm::StreamKScheduler
>;
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
}
//
// Cluster tile shape 128x64x64
// Cluster shape 1x1x1
//
TEST(SM100_device_conv3d_wgrad_implicitgemm_f16ndhwc_f16ndhwc_f16ndhwc_tensor_op_f16, 128x64x64_1x1x1) {
using ElementAct = cutlass::half_t;
using ElementFlt = cutlass::half_t;
using ElementOut = cutlass::half_t;
using ElementAcc = cutlass::half_t;
using ElementCompute = float;
using MmaTileShape = Shape<_128, Shape<_64>, Shape<_64>>;
using ClusterShape = Shape<_1,_1,_1>;
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
MmaTileShape, ClusterShape,
cutlass::epilogue::collective::EpilogueTileAuto,
ElementAcc, ElementCompute,
ElementAct, cutlass::layout::TensorKCSRT, 128 / cutlass::sizeof_bits<ElementAct>::value,
ElementOut, cutlass::layout::TensorKCSRT, 128 / cutlass::sizeof_bits<ElementOut>::value,
cutlass::epilogue::NoSmemWarpSpecialized1Sm
>::CollectiveOp;
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
cutlass::conv::Operator::kWgrad,
ElementAct, cutlass::layout::TensorNDHWC, 8,
ElementFlt, cutlass::layout::TensorNDHWC, 8,
ElementAcc,
MmaTileShape, ClusterShape,
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue,
cutlass::gemm::StreamKScheduler
>;
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
}
//
// Cluster tile shape 256x128x64
// Cluster shape 2x2x1
//
TEST(SM100_device_conv3d_wgrad_implicitgemm_f16ndhwc_f16ndhwc_f16ndhwc_tensor_op_f16, 256x128x64_2x2x1) {
using ElementAct = cutlass::half_t;
using ElementFlt = cutlass::half_t;
using ElementOut = cutlass::half_t;
using ElementAcc = cutlass::half_t;
using ElementCompute = float;
using MmaTileShape = Shape<_256, Shape<_64>, Shape<_64>>;
using ClusterShape = Shape<_2,_2,_1>;
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
MmaTileShape, ClusterShape,
cutlass::epilogue::collective::EpilogueTileAuto,
ElementAcc, ElementCompute,
ElementAct, cutlass::layout::TensorKCSRT, 128 / cutlass::sizeof_bits<ElementAct>::value,
ElementOut, cutlass::layout::TensorKCSRT, 128 / cutlass::sizeof_bits<ElementOut>::value,
cutlass::epilogue::NoSmemWarpSpecialized2Sm
>::CollectiveOp;
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
cutlass::conv::Operator::kWgrad,
ElementAct, cutlass::layout::TensorNDHWC, 8,
ElementFlt, cutlass::layout::TensorNDHWC, 8,
ElementAcc,
MmaTileShape, ClusterShape,
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue,
cutlass::gemm::StreamKScheduler
>;
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
}
//////////////////////////////////////////////////////////////////////////////////////////////////
// Dynamic cluster
//////////////////////////////////////////////////////////////////////////////////////////////////
//
// CTA tile shape 64x64x64
// preferred cluster shape 2x4x1
// fallback cluster shape 2x2x1
//
TEST(SM100_device_conv3d_wgrad_implicitgemm_f16ndhwc_f16ndhwc_f16ndhwc_tensor_op_f16, 64x64x64_preferred_2x4x1_fallback_2x2x1) {
using ElementAct = cutlass::half_t;
using ElementFlt = cutlass::half_t;
using ElementOut = cutlass::half_t;
using ElementAcc = cutlass::half_t;
using ElementCompute = float;
using MmaTileShape = Shape<_64, Shape<_64>, Shape<_64>>;
using ClusterShape = decltype(make_shape(int(0), int(0), Int<1>{}));
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
MmaTileShape, ClusterShape,
cutlass::epilogue::collective::EpilogueTileAuto,
ElementAcc, ElementCompute,
ElementAct, cutlass::layout::TensorKCSRT, 128 / cutlass::sizeof_bits<ElementAct>::value,
ElementOut, cutlass::layout::TensorKCSRT, 128 / cutlass::sizeof_bits<ElementOut>::value,
cutlass::epilogue::collective::EpilogueScheduleAuto
>::CollectiveOp;
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
cutlass::conv::Operator::kWgrad,
ElementAct, cutlass::layout::TensorNDHWC, 8,
ElementFlt, cutlass::layout::TensorNDHWC, 8,
ElementAcc,
MmaTileShape, ClusterShape,
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
cutlass::conv::collective::KernelScheduleAuto
>::CollectiveOp;
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
ProblemShape,
CollectiveMainloop,
CollectiveEpilogue,
cutlass::gemm::StreamKScheduler
>;
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>(1.0, 0.0, 0.0f, dim3(2,4,1), dim3(2,2,1)));
}
#endif // defined(CUTLASS_ARCH_MMA_SM100_SUPPORTED)

5
test/unit/core/uint128.cu
View File

@@ -58,6 +58,11 @@ TEST(uint128_t, host_arithmetic) {
T y = j;
EXPECT_TRUE(static_cast<uint64_t>(x + y) == (i + j));
EXPECT_TRUE(static_cast<uint64_t>(x * static_cast<uint64_t>(y)) == (i * j));
if (j != 0) {
EXPECT_TRUE(static_cast<uint64_t>(x / static_cast<uint64_t>(y)) == (i / j));
}
}
}

4
test/unit/cute/ampere/cooperative_gemm.cu
View File

@@ -537,6 +537,8 @@ TEST(SM80_CuTe_Ampere, CooperativeGemmLDSMx2) {
SM75_U32x2_LDSM_N{});
}
#if (__CUDACC_VER_MAJOR__ > 12) || (__CUDACC_VER_MAJOR__ == 12 && __CUDACC_VER_MINOR__ >= 8)
TEST(SM89_CuTe_Ada, CooperativeGemm_e4m3e4m3f32_MMA) {
using TA = cutlass::float_e4m3_t;
using TB = cutlass::float_e4m3_t;
@@ -609,8 +611,6 @@ TEST(SM89_CuTe_Ada, CooperativeGemm_e5m2e5m2f32_MMA) {
test_cooperative_gemm_col_major_layout<thread_block_size, MaxVecBits, TA, TB, TC>(shape_mnk, tiled_mma);
}
#if (__CUDACC_VER_MAJOR__ > 12) || (__CUDACC_VER_MAJOR__ == 12 && __CUDACC_VER_MINOR__ >= 8)
TEST(SM89_CuTe_Ada, CooperativeGemm_e4m3e4m3f16_MMA) {
using TA = cutlass::float_e4m3_t;
using TB = cutlass::float_e4m3_t;

4
test/unit/gemm/device/CMakeLists.txt
View File

@@ -645,6 +645,7 @@ endif()
if (CUTLASS_NVCC_DEVICE_COMPILE)
if (NOT CUTLASS_NVCC_ARCHS MATCHES 101|101a|101f|110|110a|110f)
cutlass_test_unit_gemm_device_add_executable(
cutlass_test_unit_gemm_device_blas3
@@ -808,7 +809,7 @@ cutlass_test_unit_gemm_device_add_executable(
hemm_cf64_cf64_cf64_tensor_op_f64_sm90.cu
)
if (NOT CUTLASS_NVCC_ARCHS MATCHES 100f|101|101a|101f|103|103a|103f)
if (NOT CUTLASS_NVCC_ARCHS MATCHES 100f|101|101a|101f|110|110a|110f|103|103a|103f)
cutlass_test_unit_gemm_device_add_executable(
cutlass_test_unit_gemm_device_blas3_gaussian
@@ -844,6 +845,7 @@ cutlass_test_unit_gemm_device_add_executable(
her2k_cf64n_cf64n_tensor_op_f64_grouped_sm80.cu
her2k_cf64h_cf64n_tensor_op_f64_grouped_sm80.cu
)
endif()
endif()