v4.0 update. (#2371)
This commit is contained in:
Junkai-Wu
@@ -30,6 +30,8 @@ add_custom_target(
|
||||
cutlass_test_unit_conv_dgrad_device
|
||||
DEPENDS
|
||||
cutlass_test_unit_conv_dgrad_device_tensorop_sm90
|
||||
cutlass_test_unit_conv_dgrad_device_tensorop_sm100
|
||||
cutlass_test_unit_conv_dgrad_device_tensorop_sm100_fusion
|
||||
)
|
||||
|
||||
cutlass_test_unit_add_executable(
|
||||
@@ -47,3 +49,43 @@ cutlass_test_unit_add_executable(
|
||||
sm90_conv3d_dgrad_implicit_gemm_f16_f16_f32_tensorop_f32.cu
|
||||
)
|
||||
|
||||
if (CUTLASS_NVCC_ARCHS MATCHES 100a)
|
||||
|
||||
set(cutlass_test_unit_conv_dgrad_device_tensorop_sm100_kernels
|
||||
sm100_conv2d_dgrad_implicit_gemm_f8_f8_f8_tensorop_f32.cu
|
||||
sm100_conv2d_dgrad_implicit_gemm_f8_f8_bf16_tensorop_f32.cu
|
||||
sm100_conv2d_dgrad_implicit_gemm_f8_f8_f16_tensorop_f32.cu
|
||||
sm100_conv2d_dgrad_implicit_gemm_f8_f8_f32_tensorop_f32.cu
|
||||
|
||||
sm100_conv3d_dgrad_implicit_gemm_f8_f8_f8_tensorop_f32.cu
|
||||
sm100_conv3d_dgrad_implicit_gemm_f8_f8_bf16_tensorop_f32.cu
|
||||
sm100_conv3d_dgrad_implicit_gemm_f8_f8_f16_tensorop_f32.cu
|
||||
sm100_conv3d_dgrad_implicit_gemm_f8_f8_f32_tensorop_f32.cu
|
||||
|
||||
sm100_conv1d_dgrad_implicit_gemm_f16_f16_f16_tensorop_f16.cu
|
||||
sm100_conv2d_dgrad_implicit_gemm_f16_f16_f16_tensorop_f16.cu
|
||||
sm100_conv3d_dgrad_implicit_gemm_f16_f16_f16_tensorop_f16.cu
|
||||
|
||||
sm100_conv1d_dgrad_implicit_gemm_f16_f16_f32_tensorop_f32.cu
|
||||
sm100_conv2d_dgrad_implicit_gemm_f16_f16_f32_tensorop_f32.cu
|
||||
sm100_conv3d_dgrad_implicit_gemm_f16_f16_f32_tensorop_f32.cu
|
||||
)
|
||||
|
||||
# Add the executable
|
||||
cutlass_test_unit_add_executable(
|
||||
cutlass_test_unit_conv_dgrad_device_tensorop_sm100
|
||||
${cutlass_test_unit_conv_dgrad_device_tensorop_sm100_kernels}
|
||||
)
|
||||
|
||||
cutlass_test_unit_add_executable(
|
||||
cutlass_test_unit_conv_dgrad_device_tensorop_sm100_fusion
|
||||
|
||||
sm100_conv2d_dgrad_implicit_gemm_f8_f8_f16_tensorop_f32_with_fusion.cu
|
||||
sm100_conv3d_dgrad_implicit_gemm_f8_f8_f16_tensorop_f32_with_fusion.cu
|
||||
|
||||
sm100_conv1d_dgrad_implicit_gemm_f16_f16_f16_tensorop_f16_with_fusion.cu
|
||||
sm100_conv2d_dgrad_implicit_gemm_f16_f16_f16_tensorop_f16_with_fusion.cu
|
||||
sm100_conv3d_dgrad_implicit_gemm_f16_f16_f16_tensorop_f16_with_fusion.cu
|
||||
)
|
||||
|
||||
endif()
|
||||
|
||||
+338
@@ -0,0 +1,338 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2023 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/*! \file
|
||||
\brief Tests for device-wide CONV interface
|
||||
*/
|
||||
|
||||
#include "cutlass_unit_test.h"
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cute/tensor.hpp"
|
||||
#include "cute/atom/mma_atom.hpp"
|
||||
|
||||
#include "cutlass/numeric_types.h"
|
||||
|
||||
#include "cutlass/conv/device/conv_universal_adapter.hpp"
|
||||
#include "cutlass/conv/kernel/conv_universal.hpp"
|
||||
#include "cutlass/conv/collective/collective_builder.hpp"
|
||||
#include "cutlass/epilogue/collective/collective_builder.hpp"
|
||||
|
||||
#include "../testbed_conv.hpp"
|
||||
using namespace cute;
|
||||
|
||||
#if defined(CUTLASS_ARCH_MMA_SM100_SUPPORTED)
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// Static cluster
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
//
|
||||
// Cluster tile shape 64x64x64
|
||||
// Cluster shape 1x1x1
|
||||
//
|
||||
TEST(SM100_device_conv1d_dgrad_implicitgemm_f16nwc_f16nwc_f16nwc_tensor_op_f16, 64x64x64_1x1x1) {
|
||||
using ElementAct = cutlass::half_t;
|
||||
using ElementFlt = cutlass::half_t;
|
||||
using ElementOut = cutlass::half_t;
|
||||
using ElementAcc = cutlass::half_t;
|
||||
using ElementCompute = float;
|
||||
using MmaTileShape = Shape<_64, _64, Shape<_64>>;
|
||||
using ClusterShape = Shape<_1,_1,_1>;
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAcc, ElementCompute,
|
||||
ElementAct, cutlass::layout::TensorNWC, 128 / sizeof_bits_v<ElementAct>,
|
||||
ElementOut, cutlass::layout::TensorNWC, 128 / sizeof_bits_v<ElementOut>,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::conv::Operator::kDgrad,
|
||||
ElementAct, cutlass::layout::TensorNWC, 8,
|
||||
ElementFlt, cutlass::layout::TensorNWC, 8,
|
||||
ElementAcc,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::conv::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
|
||||
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
|
||||
ProblemShape,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
|
||||
|
||||
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
|
||||
}
|
||||
|
||||
//
|
||||
// Cluster tile shape 128x64x64
|
||||
// Cluster shape 1x1x1
|
||||
//
|
||||
TEST(SM100_device_conv1d_dgrad_implicitgemm_f16nwc_f16nwc_f16nwc_tensor_op_f16, 128x64x64_1x1x1) {
|
||||
using ElementAct = cutlass::half_t;
|
||||
using ElementFlt = cutlass::half_t;
|
||||
using ElementOut = cutlass::half_t;
|
||||
using ElementAcc = cutlass::half_t;
|
||||
using ElementCompute = float;
|
||||
using MmaTileShape = Shape<_128, _64, Shape<_64>>;
|
||||
using ClusterShape = Shape<_1,_1,_1>;
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAcc, ElementCompute,
|
||||
ElementAct, cutlass::layout::TensorNWC, 128 / sizeof_bits_v<ElementAct>,
|
||||
ElementOut, cutlass::layout::TensorNWC, 128 / sizeof_bits_v<ElementOut>,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::conv::Operator::kDgrad,
|
||||
ElementAct, cutlass::layout::TensorNWC, 8,
|
||||
ElementFlt, cutlass::layout::TensorNWC, 8,
|
||||
ElementAcc,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::conv::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
|
||||
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
|
||||
ProblemShape,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
|
||||
|
||||
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
|
||||
}
|
||||
|
||||
//
|
||||
// Cluster tile shape 128x128x64
|
||||
// Cluster shape 1x2x1
|
||||
//
|
||||
TEST(SM100_device_conv1d_dgrad_implicitgemm_f16nwc_f16nwc_f16nwc_tensor_op_f16, 128x128x64_1x2x1) {
|
||||
using ElementAct = cutlass::half_t;
|
||||
using ElementFlt = cutlass::half_t;
|
||||
using ElementOut = cutlass::half_t;
|
||||
using ElementAcc = cutlass::half_t;
|
||||
using ElementCompute = float;
|
||||
using MmaTileShape = Shape<_128, _64, Shape<_64>>;
|
||||
using ClusterShape = Shape<_1,_2,_1>;
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAcc, ElementCompute,
|
||||
ElementAct, cutlass::layout::TensorNWC, 128 / sizeof_bits_v<ElementAct>,
|
||||
ElementOut, cutlass::layout::TensorNWC, 128 / sizeof_bits_v<ElementOut>,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::conv::Operator::kDgrad,
|
||||
ElementAct, cutlass::layout::TensorNWC, 8,
|
||||
ElementFlt, cutlass::layout::TensorNWC, 8,
|
||||
ElementAcc,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::conv::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
|
||||
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
|
||||
ProblemShape,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
|
||||
|
||||
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
|
||||
}
|
||||
|
||||
//
|
||||
// Cluster tile shape 256x64x64
|
||||
// Cluster shape 2x1x1
|
||||
//
|
||||
TEST(SM100_device_conv1d_dgrad_implicitgemm_f16nwc_f16nwc_f16nwc_tensor_op_f16, 256x64x64_2x1x1) {
|
||||
using ElementAct = cutlass::half_t;
|
||||
using ElementFlt = cutlass::half_t;
|
||||
using ElementOut = cutlass::half_t;
|
||||
using ElementAcc = cutlass::half_t;
|
||||
using ElementCompute = float;
|
||||
using MmaTileShape = Shape<_256, _64, Shape<_64>>;
|
||||
using ClusterShape = Shape<_2,_1,_1>;
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAcc, ElementCompute,
|
||||
ElementAct, cutlass::layout::TensorNWC, 128 / sizeof_bits_v<ElementAct>,
|
||||
ElementOut, cutlass::layout::TensorNWC, 128 / sizeof_bits_v<ElementOut>,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::conv::Operator::kDgrad,
|
||||
ElementAct, cutlass::layout::TensorNWC, 8,
|
||||
ElementFlt, cutlass::layout::TensorNWC, 8,
|
||||
ElementAcc,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::conv::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
|
||||
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
|
||||
ProblemShape,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
|
||||
|
||||
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
|
||||
}
|
||||
|
||||
//
|
||||
// Cluster tile shape 256x128x64
|
||||
// Cluster shape 2x2x1
|
||||
//
|
||||
TEST(SM100_device_conv1d_dgrad_implicitgemm_f16nwc_f16nwc_f16nwc_tensor_op_f16, 256x128x64_2x2x1) {
|
||||
using ElementAct = cutlass::half_t;
|
||||
using ElementFlt = cutlass::half_t;
|
||||
using ElementOut = cutlass::half_t;
|
||||
using ElementAcc = cutlass::half_t;
|
||||
using ElementCompute = float;
|
||||
using MmaTileShape = Shape<_256, _64, Shape<_64>>;
|
||||
using ClusterShape = Shape<_2,_2,_1>;
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAcc, ElementCompute,
|
||||
ElementAct, cutlass::layout::TensorNWC, 128 / sizeof_bits_v<ElementAct>,
|
||||
ElementOut, cutlass::layout::TensorNWC, 128 / sizeof_bits_v<ElementOut>,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::conv::Operator::kDgrad,
|
||||
ElementAct, cutlass::layout::TensorNWC, 8,
|
||||
ElementFlt, cutlass::layout::TensorNWC, 8,
|
||||
ElementAcc,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::conv::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
|
||||
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
|
||||
ProblemShape,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
|
||||
|
||||
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// Dynamic cluster
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
//
|
||||
// CTA tile shape 64x64x64
|
||||
// preferred cluster shape 2x4x1
|
||||
// fallback cluster shape 2x2x1
|
||||
//
|
||||
TEST(SM100_device_conv1d_dgrad_implicitgemm_f16nwc_f16nwc_f16nwc_tensor_op_f16, 64x64x64_preferred_2x4x1_fallback_2x2x1) {
|
||||
using ElementAct = cutlass::half_t;
|
||||
using ElementFlt = cutlass::half_t;
|
||||
using ElementOut = cutlass::half_t;
|
||||
using ElementAcc = cutlass::half_t;
|
||||
using ElementCompute = float;
|
||||
using MmaTileShape = Shape<_64, _64, Shape<_64>>;
|
||||
using ClusterShape = decltype(make_shape(int(0), int(0), Int<1>{}));
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAcc, ElementCompute,
|
||||
ElementAct, cutlass::layout::TensorNWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
|
||||
ElementOut, cutlass::layout::TensorNWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::conv::Operator::kDgrad,
|
||||
ElementAct, cutlass::layout::TensorNWC, 8,
|
||||
ElementFlt, cutlass::layout::TensorNWC, 8,
|
||||
ElementAcc,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::conv::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
|
||||
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
|
||||
ProblemShape,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
|
||||
|
||||
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>(1.0, 0.0, 0.0f, dim3(2,4,1), dim3(2,2,1)));
|
||||
}
|
||||
|
||||
#endif // defined(CUTLASS_ARCH_MMA_SM100_SUPPORTED)
|
||||
+190
@@ -0,0 +1,190 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2023 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/*! \file
|
||||
\brief Tests for device-wide CONV interface
|
||||
*/
|
||||
|
||||
#include "cutlass_unit_test.h"
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cute/tensor.hpp"
|
||||
#include "cute/atom/mma_atom.hpp"
|
||||
|
||||
#include "cutlass/numeric_types.h"
|
||||
|
||||
#include "cutlass/conv/device/conv_universal_adapter.hpp"
|
||||
#include "cutlass/conv/kernel/conv_universal.hpp"
|
||||
#include "cutlass/conv/collective/collective_builder.hpp"
|
||||
#include "cutlass/epilogue/collective/collective_builder.hpp"
|
||||
|
||||
#include "../testbed_conv.hpp"
|
||||
using namespace cute;
|
||||
|
||||
#if defined(CUTLASS_ARCH_MMA_SM100_SUPPORTED)
|
||||
|
||||
// alpha != 1 && beta != 0
|
||||
TEST(SM100_device_conv1d_dgrad_implicitgemm_f16nwc_f16nwc_f16nwc_tensor_op_f16, 64x64x64_1x1x1_alpha_beta) {
|
||||
using ElementAct = cutlass::half_t;
|
||||
using ElementFlt = cutlass::half_t;
|
||||
using ElementOut = cutlass::half_t;
|
||||
using ElementAcc = cutlass::half_t;
|
||||
using ElementCompute = float;
|
||||
using MmaTileShape = Shape<_64, _64, Shape<_64>>;
|
||||
using ClusterShape = Shape<_1,_1,_1>;
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAcc, ElementCompute,
|
||||
ElementAct, cutlass::layout::TensorNWC, 128 / sizeof_bits_v<ElementAct>,
|
||||
ElementOut, cutlass::layout::TensorNWC, 128 / sizeof_bits_v<ElementOut>,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::conv::Operator::kDgrad,
|
||||
ElementAct, cutlass::layout::TensorNWC, 8,
|
||||
ElementFlt, cutlass::layout::TensorNWC, 8,
|
||||
ElementAcc,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::conv::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
|
||||
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
|
||||
ProblemShape,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
|
||||
|
||||
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>(2.0, 1.0));
|
||||
}
|
||||
|
||||
// alpha != 1 && beta != 0 && bias
|
||||
TEST(SM100_device_conv1d_dgrad_implicitgemm_f16nwc_f16nwc_f16nwc_tensor_op_f16, 64x64x64_1x1x1_alpha_beta_bias) {
|
||||
using ElementAct = cutlass::half_t;
|
||||
using ElementFlt = cutlass::half_t;
|
||||
using ElementOut = cutlass::half_t;
|
||||
using ElementAcc = cutlass::half_t;
|
||||
using ElementCompute = float;
|
||||
using ElementBias = cutlass::half_t;
|
||||
using MmaTileShape = Shape<_64, _64, Shape<_64>>;
|
||||
using ClusterShape = Shape<_1,_1,_1>;
|
||||
|
||||
using FusionOperation = cutlass::epilogue::fusion::LinCombPerColBias<
|
||||
ElementOut, ElementCompute, ElementBias>;
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAcc, ElementCompute,
|
||||
ElementAct, cutlass::layout::TensorNWC, 128 / sizeof_bits_v<ElementAct>,
|
||||
ElementOut, cutlass::layout::TensorNWC, 128 / sizeof_bits_v<ElementOut>,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto,
|
||||
FusionOperation
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::conv::Operator::kDgrad,
|
||||
ElementAct, cutlass::layout::TensorNWC, 8,
|
||||
ElementFlt, cutlass::layout::TensorNWC, 8,
|
||||
ElementAcc,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::conv::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
|
||||
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
|
||||
ProblemShape,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
|
||||
|
||||
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>(2.0, 1.0));
|
||||
}
|
||||
|
||||
// alpha != 1 && beta != 0 && bias && relu
|
||||
TEST(SM100_device_conv1d_dgrad_implicitgemm_f16nwc_f16nwc_f16nwc_tensor_op_f16, 64x64x64_1x1x1_alpha_beta_bias_relu) {
|
||||
using ElementAct = cutlass::half_t;
|
||||
using ElementFlt = cutlass::half_t;
|
||||
using ElementOut = cutlass::half_t;
|
||||
using ElementAcc = cutlass::half_t;
|
||||
using ElementCompute = float;
|
||||
using ElementBias = cutlass::half_t;
|
||||
using MmaTileShape = Shape<_64, _64, Shape<_64>>;
|
||||
using ClusterShape = Shape<_1,_1,_1>;
|
||||
|
||||
using FusionOperation = cutlass::epilogue::fusion::LinCombPerColBiasEltAct<
|
||||
cutlass::epilogue::thread::ReLu, ElementOut, ElementCompute, ElementBias>;
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAcc, ElementCompute,
|
||||
ElementAct, cutlass::layout::TensorNWC, 128 / sizeof_bits_v<ElementAct>,
|
||||
ElementOut, cutlass::layout::TensorNWC, 128 / sizeof_bits_v<ElementOut>,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto,
|
||||
FusionOperation
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::conv::Operator::kDgrad,
|
||||
ElementAct, cutlass::layout::TensorNWC, 8,
|
||||
ElementFlt, cutlass::layout::TensorNWC, 8,
|
||||
ElementAcc,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::conv::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
|
||||
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
|
||||
ProblemShape,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
|
||||
|
||||
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>(2.0, 1.0));
|
||||
}
|
||||
|
||||
#endif // defined(CUTLASS_ARCH_MMA_SM100_SUPPORTED)
|
||||
+338
@@ -0,0 +1,338 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2023 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/*! \file
|
||||
\brief Tests for device-wide CONV interface
|
||||
*/
|
||||
|
||||
#include "cutlass_unit_test.h"
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cute/tensor.hpp"
|
||||
#include "cute/atom/mma_atom.hpp"
|
||||
|
||||
#include "cutlass/numeric_types.h"
|
||||
|
||||
#include "cutlass/conv/device/conv_universal_adapter.hpp"
|
||||
#include "cutlass/conv/kernel/conv_universal.hpp"
|
||||
#include "cutlass/conv/collective/collective_builder.hpp"
|
||||
#include "cutlass/epilogue/collective/collective_builder.hpp"
|
||||
|
||||
#include "../testbed_conv.hpp"
|
||||
using namespace cute;
|
||||
|
||||
#if defined(CUTLASS_ARCH_MMA_SM100_SUPPORTED)
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// Static cluster
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
//
|
||||
// Cluster tile shape 64x64x64
|
||||
// Cluster shape 1x1x1
|
||||
//
|
||||
TEST(SM100_device_conv1d_dgrad_implicitgemm_f16nwc_f16nwc_f32nwc_tensor_op_f32, 64x64x64_1x1x1) {
|
||||
using ElementAct = cutlass::half_t;
|
||||
using ElementFlt = cutlass::half_t;
|
||||
using ElementOut = float;
|
||||
using ElementAcc = float;
|
||||
using ElementCompute = float;
|
||||
using MmaTileShape = Shape<_64, _64, Shape<_64>>;
|
||||
using ClusterShape = Shape<_1,_1,_1>;
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAcc, ElementCompute,
|
||||
ElementAct, cutlass::layout::TensorNWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
|
||||
ElementOut, cutlass::layout::TensorNWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::conv::Operator::kDgrad,
|
||||
ElementAct, cutlass::layout::TensorNWC, 8,
|
||||
ElementFlt, cutlass::layout::TensorNWC, 8,
|
||||
ElementAcc,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::conv::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
|
||||
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
|
||||
ProblemShape,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
|
||||
|
||||
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
|
||||
}
|
||||
|
||||
//
|
||||
// Cluster tile shape 128x64x64
|
||||
// Cluster shape 1x1x1
|
||||
//
|
||||
TEST(SM100_device_conv1d_dgrad_implicitgemm_f16nwc_f16nwc_f32nwc_tensor_op_f32, 128x64x64_1x1x1) {
|
||||
using ElementAct = cutlass::half_t;
|
||||
using ElementFlt = cutlass::half_t;
|
||||
using ElementOut = float;
|
||||
using ElementAcc = float;
|
||||
using ElementCompute = float;
|
||||
using MmaTileShape = Shape<_128, _64, Shape<_64>>;
|
||||
using ClusterShape = Shape<_1,_1,_1>;
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAcc, ElementCompute,
|
||||
ElementAct, cutlass::layout::TensorNWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
|
||||
ElementOut, cutlass::layout::TensorNWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::conv::Operator::kDgrad,
|
||||
ElementAct, cutlass::layout::TensorNWC, 8,
|
||||
ElementFlt, cutlass::layout::TensorNWC, 8,
|
||||
ElementAcc,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::conv::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
|
||||
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
|
||||
ProblemShape,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
|
||||
|
||||
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
|
||||
}
|
||||
|
||||
//
|
||||
// Cluster tile shape 128x128x64
|
||||
// Cluster shape 1x2x1
|
||||
//
|
||||
TEST(SM100_device_conv1d_dgrad_implicitgemm_f16nwc_f16nwc_f32nwc_tensor_op_f32, 128x128x64_1x2x1) {
|
||||
using ElementAct = cutlass::half_t;
|
||||
using ElementFlt = cutlass::half_t;
|
||||
using ElementOut = float;
|
||||
using ElementAcc = float;
|
||||
using ElementCompute = float;
|
||||
using MmaTileShape = Shape<_128, _64, Shape<_64>>;
|
||||
using ClusterShape = Shape<_1,_2,_1>;
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAcc, ElementCompute,
|
||||
ElementAct, cutlass::layout::TensorNWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
|
||||
ElementOut, cutlass::layout::TensorNWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::conv::Operator::kDgrad,
|
||||
ElementAct, cutlass::layout::TensorNWC, 8,
|
||||
ElementFlt, cutlass::layout::TensorNWC, 8,
|
||||
ElementAcc,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::conv::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
|
||||
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
|
||||
ProblemShape,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
|
||||
|
||||
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
|
||||
}
|
||||
|
||||
//
|
||||
// Cluster tile shape 256x64x64
|
||||
// Cluster shape 2x1x1
|
||||
//
|
||||
TEST(SM100_device_conv1d_dgrad_implicitgemm_f16nwc_f16nwc_f32nwc_tensor_op_f32, 256x64x64_2x1x1) {
|
||||
using ElementAct = cutlass::half_t;
|
||||
using ElementFlt = cutlass::half_t;
|
||||
using ElementOut = float;
|
||||
using ElementAcc = float;
|
||||
using ElementCompute = float;
|
||||
using MmaTileShape = Shape<_256, _64, Shape<_64>>;
|
||||
using ClusterShape = Shape<_2,_1,_1>;
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAcc, ElementCompute,
|
||||
ElementAct, cutlass::layout::TensorNWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
|
||||
ElementOut, cutlass::layout::TensorNWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::conv::Operator::kDgrad,
|
||||
ElementAct, cutlass::layout::TensorNWC, 8,
|
||||
ElementFlt, cutlass::layout::TensorNWC, 8,
|
||||
ElementAcc,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::conv::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
|
||||
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
|
||||
ProblemShape,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
|
||||
|
||||
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
|
||||
}
|
||||
|
||||
//
|
||||
// Cluster tile shape 256x128x64
|
||||
// Cluster shape 2x2x1
|
||||
//
|
||||
TEST(SM100_device_conv1d_dgrad_implicitgemm_f16nwc_f16nwc_f32nwc_tensor_op_f32, 256x128x64_2x2x1) {
|
||||
using ElementAct = cutlass::half_t;
|
||||
using ElementFlt = cutlass::half_t;
|
||||
using ElementOut = float;
|
||||
using ElementAcc = float;
|
||||
using ElementCompute = float;
|
||||
using MmaTileShape = Shape<_256, _64, Shape<_64>>;
|
||||
using ClusterShape = Shape<_2,_2,_1>;
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAcc, ElementCompute,
|
||||
ElementAct, cutlass::layout::TensorNWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
|
||||
ElementOut, cutlass::layout::TensorNWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::conv::Operator::kDgrad,
|
||||
ElementAct, cutlass::layout::TensorNWC, 8,
|
||||
ElementFlt, cutlass::layout::TensorNWC, 8,
|
||||
ElementAcc,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::conv::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
|
||||
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
|
||||
ProblemShape,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
|
||||
|
||||
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// Dynamic cluster
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
//
|
||||
// CTA tile shape 64x64x64
|
||||
// preferred cluster shape 2x4x1
|
||||
// fallback cluster shape 2x2x1
|
||||
//
|
||||
TEST(SM100_device_conv1d_dgrad_implicitgemm_f16nwc_f16nwc_f32nwc_tensor_op_f32, 64x64x64_preferred_2x4x1_fallback_2x2x1) {
|
||||
using ElementAct = cutlass::half_t;
|
||||
using ElementFlt = cutlass::half_t;
|
||||
using ElementOut = float;
|
||||
using ElementAcc = float;
|
||||
using ElementCompute = float;
|
||||
using MmaTileShape = Shape<_64, _64, Shape<_64>>;
|
||||
using ClusterShape = decltype(make_shape(int(0), int(0), Int<1>{}));
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAcc, ElementCompute,
|
||||
ElementAct, cutlass::layout::TensorNWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
|
||||
ElementOut, cutlass::layout::TensorNWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::conv::Operator::kFprop,
|
||||
ElementAct, cutlass::layout::TensorNWC, 8,
|
||||
ElementFlt, cutlass::layout::TensorNWC, 8,
|
||||
ElementAcc,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::conv::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
|
||||
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
|
||||
ProblemShape,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
|
||||
|
||||
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>(1.0, 0.0, 0.0f, dim3(2,4,1), dim3(2,2,1)));
|
||||
}
|
||||
|
||||
#endif // defined(CUTLASS_ARCH_MMA_SM100_SUPPORTED)
|
||||
+338
@@ -0,0 +1,338 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2023 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/*! \file
|
||||
\brief Tests for device-wide CONV interface
|
||||
*/
|
||||
|
||||
#include "cutlass_unit_test.h"
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cute/tensor.hpp"
|
||||
#include "cute/atom/mma_atom.hpp"
|
||||
|
||||
#include "cutlass/numeric_types.h"
|
||||
|
||||
#include "cutlass/conv/device/conv_universal_adapter.hpp"
|
||||
#include "cutlass/conv/kernel/conv_universal.hpp"
|
||||
#include "cutlass/conv/collective/collective_builder.hpp"
|
||||
#include "cutlass/epilogue/collective/collective_builder.hpp"
|
||||
|
||||
#include "../testbed_conv.hpp"
|
||||
using namespace cute;
|
||||
|
||||
#if defined(CUTLASS_ARCH_MMA_SM100_SUPPORTED)
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// Static cluster
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
//
|
||||
// Cluster tile shape 64x64x64
|
||||
// Cluster shape 1x1x1
|
||||
//
|
||||
TEST(SM100_device_conv2d_dgrad_implicitgemm_f16nhwc_f16nhwc_f16nhwc_tensor_op_f16, 64x64x64_1x1x1) {
|
||||
using ElementAct = cutlass::half_t;
|
||||
using ElementFlt = cutlass::half_t;
|
||||
using ElementOut = cutlass::half_t;
|
||||
using ElementAcc = cutlass::half_t;
|
||||
using ElementCompute = float;
|
||||
using MmaTileShape = Shape<_64, _64, Shape<_64>>;
|
||||
using ClusterShape = Shape<_1,_1,_1>;
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAcc, ElementCompute,
|
||||
ElementAct, cutlass::layout::TensorNHWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
|
||||
ElementOut, cutlass::layout::TensorNHWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::conv::Operator::kDgrad,
|
||||
ElementAct, cutlass::layout::TensorNHWC, 8,
|
||||
ElementFlt, cutlass::layout::TensorNHWC, 8,
|
||||
ElementAcc,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::conv::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
|
||||
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
|
||||
ProblemShape,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
|
||||
|
||||
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
|
||||
}
|
||||
|
||||
//
|
||||
// Cluster tile shape 128x64x64
|
||||
// Cluster shape 1x1x1
|
||||
//
|
||||
TEST(SM100_device_conv2d_dgrad_implicitgemm_f16nhwc_f16nhwc_f16nhwc_tensor_op_f16, 128x64x64_1x1x1) {
|
||||
using ElementAct = cutlass::half_t;
|
||||
using ElementFlt = cutlass::half_t;
|
||||
using ElementOut = cutlass::half_t;
|
||||
using ElementAcc = cutlass::half_t;
|
||||
using ElementCompute = float;
|
||||
using MmaTileShape = Shape<_128, _64, Shape<_64>>;
|
||||
using ClusterShape = Shape<_1,_1,_1>;
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAcc, ElementCompute,
|
||||
ElementAct, cutlass::layout::TensorNHWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
|
||||
ElementOut, cutlass::layout::TensorNHWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::conv::Operator::kDgrad,
|
||||
ElementAct, cutlass::layout::TensorNHWC, 8,
|
||||
ElementFlt, cutlass::layout::TensorNHWC, 8,
|
||||
ElementAcc,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::conv::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
|
||||
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
|
||||
ProblemShape,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
|
||||
|
||||
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
|
||||
}
|
||||
|
||||
//
|
||||
// Cluster tile shape 128x128x64
|
||||
// Cluster shape 1x2x1
|
||||
//
|
||||
TEST(SM100_device_conv2d_dgrad_implicitgemm_f16nhwc_f16nhwc_f16nhwc_tensor_op_f16, 128x128x64_1x2x1) {
|
||||
using ElementAct = cutlass::half_t;
|
||||
using ElementFlt = cutlass::half_t;
|
||||
using ElementOut = cutlass::half_t;
|
||||
using ElementAcc = cutlass::half_t;
|
||||
using ElementCompute = float;
|
||||
using MmaTileShape = Shape<_128, _64, Shape<_64>>;
|
||||
using ClusterShape = Shape<_1,_2,_1>;
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAcc, ElementCompute,
|
||||
ElementAct, cutlass::layout::TensorNHWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
|
||||
ElementOut, cutlass::layout::TensorNHWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::conv::Operator::kDgrad,
|
||||
ElementAct, cutlass::layout::TensorNHWC, 8,
|
||||
ElementFlt, cutlass::layout::TensorNHWC, 8,
|
||||
ElementAcc,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::conv::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
|
||||
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
|
||||
ProblemShape,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
|
||||
|
||||
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
|
||||
}
|
||||
|
||||
//
|
||||
// Cluster tile shape 256x64x64
|
||||
// Cluster shape 2x1x1
|
||||
//
|
||||
TEST(SM100_device_conv2d_dgrad_implicitgemm_f16nhwc_f16nhwc_f16nhwc_tensor_op_f16, 256x64x64_2x1x1) {
|
||||
using ElementAct = cutlass::half_t;
|
||||
using ElementFlt = cutlass::half_t;
|
||||
using ElementOut = cutlass::half_t;
|
||||
using ElementAcc = cutlass::half_t;
|
||||
using ElementCompute = float;
|
||||
using MmaTileShape = Shape<_256, _64, Shape<_64>>;
|
||||
using ClusterShape = Shape<_2,_1,_1>;
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAcc, ElementCompute,
|
||||
ElementAct, cutlass::layout::TensorNHWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
|
||||
ElementOut, cutlass::layout::TensorNHWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::conv::Operator::kDgrad,
|
||||
ElementAct, cutlass::layout::TensorNHWC, 8,
|
||||
ElementFlt, cutlass::layout::TensorNHWC, 8,
|
||||
ElementAcc,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::conv::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
|
||||
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
|
||||
ProblemShape,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
|
||||
|
||||
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
|
||||
}
|
||||
|
||||
//
|
||||
// Cluster tile shape 256x128x64
|
||||
// Cluster shape 2x2x1
|
||||
//
|
||||
TEST(SM100_device_conv2d_dgrad_implicitgemm_f16nhwc_f16nhwc_f16nhwc_tensor_op_f16, 256x128x64_2x2x1) {
|
||||
using ElementAct = cutlass::half_t;
|
||||
using ElementFlt = cutlass::half_t;
|
||||
using ElementOut = cutlass::half_t;
|
||||
using ElementAcc = cutlass::half_t;
|
||||
using ElementCompute = float;
|
||||
using MmaTileShape = Shape<_256, _64, Shape<_64>>;
|
||||
using ClusterShape = Shape<_2,_2,_1>;
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAcc, ElementCompute,
|
||||
ElementAct, cutlass::layout::TensorNHWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
|
||||
ElementOut, cutlass::layout::TensorNHWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::conv::Operator::kDgrad,
|
||||
ElementAct, cutlass::layout::TensorNHWC, 8,
|
||||
ElementFlt, cutlass::layout::TensorNHWC, 8,
|
||||
ElementAcc,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::conv::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
|
||||
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
|
||||
ProblemShape,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
|
||||
|
||||
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// Dynamic cluster
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
//
|
||||
// CTA tile shape 64x64x64
|
||||
// preferred cluster shape 2x4x1
|
||||
// fallback cluster shape 2x2x1
|
||||
//
|
||||
TEST(SM100_device_conv2d_dgrad_implicitgemm_f16nhwc_f16nhwc_f16nhwc_tensor_op_f16, 64x64x64_preferred_2x4x1_fallback_2x2x1) {
|
||||
using ElementAct = cutlass::half_t;
|
||||
using ElementFlt = cutlass::half_t;
|
||||
using ElementOut = cutlass::half_t;
|
||||
using ElementAcc = cutlass::half_t;
|
||||
using ElementCompute = float;
|
||||
using MmaTileShape = Shape<_64, _64, Shape<_64>>;
|
||||
using ClusterShape = decltype(make_shape(int(0), int(0), Int<1>{}));
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAcc, ElementCompute,
|
||||
ElementAct, cutlass::layout::TensorNHWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
|
||||
ElementOut, cutlass::layout::TensorNHWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::conv::Operator::kDgrad,
|
||||
ElementAct, cutlass::layout::TensorNHWC, 8,
|
||||
ElementFlt, cutlass::layout::TensorNHWC, 8,
|
||||
ElementAcc,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::conv::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
|
||||
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
|
||||
ProblemShape,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
|
||||
|
||||
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>(1.0, 0.0, 0.0f, dim3(2,4,1), dim3(2,2,1)));
|
||||
}
|
||||
|
||||
#endif // defined(CUTLASS_ARCH_MMA_SM100_SUPPORTED)
|
||||
+190
@@ -0,0 +1,190 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2023 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/*! \file
|
||||
\brief Tests for device-wide CONV interface
|
||||
*/
|
||||
|
||||
#include "cutlass_unit_test.h"
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cute/tensor.hpp"
|
||||
#include "cute/atom/mma_atom.hpp"
|
||||
|
||||
#include "cutlass/numeric_types.h"
|
||||
|
||||
#include "cutlass/conv/device/conv_universal_adapter.hpp"
|
||||
#include "cutlass/conv/kernel/conv_universal.hpp"
|
||||
#include "cutlass/conv/collective/collective_builder.hpp"
|
||||
#include "cutlass/epilogue/collective/collective_builder.hpp"
|
||||
|
||||
#include "../testbed_conv.hpp"
|
||||
using namespace cute;
|
||||
|
||||
#if defined(CUTLASS_ARCH_MMA_SM100_SUPPORTED)
|
||||
|
||||
// alpha != 1 && beta != 0
|
||||
TEST(SM100_device_conv2d_dgrad_implicitgemm_f16nhwc_f16nhwc_f16nhwc_tensor_op_f16, 64x64x64_1x1x1_alpha_beta) {
|
||||
using ElementAct = cutlass::half_t;
|
||||
using ElementFlt = cutlass::half_t;
|
||||
using ElementOut = cutlass::half_t;
|
||||
using ElementAcc = cutlass::half_t;
|
||||
using ElementCompute = float;
|
||||
using MmaTileShape = Shape<_64, _64, Shape<_64>>;
|
||||
using ClusterShape = Shape<_1,_1,_1>;
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAcc, ElementCompute,
|
||||
ElementAct, cutlass::layout::TensorNHWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
|
||||
ElementOut, cutlass::layout::TensorNHWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::conv::Operator::kDgrad,
|
||||
ElementAct, cutlass::layout::TensorNHWC, 8,
|
||||
ElementFlt, cutlass::layout::TensorNHWC, 8,
|
||||
ElementAcc,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::conv::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
|
||||
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
|
||||
ProblemShape,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
|
||||
|
||||
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>(2.0, 1.0));
|
||||
}
|
||||
|
||||
// alpha != 1 && beta != 0 && bias
|
||||
TEST(SM100_device_conv2d_dgrad_implicitgemm_f16nhwc_f16nhwc_f16nhwc_tensor_op_f16, 64x64x64_1x1x1_alpha_beta_bias) {
|
||||
using ElementAct = cutlass::half_t;
|
||||
using ElementFlt = cutlass::half_t;
|
||||
using ElementOut = cutlass::half_t;
|
||||
using ElementAcc = cutlass::half_t;
|
||||
using ElementCompute = float;
|
||||
using ElementBias = cutlass::half_t;
|
||||
using MmaTileShape = Shape<_64, _64, Shape<_64>>;
|
||||
using ClusterShape = Shape<_1,_1,_1>;
|
||||
|
||||
using FusionOperation = cutlass::epilogue::fusion::LinCombPerColBias<
|
||||
ElementOut, ElementCompute, ElementBias>;
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAcc, ElementCompute,
|
||||
ElementAct, cutlass::layout::TensorNHWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
|
||||
ElementOut, cutlass::layout::TensorNHWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto,
|
||||
FusionOperation
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::conv::Operator::kDgrad,
|
||||
ElementAct, cutlass::layout::TensorNHWC, 8,
|
||||
ElementFlt, cutlass::layout::TensorNHWC, 8,
|
||||
ElementAcc,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::conv::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
|
||||
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
|
||||
ProblemShape,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
|
||||
|
||||
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>(2.0, 1.0));
|
||||
}
|
||||
|
||||
// alpha != 1 && beta != 0 && bias && relu
|
||||
TEST(SM100_device_conv2d_dgrad_implicitgemm_f16nhwc_f16nhwc_f16nhwc_tensor_op_f16, 64x64x64_1x1x1_alpha_beta_bias_relu) {
|
||||
using ElementAct = cutlass::half_t;
|
||||
using ElementFlt = cutlass::half_t;
|
||||
using ElementOut = cutlass::half_t;
|
||||
using ElementAcc = cutlass::half_t;
|
||||
using ElementCompute = float;
|
||||
using ElementBias = cutlass::half_t;
|
||||
using MmaTileShape = Shape<_64, _64, Shape<_64>>;
|
||||
using ClusterShape = Shape<_1,_1,_1>;
|
||||
|
||||
using FusionOperation = cutlass::epilogue::fusion::LinCombPerColBiasEltAct<
|
||||
cutlass::epilogue::thread::ReLu, ElementOut, ElementCompute, ElementBias>;
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAcc, ElementCompute,
|
||||
ElementAct, cutlass::layout::TensorNHWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
|
||||
ElementOut, cutlass::layout::TensorNHWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto,
|
||||
FusionOperation
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::conv::Operator::kDgrad,
|
||||
ElementAct, cutlass::layout::TensorNHWC, 8,
|
||||
ElementFlt, cutlass::layout::TensorNHWC, 8,
|
||||
ElementAcc,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::conv::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
|
||||
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
|
||||
ProblemShape,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
|
||||
|
||||
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>(2.0, 1.0));
|
||||
}
|
||||
|
||||
#endif // defined(CUTLASS_ARCH_MMA_SM100_SUPPORTED)
|
||||
+338
@@ -0,0 +1,338 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2023 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/*! \file
|
||||
\brief Tests for device-wide CONV interface
|
||||
*/
|
||||
|
||||
#include "cutlass_unit_test.h"
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cute/tensor.hpp"
|
||||
#include "cute/atom/mma_atom.hpp"
|
||||
|
||||
#include "cutlass/numeric_types.h"
|
||||
|
||||
#include "cutlass/conv/device/conv_universal_adapter.hpp"
|
||||
#include "cutlass/conv/kernel/conv_universal.hpp"
|
||||
#include "cutlass/conv/collective/collective_builder.hpp"
|
||||
#include "cutlass/epilogue/collective/collective_builder.hpp"
|
||||
|
||||
#include "../testbed_conv.hpp"
|
||||
using namespace cute;
|
||||
|
||||
#if defined(CUTLASS_ARCH_MMA_SM100_SUPPORTED)
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// Static cluster
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
//
|
||||
// Cluster tile shape 64x64x64
|
||||
// Cluster shape 1x1x1
|
||||
//
|
||||
TEST(SM100_device_conv2d_dgrad_implicitgemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f32, 64x64x64_1x1x1) {
|
||||
using ElementAct = cutlass::half_t;
|
||||
using ElementFlt = cutlass::half_t;
|
||||
using ElementOut = float;
|
||||
using ElementAcc = float;
|
||||
using ElementCompute = float;
|
||||
using MmaTileShape = Shape<_64, _64, Shape<_64>>;
|
||||
using ClusterShape = Shape<_1,_1,_1>;
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAcc, ElementCompute,
|
||||
ElementAct, cutlass::layout::TensorNHWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
|
||||
ElementOut, cutlass::layout::TensorNHWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::conv::Operator::kDgrad,
|
||||
ElementAct, cutlass::layout::TensorNHWC, 8,
|
||||
ElementFlt, cutlass::layout::TensorNHWC, 8,
|
||||
ElementAcc,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::conv::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
|
||||
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
|
||||
ProblemShape,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
|
||||
|
||||
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
|
||||
}
|
||||
|
||||
//
|
||||
// Cluster tile shape 128x64x64
|
||||
// Cluster shape 1x1x1
|
||||
//
|
||||
TEST(SM100_device_conv2d_dgrad_implicitgemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f32, 128x64x64_1x1x1) {
|
||||
using ElementAct = cutlass::half_t;
|
||||
using ElementFlt = cutlass::half_t;
|
||||
using ElementOut = float;
|
||||
using ElementAcc = float;
|
||||
using ElementCompute = float;
|
||||
using MmaTileShape = Shape<_128, _64, Shape<_64>>;
|
||||
using ClusterShape = Shape<_1,_1,_1>;
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAcc, ElementCompute,
|
||||
ElementAct, cutlass::layout::TensorNHWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
|
||||
ElementOut, cutlass::layout::TensorNHWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::conv::Operator::kDgrad,
|
||||
ElementAct, cutlass::layout::TensorNHWC, 8,
|
||||
ElementFlt, cutlass::layout::TensorNHWC, 8,
|
||||
ElementAcc,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::conv::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
|
||||
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
|
||||
ProblemShape,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
|
||||
|
||||
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
|
||||
}
|
||||
|
||||
//
|
||||
// Cluster tile shape 128x128x64
|
||||
// Cluster shape 1x2x1
|
||||
//
|
||||
TEST(SM100_device_conv2d_dgrad_implicitgemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f32, 128x128x64_1x2x1) {
|
||||
using ElementAct = cutlass::half_t;
|
||||
using ElementFlt = cutlass::half_t;
|
||||
using ElementOut = float;
|
||||
using ElementAcc = float;
|
||||
using ElementCompute = float;
|
||||
using MmaTileShape = Shape<_128, _64, Shape<_64>>;
|
||||
using ClusterShape = Shape<_1,_2,_1>;
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAcc, ElementCompute,
|
||||
ElementAct, cutlass::layout::TensorNHWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
|
||||
ElementOut, cutlass::layout::TensorNHWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::conv::Operator::kDgrad,
|
||||
ElementAct, cutlass::layout::TensorNHWC, 8,
|
||||
ElementFlt, cutlass::layout::TensorNHWC, 8,
|
||||
ElementAcc,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::conv::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
|
||||
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
|
||||
ProblemShape,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
|
||||
|
||||
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
|
||||
}
|
||||
|
||||
//
|
||||
// Cluster tile shape 256x64x64
|
||||
// Cluster shape 2x1x1
|
||||
//
|
||||
TEST(SM100_device_conv2d_dgrad_implicitgemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f32, 256x64x64_2x1x1) {
|
||||
using ElementAct = cutlass::half_t;
|
||||
using ElementFlt = cutlass::half_t;
|
||||
using ElementOut = float;
|
||||
using ElementAcc = float;
|
||||
using ElementCompute = float;
|
||||
using MmaTileShape = Shape<_256, _64, Shape<_64>>;
|
||||
using ClusterShape = Shape<_2,_1,_1>;
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAcc, ElementCompute,
|
||||
ElementAct, cutlass::layout::TensorNHWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
|
||||
ElementOut, cutlass::layout::TensorNHWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::conv::Operator::kDgrad,
|
||||
ElementAct, cutlass::layout::TensorNHWC, 8,
|
||||
ElementFlt, cutlass::layout::TensorNHWC, 8,
|
||||
ElementAcc,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::conv::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
|
||||
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
|
||||
ProblemShape,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
|
||||
|
||||
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
|
||||
}
|
||||
|
||||
//
|
||||
// Cluster tile shape 256x128x64
|
||||
// Cluster shape 2x2x1
|
||||
//
|
||||
TEST(SM100_device_conv2d_dgrad_implicitgemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f32, 256x128x64_2x2x1) {
|
||||
using ElementAct = cutlass::half_t;
|
||||
using ElementFlt = cutlass::half_t;
|
||||
using ElementOut = float;
|
||||
using ElementAcc = float;
|
||||
using ElementCompute = float;
|
||||
using MmaTileShape = Shape<_256, _64, Shape<_64>>;
|
||||
using ClusterShape = Shape<_2,_2,_1>;
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAcc, ElementCompute,
|
||||
ElementAct, cutlass::layout::TensorNHWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
|
||||
ElementOut, cutlass::layout::TensorNHWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::conv::Operator::kDgrad,
|
||||
ElementAct, cutlass::layout::TensorNHWC, 8,
|
||||
ElementFlt, cutlass::layout::TensorNHWC, 8,
|
||||
ElementAcc,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::conv::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
|
||||
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
|
||||
ProblemShape,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
|
||||
|
||||
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// Dynamic cluster
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
//
|
||||
// CTA tile shape 64x64x64
|
||||
// preferred cluster shape 2x4x1
|
||||
// fallback cluster shape 2x2x1
|
||||
//
|
||||
TEST(SM100_device_conv2d_dgrad_implicitgemm_f16nhwc_f16nhwc_f32nhwc_tensor_op_f32, 64x64x64_preferred_2x4x1_fallback_2x2x1) {
|
||||
using ElementAct = cutlass::half_t;
|
||||
using ElementFlt = cutlass::half_t;
|
||||
using ElementOut = float;
|
||||
using ElementAcc = float;
|
||||
using ElementCompute = float;
|
||||
using MmaTileShape = Shape<_64, _64, Shape<_64>>;
|
||||
using ClusterShape = decltype(make_shape(int(0), int(0), Int<1>{}));
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAcc, ElementCompute,
|
||||
ElementAct, cutlass::layout::TensorNHWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
|
||||
ElementOut, cutlass::layout::TensorNHWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::conv::Operator::kDgrad,
|
||||
ElementAct, cutlass::layout::TensorNHWC, 8,
|
||||
ElementFlt, cutlass::layout::TensorNHWC, 8,
|
||||
ElementAcc,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::conv::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
|
||||
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
|
||||
ProblemShape,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
|
||||
|
||||
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>(1.0, 0.0, 0.0f, dim3(2,4,1), dim3(2,2,1)));
|
||||
}
|
||||
|
||||
#endif // defined(CUTLASS_ARCH_MMA_SM100_SUPPORTED)
|
||||
+338
@@ -0,0 +1,338 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2023 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/*! \file
|
||||
\brief Tests for device-wide CONV interface
|
||||
*/
|
||||
|
||||
#include "cutlass_unit_test.h"
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cute/tensor.hpp"
|
||||
#include "cute/atom/mma_atom.hpp"
|
||||
|
||||
#include "cutlass/numeric_types.h"
|
||||
|
||||
#include "cutlass/conv/device/conv_universal_adapter.hpp"
|
||||
#include "cutlass/conv/kernel/conv_universal.hpp"
|
||||
#include "cutlass/conv/collective/collective_builder.hpp"
|
||||
#include "cutlass/epilogue/collective/collective_builder.hpp"
|
||||
|
||||
#include "../testbed_conv.hpp"
|
||||
using namespace cute;
|
||||
|
||||
#if defined(CUTLASS_ARCH_MMA_SM100_SUPPORTED)
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// Static cluster
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
//
|
||||
// Cluster tile shape 64x64x64
|
||||
// Cluster shape 1x1x1
|
||||
//
|
||||
TEST(SM100_device_conv2d_dgrad_implicitgemm_f8nhwc_f8nhwc_bf16nhwc_tensor_op_f32, 64x64x64_1x1x1) {
|
||||
using ElementAct = cutlass::float_e4m3_t;
|
||||
using ElementFlt = cutlass::float_e4m3_t;
|
||||
using ElementOut = cutlass::bfloat16_t;
|
||||
using ElementAcc = float;
|
||||
using ElementCompute = float;
|
||||
using MmaTileShape = Shape<_64, _64, Shape<_64>>;
|
||||
using ClusterShape = Shape<_1,_1,_1>;
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAcc, ElementCompute,
|
||||
ElementAct, cutlass::layout::TensorNHWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
|
||||
ElementOut, cutlass::layout::TensorNHWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::conv::Operator::kDgrad,
|
||||
ElementAct, cutlass::layout::TensorNHWC, 16 / sizeof(ElementAct),
|
||||
ElementFlt, cutlass::layout::TensorNHWC, 16 / sizeof(ElementFlt),
|
||||
ElementAcc,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::conv::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
|
||||
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
|
||||
ProblemShape,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
|
||||
|
||||
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
|
||||
}
|
||||
|
||||
//
|
||||
// Cluster tile shape 128x64x64
|
||||
// Cluster shape 1x1x1
|
||||
//
|
||||
TEST(SM100_device_conv2d_dgrad_implicitgemm_f8nhwc_f8nhwc_bf16nhwc_tensor_op_f32, 128x64x64_1x1x1) {
|
||||
using ElementAct = cutlass::float_e4m3_t;
|
||||
using ElementFlt = cutlass::float_e4m3_t;
|
||||
using ElementOut = cutlass::bfloat16_t;
|
||||
using ElementAcc = float;
|
||||
using ElementCompute = float;
|
||||
using MmaTileShape = Shape<_128, _64, Shape<_64>>;
|
||||
using ClusterShape = Shape<_1,_1,_1>;
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAcc, ElementCompute,
|
||||
ElementAct, cutlass::layout::TensorNHWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
|
||||
ElementOut, cutlass::layout::TensorNHWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::conv::Operator::kDgrad,
|
||||
ElementAct, cutlass::layout::TensorNHWC, 16 / sizeof(ElementAct),
|
||||
ElementFlt, cutlass::layout::TensorNHWC, 16 / sizeof(ElementFlt),
|
||||
ElementAcc,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::conv::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
|
||||
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
|
||||
ProblemShape,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
|
||||
|
||||
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
|
||||
}
|
||||
|
||||
//
|
||||
// Cluster tile shape 128x128x64
|
||||
// Cluster shape 1x2x1
|
||||
//
|
||||
TEST(SM100_device_conv2d_dgrad_implicitgemm_f8nhwc_f8nhwc_bf16nhwc_tensor_op_f32, 128x128x64_1x2x1) {
|
||||
using ElementAct = cutlass::float_e4m3_t;
|
||||
using ElementFlt = cutlass::float_e4m3_t;
|
||||
using ElementOut = cutlass::bfloat16_t;
|
||||
using ElementAcc = float;
|
||||
using ElementCompute = float;
|
||||
using MmaTileShape = Shape<_128, _64, Shape<_64>>;
|
||||
using ClusterShape = Shape<_1,_2,_1>;
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAcc, ElementCompute,
|
||||
ElementAct, cutlass::layout::TensorNHWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
|
||||
ElementOut, cutlass::layout::TensorNHWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::conv::Operator::kDgrad,
|
||||
ElementAct, cutlass::layout::TensorNHWC, 16 / sizeof(ElementAct),
|
||||
ElementFlt, cutlass::layout::TensorNHWC, 16 / sizeof(ElementFlt),
|
||||
ElementAcc,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::conv::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
|
||||
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
|
||||
ProblemShape,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
|
||||
|
||||
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
|
||||
}
|
||||
|
||||
//
|
||||
// Cluster tile shape 256x64x64
|
||||
// Cluster shape 2x1x1
|
||||
//
|
||||
TEST(SM100_device_conv2d_dgrad_implicitgemm_f8nhwc_f8nhwc_bf16nhwc_tensor_op_f32, 256x64x64_2x1x1) {
|
||||
using ElementAct = cutlass::float_e4m3_t;
|
||||
using ElementFlt = cutlass::float_e4m3_t;
|
||||
using ElementOut = cutlass::bfloat16_t;
|
||||
using ElementAcc = float;
|
||||
using ElementCompute = float;
|
||||
using MmaTileShape = Shape<_256, _64, Shape<_64>>;
|
||||
using ClusterShape = Shape<_2,_1,_1>;
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAcc, ElementCompute,
|
||||
ElementAct, cutlass::layout::TensorNHWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
|
||||
ElementOut, cutlass::layout::TensorNHWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::conv::Operator::kDgrad,
|
||||
ElementAct, cutlass::layout::TensorNHWC, 16 / sizeof(ElementAct),
|
||||
ElementFlt, cutlass::layout::TensorNHWC, 16 / sizeof(ElementFlt),
|
||||
ElementAcc,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::conv::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
|
||||
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
|
||||
ProblemShape,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
|
||||
|
||||
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
|
||||
}
|
||||
|
||||
//
|
||||
// Cluster tile shape 256x128x64
|
||||
// Cluster shape 2x2x1
|
||||
//
|
||||
TEST(SM100_device_conv2d_dgrad_implicitgemm_f8nhwc_f8nhwc_bf16nhwc_tensor_op_f32, 256x128x64_2x2x1) {
|
||||
using ElementAct = cutlass::float_e4m3_t;
|
||||
using ElementFlt = cutlass::float_e4m3_t;
|
||||
using ElementOut = cutlass::bfloat16_t;
|
||||
using ElementAcc = float;
|
||||
using ElementCompute = float;
|
||||
using MmaTileShape = Shape<_256, _64, Shape<_64>>;
|
||||
using ClusterShape = Shape<_2,_2,_1>;
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAcc, ElementCompute,
|
||||
ElementAct, cutlass::layout::TensorNHWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
|
||||
ElementOut, cutlass::layout::TensorNHWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::conv::Operator::kDgrad,
|
||||
ElementAct, cutlass::layout::TensorNHWC, 16 / sizeof(ElementAct),
|
||||
ElementFlt, cutlass::layout::TensorNHWC, 16 / sizeof(ElementFlt),
|
||||
ElementAcc,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::conv::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
|
||||
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
|
||||
ProblemShape,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
|
||||
|
||||
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// Dynamic cluster
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
//
|
||||
// CTA tile shape 64x64x64
|
||||
// preferred cluster shape 2x4x1
|
||||
// fallback cluster shape 2x2x1
|
||||
//
|
||||
TEST(SM100_device_conv2d_dgrad_implicitgemm_f8nhwc_f8nhwc_bf16nhwc_tensor_op_f32, 64x64x64_preferred_2x4x1_fallback_2x2x1) {
|
||||
using ElementAct = cutlass::float_e4m3_t;
|
||||
using ElementFlt = cutlass::float_e4m3_t;
|
||||
using ElementOut = cutlass::bfloat16_t;
|
||||
using ElementAcc = float;
|
||||
using ElementCompute = float;
|
||||
using MmaTileShape = Shape<_64, _64, Shape<_64>>;
|
||||
using ClusterShape = decltype(make_shape(int(0), int(0), Int<1>{}));
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAcc, ElementCompute,
|
||||
ElementAct, cutlass::layout::TensorNHWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
|
||||
ElementOut, cutlass::layout::TensorNHWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::conv::Operator::kDgrad,
|
||||
ElementAct, cutlass::layout::TensorNHWC, 16 / sizeof(ElementAct),
|
||||
ElementFlt, cutlass::layout::TensorNHWC, 16 / sizeof(ElementFlt),
|
||||
ElementAcc,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::conv::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
|
||||
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
|
||||
ProblemShape,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
|
||||
|
||||
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>(1.0, 0.0, 0.0f, dim3(2,4,1), dim3(2,2,1)));
|
||||
}
|
||||
|
||||
#endif // defined(CUTLASS_ARCH_MMA_SM100_SUPPORTED)
|
||||
+338
@@ -0,0 +1,338 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2023 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/*! \file
|
||||
\brief Tests for device-wide CONV interface
|
||||
*/
|
||||
|
||||
#include "cutlass_unit_test.h"
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cute/tensor.hpp"
|
||||
#include "cute/atom/mma_atom.hpp"
|
||||
|
||||
#include "cutlass/numeric_types.h"
|
||||
|
||||
#include "cutlass/conv/device/conv_universal_adapter.hpp"
|
||||
#include "cutlass/conv/kernel/conv_universal.hpp"
|
||||
#include "cutlass/conv/collective/collective_builder.hpp"
|
||||
#include "cutlass/epilogue/collective/collective_builder.hpp"
|
||||
|
||||
#include "../testbed_conv.hpp"
|
||||
using namespace cute;
|
||||
|
||||
#if defined(CUTLASS_ARCH_MMA_SM100_SUPPORTED)
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// Static cluster
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
//
|
||||
// Cluster tile shape 64x64x64
|
||||
// Cluster shape 1x1x1
|
||||
//
|
||||
TEST(SM100_device_conv2d_dgrad_implicitgemm_f8nhwc_f8nhwc_f16nhwc_tensor_op_f32, 64x64x64_1x1x1) {
|
||||
using ElementAct = cutlass::float_e4m3_t;
|
||||
using ElementFlt = cutlass::float_e4m3_t;
|
||||
using ElementOut = cutlass::half_t;
|
||||
using ElementAcc = float;
|
||||
using ElementCompute = float;
|
||||
using MmaTileShape = Shape<_64, _64, Shape<_64>>;
|
||||
using ClusterShape = Shape<_1,_1,_1>;
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAcc, ElementCompute,
|
||||
ElementAct, cutlass::layout::TensorNHWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
|
||||
ElementOut, cutlass::layout::TensorNHWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::conv::Operator::kDgrad,
|
||||
ElementAct, cutlass::layout::TensorNHWC, 16 / sizeof(ElementAct),
|
||||
ElementFlt, cutlass::layout::TensorNHWC, 16 / sizeof(ElementFlt),
|
||||
ElementAcc,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::conv::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
|
||||
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
|
||||
ProblemShape,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
|
||||
|
||||
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
|
||||
}
|
||||
|
||||
//
|
||||
// Cluster tile shape 128x64x64
|
||||
// Cluster shape 1x1x1
|
||||
//
|
||||
TEST(SM100_device_conv2d_dgrad_implicitgemm_f8nhwc_f8nhwc_f16nhwc_tensor_op_f32, 128x64x64_1x1x1) {
|
||||
using ElementAct = cutlass::float_e4m3_t;
|
||||
using ElementFlt = cutlass::float_e4m3_t;
|
||||
using ElementOut = cutlass::half_t;
|
||||
using ElementAcc = float;
|
||||
using ElementCompute = float;
|
||||
using MmaTileShape = Shape<_128, _64, Shape<_64>>;
|
||||
using ClusterShape = Shape<_1,_1,_1>;
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAcc, ElementCompute,
|
||||
ElementAct, cutlass::layout::TensorNHWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
|
||||
ElementOut, cutlass::layout::TensorNHWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::conv::Operator::kDgrad,
|
||||
ElementAct, cutlass::layout::TensorNHWC, 16 / sizeof(ElementAct),
|
||||
ElementFlt, cutlass::layout::TensorNHWC, 16 / sizeof(ElementFlt),
|
||||
ElementAcc,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::conv::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
|
||||
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
|
||||
ProblemShape,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
|
||||
|
||||
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
|
||||
}
|
||||
|
||||
//
|
||||
// Cluster tile shape 128x128x64
|
||||
// Cluster shape 1x2x1
|
||||
//
|
||||
TEST(SM100_device_conv2d_dgrad_implicitgemm_f8nhwc_f8nhwc_f16nhwc_tensor_op_f32, 128x128x64_1x2x1) {
|
||||
using ElementAct = cutlass::float_e4m3_t;
|
||||
using ElementFlt = cutlass::float_e4m3_t;
|
||||
using ElementOut = cutlass::half_t;
|
||||
using ElementAcc = float;
|
||||
using ElementCompute = float;
|
||||
using MmaTileShape = Shape<_128, _64, Shape<_64>>;
|
||||
using ClusterShape = Shape<_1,_2,_1>;
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAcc, ElementCompute,
|
||||
ElementAct, cutlass::layout::TensorNHWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
|
||||
ElementOut, cutlass::layout::TensorNHWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::conv::Operator::kDgrad,
|
||||
ElementAct, cutlass::layout::TensorNHWC, 16 / sizeof(ElementAct),
|
||||
ElementFlt, cutlass::layout::TensorNHWC, 16 / sizeof(ElementFlt),
|
||||
ElementAcc,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::conv::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
|
||||
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
|
||||
ProblemShape,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
|
||||
|
||||
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
|
||||
}
|
||||
|
||||
//
|
||||
// Cluster tile shape 256x64x64
|
||||
// Cluster shape 2x1x1
|
||||
//
|
||||
TEST(SM100_device_conv2d_dgrad_implicitgemm_f8nhwc_f8nhwc_f16nhwc_tensor_op_f32, 256x64x64_2x1x1) {
|
||||
using ElementAct = cutlass::float_e4m3_t;
|
||||
using ElementFlt = cutlass::float_e4m3_t;
|
||||
using ElementOut = cutlass::half_t;
|
||||
using ElementAcc = float;
|
||||
using ElementCompute = float;
|
||||
using MmaTileShape = Shape<_256, _64, Shape<_64>>;
|
||||
using ClusterShape = Shape<_2,_1,_1>;
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAcc, ElementCompute,
|
||||
ElementAct, cutlass::layout::TensorNHWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
|
||||
ElementOut, cutlass::layout::TensorNHWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::conv::Operator::kDgrad,
|
||||
ElementAct, cutlass::layout::TensorNHWC, 16 / sizeof(ElementAct),
|
||||
ElementFlt, cutlass::layout::TensorNHWC, 16 / sizeof(ElementFlt),
|
||||
ElementAcc,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::conv::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
|
||||
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
|
||||
ProblemShape,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
|
||||
|
||||
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
|
||||
}
|
||||
|
||||
//
|
||||
// Cluster tile shape 256x128x64
|
||||
// Cluster shape 2x2x1
|
||||
//
|
||||
TEST(SM100_device_conv2d_dgrad_implicitgemm_f8nhwc_f8nhwc_f16nhwc_tensor_op_f32, 256x128x64_2x2x1) {
|
||||
using ElementAct = cutlass::float_e4m3_t;
|
||||
using ElementFlt = cutlass::float_e4m3_t;
|
||||
using ElementOut = cutlass::half_t;
|
||||
using ElementAcc = float;
|
||||
using ElementCompute = float;
|
||||
using MmaTileShape = Shape<_256, _64, Shape<_64>>;
|
||||
using ClusterShape = Shape<_2,_2,_1>;
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAcc, ElementCompute,
|
||||
ElementAct, cutlass::layout::TensorNHWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
|
||||
ElementOut, cutlass::layout::TensorNHWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::conv::Operator::kDgrad,
|
||||
ElementAct, cutlass::layout::TensorNHWC, 16 / sizeof(ElementAct),
|
||||
ElementFlt, cutlass::layout::TensorNHWC, 16 / sizeof(ElementFlt),
|
||||
ElementAcc,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::conv::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
|
||||
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
|
||||
ProblemShape,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
|
||||
|
||||
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// Dynamic cluster
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
//
|
||||
// CTA tile shape 64x64x64
|
||||
// preferred cluster shape 2x4x1
|
||||
// fallback cluster shape 2x2x1
|
||||
//
|
||||
TEST(SM100_device_conv2d_dgrad_implicitgemm_f8nhwc_f8nhwc_f16nhwc_tensor_op_f32, 64x64x64_preferred_2x4x1_fallback_2x2x1) {
|
||||
using ElementAct = cutlass::float_e4m3_t;
|
||||
using ElementFlt = cutlass::float_e4m3_t;
|
||||
using ElementOut = cutlass::half_t;
|
||||
using ElementAcc = float;
|
||||
using ElementCompute = float;
|
||||
using MmaTileShape = Shape<_64, _64, Shape<_64>>;
|
||||
using ClusterShape = decltype(make_shape(int(0), int(0), Int<1>{}));
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAcc, ElementCompute,
|
||||
ElementAct, cutlass::layout::TensorNHWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
|
||||
ElementOut, cutlass::layout::TensorNHWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::conv::Operator::kDgrad,
|
||||
ElementAct, cutlass::layout::TensorNHWC, 16 / sizeof(ElementAct),
|
||||
ElementFlt, cutlass::layout::TensorNHWC, 16 / sizeof(ElementFlt),
|
||||
ElementAcc,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::conv::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
|
||||
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
|
||||
ProblemShape,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
|
||||
|
||||
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>(1.0, 0.0, 0.0f, dim3(2,4,1), dim3(2,2,1)));
|
||||
}
|
||||
|
||||
#endif // defined(CUTLASS_ARCH_MMA_SM100_SUPPORTED)
|
||||
+237
@@ -0,0 +1,237 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2023 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/*! \file
|
||||
\brief Tests for device-wide CONV interface
|
||||
*/
|
||||
|
||||
#include "cutlass_unit_test.h"
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cute/tensor.hpp"
|
||||
#include "cute/atom/mma_atom.hpp"
|
||||
|
||||
#include "cutlass/numeric_types.h"
|
||||
|
||||
#include "cutlass/conv/device/conv_universal_adapter.hpp"
|
||||
#include "cutlass/conv/kernel/conv_universal.hpp"
|
||||
#include "cutlass/conv/collective/collective_builder.hpp"
|
||||
#include "cutlass/epilogue/collective/collective_builder.hpp"
|
||||
|
||||
#include "../testbed_conv.hpp"
|
||||
using namespace cute;
|
||||
|
||||
#if defined(CUTLASS_ARCH_MMA_SM100_SUPPORTED)
|
||||
|
||||
// alpha != 1 && beta != 0
|
||||
TEST(SM100_device_conv2d_dgrad_implicitgemm_f8nhwc_f8nhwc_f16nhwc_tensor_op_f32, 64x64x64_1x1x1_alpha_beta) {
|
||||
using ElementAct = cutlass::float_e4m3_t;
|
||||
using ElementFlt = cutlass::float_e4m3_t;
|
||||
using ElementOut = cutlass::half_t;
|
||||
using ElementAcc = float;
|
||||
using ElementCompute = float;
|
||||
using MmaTileShape = Shape<_64, _64, Shape<_64>>;
|
||||
using ClusterShape = Shape<_1,_1,_1>;
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAcc, ElementCompute,
|
||||
ElementAct, cutlass::layout::TensorNHWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
|
||||
ElementOut, cutlass::layout::TensorNHWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::conv::Operator::kDgrad,
|
||||
ElementAct, cutlass::layout::TensorNHWC, 16 / sizeof(ElementAct),
|
||||
ElementFlt, cutlass::layout::TensorNHWC, 16 / sizeof(ElementFlt),
|
||||
ElementAcc,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::conv::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
|
||||
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
|
||||
ProblemShape,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
|
||||
|
||||
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>(2.0, 1.0));
|
||||
}
|
||||
|
||||
// alpha != 1 && beta != 0 && bias
|
||||
TEST(SM100_device_conv2d_dgrad_implicitgemm_f8nhwc_f8nhwc_f16nhwc_tensor_op_f32, 64x64x64_1x1x1_alpha_beta_bias) {
|
||||
using ElementAct = cutlass::float_e4m3_t;
|
||||
using ElementFlt = cutlass::float_e4m3_t;
|
||||
using ElementOut = cutlass::half_t;
|
||||
using ElementAcc = float;
|
||||
using ElementCompute = float;
|
||||
using ElementBias = cutlass::half_t;
|
||||
using MmaTileShape = Shape<_64, _64, Shape<_64>>;
|
||||
using ClusterShape = Shape<_1,_1,_1>;
|
||||
|
||||
using FusionOperation = cutlass::epilogue::fusion::LinCombPerColBias<
|
||||
ElementOut, ElementCompute, ElementBias>;
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAcc, ElementCompute,
|
||||
ElementAct, cutlass::layout::TensorNHWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
|
||||
ElementOut, cutlass::layout::TensorNHWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto,
|
||||
FusionOperation
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::conv::Operator::kDgrad,
|
||||
ElementAct, cutlass::layout::TensorNHWC, 16 / sizeof(ElementAct),
|
||||
ElementFlt, cutlass::layout::TensorNHWC, 16 / sizeof(ElementFlt),
|
||||
ElementAcc,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::conv::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
|
||||
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
|
||||
ProblemShape,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
|
||||
|
||||
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>(2.0, 1.0));
|
||||
}
|
||||
|
||||
// alpha != 1 && beta != 0 && bias && relu
|
||||
TEST(SM100_device_conv2d_dgrad_implicitgemm_f8nhwc_f8nhwc_f16nhwc_tensor_op_f32, 64x64x64_1x1x1_alpha_beta_bias_relu) {
|
||||
using ElementAct = cutlass::float_e4m3_t;
|
||||
using ElementFlt = cutlass::float_e4m3_t;
|
||||
using ElementOut = cutlass::half_t;
|
||||
using ElementAcc = float;
|
||||
using ElementCompute = float;
|
||||
using ElementBias = cutlass::half_t;
|
||||
using MmaTileShape = Shape<_64, _64, Shape<_64>>;
|
||||
using ClusterShape = Shape<_1,_1,_1>;
|
||||
|
||||
using FusionOperation = cutlass::epilogue::fusion::LinCombPerColBiasEltAct<
|
||||
cutlass::epilogue::thread::ReLu, ElementOut, ElementCompute, ElementBias>;
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAcc, ElementCompute,
|
||||
ElementAct, cutlass::layout::TensorNHWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
|
||||
ElementOut, cutlass::layout::TensorNHWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto,
|
||||
FusionOperation
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::conv::Operator::kDgrad,
|
||||
ElementAct, cutlass::layout::TensorNHWC, 16 / sizeof(ElementAct),
|
||||
ElementFlt, cutlass::layout::TensorNHWC, 16 / sizeof(ElementFlt),
|
||||
ElementAcc,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::conv::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
|
||||
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
|
||||
ProblemShape,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
|
||||
|
||||
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>(2.0, 1.0));
|
||||
}
|
||||
|
||||
// per-channel alpha/beta scaling && bias && relu
|
||||
TEST(SM100_device_conv2d_dgrad_implicitgemm_f8nhwc_f8nhwc_f16nhwc_tensor_op_f32, 64x64x64_1x1x1_alpha_beta_scaled_bias_relu) {
|
||||
using ElementAct = cutlass::float_e4m3_t;
|
||||
using ElementFlt = cutlass::float_e4m3_t;
|
||||
using ElementOut = cutlass::half_t;
|
||||
using ElementAcc = float;
|
||||
using ElementCompute = float;
|
||||
using ElementBias = cutlass::half_t;
|
||||
using MmaTileShape = Shape<_64, _64, Shape<_64>>;
|
||||
using ClusterShape = Shape<_1,_1,_1>;
|
||||
|
||||
using FusionOperation = cutlass::epilogue::fusion::PerColLinCombPerColBiasEltAct<
|
||||
cutlass::epilogue::thread::ReLu, ElementOut, ElementCompute, ElementBias>;
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAcc, ElementCompute,
|
||||
ElementAct, cutlass::layout::TensorNHWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
|
||||
ElementOut, cutlass::layout::TensorNHWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto,
|
||||
FusionOperation
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::conv::Operator::kDgrad,
|
||||
ElementAct, cutlass::layout::TensorNHWC, 16 / sizeof(ElementAct),
|
||||
ElementFlt, cutlass::layout::TensorNHWC, 16 / sizeof(ElementFlt),
|
||||
ElementAcc,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::conv::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
|
||||
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
|
||||
ProblemShape,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
|
||||
|
||||
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
|
||||
}
|
||||
|
||||
#endif // defined(CUTLASS_ARCH_MMA_SM100_SUPPORTED)
|
||||
+338
@@ -0,0 +1,338 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2023 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/*! \file
|
||||
\brief Tests for device-wide CONV interface
|
||||
*/
|
||||
|
||||
#include "cutlass_unit_test.h"
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cute/tensor.hpp"
|
||||
#include "cute/atom/mma_atom.hpp"
|
||||
|
||||
#include "cutlass/numeric_types.h"
|
||||
|
||||
#include "cutlass/conv/device/conv_universal_adapter.hpp"
|
||||
#include "cutlass/conv/kernel/conv_universal.hpp"
|
||||
#include "cutlass/conv/collective/collective_builder.hpp"
|
||||
#include "cutlass/epilogue/collective/collective_builder.hpp"
|
||||
|
||||
#include "../testbed_conv.hpp"
|
||||
using namespace cute;
|
||||
|
||||
#if defined(CUTLASS_ARCH_MMA_SM100_SUPPORTED)
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// Static cluster
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
//
|
||||
// Cluster tile shape 64x64x64
|
||||
// Cluster shape 1x1x1
|
||||
//
|
||||
TEST(SM100_device_conv2d_dgrad_implicitgemm_f8nhwc_f8nhwc_f32nhwc_tensor_op_f32, 64x64x64_1x1x1) {
|
||||
using ElementAct = cutlass::float_e4m3_t;
|
||||
using ElementFlt = cutlass::float_e4m3_t;
|
||||
using ElementOut = float;
|
||||
using ElementAcc = float;
|
||||
using ElementCompute = float;
|
||||
using MmaTileShape = Shape<_64, _64, Shape<_64>>;
|
||||
using ClusterShape = Shape<_1,_1,_1>;
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAcc, ElementCompute,
|
||||
ElementAct, cutlass::layout::TensorNHWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
|
||||
ElementOut, cutlass::layout::TensorNHWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::conv::Operator::kDgrad,
|
||||
ElementAct, cutlass::layout::TensorNHWC, 16 / sizeof(ElementAct),
|
||||
ElementFlt, cutlass::layout::TensorNHWC, 16 / sizeof(ElementFlt),
|
||||
ElementAcc,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::conv::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
|
||||
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
|
||||
ProblemShape,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
|
||||
|
||||
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
|
||||
}
|
||||
|
||||
//
|
||||
// Cluster tile shape 128x64x64
|
||||
// Cluster shape 1x1x1
|
||||
//
|
||||
TEST(SM100_device_conv2d_dgrad_implicitgemm_f8nhwc_f8nhwc_f32nhwc_tensor_op_f32, 128x64x64_1x1x1) {
|
||||
using ElementAct = cutlass::float_e4m3_t;
|
||||
using ElementFlt = cutlass::float_e4m3_t;
|
||||
using ElementOut = float;
|
||||
using ElementAcc = float;
|
||||
using ElementCompute = float;
|
||||
using MmaTileShape = Shape<_128, _64, Shape<_64>>;
|
||||
using ClusterShape = Shape<_1,_1,_1>;
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAcc, ElementCompute,
|
||||
ElementAct, cutlass::layout::TensorNHWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
|
||||
ElementOut, cutlass::layout::TensorNHWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::conv::Operator::kDgrad,
|
||||
ElementAct, cutlass::layout::TensorNHWC, 16 / sizeof(ElementAct),
|
||||
ElementFlt, cutlass::layout::TensorNHWC, 16 / sizeof(ElementFlt),
|
||||
ElementAcc,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::conv::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
|
||||
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
|
||||
ProblemShape,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
|
||||
|
||||
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
|
||||
}
|
||||
|
||||
//
|
||||
// Cluster tile shape 128x128x64
|
||||
// Cluster shape 1x2x1
|
||||
//
|
||||
TEST(SM100_device_conv2d_dgrad_implicitgemm_f8nhwc_f8nhwc_f32nhwc_tensor_op_f32, 128x128x64_1x2x1) {
|
||||
using ElementAct = cutlass::float_e4m3_t;
|
||||
using ElementFlt = cutlass::float_e4m3_t;
|
||||
using ElementOut = float;
|
||||
using ElementAcc = float;
|
||||
using ElementCompute = float;
|
||||
using MmaTileShape = Shape<_128, _64, Shape<_64>>;
|
||||
using ClusterShape = Shape<_1,_2,_1>;
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAcc, ElementCompute,
|
||||
ElementAct, cutlass::layout::TensorNHWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
|
||||
ElementOut, cutlass::layout::TensorNHWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::conv::Operator::kDgrad,
|
||||
ElementAct, cutlass::layout::TensorNHWC, 16 / sizeof(ElementAct),
|
||||
ElementFlt, cutlass::layout::TensorNHWC, 16 / sizeof(ElementFlt),
|
||||
ElementAcc,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::conv::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
|
||||
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
|
||||
ProblemShape,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
|
||||
|
||||
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
|
||||
}
|
||||
|
||||
//
|
||||
// Cluster tile shape 256x64x64
|
||||
// Cluster shape 2x1x1
|
||||
//
|
||||
TEST(SM100_device_conv2d_dgrad_implicitgemm_f8nhwc_f8nhwc_f32nhwc_tensor_op_f32, 256x64x64_2x1x1) {
|
||||
using ElementAct = cutlass::float_e4m3_t;
|
||||
using ElementFlt = cutlass::float_e4m3_t;
|
||||
using ElementOut = float;
|
||||
using ElementAcc = float;
|
||||
using ElementCompute = float;
|
||||
using MmaTileShape = Shape<_256, _64, Shape<_64>>;
|
||||
using ClusterShape = Shape<_2,_1,_1>;
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAcc, ElementCompute,
|
||||
ElementAct, cutlass::layout::TensorNHWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
|
||||
ElementOut, cutlass::layout::TensorNHWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::conv::Operator::kDgrad,
|
||||
ElementAct, cutlass::layout::TensorNHWC, 16 / sizeof(ElementAct),
|
||||
ElementFlt, cutlass::layout::TensorNHWC, 16 / sizeof(ElementFlt),
|
||||
ElementAcc,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::conv::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
|
||||
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
|
||||
ProblemShape,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
|
||||
|
||||
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
|
||||
}
|
||||
|
||||
//
|
||||
// Cluster tile shape 256x128x64
|
||||
// Cluster shape 2x2x1
|
||||
//
|
||||
TEST(SM100_device_conv2d_dgrad_implicitgemm_f8nhwc_f8nhwc_f32nhwc_tensor_op_f32, 256x128x64_2x2x1) {
|
||||
using ElementAct = cutlass::float_e4m3_t;
|
||||
using ElementFlt = cutlass::float_e4m3_t;
|
||||
using ElementOut = float;
|
||||
using ElementAcc = float;
|
||||
using ElementCompute = float;
|
||||
using MmaTileShape = Shape<_256, _64, Shape<_64>>;
|
||||
using ClusterShape = Shape<_2,_2,_1>;
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAcc, ElementCompute,
|
||||
ElementAct, cutlass::layout::TensorNHWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
|
||||
ElementOut, cutlass::layout::TensorNHWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::conv::Operator::kDgrad,
|
||||
ElementAct, cutlass::layout::TensorNHWC, 16 / sizeof(ElementAct),
|
||||
ElementFlt, cutlass::layout::TensorNHWC, 16 / sizeof(ElementFlt),
|
||||
ElementAcc,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::conv::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
|
||||
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
|
||||
ProblemShape,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
|
||||
|
||||
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// Dynamic cluster
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
//
|
||||
// CTA tile shape 64x64x64
|
||||
// preferred cluster shape 2x4x1
|
||||
// fallback cluster shape 2x2x1
|
||||
//
|
||||
TEST(SM100_device_conv2d_dgrad_implicitgemm_f8nhwc_f8nhwc_f32nhwc_tensor_op_f32, 64x64x64_preferred_2x4x1_fallback_2x2x1) {
|
||||
using ElementAct = cutlass::float_e4m3_t;
|
||||
using ElementFlt = cutlass::float_e4m3_t;
|
||||
using ElementOut = float;
|
||||
using ElementAcc = float;
|
||||
using ElementCompute = float;
|
||||
using MmaTileShape = Shape<_64, _64, Shape<_64>>;
|
||||
using ClusterShape = decltype(make_shape(int(0), int(0), Int<1>{}));
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAcc, ElementCompute,
|
||||
ElementAct, cutlass::layout::TensorNHWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
|
||||
ElementOut, cutlass::layout::TensorNHWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::conv::Operator::kDgrad,
|
||||
ElementAct, cutlass::layout::TensorNHWC, 16 / sizeof(ElementAct),
|
||||
ElementFlt, cutlass::layout::TensorNHWC, 16 / sizeof(ElementFlt),
|
||||
ElementAcc,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::conv::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
|
||||
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
|
||||
ProblemShape,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
|
||||
|
||||
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>(1.0, 0.0, 0.0f, dim3(2,4,1), dim3(2,2,1)));
|
||||
}
|
||||
|
||||
#endif // defined(CUTLASS_ARCH_MMA_SM100_SUPPORTED)
|
||||
+338
@@ -0,0 +1,338 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2023 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/*! \file
|
||||
\brief Tests for device-wide CONV interface
|
||||
*/
|
||||
|
||||
#include "cutlass_unit_test.h"
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cute/tensor.hpp"
|
||||
#include "cute/atom/mma_atom.hpp"
|
||||
|
||||
#include "cutlass/numeric_types.h"
|
||||
|
||||
#include "cutlass/conv/device/conv_universal_adapter.hpp"
|
||||
#include "cutlass/conv/kernel/conv_universal.hpp"
|
||||
#include "cutlass/conv/collective/collective_builder.hpp"
|
||||
#include "cutlass/epilogue/collective/collective_builder.hpp"
|
||||
|
||||
#include "../testbed_conv.hpp"
|
||||
using namespace cute;
|
||||
|
||||
#if defined(CUTLASS_ARCH_MMA_SM100_SUPPORTED)
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// Static cluster
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
//
|
||||
// Cluster tile shape 64x64x64
|
||||
// Cluster shape 1x1x1
|
||||
//
|
||||
TEST(SM100_device_conv2d_dgrad_implicitgemm_f8nhwc_f8nhwc_f8nhwc_tensor_op_f32, 64x64x64_1x1x1) {
|
||||
using ElementAct = cutlass::float_e4m3_t;
|
||||
using ElementFlt = cutlass::float_e4m3_t;
|
||||
using ElementOut = cutlass::float_e4m3_t;
|
||||
using ElementAcc = float;
|
||||
using ElementCompute = float;
|
||||
using MmaTileShape = Shape<_64, _64, Shape<_64>>;
|
||||
using ClusterShape = Shape<_1,_1,_1>;
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAcc, ElementCompute,
|
||||
ElementAct, cutlass::layout::TensorNHWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
|
||||
ElementOut, cutlass::layout::TensorNHWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::conv::Operator::kDgrad,
|
||||
ElementAct, cutlass::layout::TensorNHWC, 16 / sizeof(ElementAct),
|
||||
ElementFlt, cutlass::layout::TensorNHWC, 16 / sizeof(ElementFlt),
|
||||
ElementAcc,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::conv::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
|
||||
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
|
||||
ProblemShape,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
|
||||
|
||||
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
|
||||
}
|
||||
|
||||
//
|
||||
// Cluster tile shape 128x64x64
|
||||
// Cluster shape 1x1x1
|
||||
//
|
||||
TEST(SM100_device_conv2d_dgrad_implicitgemm_f8nhwc_f8nhwc_f8nhwc_tensor_op_f32, 128x64x64_1x1x1) {
|
||||
using ElementAct = cutlass::float_e4m3_t;
|
||||
using ElementFlt = cutlass::float_e4m3_t;
|
||||
using ElementOut = cutlass::float_e4m3_t;
|
||||
using ElementAcc = float;
|
||||
using ElementCompute = float;
|
||||
using MmaTileShape = Shape<_128, _64, Shape<_64>>;
|
||||
using ClusterShape = Shape<_1,_1,_1>;
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAcc, ElementCompute,
|
||||
ElementAct, cutlass::layout::TensorNHWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
|
||||
ElementOut, cutlass::layout::TensorNHWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::conv::Operator::kDgrad,
|
||||
ElementAct, cutlass::layout::TensorNHWC, 16 / sizeof(ElementAct),
|
||||
ElementFlt, cutlass::layout::TensorNHWC, 16 / sizeof(ElementFlt),
|
||||
ElementAcc,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::conv::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
|
||||
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
|
||||
ProblemShape,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
|
||||
|
||||
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
|
||||
}
|
||||
|
||||
//
|
||||
// Cluster tile shape 128x128x64
|
||||
// Cluster shape 1x2x1
|
||||
//
|
||||
TEST(SM100_device_conv2d_dgrad_implicitgemm_f8nhwc_f8nhwc_f8nhwc_tensor_op_f32, 128x128x64_1x2x1) {
|
||||
using ElementAct = cutlass::float_e4m3_t;
|
||||
using ElementFlt = cutlass::float_e4m3_t;
|
||||
using ElementOut = cutlass::float_e4m3_t;
|
||||
using ElementAcc = float;
|
||||
using ElementCompute = float;
|
||||
using MmaTileShape = Shape<_128, _64, Shape<_64>>;
|
||||
using ClusterShape = Shape<_1,_2,_1>;
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAcc, ElementCompute,
|
||||
ElementAct, cutlass::layout::TensorNHWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
|
||||
ElementOut, cutlass::layout::TensorNHWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::conv::Operator::kDgrad,
|
||||
ElementAct, cutlass::layout::TensorNHWC, 16 / sizeof(ElementAct),
|
||||
ElementFlt, cutlass::layout::TensorNHWC, 16 / sizeof(ElementFlt),
|
||||
ElementAcc,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::conv::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
|
||||
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
|
||||
ProblemShape,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
|
||||
|
||||
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
|
||||
}
|
||||
|
||||
//
|
||||
// Cluster tile shape 256x64x64
|
||||
// Cluster shape 2x1x1
|
||||
//
|
||||
TEST(SM100_device_conv2d_dgrad_implicitgemm_f8nhwc_f8nhwc_f8nhwc_tensor_op_f32, 256x64x64_2x1x1) {
|
||||
using ElementAct = cutlass::float_e4m3_t;
|
||||
using ElementFlt = cutlass::float_e4m3_t;
|
||||
using ElementOut = cutlass::float_e4m3_t;
|
||||
using ElementAcc = float;
|
||||
using ElementCompute = float;
|
||||
using MmaTileShape = Shape<_256, _64, Shape<_64>>;
|
||||
using ClusterShape = Shape<_2,_1,_1>;
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAcc, ElementCompute,
|
||||
ElementAct, cutlass::layout::TensorNHWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
|
||||
ElementOut, cutlass::layout::TensorNHWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::conv::Operator::kDgrad,
|
||||
ElementAct, cutlass::layout::TensorNHWC, 16 / sizeof(ElementAct),
|
||||
ElementFlt, cutlass::layout::TensorNHWC, 16 / sizeof(ElementFlt),
|
||||
ElementAcc,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::conv::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
|
||||
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
|
||||
ProblemShape,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
|
||||
|
||||
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
|
||||
}
|
||||
|
||||
//
|
||||
// Cluster tile shape 256x128x64
|
||||
// Cluster shape 2x2x1
|
||||
//
|
||||
TEST(SM100_device_conv2d_dgrad_implicitgemm_f8nhwc_f8nhwc_f8nhwc_tensor_op_f32, 256x128x64_2x2x1) {
|
||||
using ElementAct = cutlass::float_e4m3_t;
|
||||
using ElementFlt = cutlass::float_e4m3_t;
|
||||
using ElementOut = cutlass::float_e4m3_t;
|
||||
using ElementAcc = float;
|
||||
using ElementCompute = float;
|
||||
using MmaTileShape = Shape<_256, _64, Shape<_64>>;
|
||||
using ClusterShape = Shape<_2,_2,_1>;
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAcc, ElementCompute,
|
||||
ElementAct, cutlass::layout::TensorNHWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
|
||||
ElementOut, cutlass::layout::TensorNHWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::conv::Operator::kDgrad,
|
||||
ElementAct, cutlass::layout::TensorNHWC, 16 / sizeof(ElementAct),
|
||||
ElementFlt, cutlass::layout::TensorNHWC, 16 / sizeof(ElementFlt),
|
||||
ElementAcc,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::conv::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
|
||||
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
|
||||
ProblemShape,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
|
||||
|
||||
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// Dynamic cluster
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
//
|
||||
// CTA tile shape 64x64x64
|
||||
// preferred cluster shape 2x4x1
|
||||
// fallback cluster shape 2x2x1
|
||||
//
|
||||
TEST(SM100_device_conv2d_dgrad_implicitgemm_f8nhwc_f8nhwc_f8nhwc_tensor_op_f32, 64x64x64_preferred_2x4x1_fallback_2x2x1) {
|
||||
using ElementAct = cutlass::float_e4m3_t;
|
||||
using ElementFlt = cutlass::float_e4m3_t;
|
||||
using ElementOut = cutlass::float_e4m3_t;
|
||||
using ElementAcc = float;
|
||||
using ElementCompute = float;
|
||||
using MmaTileShape = Shape<_64, _64, Shape<_64>>;
|
||||
using ClusterShape = decltype(make_shape(int(0), int(0), Int<1>{}));
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAcc, ElementCompute,
|
||||
ElementAct, cutlass::layout::TensorNHWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
|
||||
ElementOut, cutlass::layout::TensorNHWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::conv::Operator::kDgrad,
|
||||
ElementAct, cutlass::layout::TensorNHWC, 16 / sizeof(ElementAct),
|
||||
ElementFlt, cutlass::layout::TensorNHWC, 16 / sizeof(ElementFlt),
|
||||
ElementAcc,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::conv::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
|
||||
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
|
||||
ProblemShape,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
|
||||
|
||||
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>(1.0, 0.0, 0.0f, dim3(2,4,1), dim3(2,2,1)));
|
||||
}
|
||||
|
||||
#endif // defined(CUTLASS_ARCH_MMA_SM100_SUPPORTED)
|
||||
+338
@@ -0,0 +1,338 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2023 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/*! \file
|
||||
\brief Tests for device-wide CONV interface
|
||||
*/
|
||||
|
||||
#include "cutlass_unit_test.h"
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cute/tensor.hpp"
|
||||
#include "cute/atom/mma_atom.hpp"
|
||||
|
||||
#include "cutlass/numeric_types.h"
|
||||
|
||||
#include "cutlass/conv/device/conv_universal_adapter.hpp"
|
||||
#include "cutlass/conv/kernel/conv_universal.hpp"
|
||||
#include "cutlass/conv/collective/collective_builder.hpp"
|
||||
#include "cutlass/epilogue/collective/collective_builder.hpp"
|
||||
|
||||
#include "../testbed_conv.hpp"
|
||||
using namespace cute;
|
||||
|
||||
#if defined(CUTLASS_ARCH_MMA_SM100_SUPPORTED)
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// Static cluster
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
//
|
||||
// Cluster tile shape 64x64x64
|
||||
// Cluster shape 1x1x1
|
||||
//
|
||||
TEST(SM100_device_conv3d_dgrad_implicitgemm_f16ndhwc_f16ndhwc_f16ndhwc_tensor_op_f16, 64x64x64_1x1x1) {
|
||||
using ElementAct = cutlass::half_t;
|
||||
using ElementFlt = cutlass::half_t;
|
||||
using ElementOut = cutlass::half_t;
|
||||
using ElementAcc = cutlass::half_t;
|
||||
using ElementCompute = float;
|
||||
using MmaTileShape = Shape<_64, _64, Shape<_64>>;
|
||||
using ClusterShape = Shape<_1,_1,_1>;
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAcc, ElementCompute,
|
||||
ElementAct, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
|
||||
ElementOut, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::conv::Operator::kDgrad,
|
||||
ElementAct, cutlass::layout::TensorNDHWC, 8,
|
||||
ElementFlt, cutlass::layout::TensorNDHWC, 8,
|
||||
ElementAcc,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::conv::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
|
||||
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
|
||||
ProblemShape,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
|
||||
|
||||
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
|
||||
}
|
||||
|
||||
//
|
||||
// Cluster tile shape 128x64x64
|
||||
// Cluster shape 1x1x1
|
||||
//
|
||||
TEST(SM100_device_conv3d_dgrad_implicitgemm_f16ndhwc_f16ndhwc_f16ndhwc_tensor_op_f16, 128x64x64_1x1x1) {
|
||||
using ElementAct = cutlass::half_t;
|
||||
using ElementFlt = cutlass::half_t;
|
||||
using ElementOut = cutlass::half_t;
|
||||
using ElementAcc = cutlass::half_t;
|
||||
using ElementCompute = float;
|
||||
using MmaTileShape = Shape<_128, _64, Shape<_64>>;
|
||||
using ClusterShape = Shape<_1,_1,_1>;
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAcc, ElementCompute,
|
||||
ElementAct, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
|
||||
ElementOut, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::conv::Operator::kDgrad,
|
||||
ElementAct, cutlass::layout::TensorNDHWC, 8,
|
||||
ElementFlt, cutlass::layout::TensorNDHWC, 8,
|
||||
ElementAcc,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::conv::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
|
||||
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
|
||||
ProblemShape,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
|
||||
|
||||
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
|
||||
}
|
||||
|
||||
//
|
||||
// Cluster tile shape 128x128x64
|
||||
// Cluster shape 1x2x1
|
||||
//
|
||||
TEST(SM100_device_conv3d_dgrad_implicitgemm_f16ndhwc_f16ndhwc_f16ndhwc_tensor_op_f16, 128x128x64_1x2x1) {
|
||||
using ElementAct = cutlass::half_t;
|
||||
using ElementFlt = cutlass::half_t;
|
||||
using ElementOut = cutlass::half_t;
|
||||
using ElementAcc = cutlass::half_t;
|
||||
using ElementCompute = float;
|
||||
using MmaTileShape = Shape<_128, _64, Shape<_64>>;
|
||||
using ClusterShape = Shape<_1,_2,_1>;
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAcc, ElementCompute,
|
||||
ElementAct, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
|
||||
ElementOut, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::conv::Operator::kDgrad,
|
||||
ElementAct, cutlass::layout::TensorNDHWC, 8,
|
||||
ElementFlt, cutlass::layout::TensorNDHWC, 8,
|
||||
ElementAcc,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::conv::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
|
||||
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
|
||||
ProblemShape,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
|
||||
|
||||
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
|
||||
}
|
||||
|
||||
//
|
||||
// Cluster tile shape 256x64x64
|
||||
// Cluster shape 2x1x1
|
||||
//
|
||||
TEST(SM100_device_conv3d_dgrad_implicitgemm_f16ndhwc_f16ndhwc_f16ndhwc_tensor_op_f16, 256x64x64_2x1x1) {
|
||||
using ElementAct = cutlass::half_t;
|
||||
using ElementFlt = cutlass::half_t;
|
||||
using ElementOut = cutlass::half_t;
|
||||
using ElementAcc = cutlass::half_t;
|
||||
using ElementCompute = float;
|
||||
using MmaTileShape = Shape<_256, _64, Shape<_64>>;
|
||||
using ClusterShape = Shape<_2,_1,_1>;
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAcc, ElementCompute,
|
||||
ElementAct, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
|
||||
ElementOut, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::conv::Operator::kDgrad,
|
||||
ElementAct, cutlass::layout::TensorNDHWC, 8,
|
||||
ElementFlt, cutlass::layout::TensorNDHWC, 8,
|
||||
ElementAcc,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::conv::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
|
||||
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
|
||||
ProblemShape,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
|
||||
|
||||
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
|
||||
}
|
||||
|
||||
//
|
||||
// Cluster tile shape 256x128x64
|
||||
// Cluster shape 2x2x1
|
||||
//
|
||||
TEST(SM100_device_conv3d_dgrad_implicitgemm_f16ndhwc_f16ndhwc_f16ndhwc_tensor_op_f16, 256x128x64_2x2x1) {
|
||||
using ElementAct = cutlass::half_t;
|
||||
using ElementFlt = cutlass::half_t;
|
||||
using ElementOut = cutlass::half_t;
|
||||
using ElementAcc = cutlass::half_t;
|
||||
using ElementCompute = float;
|
||||
using MmaTileShape = Shape<_256, _64, Shape<_64>>;
|
||||
using ClusterShape = Shape<_2,_2,_1>;
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAcc, ElementCompute,
|
||||
ElementAct, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
|
||||
ElementOut, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::conv::Operator::kDgrad,
|
||||
ElementAct, cutlass::layout::TensorNDHWC, 8,
|
||||
ElementFlt, cutlass::layout::TensorNDHWC, 8,
|
||||
ElementAcc,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::conv::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
|
||||
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
|
||||
ProblemShape,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
|
||||
|
||||
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// Dynamic cluster
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
//
|
||||
// CTA tile shape 64x64x64
|
||||
// preferred cluster shape 2x4x1
|
||||
// fallback cluster shape 2x2x1
|
||||
//
|
||||
TEST(SM100_device_conv3d_dgrad_implicitgemm_f16ndhwc_f16ndhwc_f16ndhwc_tensor_op_f16, 64x64x64_preferred_2x4x1_fallback_2x2x1) {
|
||||
using ElementAct = cutlass::half_t;
|
||||
using ElementFlt = cutlass::half_t;
|
||||
using ElementOut = cutlass::half_t;
|
||||
using ElementAcc = cutlass::half_t;
|
||||
using ElementCompute = float;
|
||||
using MmaTileShape = Shape<_64, _64, Shape<_64>>;
|
||||
using ClusterShape = decltype(make_shape(int(0), int(0), Int<1>{}));
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAcc, ElementCompute,
|
||||
ElementAct, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
|
||||
ElementOut, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::conv::Operator::kDgrad,
|
||||
ElementAct, cutlass::layout::TensorNDHWC, 8,
|
||||
ElementFlt, cutlass::layout::TensorNDHWC, 8,
|
||||
ElementAcc,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::conv::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
|
||||
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
|
||||
ProblemShape,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
|
||||
|
||||
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>(1.0, 0.0, 0.0f, dim3(2,4,1), dim3(2,2,1)));
|
||||
}
|
||||
|
||||
#endif // defined(CUTLASS_ARCH_MMA_SM100_SUPPORTED)
|
||||
+143
@@ -0,0 +1,143 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2023 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/*! \file
|
||||
\brief Tests for device-wide CONV interface
|
||||
*/
|
||||
|
||||
#include "cutlass_unit_test.h"
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cute/tensor.hpp"
|
||||
#include "cute/atom/mma_atom.hpp"
|
||||
|
||||
#include "cutlass/numeric_types.h"
|
||||
|
||||
#include "cutlass/conv/device/conv_universal_adapter.hpp"
|
||||
#include "cutlass/conv/kernel/conv_universal.hpp"
|
||||
#include "cutlass/conv/collective/collective_builder.hpp"
|
||||
#include "cutlass/epilogue/collective/collective_builder.hpp"
|
||||
|
||||
#include "../testbed_conv.hpp"
|
||||
using namespace cute;
|
||||
|
||||
#if defined(CUTLASS_ARCH_MMA_SM100_SUPPORTED)
|
||||
|
||||
// alpha != 1 && beta != 0
|
||||
TEST(SM100_device_conv3d_dgrad_implicitgemm_f16ndhwc_f16ndhwc_f16ndhwc_tensor_op_f16, 64x64x64_1x1x1_alpha_beta) {
|
||||
using ElementAct = cutlass::half_t;
|
||||
using ElementFlt = cutlass::half_t;
|
||||
using ElementOut = cutlass::half_t;
|
||||
using ElementAcc = cutlass::half_t;
|
||||
using ElementCompute = float;
|
||||
using MmaTileShape = Shape<_64, _64, Shape<_64>>;
|
||||
using ClusterShape = Shape<_1,_1,_1>;
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAcc, ElementCompute,
|
||||
ElementAct, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
|
||||
ElementOut, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::conv::Operator::kDgrad,
|
||||
ElementAct, cutlass::layout::TensorNDHWC, 8,
|
||||
ElementFlt, cutlass::layout::TensorNDHWC, 8,
|
||||
ElementAcc,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::conv::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
|
||||
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
|
||||
ProblemShape,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
|
||||
|
||||
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>(2.0, 1.0));
|
||||
}
|
||||
|
||||
// alpha != 1 && beta != 0 && bias
|
||||
TEST(SM100_device_conv3d_dgrad_implicitgemm_f16ndhwc_f16ndhwc_f16ndhwc_tensor_op_f16, 64x64x64_1x1x1_alpha_beta_bias) {
|
||||
using ElementAct = cutlass::half_t;
|
||||
using ElementFlt = cutlass::half_t;
|
||||
using ElementOut = cutlass::half_t;
|
||||
using ElementAcc = cutlass::half_t;
|
||||
using ElementCompute = float;
|
||||
using ElementBias = cutlass::half_t;
|
||||
using MmaTileShape = Shape<_64, _64, Shape<_64>>;
|
||||
using ClusterShape = Shape<_1,_1,_1>;
|
||||
|
||||
using FusionOperation = cutlass::epilogue::fusion::LinCombPerColBias<
|
||||
ElementOut, ElementCompute, ElementBias>;
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAcc, ElementCompute,
|
||||
ElementAct, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
|
||||
ElementOut, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto,
|
||||
FusionOperation
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::conv::Operator::kDgrad,
|
||||
ElementAct, cutlass::layout::TensorNDHWC, 8,
|
||||
ElementFlt, cutlass::layout::TensorNDHWC, 8,
|
||||
ElementAcc,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::conv::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
|
||||
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
|
||||
ProblemShape,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
|
||||
|
||||
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>(2.0, 1.0));
|
||||
}
|
||||
|
||||
#endif // defined(CUTLASS_ARCH_MMA_SM100_SUPPORTED)
|
||||
+338
@@ -0,0 +1,338 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2023 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/*! \file
|
||||
\brief Tests for device-wide CONV interface
|
||||
*/
|
||||
|
||||
#include "cutlass_unit_test.h"
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cute/tensor.hpp"
|
||||
#include "cute/atom/mma_atom.hpp"
|
||||
|
||||
#include "cutlass/numeric_types.h"
|
||||
|
||||
#include "cutlass/conv/device/conv_universal_adapter.hpp"
|
||||
#include "cutlass/conv/kernel/conv_universal.hpp"
|
||||
#include "cutlass/conv/collective/collective_builder.hpp"
|
||||
#include "cutlass/epilogue/collective/collective_builder.hpp"
|
||||
|
||||
#include "../testbed_conv.hpp"
|
||||
using namespace cute;
|
||||
|
||||
#if defined(CUTLASS_ARCH_MMA_SM100_SUPPORTED)
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// Static cluster
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
//
|
||||
// Cluster tile shape 64x64x64
|
||||
// Cluster shape 1x1x1
|
||||
//
|
||||
TEST(SM100_device_conv3d_dgrad_implicitgemm_f16ndhwc_f16ndhwc_f32ndhwc_tensor_op_f32, 64x64x64_1x1x1) {
|
||||
using ElementAct = cutlass::half_t;
|
||||
using ElementFlt = cutlass::half_t;
|
||||
using ElementOut = float;
|
||||
using ElementAcc = float;
|
||||
using ElementCompute = float;
|
||||
using MmaTileShape = Shape<_64, _64, Shape<_64>>;
|
||||
using ClusterShape = Shape<_1,_1,_1>;
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAcc, ElementCompute,
|
||||
ElementAct, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
|
||||
ElementOut, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::conv::Operator::kDgrad,
|
||||
ElementAct, cutlass::layout::TensorNDHWC, 8,
|
||||
ElementFlt, cutlass::layout::TensorNDHWC, 8,
|
||||
ElementAcc,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::conv::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
|
||||
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
|
||||
ProblemShape,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
|
||||
|
||||
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
|
||||
}
|
||||
|
||||
//
|
||||
// Cluster tile shape 128x64x64
|
||||
// Cluster shape 1x1x1
|
||||
//
|
||||
TEST(SM100_device_conv3d_dgrad_implicitgemm_f16ndhwc_f16ndhwc_f32ndhwc_tensor_op_f32, 128x64x64_1x1x1) {
|
||||
using ElementAct = cutlass::half_t;
|
||||
using ElementFlt = cutlass::half_t;
|
||||
using ElementOut = float;
|
||||
using ElementAcc = float;
|
||||
using ElementCompute = float;
|
||||
using MmaTileShape = Shape<_128, _64, Shape<_64>>;
|
||||
using ClusterShape = Shape<_1,_1,_1>;
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAcc, ElementCompute,
|
||||
ElementAct, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
|
||||
ElementOut, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::conv::Operator::kDgrad,
|
||||
ElementAct, cutlass::layout::TensorNDHWC, 8,
|
||||
ElementFlt, cutlass::layout::TensorNDHWC, 8,
|
||||
ElementAcc,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::conv::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
|
||||
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
|
||||
ProblemShape,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
|
||||
|
||||
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
|
||||
}
|
||||
|
||||
//
|
||||
// Cluster tile shape 128x128x64
|
||||
// Cluster shape 1x2x1
|
||||
//
|
||||
TEST(SM100_device_conv3d_dgrad_implicitgemm_f16ndhwc_f16ndhwc_f32ndhwc_tensor_op_f32, 128x128x64_1x2x1) {
|
||||
using ElementAct = cutlass::half_t;
|
||||
using ElementFlt = cutlass::half_t;
|
||||
using ElementOut = float;
|
||||
using ElementAcc = float;
|
||||
using ElementCompute = float;
|
||||
using MmaTileShape = Shape<_128, _64, Shape<_64>>;
|
||||
using ClusterShape = Shape<_1,_2,_1>;
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAcc, ElementCompute,
|
||||
ElementAct, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
|
||||
ElementOut, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::conv::Operator::kDgrad,
|
||||
ElementAct, cutlass::layout::TensorNDHWC, 8,
|
||||
ElementFlt, cutlass::layout::TensorNDHWC, 8,
|
||||
ElementAcc,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::conv::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
|
||||
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
|
||||
ProblemShape,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
|
||||
|
||||
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
|
||||
}
|
||||
|
||||
//
|
||||
// Cluster tile shape 256x64x64
|
||||
// Cluster shape 2x1x1
|
||||
//
|
||||
TEST(SM100_device_conv3d_dgrad_implicitgemm_f16ndhwc_f16ndhwc_f32ndhwc_tensor_op_f32, 256x64x64_2x1x1) {
|
||||
using ElementAct = cutlass::half_t;
|
||||
using ElementFlt = cutlass::half_t;
|
||||
using ElementOut = float;
|
||||
using ElementAcc = float;
|
||||
using ElementCompute = float;
|
||||
using MmaTileShape = Shape<_256, _64, Shape<_64>>;
|
||||
using ClusterShape = Shape<_2,_1,_1>;
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAcc, ElementCompute,
|
||||
ElementAct, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
|
||||
ElementOut, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::conv::Operator::kDgrad,
|
||||
ElementAct, cutlass::layout::TensorNDHWC, 8,
|
||||
ElementFlt, cutlass::layout::TensorNDHWC, 8,
|
||||
ElementAcc,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::conv::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
|
||||
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
|
||||
ProblemShape,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
|
||||
|
||||
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
|
||||
}
|
||||
|
||||
//
|
||||
// Cluster tile shape 256x128x64
|
||||
// Cluster shape 2x2x1
|
||||
//
|
||||
TEST(SM100_device_conv3d_dgrad_implicitgemm_f16ndhwc_f16ndhwc_f32ndhwc_tensor_op_f32, 256x128x64_2x2x1) {
|
||||
using ElementAct = cutlass::half_t;
|
||||
using ElementFlt = cutlass::half_t;
|
||||
using ElementOut = float;
|
||||
using ElementAcc = float;
|
||||
using ElementCompute = float;
|
||||
using MmaTileShape = Shape<_256, _64, Shape<_64>>;
|
||||
using ClusterShape = Shape<_2,_2,_1>;
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAcc, ElementCompute,
|
||||
ElementAct, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
|
||||
ElementOut, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::conv::Operator::kDgrad,
|
||||
ElementAct, cutlass::layout::TensorNDHWC, 8,
|
||||
ElementFlt, cutlass::layout::TensorNDHWC, 8,
|
||||
ElementAcc,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::conv::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
|
||||
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
|
||||
ProblemShape,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
|
||||
|
||||
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// Dynamic cluster
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
//
|
||||
// CTA tile shape 64x64x64
|
||||
// preferred cluster shape 2x4x1
|
||||
// fallback cluster shape 2x2x1
|
||||
//
|
||||
TEST(SM100_device_conv3d_dgrad_implicitgemm_f16ndhwc_f16ndhwc_f32ndhwc_tensor_op_f32, 64x64x64_preferred_2x4x1_fallback_2x2x1) {
|
||||
using ElementAct = cutlass::half_t;
|
||||
using ElementFlt = cutlass::half_t;
|
||||
using ElementOut = float;
|
||||
using ElementAcc = float;
|
||||
using ElementCompute = float;
|
||||
using MmaTileShape = Shape<_64, _64, Shape<_64>>;
|
||||
using ClusterShape = decltype(make_shape(int(0), int(0), Int<1>{}));
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAcc, ElementCompute,
|
||||
ElementAct, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
|
||||
ElementOut, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::conv::Operator::kDgrad,
|
||||
ElementAct, cutlass::layout::TensorNDHWC, 8,
|
||||
ElementFlt, cutlass::layout::TensorNDHWC, 8,
|
||||
ElementAcc,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::conv::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
|
||||
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
|
||||
ProblemShape,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
|
||||
|
||||
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>(1.0, 0.0, 0.0f, dim3(2,4,1), dim3(2,2,1)));
|
||||
}
|
||||
|
||||
#endif // defined(CUTLASS_ARCH_MMA_SM100_SUPPORTED)
|
||||
+338
@@ -0,0 +1,338 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2023 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/*! \file
|
||||
\brief Tests for device-wide CONV interface
|
||||
*/
|
||||
|
||||
#include "cutlass_unit_test.h"
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cute/tensor.hpp"
|
||||
#include "cute/atom/mma_atom.hpp"
|
||||
|
||||
#include "cutlass/numeric_types.h"
|
||||
|
||||
#include "cutlass/conv/device/conv_universal_adapter.hpp"
|
||||
#include "cutlass/conv/kernel/conv_universal.hpp"
|
||||
#include "cutlass/conv/collective/collective_builder.hpp"
|
||||
#include "cutlass/epilogue/collective/collective_builder.hpp"
|
||||
|
||||
#include "../testbed_conv.hpp"
|
||||
using namespace cute;
|
||||
|
||||
#if defined(CUTLASS_ARCH_MMA_SM100_SUPPORTED)
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// Static cluster
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
//
|
||||
// Cluster tile shape 64x64x64
|
||||
// Cluster shape 1x1x1
|
||||
//
|
||||
TEST(SM100_device_conv3d_dgrad_implicitgemm_f8ndhwc_f8ndhwc_bf16ndhwc_tensor_op_f32, 64x64x64_1x1x1) {
|
||||
using ElementAct = cutlass::float_e4m3_t;
|
||||
using ElementFlt = cutlass::float_e4m3_t;
|
||||
using ElementOut = cutlass::bfloat16_t;
|
||||
using ElementAcc = float;
|
||||
using ElementCompute = float;
|
||||
using MmaTileShape = Shape<_64, _64, Shape<_64>>;
|
||||
using ClusterShape = Shape<_1,_1,_1>;
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAcc, ElementCompute,
|
||||
ElementAct, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
|
||||
ElementOut, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::conv::Operator::kDgrad,
|
||||
ElementAct, cutlass::layout::TensorNDHWC, 16 / sizeof(ElementAct),
|
||||
ElementFlt, cutlass::layout::TensorNDHWC, 16 / sizeof(ElementFlt),
|
||||
ElementAcc,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::conv::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
|
||||
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
|
||||
ProblemShape,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
|
||||
|
||||
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
|
||||
}
|
||||
|
||||
//
|
||||
// Cluster tile shape 128x64x64
|
||||
// Cluster shape 1x1x1
|
||||
//
|
||||
TEST(SM100_device_conv3d_dgrad_implicitgemm_f8ndhwc_f8ndhwc_bf16ndhwc_tensor_op_f32, 128x64x64_1x1x1) {
|
||||
using ElementAct = cutlass::float_e4m3_t;
|
||||
using ElementFlt = cutlass::float_e4m3_t;
|
||||
using ElementOut = cutlass::bfloat16_t;
|
||||
using ElementAcc = float;
|
||||
using ElementCompute = float;
|
||||
using MmaTileShape = Shape<_128, _64, Shape<_64>>;
|
||||
using ClusterShape = Shape<_1,_1,_1>;
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAcc, ElementCompute,
|
||||
ElementAct, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
|
||||
ElementOut, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::conv::Operator::kDgrad,
|
||||
ElementAct, cutlass::layout::TensorNDHWC, 16 / sizeof(ElementAct),
|
||||
ElementFlt, cutlass::layout::TensorNDHWC, 16 / sizeof(ElementFlt),
|
||||
ElementAcc,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::conv::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
|
||||
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
|
||||
ProblemShape,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
|
||||
|
||||
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
|
||||
}
|
||||
|
||||
//
|
||||
// Cluster tile shape 128x128x64
|
||||
// Cluster shape 1x2x1
|
||||
//
|
||||
TEST(SM100_device_conv3d_dgrad_implicitgemm_f8ndhwc_f8ndhwc_bf16ndhwc_tensor_op_f32, 128x128x64_1x2x1) {
|
||||
using ElementAct = cutlass::float_e4m3_t;
|
||||
using ElementFlt = cutlass::float_e4m3_t;
|
||||
using ElementOut = cutlass::bfloat16_t;
|
||||
using ElementAcc = float;
|
||||
using ElementCompute = float;
|
||||
using MmaTileShape = Shape<_128, _64, Shape<_64>>;
|
||||
using ClusterShape = Shape<_1,_2,_1>;
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAcc, ElementCompute,
|
||||
ElementAct, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
|
||||
ElementOut, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::conv::Operator::kDgrad,
|
||||
ElementAct, cutlass::layout::TensorNDHWC, 16 / sizeof(ElementAct),
|
||||
ElementFlt, cutlass::layout::TensorNDHWC, 16 / sizeof(ElementFlt),
|
||||
ElementAcc,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::conv::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
|
||||
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
|
||||
ProblemShape,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
|
||||
|
||||
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
|
||||
}
|
||||
|
||||
//
|
||||
// Cluster tile shape 256x64x64
|
||||
// Cluster shape 2x1x1
|
||||
//
|
||||
TEST(SM100_device_conv3d_dgrad_implicitgemm_f8ndhwc_f8ndhwc_bf16ndhwc_tensor_op_f32, 256x64x64_2x1x1) {
|
||||
using ElementAct = cutlass::float_e4m3_t;
|
||||
using ElementFlt = cutlass::float_e4m3_t;
|
||||
using ElementOut = cutlass::bfloat16_t;
|
||||
using ElementAcc = float;
|
||||
using ElementCompute = float;
|
||||
using MmaTileShape = Shape<_256, _64, Shape<_64>>;
|
||||
using ClusterShape = Shape<_2,_1,_1>;
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAcc, ElementCompute,
|
||||
ElementAct, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
|
||||
ElementOut, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::conv::Operator::kDgrad,
|
||||
ElementAct, cutlass::layout::TensorNDHWC, 16 / sizeof(ElementAct),
|
||||
ElementFlt, cutlass::layout::TensorNDHWC, 16 / sizeof(ElementFlt),
|
||||
ElementAcc,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::conv::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
|
||||
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
|
||||
ProblemShape,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
|
||||
|
||||
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
|
||||
}
|
||||
|
||||
//
|
||||
// Cluster tile shape 256x128x64
|
||||
// Cluster shape 2x2x1
|
||||
//
|
||||
TEST(SM100_device_conv3d_dgrad_implicitgemm_f8ndhwc_f8ndhwc_bf16ndhwc_tensor_op_f32, 256x128x64_2x2x1) {
|
||||
using ElementAct = cutlass::float_e4m3_t;
|
||||
using ElementFlt = cutlass::float_e4m3_t;
|
||||
using ElementOut = cutlass::bfloat16_t;
|
||||
using ElementAcc = float;
|
||||
using ElementCompute = float;
|
||||
using MmaTileShape = Shape<_256, _64, Shape<_64>>;
|
||||
using ClusterShape = Shape<_2,_2,_1>;
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAcc, ElementCompute,
|
||||
ElementAct, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
|
||||
ElementOut, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::conv::Operator::kDgrad,
|
||||
ElementAct, cutlass::layout::TensorNDHWC, 16 / sizeof(ElementAct),
|
||||
ElementFlt, cutlass::layout::TensorNDHWC, 16 / sizeof(ElementFlt),
|
||||
ElementAcc,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::conv::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
|
||||
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
|
||||
ProblemShape,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
|
||||
|
||||
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// Dynamic cluster
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
//
|
||||
// CTA tile shape 64x64x64
|
||||
// preferred cluster shape 2x4x1
|
||||
// fallback cluster shape 2x2x1
|
||||
//
|
||||
TEST(SM100_device_conv3d_dgrad_implicitgemm_f8ndhwc_f8ndhwc_bf16ndhwc_tensor_op_f32, 64x64x64_preferred_2x4x1_fallback_2x2x1) {
|
||||
using ElementAct = cutlass::float_e4m3_t;
|
||||
using ElementFlt = cutlass::float_e4m3_t;
|
||||
using ElementOut = cutlass::bfloat16_t;
|
||||
using ElementAcc = float;
|
||||
using ElementCompute = float;
|
||||
using MmaTileShape = Shape<_64, _64, Shape<_64>>;
|
||||
using ClusterShape = decltype(make_shape(int(0), int(0), Int<1>{}));
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAcc, ElementCompute,
|
||||
ElementAct, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
|
||||
ElementOut, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::conv::Operator::kDgrad,
|
||||
ElementAct, cutlass::layout::TensorNDHWC, 16 / sizeof(ElementAct),
|
||||
ElementFlt, cutlass::layout::TensorNDHWC, 16 / sizeof(ElementFlt),
|
||||
ElementAcc,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::conv::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
|
||||
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
|
||||
ProblemShape,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
|
||||
|
||||
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>(1.0, 0.0, 0.0f, dim3(2,4,1), dim3(2,2,1)));
|
||||
}
|
||||
|
||||
#endif // defined(CUTLASS_ARCH_MMA_SM100_SUPPORTED)
|
||||
+338
@@ -0,0 +1,338 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2023 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/*! \file
|
||||
\brief Tests for device-wide CONV interface
|
||||
*/
|
||||
|
||||
#include "cutlass_unit_test.h"
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cute/tensor.hpp"
|
||||
#include "cute/atom/mma_atom.hpp"
|
||||
|
||||
#include "cutlass/numeric_types.h"
|
||||
|
||||
#include "cutlass/conv/device/conv_universal_adapter.hpp"
|
||||
#include "cutlass/conv/kernel/conv_universal.hpp"
|
||||
#include "cutlass/conv/collective/collective_builder.hpp"
|
||||
#include "cutlass/epilogue/collective/collective_builder.hpp"
|
||||
|
||||
#include "../testbed_conv.hpp"
|
||||
using namespace cute;
|
||||
|
||||
#if defined(CUTLASS_ARCH_MMA_SM100_SUPPORTED)
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// Static cluster
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
//
|
||||
// Cluster tile shape 64x64x64
|
||||
// Cluster shape 1x1x1
|
||||
//
|
||||
TEST(SM100_device_conv3d_dgrad_implicitgemm_f8ndhwc_f8ndhwc_f16ndhwc_tensor_op_f32, 64x64x64_1x1x1) {
|
||||
using ElementAct = cutlass::float_e4m3_t;
|
||||
using ElementFlt = cutlass::float_e4m3_t;
|
||||
using ElementOut = cutlass::half_t;
|
||||
using ElementAcc = float;
|
||||
using ElementCompute = float;
|
||||
using MmaTileShape = Shape<_64, _64, Shape<_64>>;
|
||||
using ClusterShape = Shape<_1,_1,_1>;
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAcc, ElementCompute,
|
||||
ElementAct, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
|
||||
ElementOut, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::conv::Operator::kDgrad,
|
||||
ElementAct, cutlass::layout::TensorNDHWC, 16 / sizeof(ElementAct),
|
||||
ElementFlt, cutlass::layout::TensorNDHWC, 16 / sizeof(ElementFlt),
|
||||
ElementAcc,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::conv::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
|
||||
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
|
||||
ProblemShape,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
|
||||
|
||||
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
|
||||
}
|
||||
|
||||
//
|
||||
// Cluster tile shape 128x64x64
|
||||
// Cluster shape 1x1x1
|
||||
//
|
||||
TEST(SM100_device_conv3d_dgrad_implicitgemm_f8ndhwc_f8ndhwc_f16ndhwc_tensor_op_f32, 128x64x64_1x1x1) {
|
||||
using ElementAct = cutlass::float_e4m3_t;
|
||||
using ElementFlt = cutlass::float_e4m3_t;
|
||||
using ElementOut = cutlass::half_t;
|
||||
using ElementAcc = float;
|
||||
using ElementCompute = float;
|
||||
using MmaTileShape = Shape<_128, _64, Shape<_64>>;
|
||||
using ClusterShape = Shape<_1,_1,_1>;
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAcc, ElementCompute,
|
||||
ElementAct, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
|
||||
ElementOut, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::conv::Operator::kDgrad,
|
||||
ElementAct, cutlass::layout::TensorNDHWC, 16 / sizeof(ElementAct),
|
||||
ElementFlt, cutlass::layout::TensorNDHWC, 16 / sizeof(ElementFlt),
|
||||
ElementAcc,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::conv::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
|
||||
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
|
||||
ProblemShape,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
|
||||
|
||||
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
|
||||
}
|
||||
|
||||
//
|
||||
// Cluster tile shape 128x128x64
|
||||
// Cluster shape 1x2x1
|
||||
//
|
||||
TEST(SM100_device_conv3d_dgrad_implicitgemm_f8ndhwc_f8ndhwc_f16ndhwc_tensor_op_f32, 128x128x64_1x2x1) {
|
||||
using ElementAct = cutlass::float_e4m3_t;
|
||||
using ElementFlt = cutlass::float_e4m3_t;
|
||||
using ElementOut = cutlass::half_t;
|
||||
using ElementAcc = float;
|
||||
using ElementCompute = float;
|
||||
using MmaTileShape = Shape<_128, _64, Shape<_64>>;
|
||||
using ClusterShape = Shape<_1,_2,_1>;
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAcc, ElementCompute,
|
||||
ElementAct, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
|
||||
ElementOut, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::conv::Operator::kDgrad,
|
||||
ElementAct, cutlass::layout::TensorNDHWC, 16 / sizeof(ElementAct),
|
||||
ElementFlt, cutlass::layout::TensorNDHWC, 16 / sizeof(ElementFlt),
|
||||
ElementAcc,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::conv::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
|
||||
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
|
||||
ProblemShape,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
|
||||
|
||||
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
|
||||
}
|
||||
|
||||
//
|
||||
// Cluster tile shape 256x64x64
|
||||
// Cluster shape 2x1x1
|
||||
//
|
||||
TEST(SM100_device_conv3d_dgrad_implicitgemm_f8ndhwc_f8ndhwc_f16ndhwc_tensor_op_f32, 256x64x64_2x1x1) {
|
||||
using ElementAct = cutlass::float_e4m3_t;
|
||||
using ElementFlt = cutlass::float_e4m3_t;
|
||||
using ElementOut = cutlass::half_t;
|
||||
using ElementAcc = float;
|
||||
using ElementCompute = float;
|
||||
using MmaTileShape = Shape<_256, _64, Shape<_64>>;
|
||||
using ClusterShape = Shape<_2,_1,_1>;
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAcc, ElementCompute,
|
||||
ElementAct, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
|
||||
ElementOut, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::conv::Operator::kDgrad,
|
||||
ElementAct, cutlass::layout::TensorNDHWC, 16 / sizeof(ElementAct),
|
||||
ElementFlt, cutlass::layout::TensorNDHWC, 16 / sizeof(ElementFlt),
|
||||
ElementAcc,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::conv::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
|
||||
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
|
||||
ProblemShape,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
|
||||
|
||||
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
|
||||
}
|
||||
|
||||
//
|
||||
// Cluster tile shape 256x128x64
|
||||
// Cluster shape 2x2x1
|
||||
//
|
||||
TEST(SM100_device_conv3d_dgrad_implicitgemm_f8ndhwc_f8ndhwc_f16ndhwc_tensor_op_f32, 256x128x64_2x2x1) {
|
||||
using ElementAct = cutlass::float_e4m3_t;
|
||||
using ElementFlt = cutlass::float_e4m3_t;
|
||||
using ElementOut = cutlass::half_t;
|
||||
using ElementAcc = float;
|
||||
using ElementCompute = float;
|
||||
using MmaTileShape = Shape<_256, _64, Shape<_64>>;
|
||||
using ClusterShape = Shape<_2,_2,_1>;
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAcc, ElementCompute,
|
||||
ElementAct, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
|
||||
ElementOut, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::conv::Operator::kDgrad,
|
||||
ElementAct, cutlass::layout::TensorNDHWC, 16 / sizeof(ElementAct),
|
||||
ElementFlt, cutlass::layout::TensorNDHWC, 16 / sizeof(ElementFlt),
|
||||
ElementAcc,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::conv::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
|
||||
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
|
||||
ProblemShape,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
|
||||
|
||||
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// Dynamic cluster
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
//
|
||||
// CTA tile shape 64x64x64
|
||||
// preferred cluster shape 2x4x1
|
||||
// fallback cluster shape 2x2x1
|
||||
//
|
||||
TEST(SM100_device_conv3d_dgrad_implicitgemm_f8ndhwc_f8ndhwc_f16ndhwc_tensor_op_f32, 64x64x64_preferred_2x4x1_fallback_2x2x1) {
|
||||
using ElementAct = cutlass::float_e4m3_t;
|
||||
using ElementFlt = cutlass::float_e4m3_t;
|
||||
using ElementOut = cutlass::half_t;
|
||||
using ElementAcc = float;
|
||||
using ElementCompute = float;
|
||||
using MmaTileShape = Shape<_64, _64, Shape<_64>>;
|
||||
using ClusterShape = decltype(make_shape(int(0), int(0), Int<1>{}));
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAcc, ElementCompute,
|
||||
ElementAct, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
|
||||
ElementOut, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::conv::Operator::kDgrad,
|
||||
ElementAct, cutlass::layout::TensorNDHWC, 16 / sizeof(ElementAct),
|
||||
ElementFlt, cutlass::layout::TensorNDHWC, 16 / sizeof(ElementFlt),
|
||||
ElementAcc,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::conv::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
|
||||
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
|
||||
ProblemShape,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
|
||||
|
||||
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>(1.0, 0.0, 0.0f, dim3(2,4,1), dim3(2,2,1)));
|
||||
}
|
||||
|
||||
#endif // defined(CUTLASS_ARCH_MMA_SM100_SUPPORTED)
|
||||
+237
@@ -0,0 +1,237 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2023 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/*! \file
|
||||
\brief Tests for device-wide CONV interface
|
||||
*/
|
||||
|
||||
#include "cutlass_unit_test.h"
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cute/tensor.hpp"
|
||||
#include "cute/atom/mma_atom.hpp"
|
||||
|
||||
#include "cutlass/numeric_types.h"
|
||||
|
||||
#include "cutlass/conv/device/conv_universal_adapter.hpp"
|
||||
#include "cutlass/conv/kernel/conv_universal.hpp"
|
||||
#include "cutlass/conv/collective/collective_builder.hpp"
|
||||
#include "cutlass/epilogue/collective/collective_builder.hpp"
|
||||
|
||||
#include "../testbed_conv.hpp"
|
||||
using namespace cute;
|
||||
|
||||
#if defined(CUTLASS_ARCH_MMA_SM100_SUPPORTED)
|
||||
|
||||
// alpha != 1 && beta != 0
|
||||
TEST(SM100_device_conv3d_dgrad_implicitgemm_f8ndhwc_f8ndhwc_f16ndhwc_tensor_op_f32, 64x64x64_1x1x1_alpha_beta) {
|
||||
using ElementAct = cutlass::float_e4m3_t;
|
||||
using ElementFlt = cutlass::float_e4m3_t;
|
||||
using ElementOut = cutlass::half_t;
|
||||
using ElementAcc = float;
|
||||
using ElementCompute = float;
|
||||
using MmaTileShape = Shape<_64, _64, Shape<_64>>;
|
||||
using ClusterShape = Shape<_1,_1,_1>;
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAcc, ElementCompute,
|
||||
ElementAct, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
|
||||
ElementOut, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::conv::Operator::kDgrad,
|
||||
ElementAct, cutlass::layout::TensorNDHWC, 16 / sizeof(ElementAct),
|
||||
ElementFlt, cutlass::layout::TensorNDHWC, 16 / sizeof(ElementFlt),
|
||||
ElementAcc,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::conv::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
|
||||
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
|
||||
ProblemShape,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
|
||||
|
||||
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>(2.0, 1.0));
|
||||
}
|
||||
|
||||
// alpha != 1 && beta != 0 && bias
|
||||
TEST(SM100_device_conv3d_dgrad_implicitgemm_f8ndhwc_f8ndhwc_f16ndhwc_tensor_op_f32, 64x64x64_1x1x1_alpha_beta_bias) {
|
||||
using ElementAct = cutlass::float_e4m3_t;
|
||||
using ElementFlt = cutlass::float_e4m3_t;
|
||||
using ElementOut = cutlass::half_t;
|
||||
using ElementAcc = float;
|
||||
using ElementCompute = float;
|
||||
using ElementBias = cutlass::half_t;
|
||||
using MmaTileShape = Shape<_64, _64, Shape<_64>>;
|
||||
using ClusterShape = Shape<_1,_1,_1>;
|
||||
|
||||
using FusionOperation = cutlass::epilogue::fusion::LinCombPerColBias<
|
||||
ElementOut, ElementCompute, ElementBias>;
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAcc, ElementCompute,
|
||||
ElementAct, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
|
||||
ElementOut, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto,
|
||||
FusionOperation
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::conv::Operator::kDgrad,
|
||||
ElementAct, cutlass::layout::TensorNDHWC, 16 / sizeof(ElementAct),
|
||||
ElementFlt, cutlass::layout::TensorNDHWC, 16 / sizeof(ElementFlt),
|
||||
ElementAcc,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::conv::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
|
||||
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
|
||||
ProblemShape,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
|
||||
|
||||
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>(2.0, 1.0));
|
||||
}
|
||||
|
||||
// alpha != 1 && beta != 0 && bias && relu
|
||||
TEST(SM100_device_conv3d_dgrad_implicitgemm_f8ndhwc_f8ndhwc_f16ndhwc_tensor_op_f32, 64x64x64_1x1x1_alpha_beta_bias_relu) {
|
||||
using ElementAct = cutlass::float_e4m3_t;
|
||||
using ElementFlt = cutlass::float_e4m3_t;
|
||||
using ElementOut = cutlass::half_t;
|
||||
using ElementAcc = float;
|
||||
using ElementCompute = float;
|
||||
using ElementBias = cutlass::half_t;
|
||||
using MmaTileShape = Shape<_64, _64, Shape<_64>>;
|
||||
using ClusterShape = Shape<_1,_1,_1>;
|
||||
|
||||
using FusionOperation = cutlass::epilogue::fusion::LinCombPerColBiasEltAct<
|
||||
cutlass::epilogue::thread::ReLu, ElementOut, ElementCompute, ElementBias>;
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAcc, ElementCompute,
|
||||
ElementAct, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
|
||||
ElementOut, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto,
|
||||
FusionOperation
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::conv::Operator::kDgrad,
|
||||
ElementAct, cutlass::layout::TensorNDHWC, 16 / sizeof(ElementAct),
|
||||
ElementFlt, cutlass::layout::TensorNDHWC, 16 / sizeof(ElementFlt),
|
||||
ElementAcc,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::conv::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
|
||||
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
|
||||
ProblemShape,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
|
||||
|
||||
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>(2.0, 1.0));
|
||||
}
|
||||
|
||||
// per-channel alpha/beta scaling && bias && relu
|
||||
TEST(SM100_device_conv3d_dgrad_implicitgemm_f8ndhwc_f8ndhwc_f16ndhwc_tensor_op_f32, 64x64x64_1x1x1_alpha_beta_scaled_bias_relu) {
|
||||
using ElementAct = cutlass::float_e4m3_t;
|
||||
using ElementFlt = cutlass::float_e4m3_t;
|
||||
using ElementOut = cutlass::half_t;
|
||||
using ElementAcc = float;
|
||||
using ElementCompute = float;
|
||||
using ElementBias = cutlass::half_t;
|
||||
using MmaTileShape = Shape<_64, _64, Shape<_64>>;
|
||||
using ClusterShape = Shape<_1,_1,_1>;
|
||||
|
||||
using FusionOperation = cutlass::epilogue::fusion::PerColLinCombPerColBiasEltAct<
|
||||
cutlass::epilogue::thread::ReLu, ElementOut, ElementCompute, ElementBias>;
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAcc, ElementCompute,
|
||||
ElementAct, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
|
||||
ElementOut, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto,
|
||||
FusionOperation
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::conv::Operator::kDgrad,
|
||||
ElementAct, cutlass::layout::TensorNDHWC, 16 / sizeof(ElementAct),
|
||||
ElementFlt, cutlass::layout::TensorNDHWC, 16 / sizeof(ElementFlt),
|
||||
ElementAcc,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::conv::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
|
||||
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
|
||||
ProblemShape,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
|
||||
|
||||
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
|
||||
}
|
||||
|
||||
#endif // defined(CUTLASS_ARCH_MMA_SM100_SUPPORTED)
|
||||
+338
@@ -0,0 +1,338 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2023 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/*! \file
|
||||
\brief Tests for device-wide CONV interface
|
||||
*/
|
||||
|
||||
#include "cutlass_unit_test.h"
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cute/tensor.hpp"
|
||||
#include "cute/atom/mma_atom.hpp"
|
||||
|
||||
#include "cutlass/numeric_types.h"
|
||||
|
||||
#include "cutlass/conv/device/conv_universal_adapter.hpp"
|
||||
#include "cutlass/conv/kernel/conv_universal.hpp"
|
||||
#include "cutlass/conv/collective/collective_builder.hpp"
|
||||
#include "cutlass/epilogue/collective/collective_builder.hpp"
|
||||
|
||||
#include "../testbed_conv.hpp"
|
||||
using namespace cute;
|
||||
|
||||
#if defined(CUTLASS_ARCH_MMA_SM100_SUPPORTED)
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// Static cluster
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
//
|
||||
// Cluster tile shape 64x64x64
|
||||
// Cluster shape 1x1x1
|
||||
//
|
||||
TEST(SM100_device_conv3d_dgrad_implicitgemm_f8ndhwc_f8ndhwc_f32ndhwc_tensor_op_f32, 64x64x64_1x1x1) {
|
||||
using ElementAct = cutlass::float_e4m3_t;
|
||||
using ElementFlt = cutlass::float_e4m3_t;
|
||||
using ElementOut = float;
|
||||
using ElementAcc = float;
|
||||
using ElementCompute = float;
|
||||
using MmaTileShape = Shape<_64, _64, Shape<_64>>;
|
||||
using ClusterShape = Shape<_1,_1,_1>;
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAcc, ElementCompute,
|
||||
ElementAct, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
|
||||
ElementOut, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::conv::Operator::kDgrad,
|
||||
ElementAct, cutlass::layout::TensorNDHWC, 16 / sizeof(ElementAct),
|
||||
ElementFlt, cutlass::layout::TensorNDHWC, 16 / sizeof(ElementFlt),
|
||||
ElementAcc,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::conv::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
|
||||
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
|
||||
ProblemShape,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
|
||||
|
||||
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
|
||||
}
|
||||
|
||||
//
|
||||
// Cluster tile shape 128x64x64
|
||||
// Cluster shape 1x1x1
|
||||
//
|
||||
TEST(SM100_device_conv3d_dgrad_implicitgemm_f8ndhwc_f8ndhwc_f32ndhwc_tensor_op_f32, 128x64x64_1x1x1) {
|
||||
using ElementAct = cutlass::float_e4m3_t;
|
||||
using ElementFlt = cutlass::float_e4m3_t;
|
||||
using ElementOut = float;
|
||||
using ElementAcc = float;
|
||||
using ElementCompute = float;
|
||||
using MmaTileShape = Shape<_128, _64, Shape<_64>>;
|
||||
using ClusterShape = Shape<_1,_1,_1>;
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAcc, ElementCompute,
|
||||
ElementAct, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
|
||||
ElementOut, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::conv::Operator::kDgrad,
|
||||
ElementAct, cutlass::layout::TensorNDHWC, 16 / sizeof(ElementAct),
|
||||
ElementFlt, cutlass::layout::TensorNDHWC, 16 / sizeof(ElementFlt),
|
||||
ElementAcc,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::conv::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
|
||||
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
|
||||
ProblemShape,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
|
||||
|
||||
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
|
||||
}
|
||||
|
||||
//
|
||||
// Cluster tile shape 128x128x64
|
||||
// Cluster shape 1x2x1
|
||||
//
|
||||
TEST(SM100_device_conv3d_dgrad_implicitgemm_f8ndhwc_f8ndhwc_f32ndhwc_tensor_op_f32, 128x128x64_1x2x1) {
|
||||
using ElementAct = cutlass::float_e4m3_t;
|
||||
using ElementFlt = cutlass::float_e4m3_t;
|
||||
using ElementOut = float;
|
||||
using ElementAcc = float;
|
||||
using ElementCompute = float;
|
||||
using MmaTileShape = Shape<_128, _64, Shape<_64>>;
|
||||
using ClusterShape = Shape<_1,_2,_1>;
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAcc, ElementCompute,
|
||||
ElementAct, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
|
||||
ElementOut, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::conv::Operator::kDgrad,
|
||||
ElementAct, cutlass::layout::TensorNDHWC, 16 / sizeof(ElementAct),
|
||||
ElementFlt, cutlass::layout::TensorNDHWC, 16 / sizeof(ElementFlt),
|
||||
ElementAcc,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::conv::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
|
||||
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
|
||||
ProblemShape,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
|
||||
|
||||
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
|
||||
}
|
||||
|
||||
//
|
||||
// Cluster tile shape 256x64x64
|
||||
// Cluster shape 2x1x1
|
||||
//
|
||||
TEST(SM100_device_conv3d_dgrad_implicitgemm_f8ndhwc_f8ndhwc_f32ndhwc_tensor_op_f32, 256x64x64_2x1x1) {
|
||||
using ElementAct = cutlass::float_e4m3_t;
|
||||
using ElementFlt = cutlass::float_e4m3_t;
|
||||
using ElementOut = float;
|
||||
using ElementAcc = float;
|
||||
using ElementCompute = float;
|
||||
using MmaTileShape = Shape<_256, _64, Shape<_64>>;
|
||||
using ClusterShape = Shape<_2,_1,_1>;
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAcc, ElementCompute,
|
||||
ElementAct, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
|
||||
ElementOut, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::conv::Operator::kDgrad,
|
||||
ElementAct, cutlass::layout::TensorNDHWC, 16 / sizeof(ElementAct),
|
||||
ElementFlt, cutlass::layout::TensorNDHWC, 16 / sizeof(ElementFlt),
|
||||
ElementAcc,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::conv::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
|
||||
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
|
||||
ProblemShape,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
|
||||
|
||||
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
|
||||
}
|
||||
|
||||
//
|
||||
// Cluster tile shape 256x128x64
|
||||
// Cluster shape 2x2x1
|
||||
//
|
||||
TEST(SM100_device_conv3d_dgrad_implicitgemm_f8ndhwc_f8ndhwc_f32ndhwc_tensor_op_f32, 256x128x64_2x2x1) {
|
||||
using ElementAct = cutlass::float_e4m3_t;
|
||||
using ElementFlt = cutlass::float_e4m3_t;
|
||||
using ElementOut = float;
|
||||
using ElementAcc = float;
|
||||
using ElementCompute = float;
|
||||
using MmaTileShape = Shape<_256, _64, Shape<_64>>;
|
||||
using ClusterShape = Shape<_2,_2,_1>;
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAcc, ElementCompute,
|
||||
ElementAct, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
|
||||
ElementOut, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::conv::Operator::kDgrad,
|
||||
ElementAct, cutlass::layout::TensorNDHWC, 16 / sizeof(ElementAct),
|
||||
ElementFlt, cutlass::layout::TensorNDHWC, 16 / sizeof(ElementFlt),
|
||||
ElementAcc,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::conv::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
|
||||
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
|
||||
ProblemShape,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
|
||||
|
||||
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// Dynamic cluster
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
//
|
||||
// CTA tile shape 64x64x64
|
||||
// preferred cluster shape 2x4x1
|
||||
// fallback cluster shape 2x2x1
|
||||
//
|
||||
TEST(SM100_device_conv3d_dgrad_implicitgemm_f8ndhwc_f8ndhwc_f32ndhwc_tensor_op_f32, 64x64x64_preferred_2x4x1_fallback_2x2x1) {
|
||||
using ElementAct = cutlass::float_e4m3_t;
|
||||
using ElementFlt = cutlass::float_e4m3_t;
|
||||
using ElementOut = float;
|
||||
using ElementAcc = float;
|
||||
using ElementCompute = float;
|
||||
using MmaTileShape = Shape<_64, _64, Shape<_64>>;
|
||||
using ClusterShape = decltype(make_shape(int(0), int(0), Int<1>{}));
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAcc, ElementCompute,
|
||||
ElementAct, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
|
||||
ElementOut, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::conv::Operator::kDgrad,
|
||||
ElementAct, cutlass::layout::TensorNDHWC, 16 / sizeof(ElementAct),
|
||||
ElementFlt, cutlass::layout::TensorNDHWC, 16 / sizeof(ElementFlt),
|
||||
ElementAcc,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::conv::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
|
||||
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
|
||||
ProblemShape,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
|
||||
|
||||
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>(1.0, 0.0, 0.0f, dim3(2,4,1), dim3(2,2,1)));
|
||||
}
|
||||
|
||||
#endif // defined(CUTLASS_ARCH_MMA_SM100_SUPPORTED)
|
||||
+338
@@ -0,0 +1,338 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2023 - 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/*! \file
|
||||
\brief Tests for device-wide CONV interface
|
||||
*/
|
||||
|
||||
#include "cutlass_unit_test.h"
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cute/tensor.hpp"
|
||||
#include "cute/atom/mma_atom.hpp"
|
||||
|
||||
#include "cutlass/numeric_types.h"
|
||||
|
||||
#include "cutlass/conv/device/conv_universal_adapter.hpp"
|
||||
#include "cutlass/conv/kernel/conv_universal.hpp"
|
||||
#include "cutlass/conv/collective/collective_builder.hpp"
|
||||
#include "cutlass/epilogue/collective/collective_builder.hpp"
|
||||
|
||||
#include "../testbed_conv.hpp"
|
||||
using namespace cute;
|
||||
|
||||
#if defined(CUTLASS_ARCH_MMA_SM100_SUPPORTED)
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// Static cluster
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
//
|
||||
// Cluster tile shape 64x64x64
|
||||
// Cluster shape 1x1x1
|
||||
//
|
||||
TEST(SM100_device_conv3d_dgrad_implicitgemm_f8ndhwc_f8ndhwc_f8ndhwc_tensor_op_f32, 64x64x64_1x1x1) {
|
||||
using ElementAct = cutlass::float_e4m3_t;
|
||||
using ElementFlt = cutlass::float_e4m3_t;
|
||||
using ElementOut = cutlass::float_e4m3_t;
|
||||
using ElementAcc = float;
|
||||
using ElementCompute = float;
|
||||
using MmaTileShape = Shape<_64, _64, Shape<_64>>;
|
||||
using ClusterShape = Shape<_1,_1,_1>;
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAcc, ElementCompute,
|
||||
ElementAct, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
|
||||
ElementOut, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::conv::Operator::kDgrad,
|
||||
ElementAct, cutlass::layout::TensorNDHWC, 16 / sizeof(ElementAct),
|
||||
ElementFlt, cutlass::layout::TensorNDHWC, 16 / sizeof(ElementFlt),
|
||||
ElementAcc,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::conv::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
|
||||
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
|
||||
ProblemShape,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
|
||||
|
||||
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
|
||||
}
|
||||
|
||||
//
|
||||
// Cluster tile shape 128x64x64
|
||||
// Cluster shape 1x1x1
|
||||
//
|
||||
TEST(SM100_device_conv3d_dgrad_implicitgemm_f8ndhwc_f8ndhwc_f8ndhwc_tensor_op_f32, 128x64x64_1x1x1) {
|
||||
using ElementAct = cutlass::float_e4m3_t;
|
||||
using ElementFlt = cutlass::float_e4m3_t;
|
||||
using ElementOut = cutlass::float_e4m3_t;
|
||||
using ElementAcc = float;
|
||||
using ElementCompute = float;
|
||||
using MmaTileShape = Shape<_128, _64, Shape<_64>>;
|
||||
using ClusterShape = Shape<_1,_1,_1>;
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAcc, ElementCompute,
|
||||
ElementAct, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
|
||||
ElementOut, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::conv::Operator::kDgrad,
|
||||
ElementAct, cutlass::layout::TensorNDHWC, 16 / sizeof(ElementAct),
|
||||
ElementFlt, cutlass::layout::TensorNDHWC, 16 / sizeof(ElementFlt),
|
||||
ElementAcc,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::conv::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
|
||||
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
|
||||
ProblemShape,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
|
||||
|
||||
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
|
||||
}
|
||||
|
||||
//
|
||||
// Cluster tile shape 128x128x64
|
||||
// Cluster shape 1x2x1
|
||||
//
|
||||
TEST(SM100_device_conv3d_dgrad_implicitgemm_f8ndhwc_f8ndhwc_f8ndhwc_tensor_op_f32, 128x128x64_1x2x1) {
|
||||
using ElementAct = cutlass::float_e4m3_t;
|
||||
using ElementFlt = cutlass::float_e4m3_t;
|
||||
using ElementOut = cutlass::float_e4m3_t;
|
||||
using ElementAcc = float;
|
||||
using ElementCompute = float;
|
||||
using MmaTileShape = Shape<_128, _64, Shape<_64>>;
|
||||
using ClusterShape = Shape<_1,_2,_1>;
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAcc, ElementCompute,
|
||||
ElementAct, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
|
||||
ElementOut, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::conv::Operator::kDgrad,
|
||||
ElementAct, cutlass::layout::TensorNDHWC, 16 / sizeof(ElementAct),
|
||||
ElementFlt, cutlass::layout::TensorNDHWC, 16 / sizeof(ElementFlt),
|
||||
ElementAcc,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::conv::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
|
||||
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
|
||||
ProblemShape,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
|
||||
|
||||
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
|
||||
}
|
||||
|
||||
//
|
||||
// Cluster tile shape 256x64x64
|
||||
// Cluster shape 2x1x1
|
||||
//
|
||||
TEST(SM100_device_conv3d_dgrad_implicitgemm_f8ndhwc_f8ndhwc_f8ndhwc_tensor_op_f32, 256x64x64_2x1x1) {
|
||||
using ElementAct = cutlass::float_e4m3_t;
|
||||
using ElementFlt = cutlass::float_e4m3_t;
|
||||
using ElementOut = cutlass::float_e4m3_t;
|
||||
using ElementAcc = float;
|
||||
using ElementCompute = float;
|
||||
using MmaTileShape = Shape<_256, _64, Shape<_64>>;
|
||||
using ClusterShape = Shape<_2,_1,_1>;
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAcc, ElementCompute,
|
||||
ElementAct, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
|
||||
ElementOut, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::conv::Operator::kDgrad,
|
||||
ElementAct, cutlass::layout::TensorNDHWC, 16 / sizeof(ElementAct),
|
||||
ElementFlt, cutlass::layout::TensorNDHWC, 16 / sizeof(ElementFlt),
|
||||
ElementAcc,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::conv::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
|
||||
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
|
||||
ProblemShape,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
|
||||
|
||||
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
|
||||
}
|
||||
|
||||
//
|
||||
// Cluster tile shape 256x128x64
|
||||
// Cluster shape 2x2x1
|
||||
//
|
||||
TEST(SM100_device_conv3d_dgrad_implicitgemm_f8ndhwc_f8ndhwc_f8ndhwc_tensor_op_f32, 256x128x64_2x2x1) {
|
||||
using ElementAct = cutlass::float_e4m3_t;
|
||||
using ElementFlt = cutlass::float_e4m3_t;
|
||||
using ElementOut = cutlass::float_e4m3_t;
|
||||
using ElementAcc = float;
|
||||
using ElementCompute = float;
|
||||
using MmaTileShape = Shape<_256, _64, Shape<_64>>;
|
||||
using ClusterShape = Shape<_2,_2,_1>;
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAcc, ElementCompute,
|
||||
ElementAct, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
|
||||
ElementOut, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::conv::Operator::kDgrad,
|
||||
ElementAct, cutlass::layout::TensorNDHWC, 16 / sizeof(ElementAct),
|
||||
ElementFlt, cutlass::layout::TensorNDHWC, 16 / sizeof(ElementFlt),
|
||||
ElementAcc,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::conv::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
|
||||
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
|
||||
ProblemShape,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
|
||||
|
||||
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>());
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// Dynamic cluster
|
||||
//////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
//
|
||||
// CTA tile shape 64x64x64
|
||||
// preferred cluster shape 2x4x1
|
||||
// fallback cluster shape 2x2x1
|
||||
//
|
||||
TEST(SM100_device_conv3d_dgrad_implicitgemm_f8ndhwc_f8ndhwc_f8ndhwc_tensor_op_f32, 64x64x64_preferred_2x4x1_fallback_2x2x1) {
|
||||
using ElementAct = cutlass::float_e4m3_t;
|
||||
using ElementFlt = cutlass::float_e4m3_t;
|
||||
using ElementOut = cutlass::float_e4m3_t;
|
||||
using ElementAcc = float;
|
||||
using ElementCompute = float;
|
||||
using MmaTileShape = Shape<_64, _64, Shape<_64>>;
|
||||
using ClusterShape = decltype(make_shape(int(0), int(0), Int<1>{}));
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
ElementAcc, ElementCompute,
|
||||
ElementAct, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementAct>::value,
|
||||
ElementOut, cutlass::layout::TensorNDHWC, 128 / cutlass::sizeof_bits<ElementOut>::value,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::conv::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::conv::Operator::kDgrad,
|
||||
ElementAct, cutlass::layout::TensorNDHWC, 16 / sizeof(ElementAct),
|
||||
ElementFlt, cutlass::layout::TensorNDHWC, 16 / sizeof(ElementFlt),
|
||||
ElementAcc,
|
||||
MmaTileShape, ClusterShape,
|
||||
cutlass::conv::collective::StageCountAutoCarveout<static_cast<int>(sizeof(typename CollectiveEpilogue::SharedStorage))>,
|
||||
cutlass::conv::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using ProblemShape=cutlass::conv::ConvProblemShape<CollectiveMainloop::DispatchPolicy::ConvOp, CollectiveMainloop::DispatchPolicy::NumSpatialDimensions>;
|
||||
using ConvKernel = cutlass::conv::kernel::ConvUniversal<
|
||||
ProblemShape,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Conv = cutlass::conv::device::ConvUniversalAdapter<ConvKernel>;
|
||||
|
||||
EXPECT_TRUE(test::conv::device::TestAllConv<Conv>(1.0, 0.0, 0.0f, dim3(2,4,1), dim3(2,2,1)));
|
||||
}
|
||||
|
||||
#endif // defined(CUTLASS_ARCH_MMA_SM100_SUPPORTED)
|
||||
Reference in New Issue
Block a user