3.6.0 update (#2005)
* 3.6.0 update * doc and swap stuff --------- Co-authored-by: yuzhai <yuzhai@nvidia.com> Co-authored-by: Haicheng Wu <haichengw@nvidia.com>
This commit is contained in:
Yujia Zhai
co-authored by
yuzhai
Haicheng Wu
yuzhai
Haicheng Wu
parent
e1cd8c7866
commit
3d261a5974
@@ -26,52 +26,30 @@
|
||||
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
add_custom_target(
|
||||
cutlass_test_unit_gemm_device
|
||||
DEPENDS
|
||||
cutlass_test_unit_gemm_device_simt
|
||||
cutlass_test_unit_gemm_device_tensorop_sm70
|
||||
cutlass_test_unit_gemm_device_tensorop_sm75
|
||||
cutlass_test_unit_gemm_device_tensorop_f16_sm80
|
||||
cutlass_test_unit_gemm_device_tensorop_f32_sm80
|
||||
cutlass_test_unit_gemm_device_tensorop_f32_tf32_sm80
|
||||
cutlass_test_unit_gemm_device_tensorop_f64
|
||||
cutlass_test_unit_gemm_device_tensorop_s32_sm80
|
||||
cutlass_test_unit_gemm_device_wmma
|
||||
cutlass_test_unit_gemm_device_tensorop_planar_complex
|
||||
cutlass_test_unit_gemm_device_sparse_tensorop_sm80
|
||||
cutlass_test_unit_gemv_device
|
||||
cutlass_test_unit_gemm_device_tensorop_sm90
|
||||
cutlass_test_unit_sparse_gemm_device_tensorop_sm90
|
||||
cutlass_test_unit_gemm_device_tensorop_cluster_multicast_sm90
|
||||
)
|
||||
add_custom_target(cutlass_test_unit_gemm_device)
|
||||
add_custom_target(test_unit_gemm_device)
|
||||
|
||||
add_custom_target(
|
||||
test_unit_gemm_device
|
||||
DEPENDS
|
||||
test_unit_gemm_device_simt
|
||||
test_unit_gemm_device_tensorop_sm70
|
||||
test_unit_gemm_device_tensorop_sm75
|
||||
test_unit_gemm_device_tensorop_f16_sm80
|
||||
test_unit_gemm_device_tensorop_f32_sm80
|
||||
test_unit_gemm_device_tensorop_f32_tf32_sm80
|
||||
test_unit_gemm_device_tensorop_f64
|
||||
test_unit_gemm_device_tensorop_s32_sm80
|
||||
test_unit_gemm_device_wmma
|
||||
test_unit_gemm_device_tensorop_planar_complex
|
||||
test_unit_gemm_device_sparse_tensorop_sm80
|
||||
test_unit_gemv_device
|
||||
test_unit_gemm_device_tensorop_sm90
|
||||
)
|
||||
################################################################################
|
||||
|
||||
add_custom_target(
|
||||
cutlass_test_unit_gemm_device_sm90
|
||||
DEPENDS
|
||||
cutlass_test_unit_gemm_device_tensorop_sm90
|
||||
cutlass_test_unit_gemm_device_tensorop_cluster_multicast_sm90
|
||||
)
|
||||
function(cutlass_test_unit_gemm_device_add_deps NAME)
|
||||
string(REGEX REPLACE "^cutlass_" "" TEST_NAME "${NAME}")
|
||||
add_dependencies(cutlass_test_unit_gemm_device ${NAME})
|
||||
add_dependencies(test_unit_gemm_device ${TEST_NAME})
|
||||
endfunction()
|
||||
|
||||
cutlass_test_unit_add_executable(
|
||||
function(cutlass_test_unit_gemm_device_add_executable NAME)
|
||||
cutlass_test_unit_add_executable(${NAME} ${ARGN} DO_NOT_LOWERCASE_TEST_NAME)
|
||||
cutlass_test_unit_gemm_device_add_deps(${NAME})
|
||||
endfunction()
|
||||
|
||||
function(cutlass_test_unit_gemm_device_add_executable_split_file NAME)
|
||||
cutlass_test_unit_add_executable_split_file(${NAME} ${ARGN} DO_NOT_LOWERCASE_TEST_NAME)
|
||||
cutlass_test_unit_gemm_device_add_deps(${NAME})
|
||||
endfunction()
|
||||
|
||||
################################################################################
|
||||
|
||||
cutlass_test_unit_gemm_device_add_executable(
|
||||
cutlass_test_unit_gemm_device_simt
|
||||
|
||||
BATCH_SOURCES ON
|
||||
@@ -126,7 +104,9 @@ cutlass_test_unit_add_executable(
|
||||
gemm_splitk_simt_sm50.cu
|
||||
)
|
||||
|
||||
cutlass_test_unit_add_executable(
|
||||
list(APPEND CUTLASS_TEST_UNIT_GEMM_DEVICE_LIST cutlass_test_unit_gemm_device_simt)
|
||||
|
||||
cutlass_test_unit_gemm_device_add_executable(
|
||||
cutlass_test_unit_gemm_device_simt_3x
|
||||
|
||||
BATCH_SOURCES ON
|
||||
@@ -139,8 +119,7 @@ cutlass_test_unit_add_executable(
|
||||
sm61_gemm_s8_s8_s32_simt.cu
|
||||
)
|
||||
|
||||
|
||||
cutlass_test_unit_add_executable(
|
||||
cutlass_test_unit_gemm_device_add_executable(
|
||||
cutlass_test_unit_gemm_device_tensorop_sm70
|
||||
|
||||
BATCH_SOURCES ON
|
||||
@@ -159,7 +138,7 @@ cutlass_test_unit_add_executable(
|
||||
gemm_splitk_tensor_op_sm70.cu
|
||||
)
|
||||
|
||||
cutlass_test_unit_add_executable(
|
||||
cutlass_test_unit_gemm_device_add_executable(
|
||||
cutlass_test_unit_gemm_device_tensorop_sm75
|
||||
|
||||
BATCH_SOURCES ON
|
||||
@@ -204,7 +183,7 @@ cutlass_test_unit_add_executable(
|
||||
|
||||
)
|
||||
|
||||
cutlass_test_unit_add_executable(
|
||||
cutlass_test_unit_gemm_device_add_executable(
|
||||
cutlass_test_unit_gemm_device_tensorop_f16_sm80
|
||||
|
||||
BATCH_SOURCES ON
|
||||
@@ -214,7 +193,7 @@ cutlass_test_unit_add_executable(
|
||||
gemm_f16n_f16t_f16t_tensor_op_f16_slicedk_sm80.cu
|
||||
)
|
||||
|
||||
cutlass_test_unit_add_executable(
|
||||
cutlass_test_unit_gemm_device_add_executable(
|
||||
cutlass_test_unit_gemm_device_tensorop_f32_sm80
|
||||
|
||||
BATCH_SOURCES ON
|
||||
@@ -236,7 +215,7 @@ cutlass_test_unit_add_executable(
|
||||
gemm_f16n_f16n_f16n_direct_store_tensor_op_f32_sm80.cu
|
||||
)
|
||||
|
||||
cutlass_test_unit_add_executable(
|
||||
cutlass_test_unit_gemm_device_add_executable(
|
||||
cutlass_test_unit_gemm_device_tensorop_f32_sm80_3x
|
||||
|
||||
sm80_gemm_s8_s8_s32_tensor_op.cu
|
||||
@@ -245,7 +224,7 @@ cutlass_test_unit_add_executable(
|
||||
)
|
||||
|
||||
|
||||
cutlass_test_unit_add_executable(
|
||||
cutlass_test_unit_gemm_device_add_executable(
|
||||
cutlass_test_unit_gemm_device_mixed_input_tensorop_sm80
|
||||
|
||||
BATCH_SOURCES ON
|
||||
@@ -286,13 +265,14 @@ cutlass_test_unit_add_executable(
|
||||
gemm_universal_s8t_s4n_s8t_mixed_input_tensor_op_s32_sm80.cu
|
||||
)
|
||||
|
||||
cutlass_test_unit_add_executable(
|
||||
cutlass_test_unit_gemm_device_add_executable(
|
||||
cutlass_test_unit_gemm_device_tensorop_sm90
|
||||
|
||||
BATCH_SOURCES ON
|
||||
BATCH_SIZE 4
|
||||
|
||||
sm90_gemm_f16_f16_f16_tensor_op.cu
|
||||
sm90_gett_f16_f16_f16_tensor_op.cu
|
||||
sm90_gemm_bf16_bf16_bf16_tensor_op_f32.cu
|
||||
sm90_gemm_s8_s8_s8_tensor_op_s32.cu
|
||||
sm90_gemm_tf32_tf32_f32_tensor_op_f32.cu
|
||||
@@ -302,7 +282,7 @@ cutlass_test_unit_add_executable(
|
||||
sm90_gemm_f8_f8_f8_tensor_op_fp32.cu
|
||||
)
|
||||
|
||||
cutlass_test_unit_add_executable(
|
||||
cutlass_test_unit_gemm_device_add_executable(
|
||||
cutlass_test_unit_gemm_device_tensorop_sm90_stream_k
|
||||
|
||||
sm90_gemm_stream_k_scheduler.cu
|
||||
@@ -311,7 +291,7 @@ cutlass_test_unit_add_executable(
|
||||
)
|
||||
|
||||
# Alignment tests
|
||||
cutlass_test_unit_add_executable(
|
||||
cutlass_test_unit_gemm_device_add_executable(
|
||||
cutlass_test_unit_gemm_device_tensorop_alignx_sm90
|
||||
|
||||
BATCH_SOURCES ON
|
||||
@@ -336,14 +316,14 @@ cutlass_test_unit_add_executable(
|
||||
)
|
||||
|
||||
# Ptr Array test
|
||||
cutlass_test_unit_add_executable(
|
||||
cutlass_test_unit_gemm_device_add_executable(
|
||||
cutlass_test_unit_gemm_device_tensorop_sm90_ptr_array
|
||||
sm90_gemm_f16_f16_f16_tensor_op_f32_ptr_array.cu
|
||||
sm90_gemm_f16_f16_f16_tensor_op_f32_ptr_array_pingpong.cu
|
||||
)
|
||||
|
||||
# Group Gemm test
|
||||
cutlass_test_unit_add_executable(
|
||||
cutlass_test_unit_gemm_device_add_executable(
|
||||
cutlass_test_unit_gemm_device_tensorop_sm90_group_gemm
|
||||
sm90_gemm_f16_f16_f16_tensor_op_f32_group_gemm.cu
|
||||
sm90_gemm_f16_f16_f16_tensor_op_f32_group_gemm_pingpong.cu
|
||||
@@ -351,31 +331,25 @@ cutlass_test_unit_add_executable(
|
||||
|
||||
# Sparse tests
|
||||
# Sparse kernels trigger an ICE in gcc 7.5
|
||||
if (NOT (CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 8.0))
|
||||
cutlass_test_unit_add_executable(
|
||||
cutlass_test_unit_sparse_gemm_device_tensorop_sm90
|
||||
if (NOT (CUTLASS_GNU_HOST_COMPILE AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 8.0))
|
||||
|
||||
cutlass_test_unit_gemm_device_add_executable(
|
||||
cutlass_test_unit_sparse_gemm_device_tensorop_sm90
|
||||
|
||||
# No batching of source to control compiler memory usage
|
||||
BATCH_SOURCES ON
|
||||
BATCH_SIZE 1
|
||||
|
||||
sm90_sparse_gemm_s8_s8_s32_tensor_op_s32.cu
|
||||
sm90_sparse_gemm_f8_f8_f32_tensor_op_f32.cu
|
||||
sm90_sparse_gemm_f16_f16_f32_tensor_op_f32.cu
|
||||
sm90_sparse_gemm_tf32_tf32_f32_tensor_op_f32.cu
|
||||
)
|
||||
|
||||
# No batching of source to control compiler memory usage
|
||||
BATCH_SOURCES ON
|
||||
BATCH_SIZE 1
|
||||
|
||||
sm90_sparse_gemm_s8_s8_s32_tensor_op_s32.cu
|
||||
sm90_sparse_gemm_f8_f8_f32_tensor_op_f32.cu
|
||||
sm90_sparse_gemm_f16_f16_f32_tensor_op_f32.cu
|
||||
sm90_sparse_gemm_tf32_tf32_f32_tensor_op_f32.cu
|
||||
)
|
||||
else()
|
||||
cutlass_test_unit_add_executable(
|
||||
cutlass_test_unit_sparse_gemm_device_tensorop_sm90
|
||||
|
||||
# No batching of source to control compiler memory usage
|
||||
BATCH_SOURCES ON
|
||||
BATCH_SIZE 1
|
||||
)
|
||||
endif()
|
||||
|
||||
# Fused epilogue tests
|
||||
cutlass_test_unit_add_executable(
|
||||
cutlass_test_unit_gemm_device_add_executable(
|
||||
cutlass_test_unit_gemm_device_tensorop_epilogue_fusion_sm90
|
||||
|
||||
BATCH_SOURCES ON
|
||||
@@ -400,7 +374,7 @@ cutlass_test_unit_add_executable(
|
||||
sm90_gemm_f8_f8_bf16_tensor_op_fp32_evt.cu
|
||||
sm90_gemm_f8_f8_f32_tensor_op_f32_cluster_warpspecialized_cooperative_evt.cu
|
||||
)
|
||||
cutlass_test_unit_add_executable(
|
||||
cutlass_test_unit_gemm_device_add_executable(
|
||||
cutlass_test_unit_gemm_device_tensorop_cluster_multicast_sm90
|
||||
|
||||
BATCH_SOURCES ON
|
||||
@@ -412,7 +386,7 @@ cutlass_test_unit_add_executable(
|
||||
sm90_gemm_f16_f16_f16_tensor_op_f32_cluster_warpspecialized_cooperative.cu
|
||||
sm90_gemm_f8_f8_f32_tensor_op_f32_cluster_warpspecialized_cooperative.cu
|
||||
)
|
||||
cutlass_test_unit_add_executable(
|
||||
cutlass_test_unit_gemm_device_add_executable(
|
||||
cutlass_test_unit_gemm_device_tensorop_gmma_rs_warpspecialized_sm90
|
||||
|
||||
BATCH_SOURCES ON
|
||||
@@ -423,7 +397,7 @@ cutlass_test_unit_add_executable(
|
||||
sm90_gemm_f16_f16_f32_tensor_op_f32_rs_cluster_warpspecialized_cooperative.cu
|
||||
)
|
||||
|
||||
cutlass_test_unit_add_executable(
|
||||
cutlass_test_unit_gemm_device_add_executable(
|
||||
cutlass_test_unit_gemm_device_tensorop_f32_tf32_sm80
|
||||
|
||||
BATCH_SOURCES ON
|
||||
@@ -443,7 +417,7 @@ cutlass_test_unit_add_executable(
|
||||
sm80_gemm_f16_f16_f32_tensor_op_f32.cu
|
||||
)
|
||||
|
||||
cutlass_test_unit_add_executable(
|
||||
cutlass_test_unit_gemm_device_add_executable(
|
||||
cutlass_test_unit_gemm_device_tensorop_f64
|
||||
|
||||
BATCH_SOURCES ON
|
||||
@@ -471,7 +445,7 @@ cutlass_test_unit_add_executable(
|
||||
gemm_cf64t_cf64n_cf64t_tensor_op_f64_gaussian_sm90.cu
|
||||
)
|
||||
|
||||
cutlass_test_unit_add_executable(
|
||||
cutlass_test_unit_gemm_device_add_executable(
|
||||
cutlass_test_unit_gemm_device_tensorop_s32_sm80
|
||||
|
||||
BATCH_SOURCES ON
|
||||
@@ -493,7 +467,7 @@ cutlass_test_unit_add_executable(
|
||||
gemm_s4n_s4t_s4n_tensor_op_s32_sm80.cu
|
||||
)
|
||||
|
||||
cutlass_test_unit_add_executable(
|
||||
cutlass_test_unit_gemm_device_add_executable(
|
||||
cutlass_test_unit_gemm_device_wmma
|
||||
|
||||
BATCH_SOURCES ON
|
||||
@@ -551,7 +525,7 @@ cutlass_test_unit_add_executable(
|
||||
gemm_f16t_f16n_f32t_singlestage_wmma_tensor_op_f32_sm70.cu
|
||||
)
|
||||
|
||||
cutlass_test_unit_add_executable(
|
||||
cutlass_test_unit_gemm_device_add_executable(
|
||||
cutlass_test_unit_gemm_device_tensorop_planar_complex
|
||||
|
||||
BATCH_SOURCES ON
|
||||
@@ -562,7 +536,7 @@ cutlass_test_unit_add_executable(
|
||||
gemm_planar_complex_f16_f16_f32_tensor_op_sm80.cu
|
||||
)
|
||||
|
||||
cutlass_test_unit_add_executable(
|
||||
cutlass_test_unit_gemm_device_add_executable(
|
||||
cutlass_test_unit_gemm_device_tensorop_sm89
|
||||
|
||||
BATCH_SOURCES ON
|
||||
@@ -574,7 +548,7 @@ cutlass_test_unit_add_executable(
|
||||
# gemm_f8t_f8n_f8t_tensor_op_f32_sparse_sm89.cu
|
||||
)
|
||||
|
||||
cutlass_test_unit_add_executable(
|
||||
cutlass_test_unit_gemm_device_add_executable(
|
||||
cutlass_test_unit_gemm_device_grouped
|
||||
|
||||
BATCH_SOURCES ON
|
||||
@@ -583,7 +557,7 @@ cutlass_test_unit_add_executable(
|
||||
gemm_grouped_sm80.cu
|
||||
)
|
||||
|
||||
cutlass_test_unit_add_executable(
|
||||
cutlass_test_unit_gemm_device_add_executable(
|
||||
cutlass_test_unit_gemm_device_grouped_scheduler
|
||||
|
||||
BATCH_SOURCES ON
|
||||
@@ -592,7 +566,7 @@ cutlass_test_unit_add_executable(
|
||||
gemm_grouped_scheduler_sm80.cu
|
||||
)
|
||||
|
||||
cutlass_test_unit_add_executable(
|
||||
cutlass_test_unit_gemm_device_add_executable(
|
||||
cutlass_test_unit_gemm_device_grouped_rank_2k_scheduler
|
||||
|
||||
BATCH_SOURCES ON
|
||||
@@ -601,7 +575,7 @@ cutlass_test_unit_add_executable(
|
||||
rank_2k_grouped_scheduler_sm80.cu
|
||||
)
|
||||
|
||||
cutlass_test_unit_add_executable(
|
||||
cutlass_test_unit_gemm_device_add_executable(
|
||||
cutlass_test_unit_gemm_device_sparse_tensorop_sm80
|
||||
|
||||
BATCH_SOURCES ON
|
||||
@@ -622,7 +596,7 @@ cutlass_test_unit_add_executable(
|
||||
gemm_s4t_s4n_s32t_tensor_op_s32_sparse_sm80.cu
|
||||
)
|
||||
|
||||
cutlass_test_unit_add_executable(
|
||||
cutlass_test_unit_gemm_device_add_executable(
|
||||
cutlass_test_unit_gemv_device
|
||||
|
||||
BATCH_SOURCES ON
|
||||
@@ -631,42 +605,22 @@ cutlass_test_unit_add_executable(
|
||||
gemv.cu
|
||||
)
|
||||
|
||||
if (NOT CUDA_COMPILER MATCHES "[Cc]lang")
|
||||
if (CUTLASS_NVCC_DEVICE_COMPILE)
|
||||
|
||||
add_dependencies(
|
||||
cutlass_test_unit_gemm_device
|
||||
cutlass_test_unit_gemm_device_gemm_with_fused_epilogue_tensorop
|
||||
cutlass_test_unit_gemm_device_add_executable(
|
||||
cutlass_test_unit_gemm_device_gemm_with_fused_epilogue_tensorop
|
||||
|
||||
gemm_with_reduction_f16n_f16n_f16n_tensorop_f32_sm75.cu
|
||||
gemm_with_broadcast_f16n_f16n_f16n_tensorop_f32_sm75.cu
|
||||
|
||||
gemm_with_reduction_f16t_f16n_f16n_tensorop_f32_sm80.cu
|
||||
)
|
||||
|
||||
add_dependencies(
|
||||
test_unit_gemm_device
|
||||
test_unit_gemm_device_gemm_with_fused_epilogue_tensorop
|
||||
)
|
||||
|
||||
cutlass_test_unit_add_executable(
|
||||
cutlass_test_unit_gemm_device_gemm_with_fused_epilogue_tensorop
|
||||
|
||||
gemm_with_reduction_f16n_f16n_f16n_tensorop_f32_sm75.cu
|
||||
gemm_with_broadcast_f16n_f16n_f16n_tensorop_f32_sm75.cu
|
||||
|
||||
gemm_with_reduction_f16t_f16n_f16n_tensorop_f32_sm80.cu
|
||||
)
|
||||
|
||||
endif()
|
||||
|
||||
if (NOT CUDA_COMPILER MATCHES "[Cc]lang")
|
||||
if (CUTLASS_NVCC_DEVICE_COMPILE)
|
||||
|
||||
add_dependencies(
|
||||
cutlass_test_unit_gemm_device
|
||||
cutlass_test_unit_gemm_device_blas3
|
||||
)
|
||||
|
||||
add_dependencies(
|
||||
test_unit_gemm_device
|
||||
test_unit_gemm_device_blas3
|
||||
)
|
||||
|
||||
cutlass_test_unit_add_executable(
|
||||
cutlass_test_unit_gemm_device_add_executable(
|
||||
cutlass_test_unit_gemm_device_blas3
|
||||
|
||||
BATCH_SOURCES ON
|
||||
@@ -833,7 +787,7 @@ cutlass_test_unit_add_executable(
|
||||
hemm_cf64_cf64_cf64_tensor_op_f64_sm90.cu
|
||||
)
|
||||
|
||||
cutlass_test_unit_add_executable(
|
||||
cutlass_test_unit_gemm_device_add_executable(
|
||||
cutlass_test_unit_gemm_device_grouped_blas3
|
||||
|
||||
BATCH_SOURCES ON
|
||||
@@ -858,13 +812,12 @@ cutlass_test_unit_add_executable(
|
||||
|
||||
endif()
|
||||
|
||||
if (NOT CUDA_COMPILER MATCHES "[Cc]lang")
|
||||
if (CUTLASS_NVCC_DEVICE_COMPILE)
|
||||
|
||||
cutlass_test_unit_add_executable(
|
||||
cutlass_test_unit_gemm_device_broadcast
|
||||
|
||||
gemm_f16t_f16n_f16t_tensor_op_f16_broadcast_sm80.cu
|
||||
)
|
||||
cutlass_test_unit_gemm_device_add_executable(
|
||||
cutlass_test_unit_gemm_device_broadcast
|
||||
gemm_f16t_f16n_f16t_tensor_op_f16_broadcast_sm80.cu
|
||||
)
|
||||
|
||||
endif()
|
||||
|
||||
|
||||
@@ -39,6 +39,7 @@
|
||||
#include <sstream>
|
||||
#include <algorithm>
|
||||
#include <random>
|
||||
#include <numeric> // std::lcm
|
||||
|
||||
#include "../../common/cutlass_unit_test.h"
|
||||
#include "cutlass/util/host_tensor.h"
|
||||
@@ -55,6 +56,7 @@
|
||||
#include "cutlass/complex.h"
|
||||
#include "cutlass/transform/device/transform_universal_adapter.hpp"
|
||||
#include "cutlass/transform/kernel/sparse_gemm_compressor.hpp"
|
||||
#include "cutlass/detail/collective.hpp"
|
||||
|
||||
#include "testbed_utils.h"
|
||||
|
||||
@@ -151,6 +153,12 @@ struct ElementScalarType<Gemm, Default, std::void_t<typename Gemm::EpilogueOutpu
|
||||
using Type = typename Gemm::EpilogueOutputOp::ElementScalar;
|
||||
};
|
||||
|
||||
template<class CollectiveEpilogue, class = void>
|
||||
struct IsSfdEpi : cute::false_type {};
|
||||
|
||||
template<class CollectiveEpilogue>
|
||||
struct IsSfdEpi<CollectiveEpilogue, cute::void_t<typename CollectiveEpilogue::FusionCallbacks::Operation::GmemLayoutTagScalefactor>> : cute::true_type {};
|
||||
|
||||
// The maximum swizzle size to use
|
||||
//
|
||||
// This class, like Splits above makes it harder to confuse
|
||||
@@ -1140,7 +1148,6 @@ struct HostCollectiveEpilogue {
|
||||
static constexpr bool IsAbsMaxEnabledAux = IsAuxOutEnabled && FusionOp::IsAbsMaxSupported &&
|
||||
(cute::is_same_v<ElementAux, cutlass::float_e4m3_t> ||
|
||||
cute::is_same_v<ElementAux, cutlass::float_e5m2_t>);
|
||||
|
||||
using Arguments = typename Gemm::GemmKernel::EpilogueArguments;
|
||||
|
||||
/// Initialization
|
||||
@@ -1454,6 +1461,22 @@ struct HostCollectiveEpilogue {
|
||||
|
||||
bool passed = equality_check(reference_D.host_view(), tensor_D.host_view());
|
||||
if(!passed) {
|
||||
#if 0
|
||||
auto [M, N, K, L] = problem_shape_MNKL;
|
||||
auto ref = cute::make_tensor(detail::make_iterator(reference_D.host_data()),
|
||||
cute::make_layout(cute::make_shape(M, N, L), stride_d));
|
||||
auto comp = cute::make_tensor(detail::make_iterator(tensor_D.host_data()),
|
||||
cute::make_layout(cute::make_shape(M, N, L), stride_d));
|
||||
for(int i=0; i<M; i++) {
|
||||
for(int j=0; j<N; j++) {
|
||||
for(int l=0; l<L; l++) {
|
||||
if(static_cast<float>(ElementD(ref(i, j, l))) != static_cast<float>((ElementD(comp(i, j, l))))) {
|
||||
printf("<m %d, n %d, l %d> ref: %f comp: %f\n", i, j, l, static_cast<float>(ElementD(ref(i, j, l))), static_cast<float>((ElementD(comp(i, j, l)))));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
#endif
|
||||
std::cout<<"D is incorrect"<<std::endl;
|
||||
}
|
||||
|
||||
@@ -1575,9 +1598,12 @@ struct HostCollectiveEpilogue {
|
||||
}
|
||||
else {
|
||||
fusion_args.alpha = alpha.at(coord_0);
|
||||
fusion_args.beta = beta.at(coord_0);
|
||||
fusion_args.alpha_ptr = alpha.device_data();
|
||||
fusion_args.beta_ptr = beta.device_data(); // if vector_scale_mode is true this is nullptr
|
||||
// Only initializing beta/beta_ptr for non-void source
|
||||
if constexpr (not cute::is_void_v<typename kernel::ElementC>) {
|
||||
fusion_args.beta = beta.at(coord_0);
|
||||
fusion_args.beta_ptr = beta.device_data(); // if vector_scale_mode is true this is nullptr
|
||||
}
|
||||
|
||||
if constexpr (IsPerRowScaleEnabled) {
|
||||
int32_t m_stride = vector_scale_mode == VectorScale::ENABLED ? 1 : 0;
|
||||
@@ -1620,6 +1646,7 @@ struct HostCollectiveEpilogue {
|
||||
// example of how to set kernel activation arguments
|
||||
// see ActivationFunctor::Arguments in activation.h for definition
|
||||
// if Arguments doesn't exist then fusion_args.activation is empty
|
||||
|
||||
if constexpr (cute::is_same_v<ActivationFunctor, cutlass::epilogue::thread::ScaledGELU_taylor<ElementCompute>>) {
|
||||
fusion_args.activation.scale = ElementCompute(1);
|
||||
}
|
||||
@@ -1713,6 +1740,7 @@ struct HostCollectiveEpilogue {
|
||||
decltype(Vbeta),
|
||||
ActivationFunctor,
|
||||
cutlass::plus<ElementCompute>
|
||||
, false /*PerColumnBias_*/
|
||||
> epilogue_params{};
|
||||
|
||||
epilogue_params.C = C;
|
||||
@@ -1779,6 +1807,7 @@ struct TestbedImpl {
|
||||
using ScheduleType = typename Gemm::GemmKernel::CollectiveMainloop::DispatchPolicy::Schedule;
|
||||
// All Collective MMA operands are defined by HostCollectiveMainloopType based on the schedule type
|
||||
using HostCollectiveMainloopType = HostCollectiveMainloop<ScheduleType, Gemm, ElementA, ElementB>;
|
||||
|
||||
using CollectiveEpilogue = cute::conditional_t<IsDefaultEpilogue<typename Gemm::GemmKernel::CollectiveEpilogue>::value || force_legacy_epilogue,
|
||||
HostCollectiveDefaultEpilogue<Gemm>,
|
||||
HostCollectiveEpilogue<Gemm>>;
|
||||
@@ -2004,7 +2033,7 @@ struct TestbedImpl {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
catch (std::exception const& e) {
|
||||
catch ([[maybe_unused]] std::exception const& e) {
|
||||
CUTLASS_TRACE_HOST("TestbedImpl::run: this->initialize threw an exception: " << e.what());
|
||||
throw;
|
||||
}
|
||||
|
||||
@@ -346,7 +346,7 @@ struct HostCollectiveMainloop {
|
||||
stride_b_host.clear();
|
||||
|
||||
auto [M, N, K, L] = cute::append<4>(problem_shapes.get_host_problem_shape(0), 1);
|
||||
L = max(problem_shapes.groups(), L);
|
||||
L = cutlass::platform::max(problem_shapes.groups(), L);
|
||||
|
||||
for(int32_t i = 0; i < L; ++i) {
|
||||
auto [M, N, K, mock_L] = cute::append<4>(problem_shapes.get_host_problem_shape(i), 1);
|
||||
@@ -380,7 +380,7 @@ struct HostCollectiveMainloop {
|
||||
|
||||
Arguments to_args(ProblemShapeType problem_shapes) {
|
||||
auto [M, N, K, L] = cute::append<4>(problem_shapes.get_host_problem_shape(0), 1);
|
||||
L = max(problem_shapes.groups(), L);
|
||||
L = cutlass::platform::max(problem_shapes.groups(), L);
|
||||
|
||||
std::vector<ElementA *> ptr_A_host(L);
|
||||
std::vector<ElementB *> ptr_B_host(L);
|
||||
@@ -587,7 +587,7 @@ struct HostCollectiveDefaultEpilogue {
|
||||
stride_d_host.clear();
|
||||
|
||||
auto [M, N, K, L] = cute::append<4>(problem_shapes.get_host_problem_shape(0), 1);
|
||||
L = max(problem_shapes.groups(), L);
|
||||
L = cutlass::platform::max(problem_shapes.groups(), L);
|
||||
|
||||
for (int32_t i = 0; i < L; ++i) {
|
||||
auto [M, N, K, mock_L] = cute::append<4>(problem_shapes.get_host_problem_shape(i), 1);
|
||||
@@ -649,7 +649,7 @@ struct HostCollectiveDefaultEpilogue {
|
||||
ElementScalar beta,
|
||||
int batch) {
|
||||
auto [M, N, K, L] = cute::append<4>(problem_shapes.get_host_problem_shape(0), 1);
|
||||
L = max(problem_shapes.groups(), L);
|
||||
L = cutlass::platform::max(problem_shapes.groups(), L);
|
||||
|
||||
tensors_D[batch].sync_host();
|
||||
EXPECT_GT(cutlass::reference::host::TensorNorm(tensors_C[batch].host_view()), 0);
|
||||
@@ -678,7 +678,7 @@ struct HostCollectiveDefaultEpilogue {
|
||||
|
||||
Arguments to_args(ProblemShapeType problem_shapes) {
|
||||
auto [M, N, K, L] = cute::append<4>(problem_shapes.get_host_problem_shape(0), 1);
|
||||
L = max(problem_shapes.groups(), L);
|
||||
L = cutlass::platform::max(problem_shapes.groups(), L);
|
||||
|
||||
std::vector<ElementC *> ptr_C_host(L);
|
||||
std::vector<ElementD *> ptr_D_host(L);
|
||||
@@ -724,8 +724,8 @@ struct HostCollectiveDefaultEpilogue {
|
||||
//
|
||||
// Allocate the GEMM workspace
|
||||
//
|
||||
auto [M, N, K, L] = cute::append<4>(problem_shapes.get_host_problem_shape(0), 1);
|
||||
L = max(problem_shapes.groups(), L);
|
||||
auto [M, N, K, L] = cute::append<4>(problem_shapes.get_host_problem_shape(batch), 1);
|
||||
L = std::max(problem_shapes.groups(), L);
|
||||
|
||||
auto coord_0 = cutlass::make_Coord(0);
|
||||
auto C = cute::make_tensor(detail::make_iterator(tensors_C[batch].host_data()),
|
||||
@@ -905,9 +905,8 @@ struct HostCollectiveEpilogue {
|
||||
references_D.clear();
|
||||
stride_c_host.clear();
|
||||
stride_d_host.clear();
|
||||
|
||||
auto [M, N, K, L] = cute::append<4>(problem_shapes.get_host_problem_shape(0), 1);
|
||||
L = max(problem_shapes.groups(), L);
|
||||
L = std::max(problem_shapes.groups(), L);
|
||||
|
||||
for (int32_t i = 0; i < L; ++i) {
|
||||
auto [M, N, K, mock_L] = cute::append<4>(problem_shapes.get_host_problem_shape(i), 1);
|
||||
@@ -1118,7 +1117,6 @@ struct HostCollectiveEpilogue {
|
||||
passed &= tmp;
|
||||
}
|
||||
}
|
||||
|
||||
return passed;
|
||||
}
|
||||
|
||||
@@ -1189,7 +1187,7 @@ struct HostCollectiveEpilogue {
|
||||
Arguments to_args(ProblemShapeType problem_shapes) {
|
||||
auto coord_0 = cutlass::make_Coord(0);
|
||||
auto [M, N, K, L] = cute::append<4>(problem_shapes.get_host_problem_shape(0), 1);
|
||||
L = max(problem_shapes.groups(), L);
|
||||
L = std::max(problem_shapes.groups(), L);
|
||||
|
||||
std::vector<ElementC *> ptr_C_host(L);
|
||||
std::vector<ElementD *> ptr_D_host(L);
|
||||
@@ -1220,19 +1218,22 @@ struct HostCollectiveEpilogue {
|
||||
device_tensors_Aux.copy_from_host(ptr_Aux_host.data());
|
||||
}
|
||||
|
||||
auto device_tensors_C_ptr = cute::is_void_v<typename kernel::ElementC> ? nullptr :
|
||||
reinterpret_cast<typename kernel::ElementC const**>(device_tensors_C.get());
|
||||
|
||||
Arguments arguments;
|
||||
if constexpr (IsGroupGemm) {
|
||||
arguments =
|
||||
{
|
||||
{},
|
||||
device_tensors_C.get(), stride_c_device.get(), device_tensors_D.get(), stride_d_device.get()
|
||||
device_tensors_C_ptr, stride_c_device.get(), device_tensors_D.get(), stride_d_device.get()
|
||||
};
|
||||
}
|
||||
else {
|
||||
arguments =
|
||||
{
|
||||
{},
|
||||
device_tensors_C.get(), stride_c_host[0], device_tensors_D.get(), stride_d_host[0]
|
||||
device_tensors_C_ptr, stride_c_host[0], device_tensors_D.get(), stride_d_host[0]
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1252,7 +1253,9 @@ struct HostCollectiveEpilogue {
|
||||
fusion_args.beta = beta.at(coord_0);
|
||||
|
||||
fusion_args.alpha_ptr = alpha.device_data();
|
||||
fusion_args.beta_ptr = beta.device_data();
|
||||
// can_implement requires beta_ptr to not be set if its voidC
|
||||
fusion_args.beta_ptr = cute::is_void_v<typename kernel::ElementC> ? nullptr :
|
||||
beta.device_data();
|
||||
|
||||
if constexpr (IsScaleFactorEnabled) {
|
||||
fusion_args.scale_a = scale_A.at(coord_0);
|
||||
@@ -1316,7 +1319,8 @@ struct HostCollectiveEpilogue {
|
||||
//
|
||||
// Allocate the GEMM workspace
|
||||
//
|
||||
auto [M, N, K, L] = cute::append<4>(problem_shapes.get_host_problem_shape(batch), 1);
|
||||
auto problem_shape_MNKL = cute::append<4>(problem_shapes.get_host_problem_shape(batch), 1);
|
||||
auto [M, N, K, L] = problem_shape_MNKL;
|
||||
auto coord_0 = cutlass::make_Coord(0);
|
||||
auto C = cute::make_tensor(detail::make_iterator(tensors_C[batch].host_data()),
|
||||
cute::make_layout(cute::make_shape(M, N, 1), stride_c_host[batch]));
|
||||
@@ -1338,7 +1342,6 @@ struct HostCollectiveEpilogue {
|
||||
cute::make_layout(cute::make_shape(M, N, cute::_1{}), cute::make_stride(cute::_1{}, cute::_0{}, M)));
|
||||
auto Vbeta = cute::make_tensor(detail::make_iterator(beta.host_data()),
|
||||
cute::make_layout(cute::make_shape(M, N, cute::_1{}), cute::make_stride(cute::_1{}, cute::_0{}, N)));
|
||||
|
||||
cutlass::reference::host::GettEpilogueParams<
|
||||
ElementScalar,
|
||||
ElementScalar,
|
||||
@@ -1518,7 +1521,7 @@ struct TestbedImpl {
|
||||
{
|
||||
using namespace cute;
|
||||
auto [M, N, K, L] = cute::append<4>(problem_shapes.get_host_problem_shape(0), 1);
|
||||
L = max(problem_shapes.groups(), L);
|
||||
L = std::max(problem_shapes.groups(), L);
|
||||
|
||||
bool passed = true;
|
||||
for (int32_t i = 0; i < L; ++i) {
|
||||
@@ -1760,7 +1763,7 @@ bool TestAll(double alpha = 1.0, double beta = 0.0, CheckEquality check_relative
|
||||
cutlass::DeviceAllocation<typename ProblemShapeType::UnderlyingProblemShape> problem_sizes_device;
|
||||
|
||||
for (int i = 0; i < batch; ++i) {
|
||||
problem_sizes_host.push_back({m, n, k});
|
||||
problem_sizes_host.push_back({m * ((i % 3) + 1), n * ((i % 4) + 1), k * ((i % 5) + 1)});
|
||||
}
|
||||
|
||||
problem_sizes_device.reset(problem_sizes_host.size());
|
||||
|
||||
@@ -0,0 +1,184 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017 - 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/*! \file
|
||||
\brief Tests for device-wide GEMM interface
|
||||
*/
|
||||
|
||||
#include <iostream>
|
||||
|
||||
#include <thrust/device_vector.h>
|
||||
#include <thrust/host_vector.h>
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cute/tensor.hpp"
|
||||
#include "cute/atom/mma_atom.hpp"
|
||||
|
||||
#include "cutlass/numeric_types.h"
|
||||
|
||||
#include "cutlass/gemm/device/gemm_universal_adapter.h"
|
||||
#include "cutlass/gemm/kernel/gemm_universal.hpp"
|
||||
#include "cutlass/epilogue/collective/collective_builder.hpp"
|
||||
#include "cutlass/gemm/collective/collective_builder.hpp"
|
||||
#include "cutlass/epilogue/collective/sm70_epilogue_vectorized.hpp"
|
||||
#include "cutlass/epilogue/collective/default_epilogue.hpp"
|
||||
#include "cutlass/epilogue/thread/linear_combination.h"
|
||||
|
||||
#include "../../common/cutlass_unit_test.h"
|
||||
|
||||
#include "cutlass/util/reference/device/gett.hpp"
|
||||
#include "cutlass/util/reference/device/tensor_compare.h"
|
||||
|
||||
#if defined(CUTLASS_ARCH_MMA_SM90_SUPPORTED)
|
||||
|
||||
using namespace cute;
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM90_Device_Gett_f16t_f16n_f16n_tensor_op_gmma_f16, 8x8x8x8x8x8) {
|
||||
|
||||
using BatModeStrides = int;
|
||||
|
||||
using RowModeStridesA = cute::Stride<int, int>;
|
||||
using RedModeStrides = cute::Stride<cute::_1, int>;
|
||||
|
||||
using ColModeStridesB = cute::Stride<int, int>;
|
||||
|
||||
using RowModeStridesC = cute::Stride<cute::_1, int>;
|
||||
using ColModeStridesC = cute::Stride<int, int>;
|
||||
|
||||
using StrideA = cute::Stride<RowModeStridesA, RedModeStrides, BatModeStrides>;
|
||||
using StrideB = cute::Stride<ColModeStridesB, RedModeStrides, BatModeStrides>;
|
||||
using StrideC = cute::Stride<RowModeStridesC, ColModeStridesC, BatModeStrides>;
|
||||
using StrideD = StrideC;
|
||||
|
||||
using TileShape = Shape<Shape<_8, _8>, Shape<_8, _8>, Shape<_8, _8>>;
|
||||
|
||||
using CollectiveOp = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::half_t, StrideA, 8,
|
||||
cutlass::half_t, StrideB, 8,
|
||||
cutlass::half_t,
|
||||
TileShape, Shape<_1,_1,_1>,
|
||||
cutlass::gemm::collective::StageCountAuto,
|
||||
cutlass::gemm::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveEpilogue = typename cutlass::epilogue::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
|
||||
TileShape, Shape<_1,_1,_1>,
|
||||
cutlass::epilogue::collective::EpilogueTileAuto,
|
||||
cutlass::half_t, cutlass::half_t,
|
||||
cutlass::half_t, StrideC, 8,
|
||||
cutlass::half_t, StrideC, 8,
|
||||
cutlass::epilogue::collective::EpilogueScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using GettKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<Shape<int,int>,
|
||||
Shape<int,int>,
|
||||
Shape<int,int>,
|
||||
int>,
|
||||
CollectiveOp,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Gett = cutlass::gemm::device::GemmUniversalAdapter<GettKernel>;
|
||||
|
||||
auto problem_shape = make_shape(
|
||||
make_shape(32,8),
|
||||
make_shape(32,4),
|
||||
make_shape(32,2),
|
||||
1
|
||||
);
|
||||
|
||||
auto [M, N, K, L] = problem_shape;
|
||||
|
||||
StrideA dA = make_stride(make_stride(64, 2048), make_stride(_1{}, 32), size(M) * size(K));
|
||||
StrideB dB = make_stride(make_stride(64, 2048), make_stride(_1{}, 32), size(N) * size(K));
|
||||
StrideC dC = make_stride(make_stride(_1{}, 32), make_stride(256, 8192), size(M) * size(N));
|
||||
StrideD dD = dC;
|
||||
|
||||
cutlass::half_t alpha = cutlass::half_t(1.0f);
|
||||
cutlass::half_t beta = cutlass::half_t(1.0f);
|
||||
|
||||
thrust::host_vector<cutlass::half_t> A_h(size(M) * size(K) * size(L));
|
||||
thrust::host_vector<cutlass::half_t> B_h(size(N) * size(K) * size(L));
|
||||
thrust::host_vector<cutlass::half_t> C_h(size(M) * size(N) * size(L));
|
||||
thrust::host_vector<cutlass::half_t> D_h(size(M) * size(N) * size(L));
|
||||
thrust::host_vector<cutlass::half_t> D_h_ref(size(M) * size(N) * size(L));
|
||||
|
||||
for (auto& a : A_h) a = cutlass::half_t(static_cast<int>(4 * (rand() / double(RAND_MAX) - 1)));
|
||||
for (auto& b : B_h) b = cutlass::half_t(static_cast<int>(4 * (rand() / double(RAND_MAX) - 1)));
|
||||
for (auto& c : C_h) c = cutlass::half_t(static_cast<int>(4 * (rand() / double(RAND_MAX) - 1)));
|
||||
for (auto& d : D_h) d = cutlass::half_t(-1);
|
||||
for (auto& d : D_h_ref) d = cutlass::half_t(-1);
|
||||
|
||||
thrust::device_vector<cutlass::half_t> A = A_h;
|
||||
thrust::device_vector<cutlass::half_t> B = B_h;
|
||||
thrust::device_vector<cutlass::half_t> C = C_h;
|
||||
thrust::device_vector<cutlass::half_t> D = D_h;
|
||||
thrust::device_vector<cutlass::half_t> D_ref = D_h_ref;
|
||||
|
||||
typename Gett::Arguments args {
|
||||
cutlass::gemm::GemmUniversalMode::kBatched,
|
||||
problem_shape,
|
||||
{A.data().get(), dA, B.data().get(), dB},
|
||||
{ {alpha, beta}, C.data().get(), dC, D.data().get(), dD}
|
||||
};
|
||||
|
||||
Gett gett;
|
||||
auto status = gett(args);
|
||||
EXPECT_TRUE(status == cutlass::Status::kSuccess);
|
||||
auto cuda_err = cudaDeviceSynchronize();
|
||||
|
||||
EXPECT_TRUE(cuda_err == cudaSuccess);
|
||||
|
||||
cutlass::reference::device::gett(
|
||||
problem_shape,
|
||||
A.data().get(), dA,
|
||||
B.data().get(), dB,
|
||||
cutlass::half_t(0.0f),
|
||||
C.data().get(), dC,
|
||||
D_ref.data().get(), dD,
|
||||
alpha, beta);
|
||||
|
||||
cuda_err = cudaDeviceSynchronize();
|
||||
EXPECT_TRUE(cuda_err == cudaSuccess);
|
||||
|
||||
bool passed = cutlass::reference::device::BlockCompareEqual(
|
||||
D.data().get(), D_ref.data().get(), D_ref.size());
|
||||
EXPECT_TRUE(passed);
|
||||
}
|
||||
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#endif // defined(CUTLASS_ARCH_MMA_SM90_SUPPORTED)
|
||||
Reference in New Issue
Block a user