Vijay Thakkar
@@ -28,6 +28,8 @@
|
||||
|
||||
include(CTest)
|
||||
|
||||
set(CUTLASS_UNIT_TEST_COMMON_DIR ${CMAKE_CURRENT_LIST_DIR}/common)
|
||||
|
||||
cutlass_add_library(
|
||||
cutlass_test_unit_infra
|
||||
OBJECT
|
||||
@@ -42,6 +44,7 @@ target_link_libraries(
|
||||
$<$<BOOL:${CUTLASS_ENABLE_CUBLAS}>:nvidia::cublas>
|
||||
gtest
|
||||
cudart
|
||||
cuda_driver
|
||||
)
|
||||
|
||||
cutlass_add_library(
|
||||
@@ -69,6 +72,12 @@ function(cutlass_test_unit_add_executable NAME)
|
||||
|
||||
target_compile_definitions(${NAME} PUBLIC CUTLASS_TARGET_NAME="${NAME}")
|
||||
|
||||
target_include_directories(
|
||||
${NAME}
|
||||
PRIVATE
|
||||
${CUTLASS_UNIT_TEST_COMMON_DIR}
|
||||
)
|
||||
|
||||
target_link_libraries(
|
||||
${NAME}
|
||||
PRIVATE
|
||||
@@ -76,6 +85,10 @@ function(cutlass_test_unit_add_executable NAME)
|
||||
cutlass_test_unit_infra_lib
|
||||
)
|
||||
|
||||
if (CUTLASS_ENABLE_OPENMP_TESTS AND OpenMP_CXX_FOUND)
|
||||
target_link_libraries(${NAME} PRIVATE OpenMP::OpenMP_CXX)
|
||||
endif()
|
||||
|
||||
string(REGEX REPLACE cutlass_ "" NAME_STEM ${NAME})
|
||||
|
||||
set(RESULT_CACHE_FILE "${CUTLASS_TEST_UNIT_RESULTS_CACHE_DIR}/cached_results_${NAME}.txt")
|
||||
@@ -99,6 +112,7 @@ add_custom_target(test_unit)
|
||||
|
||||
set(SUBDIRS
|
||||
core
|
||||
cute
|
||||
gemm
|
||||
conv
|
||||
layout
|
||||
@@ -106,6 +120,7 @@ set(SUBDIRS
|
||||
epilogue
|
||||
reduction
|
||||
util
|
||||
pipeline
|
||||
)
|
||||
|
||||
if(TARGET nvidia::nvrtc AND TARGET nvidia::cuda_driver)
|
||||
|
||||
@@ -39,6 +39,17 @@
|
||||
|
||||
#include <cstdlib>
|
||||
#include <string>
|
||||
|
||||
#include <cuda_runtime_api.h>
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Gets a CUDA device
|
||||
cudaDeviceProp GetCudaDevice();
|
||||
|
||||
/// Prints device properties
|
||||
std::ostream &operator<<(std::ostream &out, cudaDeviceProp const &device);
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Sets flags for Unit test
|
||||
@@ -52,7 +63,6 @@ int CutlassUnitTestProblemCount();
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
|
||||
// active test macro
|
||||
#define CUTLASS_TEST_LEVEL_ACTIVE(LEVEL,NAME_STATIC,NAME_DYNAMIC,...) \
|
||||
TEST(NAME_STATIC,L##LEVEL##_##NAME_DYNAMIC) __VA_ARGS__
|
||||
@@ -78,3 +88,15 @@ int CutlassUnitTestProblemCount();
|
||||
#if !defined(CUTLASS_TEST_UNIT_ENABLE_WARNINGS)
|
||||
#define CUTLASS_TEST_UNIT_ENABLE_WARNINGS false
|
||||
#endif
|
||||
|
||||
#if (__CUDACC_VER_MAJOR__ >= 12)
|
||||
#define CUDA_12_0_SM90_FEATURES_SUPPORTED true
|
||||
#else
|
||||
#define CUDA_12_0_SM90_FEATURES_SUPPORTED false
|
||||
#endif
|
||||
|
||||
#include <cutlass/cutlass.h>
|
||||
#include <cutlass/numeric_types.h>
|
||||
#include <cutlass/trace.h>
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
@@ -35,9 +35,49 @@
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Gets a CUDA device
|
||||
cudaDeviceProp GetCudaDevice() {
|
||||
|
||||
cudaError_t err;
|
||||
|
||||
int cudaDeviceId;
|
||||
err = cudaGetDevice(&cudaDeviceId);
|
||||
if (cudaSuccess != err) {
|
||||
std::cerr << "*** Error: Could not detect active GPU device ID"
|
||||
<< " [" << cudaGetErrorString(err) << "]" << std::endl;
|
||||
exit(1);
|
||||
}
|
||||
|
||||
cudaDeviceProp deviceProperties;
|
||||
err = cudaGetDeviceProperties(&deviceProperties, cudaDeviceId);
|
||||
|
||||
return deviceProperties;
|
||||
}
|
||||
|
||||
/// Prints device properties
|
||||
std::ostream &operator<<(std::ostream &out, cudaDeviceProp const &deviceProperties) {
|
||||
|
||||
int deviceMajorMinor = deviceProperties.major * 10 + deviceProperties.minor;
|
||||
if (deviceMajorMinor) {
|
||||
int32_t clock_MHz = deviceProperties.clockRate / 1000;
|
||||
out << "GPU(compute_"
|
||||
<< deviceMajorMinor << ", "
|
||||
<< deviceProperties.multiProcessorCount << " SMs @ " << clock_MHz << " MHz)";
|
||||
}
|
||||
else {
|
||||
out << "No CUDA device.";
|
||||
}
|
||||
|
||||
return out;
|
||||
}
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Sets flags for Unit test
|
||||
void FilterArchitecture() {
|
||||
// Default flags can be overwritten by --gtest_filter from commandline
|
||||
|
||||
int const kMaxDevice = 999;
|
||||
|
||||
cudaError_t err;
|
||||
|
||||
int cudaDeviceId;
|
||||
@@ -57,7 +97,6 @@ void FilterArchitecture() {
|
||||
}
|
||||
|
||||
int deviceMajorMinor = deviceProperties.major * 10 + deviceProperties.minor;
|
||||
int const kMaxDevice = 999;
|
||||
|
||||
// Defines text filters for each GEMM kernel based on minimum supported compute capability
|
||||
struct {
|
||||
@@ -78,7 +117,7 @@ void FilterArchitecture() {
|
||||
{ "SM70*", 70, 75},
|
||||
{ "SM75*", 75, kMaxDevice},
|
||||
{ "SM80*", 80, kMaxDevice},
|
||||
{ "SM90*", 90, kMaxDevice},
|
||||
{ "SM90*", 90, 90 },
|
||||
{ 0, 0, false }
|
||||
};
|
||||
|
||||
|
||||
@@ -186,6 +186,34 @@ public:
|
||||
tensor_D_reference.sync_device();
|
||||
}
|
||||
|
||||
bool sufficient() const {
|
||||
//
|
||||
// Determine SMEM requirements and waive if not satisfied
|
||||
//
|
||||
|
||||
int smem_size = int(sizeof(typename Conv2d::UnderlyingKernel::SharedStorage));
|
||||
|
||||
cudaDeviceProp properties;
|
||||
int device_idx;
|
||||
cudaError_t result = cudaGetDevice(&device_idx);
|
||||
|
||||
if (result != cudaSuccess) {
|
||||
throw std::runtime_error("cudaGetDevice() API call failed.");
|
||||
}
|
||||
|
||||
result = cudaGetDeviceProperties(&properties, device_idx);
|
||||
|
||||
if (result != cudaSuccess) {
|
||||
throw std::runtime_error("cudaGetDeviceProperties() failed");
|
||||
}
|
||||
|
||||
if (properties.sharedMemPerMultiprocessor < smem_size) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/// Executes one test
|
||||
bool run(
|
||||
cutlass::conv::Conv2dProblemSize const &problem_size,
|
||||
@@ -193,6 +221,14 @@ public:
|
||||
ElementCompute alpha = ElementCompute(1),
|
||||
ElementCompute beta = ElementCompute(0)) {
|
||||
|
||||
// Waive test if insufficient CUDA device
|
||||
if (!sufficient()) {
|
||||
if (CUTLASS_TEST_UNIT_ENABLE_WARNINGS) {
|
||||
std::cerr << "Test waived due to insufficient CUDA device." << std::endl;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
#if 0 //display conv2d problem size for debugging
|
||||
std::cout << problem_size << std::endl
|
||||
<< "alpha, beta: (" << float(alpha) << ", " << float(beta) << ")" << std::endl
|
||||
|
||||
+3
@@ -328,6 +328,7 @@ TEST(
|
||||
DepthwiseFpropProblemSizes_filter5x5()));
|
||||
}
|
||||
|
||||
#if 0
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
TEST(
|
||||
SM60_Device_Depthwise_conv2d_Fprop_Direct_Conv_Optimized_f16nhwc_f16nhwc_f16nhwc_simt_f16,
|
||||
@@ -424,3 +425,5 @@ TEST(
|
||||
EXPECT_TRUE(test::conv::device::TestSpecificDepthwiseDirectConv2d<Direct2dConv>(
|
||||
DepthwiseFpropProblemSizes_filter5x37()));
|
||||
}
|
||||
#endif
|
||||
|
||||
|
||||
@@ -152,9 +152,7 @@ TEST(NumericConversion, f32_to_fe5m2_rn_array) {
|
||||
int const kN = 27;
|
||||
using Source = float;
|
||||
using Destination = cutlass::float_e5m2_t;
|
||||
|
||||
test::core::kernel::run_test<Destination, Source, kN>();
|
||||
|
||||
}
|
||||
|
||||
TEST(NumericConversion, f16_to_fe4m3_rn) {
|
||||
@@ -250,16 +248,19 @@ TEST(NumericConversion, fe4m3_to_f32_rn) {
|
||||
test::core::kernel::run_test<Destination, Source, kN>();
|
||||
}
|
||||
|
||||
TEST(NumericConversion, fe4m3_to_f32_array) {
|
||||
int const kN = 27;
|
||||
using Source = cutlass::float_e4m3_t;
|
||||
using Destination = float;
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(NumericConversion, f32x8_to_s8x8_rn) {
|
||||
|
||||
int const kN = 8;
|
||||
using Source = float;
|
||||
using Destination = int8_t;
|
||||
test::core::kernel::run_test<Destination, Source, kN>();
|
||||
}
|
||||
|
||||
TEST(NumericConversion, fe5m2_to_f32_rn) {
|
||||
int const kN = 1;
|
||||
using Source = cutlass::float_e5m2_t;
|
||||
TEST(NumericConversion, fe4m3_to_f32_array) {
|
||||
int const kN = 27;
|
||||
using Source = cutlass::float_e4m3_t;
|
||||
using Destination = float;
|
||||
test::core::kernel::run_test<Destination, Source, kN>();
|
||||
}
|
||||
@@ -328,35 +329,3 @@ TEST(NumericConversion, fe5m2_to_bf16_array) {
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(NumericConversion, f32x8_to_s8x8_rn) {
|
||||
|
||||
int const kN = 8;
|
||||
using Source = float;
|
||||
using Destination = int8_t;
|
||||
|
||||
dim3 grid(1, 1);
|
||||
dim3 block(1, 1);
|
||||
|
||||
cutlass::HostTensor<Destination, cutlass::layout::RowMajor> destination({1, kN});
|
||||
cutlass::HostTensor<Source, cutlass::layout::RowMajor> source({1, kN});
|
||||
|
||||
for (int i = 0; i < kN; ++i) {
|
||||
source.host_data()[i] = float(i);
|
||||
}
|
||||
|
||||
source.sync_device();
|
||||
|
||||
test::core::kernel::convert<Destination, Source, kN><<< grid, block >>>(
|
||||
reinterpret_cast<cutlass::Array<Destination, kN> *>(destination.device_data()),
|
||||
reinterpret_cast<cutlass::Array<Source, kN> const *>(source.device_data())
|
||||
);
|
||||
|
||||
destination.sync_host();
|
||||
|
||||
for (int i = 0; i < kN; ++i) {
|
||||
EXPECT_TRUE(float(destination.host_data()[i]) == source.host_data()[i]);
|
||||
}
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
# Copyright (c) 2023 - 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: BSD-3-Clause
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions are met:
|
||||
#
|
||||
# 1. Redistributions of source code must retain the above copyright notice, this
|
||||
# list of conditions and the following disclaimer.
|
||||
#
|
||||
# 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
# this list of conditions and the following disclaimer in the documentation
|
||||
# and/or other materials provided with the distribution.
|
||||
#
|
||||
# 3. Neither the name of the copyright holder nor the names of its
|
||||
# contributors may be used to endorse or promote products derived from
|
||||
# this software without specific prior written permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
add_subdirectory(core)
|
||||
add_subdirectory(ampere)
|
||||
add_subdirectory(hopper)
|
||||
add_subdirectory(layout)
|
||||
|
||||
add_custom_target(
|
||||
cutlass_test_unit_cute
|
||||
DEPENDS
|
||||
cutlass_test_unit_cute_layout
|
||||
cutlass_test_unit_cute_core
|
||||
cutlass_test_unit_cute_ampere
|
||||
cutlass_test_unit_cute_hopper
|
||||
)
|
||||
|
||||
add_custom_target(
|
||||
test_unit_cute
|
||||
DEPENDS
|
||||
test_unit_cute_layout
|
||||
test_unit_cute_core
|
||||
test_unit_cute_ampere
|
||||
test_unit_cute_hopper
|
||||
)
|
||||
@@ -0,0 +1,33 @@
|
||||
# Copyright (c) 2023 - 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: BSD-3-Clause
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions are met:
|
||||
#
|
||||
# 1. Redistributions of source code must retain the above copyright notice, this
|
||||
# list of conditions and the following disclaimer.
|
||||
#
|
||||
# 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
# this list of conditions and the following disclaimer in the documentation
|
||||
# and/or other materials provided with the distribution.
|
||||
#
|
||||
# 3. Neither the name of the copyright holder nor the names of its
|
||||
# contributors may be used to endorse or promote products derived from
|
||||
# this software without specific prior written permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
cutlass_test_unit_add_executable(
|
||||
cutlass_test_unit_cute_ampere
|
||||
cp_async.cu
|
||||
ldsm.cu
|
||||
)
|
||||
@@ -0,0 +1,104 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017 - 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
#include "cutlass_unit_test.h"
|
||||
|
||||
#include <iostream>
|
||||
#include <iomanip>
|
||||
#include <utility>
|
||||
#include <type_traits>
|
||||
#include <vector>
|
||||
#include <numeric>
|
||||
|
||||
#include <thrust/host_vector.h>
|
||||
#include <thrust/device_vector.h>
|
||||
|
||||
#include <cute/tensor.hpp>
|
||||
|
||||
using namespace cute;
|
||||
|
||||
__global__ void
|
||||
test(double const* g_in, double* g_out)
|
||||
{
|
||||
extern __shared__ double smem[];
|
||||
|
||||
smem[threadIdx.x] = g_in[threadIdx.x];
|
||||
|
||||
__syncthreads();
|
||||
|
||||
g_out[threadIdx.x] = 2 * smem[threadIdx.x];
|
||||
}
|
||||
|
||||
__global__ void
|
||||
test2(double const* g_in, double* g_out)
|
||||
{
|
||||
using namespace cute;
|
||||
|
||||
extern __shared__ double smem[];
|
||||
|
||||
auto s_tensor = make_tensor(make_smem_ptr(smem + threadIdx.x), Int<1>{});
|
||||
auto g_tensor = make_tensor(make_gmem_ptr(g_in + threadIdx.x), Int<1>{});
|
||||
|
||||
copy(g_tensor, s_tensor);
|
||||
|
||||
cp_async_fence();
|
||||
cp_async_wait<0>();
|
||||
__syncthreads();
|
||||
|
||||
g_out[threadIdx.x] = 2 * smem[threadIdx.x];
|
||||
}
|
||||
|
||||
TEST(SM80_CuTe_Ampere, CpAsync)
|
||||
{
|
||||
constexpr int count = 32;
|
||||
thrust::host_vector<double> h_in(count);
|
||||
for (int i = 0; i < count; ++i) {
|
||||
h_in[i] = double(i);
|
||||
}
|
||||
|
||||
thrust::device_vector<double> d_in(h_in);
|
||||
|
||||
thrust::device_vector<double> d_out(count, -1);
|
||||
test<<<1, count, sizeof(double) * count>>>(
|
||||
thrust::raw_pointer_cast(d_in.data()),
|
||||
thrust::raw_pointer_cast(d_out.data()));
|
||||
thrust::host_vector<double> h_result = d_out;
|
||||
|
||||
thrust::device_vector<double> d_out_cp_async(count, -2);
|
||||
test2<<<1, count, sizeof(double) * count>>>(
|
||||
thrust::raw_pointer_cast(d_in.data()),
|
||||
thrust::raw_pointer_cast(d_out_cp_async.data()));
|
||||
thrust::host_vector<double> h_result_cp_async = d_out_cp_async;
|
||||
|
||||
for (int i = 0; i < count; ++i) {
|
||||
EXPECT_EQ(h_result[i], h_result_cp_async[i]);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,431 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017 - 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
#include "cutlass_unit_test.h"
|
||||
|
||||
#include <iostream>
|
||||
|
||||
#include <thrust/host_vector.h>
|
||||
#include <thrust/device_vector.h>
|
||||
|
||||
#include <cute/tensor.hpp>
|
||||
|
||||
#include <cute/atom/copy_traits_sm75.hpp>
|
||||
|
||||
|
||||
using namespace cute;
|
||||
|
||||
template <class T>
|
||||
__global__ void
|
||||
ldsm_test_device(uint16_t* g_in, uint16_t* g_out)
|
||||
{
|
||||
constexpr int count = sizeof(T) / 4;
|
||||
int tid = threadIdx.x;
|
||||
int stride = blockDim.x;
|
||||
|
||||
// load input gmem -> smem
|
||||
__shared__ uint32_t smem[32 * count];
|
||||
for (int i = 0; i < count; ++i) {
|
||||
smem[tid + (stride * i)] = reinterpret_cast<uint32_t*>(g_in)[tid + (stride * i)];
|
||||
}
|
||||
|
||||
__syncthreads();
|
||||
|
||||
uint32_t reg[count];
|
||||
for (int i = 0; i < count; ++i) {
|
||||
reg[i] = 0;
|
||||
}
|
||||
|
||||
// load smem -> rmem using LDSM
|
||||
uint128_t* smem_ptr = reinterpret_cast<uint128_t*>(smem) + tid;
|
||||
T* rmem_ptr = reinterpret_cast<T*>(reg);
|
||||
cute::copy_ldsm(smem_ptr, rmem_ptr);
|
||||
|
||||
// store output rmem -> gmem
|
||||
for (int i = 0; i < count; ++i) {
|
||||
reinterpret_cast<uint32_t*>(g_out)[tid + (stride * i)] = reg[i];
|
||||
}
|
||||
}
|
||||
|
||||
template <class TiledCopy, class SmemLayout>
|
||||
__global__ void
|
||||
ldsm_test_device_cute(uint16_t* g_in, uint16_t* g_out,
|
||||
TiledCopy tiled_copy, SmemLayout smem_layout)
|
||||
{
|
||||
using namespace cute;
|
||||
|
||||
__shared__ uint16_t smem[size(smem_layout)];
|
||||
|
||||
auto t_g_in = make_tensor(make_gmem_ptr(g_in), smem_layout);
|
||||
auto t_g_out = make_tensor(make_gmem_ptr(g_out), smem_layout);
|
||||
auto t_smem = make_tensor(make_smem_ptr(smem), smem_layout);
|
||||
|
||||
int tid = threadIdx.x;
|
||||
|
||||
// Load input gmem -> smem
|
||||
for (int i = tid; i < size(t_smem); i += size(tiled_copy)) {
|
||||
t_smem(i) = t_g_in(i);
|
||||
}
|
||||
|
||||
__syncthreads();
|
||||
|
||||
auto thr_copy = tiled_copy.get_thread_slice(tid);
|
||||
|
||||
auto tXsX = thr_copy.partition_S(t_smem); // (V,M,N)
|
||||
auto tXgX = thr_copy.partition_D(t_g_out); // (V,M,N)
|
||||
|
||||
auto tXrX = make_tensor<uint16_t>(shape(tXgX)); // (V,M,N)
|
||||
clear(tXrX); // Just to make sure
|
||||
|
||||
/*
|
||||
if (thread0()) {
|
||||
print("tXsX: " ); print(tXsX.layout()); print("\n");
|
||||
print("tXgX: " ); print(tXgX.layout()); print("\n");
|
||||
print("tXrX: " ); print(tXrX.layout()); print("\n");
|
||||
}
|
||||
*/
|
||||
|
||||
// Copy smem -> rmem via tiled_copy (LDSM, LDS)
|
||||
copy(tiled_copy, tXsX, tXrX);
|
||||
|
||||
// Output rmem -> gmem
|
||||
copy(tXrX, tXgX);
|
||||
}
|
||||
|
||||
|
||||
TEST(SM80_CuTe_Ampere, Ldsm)
|
||||
{
|
||||
constexpr int count = 1024;
|
||||
|
||||
thrust::host_vector<uint16_t> h_in(count);
|
||||
for (int i = 0; i < count; ++i) {
|
||||
h_in[i] = uint16_t(i);
|
||||
}
|
||||
thrust::device_vector<uint16_t> d_in = h_in;
|
||||
|
||||
//
|
||||
// LDSM 1x (32b)
|
||||
//
|
||||
|
||||
{
|
||||
thrust::device_vector<uint16_t> d_out(count);
|
||||
ldsm_test_device<uint32_t><<<1, 32>>>(
|
||||
thrust::raw_pointer_cast(d_in.data()),
|
||||
thrust::raw_pointer_cast(d_out.data()));
|
||||
thrust::host_vector<uint16_t> h_out = d_out;
|
||||
for (int i = 0; i < 32; ++i) {
|
||||
EXPECT_EQ(h_out[i], h_in[i]);
|
||||
}
|
||||
CUTLASS_TRACE_HOST("LDSM 1x ldsm_test_device SUCCESS\n");
|
||||
}
|
||||
|
||||
//
|
||||
// LDSM 2x (64b)
|
||||
//
|
||||
|
||||
{
|
||||
thrust::device_vector<uint16_t> d_out(count);
|
||||
ldsm_test_device<uint64_t><<<1, 32>>>(
|
||||
thrust::raw_pointer_cast(d_in.data()),
|
||||
thrust::raw_pointer_cast(d_out.data()));
|
||||
thrust::host_vector<uint16_t> h_out = d_out;
|
||||
for (int i = 0; i < 64; ++i) {
|
||||
//printf("%d %d\n", int(h_in[i]), int(h_out[i]));
|
||||
EXPECT_EQ(h_out[i], h_in[i]);
|
||||
}
|
||||
CUTLASS_TRACE_HOST("LDSM 2x ldsm_test_device SUCCESS\n");
|
||||
}
|
||||
|
||||
//
|
||||
// LDSM 4x (128b)
|
||||
//
|
||||
|
||||
{
|
||||
thrust::device_vector<uint16_t> d_out(count);
|
||||
ldsm_test_device<uint128_t><<<1, 32>>>(
|
||||
thrust::raw_pointer_cast(d_in.data()),
|
||||
thrust::raw_pointer_cast(d_out.data()));
|
||||
thrust::host_vector<uint16_t> h_out = d_out;
|
||||
for (int i = 0; i < 128; ++i) {
|
||||
//printf("%d %d\n", int(h_in[i]), int(h_out[i]));
|
||||
EXPECT_EQ(h_out[i], h_in[i]);
|
||||
}
|
||||
CUTLASS_TRACE_HOST("LDSM 4x ldsm_test_device SUCCESS\n");
|
||||
}
|
||||
|
||||
//
|
||||
// CuTe LDSM
|
||||
//
|
||||
|
||||
{
|
||||
thrust::device_vector<uint16_t> d_out(count);
|
||||
|
||||
auto smem_layout = Layout<Shape <_32,Shape <_2, _4>>,
|
||||
Stride< _2,Stride<_1,_64>>>{};
|
||||
auto tiled_copy = make_tiled_copy(Copy_Atom<SM75_U32x1_LDSM_N, uint16_t>{},
|
||||
Layout<Shape<_32,_1>>{},
|
||||
Layout<Shape< _1,_8>>{});
|
||||
|
||||
ldsm_test_device_cute<<<1, int(size(tiled_copy))>>>(
|
||||
thrust::raw_pointer_cast(d_in.data()),
|
||||
thrust::raw_pointer_cast(d_out.data()),
|
||||
tiled_copy,
|
||||
smem_layout);
|
||||
thrust::host_vector<uint16_t> h_out = d_out;
|
||||
for (int i = 0; i < size(smem_layout); ++i) {
|
||||
//printf("%d %d\n", int(h_in[i]), int(h_out[i]));
|
||||
EXPECT_EQ(h_out[i], h_in[i]);
|
||||
}
|
||||
CUTLASS_TRACE_HOST("CuTe 32x8 interleaved U32x1_LDSM_N SUCCESS\n");
|
||||
}
|
||||
|
||||
{
|
||||
thrust::device_vector<uint16_t> d_out(count);
|
||||
|
||||
auto smem_layout = Layout<Shape <_32,Shape <_2, _4>>,
|
||||
Stride< _2,Stride<_1,_64>>>{};
|
||||
auto tiled_copy = make_tiled_copy(Copy_Atom<SM75_U32x2_LDSM_N, uint16_t>{},
|
||||
Layout<Shape<_32,_1>>{},
|
||||
Layout<Shape< _1,_8>>{});
|
||||
|
||||
ldsm_test_device_cute<<<1, int(size(tiled_copy))>>>(
|
||||
thrust::raw_pointer_cast(d_in.data()),
|
||||
thrust::raw_pointer_cast(d_out.data()),
|
||||
tiled_copy,
|
||||
smem_layout);
|
||||
thrust::host_vector<uint16_t> h_out = d_out;
|
||||
for (int i = 0; i < size(smem_layout); ++i) {
|
||||
//printf("%d %d\n", int(h_in[i]), int(h_out[i]));
|
||||
EXPECT_EQ(h_out[i], h_in[i]);
|
||||
}
|
||||
CUTLASS_TRACE_HOST("CuTe 32x8 interleaved U32x2_LDSM_N SUCCESS\n");
|
||||
}
|
||||
|
||||
{
|
||||
thrust::device_vector<uint16_t> d_out(count);
|
||||
|
||||
auto smem_layout = Layout<Shape <_32,Shape <_2, _4>>,
|
||||
Stride< _2,Stride<_1,_64>>>{};
|
||||
auto tiled_copy = make_tiled_copy(Copy_Atom<SM75_U32x4_LDSM_N, uint16_t>{},
|
||||
Layout<Shape<_32,_1>>{},
|
||||
Layout<Shape< _1,_8>>{});
|
||||
|
||||
ldsm_test_device_cute<<<1, int(size(tiled_copy))>>>(
|
||||
thrust::raw_pointer_cast(d_in.data()),
|
||||
thrust::raw_pointer_cast(d_out.data()),
|
||||
tiled_copy,
|
||||
smem_layout);
|
||||
thrust::host_vector<uint16_t> h_out = d_out;
|
||||
for (int i = 0; i < size(smem_layout); ++i) {
|
||||
//printf("%d %d\n", int(h_in[i]), int(h_out[i]));
|
||||
EXPECT_EQ(h_out[i], h_in[i]);
|
||||
}
|
||||
CUTLASS_TRACE_HOST("CuTe 32x8 interleaved U32x4_LDSM_N SUCCESS\n");
|
||||
}
|
||||
|
||||
{
|
||||
thrust::device_vector<uint16_t> d_out(count);
|
||||
|
||||
auto smem_layout = Layout<Shape <_32,Shape <_2, _4>>,
|
||||
Stride< _2,Stride<_1,_64>>>{};
|
||||
auto tiled_copy = make_tiled_copy(Copy_Atom<UniversalCopy<uint16_t>, uint16_t>{},
|
||||
Layout<Shape<_32,_1>>{},
|
||||
Layout<Shape< _1,_8>>{});
|
||||
|
||||
ldsm_test_device_cute<<<1, int(size(tiled_copy))>>>(
|
||||
thrust::raw_pointer_cast(d_in.data()),
|
||||
thrust::raw_pointer_cast(d_out.data()),
|
||||
tiled_copy,
|
||||
smem_layout);
|
||||
thrust::host_vector<uint16_t> h_out = d_out;
|
||||
for (int i = 0; i < size(smem_layout); ++i) {
|
||||
//printf("%d %d\n", int(h_in[i]), int(h_out[i]));
|
||||
EXPECT_EQ(h_out[i] , h_in[i]);
|
||||
}
|
||||
CUTLASS_TRACE_HOST("CuTe 32x8 interleaved LDS.U16 SUCCESS\n");
|
||||
}
|
||||
|
||||
{
|
||||
thrust::device_vector<uint16_t> d_out(count);
|
||||
|
||||
auto smem_layout = Layout<Shape <_32,_32>,
|
||||
Stride< _1,_32>>{};
|
||||
auto tiled_copy = make_tiled_copy(Copy_Atom<SM75_U32x1_LDSM_N, uint16_t>{},
|
||||
Layout<Shape<_16,_2>>{},
|
||||
Layout<Shape< _2,_4>>{});
|
||||
|
||||
ldsm_test_device_cute<<<1, int(size(tiled_copy))>>>(
|
||||
thrust::raw_pointer_cast(d_in.data()),
|
||||
thrust::raw_pointer_cast(d_out.data()),
|
||||
tiled_copy,
|
||||
smem_layout);
|
||||
thrust::host_vector<uint16_t> h_out = d_out;
|
||||
for (int i = 0; i < size(smem_layout); ++i) {
|
||||
//printf("%d %d\n", int(h_in[i]), int(h_out[i]));
|
||||
EXPECT_EQ(h_out[i], h_in[i]);
|
||||
}
|
||||
CUTLASS_TRACE_HOST("CuTe 32x32 U32x1_LDSM_N SUCCESS\n");
|
||||
}
|
||||
|
||||
{
|
||||
thrust::device_vector<uint16_t> d_out(count);
|
||||
|
||||
auto smem_layout = Layout<Shape <_32,_32>,
|
||||
Stride< _1,_32>>{};
|
||||
auto tiled_copy = make_tiled_copy(Copy_Atom<SM75_U32x2_LDSM_N, uint16_t>{},
|
||||
Layout<Shape<_16,_2>>{},
|
||||
Layout<Shape< _2,_4>>{});
|
||||
|
||||
ldsm_test_device_cute<<<1, int(size(tiled_copy))>>>(
|
||||
thrust::raw_pointer_cast(d_in.data()),
|
||||
thrust::raw_pointer_cast(d_out.data()),
|
||||
tiled_copy,
|
||||
smem_layout);
|
||||
thrust::host_vector<uint16_t> h_out = d_out;
|
||||
for (int i = 0; i < size(smem_layout); ++i) {
|
||||
//printf("%d %d\n", int(h_in[i]), int(h_out[i]));
|
||||
EXPECT_EQ(h_out[i], h_in[i]);
|
||||
}
|
||||
CUTLASS_TRACE_HOST("CuTe 32x32 U32x2_LDSM_N SUCCESS\n");
|
||||
}
|
||||
|
||||
{
|
||||
thrust::device_vector<uint16_t> d_out(count);
|
||||
|
||||
auto smem_layout = Layout<Shape <_32,_32>,
|
||||
Stride< _1,_32>>{};
|
||||
auto tiled_copy = make_tiled_copy(Copy_Atom<SM75_U32x4_LDSM_N, uint16_t>{},
|
||||
Layout<Shape<_16,_2>>{},
|
||||
Layout<Shape< _2,_4>>{});
|
||||
|
||||
ldsm_test_device_cute<<<1, int(size(tiled_copy))>>>(
|
||||
thrust::raw_pointer_cast(d_in.data()),
|
||||
thrust::raw_pointer_cast(d_out.data()),
|
||||
tiled_copy,
|
||||
smem_layout);
|
||||
thrust::host_vector<uint16_t> h_out = d_out;
|
||||
for (int i = 0; i < size(smem_layout); ++i) {
|
||||
//printf("%d %d\n", int(h_in[i]), int(h_out[i]));
|
||||
EXPECT_EQ(h_out[i], h_in[i]);
|
||||
}
|
||||
CUTLASS_TRACE_HOST("CuTe 32x32 U32x4_LDSM_N SUCCESS\n");
|
||||
}
|
||||
|
||||
{
|
||||
thrust::device_vector<uint16_t> d_out(count);
|
||||
|
||||
auto smem_layout = Layout<Shape <_32,_32>,
|
||||
Stride< _1,_32>>{};
|
||||
auto tiled_copy = make_tiled_copy(Copy_Atom<UniversalCopy<uint16_t>, uint16_t>{},
|
||||
Layout<Shape<_16,_2>>{},
|
||||
Layout<Shape< _2,_4>>{});
|
||||
|
||||
ldsm_test_device_cute<<<1, int(size(tiled_copy))>>>(
|
||||
thrust::raw_pointer_cast(d_in.data()),
|
||||
thrust::raw_pointer_cast(d_out.data()),
|
||||
tiled_copy,
|
||||
smem_layout);
|
||||
thrust::host_vector<uint16_t> h_out = d_out;
|
||||
for (int i = 0; i < size(smem_layout); ++i) {
|
||||
//printf("%d %d\n", int(h_in[i]), int(h_out[i]));
|
||||
EXPECT_EQ(h_out[i], h_in[i]);
|
||||
}
|
||||
CUTLASS_TRACE_HOST("CuTe 32x32 LDS.U16 SUCCESS\n");
|
||||
}
|
||||
|
||||
{
|
||||
thrust::device_vector<uint16_t> d_out(count);
|
||||
|
||||
auto smem_layout = Layout<Shape <_32,_32>,
|
||||
Stride<_32, _1>>{};
|
||||
auto tiled_copy = make_tiled_copy(Copy_Atom<SM75_U16x2_LDSM_T, uint16_t>{},
|
||||
Layout<Shape<_4,_8>>{},
|
||||
Layout<Shape<_2,_1>>{});
|
||||
|
||||
ldsm_test_device_cute<<<1, int(size(tiled_copy))>>>(
|
||||
thrust::raw_pointer_cast(d_in.data()),
|
||||
thrust::raw_pointer_cast(d_out.data()),
|
||||
tiled_copy,
|
||||
smem_layout);
|
||||
thrust::host_vector<uint16_t> h_out = d_out;
|
||||
for (int i = 0; i < size(smem_layout); ++i) {
|
||||
//printf("%d %d\n", int(h_in[i]), int(h_out[i]));
|
||||
EXPECT_EQ(h_out[i], h_in[i]);
|
||||
}
|
||||
CUTLASS_TRACE_HOST("CuTe 32x32 U16x2_LDSM_T SUCCESS\n");
|
||||
}
|
||||
|
||||
{
|
||||
thrust::device_vector<uint16_t> d_out(count);
|
||||
|
||||
auto smem_layout = Layout<Shape <_32,_32>,
|
||||
Stride<_32, _1>>{};
|
||||
auto tiled_copy = make_tiled_copy(Copy_Atom<SM75_U16x4_LDSM_T, uint16_t>{},
|
||||
Layout<Shape<_4,_8>>{},
|
||||
Layout<Shape<_4,_1>>{});
|
||||
|
||||
ldsm_test_device_cute<<<1, int(size(tiled_copy))>>>(
|
||||
thrust::raw_pointer_cast(d_in.data()),
|
||||
thrust::raw_pointer_cast(d_out.data()),
|
||||
tiled_copy,
|
||||
smem_layout);
|
||||
thrust::host_vector<uint16_t> h_out = d_out;
|
||||
for (int i = 0; i < size(smem_layout); ++i) {
|
||||
//printf("%d %d\n", int(h_in[i]), int(h_out[i]));
|
||||
EXPECT_EQ(h_out[i], h_in[i]);
|
||||
}
|
||||
CUTLASS_TRACE_HOST("CuTe 32x32 U16x4_LDSM_T SUCCESS\n");
|
||||
}
|
||||
|
||||
{
|
||||
thrust::device_vector<uint16_t> d_out(count);
|
||||
|
||||
auto smem_layout = Layout<Shape <_32,_32>,
|
||||
Stride<_32, _1>>{};
|
||||
auto tiled_copy = make_tiled_copy(Copy_Atom<SM75_U16x8_LDSM_T, uint16_t>{},
|
||||
Layout<Shape<_4,_8>>{},
|
||||
Layout<Shape<_8,_1>>{});
|
||||
|
||||
ldsm_test_device_cute<<<1, int(size(tiled_copy))>>>(
|
||||
thrust::raw_pointer_cast(d_in.data()),
|
||||
thrust::raw_pointer_cast(d_out.data()),
|
||||
tiled_copy,
|
||||
smem_layout);
|
||||
thrust::host_vector<uint16_t> h_out = d_out;
|
||||
for (int i = 0; i < size(smem_layout); ++i) {
|
||||
//printf("%d %d\n", int(h_in[i]), int(h_out[i]));
|
||||
EXPECT_EQ(h_out[i], h_in[i]);
|
||||
}
|
||||
CUTLASS_TRACE_HOST("CuTe 32x32 U16x8_LDSM_T SUCCESS\n");
|
||||
}
|
||||
|
||||
CUTLASS_TRACE_HOST("PASS");
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
# Copyright (c) 2023 - 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: BSD-3-Clause
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions are met:
|
||||
#
|
||||
# 1. Redistributions of source code must retain the above copyright notice, this
|
||||
# list of conditions and the following disclaimer.
|
||||
#
|
||||
# 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
# this list of conditions and the following disclaimer in the documentation
|
||||
# and/or other materials provided with the distribution.
|
||||
#
|
||||
# 3. Neither the name of the copyright holder nor the names of its
|
||||
# contributors may be used to endorse or promote products derived from
|
||||
# this software without specific prior written permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
cutlass_test_unit_add_executable(
|
||||
cutlass_test_unit_cute_core
|
||||
|
||||
bitfield.cpp
|
||||
coalesce.cpp
|
||||
compare.cpp
|
||||
complement.cpp
|
||||
composition.cpp
|
||||
inverse_left.cpp
|
||||
inverse_right.cpp
|
||||
logical_divide.cpp
|
||||
logical_product.cpp
|
||||
mixedbits.cpp
|
||||
transform.cpp
|
||||
tuple.cpp
|
||||
)
|
||||
@@ -0,0 +1,84 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017 - 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
#include "cutlass_unit_test.h"
|
||||
|
||||
#include <iostream>
|
||||
#include <iomanip>
|
||||
#include <utility>
|
||||
#include <type_traits>
|
||||
#include <vector>
|
||||
#include <numeric>
|
||||
|
||||
#include <cute/tensor.hpp>
|
||||
#include <cute/container/bit_field.hpp>
|
||||
|
||||
using namespace cute;
|
||||
|
||||
TEST(CuTe_core, Bitfield)
|
||||
{
|
||||
for_each(make_int_range<1,65>{}, [&](auto NumBits) {
|
||||
for_each(make_int_range<0,129>{}, [&](auto BitStart) {
|
||||
|
||||
using BF = bit_field<decltype(BitStart)::value, decltype(NumBits)::value>;
|
||||
|
||||
#if 0
|
||||
printf("bit_field<%d,%d>:\n", decltype(BitStart)::value, decltype(NumBits)::value);
|
||||
printf(" value_type_bits : %d\n", BF::value_type_bits);
|
||||
printf(" storage_type_bits: %d\n", BF::storage_type_bits);
|
||||
printf(" N : %d\n", BF::N);
|
||||
printf(" idx : %d\n", BF::idx);
|
||||
printf(" bit_lo : %d\n", BF::bit_lo);
|
||||
printf(" bit_hi : %d\n", BF::bit_hi);
|
||||
printf(" mask : 0x%lx\n", uint64_t(BF::mask));
|
||||
printf(" mask_lo : 0x%lx\n", uint64_t(BF::mask_lo));
|
||||
printf(" mask_hi : 0x%lx\n", uint64_t(BF::mask_hi));
|
||||
#endif
|
||||
|
||||
// Test
|
||||
uint64_t v = decltype(NumBits)::value == 64 ? uint64_t(-1) : ((uint64_t(1) << NumBits) - 1);
|
||||
|
||||
BF bf{};
|
||||
bf = v;
|
||||
EXPECT_EQ(v, uint64_t(bf));
|
||||
});
|
||||
});
|
||||
|
||||
for_each(make_int_range<0,129>{}, [&](auto BitStart) {
|
||||
|
||||
using BF = bit_field<decltype(BitStart)::value, 32, float>;
|
||||
|
||||
BF bf{};
|
||||
bf = 3.14f;
|
||||
EXPECT_EQ(3.14f, float(bf));
|
||||
});
|
||||
|
||||
}
|
||||
@@ -0,0 +1,182 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017 - 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
#include "cutlass_unit_test.h"
|
||||
|
||||
|
||||
#include <cute/tensor.hpp>
|
||||
|
||||
using namespace cute;
|
||||
|
||||
template <class Layout>
|
||||
void
|
||||
test_coalesce(Layout const& layout)
|
||||
{
|
||||
auto coalesce_layout = coalesce(layout);
|
||||
|
||||
CUTLASS_TRACE_HOST(shape (layout) << " => " << shape (coalesce_layout));
|
||||
CUTLASS_TRACE_HOST(stride(layout) << " " << stride(coalesce_layout));
|
||||
|
||||
CUTE_STATIC_ASSERT_V(depth(coalesce_layout) <= Int<1>{});
|
||||
|
||||
ASSERT_EQ(size(coalesce_layout), size(layout));
|
||||
|
||||
for (int i = 0; i < size(layout); ++i) {
|
||||
EXPECT_EQ(coalesce_layout(i), layout(i));
|
||||
}
|
||||
}
|
||||
|
||||
TEST(CuTe_core, Coalesce)
|
||||
{
|
||||
{
|
||||
auto layout = make_layout(Int<1>{}, Int<0>{});
|
||||
|
||||
test_coalesce(layout);
|
||||
}
|
||||
|
||||
{
|
||||
auto layout = make_layout(Int<1>{}, Int<1>{});
|
||||
|
||||
test_coalesce(layout);
|
||||
}
|
||||
|
||||
{
|
||||
auto layout = make_layout(make_shape(Int<2>{}, Int<4>{}));
|
||||
|
||||
test_coalesce(layout);
|
||||
}
|
||||
|
||||
{
|
||||
auto layout = make_layout(make_shape(Int<2>{}, Int<4>{}, Int<6>{}));
|
||||
|
||||
test_coalesce(layout);
|
||||
}
|
||||
|
||||
{
|
||||
auto layout = make_layout(make_shape (Int<2>{}, Int<1>{}, Int<6>{}),
|
||||
make_stride(Int<1>{}, Int<6>{}, Int<2>{}));
|
||||
|
||||
test_coalesce(layout);
|
||||
}
|
||||
|
||||
{
|
||||
auto layout = make_layout(make_shape (Int<2>{}, Int<1>{}, Int<6>{}),
|
||||
make_stride(Int<1>{}, 7, Int<2>{}));
|
||||
|
||||
test_coalesce(layout);
|
||||
}
|
||||
|
||||
{
|
||||
auto layout = make_layout(make_shape (Int<2>{}, Int<1>{}, Int<6>{}),
|
||||
make_stride(Int<4>{}, 7, Int<8>{}));
|
||||
|
||||
test_coalesce(layout);
|
||||
}
|
||||
|
||||
{
|
||||
auto layout = make_layout(make_shape(2, Int<4>{}, Int<6>{}));
|
||||
|
||||
test_coalesce(layout);
|
||||
}
|
||||
|
||||
{
|
||||
auto layout = make_layout(make_shape(Int<2>{}, 4, Int<6>{}));
|
||||
|
||||
test_coalesce(layout);
|
||||
}
|
||||
|
||||
{
|
||||
auto layout = make_layout(make_shape(Int<2>{}, Int<4>{}, 6));
|
||||
|
||||
test_coalesce(layout);
|
||||
}
|
||||
|
||||
{
|
||||
auto layout = make_layout(make_shape(Int<2>{}, Int<4>{}), GenRowMajor{});
|
||||
|
||||
test_coalesce(layout);
|
||||
}
|
||||
|
||||
{
|
||||
auto layout = make_layout(make_shape(Int<2>{}, Int<4>{}, Int<6>{}), GenRowMajor{});
|
||||
|
||||
test_coalesce(layout);
|
||||
}
|
||||
|
||||
{
|
||||
auto layout = make_layout(make_shape(2, Int<4>{}, Int<6>{}), GenRowMajor{});
|
||||
|
||||
test_coalesce(layout);
|
||||
}
|
||||
|
||||
{
|
||||
auto layout = make_layout(make_shape(Int<2>{}, 4, Int<6>{}), GenRowMajor{});
|
||||
|
||||
test_coalesce(layout);
|
||||
}
|
||||
|
||||
{
|
||||
auto layout = make_layout(make_shape(Int<2>{}, Int<4>{}, 6), GenRowMajor{});
|
||||
|
||||
test_coalesce(layout);
|
||||
}
|
||||
|
||||
{
|
||||
auto layout = make_layout(make_shape(Int<2>{}, Int<1>{}, Int<3>{}), GenRowMajor{});
|
||||
|
||||
test_coalesce(layout);
|
||||
}
|
||||
|
||||
{
|
||||
auto layout = make_layout(make_shape(Int<2>{}, 1, Int<3>{}), GenRowMajor{});
|
||||
|
||||
test_coalesce(layout);
|
||||
}
|
||||
|
||||
{
|
||||
auto layout = make_layout(make_shape(Int<2>{}, 1, Int<3>{}), make_stride(Int<2>{}, 4, Int<4>{}));
|
||||
|
||||
test_coalesce(layout);
|
||||
}
|
||||
|
||||
{
|
||||
auto layout = make_layout(make_shape(Int<2>{}, 1, Int<3>{}), make_stride(Int<2>{}, Int<0>{}, Int<4>{}));
|
||||
|
||||
test_coalesce(layout);
|
||||
}
|
||||
|
||||
{
|
||||
auto layout = Layout<Shape<Shape<_2,_2>,Shape<_2, _2>>,
|
||||
Stride<Stride<_1,_4>,Stride<_8,_32>>>{};
|
||||
|
||||
test_coalesce(layout);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,168 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017 - 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
#include "cutlass_unit_test.h"
|
||||
|
||||
#include <cutlass/trace.h>
|
||||
#include <cute/tensor.hpp>
|
||||
|
||||
TEST(CuTe_core, Compare_simple_2d_GenColMajor)
|
||||
{
|
||||
using namespace cute;
|
||||
|
||||
// Simple 2D layout
|
||||
auto layout = make_layout(make_shape(Int<3>{}, Int<5>{}), GenColMajor{});
|
||||
CUTLASS_TRACE_HOST("Layout: " << layout);
|
||||
|
||||
for (int i = 0; i < size(layout); ++i) {
|
||||
auto coord_i = layout.get_hier_coord(i);
|
||||
|
||||
CUTLASS_TRACE_HOST(i << ": " << coord_i);
|
||||
|
||||
EXPECT_TRUE(elem_less(coord_i, shape(layout)));
|
||||
|
||||
for (int j = 0; j < size(layout); ++j) {
|
||||
auto coord_j = layout.get_hier_coord(j);
|
||||
CUTLASS_TRACE_HOST(" " << j << ": " << coord_j);
|
||||
EXPECT_TRUE(elem_less(coord_j, shape(layout)));
|
||||
|
||||
EXPECT_EQ((i < j), colex_less(coord_i,coord_j));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
TEST(CuTe_core, Compare_simple_2d_GenRowMajor)
|
||||
{
|
||||
using namespace cute;
|
||||
|
||||
auto layout = make_layout(make_shape(Int<3>{}, Int<5>{}), GenRowMajor{});
|
||||
CUTLASS_TRACE_HOST("Layout: " << layout);
|
||||
|
||||
for (int i = 0; i < size(layout); ++i) {
|
||||
auto coord_i = layout.get_hier_coord(i);
|
||||
CUTLASS_TRACE_HOST(i << ": " << coord_i);
|
||||
EXPECT_TRUE(elem_less(coord_i, shape(layout)));
|
||||
|
||||
for (int j = 0; j < size(layout); ++j) {
|
||||
auto coord_j = layout.get_hier_coord(j);
|
||||
EXPECT_TRUE(elem_less(coord_j, shape(layout)));
|
||||
|
||||
EXPECT_EQ((i < j), lex_less(coord_i,coord_j));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
TEST(CuTe_core, Compare_simple_3d_GenColMajor)
|
||||
{
|
||||
using namespace cute;
|
||||
|
||||
auto layout = make_layout(make_shape(Int<2>{}, Int<3>{}, Int<5>{}), GenColMajor{});
|
||||
CUTLASS_TRACE_HOST("Layout: " << layout);
|
||||
|
||||
for (int i = 0; i < size(layout); ++i) {
|
||||
auto coord_i = layout.get_hier_coord(i);
|
||||
CUTLASS_TRACE_HOST(i << ": " << coord_i);
|
||||
EXPECT_TRUE(elem_less(coord_i, shape(layout)));
|
||||
|
||||
for (int j = 0; j < size(layout); ++j) {
|
||||
auto coord_j = layout.get_hier_coord(j);
|
||||
EXPECT_TRUE(elem_less(coord_j, shape(layout)));
|
||||
|
||||
EXPECT_EQ((i < j), colex_less(coord_i,coord_j));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
TEST(CuTe_core, Compare_simple_3d_GenRowMajor)
|
||||
{
|
||||
using namespace cute;
|
||||
|
||||
auto layout = make_layout(make_shape(Int<2>{}, Int<3>{}, Int<5>{}), GenRowMajor{});
|
||||
CUTLASS_TRACE_HOST("Layout: " << layout);
|
||||
|
||||
for (int i = 0; i < size(layout); ++i) {
|
||||
auto coord_i = layout.get_hier_coord(i);
|
||||
CUTLASS_TRACE_HOST(i << ": " << coord_i);
|
||||
EXPECT_TRUE(elem_less(coord_i, shape(layout)));
|
||||
|
||||
for (int j = 0; j < size(layout); ++j) {
|
||||
auto coord_j = layout.get_hier_coord(j);
|
||||
EXPECT_TRUE(elem_less(coord_j, shape(layout)));
|
||||
|
||||
EXPECT_EQ((i < j), lex_less(coord_i,coord_j));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
TEST(CuTe_core, Compare_hierarchical_3d_GenColMajor)
|
||||
{
|
||||
using namespace cute;
|
||||
|
||||
auto layout = make_layout(Shape<Shape<_3,_2>,Shape<_5,_2,_2>>{}, GenColMajor{});
|
||||
CUTLASS_TRACE_HOST("Layout: " << layout);
|
||||
|
||||
for (int i = 0; i < size(layout); ++i) {
|
||||
auto coord_i = layout.get_hier_coord(i);
|
||||
CUTLASS_TRACE_HOST(i << ": " << coord_i);
|
||||
EXPECT_TRUE(elem_less(coord_i, shape(layout)));
|
||||
|
||||
for (int j = 0; j < size(layout); ++j) {
|
||||
auto coord_j = layout.get_hier_coord(j);
|
||||
EXPECT_TRUE(elem_less(coord_j, shape(layout)));
|
||||
|
||||
EXPECT_EQ((i < j), colex_less(coord_i,coord_j));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
TEST(CuTe_core, Compare_hierarchical_3d_GenRowMajor)
|
||||
{
|
||||
using namespace cute;
|
||||
auto layout = make_layout(Shape<Shape<_3,_2>,Shape<_5,_2,_2>>{}, GenRowMajor{});
|
||||
CUTLASS_TRACE_HOST("Layout: " << layout);
|
||||
|
||||
for (int i = 0; i < size(layout); ++i) {
|
||||
auto coord_i = layout.get_hier_coord(i);
|
||||
CUTLASS_TRACE_HOST(i << ": " << coord_i);
|
||||
EXPECT_TRUE(elem_less(coord_i, shape(layout)));
|
||||
|
||||
for (int j = 0; j < size(layout); ++j) {
|
||||
auto coord_j = layout.get_hier_coord(j);
|
||||
EXPECT_TRUE(elem_less(coord_j, shape(layout)));
|
||||
|
||||
EXPECT_EQ((i < j), lex_less(coord_i,coord_j));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,273 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017 - 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
#include "cutlass_unit_test.h"
|
||||
|
||||
#include <cutlass/trace.h>
|
||||
|
||||
#include <cute/tensor.hpp>
|
||||
|
||||
template <class Layout, class CoSizeHi>
|
||||
void
|
||||
test_complement(Layout const& layout, CoSizeHi const& cosize_hi)
|
||||
{
|
||||
using namespace cute;
|
||||
|
||||
auto result = complement(layout, cosize_hi);
|
||||
|
||||
CUTLASS_TRACE_HOST("complement( " << layout << ", " << cosize_hi << ") => " << result);
|
||||
|
||||
// Post-condition on the domain size of the complement (1)
|
||||
EXPECT_GE( size(result), cosize_hi / size(filter(layout)));
|
||||
// Post-condition on the codomain size of the complement (2)
|
||||
EXPECT_LE(cosize(result), cute::ceil_div(cosize_hi, cosize(layout)) * cosize(layout));
|
||||
|
||||
// Post-condition on the codomain of the complement
|
||||
for (int i = 1; i < size(result); ++i) {
|
||||
EXPECT_LT(result(i-1), result(i)); // Ordered (3)
|
||||
for (int j = 0; j < size(layout); ++j) {
|
||||
EXPECT_NE(result(i), layout(j)); // Complemented (4)
|
||||
}
|
||||
}
|
||||
|
||||
// Other observations
|
||||
EXPECT_LE(size(result),cosize(result)); // As a result of the ordered condition (3)
|
||||
EXPECT_GE(cosize(result), cosize_hi / size(filter(layout))); // As a result of (1) (2) and (5)
|
||||
if constexpr (is_static<decltype(stride(make_layout(layout,result)))>::value) { // If we can apply complement again
|
||||
EXPECT_EQ(size(complement(make_layout(layout,result))), 1); // There's no more codomain left over
|
||||
}
|
||||
}
|
||||
|
||||
template <class Layout>
|
||||
void
|
||||
test_complement(Layout const& layout)
|
||||
{
|
||||
return test_complement(layout, cosize(layout));
|
||||
}
|
||||
|
||||
TEST(CuTe_core, Complement)
|
||||
{
|
||||
using namespace cute;
|
||||
|
||||
CUTLASS_TRACE_HOST("-------------------------------");
|
||||
CUTLASS_TRACE_HOST("COMPLEMENT");
|
||||
CUTLASS_TRACE_HOST("-------------------------------");
|
||||
|
||||
{
|
||||
auto layout = Layout<_1,_0>{};
|
||||
|
||||
test_complement(layout);
|
||||
test_complement(layout, Int<2>{});
|
||||
}
|
||||
|
||||
{
|
||||
auto layout = Layout<_1,_1>{};
|
||||
|
||||
test_complement(layout);
|
||||
test_complement(layout, Int<2>{});
|
||||
}
|
||||
|
||||
{
|
||||
auto layout = Layout<_1,_2>{};
|
||||
|
||||
test_complement(layout, Int<1>{});
|
||||
test_complement(layout, Int<2>{});
|
||||
test_complement(layout, Int<8>{});
|
||||
}
|
||||
|
||||
{
|
||||
auto layout = Layout<_4,_0>{};
|
||||
|
||||
test_complement(layout, Int<1>{});
|
||||
test_complement(layout, Int<2>{});
|
||||
test_complement(layout, Int<8>{});
|
||||
}
|
||||
|
||||
{
|
||||
auto layout = Layout<_4,_1>{};
|
||||
|
||||
test_complement(layout, Int<1>{});
|
||||
test_complement(layout, Int<2>{});
|
||||
test_complement(layout, Int<8>{});
|
||||
}
|
||||
|
||||
{
|
||||
auto layout = Layout<_4,_2>{};
|
||||
|
||||
test_complement(layout, Int<1>{});
|
||||
test_complement(layout);
|
||||
test_complement(layout, Int<16>{});
|
||||
}
|
||||
|
||||
{
|
||||
auto layout = Layout<_4,_4>{};
|
||||
|
||||
test_complement(layout, Int<1>{});
|
||||
test_complement(layout);
|
||||
test_complement(layout, Int<17>{});
|
||||
}
|
||||
|
||||
{
|
||||
auto layout = Layout<Shape<_2,_4>>{};
|
||||
|
||||
test_complement(layout);
|
||||
}
|
||||
|
||||
{
|
||||
auto layout = Layout<Shape<_2,_3>>{};
|
||||
|
||||
test_complement(layout);
|
||||
}
|
||||
|
||||
{
|
||||
auto layout = Layout<Shape<_2,_4>, Stride<_1,_4>>{};
|
||||
|
||||
test_complement(layout);
|
||||
}
|
||||
|
||||
{
|
||||
auto layout = Layout<Shape<_2,_4,_8>, Stride<_8,_1,_64>>{};
|
||||
|
||||
test_complement(layout);
|
||||
}
|
||||
|
||||
{
|
||||
auto layout = Layout<Shape<_2,_4,_8>, Stride<_8,_1,_0>>{};
|
||||
|
||||
test_complement(layout);
|
||||
test_complement(layout, Int<460>{});
|
||||
}
|
||||
|
||||
{
|
||||
auto layout = make_layout(Shape<Shape<_2,_2>,Shape<_2, _2>>{},
|
||||
Stride<Stride<_1,_4>,Stride<_8,_32>>{});
|
||||
|
||||
test_complement(layout);
|
||||
}
|
||||
|
||||
{
|
||||
auto layout = make_layout(Shape<Shape<_2,_2>,Shape<_2, _2>>{},
|
||||
Stride<Stride<_1,_32>,Stride<_8,_4>>{});
|
||||
|
||||
test_complement(layout);
|
||||
}
|
||||
|
||||
// Fails due to non-injective input
|
||||
//{
|
||||
//auto layout = make_layout(Shape<Shape<_2,_2>,Shape<_2, _2>>{},
|
||||
// Stride<Stride<_1,_8>,Stride<_8,_4>>{});
|
||||
|
||||
//test_complement(layout);
|
||||
//}
|
||||
|
||||
{
|
||||
auto layout = Layout<Shape<_4,_6>, Stride<_1,_6>>{};
|
||||
|
||||
test_complement(layout);
|
||||
}
|
||||
|
||||
{
|
||||
auto layout = Layout<Shape<_4,_2>, Stride<_1,_10>>{};
|
||||
|
||||
test_complement(layout);
|
||||
}
|
||||
|
||||
{
|
||||
auto layout = Layout<Shape<_4,_2>, Stride<_1,_16>>{};
|
||||
|
||||
test_complement(layout);
|
||||
}
|
||||
|
||||
CUTLASS_TRACE_HOST("-------------------------------");
|
||||
CUTLASS_TRACE_HOST("Dynamic shapes/strides");
|
||||
CUTLASS_TRACE_HOST("-------------------------------");
|
||||
|
||||
{
|
||||
auto layout = make_layout(12);
|
||||
|
||||
test_complement(layout, 1);
|
||||
test_complement(layout);
|
||||
test_complement(layout, 53);
|
||||
test_complement(layout, 128);
|
||||
}
|
||||
|
||||
{
|
||||
auto layout = make_layout(12, 1);
|
||||
|
||||
test_complement(layout, 1);
|
||||
test_complement(layout);
|
||||
test_complement(layout, 53);
|
||||
test_complement(layout, 128);
|
||||
}
|
||||
|
||||
{
|
||||
auto layout = make_layout(12, Int<2>{});
|
||||
|
||||
test_complement(layout, 1);
|
||||
test_complement(layout);
|
||||
test_complement(layout, 53);
|
||||
test_complement(layout, 128);
|
||||
}
|
||||
|
||||
{
|
||||
auto layout = make_layout(12, 2);
|
||||
|
||||
test_complement(layout, 1);
|
||||
test_complement(layout);
|
||||
test_complement(layout, 53);
|
||||
test_complement(layout, 128);
|
||||
}
|
||||
|
||||
{
|
||||
auto layout = make_layout(make_shape(3,6),make_stride(_1{}, _3{}));
|
||||
|
||||
test_complement(layout);
|
||||
}
|
||||
|
||||
{
|
||||
auto layout = make_layout(make_shape(3,6),make_stride(_1{}, _9{}));
|
||||
|
||||
test_complement(layout);
|
||||
}
|
||||
|
||||
{
|
||||
auto layout = make_layout(make_shape(3,6),make_stride(_1{}, _10{}));
|
||||
|
||||
test_complement(layout);
|
||||
}
|
||||
|
||||
{
|
||||
auto layout = make_layout(make_shape(make_shape(2,2), make_shape(2,2)),
|
||||
Stride<Stride<_1,_4>,Stride<_8,_32>>{});
|
||||
|
||||
test_complement(layout);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,528 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017 - 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
#include "cutlass_unit_test.h"
|
||||
|
||||
#include <cutlass/trace.h>
|
||||
|
||||
#include <iostream>
|
||||
|
||||
#include <cute/tensor.hpp>
|
||||
|
||||
using namespace cute;
|
||||
|
||||
|
||||
template <class LayoutA, class LayoutB>
|
||||
void
|
||||
test_composition(const LayoutA& layoutA,
|
||||
const LayoutB& layoutB)
|
||||
{
|
||||
auto layoutR = composition(layoutA, layoutB);
|
||||
|
||||
CUTLASS_TRACE_HOST("test_composition()");
|
||||
CUTLASS_TRACE_HOST(layoutA << " o " << layoutB);
|
||||
CUTLASS_TRACE_HOST(" => ");
|
||||
CUTLASS_TRACE_HOST(layoutR);
|
||||
|
||||
// Test that layout R is compatible with layout B
|
||||
EXPECT_TRUE(compatible(layoutB, layoutR));
|
||||
|
||||
// True post-condition: Every coordinate c of layoutB with L1D(c) < size(layoutR) is a coordinate of layoutR.
|
||||
|
||||
// Test that R(c) = A(B(c)) for all coordinates c in layoutR
|
||||
for (int i = 0; i < size(layoutR); ++i) {
|
||||
EXPECT_EQ(layoutR(i), layoutA(layoutB(i)));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
TEST(CuTe_core, Composition)
|
||||
{
|
||||
CUTLASS_TRACE_HOST("-------------------------------");
|
||||
CUTLASS_TRACE_HOST("COMPOSITION" );
|
||||
CUTLASS_TRACE_HOST("-------------------------------");
|
||||
|
||||
CUTLASS_TRACE_HOST("-------------------------------");
|
||||
CUTLASS_TRACE_HOST("Simple tests" );
|
||||
CUTLASS_TRACE_HOST("-------------------------------");
|
||||
|
||||
{
|
||||
auto a = Layout<_1,_0>{};
|
||||
auto b = Layout<_1,_0>{};
|
||||
|
||||
test_composition(a, b);
|
||||
}
|
||||
|
||||
{
|
||||
auto a = Layout<_1,_0>{};
|
||||
auto b = Layout<_1,_1>{};
|
||||
|
||||
test_composition(a, b);
|
||||
}
|
||||
|
||||
{
|
||||
auto a = Layout<_1,_1>{};
|
||||
auto b = Layout<_1,_0>{};
|
||||
|
||||
test_composition(a, b);
|
||||
}
|
||||
|
||||
{
|
||||
auto a = Layout<_1,_1>{};
|
||||
auto b = Layout<_1,_1>{};
|
||||
|
||||
test_composition(a, b);
|
||||
}
|
||||
|
||||
{
|
||||
auto a = make_layout(Shape<_4>{});
|
||||
auto b = make_layout(Shape<_4>{});
|
||||
|
||||
test_composition(a, b);
|
||||
}
|
||||
|
||||
{
|
||||
auto a = make_layout(Shape<_4>{}, Stride<_2>{});
|
||||
auto b = make_layout(Shape<_4>{});
|
||||
|
||||
test_composition(a, b);
|
||||
}
|
||||
|
||||
{
|
||||
auto a = make_layout(Shape<_4>{}, Stride<_0>{});
|
||||
auto b = make_layout(Shape<_4>{});
|
||||
|
||||
test_composition(a, b);
|
||||
}
|
||||
|
||||
{
|
||||
auto a = make_layout(Shape<_4>{});
|
||||
auto b = make_layout(Shape<_4>{}, Stride<_0>{});
|
||||
|
||||
test_composition(a, b);
|
||||
}
|
||||
|
||||
{
|
||||
auto a = make_layout(Shape<_4>{});
|
||||
auto b = make_layout(Shape<_1>{}, Stride<_0>{});
|
||||
|
||||
test_composition(a, b);
|
||||
}
|
||||
|
||||
{
|
||||
auto a = make_layout(Shape<_4>{});
|
||||
auto b = make_layout(Shape<_2>{});
|
||||
|
||||
test_composition(a, b);
|
||||
}
|
||||
|
||||
{
|
||||
auto a = make_layout(Shape<_4>{}, Stride<_2>{});
|
||||
auto b = make_layout(Shape<_2>{});
|
||||
|
||||
test_composition(a, b);
|
||||
}
|
||||
|
||||
{
|
||||
auto a = make_layout(Shape<_4>{});
|
||||
auto b = make_layout(Shape<_2>{}, Stride<_2>{});
|
||||
|
||||
test_composition(a, b);
|
||||
}
|
||||
|
||||
{
|
||||
auto a = make_layout(Shape<_4>{}, Stride<_2>{});
|
||||
auto b = make_layout(Shape<_2>{}, Stride<_2>{});
|
||||
|
||||
test_composition(a, b);
|
||||
}
|
||||
|
||||
{
|
||||
auto a = make_layout(Shape<_4,_3>{});
|
||||
auto b = make_layout(Shape<_12>{});
|
||||
|
||||
test_composition(a, b);
|
||||
}
|
||||
|
||||
{
|
||||
auto a = make_layout(Shape<_12>{});
|
||||
auto b = make_layout(Shape<_4,_3>{});
|
||||
|
||||
test_composition(a, b);
|
||||
}
|
||||
|
||||
{
|
||||
auto a = make_layout(Shape<_12>{}, Stride<_2>{});
|
||||
auto b = make_layout(Shape<_4,_3>{});
|
||||
|
||||
test_composition(a, b);
|
||||
}
|
||||
|
||||
{
|
||||
auto a = make_layout(Shape<_12>{});
|
||||
auto b = make_layout(Shape<_4,_3>{}, Stride<_3,_1>{});
|
||||
|
||||
test_composition(a, b);
|
||||
}
|
||||
|
||||
{
|
||||
auto a = make_layout(Shape<_12>{}, Stride<_2>{});
|
||||
auto b = make_layout(Shape<_4,_3>{}, Stride<_3,_1>{});
|
||||
|
||||
test_composition(a, b);
|
||||
}
|
||||
|
||||
{
|
||||
auto a = make_layout(Shape<_12>{});
|
||||
auto b = make_layout(Shape<_2,_3>{}, Stride<_2,_4>{});
|
||||
|
||||
test_composition(a, b);
|
||||
}
|
||||
|
||||
{
|
||||
auto a = make_layout(Shape<_4,_3>{});
|
||||
auto b = make_layout(Shape<_4,_3>{});
|
||||
|
||||
test_composition(a, b);
|
||||
}
|
||||
|
||||
// FAILS due to b not "dividing into" a properly
|
||||
//{
|
||||
// auto a = make_layout(Shape<_4,_3>{});
|
||||
// auto b = make_layout(Shape<_6>{});
|
||||
|
||||
// test_composition(a, b);
|
||||
//}
|
||||
|
||||
{
|
||||
auto a = make_layout(Shape<_4,_3>{});
|
||||
auto b = make_layout(Shape<_6>{}, Stride<_2>{});
|
||||
|
||||
test_composition(a, b);
|
||||
}
|
||||
|
||||
{
|
||||
auto a = make_layout(Shape<_4,_3>{});
|
||||
auto b = make_layout(Shape<_6,_2>{}, Stride<_2,_1>{});
|
||||
|
||||
test_composition(a, b);
|
||||
}
|
||||
|
||||
// FAILS due to b not "dividing into" a properly
|
||||
//{
|
||||
// auto a = make_layout(Shape<_4,_3>{});
|
||||
// auto b = make_layout(Shape<_4,_3>{}, Stride<_3,_1>{});
|
||||
|
||||
// test_composition(a, b);
|
||||
//}
|
||||
|
||||
{
|
||||
auto a = make_layout(Shape<_4,_3>{}, Stride<_3,_1>{});
|
||||
auto b = make_layout(Shape<_4,_3>{});
|
||||
|
||||
test_composition(a, b);
|
||||
}
|
||||
|
||||
{
|
||||
auto a = make_layout(Shape<_4,_3>{}, Stride<_3,_1>{});
|
||||
auto b = make_layout(Shape<_12>{});
|
||||
|
||||
test_composition(a, b);
|
||||
}
|
||||
|
||||
{
|
||||
auto a = make_layout(Shape<_4,_3>{}, Stride<_3,_1>{});
|
||||
auto b = make_layout(Shape<_6>{}, Stride<_2>{});
|
||||
|
||||
test_composition(a, b);
|
||||
}
|
||||
|
||||
{
|
||||
auto a = make_layout(Shape<_4,_3>{}, Stride<_3,_1>{});
|
||||
auto b = make_layout(Shape<_6,_2>{}, Stride<_2,_1>{});
|
||||
|
||||
test_composition(a, b);
|
||||
}
|
||||
|
||||
{
|
||||
auto a = make_layout(Shape<_8,_8>{});
|
||||
auto b = make_layout(Shape<Shape<_2, _2,_2>, Shape<_2,_2, _2>>{},
|
||||
Stride<Stride<_1,_16,_4>, Stride<_8,_2,_32>>{});
|
||||
test_composition(a, b);
|
||||
}
|
||||
|
||||
{
|
||||
auto a = make_layout(Shape<_8,_8>{}, Stride<_8,_1>{});
|
||||
auto b = make_layout(Shape<Shape<_2, _2,_2>, Shape<_2,_2, _2>>{},
|
||||
Stride<Stride<_1,_16,_4>, Stride<_8,_2,_32>>{});
|
||||
|
||||
test_composition(a, b);
|
||||
}
|
||||
|
||||
{
|
||||
auto a = make_layout(Shape<Shape<_4,_2>>{}, Stride<Stride<_1,_16>>{});
|
||||
auto b = make_layout(Shape<_4,_2>{}, Stride<_2,_1>{});
|
||||
|
||||
test_composition(a, b);
|
||||
}
|
||||
|
||||
{
|
||||
auto a = make_layout(Shape<_2,_2>{}, Stride<_2,_1>{});
|
||||
auto b = make_layout(Shape<_2,_2>{}, Stride<_2,_1>{});
|
||||
|
||||
test_composition(a, b);
|
||||
}
|
||||
|
||||
{
|
||||
auto a = make_layout(Shape<_4,_8,_2>{});
|
||||
auto b = make_layout(Shape<_2,_2,_2>{}, Stride<_2,_8,_1>{});
|
||||
|
||||
test_composition(a, b);
|
||||
}
|
||||
|
||||
{
|
||||
auto a = make_layout(Shape<_4,_8,_2>{}, Stride<_2,_8,_1>{});
|
||||
auto b = make_layout(Shape<_2,_2,_2>{}, Stride<_1,_8,_2>{});
|
||||
|
||||
test_composition(a, b);
|
||||
}
|
||||
|
||||
{
|
||||
auto a = make_layout(Shape<_4,_8,_2>{}, Stride<_2,_8,_1>{});
|
||||
auto b = make_layout(Shape<_4,_2,_2>{}, Stride<_2,_8,_1>{});
|
||||
|
||||
test_composition(a, b);
|
||||
}
|
||||
|
||||
CUTLASS_TRACE_HOST("-------------------------------");
|
||||
CUTLASS_TRACE_HOST("Dynamic shapes/strides" );
|
||||
CUTLASS_TRACE_HOST("-------------------------------");
|
||||
|
||||
|
||||
{
|
||||
auto a = make_layout(12, 1);
|
||||
auto b = make_layout(_4{}, _1{});
|
||||
|
||||
test_composition(a, b);
|
||||
}
|
||||
|
||||
{
|
||||
auto a = make_layout(12, 1);
|
||||
auto b = make_layout(_4{}, 1);
|
||||
|
||||
test_composition(a, b);
|
||||
}
|
||||
|
||||
{
|
||||
auto a = make_layout(12, _1{});
|
||||
auto b = make_layout(_4{}, 1);
|
||||
|
||||
test_composition(a, b);
|
||||
}
|
||||
|
||||
{
|
||||
auto a = make_layout(12, _1{});
|
||||
auto b = make_layout(_4{}, _1{});
|
||||
|
||||
test_composition(a, b);
|
||||
}
|
||||
|
||||
{
|
||||
auto a = make_layout(make_shape(12,3), make_stride(1,24));
|
||||
auto b = make_layout(Shape<_4>{}, Stride<_1>{});
|
||||
|
||||
test_composition(a, b);
|
||||
}
|
||||
|
||||
{
|
||||
auto a = make_layout(16, 2);
|
||||
auto b = make_layout(4, 2);
|
||||
|
||||
test_composition(a, b);
|
||||
}
|
||||
|
||||
{
|
||||
auto a = make_layout(make_shape(128,24,5), make_stride(1,128,3072));
|
||||
auto b = make_layout(64, 2);
|
||||
|
||||
test_composition(a, b);
|
||||
}
|
||||
|
||||
{
|
||||
auto a = make_layout(make_shape(128,24,5), make_stride(1,128,3072));
|
||||
auto b = make_layout(480, Int<32>{});
|
||||
|
||||
test_composition(a, b);
|
||||
}
|
||||
|
||||
CUTLASS_TRACE_HOST("-------------------------------");
|
||||
CUTLASS_TRACE_HOST("cosize(b) > size(a) and divisibility");
|
||||
CUTLASS_TRACE_HOST("-------------------------------");
|
||||
|
||||
{
|
||||
auto a = make_layout(Shape<_1>{}, Stride<_0>{});
|
||||
auto b = make_layout(Shape<_4>{});
|
||||
|
||||
test_composition(a, b);
|
||||
}
|
||||
|
||||
{
|
||||
auto a = make_layout(Shape<_1>{}, Stride<_1>{});
|
||||
auto b = make_layout(Shape<_4>{});
|
||||
|
||||
test_composition(a, b);
|
||||
}
|
||||
|
||||
{
|
||||
auto a = make_layout(Shape<_4>{});
|
||||
auto b = make_layout(Shape<_4>{}, Stride<_2>{});
|
||||
|
||||
test_composition(a, b);
|
||||
}
|
||||
|
||||
// Last mode gets extended
|
||||
{
|
||||
auto a = make_layout(Shape<_4,_3>{}, Stride<_3,_1>{});
|
||||
auto b = make_layout(Shape<_24>{});
|
||||
|
||||
test_composition(a, b);
|
||||
}
|
||||
|
||||
// Last mode extension even without last mode divisibility
|
||||
{
|
||||
auto a = make_layout(Shape<_4,_3>{}, Stride<_3,_1>{});
|
||||
auto b = make_layout(Shape<_8>{});
|
||||
|
||||
test_composition(a, b);
|
||||
}
|
||||
|
||||
// Capping a Layout with 1:0 forces divisibility and extends in stride-0
|
||||
{
|
||||
auto a = make_layout(Shape<_4,_3,_1>{}, Stride<_3,_1,_0>{});
|
||||
auto b = make_layout(Shape<_24>{});
|
||||
|
||||
test_composition(a, b);
|
||||
}
|
||||
|
||||
{
|
||||
auto a = make_layout(3, _1{});
|
||||
auto b = make_layout(_4{}, _1{});
|
||||
|
||||
test_composition(a, b);
|
||||
}
|
||||
|
||||
{
|
||||
auto a = make_layout(make_shape(48,24,5), make_stride(_1{},128,3072));
|
||||
auto b = make_layout(32, Int<1>{});
|
||||
|
||||
test_composition(a, b);
|
||||
}
|
||||
|
||||
CUTLASS_TRACE_HOST("-------------------------------");
|
||||
CUTLASS_TRACE_HOST("Swizzle composition" );
|
||||
CUTLASS_TRACE_HOST("-------------------------------");
|
||||
|
||||
{
|
||||
auto a = Layout<Shape<_8,_8>, Stride<_8,_1>>{};
|
||||
auto b = composition(Swizzle<2,0,-3>{}, Layout<Shape<_8,_8>, Stride<_8,_1>>{});
|
||||
|
||||
test_composition(a, b);
|
||||
}
|
||||
|
||||
{
|
||||
auto a = composition(Swizzle<2,0, 3>{}, Layout<Shape<_8,_8>, Stride<_8,_1>>{});
|
||||
auto b = composition(Swizzle<2,0,-3>{}, Layout<Shape<_8,_8>, Stride<_8,_1>>{});
|
||||
|
||||
test_composition(a, b);
|
||||
}
|
||||
|
||||
CUTLASS_TRACE_HOST("-------------------------------");
|
||||
CUTLASS_TRACE_HOST("BETA: Negative strides" );
|
||||
CUTLASS_TRACE_HOST("-------------------------------");
|
||||
|
||||
{
|
||||
auto a = make_layout(Shape<_4>{}, Stride<_m1>{});
|
||||
auto b = make_layout(Shape<_4>{}, Stride<_1>{});
|
||||
|
||||
test_composition(a, b);
|
||||
}
|
||||
|
||||
{
|
||||
auto a = make_layout(Shape<_4>{}, Stride<_1>{});
|
||||
auto b = make_layout(Shape<_4>{}, Stride<_m1>{});
|
||||
|
||||
test_composition(a, b);
|
||||
}
|
||||
|
||||
{
|
||||
auto a = make_layout(Shape<_4>{}, Stride<_m1>{});
|
||||
auto b = make_layout(Shape<_4>{}, Stride<_m1>{});
|
||||
|
||||
test_composition(a, b);
|
||||
}
|
||||
|
||||
{
|
||||
auto a = make_layout(Shape<_4>{}, Stride<_1>{});
|
||||
auto b = make_layout(Shape<_4>{}, Stride<_m2>{});
|
||||
|
||||
test_composition(a, b);
|
||||
}
|
||||
|
||||
{
|
||||
auto a = make_layout(Shape<_4,_4>{}, Stride<_m1,_1>{});
|
||||
auto b = make_layout(Shape<_2,_4,_2>{});
|
||||
|
||||
test_composition(a, b);
|
||||
}
|
||||
|
||||
{
|
||||
auto a = make_layout(Shape<_4,_4>{}, Stride<_m1,_1>{});
|
||||
auto b = make_layout(Shape<_2,_4,_2>{}, Stride<_1,_4,_2>{});
|
||||
|
||||
test_composition(a, b);
|
||||
}
|
||||
|
||||
// The SM80 fp64 MMA NT problem
|
||||
{
|
||||
auto a = make_layout(Shape<_1,Shape<_2,_4>>{}, Stride<_0,Stride<_m1,_512>>{});
|
||||
auto b = make_layout(_2{}, _m1{});
|
||||
|
||||
test_composition(a, b);
|
||||
}
|
||||
|
||||
{
|
||||
auto a = make_layout(Shape<_1,Shape<_2,_4>>{}, Stride<_0,Stride<_m1,_512>>{});
|
||||
auto b = make_layout(_4{}, _m1{});
|
||||
|
||||
test_composition(a, b);
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,183 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017 - 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
#include "cutlass_unit_test.h"
|
||||
|
||||
#include <cutlass/trace.h>
|
||||
|
||||
#include <iostream>
|
||||
|
||||
#include <cute/tensor.hpp>
|
||||
|
||||
using namespace cute;
|
||||
|
||||
template <class Layout>
|
||||
void
|
||||
test_left_inverse(Layout const& layout)
|
||||
{
|
||||
auto inv_layout = left_inverse(layout);
|
||||
|
||||
CUTLASS_TRACE_HOST(layout << " ^ -1\n" << " => \n" << inv_layout);
|
||||
|
||||
for (int i = 0; i < size(layout); ++i) {
|
||||
//printf("%3d: %3d %3d\n", i, int(layout(i)), int(inv_layout(layout(i))));
|
||||
EXPECT_EQ(inv_layout(layout(i)), i);
|
||||
}
|
||||
|
||||
CUTLASS_TRACE_HOST("Composition: " << coalesce(composition(inv_layout, layout)));
|
||||
}
|
||||
|
||||
TEST(CuTe_core, Inverse_left)
|
||||
{
|
||||
{
|
||||
auto layout = Layout<Shape <_1>,
|
||||
Stride<_0>>{};
|
||||
|
||||
test_left_inverse(layout);
|
||||
}
|
||||
|
||||
{
|
||||
auto layout = Layout<Shape <Shape <_1,_1>>,
|
||||
Stride<Stride<_0,_0>>>{};
|
||||
|
||||
test_left_inverse(layout);
|
||||
}
|
||||
|
||||
{
|
||||
auto layout = Layout<Shape <_1>,
|
||||
Stride<_1>>{};
|
||||
|
||||
test_left_inverse(layout);
|
||||
}
|
||||
|
||||
{
|
||||
auto layout = Layout<Shape <_4>,
|
||||
Stride<_1>>{};
|
||||
|
||||
test_left_inverse(layout);
|
||||
}
|
||||
|
||||
{
|
||||
auto layout = Layout<Shape <_4>,
|
||||
Stride<_2>>{};
|
||||
|
||||
test_left_inverse(layout);
|
||||
}
|
||||
|
||||
{
|
||||
auto layout = Layout<Shape <_8, _4>>{};
|
||||
|
||||
test_left_inverse(layout);
|
||||
}
|
||||
|
||||
{
|
||||
auto layout = Layout<Shape <_8, _4>,
|
||||
Stride<_4, _1>>{};
|
||||
|
||||
test_left_inverse(filter(layout));
|
||||
}
|
||||
|
||||
{
|
||||
auto layout = Layout<Shape< _2,_4,_6>>{};
|
||||
|
||||
test_left_inverse(layout);
|
||||
}
|
||||
|
||||
{
|
||||
auto layout = Layout<Shape <_2,_4,_6>,
|
||||
Stride<_4,_1,_8>>{};
|
||||
|
||||
test_left_inverse(layout);
|
||||
}
|
||||
|
||||
{
|
||||
auto layout = Layout<Shape <_4, _2>,
|
||||
Stride<_1,_16>>{};
|
||||
|
||||
test_left_inverse(layout);
|
||||
}
|
||||
|
||||
//
|
||||
// Swizzle left_inverse
|
||||
//
|
||||
|
||||
{
|
||||
auto layout = ComposedLayout<Swizzle<1,0,2>, _0, Layout<Shape <_4, _4>,
|
||||
Stride<_1, _4>>>{};
|
||||
|
||||
test_left_inverse(layout);
|
||||
}
|
||||
|
||||
{
|
||||
auto layout = ComposedLayout<Swizzle<1,0,2>, _0, Layout<Shape <_4, _4>,
|
||||
Stride<_4, _1>>>{};
|
||||
|
||||
test_left_inverse(layout);
|
||||
}
|
||||
|
||||
{
|
||||
auto layout = ComposedLayout<Swizzle<1,0,1>, _0, Layout<Shape <_4, _4>,
|
||||
Stride<_8, _1>>>{};
|
||||
|
||||
test_left_inverse(layout);
|
||||
}
|
||||
|
||||
//
|
||||
// Negative strides (beta support)
|
||||
// Post-conditions/layout indexing aren't generalized enough to support these yet
|
||||
// However, the composition post-condition is general enough.
|
||||
{
|
||||
auto layout = make_layout(Shape<_4>{}, Stride<Int<-1>>{});
|
||||
|
||||
test_left_inverse(layout);
|
||||
}
|
||||
|
||||
//{
|
||||
//auto layout = Layout<Shape < _2,_4>,
|
||||
// Stride<_m1,_2>>{};
|
||||
|
||||
//test_left_inverse(layout);
|
||||
//}
|
||||
|
||||
//{
|
||||
//auto layout = Layout<Shape < _2, _4>,
|
||||
// Stride< _4,_m1>>{};
|
||||
|
||||
//test_left_inverse(layout);
|
||||
//}
|
||||
|
||||
//{
|
||||
//auto layout = Layout<Shape < _2, _4, _6>,
|
||||
// Stride<_m1,_12,_m2>>{};
|
||||
|
||||
//test_left_inverse(layout);
|
||||
//}
|
||||
}
|
||||
@@ -0,0 +1,255 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017 - 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
#include "cutlass_unit_test.h"
|
||||
|
||||
#include <cutlass/trace.h>
|
||||
#include <iostream>
|
||||
|
||||
#include <cute/tensor.hpp>
|
||||
|
||||
using namespace cute;
|
||||
|
||||
template <class Layout>
|
||||
void
|
||||
test_right_inverse(Layout const& layout)
|
||||
{
|
||||
auto inv_layout = right_inverse(layout);
|
||||
|
||||
CUTLASS_TRACE_HOST(layout << " ^ -1\n" << " => \n" << inv_layout);
|
||||
CUTLASS_TRACE_HOST("Composition: " << coalesce(composition(layout, inv_layout)) << std::endl);
|
||||
|
||||
for (int i = 0; i < size(inv_layout); ++i) {
|
||||
//printf("%3d: %3d %3d\n", i, int(inv_layout(i)), int(layout(inv_layout(i))));
|
||||
EXPECT_EQ(layout(inv_layout(i)), i);
|
||||
}
|
||||
}
|
||||
|
||||
TEST(CuTe_core, Inverse_right)
|
||||
{
|
||||
CUTLASS_TRACE_HOST("-------------------------------");
|
||||
CUTLASS_TRACE_HOST("RIGHT INVERSE" );
|
||||
CUTLASS_TRACE_HOST("-------------------------------");
|
||||
|
||||
CUTLASS_TRACE_HOST("-------------------------------");
|
||||
CUTLASS_TRACE_HOST("Simple tests" );
|
||||
CUTLASS_TRACE_HOST("-------------------------------");
|
||||
|
||||
{
|
||||
auto layout = Layout<_1, _0>{};
|
||||
|
||||
test_right_inverse(layout);
|
||||
}
|
||||
|
||||
{
|
||||
auto layout = Layout<_1, _1>{};
|
||||
|
||||
test_right_inverse(layout);
|
||||
}
|
||||
|
||||
{
|
||||
auto layout = Layout<Shape <_4>,
|
||||
Stride<_0>>{};
|
||||
|
||||
test_right_inverse(layout);
|
||||
}
|
||||
|
||||
{
|
||||
auto layout = Layout<Shape <Shape <_1,_1>>,
|
||||
Stride<Stride<_0,_0>>>{};
|
||||
|
||||
test_right_inverse(layout);
|
||||
}
|
||||
|
||||
{
|
||||
auto layout = Layout<Shape <Shape <_3,_7>>,
|
||||
Stride<Stride<_0,_0>>>{};
|
||||
|
||||
test_right_inverse(layout);
|
||||
}
|
||||
|
||||
{
|
||||
auto layout = Layout<Shape <_1>,
|
||||
Stride<_1>>{};
|
||||
|
||||
test_right_inverse(layout);
|
||||
}
|
||||
|
||||
{
|
||||
auto layout = Layout<Shape <_4>,
|
||||
Stride<_1>>{};
|
||||
|
||||
test_right_inverse(layout);
|
||||
}
|
||||
|
||||
{
|
||||
auto layout = Layout<Shape <_4>,
|
||||
Stride<_2>>{};
|
||||
|
||||
test_right_inverse(layout);
|
||||
}
|
||||
|
||||
{
|
||||
auto layout = Layout<Shape <_2,_4>,
|
||||
Stride<_0,_2>>{};
|
||||
|
||||
test_right_inverse(layout);
|
||||
}
|
||||
|
||||
{
|
||||
auto layout = Layout<Shape <_8, _4>>{};
|
||||
|
||||
test_right_inverse(layout);
|
||||
}
|
||||
|
||||
{
|
||||
auto layout = Layout<Shape <_8, _4>,
|
||||
Stride<_4, _1>>{};
|
||||
|
||||
test_right_inverse(layout);
|
||||
}
|
||||
|
||||
{
|
||||
auto layout = Layout<Shape< _2,_4,_6>>{};
|
||||
|
||||
test_right_inverse(layout);
|
||||
}
|
||||
|
||||
{
|
||||
auto layout = Layout<Shape <_2,_4,_6>,
|
||||
Stride<_4,_1,_8>>{};
|
||||
|
||||
test_right_inverse(layout);
|
||||
}
|
||||
|
||||
{
|
||||
auto layout = Layout<Shape <_2,_4,_4,_6>,
|
||||
Stride<_4,_1,_0,_8>>{};
|
||||
|
||||
test_right_inverse(layout);
|
||||
}
|
||||
|
||||
{
|
||||
auto layout = Layout<Shape <_4, _2>,
|
||||
Stride<_1,_16>>{};
|
||||
|
||||
test_right_inverse(layout);
|
||||
}
|
||||
|
||||
{
|
||||
auto layout = Layout<Shape <_4, _2>,
|
||||
Stride<_1, _5>>{};
|
||||
|
||||
test_right_inverse(layout);
|
||||
}
|
||||
|
||||
CUTLASS_TRACE_HOST("-------------------------------");
|
||||
CUTLASS_TRACE_HOST("Dynamic shapes/strides" );
|
||||
CUTLASS_TRACE_HOST("-------------------------------");
|
||||
|
||||
{
|
||||
auto layout = make_layout(Shape<_4, _2>{}, make_stride(Int<1>{}, 4));
|
||||
|
||||
test_right_inverse(layout);
|
||||
}
|
||||
|
||||
{
|
||||
auto layout = make_layout(make_shape(_4{}, 2), make_stride(Int<1>{}, 4));
|
||||
|
||||
test_right_inverse(layout);
|
||||
}
|
||||
|
||||
{
|
||||
auto layout = make_layout(make_shape(4, 2), make_stride(Int<1>{}, 4));
|
||||
|
||||
test_right_inverse(layout);
|
||||
}
|
||||
|
||||
CUTLASS_TRACE_HOST("-------------------------------");
|
||||
CUTLASS_TRACE_HOST("Swizzle layouts" );
|
||||
CUTLASS_TRACE_HOST("-------------------------------");
|
||||
|
||||
{
|
||||
auto layout = ComposedLayout<Swizzle<1,0,2>, _0, Layout<Shape <_4, _4>,
|
||||
Stride<_1, _4>>>{};
|
||||
|
||||
test_right_inverse(layout);
|
||||
}
|
||||
|
||||
{
|
||||
auto layout = ComposedLayout<Swizzle<1,0,2>, _0, Layout<Shape <_4, _4>,
|
||||
Stride<_4, _1>>>{};
|
||||
|
||||
test_right_inverse(layout);
|
||||
}
|
||||
|
||||
{
|
||||
auto layout = ComposedLayout<Swizzle<1,0,1>, _0, Layout<Shape <_4, _4>,
|
||||
Stride<_8, _1>>>{};
|
||||
|
||||
test_right_inverse(layout);
|
||||
}
|
||||
|
||||
CUTLASS_TRACE_HOST("-------------------------------");
|
||||
CUTLASS_TRACE_HOST("BETA: Negative strides" );
|
||||
CUTLASS_TRACE_HOST("-------------------------------");
|
||||
|
||||
// Negative strides (beta support)
|
||||
// Post-conditions/layout indexing aren't generalized enough to support these yet
|
||||
// However, the composition post-condition is general enough.
|
||||
{
|
||||
auto layout = make_layout(Shape<_4>{}, Stride<Int<-1>>{});
|
||||
|
||||
test_right_inverse(layout);
|
||||
}
|
||||
|
||||
//{
|
||||
//auto layout = Layout<Shape < _2,_4>,
|
||||
// Stride<_m1,_2>>{};
|
||||
|
||||
//test_right_inverse(layout);
|
||||
//}
|
||||
|
||||
//{
|
||||
//auto layout = Layout<Shape < _2, _4>,
|
||||
// Stride< _4,_m1>>{};
|
||||
|
||||
//test_right_inverse(layout);
|
||||
//}
|
||||
|
||||
//{
|
||||
//auto layout = Layout<Shape < _2, _4, _6>,
|
||||
// Stride<_m1,_12,_m2>>{};
|
||||
|
||||
//test_right_inverse(layout);
|
||||
//}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,253 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017 - 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
#include "cutlass_unit_test.h"
|
||||
|
||||
#include <cutlass/trace.h>
|
||||
|
||||
#include <cute/tensor.hpp>
|
||||
|
||||
using namespace cute;
|
||||
|
||||
template <class LayoutA, class LayoutB>
|
||||
void
|
||||
test_logical_divide(LayoutA const& layoutA,
|
||||
LayoutB const& layoutB)
|
||||
{
|
||||
auto layoutR = logical_divide(layoutA, layoutB);
|
||||
|
||||
CUTLASS_TRACE_HOST("test_logical_divide()");
|
||||
CUTLASS_TRACE_HOST(shape(layoutA) << " / " << shape(layoutB) << " => " << shape(layoutR) );
|
||||
CUTLASS_TRACE_HOST(stride(layoutA) << " " << stride(layoutB) << " => " << stride(layoutR));
|
||||
|
||||
// Test that layout R is compatible with layout B
|
||||
ASSERT_EQ(rank(layoutR), 2);
|
||||
ASSERT_TRUE(compatible(layoutB, layout<0>(layoutR)));
|
||||
}
|
||||
|
||||
TEST(CuTe_core, Logical_divide)
|
||||
{
|
||||
{
|
||||
auto layout = Layout<_1,_0>{};
|
||||
auto tile = Layout<_1,_0>{};
|
||||
|
||||
test_logical_divide(layout, tile);
|
||||
}
|
||||
|
||||
{
|
||||
auto layout = Layout<_1,_0>{};
|
||||
auto tile = Layout<_1,_1>{};
|
||||
|
||||
test_logical_divide(layout, tile);
|
||||
}
|
||||
|
||||
{
|
||||
auto layout = Layout<_1,_1>{};
|
||||
auto tile = Layout<_1,_0>{};
|
||||
|
||||
test_logical_divide(layout, tile);
|
||||
}
|
||||
|
||||
{
|
||||
auto layout = Layout<_1,_1>{};
|
||||
auto tile = Layout<_1,_1>{};
|
||||
|
||||
test_logical_divide(layout, tile);
|
||||
}
|
||||
|
||||
{
|
||||
auto layout = Layout<_6,_1>{};
|
||||
auto tile = Layout<_2,_1>{};
|
||||
|
||||
test_logical_divide(layout, tile);
|
||||
}
|
||||
|
||||
{
|
||||
auto layout = Layout<_6,_1>{};
|
||||
auto tile = Layout<_2,_3>{};
|
||||
|
||||
test_logical_divide(layout, tile);
|
||||
}
|
||||
|
||||
{
|
||||
auto layout = Layout<_6,_1>{};
|
||||
auto tile = Layout<Shape<_2,_3>,Stride<_3,_1>>{};
|
||||
|
||||
test_logical_divide(layout, tile);
|
||||
}
|
||||
|
||||
{
|
||||
auto layout = Layout<_6,_2>{};
|
||||
auto tile = Layout<_2,_1>{};
|
||||
|
||||
test_logical_divide(layout, tile);
|
||||
}
|
||||
|
||||
{
|
||||
auto layout = Layout<_6,_2>{};
|
||||
auto tile = Layout<_2,_3>{};
|
||||
|
||||
test_logical_divide(layout, tile);
|
||||
}
|
||||
|
||||
{
|
||||
auto layout = Layout<_6,_2>{};
|
||||
auto tile = Layout<Shape<_2,_3>,Stride<_3,_1>>{};
|
||||
|
||||
test_logical_divide(layout, tile);
|
||||
}
|
||||
|
||||
{
|
||||
auto layout = Layout<Shape<_6,_6>,Stride<_1,_12>>{};
|
||||
auto tile = Layout<Shape<_6,_3>,Stride<_3,_1>>{};
|
||||
|
||||
test_logical_divide(layout, tile);
|
||||
}
|
||||
|
||||
{
|
||||
auto layout = Layout<Shape<_6,_6>,Stride<_12,_1>>{};
|
||||
auto tile = Layout<Shape<_6,_3>,Stride<_3,_1>>{};
|
||||
|
||||
test_logical_divide(layout, tile);
|
||||
}
|
||||
|
||||
{
|
||||
auto layout = Layout<_32>{};
|
||||
auto tile = Layout<_2,_8>{};
|
||||
|
||||
test_logical_divide(layout, tile);
|
||||
}
|
||||
|
||||
{
|
||||
auto layout = Layout<Shape<_4,_1>,Stride<_1,_1>>{};
|
||||
auto tile = Layout<_2,_1>{};
|
||||
|
||||
test_logical_divide(layout, tile);
|
||||
}
|
||||
|
||||
{
|
||||
auto layout = Layout<Shape<_4,_1>,Stride<_1,_1>>{};
|
||||
auto tile = Layout<_2,_2>{};
|
||||
|
||||
test_logical_divide(layout, tile);
|
||||
}
|
||||
|
||||
{
|
||||
auto layout = Layout<Shape<_8,_8>,Stride<_1,_8>>{};
|
||||
auto tile = Layout<_32,_2>{};
|
||||
|
||||
test_logical_divide(layout, tile);
|
||||
}
|
||||
|
||||
{
|
||||
auto layout = Layout<Shape<_8,_8>,Stride<_8,_1>>{};
|
||||
auto tile = Layout<_32,_2>{};
|
||||
|
||||
test_logical_divide(layout, tile);
|
||||
}
|
||||
|
||||
//
|
||||
// Dynamic
|
||||
//
|
||||
|
||||
{
|
||||
auto layout = make_layout(2);
|
||||
auto tile = Layout<_32>{};
|
||||
|
||||
test_logical_divide(layout, tile);
|
||||
|
||||
// Enforcement for dynamic cases
|
||||
auto result = logical_divide(layout, tile);
|
||||
static_assert(decltype(shape<0>(result) == Int<32>{})::value);
|
||||
static_assert(decltype(stride<0>(result) == Int<1>{})::value);
|
||||
assert(shape<1>(result) == 1);
|
||||
static_assert(decltype(stride<1>(result) == Int<32>{})::value);
|
||||
}
|
||||
|
||||
{
|
||||
auto layout = make_layout(48);
|
||||
auto tile = Layout<_32>{};
|
||||
|
||||
test_logical_divide(layout, tile);
|
||||
|
||||
// Enforcement for dynamic cases
|
||||
auto result = logical_divide(layout, tile);
|
||||
static_assert(decltype(shape<0>(result) == Int<32>{})::value);
|
||||
static_assert(decltype(stride<0>(result) == Int<1>{})::value);
|
||||
assert(shape<1>(result) == 2);
|
||||
static_assert(decltype(stride<1>(result) == Int<32>{})::value);
|
||||
}
|
||||
|
||||
{
|
||||
auto layout = make_layout(96);
|
||||
auto tile = Layout<_32,_2>{};
|
||||
|
||||
test_logical_divide(layout, tile);
|
||||
}
|
||||
|
||||
{
|
||||
auto layout = make_layout(32);
|
||||
auto tile = Layout<Int<48>>{};
|
||||
|
||||
test_logical_divide(layout, tile);
|
||||
|
||||
// Enforcement for dynamic cases
|
||||
auto result = logical_divide(layout, tile);
|
||||
static_assert(decltype(shape<0>(result) == Int<48>{})::value);
|
||||
static_assert(decltype(stride<0>(result) == Int<1>{})::value);
|
||||
assert(shape<1>(result) == 1);
|
||||
static_assert(decltype(stride<1>(result) == Int<48>{})::value);
|
||||
}
|
||||
|
||||
// DISALLOWED
|
||||
//{
|
||||
//auto layout = make_layout(make_shape(128,4,3), make_stride(1,512,0));
|
||||
//auto tile = Layout<_32>{};
|
||||
|
||||
//test_logical_divide(layout, tile);
|
||||
//}
|
||||
|
||||
//{
|
||||
//auto layout = make_layout(make_shape(128,4,3), make_stride(1,512,0));
|
||||
//auto tile = Layout<_32,_2>{};
|
||||
|
||||
//CUTLASS_TRACE_HOST("complement: " << complement(tile, size(layout)));
|
||||
//test_logical_divide(layout, tile);
|
||||
//}
|
||||
|
||||
//{
|
||||
//auto layout = make_layout(make_shape(16,4,3), make_stride(1,512,0));
|
||||
//auto tile = Layout<_32>{};
|
||||
|
||||
//CUTLASS_TRACE_HOST("complement: " << complement(tile, size(layout)));
|
||||
//test_logical_divide(layout, tile);
|
||||
//}
|
||||
}
|
||||
@@ -0,0 +1,218 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017 - 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
#include "cutlass_unit_test.h"
|
||||
|
||||
#include <cutlass/trace.h>
|
||||
#include <cute/tensor.hpp>
|
||||
|
||||
using namespace cute;
|
||||
|
||||
template <class LayoutA, class LayoutB>
|
||||
void
|
||||
test_logical_product(LayoutA const& layoutA,
|
||||
LayoutB const& layoutB)
|
||||
{
|
||||
auto layoutR = logical_product(layoutA, layoutB);
|
||||
|
||||
CUTLASS_TRACE_HOST(shape(layoutA) << " x " << shape(layoutB) << " => " << shape(layoutR) );
|
||||
CUTLASS_TRACE_HOST(stride(layoutA) << " " << stride(layoutB) << " => " << stride(layoutR));
|
||||
|
||||
// Test that layout R is compatible with layout B
|
||||
ASSERT_EQ(rank(layoutR), 2);
|
||||
//assert(compatible(layoutB, layout<0>(layoutR)));
|
||||
//assert(consistent(layoutA, layout<1>(layoutR)));
|
||||
|
||||
// True post-condition:
|
||||
|
||||
}
|
||||
|
||||
TEST(CuTe_core, Logical_product)
|
||||
{
|
||||
{
|
||||
auto vec = Layout<_1,_0>{};
|
||||
auto tile = Layout<_1,_0>{};
|
||||
|
||||
test_logical_product(vec, tile);
|
||||
}
|
||||
|
||||
{
|
||||
auto vec = Layout<_1,_1>{};
|
||||
auto tile = Layout<_1,_0>{};
|
||||
|
||||
test_logical_product(vec, tile);
|
||||
}
|
||||
|
||||
{
|
||||
auto vec = Layout<_1,_0>{};
|
||||
auto tile = Layout<_1,_1>{};
|
||||
|
||||
test_logical_product(vec, tile);
|
||||
}
|
||||
|
||||
{
|
||||
auto vec = Layout<_1,_1>{};
|
||||
auto tile = Layout<_1,_1>{};
|
||||
|
||||
test_logical_product(vec, tile);
|
||||
}
|
||||
|
||||
{
|
||||
auto vec = Layout<_3,_1>{};
|
||||
auto tile = Layout<_4,_0>{};
|
||||
|
||||
test_logical_product(vec, tile);
|
||||
}
|
||||
|
||||
{
|
||||
auto vec = Layout<_3,_0>{};
|
||||
auto tile = Layout<_4,_1>{};
|
||||
|
||||
test_logical_product(vec, tile);
|
||||
}
|
||||
|
||||
{
|
||||
auto vec = Layout<_3,_0>{};
|
||||
auto tile = Layout<_4,_0>{};
|
||||
|
||||
test_logical_product(vec, tile);
|
||||
}
|
||||
|
||||
{
|
||||
auto vec = Layout<_3,_2>{};
|
||||
auto tile = Layout<_4,_1>{};
|
||||
|
||||
test_logical_product(vec, tile);
|
||||
}
|
||||
|
||||
{
|
||||
auto vec = make_layout(Shape<_3>{});
|
||||
auto tile = make_layout(Shape<_2,_4>{});
|
||||
|
||||
test_logical_product(vec, tile);
|
||||
}
|
||||
|
||||
{
|
||||
auto vec = make_layout(Shape<_2,_4>{});
|
||||
auto tile = make_layout(Shape<_3>{});
|
||||
|
||||
test_logical_product(vec, tile);
|
||||
}
|
||||
|
||||
{
|
||||
auto vec = make_layout(Shape<_8,Shape<_2,_2>>{});
|
||||
auto tile = make_layout(Shape<_4>{}, Stride<_2>{});
|
||||
|
||||
test_logical_product(vec, tile);
|
||||
}
|
||||
|
||||
{
|
||||
auto vec = make_layout(Shape<_2,_2>{});
|
||||
auto tile = make_layout(Shape<_3,_3>{}, Stride<_3,_1>{});
|
||||
|
||||
test_logical_product(vec, tile);
|
||||
}
|
||||
|
||||
{
|
||||
auto vec = make_layout(Shape<_3>{}, Stride<_32>{});
|
||||
auto tile = make_layout(Shape<_32>{});
|
||||
|
||||
test_logical_product(vec, tile);
|
||||
}
|
||||
|
||||
{
|
||||
auto vec = make_layout(Shape<_3>{}, Stride<_2>{});
|
||||
auto tile = make_layout(Shape<_4>{});
|
||||
|
||||
test_logical_product(vec, tile);
|
||||
}
|
||||
|
||||
{
|
||||
auto vec = make_layout(Shape<_3>{}, Stride<_32>{});
|
||||
auto tile = make_layout(Shape<_128>{});
|
||||
|
||||
test_logical_product(vec, tile);
|
||||
}
|
||||
|
||||
{
|
||||
auto vec = make_layout(Shape<_3>{}, Stride<_32>{});
|
||||
auto tile = make_layout(Shape<_8,_8>{});
|
||||
|
||||
test_logical_product(vec, tile);
|
||||
}
|
||||
|
||||
{
|
||||
auto vec = make_layout(Shape<_3>{}, Stride<_32>{});
|
||||
auto tile = make_layout(Shape<_8,_8>{}, Stride<_8,_1>{});
|
||||
|
||||
test_logical_product(vec, tile);
|
||||
}
|
||||
|
||||
{
|
||||
auto vec = make_layout(Shape<Shape<_4,_2>>{}, Stride<Stride<_1,_16>>{});
|
||||
auto tile = make_layout(Shape<_4,_4>{});
|
||||
|
||||
test_logical_product(vec, tile);
|
||||
}
|
||||
|
||||
{
|
||||
auto vec = make_layout(Shape<Shape<_4,_2>>{}, Stride<Stride<_1,_16>>{});
|
||||
auto tile = make_layout(Shape<_4,_2>{}, Stride<_2,_1>{});
|
||||
|
||||
test_logical_product(vec, tile);
|
||||
}
|
||||
|
||||
{
|
||||
auto vec = make_layout(Shape<Shape<_2,_2>,Shape<_2, _2>>{},
|
||||
Stride<Stride<_1,_4>,Stride<_8,_32>>{});
|
||||
auto tile = make_layout(Shape<_2,_2>{}, Stride<_1,_2>{});
|
||||
|
||||
test_logical_product(vec, tile);
|
||||
}
|
||||
|
||||
{
|
||||
auto vec = make_layout(Shape<Shape<_2,_2>,Shape<_2, _2>>{},
|
||||
Stride<Stride<_1,_4>,Stride<_8,_32>>{});
|
||||
auto tile = make_layout(Shape<_2,_2>{},
|
||||
Stride<_2,_1>{});
|
||||
|
||||
test_logical_product(vec, tile);
|
||||
}
|
||||
|
||||
{
|
||||
auto vec = make_layout(Shape <Shape <_4,_6>>{},
|
||||
Stride<Stride<_1,_6>>{});
|
||||
auto tile = make_layout(Shape <_3>{},
|
||||
Stride<_1>{});
|
||||
|
||||
test_logical_product(vec, tile);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017 - 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
#include "cutlass_unit_test.h"
|
||||
|
||||
#include <cutlass/trace.h>
|
||||
#include <cute/swizzle.hpp>
|
||||
|
||||
TEST(CuTe_core, MixedBits) {
|
||||
using namespace cute;
|
||||
|
||||
auto uzero = cute::integral_constant<uint32_t, 0>{};
|
||||
|
||||
for_each(make_integer_sequence<uint32_t, 8>{}, [&](auto S0) {
|
||||
for_each(make_integer_sequence<uint32_t, 8>{}, [&](auto F0) {
|
||||
for_each(make_integer_sequence<uint32_t, 8>{}, [&](auto S1) {
|
||||
for_each(make_integer_sequence<uint32_t, 8>{}, [&](auto F1) {
|
||||
if constexpr (decltype(S0 == uzero || S1 == uzero)::value) {
|
||||
return;
|
||||
} else if constexpr (decltype((S0 & F0) != uzero || (S1 & F1) != uzero)::value) {
|
||||
return;
|
||||
} else {
|
||||
for (uint32_t d0 = 0; d0 < 8; ++d0) {
|
||||
if ((d0 & F0) != d0) { continue; } // Skip repeats
|
||||
for (uint32_t d1 = 0; d1 < 8; ++d1) {
|
||||
if ((d1 & F1) != d1) { continue; } // Skip repeats
|
||||
auto m0 = make_mixed_bits(S0, d0, F0);
|
||||
auto m1 = make_mixed_bits(S1, d1, F1);
|
||||
//print(m0); print(" & "); print(m1); print(" = "); print(m0 & m1); print("\n");
|
||||
EXPECT_EQ(to_integral(m0 & m1), to_integral(m0) & to_integral(m1));
|
||||
//print(m0); print(" | "); print(m1); print(" = "); print(m0 | m1); print("\n");
|
||||
EXPECT_EQ(to_integral(m0 | m1), to_integral(m0) | to_integral(m1));
|
||||
//print(m0); print(" ^ "); print(m1); print(" = "); print(m0 ^ m1); print("\n");
|
||||
EXPECT_EQ(to_integral(m0 ^ m1), to_integral(m0) ^ to_integral(m1));
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017 - 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
#include "cutlass_unit_test.h"
|
||||
|
||||
#include <cutlass/trace.h>
|
||||
#include <cute/tensor.hpp>
|
||||
#include <cute/numeric/complex.hpp>
|
||||
|
||||
TEST(CuTe_core, Transform) {
|
||||
using namespace cute;
|
||||
complex<float> array[4] = {{0,0}, {1,0}, {0,1}, {1,1}};
|
||||
complex<float> correct[4] = {{0,0}, {1,0}, {0,-1}, {1,-1}};
|
||||
auto tensor = make_tensor(static_cast<complex<float>*>(array), make_layout(make_shape(4)));
|
||||
conjugate conj;
|
||||
transform(tensor, conj);
|
||||
for (int i = 0; i < 4; ++i)
|
||||
{
|
||||
EXPECT_EQ(tensor(i), correct[i]);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,266 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017 - 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
#include "cutlass_unit_test.h"
|
||||
|
||||
#include <cutlass/trace.h>
|
||||
#include <cute/tensor.hpp>
|
||||
|
||||
TEST(CuTe_core, Tuple)
|
||||
{
|
||||
using namespace cute;
|
||||
|
||||
CUTLASS_TRACE_HOST("-------------------------------");
|
||||
CUTLASS_TRACE_HOST("SIMPLE STATIC AND DYNAMIC TUPLES");
|
||||
CUTLASS_TRACE_HOST("-------------------------------");
|
||||
|
||||
using tuple_2d_s_type = tuple<_8, _4>; // (8,4)
|
||||
using tuple_3d_s_type = tuple<_8, _4, _2>; // (8,4,2)
|
||||
using tuple_3h_s_type = tuple<tuple<_1, _2>, _8, _2>; // ((1,2),8,2)
|
||||
|
||||
using tuple_2d_d_type = tuple<int, int>; // (8,4)
|
||||
using tuple_3d_d_type = tuple<int, int, int>; // (8,4,2)
|
||||
using tuple_3h_d_type = tuple<tuple<int, int>, int, int>; // ((1,2),8,2)
|
||||
|
||||
using tuple_2d_m_type = tuple<_8, int>; // (8,4)
|
||||
using tuple_3d_m_type = tuple<int, int, _2>; // (8,4,2)
|
||||
using tuple_3h_m_type = tuple<tuple<int, _2>, int, int>; // ((1,2),8,2)
|
||||
|
||||
tuple_2d_s_type tuple_2d_s;
|
||||
tuple_3d_s_type tuple_3d_s;
|
||||
tuple_3h_s_type tuple_3h_s;
|
||||
|
||||
tuple_2d_d_type tuple_2d_d(8,4);
|
||||
tuple_3d_d_type tuple_3d_d(8,4,2);
|
||||
tuple_3h_d_type tuple_3h_d(tuple<int,int>(1,2),8,2);
|
||||
|
||||
tuple_2d_m_type tuple_2d_m(_8{}, 4);
|
||||
tuple_3d_m_type tuple_3d_m(8,4,_2{});
|
||||
tuple_3h_m_type tuple_3h_m(tuple<int,_2>(1,_2{}),8,2);
|
||||
|
||||
CUTLASS_TRACE_HOST(tuple_2d_s << (is_static<tuple_2d_s_type>::value ? " Static " : " Dynamic ")
|
||||
<< "sizeof = " << sizeof(tuple_2d_s_type));
|
||||
ASSERT_TRUE(is_static<tuple_2d_s_type>::value == true);
|
||||
ASSERT_TRUE(sizeof(tuple_2d_s_type) == 1);
|
||||
ASSERT_TRUE(std::is_empty<tuple_2d_s_type>::value);
|
||||
|
||||
CUTLASS_TRACE_HOST(tuple_3d_s << (is_static<tuple_3d_s_type>::value ? " Static " : " Dynamic ")
|
||||
<< "sizeof = " << sizeof(tuple_3d_s_type));
|
||||
ASSERT_TRUE(is_static<tuple_3d_s_type>::value == true);
|
||||
ASSERT_TRUE(sizeof(tuple_3d_s_type) == 1);
|
||||
ASSERT_TRUE(std::is_empty<tuple_3d_s_type>::value);
|
||||
|
||||
CUTLASS_TRACE_HOST(tuple_3h_s << (is_static<tuple_3h_s_type>::value ? " Static " : " Dynamic ")
|
||||
<< "sizeof = " << sizeof(tuple_3h_s_type));
|
||||
ASSERT_TRUE(is_static<tuple_3h_s_type>::value == true);
|
||||
ASSERT_TRUE(sizeof(tuple_3h_s_type) == 1);
|
||||
ASSERT_TRUE(std::is_empty<tuple_3h_s_type>::value);
|
||||
|
||||
CUTLASS_TRACE_HOST(tuple_2d_d << (is_static<tuple_2d_d_type>::value ? " Static " : " Dynamic ")
|
||||
<< "sizeof = " << sizeof(tuple_2d_d_type));
|
||||
ASSERT_TRUE(is_static<tuple_2d_d_type>::value == false);
|
||||
ASSERT_TRUE(sizeof(tuple_2d_d_type) == 8);
|
||||
ASSERT_TRUE(!std::is_empty<tuple_2d_d_type>::value);
|
||||
|
||||
CUTLASS_TRACE_HOST(tuple_3d_d << (is_static<tuple_3d_d_type>::value ? " Static " : " Dynamic ")
|
||||
<< "sizeof = " << sizeof(tuple_3d_d_type));
|
||||
ASSERT_TRUE(is_static<tuple_3d_d_type>::value == false);
|
||||
ASSERT_TRUE(sizeof(tuple_3d_d_type) == 12);
|
||||
ASSERT_TRUE(!std::is_empty<tuple_3d_d_type>::value);
|
||||
|
||||
CUTLASS_TRACE_HOST(tuple_3h_d << (is_static<tuple_3h_d_type>::value ? " Static " : " Dynamic ")
|
||||
<< "sizeof = " << sizeof(tuple_3h_d_type));
|
||||
ASSERT_TRUE(is_static<tuple_3h_d_type>::value == false);
|
||||
ASSERT_TRUE(sizeof(tuple_3h_d_type) == 16);
|
||||
ASSERT_TRUE(!std::is_empty<tuple_3h_d_type>::value);
|
||||
|
||||
CUTLASS_TRACE_HOST(tuple_2d_m << (is_static<tuple_2d_m_type>::value ? " Static " : " Dynamic ")
|
||||
<< "sizeof = " << sizeof(tuple_2d_m_type));
|
||||
ASSERT_TRUE(is_static<tuple_2d_m_type>::value == false);
|
||||
ASSERT_TRUE(sizeof(tuple_2d_m_type) == 4);
|
||||
ASSERT_TRUE(!std::is_empty<tuple_2d_m_type>::value);
|
||||
|
||||
CUTLASS_TRACE_HOST(tuple_3d_m << (is_static<tuple_3d_m_type>::value ? " Static " : " Dynamic ")
|
||||
<< "sizeof = " << sizeof(tuple_3d_m_type));
|
||||
ASSERT_TRUE(is_static<tuple_3d_m_type>::value == false);
|
||||
ASSERT_TRUE(sizeof(tuple_3d_m_type) == 8);
|
||||
ASSERT_TRUE(!std::is_empty<tuple_3d_m_type>::value);
|
||||
|
||||
CUTLASS_TRACE_HOST(tuple_3h_m << (is_static<tuple_3h_m_type>::value ? " Static " : " Dynamic ")
|
||||
<< "sizeof = " << sizeof(tuple_3h_m_type));
|
||||
ASSERT_TRUE(is_static<tuple_3h_m_type>::value == false);
|
||||
ASSERT_TRUE(sizeof(tuple_3h_m_type) == 12);
|
||||
ASSERT_TRUE(!std::is_empty<tuple_3h_m_type>::value);
|
||||
|
||||
CUTLASS_TRACE_HOST("-------------------------------");
|
||||
CUTLASS_TRACE_HOST("SIMPLE TUPLE OPS");
|
||||
CUTLASS_TRACE_HOST("-------------------------------");
|
||||
|
||||
CUTLASS_TRACE_HOST("product(" << tuple_2d_s << ") => " << product(tuple_2d_s));
|
||||
CUTE_STATIC_ASSERT_V(product(tuple_2d_s) == _32{});
|
||||
CUTLASS_TRACE_HOST("product(" << tuple_3d_s << ") => " << product(tuple_3d_s));
|
||||
CUTE_STATIC_ASSERT_V(product(tuple_3d_s) == _64{});
|
||||
CUTLASS_TRACE_HOST("product(" << tuple_3h_s << ") => " << product(tuple_3h_s));
|
||||
CUTE_STATIC_ASSERT_V(product(tuple_3h_s) == _32{});
|
||||
|
||||
CUTLASS_TRACE_HOST("product(" << tuple_2d_d << ") => " << product(tuple_2d_d));
|
||||
ASSERT_TRUE(product(tuple_2d_d) == 32);
|
||||
CUTLASS_TRACE_HOST("product(" << tuple_3d_d << ") => " << product(tuple_3d_d));
|
||||
ASSERT_TRUE(product(tuple_3d_d) == 64);
|
||||
CUTLASS_TRACE_HOST("product(" << tuple_3h_d << ") => " << product(tuple_3h_d));
|
||||
ASSERT_TRUE(product(tuple_3h_d) == 32);
|
||||
|
||||
CUTLASS_TRACE_HOST("product(" << tuple_2d_m << ") => " << product(tuple_2d_m));
|
||||
ASSERT_TRUE(product(tuple_2d_m) == 32);
|
||||
CUTLASS_TRACE_HOST("product(" << tuple_3d_m << ") => " << product(tuple_3d_m));
|
||||
ASSERT_TRUE(product(tuple_3d_m) == 64);
|
||||
CUTLASS_TRACE_HOST("product(" << tuple_3h_m << ") => " << product(tuple_3h_m));
|
||||
ASSERT_TRUE(product(tuple_3h_m) == 32);
|
||||
|
||||
CUTLASS_TRACE_HOST("max(" << tuple_2d_s << ") => " << max(tuple_2d_s));
|
||||
CUTE_STATIC_ASSERT_V(max(tuple_2d_s) == _8{});
|
||||
CUTLASS_TRACE_HOST("max(" << tuple_3d_s << ") => " << max(tuple_3d_s));
|
||||
CUTE_STATIC_ASSERT_V(max(tuple_3d_s) == _8{});
|
||||
CUTLASS_TRACE_HOST("max(" << tuple_3h_s << ") => " << max(tuple_3h_s));
|
||||
CUTE_STATIC_ASSERT_V(max(tuple_3h_s) == _8{});
|
||||
|
||||
CUTLASS_TRACE_HOST("max(" << tuple_2d_d << ") => " << max(tuple_2d_d));
|
||||
ASSERT_TRUE(max(tuple_2d_d) == 8);
|
||||
CUTLASS_TRACE_HOST("max(" << tuple_3d_d << ") => " << max(tuple_3d_d));
|
||||
ASSERT_TRUE(max(tuple_3d_d) == 8);
|
||||
CUTLASS_TRACE_HOST("max(" << tuple_3h_d << ") => " << max(tuple_3h_d));
|
||||
ASSERT_TRUE(max(tuple_3h_d) == 8);
|
||||
|
||||
CUTLASS_TRACE_HOST("max(" << tuple_2d_m << ") => " << max(tuple_2d_m));
|
||||
ASSERT_TRUE(max(tuple_2d_m) == 8);
|
||||
CUTLASS_TRACE_HOST("max(" << tuple_3d_m << ") => " << max(tuple_3d_m));
|
||||
ASSERT_TRUE(max(tuple_3d_m) == 8);
|
||||
CUTLASS_TRACE_HOST("max(" << tuple_3h_m << ") => " << max(tuple_3h_m));
|
||||
ASSERT_TRUE(max(tuple_3h_m) == 8);
|
||||
|
||||
// 2d s|d|m
|
||||
CUTLASS_TRACE_HOST("inner_product(" << tuple_2d_s << ", " << tuple_2d_s << ") => "
|
||||
<< inner_product(tuple_2d_s, tuple_2d_s));
|
||||
CUTE_STATIC_ASSERT_V(inner_product(tuple_2d_s, tuple_2d_s) == Int<80>{});
|
||||
CUTLASS_TRACE_HOST("inner_product(" << tuple_2d_d << ", " << tuple_2d_d << ") => "
|
||||
<< inner_product(tuple_2d_d, tuple_2d_d));
|
||||
ASSERT_TRUE(inner_product(tuple_2d_d, tuple_2d_d) == 80);
|
||||
CUTLASS_TRACE_HOST("inner_product(" << tuple_2d_m << ", " << tuple_2d_m << ") => "
|
||||
<< inner_product(tuple_2d_m, tuple_2d_m));
|
||||
ASSERT_TRUE(inner_product(tuple_2d_m, tuple_2d_m) == 80);
|
||||
|
||||
// 3d s|d|m
|
||||
CUTLASS_TRACE_HOST("inner_product(" << tuple_3d_s << ", " << tuple_3d_s << ") => "
|
||||
<< inner_product(tuple_3d_s, tuple_3d_s));
|
||||
CUTE_STATIC_ASSERT_V(inner_product(tuple_3d_s, tuple_3d_s) == Int<84>{});
|
||||
CUTLASS_TRACE_HOST("inner_product(" << tuple_3d_d << ", " << tuple_3d_d << ") => "
|
||||
<< inner_product(tuple_3d_d, tuple_3d_d));
|
||||
ASSERT_TRUE(inner_product(tuple_3d_d, tuple_3d_d) == 84);
|
||||
CUTLASS_TRACE_HOST("inner_product(" << tuple_3d_m << ", " << tuple_3d_m << ") => "
|
||||
<< inner_product(tuple_3d_m, tuple_3d_m));
|
||||
ASSERT_TRUE(inner_product(tuple_3d_m, tuple_3d_m) == 84);
|
||||
|
||||
// 3h s|d|m
|
||||
CUTLASS_TRACE_HOST("inner_product(" << tuple_3h_s << ", " << tuple_3h_s << ") => "
|
||||
<< inner_product(tuple_3h_s, tuple_3h_s));
|
||||
CUTE_STATIC_ASSERT_V(inner_product(tuple_3h_s, tuple_3h_s) == Int<73>{});
|
||||
CUTLASS_TRACE_HOST("inner_product(" << tuple_3h_d << ", " << tuple_3h_d << ") => "
|
||||
<< inner_product(tuple_3h_d, tuple_3h_d));
|
||||
ASSERT_TRUE(inner_product(tuple_3h_d, tuple_3h_d) == 73);
|
||||
CUTLASS_TRACE_HOST("inner_product(" << tuple_3h_m << ", " << tuple_3h_m << ") => "
|
||||
<< inner_product(tuple_3h_m, tuple_3h_m));
|
||||
ASSERT_TRUE(inner_product(tuple_3h_m, tuple_3h_m) == 73);
|
||||
|
||||
CUTLASS_TRACE_HOST("col_major(" << tuple_2d_s << ") => " << compact_col_major(tuple_2d_s));
|
||||
CUTE_STATIC_ASSERT_V((compact_col_major(tuple_2d_s) == make_tuple(_1{},_8{})));
|
||||
CUTLASS_TRACE_HOST("col_major(" << tuple_3d_s << ") => " << compact_col_major(tuple_3d_s));
|
||||
CUTE_STATIC_ASSERT_V((compact_col_major(tuple_3d_s) == make_tuple(_1{},_8{},_32{})));
|
||||
CUTLASS_TRACE_HOST("col_major(" << tuple_3h_s << ") => " << compact_col_major(tuple_3h_s));
|
||||
CUTE_STATIC_ASSERT_V((compact_col_major(tuple_3h_s) == make_tuple(make_tuple(_0{},_1{}),_2{},_16{})));
|
||||
|
||||
CUTLASS_TRACE_HOST("col_major(" << tuple_2d_d << ") => " << compact_col_major(tuple_2d_d));
|
||||
ASSERT_TRUE((compact_col_major(tuple_2d_d) == make_tuple(_1{},8)));
|
||||
CUTLASS_TRACE_HOST("col_major(" << tuple_3d_d << ") => " << compact_col_major(tuple_3d_d));
|
||||
ASSERT_TRUE((compact_col_major(tuple_3d_d) == make_tuple(_1{},8,32)));
|
||||
CUTLASS_TRACE_HOST("col_major(" << tuple_3h_d << ") => " << compact_col_major(tuple_3h_d));
|
||||
ASSERT_TRUE((compact_col_major(tuple_3h_d) == make_tuple(make_tuple(_1{},1),2,16)));
|
||||
|
||||
CUTLASS_TRACE_HOST("col_major(" << tuple_2d_m << ") => " << compact_col_major(tuple_2d_m));
|
||||
ASSERT_TRUE((compact_col_major(tuple_2d_m) == make_tuple(_1{},_8{})));
|
||||
CUTLASS_TRACE_HOST("col_major(" << tuple_3d_m << ") => " << compact_col_major(tuple_3d_m));
|
||||
ASSERT_TRUE((compact_col_major(tuple_3d_m) == make_tuple(_1{},8,32)));
|
||||
CUTLASS_TRACE_HOST("col_major(" << tuple_3h_m << ") => " << compact_col_major(tuple_3h_m));
|
||||
ASSERT_TRUE((compact_col_major(tuple_3h_m) == make_tuple(make_tuple(_1{},1),2,16)));
|
||||
|
||||
CUTLASS_TRACE_HOST("-------------------------------");
|
||||
CUTLASS_TRACE_HOST("SLICING TUPLES");
|
||||
CUTLASS_TRACE_HOST("-------------------------------");
|
||||
|
||||
{
|
||||
auto a = Coord<_2,_3,_4,Coord<_5,_6>>{};
|
||||
|
||||
CUTLASS_TRACE_HOST("a = " << a);
|
||||
|
||||
CUTLASS_TRACE_HOST("a(1) = " << slice(1, a));
|
||||
|
||||
CUTLASS_TRACE_HOST("a(_) = " << slice(_, a));
|
||||
|
||||
CUTLASS_TRACE_HOST("a(_,1,_,_) = " << slice(make_coord(_,1,_,_), a));
|
||||
|
||||
CUTLASS_TRACE_HOST("a(_,1,_,(_,_)) = " << slice(make_coord(_,1,_,make_coord(_,_)), a));
|
||||
|
||||
CUTLASS_TRACE_HOST("a(_,1,_,(_,2)) = " << slice(make_coord(_,1,_,make_coord(_,2)), a));
|
||||
|
||||
CUTLASS_TRACE_HOST("a(_,1,_,(1,2)) = " << slice(make_coord(_,1,_,make_coord(1,2)), a));
|
||||
}
|
||||
|
||||
CUTLASS_TRACE_HOST("-------------------------------");
|
||||
CUTLASS_TRACE_HOST("DICING TUPLES");
|
||||
CUTLASS_TRACE_HOST("-------------------------------");
|
||||
|
||||
{
|
||||
auto a = Coord<_2,_3,_4,Coord<_5,_6>>{};
|
||||
|
||||
CUTLASS_TRACE_HOST("a = " << a);
|
||||
|
||||
CUTLASS_TRACE_HOST("a(1) = " << dice(1, a));
|
||||
|
||||
CUTLASS_TRACE_HOST("a(_) = " << dice(_, a));
|
||||
|
||||
CUTLASS_TRACE_HOST("a(_,1,_,_) = " << dice(make_coord(_,1,_,_), a));
|
||||
|
||||
CUTLASS_TRACE_HOST("a(_,1,_,(_,_)) = " << dice(make_coord(_,1,_,make_coord(_,_)), a));
|
||||
|
||||
CUTLASS_TRACE_HOST("a(_,1,_,(_,2)) = " << dice(make_coord(_,1,_,make_coord(_,2)), a));
|
||||
|
||||
CUTLASS_TRACE_HOST("a(_,1,_,(1,2)) = " << dice(make_coord(_,1,_,make_coord(1,2)), a));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
# Copyright (c) 2023 - 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: BSD-3-Clause
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions are met:
|
||||
#
|
||||
# 1. Redistributions of source code must retain the above copyright notice, this
|
||||
# list of conditions and the following disclaimer.
|
||||
#
|
||||
# 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
# this list of conditions and the following disclaimer in the documentation
|
||||
# and/or other materials provided with the distribution.
|
||||
#
|
||||
# 3. Neither the name of the copyright holder nor the names of its
|
||||
# contributors may be used to endorse or promote products derived from
|
||||
# this software without specific prior written permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
add_custom_target(
|
||||
cutlass_test_unit_cute_hopper
|
||||
DEPENDS
|
||||
cutlass_test_unit_cute_hopper_stsm
|
||||
cutlass_test_unit_cute_hopper_tma_load
|
||||
cutlass_test_unit_cute_hopper_tma_store
|
||||
)
|
||||
|
||||
add_custom_target(
|
||||
test_unit_cute_hopper
|
||||
DEPENDS
|
||||
test_unit_cute_hopper_stsm
|
||||
test_unit_cute_hopper_tma_load
|
||||
test_unit_cute_hopper_tma_store
|
||||
)
|
||||
|
||||
cutlass_test_unit_add_executable(
|
||||
cutlass_test_unit_cute_hopper_stsm
|
||||
stsm.cu
|
||||
)
|
||||
|
||||
cutlass_test_unit_add_executable(
|
||||
cutlass_test_unit_cute_hopper_tma_load
|
||||
tma_load.cu
|
||||
)
|
||||
|
||||
cutlass_test_unit_add_executable(
|
||||
cutlass_test_unit_cute_hopper_tma_store
|
||||
tma_store.cu
|
||||
)
|
||||
@@ -0,0 +1,426 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017 - 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
#include "cutlass_unit_test.h"
|
||||
|
||||
#include <iostream>
|
||||
|
||||
#include <thrust/host_vector.h>
|
||||
#include <thrust/device_vector.h>
|
||||
|
||||
#include <cute/tensor.hpp>
|
||||
#include <cute/arch/copy_sm90.hpp>
|
||||
|
||||
using namespace cute;
|
||||
|
||||
template<class T>
|
||||
__global__ void
|
||||
stsm_test_device(uint16_t* g_in, uint16_t* g_out)
|
||||
{
|
||||
constexpr int count = sizeof(T) / 4;
|
||||
int tid = threadIdx.x;
|
||||
int stride = blockDim.x;
|
||||
|
||||
// load input gmem -> rmem
|
||||
uint32_t reg[count];
|
||||
for (int i = 0; i < (sizeof(T) / 4); i++) {
|
||||
reg[i] = reinterpret_cast<uint32_t*>(g_in)[tid + (stride * i)];
|
||||
}
|
||||
|
||||
__shared__ uint32_t smem[32 * count];
|
||||
|
||||
// load rmem -> smem using STSM
|
||||
uint128_t* smem_ptr = reinterpret_cast<uint128_t*>(smem) + tid;
|
||||
T* rmem_ptr = reinterpret_cast<T*>(reg);
|
||||
cute::copy_stsm(rmem_ptr, smem_ptr);
|
||||
|
||||
__syncthreads();
|
||||
|
||||
// store output smem -> gmem
|
||||
for (int i = 0; i < (sizeof(T) / 4); i++) {
|
||||
reinterpret_cast<uint32_t*>(g_out)[tid + (stride * i)] = smem[tid + (stride * i)];
|
||||
}
|
||||
}
|
||||
|
||||
template <class TiledCopy, class SmemLayout>
|
||||
__global__ void
|
||||
stsm_test_device_cute(uint16_t* g_in, uint16_t* g_out,
|
||||
TiledCopy tiled_copy, SmemLayout smem_layout)
|
||||
{
|
||||
using namespace cute;
|
||||
|
||||
__shared__ uint16_t smem[size(smem_layout)];
|
||||
|
||||
Tensor t_g_in = make_tensor(make_gmem_ptr(g_in), smem_layout);
|
||||
Tensor t_g_out = make_tensor(make_gmem_ptr(g_out), smem_layout);
|
||||
Tensor t_smem = make_tensor(make_smem_ptr(smem), smem_layout);
|
||||
|
||||
int tid = threadIdx.x;
|
||||
|
||||
auto thr_copy = tiled_copy.get_thread_slice(tid);
|
||||
|
||||
Tensor tXgX = thr_copy.partition_S(t_g_in); // (V,M,N)
|
||||
Tensor tXsX = thr_copy.partition_D(t_smem); // (V,M,N)
|
||||
|
||||
Tensor tXrX = make_tensor<uint16_t>(shape(tXgX)); // (V,M,N)
|
||||
clear(tXrX); // Just to make sure
|
||||
|
||||
/*
|
||||
if (thread0()) {
|
||||
print("tXsX: " ); print(tXsX.layout()); print("\n");
|
||||
print("tXgX: " ); print(tXgX.layout()); print("\n");
|
||||
print("tXrX: " ); print(tXrX.layout()); print("\n");
|
||||
}
|
||||
*/
|
||||
|
||||
// Load input gmem -> rmem
|
||||
copy(tXgX, tXrX);
|
||||
|
||||
// Copy rmem -> smem via tiled_copy (STSM, STS)
|
||||
copy(tiled_copy, tXrX, tXsX);
|
||||
|
||||
// Output smem -> gmem
|
||||
for (int i = tid; i < size(t_smem); i += size(tiled_copy)) {
|
||||
t_g_out(i) = t_smem(i);
|
||||
}
|
||||
}
|
||||
|
||||
#if CUDA_12_0_SM90_FEATURES_SUPPORTED
|
||||
TEST(SM90_CuTe_Hopper, Stsm)
|
||||
{
|
||||
constexpr int count = 1024;
|
||||
|
||||
thrust::host_vector<uint16_t> h_in(count);
|
||||
for (int i = 0; i < count; ++i) {
|
||||
h_in[i] = uint16_t(i);
|
||||
}
|
||||
thrust::device_vector<uint16_t> d_in = h_in;
|
||||
|
||||
//
|
||||
// STSM 1x (32b)
|
||||
//
|
||||
|
||||
{
|
||||
thrust::device_vector<uint16_t> d_out(count);
|
||||
stsm_test_device<uint32_t><<<1, 32>>>(
|
||||
thrust::raw_pointer_cast(d_in.data()),
|
||||
thrust::raw_pointer_cast(d_out.data()));
|
||||
thrust::host_vector<uint16_t> h_out = d_out;
|
||||
for (int i = 0; i < 32; ++i) {
|
||||
//printf("%d %d\n", int(h_in[i]), int(h_out[i]));
|
||||
EXPECT_EQ(h_out[i], h_in[i]);
|
||||
}
|
||||
CUTLASS_TRACE_HOST("STSM 1x stsm_test_device SUCCESS\n");
|
||||
}
|
||||
|
||||
//
|
||||
// STSM 2x (64b)
|
||||
//
|
||||
|
||||
{
|
||||
thrust::device_vector<uint16_t> d_out(count);
|
||||
stsm_test_device<uint64_t><<<1, 32>>>(
|
||||
thrust::raw_pointer_cast(d_in.data()),
|
||||
thrust::raw_pointer_cast(d_out.data()));
|
||||
thrust::host_vector<uint16_t> h_out = d_out;
|
||||
for (int i = 0; i < 64; ++i) {
|
||||
//printf("%d %d\n", int(h_in[i]), int(h_out[i]));
|
||||
EXPECT_EQ(h_out[i], h_in[i]);
|
||||
}
|
||||
CUTLASS_TRACE_HOST("STSM 2x stsm_test_device SUCCESS\n");
|
||||
}
|
||||
|
||||
//
|
||||
// STSM 4x (128b)
|
||||
//
|
||||
|
||||
{
|
||||
thrust::device_vector<uint16_t> d_out(count);
|
||||
stsm_test_device<uint128_t><<<1, 32>>>(
|
||||
thrust::raw_pointer_cast(d_in.data()),
|
||||
thrust::raw_pointer_cast(d_out.data()));
|
||||
thrust::host_vector<uint16_t> h_out = d_out;
|
||||
for (int i = 0; i < 128; ++i) {
|
||||
//printf("%d %d\n", int(h_in[i]), int(h_out[i]));
|
||||
EXPECT_EQ(h_out[i], h_in[i]);
|
||||
}
|
||||
CUTLASS_TRACE_HOST("STSM 4x stsm_test_device SUCCESS\n");
|
||||
}
|
||||
|
||||
//
|
||||
// CuTe STSM
|
||||
//
|
||||
|
||||
{
|
||||
thrust::device_vector<uint16_t> d_out(count);
|
||||
|
||||
auto smem_layout = Layout<Shape <_32,Shape <_2, _4>>,
|
||||
Stride< _2,Stride<_1,_64>>>{};
|
||||
auto tiled_copy = make_tiled_copy(Copy_Atom<SM90_U32x1_STSM_N, uint16_t>{},
|
||||
Layout<Shape<_32,_1>>{},
|
||||
Layout<Shape< _1,_8>>{});
|
||||
|
||||
stsm_test_device_cute<<<1, int(size(tiled_copy))>>>(
|
||||
thrust::raw_pointer_cast(d_in.data()),
|
||||
thrust::raw_pointer_cast(d_out.data()),
|
||||
tiled_copy,
|
||||
smem_layout);
|
||||
thrust::host_vector<uint16_t> h_out = d_out;
|
||||
for (int i = 0; i < size(smem_layout); ++i) {
|
||||
//printf("%d %d\n", int(h_in[i]), int(h_out[i]));
|
||||
EXPECT_EQ(h_out[i], h_in[i]);
|
||||
}
|
||||
CUTLASS_TRACE_HOST("CuTe 32x8 interleaved U32x1_STSM_N SUCCESS\n");
|
||||
}
|
||||
|
||||
{
|
||||
thrust::device_vector<uint16_t> d_out(count);
|
||||
|
||||
auto smem_layout = Layout<Shape <_32,Shape <_2, _4>>,
|
||||
Stride< _2,Stride<_1,_64>>>{};
|
||||
auto tiled_copy = make_tiled_copy(Copy_Atom<SM90_U32x2_STSM_N, uint16_t>{},
|
||||
Layout<Shape<_32,_1>>{},
|
||||
Layout<Shape< _1,_8>>{});
|
||||
|
||||
stsm_test_device_cute<<<1, int(size(tiled_copy))>>>(
|
||||
thrust::raw_pointer_cast(d_in.data()),
|
||||
thrust::raw_pointer_cast(d_out.data()),
|
||||
tiled_copy,
|
||||
smem_layout);
|
||||
thrust::host_vector<uint16_t> h_out = d_out;
|
||||
for (int i = 0; i < size(smem_layout); ++i) {
|
||||
//printf("%d %d\n", int(h_in[i]), int(h_out[i]));
|
||||
EXPECT_EQ(h_out[i], h_in[i]);
|
||||
}
|
||||
CUTLASS_TRACE_HOST("CuTe 32x8 interleaved U32x2_STSM_N SUCCESS\n");
|
||||
}
|
||||
|
||||
{
|
||||
thrust::device_vector<uint16_t> d_out(count);
|
||||
|
||||
auto smem_layout = Layout<Shape <_32,Shape <_2, _4>>,
|
||||
Stride< _2,Stride<_1,_64>>>{};
|
||||
auto tiled_copy = make_tiled_copy(Copy_Atom<SM90_U32x4_STSM_N, uint16_t>{},
|
||||
Layout<Shape<_32,_1>>{},
|
||||
Layout<Shape< _1,_8>>{});
|
||||
|
||||
stsm_test_device_cute<<<1, int(size(tiled_copy))>>>(
|
||||
thrust::raw_pointer_cast(d_in.data()),
|
||||
thrust::raw_pointer_cast(d_out.data()),
|
||||
tiled_copy,
|
||||
smem_layout);
|
||||
thrust::host_vector<uint16_t> h_out = d_out;
|
||||
for (int i = 0; i < size(smem_layout); ++i) {
|
||||
//printf("%d %d\n", int(h_in[i]), int(h_out[i]));
|
||||
EXPECT_EQ(h_out[i], h_in[i]);
|
||||
}
|
||||
CUTLASS_TRACE_HOST("CuTe 32x8 interleaved U32x4_STSM_N SUCCESS\n");
|
||||
}
|
||||
|
||||
{
|
||||
thrust::device_vector<uint16_t> d_out(count);
|
||||
|
||||
auto smem_layout = Layout<Shape <_32,Shape <_2, _4>>,
|
||||
Stride< _2,Stride<_1,_64>>>{};
|
||||
auto tiled_copy = make_tiled_copy(Copy_Atom<UniversalCopy<uint16_t>, uint16_t>{},
|
||||
Layout<Shape<_32,_1>>{},
|
||||
Layout<Shape< _1,_8>>{});
|
||||
|
||||
stsm_test_device_cute<<<1, int(size(tiled_copy))>>>(
|
||||
thrust::raw_pointer_cast(d_in.data()),
|
||||
thrust::raw_pointer_cast(d_out.data()),
|
||||
tiled_copy,
|
||||
smem_layout);
|
||||
thrust::host_vector<uint16_t> h_out = d_out;
|
||||
for (int i = 0; i < size(smem_layout); ++i) {
|
||||
//printf("%d %d\n", int(h_in[i]), int(h_out[i]));
|
||||
EXPECT_EQ(h_out[i], h_in[i]);
|
||||
}
|
||||
CUTLASS_TRACE_HOST("CuTe 32x8 interleaved STS.U16 SUCCESS\n");
|
||||
}
|
||||
|
||||
{
|
||||
thrust::device_vector<uint16_t> d_out(count);
|
||||
|
||||
auto smem_layout = Layout<Shape <_32,_32>,
|
||||
Stride< _1,_32>>{};
|
||||
auto tiled_copy = make_tiled_copy(Copy_Atom<SM90_U32x1_STSM_N, uint16_t>{},
|
||||
Layout<Shape<_16,_2>>{},
|
||||
Layout<Shape< _2,_4>>{});
|
||||
|
||||
stsm_test_device_cute<<<1, int(size(tiled_copy))>>>(
|
||||
thrust::raw_pointer_cast(d_in.data()),
|
||||
thrust::raw_pointer_cast(d_out.data()),
|
||||
tiled_copy,
|
||||
smem_layout);
|
||||
thrust::host_vector<uint16_t> h_out = d_out;
|
||||
for (int i = 0; i < size(smem_layout); ++i) {
|
||||
//printf("%d %d\n", int(h_in[i]), int(h_out[i]));
|
||||
EXPECT_EQ(h_out[i], h_in[i]);
|
||||
}
|
||||
CUTLASS_TRACE_HOST("CuTe 32x32 U32x1_STSM_N SUCCESS\n");
|
||||
}
|
||||
|
||||
{
|
||||
thrust::device_vector<uint16_t> d_out(count);
|
||||
|
||||
auto smem_layout = Layout<Shape <_32,_32>,
|
||||
Stride< _1,_32>>{};
|
||||
auto tiled_copy = make_tiled_copy(Copy_Atom<SM90_U32x2_STSM_N, uint16_t>{},
|
||||
Layout<Shape<_16,_2>>{},
|
||||
Layout<Shape< _2,_4>>{});
|
||||
|
||||
stsm_test_device_cute<<<1, int(size(tiled_copy))>>>(
|
||||
thrust::raw_pointer_cast(d_in.data()),
|
||||
thrust::raw_pointer_cast(d_out.data()),
|
||||
tiled_copy,
|
||||
smem_layout);
|
||||
thrust::host_vector<uint16_t> h_out = d_out;
|
||||
for (int i = 0; i < size(smem_layout); ++i) {
|
||||
//printf("%d %d\n", int(h_in[i]), int(h_out[i]));
|
||||
EXPECT_EQ(h_out[i], h_in[i]);
|
||||
}
|
||||
CUTLASS_TRACE_HOST("CuTe 32x32 U32x2_STSM_N SUCCESS\n");
|
||||
}
|
||||
|
||||
{
|
||||
thrust::device_vector<uint16_t> d_out(count);
|
||||
|
||||
auto smem_layout = Layout<Shape <_32,_32>,
|
||||
Stride< _1,_32>>{};
|
||||
auto tiled_copy = make_tiled_copy(Copy_Atom<SM90_U32x4_STSM_N, uint16_t>{},
|
||||
Layout<Shape<_16,_2>>{},
|
||||
Layout<Shape< _2,_4>>{});
|
||||
|
||||
stsm_test_device_cute<<<1, int(size(tiled_copy))>>>(
|
||||
thrust::raw_pointer_cast(d_in.data()),
|
||||
thrust::raw_pointer_cast(d_out.data()),
|
||||
tiled_copy,
|
||||
smem_layout);
|
||||
thrust::host_vector<uint16_t> h_out = d_out;
|
||||
for (int i = 0; i < size(smem_layout); ++i) {
|
||||
//printf("%d %d\n", int(h_in[i]), int(h_out[i]));
|
||||
EXPECT_EQ(h_out[i], h_in[i]);
|
||||
}
|
||||
CUTLASS_TRACE_HOST("CuTe 32x32 U32x4_STSM_N SUCCESS\n");
|
||||
}
|
||||
|
||||
{
|
||||
thrust::device_vector<uint16_t> d_out(count);
|
||||
|
||||
auto smem_layout = Layout<Shape <_32,_32>,
|
||||
Stride< _1,_32>>{};
|
||||
auto tiled_copy = make_tiled_copy(Copy_Atom<UniversalCopy<uint16_t>, uint16_t>{},
|
||||
Layout<Shape<_16,_2>>{},
|
||||
Layout<Shape< _2,_4>>{});
|
||||
|
||||
stsm_test_device_cute<<<1, int(size(tiled_copy))>>>(
|
||||
thrust::raw_pointer_cast(d_in.data()),
|
||||
thrust::raw_pointer_cast(d_out.data()),
|
||||
tiled_copy,
|
||||
smem_layout);
|
||||
thrust::host_vector<uint16_t> h_out = d_out;
|
||||
for (int i = 0; i < size(smem_layout); ++i) {
|
||||
//printf("%d %d\n", int(h_in[i]), int(h_out[i]));
|
||||
EXPECT_EQ(h_out[i], h_in[i]);
|
||||
}
|
||||
CUTLASS_TRACE_HOST("CuTe 32x32 STS.U16 SUCCESS\n");
|
||||
}
|
||||
|
||||
{
|
||||
thrust::device_vector<uint16_t> d_out(count);
|
||||
|
||||
auto smem_layout = Layout<Shape <_32,_32>,
|
||||
Stride<_32, _1>>{};
|
||||
auto tiled_copy = make_tiled_copy(Copy_Atom<SM90_U16x2_STSM_T, uint16_t>{},
|
||||
Layout<Shape<_4,_8>>{},
|
||||
Layout<Shape<_2,_1>>{});
|
||||
|
||||
stsm_test_device_cute<<<1, int(size(tiled_copy))>>>(
|
||||
thrust::raw_pointer_cast(d_in.data()),
|
||||
thrust::raw_pointer_cast(d_out.data()),
|
||||
tiled_copy,
|
||||
smem_layout);
|
||||
thrust::host_vector<uint16_t> h_out = d_out;
|
||||
for (int i = 0; i < size(smem_layout); ++i) {
|
||||
//printf("%d %d\n", int(h_in[i]), int(h_out[i]));
|
||||
EXPECT_EQ(h_out[i], h_in[i]);
|
||||
}
|
||||
CUTLASS_TRACE_HOST("CuTe 32x32 U16x2_STSM_T SUCCESS\n");
|
||||
}
|
||||
|
||||
{
|
||||
thrust::device_vector<uint16_t> d_out(count);
|
||||
|
||||
auto smem_layout = Layout<Shape <_32,_32>,
|
||||
Stride<_32, _1>>{};
|
||||
auto tiled_copy = make_tiled_copy(Copy_Atom<SM90_U16x4_STSM_T, uint16_t>{},
|
||||
Layout<Shape<_4,_8>>{},
|
||||
Layout<Shape<_4,_1>>{});
|
||||
|
||||
stsm_test_device_cute<<<1, int(size(tiled_copy))>>>(
|
||||
thrust::raw_pointer_cast(d_in.data()),
|
||||
thrust::raw_pointer_cast(d_out.data()),
|
||||
tiled_copy,
|
||||
smem_layout);
|
||||
thrust::host_vector<uint16_t> h_out = d_out;
|
||||
for (int i = 0; i < size(smem_layout); ++i) {
|
||||
//printf("%d %d\n", int(h_in[i]), int(h_out[i]));
|
||||
EXPECT_EQ(h_out[i], h_in[i]);
|
||||
}
|
||||
CUTLASS_TRACE_HOST("CuTe 32x32 U16x4_STSM_T SUCCESS\n");
|
||||
}
|
||||
|
||||
{
|
||||
thrust::device_vector<uint16_t> d_out(count);
|
||||
|
||||
auto smem_layout = Layout<Shape <_32,_32>,
|
||||
Stride<_32, _1>>{};
|
||||
auto tiled_copy = make_tiled_copy(Copy_Atom<SM90_U16x8_STSM_T, uint16_t>{},
|
||||
Layout<Shape<_4,_8>>{},
|
||||
Layout<Shape<_8,_1>>{});
|
||||
|
||||
stsm_test_device_cute<<<1, int(size(tiled_copy))>>>(
|
||||
thrust::raw_pointer_cast(d_in.data()),
|
||||
thrust::raw_pointer_cast(d_out.data()),
|
||||
tiled_copy,
|
||||
smem_layout);
|
||||
thrust::host_vector<uint16_t> h_out = d_out;
|
||||
for (int i = 0; i < size(smem_layout); ++i) {
|
||||
//printf("%d %d\n", int(h_in[i]), int(h_out[i]));
|
||||
EXPECT_EQ(h_out[i], h_in[i]);
|
||||
}
|
||||
CUTLASS_TRACE_HOST("CuTe 32x32 U16x8_STSM_T SUCCESS\n");
|
||||
}
|
||||
|
||||
CUTLASS_TRACE_HOST("PASS");
|
||||
}
|
||||
#endif
|
||||
@@ -0,0 +1,495 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017 - 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
#include "cutlass_unit_test.h"
|
||||
|
||||
#include <iostream>
|
||||
|
||||
#include <thrust/host_vector.h>
|
||||
#include <thrust/device_vector.h>
|
||||
|
||||
#include <cute/tensor.hpp>
|
||||
|
||||
using namespace cute;
|
||||
|
||||
template <class ElementType, class SmemLayout>
|
||||
struct SharedStorage
|
||||
{
|
||||
cute::array_aligned<ElementType, cute::cosize_v<SmemLayout>> smem;
|
||||
cute::uint64_t tma_load_mbar[1];
|
||||
};
|
||||
|
||||
// __grid_constant__ was introduced in CUDA 11.7.
|
||||
#if ((__CUDACC_VER_MAJOR__ >= 12) || ((__CUDACC_VER_MAJOR__ == 11) && (__CUDACC_VER_MINOR__ >= 7)))
|
||||
# define CUTE_GRID_CONSTANT_SUPPORTED
|
||||
#endif
|
||||
|
||||
// __grid_constant__ can be enabled only on SM70+
|
||||
#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 700))
|
||||
# define CUTE_GRID_CONSTANT_ENABLED
|
||||
#endif
|
||||
|
||||
#if ! defined(CUTE_GRID_CONSTANT)
|
||||
# if defined(CUTE_GRID_CONSTANT_SUPPORTED) && defined(CUTE_GRID_CONSTANT_ENABLED)
|
||||
# define CUTE_GRID_CONSTANT __grid_constant__
|
||||
# else
|
||||
# define CUTE_GRID_CONSTANT
|
||||
# endif
|
||||
#endif
|
||||
|
||||
#if CUDA_12_0_SM90_FEATURES_SUPPORTED
|
||||
template <class T, class TiledCopy, class GmemLayout, class SmemLayout>
|
||||
__global__ void
|
||||
tma_test_device_cute(T const* g_in, T* g_out,
|
||||
CUTE_GRID_CONSTANT TiledCopy const tma,
|
||||
GmemLayout gmem_layout, SmemLayout smem_layout)
|
||||
{
|
||||
assert(product_each(shape(gmem_layout)) == product_each(smem_layout.shape()));
|
||||
|
||||
// Use Shared Storage structure to allocate and distribute aligned SMEM addresses
|
||||
extern __shared__ char shared_memory[];
|
||||
using SharedStorage = SharedStorage<T, SmemLayout>;
|
||||
SharedStorage& shared_storage = *reinterpret_cast<SharedStorage*>(shared_memory);
|
||||
|
||||
// Shared memory barriers use 64bits in SMEM for synchronization
|
||||
uint64_t* tma_load_mbar = shared_storage.tma_load_mbar;
|
||||
// Construct SMEM tensor
|
||||
Tensor sA = make_tensor(make_smem_ptr(shared_storage.smem.data()), smem_layout);
|
||||
|
||||
#if 0
|
||||
|
||||
//
|
||||
// Read in trivially
|
||||
//
|
||||
|
||||
Tensor gA_in = make_tensor(make_gmem_ptr(g_in), gmem_layout);
|
||||
|
||||
// Input gmem -> smem
|
||||
for (int i = threadIdx.x; i < size(sA); i += blockDim.x) {
|
||||
sA(i) = gA_in(i);
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
#else
|
||||
|
||||
// TMA requires special handling of strides to deal with coord codomain mapping
|
||||
// Represent the full tensors -- get these from TMA
|
||||
Tensor gA = tma.get_tma_tensor(shape(gmem_layout));
|
||||
|
||||
//
|
||||
// Prepare the TMA_LOAD
|
||||
//
|
||||
|
||||
auto cta_tma = tma.get_slice(Int<0>{}); // CTA slice
|
||||
|
||||
Tensor tAgA = cta_tma.partition_S(gA); // (TMA,TMA_M,TMA_N)
|
||||
Tensor tAsA = cta_tma.partition_D(sA); // (TMA,TMA_M,TMA_N)
|
||||
|
||||
#if 0
|
||||
if (thread0()) {
|
||||
print(" gA: "); print(gA.data()); print(" o "); print(gA.layout()); print("\n");
|
||||
print("tAgA: "); print(tAgA.data()); print(" o "); print(tAgA.layout()); print("\n");
|
||||
print(" sA: "); print(sA.data()); print(" o "); print(sA.layout()); print("\n");
|
||||
print("tAsA: "); print(tAsA.data()); print(" o "); print(tAsA.layout()); print("\n");
|
||||
}
|
||||
#endif
|
||||
|
||||
//
|
||||
// Perform the TMA_LOAD
|
||||
//
|
||||
|
||||
// Group the TMA_M and TMA_N modes
|
||||
Tensor tAgA_2 = group_modes<1,rank(tAgA)>(tAgA); // (TMA,Rest)
|
||||
Tensor tAsA_TR = group_modes<1,rank(tAsA)>(tAsA); // (TMA,Rest)
|
||||
static_assert(size<1>(tAsA_TR) == 1);
|
||||
Tensor tAsA_2 = tAsA_TR(_,0);
|
||||
|
||||
// Loop over the TMA stages, using smem as our buffer
|
||||
for (int stage = 0; stage < size<1>(tAgA_2); ++stage)
|
||||
{
|
||||
// Set the bytes transferred in this TMA transaction (may involve multiple issues)
|
||||
constexpr int kTmaTransactionBytes = size(sA) * sizeof(T);
|
||||
|
||||
if (threadIdx.x == 0)
|
||||
{
|
||||
/// Initialize shared memory barrier
|
||||
tma_load_mbar[0] = 0;
|
||||
cute::initialize_barrier(tma_load_mbar[0], 1 /*numThreads*/);
|
||||
cute::set_barrier_transaction_bytes(tma_load_mbar[0], kTmaTransactionBytes);
|
||||
|
||||
copy(tma.with(tma_load_mbar[0]), tAgA_2(_,stage), tAsA_2);
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
/// Wait on the shared memory barrier until the phase bit flips from kPhaseBit value
|
||||
constexpr int kPhaseBit = 0;
|
||||
cute::wait_barrier(tma_load_mbar[0], kPhaseBit);
|
||||
|
||||
#endif
|
||||
|
||||
//
|
||||
// Write out trivially
|
||||
//
|
||||
|
||||
Tensor gA_out = make_tensor(make_gmem_ptr(g_out), gmem_layout);
|
||||
// Do the same slicing and grouping as sA
|
||||
Tensor tAgA_out = cta_tma.partition_D(gA_out); // (TMA,TMA_M,TMA_N)
|
||||
Tensor tAgA_2_out = group_modes<1,rank(tAgA_out)>(tAgA_out); // (TMA,Rest)
|
||||
|
||||
// Output smem -> gmem
|
||||
for (int i = threadIdx.x; i < size(tAsA_2); i += blockDim.x) {
|
||||
tAgA_2_out(i,stage) = tAsA_2(i);
|
||||
}
|
||||
__syncthreads();
|
||||
}
|
||||
}
|
||||
|
||||
TEST(SM90_CuTe_Hopper, Tma_load_32x32_Col)
|
||||
{
|
||||
using T = half_t;
|
||||
Layout smem_layout = Layout<Shape<_32,_32>, Stride<_1,_32>>{};
|
||||
Layout gmem_layout = smem_layout;
|
||||
|
||||
thrust::host_vector<T> h_in(size(gmem_layout));
|
||||
for (int i = 0; i < h_in.size(); ++i) { h_in[i] = T(i); }
|
||||
thrust::device_vector<T> d_in = h_in;
|
||||
thrust::device_vector<T> d_out(h_in.size(), T(-1));
|
||||
|
||||
Tensor gA = make_tensor(d_in.data().get(), gmem_layout);
|
||||
auto tma = make_tma_copy(SM90_TMA_LOAD{}, gA, smem_layout);
|
||||
//print("TMA Box size: "); print(typename decltype(tma)::Tiler_MN{}); print("\n");
|
||||
|
||||
int smem_size = int(sizeof(SharedStorage<T, decltype(smem_layout)>));
|
||||
tma_test_device_cute<<<1, 128, smem_size>>>(
|
||||
thrust::raw_pointer_cast(d_in.data()),
|
||||
thrust::raw_pointer_cast(d_out.data()),
|
||||
tma,
|
||||
gmem_layout,
|
||||
smem_layout);
|
||||
|
||||
thrust::host_vector<T> h_out = d_out;
|
||||
for (int i = 0; i < size(smem_layout); ++i) {
|
||||
//printf("%d %d\n", int(h_in[i]), int(h_out[i]));
|
||||
EXPECT_EQ(h_out[i], h_in[i]);
|
||||
}
|
||||
CUTLASS_TRACE_HOST("CuTe TMA_LOAD 32x32 ColMajor SUCCESS\n");
|
||||
}
|
||||
|
||||
TEST(SM90_CuTe_Hopper, Tma_load_32x32_Row)
|
||||
{
|
||||
using T = half_t;
|
||||
Layout smem_layout = Layout<Shape<_32,_32>, Stride<_32,_1>>{};
|
||||
Layout gmem_layout = smem_layout;
|
||||
|
||||
thrust::host_vector<T> h_in(size(gmem_layout));
|
||||
for (int i = 0; i < h_in.size(); ++i) { h_in[i] = T(i); }
|
||||
thrust::device_vector<T> d_in = h_in;
|
||||
thrust::device_vector<T> d_out(h_in.size(), T(-1));
|
||||
|
||||
Tensor gA = make_tensor(d_in.data().get(), gmem_layout);
|
||||
auto tma = make_tma_copy(SM90_TMA_LOAD{}, gA, smem_layout);
|
||||
//print("TMA Box size: "); print(typename decltype(tma)::Tiler_MN{}); print("\n");
|
||||
|
||||
int smem_size = int(sizeof(SharedStorage<T, decltype(smem_layout)>));
|
||||
tma_test_device_cute<<<1, 128, smem_size>>>(
|
||||
thrust::raw_pointer_cast(d_in.data()),
|
||||
thrust::raw_pointer_cast(d_out.data()),
|
||||
tma,
|
||||
gmem_layout,
|
||||
smem_layout);
|
||||
|
||||
thrust::host_vector<T> h_out = d_out;
|
||||
for (int i = 0; i < size(smem_layout); ++i) {
|
||||
//printf("%d %d\n", int(h_in[i]), int(h_out[i]));
|
||||
EXPECT_EQ(h_out[i], h_in[i]);
|
||||
}
|
||||
CUTLASS_TRACE_HOST("CuTe TMA_LOAD 32x32 RowMajor SUCCESS\n");
|
||||
}
|
||||
|
||||
TEST(SM90_CuTe_Hopper, Tma_load_GMMA_SW128_MN)
|
||||
{
|
||||
using T = half_t;
|
||||
auto smem_layout = GMMA::Layout_MN_SW128_Atom<T>{};
|
||||
Layout gmem_layout = make_layout(make_shape(size<0>(smem_layout), size<1>(smem_layout)), GenColMajor{});
|
||||
|
||||
thrust::host_vector<T> h_in(size(gmem_layout));
|
||||
for (int i = 0; i < h_in.size(); ++i) { h_in[i] = T(i); }
|
||||
thrust::device_vector<T> d_in = h_in;
|
||||
thrust::device_vector<T> d_out(h_in.size(), T(-1));
|
||||
|
||||
Tensor gA = make_tensor(d_in.data().get(), gmem_layout);
|
||||
auto tma = make_tma_copy(SM90_TMA_LOAD{}, gA, smem_layout);
|
||||
//print("TMA Box size: "); print(typename decltype(tma)::Tiler_MN{}); print("\n");
|
||||
|
||||
int smem_size = int(sizeof(SharedStorage<T, decltype(smem_layout)>));
|
||||
tma_test_device_cute<<<1, 128, smem_size>>>(
|
||||
thrust::raw_pointer_cast(d_in.data()),
|
||||
thrust::raw_pointer_cast(d_out.data()),
|
||||
tma,
|
||||
gmem_layout,
|
||||
smem_layout);
|
||||
|
||||
thrust::host_vector<T> h_out = d_out;
|
||||
for (int i = 0; i < size(smem_layout); ++i) {
|
||||
//printf("%d %d\n", int(h_in[i]), int(h_out[i]));
|
||||
EXPECT_EQ(h_out[i], h_in[i]);
|
||||
}
|
||||
CUTLASS_TRACE_HOST("CuTe TMA_LOAD GMMA::Layout_MN_SW128_Atom<T> SUCCESS\n");
|
||||
}
|
||||
|
||||
TEST(SM90_CuTe_Hopper, Tma_load_GMMA_SW128_K)
|
||||
{
|
||||
using T = half_t;
|
||||
auto smem_layout = GMMA::Layout_K_SW128_Atom<T>{};
|
||||
Layout gmem_layout = make_layout(make_shape(size<0>(smem_layout), size<1>(smem_layout)), GenRowMajor{});
|
||||
|
||||
thrust::host_vector<T> h_in(size(gmem_layout));
|
||||
for (int i = 0; i < h_in.size(); ++i) { h_in[i] = T(i); }
|
||||
thrust::device_vector<T> d_in = h_in;
|
||||
thrust::device_vector<T> d_out(h_in.size(), T(-1));
|
||||
|
||||
Tensor gA = make_tensor(d_in.data().get(), gmem_layout);
|
||||
auto tma = make_tma_copy(SM90_TMA_LOAD{}, gA, smem_layout);
|
||||
//print("TMA Box size: "); print(typename decltype(tma)::Tiler_MN{}); print("\n");
|
||||
|
||||
int smem_size = int(sizeof(SharedStorage<T, decltype(smem_layout)>));
|
||||
tma_test_device_cute<<<1, 128, smem_size>>>(
|
||||
thrust::raw_pointer_cast(d_in.data()),
|
||||
thrust::raw_pointer_cast(d_out.data()),
|
||||
tma,
|
||||
gmem_layout,
|
||||
smem_layout);
|
||||
|
||||
thrust::host_vector<T> h_out = d_out;
|
||||
for (int i = 0; i < size(smem_layout); ++i) {
|
||||
//printf("%d %d\n", int(h_in[i]), int(h_out[i]));
|
||||
EXPECT_EQ(h_out[i], h_in[i]);
|
||||
}
|
||||
CUTLASS_TRACE_HOST("CuTe TMA_LOAD GMMA::Layout_K_SW128_Atom<T> SUCCESS\n");
|
||||
}
|
||||
|
||||
TEST(SM90_CuTe_Hopper, Tma_load_GMMA_SW128_MN_Multi)
|
||||
{
|
||||
using T = half_t;
|
||||
auto smem_layout = tile_to_shape(GMMA::Layout_MN_SW128_Atom<T>{}, Shape<Int<128>,Int<128>>{});
|
||||
Layout gmem_layout = make_layout(make_shape(size<0>(smem_layout), size<1>(smem_layout)), GenColMajor{});
|
||||
|
||||
thrust::host_vector<T> h_in(size(gmem_layout));
|
||||
for (int i = 0; i < h_in.size(); ++i) { h_in[i] = T(i); }
|
||||
thrust::device_vector<T> d_in = h_in;
|
||||
thrust::device_vector<T> d_out(h_in.size(), T(-1));
|
||||
|
||||
Tensor gA = make_tensor(d_in.data().get(), gmem_layout);
|
||||
auto tma = make_tma_copy(SM90_TMA_LOAD{}, gA, smem_layout);
|
||||
//print("TMA Box size: "); print(typename decltype(tma)::Tiler_MN{}); print("\n");
|
||||
|
||||
int smem_size = int(sizeof(SharedStorage<T, decltype(smem_layout)>));
|
||||
tma_test_device_cute<<<1, 128, smem_size>>>(
|
||||
thrust::raw_pointer_cast(d_in.data()),
|
||||
thrust::raw_pointer_cast(d_out.data()),
|
||||
tma,
|
||||
gmem_layout,
|
||||
smem_layout);
|
||||
|
||||
thrust::host_vector<T> h_out = d_out;
|
||||
for (int i = 0; i < size(smem_layout); ++i) {
|
||||
//printf("%d %d\n", int(h_in[i]), int(h_out[i]));
|
||||
EXPECT_EQ(h_out[i], h_in[i]);
|
||||
}
|
||||
CUTLASS_TRACE_HOST("CuTe TMA_LOAD GMMA::Layout_MN_SW128_Atom<T> Multi SUCCESS\n");
|
||||
}
|
||||
|
||||
TEST(SM90_CuTe_Hopper, Tma_load_GMMA_SW128_MN_Multi2)
|
||||
{
|
||||
using T = half_t;
|
||||
// Tile the GMMA::Layout atom in the K-mode first, then the M-mode to get a bigger box size
|
||||
auto smem_layout = tile_to_shape(GMMA::Layout_MN_SW128_Atom<T>{}, Shape<Int<128>,Int<128>>{}, Step<_2,_1>{});
|
||||
Layout gmem_layout = make_layout(make_shape(size<0>(smem_layout), size<1>(smem_layout)), GenColMajor{});
|
||||
|
||||
thrust::host_vector<T> h_in(size(gmem_layout));
|
||||
for (int i = 0; i < h_in.size(); ++i) { h_in[i] = T(i); }
|
||||
thrust::device_vector<T> d_in = h_in;
|
||||
thrust::device_vector<T> d_out(h_in.size(), T(-1));
|
||||
|
||||
Tensor gA = make_tensor(d_in.data().get(), gmem_layout);
|
||||
auto tma = make_tma_copy(SM90_TMA_LOAD{}, gA, smem_layout);
|
||||
//print("TMA Box size: "); print(typename decltype(tma)::Tiler_MN{}); print("\n");
|
||||
|
||||
int smem_size = int(sizeof(SharedStorage<T, decltype(smem_layout)>));
|
||||
tma_test_device_cute<<<1, 128, smem_size>>>(
|
||||
thrust::raw_pointer_cast(d_in.data()),
|
||||
thrust::raw_pointer_cast(d_out.data()),
|
||||
tma,
|
||||
gmem_layout,
|
||||
smem_layout);
|
||||
|
||||
thrust::host_vector<T> h_out = d_out;
|
||||
for (int i = 0; i < size(smem_layout); ++i) {
|
||||
//printf("%d %d\n", int(h_in[i]), int(h_out[i]));
|
||||
EXPECT_EQ(h_out[i], h_in[i]);
|
||||
}
|
||||
CUTLASS_TRACE_HOST("CuTe TMA_LOAD GMMA::Layout_MN_SW128_Atom<T> Multi SUCCESS\n");
|
||||
}
|
||||
|
||||
TEST(SM90_CuTe_Hopper, Tma_load_GMMA_SW128_MN_Multi_Dyn)
|
||||
{
|
||||
using T = half_t;
|
||||
auto smem_layout = tile_to_shape(GMMA::Layout_MN_SW128_Atom<T>{}, Shape<Int<128>,Int<128>>{}, Step<_2,_1>{});
|
||||
Layout gmem_layout = make_layout(make_shape(128, 128), GenColMajor{});
|
||||
|
||||
thrust::host_vector<T> h_in(size(gmem_layout));
|
||||
for (int i = 0; i < h_in.size(); ++i) { h_in[i] = T(i); }
|
||||
thrust::device_vector<T> d_in = h_in;
|
||||
thrust::device_vector<T> d_out(h_in.size(), T(-1));
|
||||
|
||||
Tensor gA = make_tensor(d_in.data().get(), gmem_layout);
|
||||
auto tma = make_tma_copy(SM90_TMA_LOAD{}, gA, smem_layout);
|
||||
//print("TMA Box size: "); print(typename decltype(tma)::Tiler_MN{}); print("\n");
|
||||
|
||||
int smem_size = int(sizeof(SharedStorage<T, decltype(smem_layout)>));
|
||||
tma_test_device_cute<<<1, 128, smem_size>>>(
|
||||
thrust::raw_pointer_cast(d_in.data()),
|
||||
thrust::raw_pointer_cast(d_out.data()),
|
||||
tma,
|
||||
gmem_layout,
|
||||
smem_layout);
|
||||
|
||||
thrust::host_vector<T> h_out = d_out;
|
||||
for (int i = 0; i < size(smem_layout); ++i) {
|
||||
//printf("%d %d\n", int(h_in[i]), int(h_out[i]));
|
||||
EXPECT_EQ(h_out[i], h_in[i]);
|
||||
}
|
||||
CUTLASS_TRACE_HOST("CuTe TMA_LOAD GMMA::Layout_MN_SW128_Atom<T> Multi SUCCESS\n");
|
||||
}
|
||||
|
||||
TEST(SM90_CuTe_Hopper, Tma_load_32x32_Multimode)
|
||||
{
|
||||
using T = half_t;
|
||||
auto smem_layout = Layout<Shape<_32,_32>, Stride<_32,_1>>{};
|
||||
Layout gmem_layout = make_layout(make_shape(make_shape(8,4), 32), GenRowMajor{});
|
||||
|
||||
//auto smem_layout = Layout<Shape<_32,_32>>{};
|
||||
//Layout gmem_layout = make_layout(make_shape(make_shape(8,4), 32), GenColMajor{});
|
||||
|
||||
thrust::host_vector<T> h_in(size(gmem_layout));
|
||||
for (int i = 0; i < h_in.size(); ++i) { h_in[i] = T(i); }
|
||||
thrust::device_vector<T> d_in = h_in;
|
||||
thrust::device_vector<T> d_out(h_in.size(), T(-1));
|
||||
|
||||
Tensor gA = make_tensor(d_in.data().get(), gmem_layout);
|
||||
auto tma = make_tma_copy(SM90_TMA_LOAD{}, gA, smem_layout);
|
||||
//print("TMA Box size: "); print(typename decltype(tma)::Tiler_MN{}); print("\n");
|
||||
|
||||
int smem_size = int(sizeof(SharedStorage<T, decltype(smem_layout)>));
|
||||
tma_test_device_cute<<<1, 128, smem_size>>>(
|
||||
thrust::raw_pointer_cast(d_in.data()),
|
||||
thrust::raw_pointer_cast(d_out.data()),
|
||||
tma,
|
||||
gmem_layout,
|
||||
smem_layout);
|
||||
|
||||
thrust::host_vector<T> h_out = d_out;
|
||||
for (int i = 0; i < size(smem_layout); ++i) {
|
||||
//printf("%d %d\n", int(h_in[i]), int(h_out[i]));
|
||||
EXPECT_EQ(h_out[i], h_in[i]);
|
||||
}
|
||||
CUTLASS_TRACE_HOST("CuTe TMA_LOAD GMMA::Layout_MN_SW128_Atom<T> Multi SUCCESS\n");
|
||||
}
|
||||
|
||||
TEST(SM90_CuTe_Hopper, Tma_load_Tensor_blocking)
|
||||
{
|
||||
using T = half_t;
|
||||
auto gmem_layout = make_shape(make_shape(336,40),make_shape(32,656)); // GMEM
|
||||
auto cta_tile = make_shape(make_shape(_16{},_8{}),make_shape(_32{},_2{})); // GMEM Tiling:
|
||||
// Take 16-elem from m0, 8-elem from m1,
|
||||
// Take 32-elem from k0, 2-elem from k1
|
||||
auto smem_layout = make_layout(cta_tile); // Col-Major SMEM
|
||||
|
||||
thrust::host_vector<T> h_in(size(gmem_layout));
|
||||
for (int i = 0; i < h_in.size(); ++i) { h_in[i] = T(i); }
|
||||
thrust::device_vector<T> d_in = h_in;
|
||||
thrust::device_vector<T> d_out(h_in.size(), T(-1));
|
||||
|
||||
Tensor gA = make_tensor(d_in.data().get(), gmem_layout);
|
||||
auto tma = make_tma_copy(SM90_TMA_LOAD{}, gA, smem_layout, cta_tile, Int<1>{});
|
||||
//print("TMA Box size: "); print(typename decltype(tma)::Tiler_MN{}); print("\n");
|
||||
|
||||
int smem_size = int(sizeof(SharedStorage<T, decltype(smem_layout)>));
|
||||
tma_test_device_cute<<<1, 128, smem_size>>>(
|
||||
thrust::raw_pointer_cast(d_in.data()),
|
||||
thrust::raw_pointer_cast(d_out.data()),
|
||||
tma,
|
||||
gmem_layout,
|
||||
smem_layout);
|
||||
|
||||
thrust::host_vector<T> h_out = d_out;
|
||||
for (int i = 0; i < size(smem_layout); ++i) {
|
||||
//printf("%d %d\n", int(h_in[i]), int(h_out[i]));
|
||||
EXPECT_EQ(h_out[i], h_in[i]);
|
||||
}
|
||||
CUTLASS_TRACE_HOST("CuTe TMA_LOAD Tensor blocking SUCCESS\n");
|
||||
}
|
||||
|
||||
TEST(SM90_CuTe_Hopper, Tma_load_Tensor_blocking_2)
|
||||
{
|
||||
using T = half_t;
|
||||
auto gmem_layout = make_shape(make_shape(32,40),make_shape(make_shape(8,8),656)); // GMEM
|
||||
auto cta_tile = make_shape(_128{},make_shape(_32{},_2{})); // GMEM Tiling:
|
||||
// Take 128-elem from m: m0 must divide 128,
|
||||
// m-last may be predicated
|
||||
// Take 32-elem from k0, 2-elem from k1
|
||||
auto smem_layout = make_layout(cta_tile); // Col-Major SMEM
|
||||
|
||||
thrust::host_vector<T> h_in(size(gmem_layout));
|
||||
for (int i = 0; i < h_in.size(); ++i) { h_in[i] = T(i); }
|
||||
thrust::device_vector<T> d_in = h_in;
|
||||
thrust::device_vector<T> d_out(h_in.size(), T(-1));
|
||||
|
||||
Tensor gA = make_tensor(d_in.data().get(), gmem_layout);
|
||||
auto tma = make_tma_copy(SM90_TMA_LOAD{}, gA, smem_layout, cta_tile, Int<1>{});
|
||||
//print("TMA Box size: "); print(typename decltype(tma)::Tiler_MN{}); print("\n");
|
||||
|
||||
int smem_size = int(sizeof(SharedStorage<T, decltype(smem_layout)>));
|
||||
tma_test_device_cute<<<1, 128, smem_size>>>(
|
||||
thrust::raw_pointer_cast(d_in.data()),
|
||||
thrust::raw_pointer_cast(d_out.data()),
|
||||
tma,
|
||||
gmem_layout,
|
||||
smem_layout);
|
||||
|
||||
thrust::host_vector<T> h_out = d_out;
|
||||
for (int i = 0; i < size(smem_layout); ++i) {
|
||||
//printf("%d %d\n", int(h_in[i]), int(h_out[i]));
|
||||
EXPECT_EQ(h_out[i], h_in[i]);
|
||||
}
|
||||
CUTLASS_TRACE_HOST("CuTe TMA_LOAD Tensor blocking 2 SUCCESS\n");
|
||||
}
|
||||
#endif
|
||||
@@ -0,0 +1,384 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017 - 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
#include "cutlass_unit_test.h"
|
||||
|
||||
#include <iostream>
|
||||
|
||||
#include <thrust/host_vector.h>
|
||||
#include <thrust/device_vector.h>
|
||||
|
||||
#include <cute/tensor.hpp>
|
||||
|
||||
using namespace cute;
|
||||
|
||||
template <class ElementType, class SmemLayout>
|
||||
struct SharedStorage
|
||||
{
|
||||
cute::array_aligned<ElementType, cute::cosize_v<SmemLayout>> smem;
|
||||
};
|
||||
|
||||
// __grid_constant__ was introduced in CUDA 11.7.
|
||||
#if ((__CUDACC_VER_MAJOR__ >= 12) || ((__CUDACC_VER_MAJOR__ == 11) && (__CUDACC_VER_MINOR__ >= 7)))
|
||||
# define CUTE_GRID_CONSTANT_SUPPORTED
|
||||
#endif
|
||||
|
||||
// __grid_constant__ can be enabled only on SM70+
|
||||
#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 700))
|
||||
# define CUTE_GRID_CONSTANT_ENABLED
|
||||
#endif
|
||||
|
||||
#if ! defined(CUTE_GRID_CONSTANT)
|
||||
# if defined(CUTE_GRID_CONSTANT_SUPPORTED) && defined(CUTE_GRID_CONSTANT_ENABLED)
|
||||
# define CUTE_GRID_CONSTANT __grid_constant__
|
||||
# else
|
||||
# define CUTE_GRID_CONSTANT
|
||||
# endif
|
||||
#endif
|
||||
|
||||
#if CUDA_12_0_SM90_FEATURES_SUPPORTED
|
||||
template <class T, class TiledCopy, class GmemLayout, class SmemLayout>
|
||||
__global__ void
|
||||
tma_test_device_cute(T const* g_in, T* g_out,
|
||||
CUTE_GRID_CONSTANT TiledCopy const tma,
|
||||
GmemLayout gmem_layout, SmemLayout smem_layout)
|
||||
{
|
||||
// Use Shared Storage structure to allocate and distribute aligned SMEM addresses
|
||||
extern __shared__ char shared_memory[];
|
||||
using SharedStorage = SharedStorage<T, SmemLayout>;
|
||||
SharedStorage& shared_storage = *reinterpret_cast<SharedStorage*>(shared_memory);
|
||||
// Construct SMEM tensor
|
||||
Tensor sA = make_tensor(make_smem_ptr(shared_storage.smem.data()), smem_layout);
|
||||
|
||||
//
|
||||
// Read in trivially
|
||||
//
|
||||
|
||||
Tensor gA_in = make_tensor(make_gmem_ptr(g_in), gmem_layout);
|
||||
|
||||
// Input gmem -> smem
|
||||
for (int i = threadIdx.x; i < size(sA); i += blockDim.x) {
|
||||
sA(i) = gA_in(i);
|
||||
}
|
||||
|
||||
__syncthreads();
|
||||
|
||||
#if 0
|
||||
|
||||
//
|
||||
// Write out trivially
|
||||
//
|
||||
|
||||
Tensor gA_out = make_tensor(make_gmem_ptr(g_out), gmem_layout);
|
||||
|
||||
// Output smem -> gmem
|
||||
for (int i = threadIdx.x; i < size(sA); i += blockDim.x) {
|
||||
gA_out(i) = sA(i);
|
||||
}
|
||||
|
||||
#else
|
||||
|
||||
// TMA requires special handling of strides to deal with coord codomain mapping
|
||||
// Represent the full tensors -- get these from TMA
|
||||
Tensor gA = tma.get_tma_tensor(shape(gmem_layout));
|
||||
|
||||
//
|
||||
// Prepare the TMA_STORE
|
||||
//
|
||||
|
||||
auto cta_tma = tma.get_slice(Int<0>{}); // CTA slice
|
||||
|
||||
Tensor tAsA = cta_tma.partition_S(sA);
|
||||
Tensor tAgA = cta_tma.partition_D(gA);
|
||||
|
||||
//
|
||||
// Perform the TMA_STORE
|
||||
//
|
||||
|
||||
if (threadIdx.x == 0) {
|
||||
copy(tma, tAsA, tAgA);
|
||||
}
|
||||
|
||||
#endif
|
||||
}
|
||||
|
||||
TEST(SM90_CuTe_Hopper, Tma_Store_32x32_Col)
|
||||
{
|
||||
using T = half_t;
|
||||
Layout smem_layout = Layout<Shape<_32,_32>, Stride<_1,_32>>{};
|
||||
Layout gmem_layout = smem_layout;
|
||||
|
||||
thrust::host_vector<T> h_in(size(smem_layout));
|
||||
for (int i = 0; i < h_in.size(); ++i) { h_in[i] = T(i); }
|
||||
thrust::device_vector<T> d_in = h_in;
|
||||
thrust::device_vector<T> d_out(h_in.size(), T(-1));
|
||||
|
||||
Tensor gA = make_tensor(d_out.data().get(), gmem_layout);
|
||||
auto tma = make_tma_copy(SM90_TMA_STORE{}, gA, smem_layout);
|
||||
//print("TMA Box size: "); print(typename decltype(tma)::Tiler_MN{}); print("\n");
|
||||
|
||||
int smem_size = int(sizeof(SharedStorage<T, decltype(smem_layout)>));
|
||||
tma_test_device_cute<<<1, 128, smem_size>>>(
|
||||
thrust::raw_pointer_cast(d_in.data()),
|
||||
thrust::raw_pointer_cast(d_out.data()),
|
||||
tma,
|
||||
gmem_layout,
|
||||
smem_layout);
|
||||
|
||||
thrust::host_vector<T> h_out = d_out;
|
||||
for (int i = 0; i < size(smem_layout); ++i) {
|
||||
//printf("%d %d\n", int(h_in[i]), int(h_out[i]));
|
||||
EXPECT_EQ(h_out[i], h_in[i]);
|
||||
}
|
||||
CUTLASS_TRACE_HOST("CuTe TMA_STORE 32x32 ColMajor SUCCESS\n");
|
||||
}
|
||||
|
||||
TEST(SM90_CuTe_Hopper, Tma_Store_32x32_Row)
|
||||
{
|
||||
using T = half_t;
|
||||
Layout smem_layout = Layout<Shape<_32,_32>, Stride<_32,_1>>{};
|
||||
Layout gmem_layout = smem_layout;
|
||||
|
||||
thrust::host_vector<T> h_in(size(smem_layout));
|
||||
for (int i = 0; i < h_in.size(); ++i) { h_in[i] = T(i); }
|
||||
thrust::device_vector<T> d_in = h_in;
|
||||
thrust::device_vector<T> d_out(h_in.size(), T(-1));
|
||||
|
||||
Tensor gA = make_tensor(d_out.data().get(), gmem_layout);
|
||||
auto tma = make_tma_copy(SM90_TMA_STORE{}, gA, smem_layout);
|
||||
//print("TMA Box size: "); print(typename decltype(tma)::Tiler_MN{}); print("\n");
|
||||
|
||||
int smem_size = int(sizeof(SharedStorage<T, decltype(smem_layout)>));
|
||||
tma_test_device_cute<<<1, 128, smem_size>>>(
|
||||
thrust::raw_pointer_cast(d_in.data()),
|
||||
thrust::raw_pointer_cast(d_out.data()),
|
||||
tma,
|
||||
gmem_layout,
|
||||
smem_layout);
|
||||
|
||||
thrust::host_vector<T> h_out = d_out;
|
||||
for (int i = 0; i < size(smem_layout); ++i) {
|
||||
//printf("%d %d\n", int(h_in[i]), int(h_out[i]));
|
||||
EXPECT_EQ(h_out[i], h_in[i]);
|
||||
}
|
||||
CUTLASS_TRACE_HOST("CuTe TMA_STORE 32x32 RowMajor SUCCESS\n");
|
||||
}
|
||||
|
||||
TEST(SM90_CuTe_Hopper, Tma_Store_GMMA_SW128_MN)
|
||||
{
|
||||
using T = half_t;
|
||||
auto smem_layout = GMMA::Layout_MN_SW128_Atom<T>{};
|
||||
Layout gmem_layout = make_layout(make_shape(size<0>(smem_layout), size<1>(smem_layout)), GenColMajor{});
|
||||
|
||||
thrust::host_vector<T> h_in(size(smem_layout));
|
||||
for (int i = 0; i < h_in.size(); ++i) { h_in[i] = T(i); }
|
||||
thrust::device_vector<T> d_in = h_in;
|
||||
thrust::device_vector<T> d_out(h_in.size(), T(-1));
|
||||
|
||||
Tensor gA = make_tensor(d_out.data().get(), gmem_layout);
|
||||
auto tma = make_tma_copy(SM90_TMA_STORE{}, gA, smem_layout);
|
||||
//print("TMA Box size: "); print(typename decltype(tma)::Tiler_MN{}); print("\n");
|
||||
|
||||
int smem_size = int(sizeof(SharedStorage<T, decltype(smem_layout)>));
|
||||
tma_test_device_cute<<<1, 128, smem_size>>>(
|
||||
thrust::raw_pointer_cast(d_in.data()),
|
||||
thrust::raw_pointer_cast(d_out.data()),
|
||||
tma,
|
||||
gmem_layout,
|
||||
smem_layout);
|
||||
|
||||
thrust::host_vector<T> h_out = d_out;
|
||||
for (int i = 0; i < size(smem_layout); ++i) {
|
||||
//printf("%d %d\n", int(h_in[i]), int(h_out[i]));
|
||||
EXPECT_EQ(h_out[i], h_in[i]);
|
||||
}
|
||||
CUTLASS_TRACE_HOST("CuTe TMA_STORE GMMA::Layout_MN_SW128_Atom<T> SUCCESS\n");
|
||||
}
|
||||
|
||||
TEST(SM90_CuTe_Hopper, Tma_Store_GMMA_SW128_K)
|
||||
{
|
||||
using T = half_t;
|
||||
auto smem_layout = GMMA::Layout_K_SW128_Atom<T>{};
|
||||
Layout gmem_layout = make_layout(make_shape(size<0>(smem_layout), size<1>(smem_layout)), GenRowMajor{});
|
||||
|
||||
thrust::host_vector<T> h_in(size(smem_layout));
|
||||
for (int i = 0; i < h_in.size(); ++i) { h_in[i] = T(i); }
|
||||
thrust::device_vector<T> d_in = h_in;
|
||||
thrust::device_vector<T> d_out(h_in.size(), T(-1));
|
||||
|
||||
Tensor gA = make_tensor(d_out.data().get(), gmem_layout);
|
||||
auto tma = make_tma_copy(SM90_TMA_STORE{}, gA, smem_layout);
|
||||
//print("TMA Box size: "); print(typename decltype(tma)::Tiler_MN{}); print("\n");
|
||||
|
||||
int smem_size = int(sizeof(SharedStorage<T, decltype(smem_layout)>));
|
||||
tma_test_device_cute<<<1, 128, smem_size>>>(
|
||||
thrust::raw_pointer_cast(d_in.data()),
|
||||
thrust::raw_pointer_cast(d_out.data()),
|
||||
tma,
|
||||
gmem_layout,
|
||||
smem_layout);
|
||||
|
||||
thrust::host_vector<T> h_out = d_out;
|
||||
for (int i = 0; i < size(smem_layout); ++i) {
|
||||
//printf("%d %d\n", int(h_in[i]), int(h_out[i]));
|
||||
EXPECT_EQ(h_out[i], h_in[i]);
|
||||
}
|
||||
CUTLASS_TRACE_HOST("CuTe TMA_STORE GMMA::Layout_K_SW128_Atom<T> SUCCESS\n");
|
||||
}
|
||||
|
||||
TEST(SM90_CuTe_Hopper, Tma_Store_GMMA_SW128_MN_Multi)
|
||||
{
|
||||
using T = half_t;
|
||||
auto smem_layout = tile_to_shape(GMMA::Layout_MN_SW128_Atom<T>{}, Shape<Int<128>,Int<128>>{});
|
||||
Layout gmem_layout = make_layout(make_shape(size<0>(smem_layout), size<1>(smem_layout)), GenColMajor{});
|
||||
|
||||
thrust::host_vector<T> h_in(size(smem_layout));
|
||||
for (int i = 0; i < h_in.size(); ++i) { h_in[i] = T(i); }
|
||||
thrust::device_vector<T> d_in = h_in;
|
||||
thrust::device_vector<T> d_out(h_in.size(), T(-1));
|
||||
|
||||
Tensor gA = make_tensor(d_out.data().get(), gmem_layout);
|
||||
auto tma = make_tma_copy(SM90_TMA_STORE{}, gA, smem_layout);
|
||||
//print("TMA Box size: "); print(typename decltype(tma)::Tiler_MN{}); print("\n");
|
||||
|
||||
int smem_size = int(sizeof(SharedStorage<T, decltype(smem_layout)>));
|
||||
tma_test_device_cute<<<1, 128, smem_size>>>(
|
||||
thrust::raw_pointer_cast(d_in.data()),
|
||||
thrust::raw_pointer_cast(d_out.data()),
|
||||
tma,
|
||||
gmem_layout,
|
||||
smem_layout);
|
||||
|
||||
thrust::host_vector<T> h_out = d_out;
|
||||
for (int i = 0; i < size(smem_layout); ++i) {
|
||||
//printf("%d %d\n", int(h_in[i]), int(h_out[i]));
|
||||
EXPECT_EQ(h_out[i], h_in[i]);
|
||||
}
|
||||
CUTLASS_TRACE_HOST("CuTe TMA_STORE GMMA::Layout_MN_SW128_Atom<T> Multi SUCCESS\n");
|
||||
}
|
||||
|
||||
TEST(SM90_CuTe_Hopper, Tma_Store_GMMA_SW128_MN_Multi2)
|
||||
{
|
||||
using T = half_t;
|
||||
// Tile the GMMA::Layout atom in the K-mode first, then the M-mode to get a bigger box size
|
||||
auto smem_layout = tile_to_shape(GMMA::Layout_MN_SW128_Atom<T>{}, Shape<Int<128>,Int<128>>{}, Step<_2,_1>{});
|
||||
Layout gmem_layout = make_layout(make_shape(size<0>(smem_layout), size<1>(smem_layout)), GenColMajor{});
|
||||
|
||||
thrust::host_vector<T> h_in(size(smem_layout));
|
||||
for (int i = 0; i < h_in.size(); ++i) { h_in[i] = T(i); }
|
||||
thrust::device_vector<T> d_in = h_in;
|
||||
thrust::device_vector<T> d_out(h_in.size(), T(-1));
|
||||
|
||||
Tensor gA = make_tensor(d_out.data().get(), gmem_layout);
|
||||
auto tma = make_tma_copy(SM90_TMA_STORE{}, gA, smem_layout);
|
||||
//print("TMA Box size: "); print(typename decltype(tma)::Tiler_MN{}); print("\n");
|
||||
|
||||
int smem_size = int(sizeof(SharedStorage<T, decltype(smem_layout)>));
|
||||
tma_test_device_cute<<<1, 128, smem_size>>>(
|
||||
thrust::raw_pointer_cast(d_in.data()),
|
||||
thrust::raw_pointer_cast(d_out.data()),
|
||||
tma,
|
||||
gmem_layout,
|
||||
smem_layout);
|
||||
|
||||
thrust::host_vector<T> h_out = d_out;
|
||||
for (int i = 0; i < size(smem_layout); ++i) {
|
||||
//printf("%d %d\n", int(h_in[i]), int(h_out[i]));
|
||||
EXPECT_EQ(h_out[i], h_in[i]);
|
||||
}
|
||||
CUTLASS_TRACE_HOST("CuTe TMA_STORE GMMA::Layout_MN_SW128_Atom<T> Multi SUCCESS\n");
|
||||
}
|
||||
|
||||
TEST(SM90_CuTe_Hopper, Tma_Store_GMMA_SW128_MN_Multi_Dyn)
|
||||
{
|
||||
using T = half_t;
|
||||
auto smem_layout = tile_to_shape(GMMA::Layout_MN_SW128_Atom<T>{}, Shape<Int<128>,Int<128>>{}, Step<_2,_1>{});
|
||||
Layout gmem_layout = make_layout(make_shape(128, 128), GenColMajor{});
|
||||
|
||||
thrust::host_vector<T> h_in(size(smem_layout));
|
||||
for (int i = 0; i < h_in.size(); ++i) { h_in[i] = T(i); }
|
||||
thrust::device_vector<T> d_in = h_in;
|
||||
thrust::device_vector<T> d_out(h_in.size(), T(-1));
|
||||
|
||||
Tensor gA = make_tensor(d_out.data().get(), gmem_layout);
|
||||
auto tma = make_tma_copy(SM90_TMA_STORE{}, gA, smem_layout);
|
||||
//print("TMA Box size: "); print(typename decltype(tma)::Tiler_MN{}); print("\n");
|
||||
|
||||
int smem_size = int(sizeof(SharedStorage<T, decltype(smem_layout)>));
|
||||
tma_test_device_cute<<<1, 128, smem_size>>>(
|
||||
thrust::raw_pointer_cast(d_in.data()),
|
||||
thrust::raw_pointer_cast(d_out.data()),
|
||||
tma,
|
||||
gmem_layout,
|
||||
smem_layout);
|
||||
|
||||
thrust::host_vector<T> h_out = d_out;
|
||||
for (int i = 0; i < size(smem_layout); ++i) {
|
||||
//printf("%d %d\n", int(h_in[i]), int(h_out[i]));
|
||||
EXPECT_EQ(h_out[i], h_in[i]);
|
||||
}
|
||||
CUTLASS_TRACE_HOST("CuTe TMA_STORE GMMA::Layout_MN_SW128_Atom<T> Multi SUCCESS\n");
|
||||
}
|
||||
|
||||
TEST(SM90_CuTe_Hopper, Tma_Store_32x32_Multimode)
|
||||
{
|
||||
using T = half_t;
|
||||
auto smem_layout = Layout<Shape<_32,_32>, Stride<_32,_1>>{};
|
||||
Layout gmem_layout = make_layout(make_shape(make_shape(8,4), 32), GenRowMajor{});
|
||||
|
||||
//auto smem_layout = Layout<Shape<_32,_32>>{};
|
||||
//Layout gmem_layout = make_layout(make_shape(make_shape(8,4), 32), GenColMajor{});
|
||||
|
||||
thrust::host_vector<T> h_in(size(smem_layout));
|
||||
for (int i = 0; i < h_in.size(); ++i) { h_in[i] = T(i); }
|
||||
thrust::device_vector<T> d_in = h_in;
|
||||
thrust::device_vector<T> d_out(h_in.size(), T(-1));
|
||||
|
||||
Tensor gA = make_tensor(d_out.data().get(), gmem_layout);
|
||||
auto tma = make_tma_copy(SM90_TMA_STORE{}, gA, smem_layout);
|
||||
//print("TMA Box size: "); print(typename decltype(tma)::Tiler_MN{}); print("\n");
|
||||
|
||||
int smem_size = int(sizeof(SharedStorage<T, decltype(smem_layout)>));
|
||||
tma_test_device_cute<<<1, 128, smem_size>>>(
|
||||
thrust::raw_pointer_cast(d_in.data()),
|
||||
thrust::raw_pointer_cast(d_out.data()),
|
||||
tma,
|
||||
gmem_layout,
|
||||
smem_layout);
|
||||
|
||||
thrust::host_vector<T> h_out = d_out;
|
||||
for (int i = 0; i < size(smem_layout); ++i) {
|
||||
//printf("%d %d\n", int(h_in[i]), int(h_out[i]));
|
||||
EXPECT_EQ(h_out[i], h_in[i]);
|
||||
}
|
||||
CUTLASS_TRACE_HOST("CuTe TMA_STORE GMMA::Layout_MN_SW128_Atom<T> Multi SUCCESS\n");
|
||||
}
|
||||
#endif
|
||||
@@ -0,0 +1,32 @@
|
||||
# Copyright (c) 2023 - 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: BSD-3-Clause
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions are met:
|
||||
#
|
||||
# 1. Redistributions of source code must retain the above copyright notice, this
|
||||
# list of conditions and the following disclaimer.
|
||||
#
|
||||
# 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
# this list of conditions and the following disclaimer in the documentation
|
||||
# and/or other materials provided with the distribution.
|
||||
#
|
||||
# 3. Neither the name of the copyright holder nor the names of its
|
||||
# contributors may be used to endorse or promote products derived from
|
||||
# this software without specific prior written permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
cutlass_test_unit_add_executable(
|
||||
cutlass_test_unit_cute_layout
|
||||
layout_operator.cu
|
||||
)
|
||||
@@ -0,0 +1,136 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2023 - 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
/*! \file
|
||||
\brief Unit tests Generic CuTe Layouts
|
||||
*/
|
||||
|
||||
#include "../../common/cutlass_unit_test.h"
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cutlass/layout/layout.h"
|
||||
#include "cutlass/matrix_coord.h"
|
||||
|
||||
// Cute includes
|
||||
#include <cute/layout.hpp>
|
||||
#include <cute/int_tuple.hpp>
|
||||
|
||||
using namespace cutlass;
|
||||
using namespace cute;
|
||||
|
||||
namespace test {
|
||||
namespace layout {
|
||||
|
||||
template <typename GenericLayout, typename Layout>
|
||||
struct Testbed {
|
||||
|
||||
|
||||
Testbed() {}
|
||||
|
||||
bool run() {
|
||||
GenericLayout generic_layout;
|
||||
Layout layout = Layout::packed({size<0>(generic_layout), size<1>(generic_layout)});
|
||||
|
||||
for (int m = 0; m < size<0>(generic_layout); m++) {
|
||||
for (int n = 0; n < size<1>(generic_layout); n++) {
|
||||
if (generic_layout(m, n) != layout({m, n})) return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
};
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
// Test Generic CuTe Layouts
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Canonical Layouts
|
||||
|
||||
TEST(GenericLayout, ColumnMajor) {
|
||||
using GenericLayout = cute::Layout<Shape<_8, _4>, Stride<_1, _8>>;
|
||||
using Layout = cutlass::layout::ColumnMajor;
|
||||
|
||||
test::layout::Testbed<GenericLayout, Layout> testbed;
|
||||
|
||||
EXPECT_TRUE(testbed.run());
|
||||
}
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(GenericLayout, RowMajor) {
|
||||
using GenericLayout = cute::Layout<Shape<_8, _4>, Stride<_4, _1>>;
|
||||
using Layout = cutlass::layout::RowMajor;
|
||||
|
||||
test::layout::Testbed<GenericLayout, Layout> testbed;
|
||||
|
||||
EXPECT_TRUE(testbed.run());
|
||||
}
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
|
||||
|
||||
/// Swizzle Shared Memory layouts
|
||||
|
||||
TEST(GenericLayout, RowMajorTensorOpMultiplicandCrosswise) {
|
||||
|
||||
using GenericLayout = decltype(
|
||||
composition(
|
||||
Swizzle<3,3,3>{},
|
||||
Layout<Shape<_128, _64>, Stride<_64, _1>>{})
|
||||
);
|
||||
|
||||
using Layout = cutlass::layout::RowMajorTensorOpMultiplicandCrosswise<
|
||||
cutlass::sizeof_bits<cutlass::half_t>::value, 64>;
|
||||
|
||||
test::layout::Testbed<GenericLayout, Layout> testbed;
|
||||
|
||||
EXPECT_TRUE(testbed.run());
|
||||
}
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(GenericLayout, ColumnMajorTensorOpMultiplicandCongruous) {
|
||||
|
||||
using GenericLayout = decltype(
|
||||
composition(
|
||||
Swizzle<3,3,4>{},
|
||||
Layout<Shape<_128, _64>>{})
|
||||
);
|
||||
|
||||
using Layout = cutlass::layout::ColumnMajorTensorOpMultiplicandCongruous<
|
||||
cutlass::sizeof_bits<cutlass::half_t>::value, 64>;
|
||||
|
||||
|
||||
test::layout::Testbed<GenericLayout, Layout> testbed;
|
||||
|
||||
EXPECT_TRUE(testbed.run());
|
||||
}
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
@@ -41,6 +41,8 @@ add_custom_target(
|
||||
cutlass_test_unit_gemm_device_tensorop_planar_complex
|
||||
cutlass_test_unit_gemm_device_sparse_tensorop_sm80
|
||||
cutlass_test_unit_gemv_device
|
||||
cutlass_test_unit_gemm_device_tensorop_sm90
|
||||
cutlass_test_unit_gemm_device_tensorop_cluster_multicast_sm90
|
||||
)
|
||||
|
||||
add_custom_target(
|
||||
@@ -58,6 +60,14 @@ add_custom_target(
|
||||
test_unit_gemm_device_tensorop_planar_complex
|
||||
test_unit_gemm_device_sparse_tensorop_sm80
|
||||
test_unit_gemv_device
|
||||
test_unit_gemm_device_tensorop_sm90
|
||||
)
|
||||
|
||||
add_custom_target(
|
||||
cutlass_test_unit_gemm_device_sm90
|
||||
DEPENDS
|
||||
cutlass_test_unit_gemm_device_tensorop_sm90
|
||||
cutlass_test_unit_gemm_device_tensorop_cluster_multicast_sm90
|
||||
)
|
||||
|
||||
cutlass_test_unit_add_executable(
|
||||
@@ -78,7 +88,7 @@ cutlass_test_unit_add_executable(
|
||||
simt_cgemm_nt_sm50.cu
|
||||
simt_cgemm_tn_sm50.cu
|
||||
simt_cgemm_tt_sm50.cu
|
||||
|
||||
|
||||
simt_qgemm_nn_sm50.cu
|
||||
simt_qgemm_nt_sm50.cu
|
||||
simt_qgemm_tn_sm50.cu
|
||||
@@ -88,33 +98,48 @@ cutlass_test_unit_add_executable(
|
||||
simt_dgemm_nt_sm50.cu
|
||||
simt_dgemm_tn_sm50.cu
|
||||
simt_dgemm_tt_sm50.cu
|
||||
|
||||
|
||||
simt_hgemm_nn_sm50.cu
|
||||
simt_hgemm_nt_sm50.cu
|
||||
simt_hgemm_tn_sm50.cu
|
||||
simt_hgemm_tt_sm50.cu
|
||||
|
||||
|
||||
simt_igemm_nn_sm50.cu
|
||||
simt_igemm_nt_sm50.cu
|
||||
simt_igemm_tn_sm50.cu
|
||||
simt_igemm_tt_sm50.cu
|
||||
|
||||
|
||||
simt_int8_igemm_sm61_sliced_k.cu
|
||||
simt_int8_igemm_sm61.cu
|
||||
|
||||
|
||||
simt_sgemm_nn_sm50.cu
|
||||
simt_sgemm_nt_sm50.cu
|
||||
simt_sgemm_tn_sm50.cu
|
||||
simt_sgemm_tt_sm50.cu
|
||||
|
||||
|
||||
simt_zgemm_nn_sm50.cu
|
||||
simt_zgemm_nt_sm50.cu
|
||||
simt_zgemm_tn_sm50.cu
|
||||
simt_zgemm_tt_sm50.cu
|
||||
|
||||
|
||||
gemm_splitk_simt_sm50.cu
|
||||
)
|
||||
|
||||
cutlass_test_unit_add_executable(
|
||||
cutlass_test_unit_gemm_device_simt_3x
|
||||
|
||||
BATCH_SOURCES ON
|
||||
BATCH_SIZE 4
|
||||
|
||||
|
||||
sm50_gemm_f32_f32_f32_simt.cu
|
||||
sm80_gemm_f32_f32_f32_simt.cu
|
||||
sm50_gemm_f64_f64_f64_simt.cu
|
||||
sm80_gemm_f64_f64_f64_simt.cu
|
||||
sm61_gemm_s8_s8_s32_simt.cu
|
||||
)
|
||||
|
||||
|
||||
cutlass_test_unit_add_executable(
|
||||
cutlass_test_unit_gemm_device_tensorop_sm70
|
||||
|
||||
@@ -209,6 +234,51 @@ cutlass_test_unit_add_executable(
|
||||
gemm_f16n_f16n_f16n_direct_store_tensor_op_f32_sm80.cu
|
||||
)
|
||||
|
||||
cutlass_test_unit_add_executable(
|
||||
cutlass_test_unit_gemm_device_tensorop_f32_sm80_3x
|
||||
|
||||
sm80_gemm_s8_s8_s32_tensor_op.cu
|
||||
sm80_gemm_f16_f16_f32_tensor_op_f32.cu
|
||||
sm80_gemm_tf32_tf32_f32_tensor_op_f32.cu
|
||||
)
|
||||
|
||||
cutlass_test_unit_add_executable(
|
||||
cutlass_test_unit_gemm_device_tensorop_sm90
|
||||
|
||||
BATCH_SOURCES ON
|
||||
BATCH_SIZE 4
|
||||
|
||||
sm90_gemm_f16_f16_f16_tensor_op.cu
|
||||
sm90_gemm_bf16_bf16_bf16_tensor_op_f32.cu
|
||||
sm90_gemm_s8_s8_s8_tensor_op_s32.cu
|
||||
sm90_gemm_tf32_tf32_f32_tensor_op_f32.cu
|
||||
sm90_gemm_f32_f32_f32_tensor_op_f32.cu
|
||||
)
|
||||
|
||||
# Alignment tests
|
||||
cutlass_test_unit_add_executable(
|
||||
cutlass_test_unit_gemm_device_tensorop_alignx_sm90
|
||||
|
||||
BATCH_SOURCES ON
|
||||
BATCH_SIZE 4
|
||||
sm90_gemm_f16_f16_f16_alignx_tensor_op.cu
|
||||
sm90_gemm_bf16_bf16_bf16_alignx_tensor_op_f32.cu
|
||||
sm90_gemm_s8_s8_s8_alignx_tensor_op_s32.cu
|
||||
sm90_gemm_tf32_tf32_f32_alignx_tensor_op_f32.cu
|
||||
)
|
||||
|
||||
|
||||
cutlass_test_unit_add_executable(
|
||||
cutlass_test_unit_gemm_device_tensorop_cluster_multicast_sm90
|
||||
|
||||
BATCH_SOURCES ON
|
||||
BATCH_SIZE 4
|
||||
|
||||
sm90_gemm_f16_f16_f16_tensor_op_f32_cluster_unspecialized.cu
|
||||
sm90_gemm_f16_f16_f16_tensor_op_f32_cluster_warpspecialized.cu
|
||||
sm90_gemm_f16_f16_f16_tensor_op_f32_cluster_warpspecialized_persistent.cu
|
||||
)
|
||||
|
||||
cutlass_test_unit_add_executable(
|
||||
cutlass_test_unit_gemm_device_tensorop_f32_tf32_sm80
|
||||
|
||||
@@ -226,6 +296,7 @@ cutlass_test_unit_add_executable(
|
||||
gemm_f32n_f32n_f32t_tensor_op_f32_sm80.cu
|
||||
gemm_f32n_f32n_f32t_tensor_op_bf16_f32_sm80.cu
|
||||
|
||||
sm80_gemm_f16_f16_f32_tensor_op_f32.cu
|
||||
)
|
||||
|
||||
cutlass_test_unit_add_executable(
|
||||
@@ -247,6 +318,9 @@ cutlass_test_unit_add_executable(
|
||||
# SM90 device level tests
|
||||
gemm_f64n_f64t_f64t_tensor_op_f64_sm90.cu
|
||||
gemm_f64t_f64n_f64t_tensor_op_f64_sm90.cu
|
||||
|
||||
sm80_gemm_f64_f64_f64_tensor_op_f64.cu
|
||||
|
||||
gemm_cf64n_cf64t_cf64t_tensor_op_f64_sm90.cu
|
||||
gemm_cf64t_cf64n_cf64t_tensor_op_f64_sm90.cu
|
||||
gemm_cf64n_cf64t_cf64t_tensor_op_f64_gaussian_sm90.cu
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -50,7 +50,7 @@
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#if defined(CUTLASS_ARCH_MMA_SM90_SUPPORTED)
|
||||
#if defined(CUTLASS_ARCH_MMA_SM90_F64_MMA_ENABLED)
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
@@ -193,6 +193,6 @@ TEST(SM90_Device_Gemm_cf64n_cf64t_cf64t_tensor_op_f64_gaussian, 64x64x8_16x32x8)
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#endif // #if defined(CUTLASS_ARCH_MMA_SM90_SUPPORTED)
|
||||
#endif // #if defined(CUTLASS_ARCH_MMA_SM90_F64_MMA_ENABLED)
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
@@ -50,7 +50,7 @@
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#if defined(CUTLASS_ARCH_MMA_SM90_SUPPORTED)
|
||||
#if defined(CUTLASS_ARCH_MMA_SM90_F64_MMA_ENABLED)
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
@@ -247,6 +247,6 @@ TEST(SM90_Device_Gemm_cf64n_cf64t_cf64t_tensor_op_f64, 64x64x8_32x32x8) {
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#endif // #if defined(CUTLASS_ARCH_MMA_SM90_SUPPORTED)
|
||||
#endif // #if defined(CUTLASS_ARCH_MMA_SM90_F64_MMA_ENABLED)
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
@@ -50,7 +50,7 @@
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#if defined(CUTLASS_ARCH_MMA_SM90_SUPPORTED)
|
||||
#if defined(CUTLASS_ARCH_MMA_SM90_F64_MMA_ENABLED)
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
@@ -191,7 +191,7 @@ TEST(SM90_Device_Gemm_cf64t_cf64n_cf64t_tensor_op_f64_gaussian, 64x64x16_32x16x1
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#endif // #if defined(CUTLASS_ARCH_MMA_SM90_SUPPORTED)
|
||||
#endif // #if defined(CUTLASS_ARCH_MMA_SM90_F64_MMA_ENABLED)
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
|
||||
@@ -50,7 +50,7 @@
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#if defined(CUTLASS_ARCH_MMA_SM90_SUPPORTED)
|
||||
#if defined(CUTLASS_ARCH_MMA_SM90_F64_MMA_ENABLED)
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
@@ -299,7 +299,7 @@ TEST(SM90_Device_Gemm_cf64t_cf64n_cf64t_tensor_op_f64, 128x64x16_32x32x16) {
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#endif // #if defined(CUTLASS_ARCH_MMA_SM90_SUPPORTED)
|
||||
#endif // #if defined(CUTLASS_ARCH_MMA_SM90_F64_MMA_ENABLED)
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
|
||||
@@ -46,7 +46,7 @@
|
||||
|
||||
#include "testbed.h"
|
||||
|
||||
#if defined(CUTLASS_ARCH_MMA_SM90_SUPPORTED)
|
||||
#if defined(CUTLASS_ARCH_MMA_SM90_F64_MMA_ENABLED)
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
@@ -220,4 +220,4 @@ TEST(SM90_Device_Gemm_f64n_f64t_f64t_tensor_op_f64, 128x128x16_32x64x16_16x8x4)
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#endif // if defined(CUTLASS_ARCH_MMA_SM90_SUPPORTED)
|
||||
#endif // if defined(CUTLASS_ARCH_MMA_SM90_F64_MMA_ENABLED)
|
||||
|
||||
@@ -46,7 +46,7 @@
|
||||
|
||||
#include "testbed.h"
|
||||
|
||||
#if defined(CUTLASS_ARCH_MMA_SM90_SUPPORTED)
|
||||
#if defined(CUTLASS_ARCH_MMA_SM90_F64_MMA_ENABLED)
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
@@ -220,4 +220,4 @@ TEST(SM90_Device_Gemm_f64t_f64n_f64t_tensor_op_f64, 128x128x16_32x64x16_16x8x4)
|
||||
}
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#endif // if (CUTLASS_ARCH_MMA_SM90_SUPPORTED)
|
||||
#endif // if (CUTLASS_ARCH_MMA_SM90_F64_MMA_ENABLED)
|
||||
|
||||
@@ -0,0 +1,717 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017 - 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/*! \file
|
||||
\brief Tests for device-wide GEMM interface
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <iostream>
|
||||
#include <fstream>
|
||||
#include <sstream>
|
||||
|
||||
#include "../../common/cutlass_unit_test.h"
|
||||
|
||||
#include "cutlass/util/host_tensor.h"
|
||||
#include "cutlass/util/tensor_view_io.h"
|
||||
#include "cutlass/util/distribution.h"
|
||||
#include "cutlass/util/packed_stride.hpp"
|
||||
#include "cutlass/util/reference/host/tensor_fill.h"
|
||||
#include "cutlass/util/reference/host/tensor_copy.h"
|
||||
#include "cutlass/util/reference/host/tensor_compare.h"
|
||||
#include "cutlass/util/reference/host/tensor_norm.h"
|
||||
#include "cutlass/util/reference/host/gett.hpp"
|
||||
|
||||
#include "testbed_utils.h"
|
||||
|
||||
#include "cutlass/kernel_hardware_info.hpp"
|
||||
#include "cutlass/layout/matrix.h"
|
||||
#include "cutlass/matrix_coord.h"
|
||||
#include "cutlass/gemm/gemm.h"
|
||||
|
||||
#include "cute/int_tuple.hpp"
|
||||
|
||||
namespace test {
|
||||
namespace gemm {
|
||||
namespace device {
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
namespace detail{
|
||||
|
||||
template <typename Gemm>
|
||||
struct TestbedImpl {
|
||||
// Kernel data types
|
||||
using ElementA = typename Gemm::GemmKernel::ElementA;
|
||||
using StrideA = typename Gemm::GemmKernel::StrideA;
|
||||
using ElementB = typename Gemm::GemmKernel::ElementB;
|
||||
using StrideB = typename Gemm::GemmKernel::StrideB;
|
||||
using ElementC = typename Gemm::GemmKernel::ElementC;
|
||||
using StrideC = typename Gemm::GemmKernel::StrideC;
|
||||
using ElementD = typename Gemm::GemmKernel::ElementD;
|
||||
using StrideD = typename Gemm::GemmKernel::StrideD;
|
||||
using ElementAccumulator = typename Gemm::GemmKernel::ElementAccumulator;
|
||||
using ElementCompute = typename Gemm::GemmKernel::CollectiveEpilogue::ElementCompute;
|
||||
using ElementScalar = typename Gemm::GemmKernel::CollectiveEpilogue::ElementScalar;
|
||||
using ProblemShapeType = typename Gemm::GemmKernel::ProblemShape;
|
||||
|
||||
static_assert(rank(StrideC{}) == 3, "StrideCD must be rank-3: [M, N, L]");
|
||||
static_assert(rank(StrideD{}) == 3, "StrideCD must be rank-3: [M, N, L]");
|
||||
|
||||
// Looks at Cute Stride to check Row / Column Major
|
||||
template<typename Stride>
|
||||
static constexpr bool is_row_or_col_major(){
|
||||
int stride_0 = int(cute::size<0>(Stride{}));
|
||||
int stride_1 = int(cute::size<1>(Stride{}));
|
||||
int depth = cute::depth(Stride{});
|
||||
return ((stride_0 == 1) || (stride_1 == 1)) && (depth == 1);
|
||||
}
|
||||
|
||||
// Note: this limitation comes from testbed / not the library
|
||||
static_assert(is_row_or_col_major<StrideA>(),
|
||||
"ERROR : A Layout is neither Row / Column Major)");
|
||||
static_assert(is_row_or_col_major<StrideB>(),
|
||||
"ERROR : B Layout is neither Row / Column Major)");
|
||||
static_assert(is_row_or_col_major<StrideC>(),
|
||||
"ERROR : C Layout is neither Row / Column Major)");
|
||||
static_assert(is_row_or_col_major<StrideD>(),
|
||||
"ERROR : D Layout is neither Row / Column Major)");
|
||||
|
||||
// Deduce Cutlass Layouts (RowMajor & ColumnMajor)
|
||||
using LayoutTagA = decltype(cutlass::gemm::detail::stride_to_layout_tag_A<StrideA>());
|
||||
using LayoutTagB = decltype(cutlass::gemm::detail::stride_to_layout_tag_B<StrideB>());
|
||||
using LayoutTagC = decltype(cutlass::gemm::detail::stride_to_layout_tag_A<StrideC>());
|
||||
using LayoutTagD = decltype(cutlass::gemm::detail::stride_to_layout_tag_A<StrideD>());
|
||||
using LayoutTagPackedVector = cutlass::layout::PackedVectorLayout;
|
||||
|
||||
/// Initialization
|
||||
StrideA stride_a;
|
||||
StrideB stride_b;
|
||||
StrideC stride_c;
|
||||
StrideD stride_d;
|
||||
typename LayoutTagA::Stride stride_factor_A;
|
||||
typename LayoutTagB::Stride stride_factor_B;
|
||||
typename LayoutTagC::Stride stride_factor_C;
|
||||
typename LayoutTagD::Stride stride_factor_D;
|
||||
cutlass::Distribution::Kind init_A;
|
||||
cutlass::Distribution::Kind init_B;
|
||||
cutlass::Distribution::Kind init_C;
|
||||
uint64_t seed;
|
||||
static constexpr uint64_t kDefaultSeed = 4096;
|
||||
|
||||
cutlass::HostTensor<ElementA, LayoutTagA> tensor_A;
|
||||
cutlass::HostTensor<ElementB, LayoutTagB> tensor_B;
|
||||
cutlass::HostTensor<ElementC, LayoutTagC> tensor_C;
|
||||
cutlass::HostTensor<ElementD, LayoutTagD> tensor_D;
|
||||
cutlass::HostTensor<ElementD, LayoutTagD> reference_D;
|
||||
uint32_t sm_count;
|
||||
|
||||
// Used to force multi-wave tests for persistent kernel schedules
|
||||
constexpr static int MaxSmCount = 16;
|
||||
|
||||
//
|
||||
// Methods
|
||||
//
|
||||
|
||||
TestbedImpl(
|
||||
cutlass::Distribution::Kind init_A_ = cutlass::Distribution::Uniform,
|
||||
cutlass::Distribution::Kind init_B_ = cutlass::Distribution::Uniform,
|
||||
cutlass::Distribution::Kind init_C_ = cutlass::Distribution::Uniform,
|
||||
uint64_t seed_ = kDefaultSeed
|
||||
):
|
||||
stride_factor_A(typename LayoutTagA::Stride()),
|
||||
stride_factor_B(typename LayoutTagB::Stride()),
|
||||
stride_factor_C(typename LayoutTagC::Stride()),
|
||||
stride_factor_D(typename LayoutTagD::Stride()),
|
||||
init_A(init_A_), init_B(init_B_), init_C(init_C_), seed(seed_) { }
|
||||
|
||||
TestbedImpl(
|
||||
typename LayoutTagA::Stride stride_factor_A_,
|
||||
typename LayoutTagB::Stride stride_factor_B_,
|
||||
typename LayoutTagC::Stride stride_factor_C_,
|
||||
typename LayoutTagD::Stride stride_factor_D_,
|
||||
cutlass::Distribution::Kind init_A_ = cutlass::Distribution::Uniform,
|
||||
cutlass::Distribution::Kind init_B_ = cutlass::Distribution::Uniform,
|
||||
cutlass::Distribution::Kind init_C_ = cutlass::Distribution::Uniform,
|
||||
uint64_t seed_ = kDefaultSeed
|
||||
):
|
||||
stride_factor_A(stride_factor_A_),
|
||||
stride_factor_B(stride_factor_B_),
|
||||
stride_factor_C(stride_factor_C_),
|
||||
stride_factor_D(stride_factor_D_),
|
||||
init_A(init_A_), init_B(init_B_), init_C(init_C_), seed(seed_) { }
|
||||
|
||||
/// Helper to initialize a tensor view
|
||||
template <typename Element, typename Layout>
|
||||
bool initialize_tensor(
|
||||
cutlass::TensorView<Element, Layout> view,
|
||||
cutlass::Distribution::Kind dist_kind,
|
||||
uint64_t seed) {
|
||||
|
||||
if (dist_kind == cutlass::Distribution::Uniform) {
|
||||
double scope_max, scope_min;
|
||||
int bits_input = cutlass::sizeof_bits<Element>::value;
|
||||
int bits_output = cutlass::sizeof_bits<ElementD>::value;
|
||||
|
||||
if (bits_input == 1) {
|
||||
scope_max = 2;
|
||||
scope_min = 0;
|
||||
}
|
||||
else if (bits_input <= 8) {
|
||||
scope_max = 2;
|
||||
scope_min = -2;
|
||||
}
|
||||
else if (bits_output == 16) {
|
||||
scope_max = 5;
|
||||
scope_min = -5;
|
||||
}
|
||||
else {
|
||||
scope_max = 8;
|
||||
scope_min = -8;
|
||||
}
|
||||
cutlass::reference::host::TensorFillRandomUniform(
|
||||
view, seed, scope_max, scope_min, 0);
|
||||
}
|
||||
|
||||
else if (dist_kind == cutlass::Distribution::Identity) {
|
||||
cutlass::reference::host::TensorFillIdentity(view);
|
||||
}
|
||||
|
||||
else if (dist_kind == cutlass::Distribution::Gaussian) {
|
||||
cutlass::reference::host::TensorFillRandomGaussian(view, seed, 0, 0.5);
|
||||
}
|
||||
|
||||
else if (dist_kind == cutlass::Distribution::Sequential) {
|
||||
cutlass::reference::host::BlockFillSequential(
|
||||
view.data(), view.capacity());
|
||||
}
|
||||
|
||||
else {
|
||||
EXPECT_TRUE(false) << "Not implemented";
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/// Initializes data structures
|
||||
void initialize(ProblemShapeType problem_size) {
|
||||
//
|
||||
// Allocate the GEMM workspace
|
||||
//
|
||||
auto problem_shape_MNKL = cute::append<4>(problem_size, 1);
|
||||
auto M = cute::size<0>(problem_shape_MNKL);
|
||||
auto N = cute::size<1>(problem_shape_MNKL);
|
||||
auto K = cute::size<2>(problem_shape_MNKL);
|
||||
auto L = cute::size<3>(problem_shape_MNKL);
|
||||
|
||||
stride_a = make_cute_packed_stride(StrideA{}, cute::make_shape(M, K, L));
|
||||
stride_b = make_cute_packed_stride(StrideB{}, cute::make_shape(N, K, L));
|
||||
stride_c = make_cute_packed_stride(StrideC{}, cute::make_shape(M, N, L));
|
||||
stride_d = make_cute_packed_stride(StrideD{}, cute::make_shape(M, N, L));
|
||||
|
||||
// 2.x host tensor does not natively contain a batch stride or coord, so we spoof if by folding it into the outer mode
|
||||
auto a_coord = cutlass::make_Coord(M * L, K);
|
||||
auto c_coord = cutlass::make_Coord(M * L, N);
|
||||
// Cutlass has Row/Col major refers to MxK times KxN matrix product,
|
||||
// so the HostTensorB should be treated as KxN in "coord"'s view
|
||||
auto b_coord = cutlass::make_Coord(K, N * L);
|
||||
|
||||
|
||||
tensor_A.resize(a_coord, cutlass::layout::Affine2Layout_Factory<LayoutTagA>::layout_factory(a_coord, stride_factor_A));
|
||||
tensor_B.resize(b_coord, cutlass::layout::Affine2Layout_Factory<LayoutTagB>::layout_factory(b_coord, stride_factor_B));
|
||||
tensor_C.resize(c_coord, cutlass::layout::Affine2Layout_Factory<LayoutTagC>::layout_factory(c_coord, stride_factor_C));
|
||||
tensor_D.resize(c_coord, cutlass::layout::Affine2Layout_Factory<LayoutTagD>::layout_factory(c_coord, stride_factor_D));
|
||||
reference_D.resize(c_coord, cutlass::layout::Affine2Layout_Factory<LayoutTagD>::layout_factory(c_coord, stride_factor_D), false);
|
||||
|
||||
EXPECT_TRUE(initialize_tensor(tensor_A.host_view(), init_A, seed + 2022));
|
||||
EXPECT_TRUE(initialize_tensor(tensor_B.host_view(), init_B, seed + 2021));
|
||||
EXPECT_TRUE(initialize_tensor(tensor_C.host_view(), init_C, seed + 2020));
|
||||
|
||||
// It is possible to randomly initialize to all zeros, so override this with non-zeros
|
||||
// in the upper left corner of each operand.
|
||||
tensor_A.host_view().at({0, 0}) = ElementA(1);
|
||||
tensor_B.host_view().at({0, 0}) = ElementB(1);
|
||||
tensor_C.host_view().at(cutlass::make_Coord(0, 0)) = ElementC(1);
|
||||
|
||||
cutlass::reference::host::TensorCopy(reference_D.host_view(), tensor_C.host_view());
|
||||
|
||||
tensor_A.sync_device();
|
||||
tensor_B.sync_device();
|
||||
tensor_C.sync_device();
|
||||
tensor_D.sync_device();
|
||||
}
|
||||
|
||||
/// Compares computed reference with device reference and outputs to a file if incorrect
|
||||
bool compare_reference(
|
||||
cute::Shape<int,int,int,int> problem_shape_MNKL,
|
||||
ElementScalar alpha,
|
||||
ElementScalar beta
|
||||
) {
|
||||
auto [M, N, K, L] = problem_shape_MNKL;
|
||||
|
||||
tensor_D.sync_host();
|
||||
EXPECT_GT(cutlass::reference::host::TensorNorm(tensor_A.host_view()), 0);
|
||||
EXPECT_GT(cutlass::reference::host::TensorNorm(tensor_B.host_view()), 0);
|
||||
EXPECT_GT(cutlass::reference::host::TensorNorm(tensor_C.host_view()), 0);
|
||||
|
||||
if (tensor_D.size() > 1) {
|
||||
EXPECT_GT(cutlass::reference::host::TensorNorm(tensor_D.host_view()), 0);
|
||||
}
|
||||
|
||||
if (reference_D.size() > 1) {
|
||||
EXPECT_GT(cutlass::reference::host::TensorNorm(reference_D.host_view()), 0);
|
||||
}
|
||||
|
||||
bool passed = cutlass::reference::host::TensorEquals(reference_D.host_view(), tensor_D.host_view());
|
||||
|
||||
EXPECT_TRUE(passed);
|
||||
if (!passed) {
|
||||
std::stringstream fname;
|
||||
fname << "error_Gemm_device_"
|
||||
<< M << "x" << N << "x" << K << "x" << L << "_"
|
||||
<< cute::get<0>(typename Gemm::GemmKernel::TileShape{}) << "_"
|
||||
<< cute::get<1>(typename Gemm::GemmKernel::TileShape{}) << "_"
|
||||
<< cute::get<2>(typename Gemm::GemmKernel::TileShape{}) << ".txt";
|
||||
|
||||
std::ofstream file(fname.str());
|
||||
file
|
||||
<< "problem: " << ' ' << M << "x" << N << "x" << K << ", Batch count = " << L
|
||||
<< ", alpha: " << float(alpha) << ", beta: " << float(beta) << "\n\n";
|
||||
|
||||
file
|
||||
<< "A =\n" << tensor_A.host_view()
|
||||
<< "\nB =\n" << tensor_B.host_view()
|
||||
<< "\nC =\n" << tensor_C.host_view()
|
||||
<< "\n\nReference =\n" << reference_D.host_view()
|
||||
<< "\n\nComputed =\n" << tensor_D.host_view();
|
||||
}
|
||||
|
||||
return passed;
|
||||
}
|
||||
|
||||
/// Verifies the result is a GEMM
|
||||
bool verify(
|
||||
ProblemShapeType problem_size,
|
||||
ElementScalar alpha,
|
||||
ElementScalar beta
|
||||
) {
|
||||
auto problem_shape_MNKL = cute::append<4>(problem_size, 1);
|
||||
auto M = cute::size<0>(problem_shape_MNKL);
|
||||
auto N = cute::size<1>(problem_shape_MNKL);
|
||||
auto K = cute::size<2>(problem_shape_MNKL);
|
||||
auto L = cute::size<3>(problem_shape_MNKL);
|
||||
|
||||
auto A = cute::make_tensor(tensor_A.host_data(),
|
||||
cute::make_layout(cute::make_shape(M, K, L), stride_a));
|
||||
auto B = cute::make_tensor(tensor_B.host_data(),
|
||||
cute::make_layout(cute::make_shape(N, K, L), stride_b));
|
||||
auto C = cute::make_tensor(tensor_C.host_data(),
|
||||
cute::make_layout(cute::make_shape(M, N, L), stride_c));
|
||||
auto D = cute::make_tensor(reference_D.host_data(),
|
||||
cute::make_layout(cute::make_shape(M, N, L), stride_d));
|
||||
cutlass::reference::host::GettMainloopParams<ElementAccumulator, decltype(A), decltype(B)> mainloop_params{A, B};
|
||||
|
||||
cutlass::reference::host::GettEpilogueParams<
|
||||
ElementScalar,
|
||||
ElementAccumulator,
|
||||
ElementCompute,
|
||||
decltype(C),
|
||||
decltype(D)
|
||||
>
|
||||
epilogue_params{
|
||||
alpha, beta,
|
||||
C, D
|
||||
};
|
||||
|
||||
cutlass::reference::host::Gemm3x(mainloop_params, epilogue_params);
|
||||
|
||||
return compare_reference(
|
||||
problem_shape_MNKL, alpha, beta
|
||||
);
|
||||
}
|
||||
|
||||
/// Determine if the CUDA device is sufficient to run the kernel
|
||||
bool sufficient() {
|
||||
//
|
||||
// Determine SMEM requirements and waive if not satisfied
|
||||
//
|
||||
|
||||
int smem_size = Gemm::GemmKernel::SharedStorageSize;
|
||||
|
||||
int device_idx;
|
||||
cudaError_t result = cudaGetDevice(&device_idx);
|
||||
|
||||
if (result != cudaSuccess) {
|
||||
throw std::runtime_error("cudaGetDevice() API call failed.");
|
||||
}
|
||||
|
||||
cudaDeviceProp properties;
|
||||
result = cudaGetDeviceProperties(&properties, device_idx);
|
||||
this->sm_count = properties.multiProcessorCount;
|
||||
|
||||
if (result != cudaSuccess) {
|
||||
throw std::runtime_error("cudaGetDeviceProperties() failed");
|
||||
}
|
||||
|
||||
if (properties.sharedMemPerBlockOptin < smem_size) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
bool profile(
|
||||
ProblemShapeType problem_size,
|
||||
int iterations,
|
||||
Gemm& gemm_op,
|
||||
typename Gemm::Arguments& arguments,
|
||||
cutlass::device_memory::allocation<uint8_t>& workspace) {
|
||||
int M = cute::size<0>(problem_size);
|
||||
int N = cute::size<1>(problem_size);
|
||||
int K = cute::size<2>(problem_size);
|
||||
int L = 1;
|
||||
if constexpr(cute::rank(ProblemShapeType{}) == 4) {
|
||||
L = cute::size<3>(problem_size);
|
||||
}
|
||||
|
||||
|
||||
cutlass::Status status;
|
||||
//
|
||||
// Run the GEMM
|
||||
//
|
||||
cudaError_t result;
|
||||
|
||||
for (int iter = 0; iter < iterations; ++iter) {
|
||||
status = gemm_op(arguments, workspace.get());
|
||||
if (status != cutlass::Status::kSuccess) {
|
||||
EXPECT_TRUE(status == cutlass::Status::kSuccess) << to_string(status);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
result = cudaDeviceSynchronize();
|
||||
if (result != cudaSuccess) {
|
||||
EXPECT_EQ(result, cudaSuccess) << "Error at Kernel Sync.";
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/// Executes one test
|
||||
bool run(
|
||||
ProblemShapeType problem_size,
|
||||
ElementScalar alpha = ElementScalar(1),
|
||||
ElementScalar beta = ElementScalar(0),
|
||||
bool profiling = false,
|
||||
int iterations = 20
|
||||
) {
|
||||
// Fail test if insufficient CUDA device
|
||||
if (!sufficient()) {
|
||||
std::cout << "Test failed due to insufficient CUDA device." << std::endl;
|
||||
return false;
|
||||
}
|
||||
|
||||
this->initialize(problem_size);
|
||||
|
||||
//
|
||||
// Initialize the GEMM operator
|
||||
//
|
||||
|
||||
typename Gemm::Arguments arguments;
|
||||
cutlass::KernelHardwareInfo hw_info;
|
||||
hw_info.device_id = 0;
|
||||
if (not profiling) {
|
||||
this->sm_count = min(MaxSmCount, cutlass::KernelHardwareInfo::query_device_multiprocessor_count(hw_info.device_id));
|
||||
hw_info.sm_count = this->sm_count;
|
||||
}
|
||||
else {
|
||||
this->sm_count = cutlass::KernelHardwareInfo::query_device_multiprocessor_count(hw_info.device_id);
|
||||
hw_info.sm_count = this->sm_count;
|
||||
}
|
||||
|
||||
// DefaultEpilogue
|
||||
arguments = typename Gemm::Arguments{
|
||||
cutlass::gemm::GemmUniversalMode::kGemm,
|
||||
problem_size,
|
||||
tensor_A.device_data(),
|
||||
stride_a,
|
||||
tensor_B.device_data(),
|
||||
stride_b,
|
||||
{tensor_C.device_data(), stride_c, tensor_D.device_data(), stride_d, {alpha, beta}},
|
||||
hw_info
|
||||
};
|
||||
Gemm gemm_op;
|
||||
|
||||
size_t workspace_size = Gemm::get_workspace_size(arguments);
|
||||
cutlass::device_memory::allocation<uint8_t> workspace(workspace_size);
|
||||
|
||||
cutlass::Status status = gemm_op.can_implement(arguments);
|
||||
|
||||
if (status != cutlass::Status::kSuccess) {
|
||||
cudaError_t error = cudaGetLastError();
|
||||
std::cerr << "This test is not supported: " << cudaGetErrorString(error) << "\n";
|
||||
return true;
|
||||
}
|
||||
|
||||
//
|
||||
// Run the GEMM
|
||||
//
|
||||
|
||||
if (profiling) {
|
||||
return profile(problem_size, iterations, gemm_op, arguments, workspace);
|
||||
}
|
||||
else {
|
||||
cudaError_t result;
|
||||
status = gemm_op.initialize(arguments, workspace.get());
|
||||
status = gemm_op.run();
|
||||
result = cudaDeviceSynchronize();
|
||||
if (result != cudaSuccess) {
|
||||
EXPECT_EQ(result, cudaSuccess) << "Error at Kernel Sync.";
|
||||
return false;
|
||||
}
|
||||
|
||||
EXPECT_TRUE(status == cutlass::Status::kSuccess) << to_string(status);
|
||||
|
||||
//
|
||||
// Verify
|
||||
//
|
||||
bool passed = this->verify(
|
||||
problem_size, alpha, beta
|
||||
);
|
||||
if (!passed) {
|
||||
std::cout << "Error : Failed : with alpha: " << float(alpha) << ", beta: " << float(beta)
|
||||
<< "\n";
|
||||
}
|
||||
|
||||
return passed;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace detail
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
template <typename Gemm>
|
||||
struct Testbed {
|
||||
|
||||
using TestBedImplementation = typename detail::TestbedImpl<Gemm>;
|
||||
|
||||
using ElementAccumulator = typename Gemm::GemmKernel::ElementAccumulator;
|
||||
using ElementCompute = typename Gemm::GemmKernel::CollectiveEpilogue::ElementCompute;
|
||||
using ElementScalar = typename Gemm::GemmKernel::CollectiveEpilogue::ElementScalar;
|
||||
using LayoutTagA = typename TestBedImplementation::LayoutTagA;
|
||||
using LayoutTagB = typename TestBedImplementation::LayoutTagB;
|
||||
using LayoutTagC = typename TestBedImplementation::LayoutTagC;
|
||||
using LayoutTagD = typename TestBedImplementation::LayoutTagD;
|
||||
|
||||
// Detail Implementation
|
||||
TestBedImplementation impl_;
|
||||
|
||||
//
|
||||
// Methods
|
||||
//
|
||||
Testbed(
|
||||
cutlass::Distribution::Kind init_A_ = cutlass::Distribution::Uniform,
|
||||
cutlass::Distribution::Kind init_B_ = cutlass::Distribution::Uniform,
|
||||
cutlass::Distribution::Kind init_C_ = cutlass::Distribution::Uniform,
|
||||
uint64_t seed_ = TestBedImplementation::kDefaultSeed)
|
||||
: impl_(init_A_, init_B_, init_C_, seed_) {}
|
||||
|
||||
Testbed(
|
||||
typename LayoutTagA::Stride stride_factor_A_,
|
||||
typename LayoutTagB::Stride stride_factor_B_,
|
||||
typename LayoutTagC::Stride stride_factor_C_,
|
||||
typename LayoutTagD::Stride stride_factor_D_,
|
||||
cutlass::Distribution::Kind init_A_ = cutlass::Distribution::Uniform,
|
||||
cutlass::Distribution::Kind init_B_ = cutlass::Distribution::Uniform,
|
||||
cutlass::Distribution::Kind init_C_ = cutlass::Distribution::Uniform,
|
||||
uint64_t seed_ = TestBedImplementation::kDefaultSeed)
|
||||
: impl_(stride_factor_A_,
|
||||
stride_factor_B_,
|
||||
stride_factor_C_,
|
||||
stride_factor_D_,
|
||||
init_A_,
|
||||
init_B_,
|
||||
init_C_,
|
||||
seed_) {}
|
||||
|
||||
/// Executes one test
|
||||
bool run(
|
||||
typename TestBedImplementation::ProblemShapeType problem_size,
|
||||
ElementScalar alpha = ElementScalar(1),
|
||||
ElementScalar beta = ElementScalar(0),
|
||||
bool profiling = false,
|
||||
int iterations = 20
|
||||
) {
|
||||
return impl_.run(
|
||||
problem_size, alpha, beta, profiling, iterations
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
template <typename Gemm>
|
||||
bool TestAll() {
|
||||
using ElementScalar = typename Gemm::GemmKernel::CollectiveEpilogue::ElementScalar;
|
||||
using ProblemShapeType = typename Gemm::GemmKernel::ProblemShape;
|
||||
|
||||
int max_alignment = std::max(Gemm::kAlignmentA, Gemm::kAlignmentB);
|
||||
std::vector<int> problem_size_m = {max_alignment, 512 - 3 * max_alignment};
|
||||
std::vector<int> problem_size_n = {max_alignment, 512 - 2 * max_alignment};
|
||||
|
||||
if constexpr (std::is_same_v<typename Gemm::GemmKernel::DispatchPolicy::Schedule,
|
||||
cutlass::gemm::KernelTmaWarpSpecializedPersistent>) {
|
||||
problem_size_m.push_back(768);
|
||||
problem_size_n.push_back(768);
|
||||
}
|
||||
|
||||
constexpr int Stages = Gemm::GemmKernel::DispatchPolicy::Stages;
|
||||
constexpr int TileShapeK = cute::size<2>(typename Gemm::GemmKernel::TileShape{});
|
||||
|
||||
std::vector<int> problem_size_k = {max_alignment, TileShapeK * (Stages + 1) - max_alignment};
|
||||
|
||||
Testbed<Gemm> testbed;
|
||||
bool passed = true;
|
||||
|
||||
for (int m : problem_size_m) {
|
||||
for (int n : problem_size_n) {
|
||||
for (int k : problem_size_k) {
|
||||
ProblemShapeType problem_size;
|
||||
if constexpr (cute::rank(ProblemShapeType{}) == 4) {
|
||||
problem_size = ProblemShapeType{m, n, k, /* l */ 1};
|
||||
}
|
||||
else {
|
||||
problem_size = ProblemShapeType{m, n, k};
|
||||
}
|
||||
|
||||
passed = testbed.run(
|
||||
problem_size,
|
||||
cutlass::from_real<ElementScalar>(1),
|
||||
cutlass::from_real<ElementScalar>(0)
|
||||
);
|
||||
|
||||
if (!passed) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// if we do support batched GEMM, just run one test on it to save on test time
|
||||
if constexpr (cute::rank(ProblemShapeType{}) == 4) {
|
||||
auto problem_size = ProblemShapeType{256 + max_alignment, 256 + max_alignment, 160 + max_alignment, /* l */ 3};
|
||||
passed = testbed.run(
|
||||
problem_size,
|
||||
cutlass::from_real<ElementScalar>(1),
|
||||
cutlass::from_real<ElementScalar>(0)
|
||||
);
|
||||
|
||||
if (!passed) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return passed;
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
template <typename Gemm>
|
||||
bool TestGemmPerf(int iterations = 20) {
|
||||
using ProblemShapeType = typename Gemm::GemmKernel::ProblemShape;
|
||||
using ElementAccumulator = typename Gemm::GemmKernel::ElementAccumulator;
|
||||
using ElementScalar = ElementAccumulator;
|
||||
bool passed = true;
|
||||
|
||||
std::vector<int> problem_size_m = { 4608 };
|
||||
std::vector<int> problem_size_n = { 4608 };
|
||||
std::vector<int> problem_size_k = { 8192 };
|
||||
|
||||
Testbed<Gemm> testbed;
|
||||
|
||||
for (int m : problem_size_m) {
|
||||
for (int n : problem_size_n) {
|
||||
for (int k : problem_size_k) {
|
||||
ProblemShapeType problem_size;
|
||||
if constexpr (cute::rank(ProblemShapeType{}) == 4) {
|
||||
problem_size = ProblemShapeType{m, n, k, /* l */ 1};
|
||||
}
|
||||
else {
|
||||
problem_size = ProblemShapeType{m, n, k};
|
||||
}
|
||||
|
||||
passed = testbed.run(
|
||||
problem_size,
|
||||
cutlass::from_real<ElementScalar>(1),
|
||||
cutlass::from_real<ElementScalar>(0),
|
||||
true,
|
||||
iterations
|
||||
);
|
||||
|
||||
if (!passed) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// if we do support batched GEMM, just run it once
|
||||
if constexpr (cute::rank(ProblemShapeType{}) == 4) {
|
||||
auto problem_size = ProblemShapeType{problem_size_m[0], problem_size_n[0], problem_size_k[0], /* l */ 4};
|
||||
passed = testbed.run(
|
||||
problem_size,
|
||||
cutlass::from_real<ElementScalar>(1),
|
||||
cutlass::from_real<ElementScalar>(0),
|
||||
true,
|
||||
iterations
|
||||
);
|
||||
|
||||
if (!passed) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return passed;
|
||||
}
|
||||
|
||||
|
||||
} // namespace device
|
||||
} // namespace gemm
|
||||
} // namespace test
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
@@ -48,7 +48,7 @@
|
||||
|
||||
#include "testbed_symm_universal.h"
|
||||
|
||||
#if defined(CUTLASS_ARCH_MMA_SM90_SUPPORTED)
|
||||
#if defined(CUTLASS_ARCH_MMA_SM90_F64_MMA_ENABLED)
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
@@ -132,4 +132,4 @@ TEST(SM90_Device_Hemm_cf64h_cf64n_rs_u_tensor_op_f64, 64x64x16_32x32x16) {
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#endif // #if defined(CUTLASS_ARCH_MMA_SM90_SUPPORTED)
|
||||
#endif // #if defined(CUTLASS_ARCH_MMA_SM90_F64_MMA_ENABLED)
|
||||
|
||||
@@ -46,7 +46,7 @@
|
||||
|
||||
#include "testbed_rank2k_universal.h"
|
||||
|
||||
#if defined(CUTLASS_ARCH_MMA_SM90_SUPPORTED)
|
||||
#if defined(CUTLASS_ARCH_MMA_SM90_F64_MMA_ENABLED)
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
@@ -146,4 +146,4 @@ TEST(SM90_Device_Her2k_cf64c_cf64n_u_tensor_op_f64, 32x32x16_16x16x16) {
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#endif // #if defined(CUTLASS_ARCH_MMA_SM90_SUPPORTED)
|
||||
#endif // #if defined(CUTLASS_ARCH_MMA_SM90_F64_MMA_ENABLED)
|
||||
|
||||
@@ -46,7 +46,7 @@
|
||||
|
||||
#include "testbed_rank_k_universal.h"
|
||||
|
||||
#if defined(CUTLASS_ARCH_MMA_SM90_SUPPORTED)
|
||||
#if defined(CUTLASS_ARCH_MMA_SM90_F64_MMA_ENABLED)
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// HERK operator on CUBLAS_OP_C (row-major + conj) input layouts
|
||||
@@ -90,4 +90,4 @@ TEST(SM90_Device_Herk_cf64h_cf64n_l_tensor_op_f64, 64x64x16_32x32x16) {
|
||||
}
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#endif // #if defined(CUTLASS_ARCH_MMA_SM90_SUPPORTED)
|
||||
#endif // #if defined(CUTLASS_ARCH_MMA_SM90_F64_MMA_ENABLED)
|
||||
|
||||
@@ -58,6 +58,11 @@ namespace device {
|
||||
|
||||
template <typename Gemm>
|
||||
struct MultistageTestbed {
|
||||
|
||||
using ElementA = typename Gemm::ElementA;
|
||||
using ElementB = typename Gemm::ElementB;
|
||||
using ElementC = typename Gemm::ElementC;
|
||||
|
||||
using ElementAccumulator = typename Gemm::ElementAccumulator;
|
||||
using ElementCompute =
|
||||
typename Gemm::GemmKernel::Epilogue::OutputOp::ElementCompute;
|
||||
|
||||
@@ -59,6 +59,9 @@ namespace device {
|
||||
template <typename Gemm, int InterleavedK>
|
||||
struct MultistageInterleavedTestbed {
|
||||
|
||||
using ElementA = typename Gemm::ElementA;
|
||||
using ElementB = typename Gemm::ElementB;
|
||||
using ElementC = typename Gemm::ElementC;
|
||||
using ElementAccumulator = typename Gemm::ElementAccumulator;
|
||||
using ElementCompute = typename Gemm::GemmKernel::Epilogue::OutputOp::ElementCompute;
|
||||
|
||||
@@ -110,12 +113,49 @@ struct MultistageInterleavedTestbed {
|
||||
return true;
|
||||
}
|
||||
|
||||
/// Returns true if the CUDA device is sufficient to execute the kernel.
|
||||
bool sufficient() const {
|
||||
//
|
||||
// Determine SMEM requirements and waive if not satisfied
|
||||
//
|
||||
|
||||
int smem_size = int(sizeof(typename Gemm::GemmKernel::SharedStorage));
|
||||
|
||||
cudaDeviceProp properties;
|
||||
int device_idx;
|
||||
cudaError_t result = cudaGetDevice(&device_idx);
|
||||
|
||||
if (result != cudaSuccess) {
|
||||
throw std::runtime_error("cudaGetDevice() API call failed.");
|
||||
}
|
||||
|
||||
result = cudaGetDeviceProperties(&properties, device_idx);
|
||||
|
||||
if (result != cudaSuccess) {
|
||||
throw std::runtime_error("cudaGetDeviceProperties() failed");
|
||||
}
|
||||
|
||||
if (properties.sharedMemPerMultiprocessor < smem_size) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/// Executes one test
|
||||
bool run(
|
||||
cutlass::gemm::GemmCoord problem_size,
|
||||
ElementCompute alpha = ElementCompute(1),
|
||||
ElementCompute beta = ElementCompute(0)) {
|
||||
|
||||
// Waive test if insufficient CUDA device
|
||||
if (!sufficient()) {
|
||||
if (CUTLASS_TEST_UNIT_ENABLE_WARNINGS) {
|
||||
std::cerr << "Test waived due to insufficient CUDA device." << std::endl;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
//
|
||||
// Allocate the GEMM workspace
|
||||
//
|
||||
|
||||
@@ -0,0 +1,135 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017 - 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/*! \file
|
||||
\brief Tests for device-wide GEMM interface
|
||||
*/
|
||||
|
||||
#include <iostream>
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cute/tensor.hpp"
|
||||
#include "cute/atom/mma_atom.hpp"
|
||||
|
||||
#include "cutlass/numeric_types.h"
|
||||
|
||||
#include "cutlass/gemm/device/gemm_universal_adapter.h"
|
||||
#include "default_gemm_configuration.hpp"
|
||||
|
||||
#include "../../common/cutlass_unit_test.h"
|
||||
|
||||
#include "gemm_testbed_3x.hpp"
|
||||
|
||||
using namespace cute;
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM50_Device_Gemm_f32n_f32n_f32n_simt_f32, 128x128x64_64x64x64) {
|
||||
using Config = cutlass::gemm::device::DefaultGemmConfigurationToCutlass3Types<
|
||||
cutlass::arch::OpClassSimt, cutlass::arch::Sm50,
|
||||
float, cutlass::layout::ColumnMajor,
|
||||
float, cutlass::layout::ColumnMajor,
|
||||
float, cutlass::layout::ColumnMajor,
|
||||
float>;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
Config::CollectiveMainloop,
|
||||
Config::CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM50_Device_Gemm_f32n_f32t_f32n_simt_f32, 128x128x64_64x64x64) {
|
||||
|
||||
using Config = cutlass::gemm::device::DefaultGemmConfigurationToCutlass3Types<
|
||||
cutlass::arch::OpClassSimt, cutlass::arch::Sm50,
|
||||
float, cutlass::layout::ColumnMajor,
|
||||
float, cutlass::layout::RowMajor,
|
||||
float, cutlass::layout::ColumnMajor,
|
||||
float>;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
Config::CollectiveMainloop,
|
||||
Config::CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM50_Device_Gemm_f32t_f32n_f32n_simt_f32, 128x128x64_64x64x64) {
|
||||
|
||||
using Config = cutlass::gemm::device::DefaultGemmConfigurationToCutlass3Types<
|
||||
cutlass::arch::OpClassSimt, cutlass::arch::Sm50,
|
||||
float, cutlass::layout::RowMajor,
|
||||
float, cutlass::layout::ColumnMajor,
|
||||
float, cutlass::layout::ColumnMajor,
|
||||
float>;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
Config::CollectiveMainloop,
|
||||
Config::CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM50_Device_Gemm_f32t_f32t_f32n_simt_f32, 128x128x64_64x64x64) {
|
||||
|
||||
using Config = cutlass::gemm::device::DefaultGemmConfigurationToCutlass3Types<
|
||||
cutlass::arch::OpClassSimt, cutlass::arch::Sm50,
|
||||
float, cutlass::layout::RowMajor,
|
||||
float, cutlass::layout::RowMajor,
|
||||
float, cutlass::layout::ColumnMajor,
|
||||
float>;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
Config::CollectiveMainloop,
|
||||
Config::CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
@@ -0,0 +1,134 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017 - 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/*! \file
|
||||
\brief Tests for device-wide GEMM interface
|
||||
*/
|
||||
|
||||
#include <iostream>
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cute/tensor.hpp"
|
||||
#include "cute/atom/mma_atom.hpp"
|
||||
|
||||
#include "cutlass/numeric_types.h"
|
||||
|
||||
#include "cutlass/gemm/device/gemm_universal_adapter.h"
|
||||
#include "default_gemm_configuration.hpp"
|
||||
|
||||
#include "../../common/cutlass_unit_test.h"
|
||||
|
||||
#include "gemm_testbed_3x.hpp"
|
||||
|
||||
using namespace cute;
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM50_Device_Gemm_f64n_f64n_f64n_simt_f64, 128x128x64_64x64x64) {
|
||||
using Config = cutlass::gemm::device::DefaultGemmConfigurationToCutlass3Types<
|
||||
cutlass::arch::OpClassSimt, cutlass::arch::Sm50,
|
||||
double, cutlass::layout::ColumnMajor,
|
||||
double, cutlass::layout::ColumnMajor,
|
||||
double, cutlass::layout::ColumnMajor,
|
||||
double>;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
Config::CollectiveMainloop,
|
||||
Config::CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
TEST(SM50_Device_Gemm_f64n_f64t_f64n_simt_f64, 128x128x64_64x64x64) {
|
||||
|
||||
using Config = cutlass::gemm::device::DefaultGemmConfigurationToCutlass3Types<
|
||||
cutlass::arch::OpClassSimt, cutlass::arch::Sm50,
|
||||
double, cutlass::layout::ColumnMajor,
|
||||
double, cutlass::layout::RowMajor,
|
||||
double, cutlass::layout::ColumnMajor,
|
||||
double>;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
Config::CollectiveMainloop,
|
||||
Config::CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM50_Device_Gemm_f64t_f64n_f64n_simt_f64, 128x128x64_64x64x64) {
|
||||
|
||||
using Config = cutlass::gemm::device::DefaultGemmConfigurationToCutlass3Types<
|
||||
cutlass::arch::OpClassSimt, cutlass::arch::Sm50,
|
||||
double, cutlass::layout::RowMajor,
|
||||
double, cutlass::layout::ColumnMajor,
|
||||
double, cutlass::layout::ColumnMajor,
|
||||
double>;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
Config::CollectiveMainloop,
|
||||
Config::CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM50_Device_Gemm_f64t_f64t_f64n_simt_f64, 128x128x64_64x64x64) {
|
||||
|
||||
using Config = cutlass::gemm::device::DefaultGemmConfigurationToCutlass3Types<
|
||||
cutlass::arch::OpClassSimt, cutlass::arch::Sm50,
|
||||
double, cutlass::layout::RowMajor,
|
||||
double, cutlass::layout::RowMajor,
|
||||
double, cutlass::layout::ColumnMajor,
|
||||
double>;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
Config::CollectiveMainloop,
|
||||
Config::CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
@@ -0,0 +1,136 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017 - 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/*! \file
|
||||
\brief Tests for device-wide GEMM interface
|
||||
*/
|
||||
|
||||
#include <iostream>
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cute/tensor.hpp"
|
||||
#include "cute/atom/mma_atom.hpp"
|
||||
|
||||
#include "cutlass/numeric_types.h"
|
||||
|
||||
#include "cutlass/gemm/device/gemm_universal_adapter.h"
|
||||
#include "default_gemm_configuration.hpp"
|
||||
|
||||
#include "../../common/cutlass_unit_test.h"
|
||||
|
||||
#include "gemm_testbed_3x.hpp"
|
||||
|
||||
using namespace cute;
|
||||
|
||||
//#if defined(CUTLASS_ARCH_MMA_SM61_SUPPORTED)
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM61_Device_Gemm_s8n_s8n_s32n_simt_s32, 128x128x64_64x64x64) {
|
||||
using Config = cutlass::gemm::device::DefaultGemmConfigurationToCutlass3Types<
|
||||
cutlass::arch::OpClassSimt, cutlass::arch::Sm50,
|
||||
int8_t, cutlass::layout::ColumnMajor,
|
||||
int8_t, cutlass::layout::ColumnMajor,
|
||||
int32_t, cutlass::layout::ColumnMajor,
|
||||
int32_t>;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
Config::CollectiveMainloop,
|
||||
Config::CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM61_Device_Gemm_s8n_s8t_s32n_simt_s32, 128x128x64_64x64x64) {
|
||||
using Config = cutlass::gemm::device::DefaultGemmConfigurationToCutlass3Types<
|
||||
cutlass::arch::OpClassSimt, cutlass::arch::Sm50,
|
||||
int8_t, cutlass::layout::ColumnMajor,
|
||||
int8_t, cutlass::layout::RowMajor,
|
||||
int32_t, cutlass::layout::ColumnMajor,
|
||||
int32_t>;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
Config::CollectiveMainloop,
|
||||
Config::CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM61_Device_Gemm_s8t_s8n_s32n_simt_s32, 128x128x64_64x64x64) {
|
||||
using Config = cutlass::gemm::device::DefaultGemmConfigurationToCutlass3Types<
|
||||
cutlass::arch::OpClassSimt, cutlass::arch::Sm50,
|
||||
int8_t, cutlass::layout::RowMajor,
|
||||
int8_t, cutlass::layout::ColumnMajor,
|
||||
int32_t, cutlass::layout::ColumnMajor,
|
||||
int32_t>;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
Config::CollectiveMainloop,
|
||||
Config::CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM61_Device_Gemm_s8t_s8t_s32n_simt_s32, 128x128x64_64x64x64) {
|
||||
using Config = cutlass::gemm::device::DefaultGemmConfigurationToCutlass3Types<
|
||||
cutlass::arch::OpClassSimt, cutlass::arch::Sm50,
|
||||
int8_t, cutlass::layout::RowMajor,
|
||||
int8_t, cutlass::layout::RowMajor,
|
||||
int32_t, cutlass::layout::ColumnMajor,
|
||||
int32_t>;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
Config::CollectiveMainloop,
|
||||
Config::CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
//#endif // #if defined(CUTLASS_ARCH_MMA_SM61_SUPPORTED)
|
||||
@@ -0,0 +1,136 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017 - 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/*! \file
|
||||
\brief Tests for device-wide GEMM interface
|
||||
*/
|
||||
|
||||
#include <iostream>
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cute/tensor.hpp"
|
||||
#include "cute/atom/mma_atom.hpp"
|
||||
|
||||
#include "cutlass/numeric_types.h"
|
||||
|
||||
#include "cutlass/gemm/device/gemm_universal_adapter.h"
|
||||
#include "default_gemm_configuration.hpp"
|
||||
|
||||
#include "../../common/cutlass_unit_test.h"
|
||||
|
||||
#include "gemm_testbed_3x.hpp"
|
||||
|
||||
using namespace cute;
|
||||
|
||||
//#if defined(CUTLASS_ARCH_MMA_SM80_SUPPORTED)
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
#if 1
|
||||
TEST(SM80_Device_Gemm_f16t_f16n_f32t_tensor_op_f32_3x, 128x128x32_64x64x32) {
|
||||
using Config = cutlass::gemm::device::DefaultGemmConfigurationToCutlass3Types<
|
||||
cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
|
||||
cutlass::half_t, cutlass::layout::RowMajor,
|
||||
cutlass::half_t, cutlass::layout::ColumnMajor,
|
||||
float, cutlass::layout::RowMajor,
|
||||
float>;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
Config::CollectiveMainloop,
|
||||
Config::CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
|
||||
}
|
||||
#endif
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
#if 1
|
||||
TEST(SM80_Device_Gemm_f16n_f16t_f32t_tensor_op_f32_3x, 128x128x32_64x64x32) {
|
||||
using Config = cutlass::gemm::device::DefaultGemmConfigurationToCutlass3Types<
|
||||
cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
|
||||
cutlass::half_t, cutlass::layout::ColumnMajor,
|
||||
cutlass::half_t, cutlass::layout::RowMajor,
|
||||
float, cutlass::layout::RowMajor,
|
||||
float>;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
Config::CollectiveMainloop,
|
||||
Config::CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM80_Device_Gemm_f16n_f16n_f32t_tensor_op_f32_3x, 128x128x32_64x64x32) {
|
||||
using Config = cutlass::gemm::device::DefaultGemmConfigurationToCutlass3Types<
|
||||
cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
|
||||
cutlass::half_t, cutlass::layout::ColumnMajor,
|
||||
cutlass::half_t, cutlass::layout::ColumnMajor,
|
||||
float, cutlass::layout::RowMajor,
|
||||
float>;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
Config::CollectiveMainloop,
|
||||
Config::CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM80_Device_Gemm_f16t_f16t_f32t_tensor_op_f32_3x, 128x128x32_64x64x32) {
|
||||
using Config = cutlass::gemm::device::DefaultGemmConfigurationToCutlass3Types<
|
||||
cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
|
||||
cutlass::half_t, cutlass::layout::RowMajor,
|
||||
cutlass::half_t, cutlass::layout::RowMajor,
|
||||
float, cutlass::layout::RowMajor,
|
||||
float>;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
Config::CollectiveMainloop,
|
||||
Config::CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
|
||||
}
|
||||
#endif
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
//#endif // #if defined(CUTLASS_ARCH_MMA_SM80_SUPPORTED)
|
||||
@@ -0,0 +1,135 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017 - 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/*! \file
|
||||
\brief Tests for device-wide GEMM interface
|
||||
*/
|
||||
|
||||
#include <iostream>
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cute/tensor.hpp"
|
||||
#include "cute/atom/mma_atom.hpp"
|
||||
|
||||
#include "cutlass/numeric_types.h"
|
||||
|
||||
#include "cutlass/gemm/device/gemm_universal_adapter.h"
|
||||
#include "default_gemm_configuration.hpp"
|
||||
|
||||
#include "../../common/cutlass_unit_test.h"
|
||||
|
||||
#include "gemm_testbed_3x.hpp"
|
||||
|
||||
using namespace cute;
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM80_Device_Gemm_f32n_f32n_f32n_simt_f32, 128x128x64_64x64x64) {
|
||||
using Config = cutlass::gemm::device::DefaultGemmConfigurationToCutlass3Types<
|
||||
cutlass::arch::OpClassSimt, cutlass::arch::Sm80,
|
||||
float, cutlass::layout::ColumnMajor,
|
||||
float, cutlass::layout::ColumnMajor,
|
||||
float, cutlass::layout::ColumnMajor,
|
||||
float>;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
Config::CollectiveMainloop,
|
||||
Config::CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM80_Device_Gemm_f32n_f32t_f32n_simt_f32, 128x128x64_64x64x64) {
|
||||
|
||||
using Config = cutlass::gemm::device::DefaultGemmConfigurationToCutlass3Types<
|
||||
cutlass::arch::OpClassSimt, cutlass::arch::Sm80,
|
||||
float, cutlass::layout::ColumnMajor,
|
||||
float, cutlass::layout::RowMajor,
|
||||
float, cutlass::layout::ColumnMajor,
|
||||
float>;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
Config::CollectiveMainloop,
|
||||
Config::CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM80_Device_Gemm_f32t_f32n_f32n_simt_f32, 128x128x64_64x64x64) {
|
||||
|
||||
using Config = cutlass::gemm::device::DefaultGemmConfigurationToCutlass3Types<
|
||||
cutlass::arch::OpClassSimt, cutlass::arch::Sm80,
|
||||
float, cutlass::layout::RowMajor,
|
||||
float, cutlass::layout::ColumnMajor,
|
||||
float, cutlass::layout::ColumnMajor,
|
||||
float>;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
Config::CollectiveMainloop,
|
||||
Config::CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM80_Device_Gemm_f32t_f32t_f32n_simt_f32, 128x128x64_64x64x64) {
|
||||
|
||||
using Config = cutlass::gemm::device::DefaultGemmConfigurationToCutlass3Types<
|
||||
cutlass::arch::OpClassSimt, cutlass::arch::Sm80,
|
||||
float, cutlass::layout::RowMajor,
|
||||
float, cutlass::layout::RowMajor,
|
||||
float, cutlass::layout::ColumnMajor,
|
||||
float>;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
Config::CollectiveMainloop,
|
||||
Config::CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
@@ -0,0 +1,134 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017 - 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/*! \file
|
||||
\brief Tests for device-wide GEMM interface
|
||||
*/
|
||||
|
||||
#include <iostream>
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cute/tensor.hpp"
|
||||
#include "cute/atom/mma_atom.hpp"
|
||||
|
||||
#include "cutlass/numeric_types.h"
|
||||
|
||||
#include "cutlass/gemm/device/gemm_universal_adapter.h"
|
||||
#include "default_gemm_configuration.hpp"
|
||||
|
||||
#include "../../common/cutlass_unit_test.h"
|
||||
|
||||
#include "gemm_testbed_3x.hpp"
|
||||
|
||||
using namespace cute;
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM80_Device_Gemm_f64n_f64n_f64n_simt_f64, 128x128x64_64x64x64) {
|
||||
using Config = cutlass::gemm::device::DefaultGemmConfigurationToCutlass3Types<
|
||||
cutlass::arch::OpClassSimt, cutlass::arch::Sm80,
|
||||
double, cutlass::layout::ColumnMajor,
|
||||
double, cutlass::layout::ColumnMajor,
|
||||
double, cutlass::layout::ColumnMajor,
|
||||
double>;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
Config::CollectiveMainloop,
|
||||
Config::CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
TEST(SM80_Device_Gemm_f64n_f64t_f64n_simt_f64, 128x128x64_64x64x64) {
|
||||
|
||||
using Config = cutlass::gemm::device::DefaultGemmConfigurationToCutlass3Types<
|
||||
cutlass::arch::OpClassSimt, cutlass::arch::Sm80,
|
||||
double, cutlass::layout::ColumnMajor,
|
||||
double, cutlass::layout::RowMajor,
|
||||
double, cutlass::layout::ColumnMajor,
|
||||
double>;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
Config::CollectiveMainloop,
|
||||
Config::CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM80_Device_Gemm_f64t_f64n_f64n_simt_f64, 128x128x64_64x64x64) {
|
||||
|
||||
using Config = cutlass::gemm::device::DefaultGemmConfigurationToCutlass3Types<
|
||||
cutlass::arch::OpClassSimt, cutlass::arch::Sm80,
|
||||
double, cutlass::layout::RowMajor,
|
||||
double, cutlass::layout::ColumnMajor,
|
||||
double, cutlass::layout::ColumnMajor,
|
||||
double>;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
Config::CollectiveMainloop,
|
||||
Config::CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM80_Device_Gemm_f64t_f64t_f64n_simt_f64, 128x128x64_64x64x64) {
|
||||
|
||||
using Config = cutlass::gemm::device::DefaultGemmConfigurationToCutlass3Types<
|
||||
cutlass::arch::OpClassSimt, cutlass::arch::Sm80,
|
||||
double, cutlass::layout::RowMajor,
|
||||
double, cutlass::layout::RowMajor,
|
||||
double, cutlass::layout::ColumnMajor,
|
||||
double>;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
Config::CollectiveMainloop,
|
||||
Config::CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
@@ -0,0 +1,98 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017 - 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/*! \file
|
||||
\brief Tests for device-wide GEMM interface
|
||||
*/
|
||||
|
||||
#include <iostream>
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cute/tensor.hpp"
|
||||
#include "cute/atom/mma_atom.hpp"
|
||||
|
||||
#include "cutlass/numeric_types.h"
|
||||
|
||||
#include "cutlass/gemm/device/gemm_universal_adapter.h"
|
||||
#include "default_gemm_configuration.hpp"
|
||||
|
||||
#include "../../common/cutlass_unit_test.h"
|
||||
|
||||
#include "gemm_testbed_3x.hpp"
|
||||
|
||||
using namespace cute;
|
||||
|
||||
//#if defined(CUTLASS_ARCH_MMA_SM80_SUPPORTED)
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM80_Device_Gemm_f64n_f64t_f64n_tensor_op_f64, 128x128x64_64x64x64) {
|
||||
|
||||
using Config = cutlass::gemm::device::DefaultGemmConfigurationToCutlass3Types<
|
||||
cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
|
||||
double, cutlass::layout::ColumnMajor,
|
||||
double, cutlass::layout::ColumnMajor,
|
||||
double, cutlass::layout::ColumnMajor,
|
||||
double>;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
Config::CollectiveMainloop,
|
||||
Config::CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM80_Device_Gemm_f64t_f64n_f64n_tensor_op_f64, 128x128x64_64x64x64) {
|
||||
|
||||
using Config = cutlass::gemm::device::DefaultGemmConfigurationToCutlass3Types<
|
||||
cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
|
||||
double, cutlass::layout::RowMajor,
|
||||
double, cutlass::layout::ColumnMajor,
|
||||
double, cutlass::layout::ColumnMajor,
|
||||
double>;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
Config::CollectiveMainloop,
|
||||
Config::CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
// #endif
|
||||
@@ -0,0 +1,94 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017 - 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/*! \file
|
||||
\brief Tests for device-wide GEMM interface
|
||||
*/
|
||||
|
||||
#include <iostream>
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cute/tensor.hpp"
|
||||
#include "cute/atom/mma_atom.hpp"
|
||||
|
||||
#include "cutlass/numeric_types.h"
|
||||
|
||||
#include "cutlass/gemm/device/gemm_universal_adapter.h"
|
||||
#include "default_gemm_configuration.hpp"
|
||||
|
||||
#include "../../common/cutlass_unit_test.h"
|
||||
|
||||
#include "gemm_testbed_3x.hpp"
|
||||
|
||||
using namespace cute;
|
||||
|
||||
//#if defined(CUTLASS_ARCH_MMA_SM80_SUPPORTED)
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(DISABLED_SM80_Device_Gemm_s8n_s8n_s32n_tensor_op_s32, 128x128x32_64x64x64) {
|
||||
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(DISABLED_SM80_Device_Gemm_s8n_s8t_s32n_tensor_op_s32, 128x128x32_64x64x64) {
|
||||
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM80_Device_Gemm_s8t_s8n_s32n_tensor_op_s32, 128x128x32_64x64x64) {
|
||||
using Config = cutlass::gemm::device::DefaultGemmConfigurationToCutlass3Types<
|
||||
cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
|
||||
int8_t, cutlass::layout::RowMajor,
|
||||
int8_t, cutlass::layout::ColumnMajor,
|
||||
int32_t, cutlass::layout::ColumnMajor,
|
||||
int32_t>;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
Config::CollectiveMainloop,
|
||||
Config::CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(DISABLED_SM80_Device_Gemm_s8t_s8t_s32n_tensor_op_s32, 128x128x32_64x64x64) {
|
||||
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
//#endif // #if defined(CUTLASS_ARCH_MMA_SM80_SUPPORTED)
|
||||
@@ -0,0 +1,135 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017 - 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/*! \file
|
||||
\brief Tests for device-wide GEMM interface
|
||||
*/
|
||||
|
||||
#include <iostream>
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cute/tensor.hpp"
|
||||
#include "cute/atom/mma_atom.hpp"
|
||||
|
||||
#include "cutlass/numeric_types.h"
|
||||
|
||||
#include "cutlass/gemm/device/gemm_universal_adapter.h"
|
||||
#include "default_gemm_configuration.hpp"
|
||||
|
||||
#include "../../common/cutlass_unit_test.h"
|
||||
|
||||
#include "gemm_testbed_3x.hpp"
|
||||
|
||||
using namespace cute;
|
||||
|
||||
|
||||
//#if defined(CUTLASS_ARCH_MMA_SM80_SUPPORTED)
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM80_Device_Gemm_tf32n_tf32n_f32n_tensor_op_f32, 128x128x32_64x64x64) {
|
||||
using Config = cutlass::gemm::device::DefaultGemmConfigurationToCutlass3Types<
|
||||
cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
|
||||
cutlass::tfloat32_t, cutlass::layout::ColumnMajor,
|
||||
cutlass::tfloat32_t, cutlass::layout::ColumnMajor,
|
||||
float, cutlass::layout::RowMajor,
|
||||
float>;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
Config::CollectiveMainloop,
|
||||
Config::CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM80_Device_Gemm_tf32n_tf32t_f32n_tensor_op_f32, 128x128x32_64x64x64) {
|
||||
using Config = cutlass::gemm::device::DefaultGemmConfigurationToCutlass3Types<
|
||||
cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
|
||||
cutlass::tfloat32_t, cutlass::layout::ColumnMajor,
|
||||
cutlass::tfloat32_t, cutlass::layout::RowMajor,
|
||||
float, cutlass::layout::RowMajor,
|
||||
float>;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
Config::CollectiveMainloop,
|
||||
Config::CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM80_Device_Gemm_tf32t_tf32n_f32n_tensor_op_f32, 128x128x32_64x64x64) {
|
||||
using Config = cutlass::gemm::device::DefaultGemmConfigurationToCutlass3Types<
|
||||
cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
|
||||
cutlass::tfloat32_t, cutlass::layout::RowMajor,
|
||||
cutlass::tfloat32_t, cutlass::layout::ColumnMajor,
|
||||
float, cutlass::layout::RowMajor,
|
||||
float>;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
Config::CollectiveMainloop,
|
||||
Config::CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
|
||||
}
|
||||
|
||||
TEST(SM80_Device_Gemm_tf32t_tf32t_f32n_tensor_op_f32, 128x128x32_64x64x64) {
|
||||
using Config = cutlass::gemm::device::DefaultGemmConfigurationToCutlass3Types<
|
||||
cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
|
||||
cutlass::tfloat32_t, cutlass::layout::RowMajor,
|
||||
cutlass::tfloat32_t, cutlass::layout::RowMajor,
|
||||
float, cutlass::layout::RowMajor,
|
||||
float>;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
Config::CollectiveMainloop,
|
||||
Config::CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
//#endif // #if defined(CUTLASS_ARCH_MMA_SM80_SUPPORTED)
|
||||
@@ -0,0 +1,188 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017 - 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/*! \file
|
||||
\brief Tests for device-wide GEMM interface
|
||||
*/
|
||||
|
||||
#include <iostream>
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cute/tensor.hpp"
|
||||
#include "cute/atom/mma_atom.hpp"
|
||||
|
||||
#include "cutlass/numeric_types.h"
|
||||
|
||||
#include "cutlass/gemm/gemm.h"
|
||||
#include "cutlass/gemm/device/gemm_universal_adapter.h"
|
||||
#include "cutlass/gemm/kernel/gemm_universal.hpp"
|
||||
#include "cutlass/gemm/collective/collective_builder.hpp"
|
||||
#include "cutlass/epilogue/collective/default_epilogue.hpp"
|
||||
#include "cutlass/epilogue/thread/linear_combination.h"
|
||||
|
||||
#include "../../common/cutlass_unit_test.h"
|
||||
|
||||
#include "gemm_testbed_3x.hpp"
|
||||
|
||||
#if defined(CUTLASS_ARCH_MMA_SM90_SUPPORTED)
|
||||
|
||||
using namespace cute;
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM90_Device_Gemm_bf16t_bf16t_bf16n_align8_tensor_op_gmma_f32, 64x128x64) {
|
||||
using LayoutA = cutlass::layout::RowMajor;
|
||||
using LayoutB = cutlass::layout::RowMajor;
|
||||
using LayoutC = cutlass::layout::ColumnMajor;
|
||||
|
||||
using CollectiveOp = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::bfloat16_t, LayoutA, 8,
|
||||
cutlass::bfloat16_t, LayoutB, 8,
|
||||
float,
|
||||
Shape<_64,_128,_64>, Shape<_1,_1,_1>,
|
||||
cutlass::gemm::collective::StageCountAuto,
|
||||
cutlass::gemm::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using EpilogueOp = cutlass::epilogue::collective::DefaultEpilogue<
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::epilogue::thread::LinearCombination<cutlass::bfloat16_t, 1, float, float>>;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
CollectiveOp,
|
||||
EpilogueOp
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM90_Device_Gemm_bf16t_bf16n_bf16n_align4_tensor_op_gmma_f32, 64x128x64) {
|
||||
using LayoutA = cutlass::layout::RowMajor;
|
||||
using LayoutB = cutlass::layout::ColumnMajor;
|
||||
using LayoutC = cutlass::layout::ColumnMajor;
|
||||
|
||||
using CollectiveOp = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::bfloat16_t, LayoutA, 4,
|
||||
cutlass::bfloat16_t, LayoutB, 4,
|
||||
float,
|
||||
Shape<_64,_128,_64>, Shape<_1,_1,_1>,
|
||||
cutlass::gemm::collective::StageCountAuto,
|
||||
cutlass::gemm::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using EpilogueOp = cutlass::epilogue::collective::DefaultEpilogue<
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::epilogue::thread::LinearCombination<cutlass::bfloat16_t, 1, float, float>>;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
CollectiveOp,
|
||||
EpilogueOp
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM90_Device_Gemm_bf16n_bf16t_bf16n_align2_tensor_op_gmma_f32, 64x128x64) {
|
||||
using LayoutA = cutlass::layout::ColumnMajor;
|
||||
using LayoutB = cutlass::layout::RowMajor;
|
||||
using LayoutC = cutlass::layout::ColumnMajor;
|
||||
|
||||
using CollectiveOp = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::bfloat16_t, LayoutA, 2,
|
||||
cutlass::bfloat16_t, LayoutB, 2,
|
||||
float,
|
||||
Shape<_64,_128,_64>, Shape<_1,_1,_1>,
|
||||
cutlass::gemm::collective::StageCountAuto,
|
||||
cutlass::gemm::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using EpilogueOp = cutlass::epilogue::collective::DefaultEpilogue<
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::epilogue::thread::LinearCombination<cutlass::bfloat16_t, 1, float, float>>;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
CollectiveOp,
|
||||
EpilogueOp
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM90_Device_Gemm_bf16n_bf16n_bf16n_align8_tensor_op_gmma_f32, 64x128x64) {
|
||||
using LayoutA = cutlass::layout::ColumnMajor;
|
||||
using LayoutB = cutlass::layout::ColumnMajor;
|
||||
using LayoutC = cutlass::layout::ColumnMajor;
|
||||
|
||||
using CollectiveOp = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::bfloat16_t, LayoutA, 8,
|
||||
cutlass::bfloat16_t, LayoutB, 8,
|
||||
float,
|
||||
Shape<_64,_128,_64>, Shape<_1,_1,_1>,
|
||||
cutlass::gemm::collective::StageCountAuto,
|
||||
cutlass::gemm::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using EpilogueOp = cutlass::epilogue::collective::DefaultEpilogue<
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::epilogue::thread::LinearCombination<cutlass::bfloat16_t, 1, float, float>>;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
CollectiveOp,
|
||||
EpilogueOp
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#endif // defined(CUTLASS_ARCH_MMA_SM90_SUPPORTED)
|
||||
@@ -0,0 +1,187 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017 - 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/*! \file
|
||||
\brief Tests for device-wide GEMM interface
|
||||
*/
|
||||
|
||||
#include <iostream>
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cute/tensor.hpp"
|
||||
#include "cute/atom/mma_atom.hpp"
|
||||
|
||||
#include "cutlass/numeric_types.h"
|
||||
|
||||
#include "cutlass/gemm/device/gemm_universal_adapter.h"
|
||||
#include "cutlass/gemm/kernel/gemm_universal.hpp"
|
||||
#include "cutlass/gemm/collective/collective_builder.hpp"
|
||||
#include "cutlass/epilogue/collective/default_epilogue.hpp"
|
||||
#include "cutlass/epilogue/thread/linear_combination.h"
|
||||
|
||||
#include "../../common/cutlass_unit_test.h"
|
||||
|
||||
#include "gemm_testbed_3x.hpp"
|
||||
|
||||
#if defined(CUTLASS_ARCH_MMA_SM90_SUPPORTED)
|
||||
|
||||
using namespace cute;
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM90_Device_Gemm_bf16t_bf16t_bf16n_tensor_op_gmma_f32, 64x128x64) {
|
||||
using LayoutA = cutlass::layout::RowMajor;
|
||||
using LayoutB = cutlass::layout::RowMajor;
|
||||
using LayoutC = cutlass::layout::ColumnMajor;
|
||||
|
||||
using CollectiveOp = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::bfloat16_t, LayoutA, 8,
|
||||
cutlass::bfloat16_t, LayoutB, 8,
|
||||
float,
|
||||
Shape<_64,_128,_64>, Shape<_1,_1,_1>,
|
||||
cutlass::gemm::collective::StageCountAuto,
|
||||
cutlass::gemm::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using EpilogueOp = cutlass::epilogue::collective::DefaultEpilogue<
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::epilogue::thread::LinearCombination<cutlass::bfloat16_t, 1, float, float>>;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
CollectiveOp,
|
||||
EpilogueOp
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM90_Device_Gemm_bf16t_bf16n_bf16n_tensor_op_gmma_f32, 64x128x64) {
|
||||
using LayoutA = cutlass::layout::RowMajor;
|
||||
using LayoutB = cutlass::layout::ColumnMajor;
|
||||
using LayoutC = cutlass::layout::ColumnMajor;
|
||||
|
||||
using CollectiveOp = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::bfloat16_t, LayoutA, 8,
|
||||
cutlass::bfloat16_t, LayoutB, 8,
|
||||
float,
|
||||
Shape<_64,_128,_64>, Shape<_1,_1,_1>,
|
||||
cutlass::gemm::collective::StageCountAuto,
|
||||
cutlass::gemm::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using EpilogueOp = cutlass::epilogue::collective::DefaultEpilogue<
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::epilogue::thread::LinearCombination<cutlass::bfloat16_t, 1, float, float>>;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
CollectiveOp,
|
||||
EpilogueOp
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM90_Device_Gemm_bf16n_bf16t_bf16n_tensor_op_gmma_f32, 64x128x64) {
|
||||
using LayoutA = cutlass::layout::ColumnMajor;
|
||||
using LayoutB = cutlass::layout::RowMajor;
|
||||
using LayoutC = cutlass::layout::ColumnMajor;
|
||||
|
||||
using CollectiveOp = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::bfloat16_t, LayoutA, 8,
|
||||
cutlass::bfloat16_t, LayoutB, 8,
|
||||
float,
|
||||
Shape<_64,_128,_64>, Shape<_1,_1,_1>,
|
||||
cutlass::gemm::collective::StageCountAuto,
|
||||
cutlass::gemm::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using EpilogueOp = cutlass::epilogue::collective::DefaultEpilogue<
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::epilogue::thread::LinearCombination<cutlass::bfloat16_t, 1, float, float>>;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
CollectiveOp,
|
||||
EpilogueOp
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM90_Device_Gemm_bf16n_bf16n_bf16n_tensor_op_gmma_f32, 64x128x64) {
|
||||
using LayoutA = cutlass::layout::ColumnMajor;
|
||||
using LayoutB = cutlass::layout::ColumnMajor;
|
||||
using LayoutC = cutlass::layout::ColumnMajor;
|
||||
|
||||
using CollectiveOp = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::bfloat16_t, LayoutA, 8,
|
||||
cutlass::bfloat16_t, LayoutB, 8,
|
||||
float,
|
||||
Shape<_64,_128,_64>, Shape<_1,_1,_1>,
|
||||
cutlass::gemm::collective::StageCountAuto,
|
||||
cutlass::gemm::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using EpilogueOp = cutlass::epilogue::collective::DefaultEpilogue<
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::epilogue::thread::LinearCombination<cutlass::bfloat16_t, 1, float, float>>;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
CollectiveOp,
|
||||
EpilogueOp
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#endif // defined(CUTLASS_ARCH_MMA_SM90_SUPPORTED)
|
||||
@@ -0,0 +1,449 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017 - 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/*! \file
|
||||
\brief Tests for device-wide GEMM interface
|
||||
*/
|
||||
|
||||
#include <iostream>
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cute/tensor.hpp"
|
||||
#include "cute/atom/mma_atom.hpp"
|
||||
|
||||
#include "cutlass/numeric_types.h"
|
||||
|
||||
#include "cutlass/gemm/gemm.h"
|
||||
#include "cutlass/gemm/device/gemm_universal_adapter.h"
|
||||
#include "cutlass/gemm/kernel/gemm_universal.hpp"
|
||||
#include "cutlass/gemm/collective/collective_builder.hpp"
|
||||
#include "cutlass/epilogue/collective/default_epilogue.hpp"
|
||||
#include "cutlass/epilogue/thread/linear_combination.h"
|
||||
|
||||
#include "../../common/cutlass_unit_test.h"
|
||||
|
||||
#include "gemm_testbed_3x.hpp"
|
||||
|
||||
#if defined(CUTLASS_ARCH_MMA_SM90_SUPPORTED)
|
||||
|
||||
using namespace cute;
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
///////////////////////////////////// TT //////////////////////////////////////
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM90_Device_Gemm_f16t_f16t_f16n_align8_tensor_op_gmma_f32, 64x128x64) {
|
||||
using LayoutA = cutlass::layout::RowMajor;
|
||||
using LayoutB = cutlass::layout::RowMajor;
|
||||
using LayoutC = cutlass::layout::ColumnMajor;
|
||||
|
||||
using CollectiveOp = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::half_t, LayoutA, 8,
|
||||
cutlass::half_t, LayoutB, 8,
|
||||
float,
|
||||
Shape<_64,_128,_64>, Shape<_1,_1,_1>,
|
||||
cutlass::gemm::collective::StageCountAuto,
|
||||
cutlass::gemm::KernelMultistage
|
||||
>::CollectiveOp;
|
||||
|
||||
using EpilogueOp = cutlass::epilogue::collective::DefaultEpilogue<
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::epilogue::thread::LinearCombination<cutlass::half_t, 1, float, float>>;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
CollectiveOp,
|
||||
EpilogueOp
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
|
||||
}
|
||||
|
||||
TEST(SM90_Device_Gemm_f16t_f16t_f16n_align4_tensor_op_gmma_f32, 64x128x64) {
|
||||
using LayoutA = cutlass::layout::RowMajor;
|
||||
using LayoutB = cutlass::layout::RowMajor;
|
||||
using LayoutC = cutlass::layout::ColumnMajor;
|
||||
|
||||
using CollectiveOp = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::half_t, LayoutA, 4,
|
||||
cutlass::half_t, LayoutB, 4,
|
||||
float,
|
||||
Shape<_64,_128,_64>, Shape<_1,_1,_1>,
|
||||
cutlass::gemm::collective::StageCountAuto,
|
||||
cutlass::gemm::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using EpilogueOp = cutlass::epilogue::collective::DefaultEpilogue<
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::epilogue::thread::LinearCombination<cutlass::half_t, 1, float, float>>;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
CollectiveOp,
|
||||
EpilogueOp
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
|
||||
}
|
||||
|
||||
|
||||
TEST(SM90_Device_Gemm_f16t_f16t_f16n_align2_tensor_op_gmma_f32, 64x128x64) {
|
||||
using LayoutA = cutlass::layout::RowMajor;
|
||||
using LayoutB = cutlass::layout::RowMajor;
|
||||
using LayoutC = cutlass::layout::ColumnMajor;
|
||||
|
||||
using CollectiveOp = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::half_t, LayoutA, 2,
|
||||
cutlass::half_t, LayoutB, 2,
|
||||
float,
|
||||
Shape<_64,_128,_64>, Shape<_1,_1,_1>,
|
||||
cutlass::gemm::collective::StageCountAuto,
|
||||
cutlass::gemm::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using EpilogueOp = cutlass::epilogue::collective::DefaultEpilogue<
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::epilogue::thread::LinearCombination<cutlass::half_t, 1, float, float>>;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
CollectiveOp,
|
||||
EpilogueOp
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
///////////////////////////////////// TN //////////////////////////////////////
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM90_Device_Gemm_f16t_f16n_f16n_align8_tensor_op_gmma_f32, 64x128x64) {
|
||||
using LayoutA = cutlass::layout::RowMajor;
|
||||
using LayoutB = cutlass::layout::ColumnMajor;
|
||||
using LayoutC = cutlass::layout::ColumnMajor;
|
||||
|
||||
using CollectiveOp = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::half_t, LayoutA, 8,
|
||||
cutlass::half_t, LayoutB, 8,
|
||||
float,
|
||||
Shape<_64,_128,_64>, Shape<_1,_1,_1>,
|
||||
cutlass::gemm::collective::StageCountAuto,
|
||||
cutlass::gemm::KernelMultistage
|
||||
>::CollectiveOp;
|
||||
|
||||
using EpilogueOp = cutlass::epilogue::collective::DefaultEpilogue<
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::epilogue::thread::LinearCombination<cutlass::half_t, 1, float, float>>;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
CollectiveOp,
|
||||
EpilogueOp
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM90_Device_Gemm_f16t_f16n_f16n_align4_tensor_op_gmma_f32, 64x128x64) {
|
||||
using LayoutA = cutlass::layout::RowMajor;
|
||||
using LayoutB = cutlass::layout::ColumnMajor;
|
||||
using LayoutC = cutlass::layout::ColumnMajor;
|
||||
|
||||
using CollectiveOp = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::half_t, LayoutA, 4,
|
||||
cutlass::half_t, LayoutB, 4,
|
||||
float,
|
||||
Shape<_64,_128,_64>, Shape<_1,_1,_1>,
|
||||
cutlass::gemm::collective::StageCountAuto,
|
||||
cutlass::gemm::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using EpilogueOp = cutlass::epilogue::collective::DefaultEpilogue<
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::epilogue::thread::LinearCombination<cutlass::half_t, 1, float, float>>;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
CollectiveOp,
|
||||
EpilogueOp
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM90_Device_Gemm_f16t_f16n_f16n_align2_tensor_op_gmma_f32, 64x128x64) {
|
||||
using LayoutA = cutlass::layout::RowMajor;
|
||||
using LayoutB = cutlass::layout::ColumnMajor;
|
||||
using LayoutC = cutlass::layout::ColumnMajor;
|
||||
|
||||
using CollectiveOp = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::half_t, LayoutA, 2,
|
||||
cutlass::half_t, LayoutB, 2,
|
||||
float,
|
||||
Shape<_64,_128,_64>, Shape<_1,_1,_1>,
|
||||
cutlass::gemm::collective::StageCountAuto,
|
||||
cutlass::gemm::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using EpilogueOp = cutlass::epilogue::collective::DefaultEpilogue<
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::epilogue::thread::LinearCombination<cutlass::half_t, 1, float, float>>;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
CollectiveOp,
|
||||
EpilogueOp
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
///////////////////////////////////// NT //////////////////////////////////////
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM90_Device_Gemm_f16n_f16t_f16n_align8_tensor_op_gmma_f32, 64x128x64) {
|
||||
using LayoutA = cutlass::layout::ColumnMajor;
|
||||
using LayoutB = cutlass::layout::RowMajor;
|
||||
using LayoutC = cutlass::layout::ColumnMajor;
|
||||
|
||||
using CollectiveOp = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::half_t, LayoutA, 8,
|
||||
cutlass::half_t, LayoutB, 8,
|
||||
float,
|
||||
Shape<_64,_128,_64>, Shape<_1,_1,_1>,
|
||||
cutlass::gemm::collective::StageCountAuto,
|
||||
cutlass::gemm::KernelMultistage
|
||||
>::CollectiveOp;
|
||||
|
||||
using EpilogueOp = cutlass::epilogue::collective::DefaultEpilogue<
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::epilogue::thread::LinearCombination<cutlass::half_t, 1, float, float>>;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
CollectiveOp,
|
||||
EpilogueOp
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM90_Device_Gemm_f16n_f16t_f16n_align4_tensor_op_gmma_f32, 64x128x64) {
|
||||
using LayoutA = cutlass::layout::ColumnMajor;
|
||||
using LayoutB = cutlass::layout::RowMajor;
|
||||
using LayoutC = cutlass::layout::ColumnMajor;
|
||||
|
||||
using CollectiveOp = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::half_t, LayoutA, 4,
|
||||
cutlass::half_t, LayoutB, 4,
|
||||
float,
|
||||
Shape<_64,_128,_64>, Shape<_1,_1,_1>,
|
||||
cutlass::gemm::collective::StageCountAuto,
|
||||
cutlass::gemm::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using EpilogueOp = cutlass::epilogue::collective::DefaultEpilogue<
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::epilogue::thread::LinearCombination<cutlass::half_t, 1, float, float>>;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
CollectiveOp,
|
||||
EpilogueOp
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM90_Device_Gemm_f16n_f16t_f16n_align2_tensor_op_gmma_f32, 64x128x64) {
|
||||
using LayoutA = cutlass::layout::ColumnMajor;
|
||||
using LayoutB = cutlass::layout::RowMajor;
|
||||
using LayoutC = cutlass::layout::ColumnMajor;
|
||||
|
||||
using CollectiveOp = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::half_t, LayoutA, 2,
|
||||
cutlass::half_t, LayoutB, 2,
|
||||
float,
|
||||
Shape<_64,_128,_64>, Shape<_1,_1,_1>,
|
||||
cutlass::gemm::collective::StageCountAuto,
|
||||
cutlass::gemm::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using EpilogueOp = cutlass::epilogue::collective::DefaultEpilogue<
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::epilogue::thread::LinearCombination<cutlass::half_t, 1, float, float>>;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
CollectiveOp,
|
||||
EpilogueOp
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
///////////////////////////////////// NN //////////////////////////////////////
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM90_Device_Gemm_f16n_f16n_f16n_align8_tensor_op_gmma_f32, 64x128x64) {
|
||||
using LayoutA = cutlass::layout::ColumnMajor;
|
||||
using LayoutB = cutlass::layout::ColumnMajor;
|
||||
using LayoutC = cutlass::layout::ColumnMajor;
|
||||
|
||||
using CollectiveOp = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::half_t, LayoutA, 8,
|
||||
cutlass::half_t, LayoutB, 8,
|
||||
float,
|
||||
Shape<_64,_128,_64>, Shape<_1,_1,_1>,
|
||||
cutlass::gemm::collective::StageCountAuto,
|
||||
cutlass::gemm::KernelMultistage
|
||||
>::CollectiveOp;
|
||||
|
||||
using EpilogueOp = cutlass::epilogue::collective::DefaultEpilogue<
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::epilogue::thread::LinearCombination<cutlass::half_t, 1, float, float>>;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
CollectiveOp,
|
||||
EpilogueOp
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM90_Device_Gemm_f16n_f16n_f16n_align4_tensor_op_gmma_f32, 64x128x64) {
|
||||
using LayoutA = cutlass::layout::ColumnMajor;
|
||||
using LayoutB = cutlass::layout::ColumnMajor;
|
||||
using LayoutC = cutlass::layout::ColumnMajor;
|
||||
|
||||
using CollectiveOp = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::half_t, LayoutA, 4,
|
||||
cutlass::half_t, LayoutB, 4,
|
||||
float,
|
||||
Shape<_64,_128,_64>, Shape<_1,_1,_1>,
|
||||
cutlass::gemm::collective::StageCountAuto,
|
||||
cutlass::gemm::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using EpilogueOp = cutlass::epilogue::collective::DefaultEpilogue<
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::epilogue::thread::LinearCombination<cutlass::half_t, 1, float, float>>;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
CollectiveOp,
|
||||
EpilogueOp
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM90_Device_Gemm_f16n_f16n_f16n_align2_tensor_op_gmma_f32, 64x128x64) {
|
||||
using LayoutA = cutlass::layout::ColumnMajor;
|
||||
using LayoutB = cutlass::layout::ColumnMajor;
|
||||
using LayoutC = cutlass::layout::ColumnMajor;
|
||||
|
||||
using CollectiveOp = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::half_t, LayoutA, 2,
|
||||
cutlass::half_t, LayoutB, 2,
|
||||
float,
|
||||
Shape<_64,_128,_64>, Shape<_1,_1,_1>,
|
||||
cutlass::gemm::collective::StageCountAuto,
|
||||
cutlass::gemm::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using EpilogueOp = cutlass::epilogue::collective::DefaultEpilogue<
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::epilogue::thread::LinearCombination<cutlass::half_t, 1, float, float>>;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
CollectiveOp,
|
||||
EpilogueOp
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#endif // defined(CUTLASS_ARCH_MMA_SM90_SUPPORTED)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,582 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017 - 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/*! \file
|
||||
\brief Tests for device-wide GEMM interface
|
||||
*/
|
||||
|
||||
#include <iostream>
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cute/tensor.hpp"
|
||||
#include "cute/atom/mma_atom.hpp"
|
||||
|
||||
#include "cutlass/numeric_types.h"
|
||||
|
||||
#include "cutlass/gemm/device/gemm_universal_adapter.h"
|
||||
#include "cutlass/gemm/kernel/gemm_universal.hpp"
|
||||
#include "cutlass/gemm/collective/collective_builder.hpp"
|
||||
#include "cutlass/epilogue/collective/default_epilogue.hpp"
|
||||
#include "cutlass/epilogue/thread/linear_combination.h"
|
||||
|
||||
#include "../../common/cutlass_unit_test.h"
|
||||
|
||||
#include "gemm_testbed_3x.hpp"
|
||||
|
||||
#if defined(CUTLASS_ARCH_MMA_SM90_SUPPORTED)
|
||||
|
||||
using namespace cute;
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
/////////////////////////////// Cluster 2x2x1 ////////////////////////////////
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM90_Device_Gemm_f16t_f16t_f32n_tensor_op_gmma_f32_unspecialized, 64x128x64_2x2x1) {
|
||||
using LayoutA = cutlass::layout::RowMajor;
|
||||
using LayoutB = cutlass::layout::RowMajor;
|
||||
using LayoutC = cutlass::layout::ColumnMajor;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::half_t, LayoutA, 8,
|
||||
cutlass::half_t, LayoutB, 8,
|
||||
float,
|
||||
Shape<_64,_128,_64>, Shape<_2,_2,_1>,
|
||||
cutlass::gemm::collective::StageCountAuto,
|
||||
cutlass::gemm::KernelTma
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveEpilogue = cutlass::epilogue::collective::DefaultEpilogue<
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::epilogue::thread::LinearCombination<cutlass::half_t, 1, float, float>>;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM90_Device_Gemm_f16t_f16n_f32n_tensor_op_gmma_f32_unspecialized, 64x128x64_2x2x1) {
|
||||
using LayoutA = cutlass::layout::RowMajor;
|
||||
using LayoutB = cutlass::layout::ColumnMajor;
|
||||
using LayoutC = cutlass::layout::ColumnMajor;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::half_t, LayoutA, 8,
|
||||
cutlass::half_t, LayoutB, 8,
|
||||
float,
|
||||
Shape<_64,_128,_64>, Shape<_2,_2,_1>,
|
||||
cutlass::gemm::collective::StageCountAuto,
|
||||
cutlass::gemm::KernelTma
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveEpilogue = cutlass::epilogue::collective::DefaultEpilogue<
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::epilogue::thread::LinearCombination<cutlass::half_t, 1, float, float>>;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM90_Device_Gemm_f16n_f16t_f32n_tensor_op_gmma_f32_unspecialized, 64x128x64_2x2x1) {
|
||||
using LayoutA = cutlass::layout::ColumnMajor;
|
||||
using LayoutB = cutlass::layout::RowMajor;
|
||||
using LayoutC = cutlass::layout::ColumnMajor;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::half_t, LayoutA, 8,
|
||||
cutlass::half_t, LayoutB, 8,
|
||||
float,
|
||||
Shape<_64,_128,_64>, Shape<_2,_2,_1>,
|
||||
cutlass::gemm::collective::StageCountAuto,
|
||||
cutlass::gemm::KernelTma
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveEpilogue = cutlass::epilogue::collective::DefaultEpilogue<
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::epilogue::thread::LinearCombination<cutlass::half_t, 1, float, float>>;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM90_Device_Gemm_f16n_f16n_f32n_tensor_op_gmma_f32_unspecialized, 64x128x64_2x2x1) {
|
||||
using LayoutA = cutlass::layout::ColumnMajor;
|
||||
using LayoutB = cutlass::layout::ColumnMajor;
|
||||
using LayoutC = cutlass::layout::ColumnMajor;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::half_t, LayoutA, 8,
|
||||
cutlass::half_t, LayoutB, 8,
|
||||
float,
|
||||
Shape<_64,_128,_64>, Shape<_2,_2,_1>,
|
||||
cutlass::gemm::collective::StageCountAuto,
|
||||
cutlass::gemm::KernelTma
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveEpilogue = cutlass::epilogue::collective::DefaultEpilogue<
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::epilogue::thread::LinearCombination<cutlass::half_t, 1, float, float>>;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
/////////////////////////////// Cluster 4x1x1 ////////////////////////////////
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
|
||||
TEST(SM90_Device_Gemm_f16t_f16t_f32n_tensor_op_gmma_f32_unspecialized, 64x128x64_4x1x1) {
|
||||
using LayoutA = cutlass::layout::RowMajor;
|
||||
using LayoutB = cutlass::layout::RowMajor;
|
||||
using LayoutC = cutlass::layout::ColumnMajor;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::half_t, LayoutA, 8,
|
||||
cutlass::half_t, LayoutB, 8,
|
||||
float,
|
||||
Shape<_64,_128,_64>, Shape<_4,_1,_1>,
|
||||
cutlass::gemm::collective::StageCountAuto,
|
||||
cutlass::gemm::KernelTma
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveEpilogue = cutlass::epilogue::collective::DefaultEpilogue<
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::epilogue::thread::LinearCombination<cutlass::half_t, 1, float, float>>;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM90_Device_Gemm_f16t_f16n_f32n_tensor_op_gmma_f32_unspecialized, 64x128x64_4x1x1) {
|
||||
using LayoutA = cutlass::layout::RowMajor;
|
||||
using LayoutB = cutlass::layout::ColumnMajor;
|
||||
using LayoutC = cutlass::layout::ColumnMajor;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::half_t, LayoutA, 8,
|
||||
cutlass::half_t, LayoutB, 8,
|
||||
float,
|
||||
Shape<_64,_128,_64>, Shape<_4,_1,_1>,
|
||||
cutlass::gemm::collective::StageCountAuto,
|
||||
cutlass::gemm::KernelTma
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveEpilogue = cutlass::epilogue::collective::DefaultEpilogue<
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::epilogue::thread::LinearCombination<cutlass::half_t, 1, float, float>>;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM90_Device_Gemm_f16n_f16t_f32n_tensor_op_gmma_f32_unspecialized, 64x128x64_4x1x1) {
|
||||
using LayoutA = cutlass::layout::ColumnMajor;
|
||||
using LayoutB = cutlass::layout::RowMajor;
|
||||
using LayoutC = cutlass::layout::ColumnMajor;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::half_t, LayoutA, 8,
|
||||
cutlass::half_t, LayoutB, 8,
|
||||
float,
|
||||
Shape<_64,_128,_64>, Shape<_4,_1,_1>,
|
||||
cutlass::gemm::collective::StageCountAuto,
|
||||
cutlass::gemm::KernelTma
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveEpilogue = cutlass::epilogue::collective::DefaultEpilogue<
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::epilogue::thread::LinearCombination<cutlass::half_t, 1, float, float>>;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM90_Device_Gemm_f16n_f16n_f32n_tensor_op_gmma_f32_unspecialized, 64x128x64_4x1x1) {
|
||||
using LayoutA = cutlass::layout::ColumnMajor;
|
||||
using LayoutB = cutlass::layout::ColumnMajor;
|
||||
using LayoutC = cutlass::layout::ColumnMajor;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::half_t, LayoutA, 8,
|
||||
cutlass::half_t, LayoutB, 8,
|
||||
float,
|
||||
Shape<_64,_128,_64>, Shape<_4,_1,_1>,
|
||||
cutlass::gemm::collective::StageCountAuto,
|
||||
cutlass::gemm::KernelTma
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveEpilogue = cutlass::epilogue::collective::DefaultEpilogue<
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::epilogue::thread::LinearCombination<cutlass::half_t, 1, float, float>>;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
|
||||
}
|
||||
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
/////////////////////////////// Cluster 1x4x1 ////////////////////////////////
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
|
||||
TEST(SM90_Device_Gemm_f16t_f16t_f32n_tensor_op_gmma_f32_unspecialized, 64x128x64_1x4x1) {
|
||||
using LayoutA = cutlass::layout::RowMajor;
|
||||
using LayoutB = cutlass::layout::RowMajor;
|
||||
using LayoutC = cutlass::layout::ColumnMajor;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::half_t, LayoutA, 8,
|
||||
cutlass::half_t, LayoutB, 8,
|
||||
float,
|
||||
Shape<_64,_128,_64>, Shape<_1,_4,_1>,
|
||||
cutlass::gemm::collective::StageCountAuto,
|
||||
cutlass::gemm::KernelTma
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveEpilogue = cutlass::epilogue::collective::DefaultEpilogue<
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::epilogue::thread::LinearCombination<cutlass::half_t, 1, float, float>>;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM90_Device_Gemm_f16t_f16n_f32n_tensor_op_gmma_f32_unspecialized, 64x128x64_1x4x1) {
|
||||
using LayoutA = cutlass::layout::RowMajor;
|
||||
using LayoutB = cutlass::layout::ColumnMajor;
|
||||
using LayoutC = cutlass::layout::ColumnMajor;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::half_t, LayoutA, 8,
|
||||
cutlass::half_t, LayoutB, 8,
|
||||
float,
|
||||
Shape<_64,_128,_64>, Shape<_1,_4,_1>,
|
||||
cutlass::gemm::collective::StageCountAuto,
|
||||
cutlass::gemm::KernelTma
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveEpilogue = cutlass::epilogue::collective::DefaultEpilogue<
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::epilogue::thread::LinearCombination<cutlass::half_t, 1, float, float>>;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM90_Device_Gemm_f16n_f16t_f32n_tensor_op_gmma_f32_unspecialized, 64x128x64_1x4x1) {
|
||||
using LayoutA = cutlass::layout::ColumnMajor;
|
||||
using LayoutB = cutlass::layout::RowMajor;
|
||||
using LayoutC = cutlass::layout::ColumnMajor;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::half_t, LayoutA, 8,
|
||||
cutlass::half_t, LayoutB, 8,
|
||||
float,
|
||||
Shape<_64,_128,_64>, Shape<_1,_4,_1>,
|
||||
cutlass::gemm::collective::StageCountAuto,
|
||||
cutlass::gemm::KernelTma
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveEpilogue = cutlass::epilogue::collective::DefaultEpilogue<
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::epilogue::thread::LinearCombination<cutlass::half_t, 1, float, float>>;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM90_Device_Gemm_f16n_f16n_f32n_tensor_op_gmma_f32_unspecialized, 64x128x64_1x4x1) {
|
||||
using LayoutA = cutlass::layout::ColumnMajor;
|
||||
using LayoutB = cutlass::layout::ColumnMajor;
|
||||
using LayoutC = cutlass::layout::ColumnMajor;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::half_t, LayoutA, 8,
|
||||
cutlass::half_t, LayoutB, 8,
|
||||
float,
|
||||
Shape<_64,_128,_64>, Shape<_1,_4,_1>,
|
||||
cutlass::gemm::collective::StageCountAuto,
|
||||
cutlass::gemm::KernelTma
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveEpilogue = cutlass::epilogue::collective::DefaultEpilogue<
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::epilogue::thread::LinearCombination<cutlass::half_t, 1, float, float>>;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
|
||||
}
|
||||
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
/////////////////////////////// Cluster 2x4x1 ////////////////////////////////
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
|
||||
TEST(SM90_Device_Gemm_f16t_f16t_f32n_tensor_op_gmma_f32_unspecialized, 64x128x64_2x4x1) {
|
||||
using LayoutA = cutlass::layout::RowMajor;
|
||||
using LayoutB = cutlass::layout::RowMajor;
|
||||
using LayoutC = cutlass::layout::ColumnMajor;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::half_t, LayoutA, 8,
|
||||
cutlass::half_t, LayoutB, 8,
|
||||
float,
|
||||
Shape<_64,_128,_64>, Shape<_2,_4,_1>,
|
||||
cutlass::gemm::collective::StageCountAuto,
|
||||
cutlass::gemm::KernelTma
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveEpilogue = cutlass::epilogue::collective::DefaultEpilogue<
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::epilogue::thread::LinearCombination<cutlass::half_t, 1, float, float>>;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM90_Device_Gemm_f16t_f16n_f32n_tensor_op_gmma_f32_unspecialized, 64x128x64_2x4x1) {
|
||||
using LayoutA = cutlass::layout::RowMajor;
|
||||
using LayoutB = cutlass::layout::ColumnMajor;
|
||||
using LayoutC = cutlass::layout::ColumnMajor;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::half_t, LayoutA, 8,
|
||||
cutlass::half_t, LayoutB, 8,
|
||||
float,
|
||||
Shape<_64,_128,_64>, Shape<_2,_4,_1>,
|
||||
cutlass::gemm::collective::StageCountAuto,
|
||||
cutlass::gemm::KernelTma
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveEpilogue = cutlass::epilogue::collective::DefaultEpilogue<
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::epilogue::thread::LinearCombination<cutlass::half_t, 1, float, float>>;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM90_Device_Gemm_f16n_f16t_f32n_tensor_op_gmma_f32_unspecialized, 64x128x64_2x4x1) {
|
||||
using LayoutA = cutlass::layout::ColumnMajor;
|
||||
using LayoutB = cutlass::layout::RowMajor;
|
||||
using LayoutC = cutlass::layout::ColumnMajor;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::half_t, LayoutA, 8,
|
||||
cutlass::half_t, LayoutB, 8,
|
||||
float,
|
||||
Shape<_64,_128,_64>, Shape<_2,_4,_1>,
|
||||
cutlass::gemm::collective::StageCountAuto,
|
||||
cutlass::gemm::KernelTma
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveEpilogue = cutlass::epilogue::collective::DefaultEpilogue<
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::epilogue::thread::LinearCombination<cutlass::half_t, 1, float, float>>;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM90_Device_Gemm_f16n_f16n_f32n_tensor_op_gmma_f32_unspecialized, 64x128x64_2x4x1) {
|
||||
using LayoutA = cutlass::layout::ColumnMajor;
|
||||
using LayoutB = cutlass::layout::ColumnMajor;
|
||||
using LayoutC = cutlass::layout::ColumnMajor;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::half_t, LayoutA, 8,
|
||||
cutlass::half_t, LayoutB, 8,
|
||||
float,
|
||||
Shape<_64,_128,_64>, Shape<_2,_4,_1>,
|
||||
cutlass::gemm::collective::StageCountAuto,
|
||||
cutlass::gemm::KernelTma
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveEpilogue = cutlass::epilogue::collective::DefaultEpilogue<
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::epilogue::thread::LinearCombination<cutlass::half_t, 1, float, float>>;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
|
||||
}
|
||||
|
||||
#endif // defined(CUTLASS_ARCH_MMA_SM90_SUPPORTED)
|
||||
@@ -0,0 +1,582 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017 - 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/*! \file
|
||||
\brief Tests for device-wide GEMM interface
|
||||
*/
|
||||
|
||||
#include <iostream>
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cute/tensor.hpp"
|
||||
#include "cute/atom/mma_atom.hpp"
|
||||
|
||||
#include "cutlass/numeric_types.h"
|
||||
|
||||
#include "cutlass/gemm/device/gemm_universal_adapter.h"
|
||||
#include "cutlass/gemm/kernel/gemm_universal.hpp"
|
||||
#include "cutlass/gemm/collective/collective_builder.hpp"
|
||||
#include "cutlass/epilogue/collective/default_epilogue.hpp"
|
||||
#include "cutlass/epilogue/thread/linear_combination.h"
|
||||
|
||||
#include "../../common/cutlass_unit_test.h"
|
||||
|
||||
#include "gemm_testbed_3x.hpp"
|
||||
|
||||
#if defined(CUTLASS_ARCH_MMA_SM90_SUPPORTED)
|
||||
|
||||
using namespace cute;
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
/////////////////////////////// Cluster 2x2x1 ////////////////////////////////
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM90_Device_Gemm_f16t_f16t_f32n_tensor_op_gmma_f32_warpspecialized, 64x128x64_2x2x1) {
|
||||
using LayoutA = cutlass::layout::RowMajor;
|
||||
using LayoutB = cutlass::layout::RowMajor;
|
||||
using LayoutC = cutlass::layout::ColumnMajor;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::half_t, LayoutA, 8,
|
||||
cutlass::half_t, LayoutB, 8,
|
||||
float,
|
||||
Shape<_64,_128,_64>, Shape<_2,_2,_1>,
|
||||
cutlass::gemm::collective::StageCountAuto,
|
||||
cutlass::gemm::KernelTmaWarpSpecialized
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveEpilogue = cutlass::epilogue::collective::DefaultEpilogue<
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::epilogue::thread::LinearCombination<cutlass::half_t, 1, float, float>>;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM90_Device_Gemm_f16t_f16n_f32n_tensor_op_gmma_f32_warpspecialized, 64x128x64_2x2x1) {
|
||||
using LayoutA = cutlass::layout::RowMajor;
|
||||
using LayoutB = cutlass::layout::ColumnMajor;
|
||||
using LayoutC = cutlass::layout::ColumnMajor;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::half_t, LayoutA, 8,
|
||||
cutlass::half_t, LayoutB, 8,
|
||||
float,
|
||||
Shape<_64,_128,_64>, Shape<_2,_2,_1>,
|
||||
cutlass::gemm::collective::StageCountAuto,
|
||||
cutlass::gemm::KernelTmaWarpSpecialized
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveEpilogue = cutlass::epilogue::collective::DefaultEpilogue<
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::epilogue::thread::LinearCombination<cutlass::half_t, 1, float, float>>;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM90_Device_Gemm_f16n_f16t_f32n_tensor_op_gmma_f32_warpspecialized, 64x128x64_2x2x1) {
|
||||
using LayoutA = cutlass::layout::ColumnMajor;
|
||||
using LayoutB = cutlass::layout::RowMajor;
|
||||
using LayoutC = cutlass::layout::ColumnMajor;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::half_t, LayoutA, 8,
|
||||
cutlass::half_t, LayoutB, 8,
|
||||
float,
|
||||
Shape<_64,_128,_64>, Shape<_2,_2,_1>,
|
||||
cutlass::gemm::collective::StageCountAuto,
|
||||
cutlass::gemm::KernelTmaWarpSpecialized
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveEpilogue = cutlass::epilogue::collective::DefaultEpilogue<
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::epilogue::thread::LinearCombination<cutlass::half_t, 1, float, float>>;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM90_Device_Gemm_f16n_f16n_f32n_tensor_op_gmma_f32_warpspecialized, 64x128x64_2x2x1) {
|
||||
using LayoutA = cutlass::layout::ColumnMajor;
|
||||
using LayoutB = cutlass::layout::ColumnMajor;
|
||||
using LayoutC = cutlass::layout::ColumnMajor;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::half_t, LayoutA, 8,
|
||||
cutlass::half_t, LayoutB, 8,
|
||||
float,
|
||||
Shape<_64,_128,_64>, Shape<_2,_2,_1>,
|
||||
cutlass::gemm::collective::StageCountAuto,
|
||||
cutlass::gemm::KernelTmaWarpSpecialized
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveEpilogue = cutlass::epilogue::collective::DefaultEpilogue<
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::epilogue::thread::LinearCombination<cutlass::half_t, 1, float, float>>;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
/////////////////////////////// Cluster 4x1x1 ////////////////////////////////
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
|
||||
TEST(SM90_Device_Gemm_f16t_f16t_f32n_tensor_op_gmma_f32_warpspecialized, 64x128x64_4x1x1) {
|
||||
using LayoutA = cutlass::layout::RowMajor;
|
||||
using LayoutB = cutlass::layout::RowMajor;
|
||||
using LayoutC = cutlass::layout::ColumnMajor;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::half_t, LayoutA, 8,
|
||||
cutlass::half_t, LayoutB, 8,
|
||||
float,
|
||||
Shape<_64,_128,_64>, Shape<_4,_1,_1>,
|
||||
cutlass::gemm::collective::StageCountAuto,
|
||||
cutlass::gemm::KernelTmaWarpSpecialized
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveEpilogue = cutlass::epilogue::collective::DefaultEpilogue<
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::epilogue::thread::LinearCombination<cutlass::half_t, 1, float, float>>;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM90_Device_Gemm_f16t_f16n_f32n_tensor_op_gmma_f32_warpspecialized, 64x128x64_4x1x1) {
|
||||
using LayoutA = cutlass::layout::RowMajor;
|
||||
using LayoutB = cutlass::layout::ColumnMajor;
|
||||
using LayoutC = cutlass::layout::ColumnMajor;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::half_t, LayoutA, 8,
|
||||
cutlass::half_t, LayoutB, 8,
|
||||
float,
|
||||
Shape<_64,_128,_64>, Shape<_4,_1,_1>,
|
||||
cutlass::gemm::collective::StageCountAuto,
|
||||
cutlass::gemm::KernelTmaWarpSpecialized
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveEpilogue = cutlass::epilogue::collective::DefaultEpilogue<
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::epilogue::thread::LinearCombination<cutlass::half_t, 1, float, float>>;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM90_Device_Gemm_f16n_f16t_f32n_tensor_op_gmma_f32_warpspecialized, 64x128x64_4x1x1) {
|
||||
using LayoutA = cutlass::layout::ColumnMajor;
|
||||
using LayoutB = cutlass::layout::RowMajor;
|
||||
using LayoutC = cutlass::layout::ColumnMajor;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::half_t, LayoutA, 8,
|
||||
cutlass::half_t, LayoutB, 8,
|
||||
float,
|
||||
Shape<_64,_128,_64>, Shape<_4,_1,_1>,
|
||||
cutlass::gemm::collective::StageCountAuto,
|
||||
cutlass::gemm::KernelTmaWarpSpecialized
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveEpilogue = cutlass::epilogue::collective::DefaultEpilogue<
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::epilogue::thread::LinearCombination<cutlass::half_t, 1, float, float>>;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM90_Device_Gemm_f16n_f16n_f32n_tensor_op_gmma_f32_warpspecialized, 64x128x64_4x1x1) {
|
||||
using LayoutA = cutlass::layout::ColumnMajor;
|
||||
using LayoutB = cutlass::layout::ColumnMajor;
|
||||
using LayoutC = cutlass::layout::ColumnMajor;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::half_t, LayoutA, 8,
|
||||
cutlass::half_t, LayoutB, 8,
|
||||
float,
|
||||
Shape<_64,_128,_64>, Shape<_4,_1,_1>,
|
||||
cutlass::gemm::collective::StageCountAuto,
|
||||
cutlass::gemm::KernelTmaWarpSpecialized
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveEpilogue = cutlass::epilogue::collective::DefaultEpilogue<
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::epilogue::thread::LinearCombination<cutlass::half_t, 1, float, float>>;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
|
||||
}
|
||||
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
/////////////////////////////// Cluster 1x4x1 ////////////////////////////////
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
|
||||
TEST(SM90_Device_Gemm_f16t_f16t_f32n_tensor_op_gmma_f32_warpspecialized, 64x128x64_1x4x1) {
|
||||
using LayoutA = cutlass::layout::RowMajor;
|
||||
using LayoutB = cutlass::layout::RowMajor;
|
||||
using LayoutC = cutlass::layout::ColumnMajor;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::half_t, LayoutA, 8,
|
||||
cutlass::half_t, LayoutB, 8,
|
||||
float,
|
||||
Shape<_64,_128,_64>, Shape<_1,_4,_1>,
|
||||
cutlass::gemm::collective::StageCountAuto,
|
||||
cutlass::gemm::KernelTmaWarpSpecialized
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveEpilogue = cutlass::epilogue::collective::DefaultEpilogue<
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::epilogue::thread::LinearCombination<cutlass::half_t, 1, float, float>>;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM90_Device_Gemm_f16t_f16n_f32n_tensor_op_gmma_f32_warpspecialized, 64x128x64_1x4x1) {
|
||||
using LayoutA = cutlass::layout::RowMajor;
|
||||
using LayoutB = cutlass::layout::ColumnMajor;
|
||||
using LayoutC = cutlass::layout::ColumnMajor;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::half_t, LayoutA, 8,
|
||||
cutlass::half_t, LayoutB, 8,
|
||||
float,
|
||||
Shape<_64,_128,_64>, Shape<_1,_4,_1>,
|
||||
cutlass::gemm::collective::StageCountAuto,
|
||||
cutlass::gemm::KernelTmaWarpSpecialized
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveEpilogue = cutlass::epilogue::collective::DefaultEpilogue<
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::epilogue::thread::LinearCombination<cutlass::half_t, 1, float, float>>;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM90_Device_Gemm_f16n_f16t_f32n_tensor_op_gmma_f32_warpspecialized, 64x128x64_1x4x1) {
|
||||
using LayoutA = cutlass::layout::ColumnMajor;
|
||||
using LayoutB = cutlass::layout::RowMajor;
|
||||
using LayoutC = cutlass::layout::ColumnMajor;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::half_t, LayoutA, 8,
|
||||
cutlass::half_t, LayoutB, 8,
|
||||
float,
|
||||
Shape<_64,_128,_64>, Shape<_1,_4,_1>,
|
||||
cutlass::gemm::collective::StageCountAuto,
|
||||
cutlass::gemm::KernelTmaWarpSpecialized
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveEpilogue = cutlass::epilogue::collective::DefaultEpilogue<
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::epilogue::thread::LinearCombination<cutlass::half_t, 1, float, float>>;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM90_Device_Gemm_f16n_f16n_f32n_tensor_op_gmma_f32_warpspecialized, 64x128x64_1x4x1) {
|
||||
using LayoutA = cutlass::layout::ColumnMajor;
|
||||
using LayoutB = cutlass::layout::ColumnMajor;
|
||||
using LayoutC = cutlass::layout::ColumnMajor;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::half_t, LayoutA, 8,
|
||||
cutlass::half_t, LayoutB, 8,
|
||||
float,
|
||||
Shape<_64,_128,_64>, Shape<_1,_4,_1>,
|
||||
cutlass::gemm::collective::StageCountAuto,
|
||||
cutlass::gemm::KernelTmaWarpSpecialized
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveEpilogue = cutlass::epilogue::collective::DefaultEpilogue<
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::epilogue::thread::LinearCombination<cutlass::half_t, 1, float, float>>;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
|
||||
}
|
||||
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
/////////////////////////////// Cluster 2x4x1 ////////////////////////////////
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
|
||||
TEST(SM90_Device_Gemm_f16t_f16t_f32n_tensor_op_gmma_f32_warpspecialized, 64x128x64_2x4x1) {
|
||||
using LayoutA = cutlass::layout::RowMajor;
|
||||
using LayoutB = cutlass::layout::RowMajor;
|
||||
using LayoutC = cutlass::layout::ColumnMajor;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::half_t, LayoutA, 8,
|
||||
cutlass::half_t, LayoutB, 8,
|
||||
float,
|
||||
Shape<_64,_128,_64>, Shape<_2,_4,_1>,
|
||||
cutlass::gemm::collective::StageCountAuto,
|
||||
cutlass::gemm::KernelTmaWarpSpecialized
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveEpilogue = cutlass::epilogue::collective::DefaultEpilogue<
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::epilogue::thread::LinearCombination<cutlass::half_t, 1, float, float>>;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM90_Device_Gemm_f16t_f16n_f32n_tensor_op_gmma_f32_warpspecialized, 64x128x64_2x4x1) {
|
||||
using LayoutA = cutlass::layout::RowMajor;
|
||||
using LayoutB = cutlass::layout::ColumnMajor;
|
||||
using LayoutC = cutlass::layout::ColumnMajor;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::half_t, LayoutA, 8,
|
||||
cutlass::half_t, LayoutB, 8,
|
||||
float,
|
||||
Shape<_64,_128,_64>, Shape<_2,_4,_1>,
|
||||
cutlass::gemm::collective::StageCountAuto,
|
||||
cutlass::gemm::KernelTmaWarpSpecialized
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveEpilogue = cutlass::epilogue::collective::DefaultEpilogue<
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::epilogue::thread::LinearCombination<cutlass::half_t, 1, float, float>>;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM90_Device_Gemm_f16n_f16t_f32n_tensor_op_gmma_f32_warpspecialized, 64x128x64_2x4x1) {
|
||||
using LayoutA = cutlass::layout::ColumnMajor;
|
||||
using LayoutB = cutlass::layout::RowMajor;
|
||||
using LayoutC = cutlass::layout::ColumnMajor;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::half_t, LayoutA, 8,
|
||||
cutlass::half_t, LayoutB, 8,
|
||||
float,
|
||||
Shape<_64,_128,_64>, Shape<_2,_4,_1>,
|
||||
cutlass::gemm::collective::StageCountAuto,
|
||||
cutlass::gemm::KernelTmaWarpSpecialized
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveEpilogue = cutlass::epilogue::collective::DefaultEpilogue<
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::epilogue::thread::LinearCombination<cutlass::half_t, 1, float, float>>;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM90_Device_Gemm_f16n_f16n_f32n_tensor_op_gmma_f32_warpspecialized, 64x128x64_2x4x1) {
|
||||
using LayoutA = cutlass::layout::ColumnMajor;
|
||||
using LayoutB = cutlass::layout::ColumnMajor;
|
||||
using LayoutC = cutlass::layout::ColumnMajor;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::half_t, LayoutA, 8,
|
||||
cutlass::half_t, LayoutB, 8,
|
||||
float,
|
||||
Shape<_64,_128,_64>, Shape<_2,_4,_1>,
|
||||
cutlass::gemm::collective::StageCountAuto,
|
||||
cutlass::gemm::KernelTmaWarpSpecialized
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveEpilogue = cutlass::epilogue::collective::DefaultEpilogue<
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::epilogue::thread::LinearCombination<cutlass::half_t, 1, float, float>>;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
|
||||
}
|
||||
|
||||
#endif // defined(CUTLASS_ARCH_MMA_SM90_SUPPORTED)
|
||||
+1018
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,86 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/*! \file
|
||||
\brief Tests for device-wide GEMM interface
|
||||
*/
|
||||
|
||||
#include <iostream>
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cute/tensor.hpp"
|
||||
#include "cute/atom/mma_atom.hpp"
|
||||
|
||||
#include "cutlass/numeric_types.h"
|
||||
|
||||
#include "cutlass/gemm/device/gemm_universal_adapter.h"
|
||||
#include "cutlass/gemm/kernel/gemm_universal.hpp"
|
||||
#include "cutlass/gemm/collective/collective_builder.hpp"
|
||||
#include "cutlass/epilogue/collective/default_epilogue.hpp"
|
||||
#include "cutlass/epilogue/collective/default_transposed_epilogue.hpp"
|
||||
#include "cutlass/epilogue/thread/linear_combination.h"
|
||||
|
||||
#include "../../common/cutlass_unit_test.h"
|
||||
|
||||
#include "gemm_testbed_3x.hpp"
|
||||
|
||||
#if defined(CUTLASS_ARCH_MMA_SM90_SUPPORTED)
|
||||
|
||||
using namespace cute;
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM90_Device_Gemm_f32t_f32n_f32n_tensor_op_gmma_f32, 64x128x32_1x2x1) {
|
||||
using LayoutA = cutlass::layout::RowMajor;
|
||||
using LayoutB = cutlass::layout::ColumnMajor;
|
||||
using LayoutC = cutlass::layout::ColumnMajor;
|
||||
|
||||
using CollectiveMainloop = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
|
||||
float, LayoutA, 4,
|
||||
float, LayoutB, 4,
|
||||
float,
|
||||
Shape<_64,_128,_128>, Shape<_1,_2,_1>,
|
||||
cutlass::gemm::collective::StageCountAuto,
|
||||
cutlass::gemm::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using CollectiveEpilogue = cutlass::epilogue::collective::DefaultEpilogue<
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::epilogue::thread::LinearCombination<float, 1, float, float>>;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
CollectiveMainloop,
|
||||
CollectiveEpilogue
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#endif // defined(CUTLASS_ARCH_MMA_SM90_SUPPORTED)
|
||||
@@ -0,0 +1,152 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017 - 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/*! \file
|
||||
\brief Tests for device-wide GEMM interface
|
||||
*/
|
||||
|
||||
#include <iostream>
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cute/tensor.hpp"
|
||||
#include "cute/atom/mma_atom.hpp"
|
||||
|
||||
#include "cutlass/numeric_types.h"
|
||||
|
||||
#include "cutlass/gemm/gemm.h"
|
||||
#include "cutlass/gemm/device/gemm_universal_adapter.h"
|
||||
#include "cutlass/gemm/kernel/gemm_universal.hpp"
|
||||
#include "cutlass/gemm/collective/collective_builder.hpp"
|
||||
#include "cutlass/epilogue/collective/default_epilogue.hpp"
|
||||
#include "cutlass/epilogue/thread/linear_combination.h"
|
||||
|
||||
#include "../../common/cutlass_unit_test.h"
|
||||
|
||||
#include "gemm_testbed_3x.hpp"
|
||||
|
||||
#if defined(CUTLASS_ARCH_MMA_SM90_SUPPORTED)
|
||||
|
||||
using namespace cute;
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM90_Device_Gemm_s8t_s8n_s8n_align8_tensor_op_gmma_s32, 64x128x128) {
|
||||
using LayoutA = cutlass::layout::RowMajor;
|
||||
using LayoutB = cutlass::layout::ColumnMajor;
|
||||
using LayoutC = cutlass::layout::ColumnMajor;
|
||||
|
||||
using CollectiveOp = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
|
||||
int8_t, LayoutA, 8,
|
||||
int8_t, LayoutB, 8,
|
||||
int32_t,
|
||||
Shape<_64,_128,_128>, Shape<_1,_1,_1>,
|
||||
cutlass::gemm::collective::StageCountAuto,
|
||||
cutlass::gemm::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using EpilogueOp = cutlass::epilogue::collective::DefaultEpilogue<
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::epilogue::thread::LinearCombination<int8_t, 1, int32_t, int32_t>>;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
CollectiveOp,
|
||||
EpilogueOp
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
|
||||
}
|
||||
|
||||
TEST(SM90_Device_Gemm_s8t_s8n_s8n_align16_tensor_op_gmma_s32, 128x128x128) {
|
||||
using LayoutA = cutlass::layout::RowMajor;
|
||||
using LayoutB = cutlass::layout::ColumnMajor;
|
||||
using LayoutC = cutlass::layout::ColumnMajor;
|
||||
|
||||
using CollectiveOp = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
|
||||
int8_t, LayoutA, 16,
|
||||
int8_t, LayoutB, 16,
|
||||
int32_t,
|
||||
Shape<_128,_128,_128>, Shape<_1,_1,_1>,
|
||||
cutlass::gemm::collective::StageCountAuto,
|
||||
cutlass::gemm::KernelMultistage
|
||||
>::CollectiveOp;
|
||||
|
||||
using EpilogueOp = cutlass::epilogue::collective::DefaultEpilogue<
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::epilogue::thread::LinearCombination<int8_t, 1, int32_t, int32_t>>;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
CollectiveOp,
|
||||
EpilogueOp
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
|
||||
}
|
||||
|
||||
TEST(SM90_Device_Gemm_s8t_s8n_s8n_align4_tensor_op_gmma_s32, 128x64x128) {
|
||||
using LayoutA = cutlass::layout::RowMajor;
|
||||
using LayoutB = cutlass::layout::ColumnMajor;
|
||||
using LayoutC = cutlass::layout::ColumnMajor;
|
||||
|
||||
using CollectiveOp = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
|
||||
int8_t, LayoutA, 4,
|
||||
int8_t, LayoutB, 4,
|
||||
int32_t,
|
||||
Shape<_128,_64,_128>, Shape<_1,_1,_1>,
|
||||
cutlass::gemm::collective::StageCountAuto,
|
||||
cutlass::gemm::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using EpilogueOp = cutlass::epilogue::collective::DefaultEpilogue<
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::epilogue::thread::LinearCombination<int8_t, 1, int32_t, int32_t>>;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
CollectiveOp,
|
||||
EpilogueOp
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#endif // defined(CUTLASS_ARCH_MMA_SM90_SUPPORTED)
|
||||
@@ -0,0 +1,243 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017 - 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/*! \file
|
||||
\brief Tests for device-wide GEMM interface
|
||||
*/
|
||||
|
||||
#include <iostream>
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cute/tensor.hpp"
|
||||
#include "cute/atom/mma_atom.hpp"
|
||||
|
||||
#include "cutlass/numeric_types.h"
|
||||
|
||||
#include "cutlass/gemm/device/gemm_universal_adapter.h"
|
||||
#include "cutlass/gemm/kernel/gemm_universal.hpp"
|
||||
#include "cutlass/gemm/collective/collective_builder.hpp"
|
||||
#include "cutlass/epilogue/collective/default_epilogue.hpp"
|
||||
#include "cutlass/epilogue/thread/linear_combination.h"
|
||||
|
||||
#include "../../common/cutlass_unit_test.h"
|
||||
|
||||
#include "gemm_testbed_3x.hpp"
|
||||
|
||||
#if defined(CUTLASS_ARCH_MMA_SM90_SUPPORTED)
|
||||
|
||||
using namespace cute;
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM90_Device_Gemm_s8t_s8n_s8n_tensor_op_gmma_s32, 64x128x128) {
|
||||
using LayoutA = cutlass::layout::RowMajor;
|
||||
using LayoutB = cutlass::layout::ColumnMajor;
|
||||
using LayoutC = cutlass::layout::ColumnMajor;
|
||||
|
||||
using CollectiveOp = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
|
||||
int8_t, LayoutA, 16,
|
||||
int8_t, LayoutB, 16,
|
||||
int32_t,
|
||||
Shape<_64,_128,_128>, Shape<_1,_1,_1>,
|
||||
cutlass::gemm::collective::StageCountAuto,
|
||||
cutlass::gemm::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using EpilogueOp = cutlass::epilogue::collective::DefaultEpilogue<
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::epilogue::thread::LinearCombination<int8_t, 1, int32_t, int32_t>>;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
CollectiveOp,
|
||||
EpilogueOp
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM90_Device_Gemm_s8t_s8n_s8n_tensor_op_gmma_s32, 64x128x128_1x2x1) {
|
||||
using LayoutA = cutlass::layout::RowMajor;
|
||||
using LayoutB = cutlass::layout::ColumnMajor;
|
||||
using LayoutC = cutlass::layout::ColumnMajor;
|
||||
|
||||
using CollectiveOp = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
|
||||
int8_t, LayoutA, 16,
|
||||
int8_t, LayoutB, 16,
|
||||
int32_t,
|
||||
Shape<_64,_128,_128>, Shape<_1,_2,_1>,
|
||||
cutlass::gemm::collective::StageCountAuto,
|
||||
cutlass::gemm::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using EpilogueOp = cutlass::epilogue::collective::DefaultEpilogue<
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::epilogue::thread::LinearCombination<int8_t, 1, int32_t, int32_t>>;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
CollectiveOp,
|
||||
EpilogueOp
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
|
||||
}
|
||||
|
||||
TEST(SM90_Device_Gemm_s8t_s8n_s8n_tensor_op_gmma_s32, 128x128x128) {
|
||||
using LayoutA = cutlass::layout::RowMajor;
|
||||
using LayoutB = cutlass::layout::ColumnMajor;
|
||||
using LayoutC = cutlass::layout::ColumnMajor;
|
||||
|
||||
using CollectiveOp = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
|
||||
int8_t, LayoutA, 16,
|
||||
int8_t, LayoutB, 16,
|
||||
int32_t,
|
||||
Shape<_128,_128,_128>, Shape<_1,_1,_1>,
|
||||
cutlass::gemm::collective::StageCountAuto,
|
||||
cutlass::gemm::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using EpilogueOp = cutlass::epilogue::collective::DefaultEpilogue<
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::epilogue::thread::LinearCombination<int8_t, 1, int32_t, int32_t>>;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
CollectiveOp,
|
||||
EpilogueOp
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
|
||||
}
|
||||
|
||||
TEST(SM90_Device_Gemm_s8t_s8n_s8n_tensor_op_gmma_s32, 128x128x128_1x2x1) {
|
||||
using LayoutA = cutlass::layout::RowMajor;
|
||||
using LayoutB = cutlass::layout::ColumnMajor;
|
||||
using LayoutC = cutlass::layout::ColumnMajor;
|
||||
|
||||
using CollectiveOp = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
|
||||
int8_t, LayoutA, 16,
|
||||
int8_t, LayoutB, 16,
|
||||
int32_t,
|
||||
Shape<_128,_128,_128>, Shape<_1,_2,_1>,
|
||||
cutlass::gemm::collective::StageCountAuto,
|
||||
cutlass::gemm::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using EpilogueOp = cutlass::epilogue::collective::DefaultEpilogue<
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::epilogue::thread::LinearCombination<int8_t, 1, int32_t, int32_t>>;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
CollectiveOp,
|
||||
EpilogueOp
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
|
||||
}
|
||||
|
||||
TEST(SM90_Device_Gemm_s8t_s8n_s8n_tensor_op_gmma_s32, 128x128x128_2x1x1) {
|
||||
using LayoutA = cutlass::layout::RowMajor;
|
||||
using LayoutB = cutlass::layout::ColumnMajor;
|
||||
using LayoutC = cutlass::layout::ColumnMajor;
|
||||
|
||||
using CollectiveOp = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
|
||||
int8_t, LayoutA, 16,
|
||||
int8_t, LayoutB, 16,
|
||||
int32_t,
|
||||
Shape<_128,_128,_128>, Shape<_2,_1,_1>,
|
||||
cutlass::gemm::collective::StageCountAuto,
|
||||
cutlass::gemm::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using EpilogueOp = cutlass::epilogue::collective::DefaultEpilogue<
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::epilogue::thread::LinearCombination<int8_t, 1, int32_t, int32_t>>;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
CollectiveOp,
|
||||
EpilogueOp
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
|
||||
}
|
||||
|
||||
TEST(SM90_Device_Gemm_s8t_s8n_s8n_tensor_op_gmma_s32, 128x128x128_2x2x1) {
|
||||
using LayoutA = cutlass::layout::RowMajor;
|
||||
using LayoutB = cutlass::layout::ColumnMajor;
|
||||
using LayoutC = cutlass::layout::ColumnMajor;
|
||||
|
||||
using CollectiveOp = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
|
||||
int8_t, LayoutA, 16,
|
||||
int8_t, LayoutB, 16,
|
||||
int32_t,
|
||||
Shape<_128,_128,_128>, Shape<_2,_2,_1>,
|
||||
cutlass::gemm::collective::StageCountAuto,
|
||||
cutlass::gemm::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using EpilogueOp = cutlass::epilogue::collective::DefaultEpilogue<
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::epilogue::thread::LinearCombination<int8_t, 1, int32_t, int32_t>>;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
CollectiveOp,
|
||||
EpilogueOp
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#endif // defined(CUTLASS_ARCH_MMA_SM90_SUPPORTED)
|
||||
@@ -0,0 +1,151 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017 - 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/*! \file
|
||||
\brief Tests for device-wide GEMM interface
|
||||
*/
|
||||
|
||||
#include <iostream>
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cute/tensor.hpp"
|
||||
#include "cute/atom/mma_atom.hpp"
|
||||
|
||||
#include "cutlass/numeric_types.h"
|
||||
|
||||
#include "cutlass/gemm/device/gemm_universal_adapter.h"
|
||||
#include "cutlass/gemm/kernel/gemm_universal.hpp"
|
||||
#include "cutlass/gemm/collective/collective_builder.hpp"
|
||||
#include "cutlass/epilogue/collective/default_epilogue.hpp"
|
||||
#include "cutlass/epilogue/thread/linear_combination.h"
|
||||
|
||||
#include "../../common/cutlass_unit_test.h"
|
||||
|
||||
#include "gemm_testbed_3x.hpp"
|
||||
|
||||
#if defined(CUTLASS_ARCH_MMA_SM90_SUPPORTED)
|
||||
|
||||
using namespace cute;
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM90_Device_Gemm_tf32t_tf32n_f32n_align4_tensor_op_gmma_f32, 64x128x32) {
|
||||
using LayoutA = cutlass::layout::RowMajor;
|
||||
using LayoutB = cutlass::layout::ColumnMajor;
|
||||
using LayoutC = cutlass::layout::ColumnMajor;
|
||||
|
||||
using CollectiveOp = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
|
||||
tfloat32_t, LayoutA, 4,
|
||||
tfloat32_t, LayoutB, 4,
|
||||
float,
|
||||
Shape<_64,_128,_32>, Shape<_1,_1,_1>,
|
||||
cutlass::gemm::collective::StageCountAuto,
|
||||
cutlass::gemm::KernelMultistage
|
||||
>::CollectiveOp;
|
||||
|
||||
using EpilogueOp = cutlass::epilogue::collective::DefaultEpilogue<
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::epilogue::thread::LinearCombination<float, 1, float, float>>;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
CollectiveOp,
|
||||
EpilogueOp
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
|
||||
}
|
||||
|
||||
TEST(SM90_Device_Gemm_tf32t_tf32n_f32n_align2_tensor_op_gmma_f32, 64x64x32) {
|
||||
using LayoutA = cutlass::layout::RowMajor;
|
||||
using LayoutB = cutlass::layout::ColumnMajor;
|
||||
using LayoutC = cutlass::layout::ColumnMajor;
|
||||
|
||||
using CollectiveOp = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::tfloat32_t, LayoutA, 2,
|
||||
cutlass::tfloat32_t, LayoutB, 2,
|
||||
float,
|
||||
Shape<_64,_64,_32>, Shape<_1,_1,_1>,
|
||||
cutlass::gemm::collective::StageCountAuto,
|
||||
cutlass::gemm::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using EpilogueOp = cutlass::epilogue::collective::DefaultEpilogue<
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::epilogue::thread::LinearCombination<float, 1, float, float>>;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
CollectiveOp,
|
||||
EpilogueOp
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
|
||||
}
|
||||
|
||||
TEST(SM90_Device_Gemm_tf32t_tf32n_f32n_align1_tensor_op_gmma_f32, 128x64x32) {
|
||||
using LayoutA = cutlass::layout::RowMajor;
|
||||
using LayoutB = cutlass::layout::ColumnMajor;
|
||||
using LayoutC = cutlass::layout::ColumnMajor;
|
||||
|
||||
using CollectiveOp = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::tfloat32_t, LayoutA, 1,
|
||||
cutlass::tfloat32_t, LayoutB, 1,
|
||||
float,
|
||||
Shape<_128,_64,_32>, Shape<_1,_1,_1>,
|
||||
cutlass::gemm::collective::StageCountAuto,
|
||||
cutlass::gemm::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using EpilogueOp = cutlass::epilogue::collective::DefaultEpilogue<
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::epilogue::thread::LinearCombination<float, 1, float, float>>;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
CollectiveOp,
|
||||
EpilogueOp
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#endif // defined(CUTLASS_ARCH_MMA_SM90_SUPPORTED)
|
||||
@@ -0,0 +1,185 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017 - 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/*! \file
|
||||
\brief Tests for device-wide GEMM interface
|
||||
*/
|
||||
|
||||
#include <iostream>
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cute/tensor.hpp"
|
||||
#include "cute/atom/mma_atom.hpp"
|
||||
|
||||
#include "cutlass/numeric_types.h"
|
||||
|
||||
#include "cutlass/gemm/device/gemm_universal_adapter.h"
|
||||
#include "cutlass/gemm/kernel/gemm_universal.hpp"
|
||||
#include "cutlass/gemm/collective/collective_builder.hpp"
|
||||
#include "cutlass/epilogue/collective/default_epilogue.hpp"
|
||||
#include "cutlass/epilogue/thread/linear_combination.h"
|
||||
|
||||
#include "../../common/cutlass_unit_test.h"
|
||||
|
||||
#include "gemm_testbed_3x.hpp"
|
||||
|
||||
#if defined(CUTLASS_ARCH_MMA_SM90_SUPPORTED)
|
||||
|
||||
using namespace cute;
|
||||
|
||||
TEST(SM90_Device_Gemm_tf32t_tf32n_f32n_tensor_op_gmma_f32, 64x128x32) {
|
||||
using LayoutA = cutlass::layout::RowMajor;
|
||||
using LayoutB = cutlass::layout::ColumnMajor;
|
||||
using LayoutC = cutlass::layout::ColumnMajor;
|
||||
|
||||
using CollectiveOp = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::tfloat32_t, LayoutA, 4,
|
||||
cutlass::tfloat32_t, LayoutB, 4,
|
||||
float,
|
||||
Shape<_64,_128,_32>, Shape<_1,_1,_1>,
|
||||
cutlass::gemm::collective::StageCountAuto,
|
||||
cutlass::gemm::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using EpilogueOp = cutlass::epilogue::collective::DefaultEpilogue<
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::epilogue::thread::LinearCombination<float, 1, float, float>>;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
CollectiveOp,
|
||||
EpilogueOp
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM90_Device_Gemm_tf32n_tf32n_f32n_tensor_op_gmma_f32, 64x128x32) {
|
||||
using LayoutA = cutlass::layout::ColumnMajor;
|
||||
using LayoutB = cutlass::layout::ColumnMajor;
|
||||
using LayoutC = cutlass::layout::ColumnMajor;
|
||||
|
||||
using CollectiveOp = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::tfloat32_t, LayoutA, 1,
|
||||
cutlass::tfloat32_t, LayoutB, 4,
|
||||
float,
|
||||
Shape<_64,_128,_32>, Shape<_1,_1,_1>,
|
||||
cutlass::gemm::collective::StageCountAuto,
|
||||
cutlass::gemm::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using EpilogueOp = cutlass::epilogue::collective::DefaultEpilogue<
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::epilogue::thread::LinearCombination<float, 1, float, float>>;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
CollectiveOp,
|
||||
EpilogueOp
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM90_Device_Gemm_tf32n_tf32t_f32n_tensor_op_gmma_f32, 64x128x32) {
|
||||
using LayoutA = cutlass::layout::ColumnMajor;
|
||||
using LayoutB = cutlass::layout::RowMajor;
|
||||
using LayoutC = cutlass::layout::ColumnMajor;
|
||||
|
||||
using CollectiveOp = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::tfloat32_t, LayoutA, 1,
|
||||
cutlass::tfloat32_t, LayoutB, 1,
|
||||
float,
|
||||
Shape<_64,_128,_32>, Shape<_1,_1,_1>,
|
||||
cutlass::gemm::collective::StageCountAuto,
|
||||
cutlass::gemm::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using EpilogueOp = cutlass::epilogue::collective::DefaultEpilogue<
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::epilogue::thread::LinearCombination<float, 1, float, float>>;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
CollectiveOp,
|
||||
EpilogueOp
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM90_Device_Gemm_tf32t_tf32t_f32n_tensor_op_gmma_f32, 64x128x32) {
|
||||
using LayoutA = cutlass::layout::RowMajor;
|
||||
using LayoutB = cutlass::layout::RowMajor;
|
||||
using LayoutC = cutlass::layout::ColumnMajor;
|
||||
|
||||
using CollectiveOp = typename cutlass::gemm::collective::CollectiveBuilder<
|
||||
cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp,
|
||||
cutlass::tfloat32_t, LayoutA, 4,
|
||||
cutlass::tfloat32_t, LayoutB, 1,
|
||||
float,
|
||||
Shape<_64,_128,_32>, Shape<_1,_1,_1>,
|
||||
cutlass::gemm::collective::StageCountAuto,
|
||||
cutlass::gemm::collective::KernelScheduleAuto
|
||||
>::CollectiveOp;
|
||||
|
||||
using EpilogueOp = cutlass::epilogue::collective::DefaultEpilogue<
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::gemm::TagToStrideC_t<LayoutC>,
|
||||
cutlass::epilogue::thread::LinearCombination<float, 1, float, float>>;
|
||||
|
||||
using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
|
||||
Shape<int,int,int,int>,
|
||||
CollectiveOp,
|
||||
EpilogueOp
|
||||
>;
|
||||
|
||||
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
|
||||
EXPECT_TRUE(test::gemm::device::TestAll<Gemm>());
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#endif // defined(CUTLASS_ARCH_MMA_SM90_SUPPORTED)
|
||||
@@ -48,7 +48,7 @@
|
||||
|
||||
#include "testbed_symm_universal.h"
|
||||
|
||||
#if defined(CUTLASS_ARCH_MMA_SM90_SUPPORTED)
|
||||
#if defined(CUTLASS_ARCH_MMA_SM90_F64_MMA_ENABLED)
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
@@ -130,4 +130,4 @@ TEST(SM90_Device_Symm_cf64n_cf64n_rs_u_tensor_op_f64, 64x64x16_32x32x16) {
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#endif // #if defined(CUTLASS_ARCH_MMA_SM90_SUPPORTED)
|
||||
#endif // #if defined(CUTLASS_ARCH_MMA_SM90_F64_MMA_ENABLED)
|
||||
|
||||
@@ -47,7 +47,7 @@
|
||||
|
||||
#include "testbed_symm_universal.h"
|
||||
|
||||
#if defined(CUTLASS_ARCH_MMA_SM90_SUPPORTED)
|
||||
#if defined(CUTLASS_ARCH_MMA_SM90_F64_MMA_ENABLED)
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
@@ -132,4 +132,4 @@ TEST(SM90_Device_Symm_f64t_f64t_ls_l_tensor_op_f64, 128x128x16_32x64x16) {
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
#endif // #if defined(CUTLASS_ARCH_MMA_SM90_SUPPORTED)
|
||||
#endif // #if defined(CUTLASS_ARCH_MMA_SM90_F64_MMA_ENABLED)
|
||||
|
||||
@@ -47,7 +47,7 @@
|
||||
|
||||
#include "testbed_rank2k_universal.h"
|
||||
|
||||
#if defined(CUTLASS_ARCH_MMA_SM90_SUPPORTED)
|
||||
#if defined(CUTLASS_ARCH_MMA_SM90_F64_MMA_ENABLED)
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
@@ -147,4 +147,4 @@ TEST(SM90_Device_Syr2k_cf64n_cf64t_u_tensor_op_f64, 32x32x16_16x16x16) {
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#endif // #if defined(CUTLASS_ARCH_MMA_SM90_SUPPORTED)
|
||||
#endif // #if defined(CUTLASS_ARCH_MMA_SM90_F64_MMA_ENABLED)
|
||||
|
||||
@@ -47,7 +47,7 @@
|
||||
|
||||
#include "testbed_rank2k_universal.h"
|
||||
|
||||
#if defined(CUTLASS_ARCH_MMA_SM90_SUPPORTED)
|
||||
#if defined(CUTLASS_ARCH_MMA_SM90_F64_MMA_ENABLED)
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
@@ -131,4 +131,4 @@ TEST(SM90_Device_Syr2k_f64t_f64n_l_tensor_op_f64, 128x128x16_32x64x16) {
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#endif // #if defined(CUTLASS_ARCH_MMA_SM90_SUPPORTED)
|
||||
#endif // #if defined(CUTLASS_ARCH_MMA_SM90_F64_MMA_ENABLED)
|
||||
|
||||
@@ -47,7 +47,7 @@
|
||||
|
||||
#include "testbed_rank_k_universal.h"
|
||||
|
||||
#if defined(CUTLASS_ARCH_MMA_SM90_SUPPORTED)
|
||||
#if defined(CUTLASS_ARCH_MMA_SM90_F64_MMA_ENABLED)
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
@@ -133,4 +133,4 @@ TEST(SM90_Device_Syrk_cf64n_cf64t_l_tensor_op_f64_gaussian, 32x32x16_16x16x16) {
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#endif // #if defined(CUTLASS_ARCH_MMA_SM90_SUPPORTED)
|
||||
#endif // #if defined(CUTLASS_ARCH_MMA_SM90_F64_MMA_ENABLED)
|
||||
|
||||
@@ -47,7 +47,7 @@
|
||||
|
||||
#include "testbed_rank_k_universal.h"
|
||||
|
||||
#if defined(CUTLASS_ARCH_MMA_SM90_SUPPORTED)
|
||||
#if defined(CUTLASS_ARCH_MMA_SM90_F64_MMA_ENABLED)
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
@@ -123,4 +123,4 @@ TEST(SM90_Device_Syrk_f64t_f64n_l_tensor_op_f64, 32x32x16_16x16x16) {
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#endif // #if defined(CUTLASS_ARCH_MMA_SM90_SUPPORTED)
|
||||
#endif // #if defined(CUTLASS_ARCH_MMA_SM90_F64_MMA_ENABLED)
|
||||
|
||||
@@ -65,6 +65,9 @@ namespace device {
|
||||
template <typename Gemm, bool Relu = false>
|
||||
struct Testbed {
|
||||
|
||||
using ElementA = typename Gemm::ElementA;
|
||||
using ElementB = typename Gemm::ElementB;
|
||||
using ElementC = typename Gemm::ElementC;
|
||||
using ElementAccumulator = typename Gemm::ElementAccumulator;
|
||||
using ElementCompute = typename Gemm::GemmKernel::Epilogue::OutputOp::ElementCompute;
|
||||
|
||||
|
||||
@@ -63,6 +63,9 @@ template <typename Gemm>
|
||||
struct TestbedComplex : public Testbed<Gemm> {
|
||||
|
||||
using Base = Testbed<Gemm>;
|
||||
using ElementA = typename Gemm::ElementA;
|
||||
using ElementB = typename Gemm::ElementB;
|
||||
using ElementC = typename Gemm::ElementC;
|
||||
using ElementAccumulator = typename Gemm::ElementAccumulator;
|
||||
using ElementCompute = typename Gemm::GemmKernel::Epilogue::OutputOp::ElementCompute;
|
||||
|
||||
@@ -131,7 +134,7 @@ struct TestbedComplex : public Testbed<Gemm> {
|
||||
if (properties.sharedMemPerBlockOptin < smem_size) {
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
@@ -100,6 +100,8 @@ template <
|
||||
>
|
||||
struct TestbedGemmWithBroadcast {
|
||||
|
||||
using ElementA = typename Gemm::ElementA;
|
||||
using ElementB = typename Gemm::ElementB;
|
||||
using OutputOp = typename Gemm::GemmKernel::Epilogue::OutputOp;
|
||||
using ElementC = typename Gemm::ElementC;
|
||||
using ElementAccumulator = typename Gemm::ElementAccumulator;
|
||||
|
||||
@@ -61,6 +61,7 @@ namespace device {
|
||||
|
||||
template <typename Gemm, typename BinaryOp>
|
||||
struct GemmWithReductionReference {
|
||||
|
||||
using ElementAccumulator = typename Gemm::ElementAccumulator;
|
||||
using ElementCompute = typename Gemm::GemmKernel::Epilogue::ElementCompute;
|
||||
using ElementC = typename Gemm::ElementC;
|
||||
@@ -93,6 +94,9 @@ template <
|
||||
>
|
||||
struct TestbedGemmWithReduction {
|
||||
|
||||
using ElementA = typename Gemm::ElementA;
|
||||
using ElementB = typename Gemm::ElementB;
|
||||
using ElementC = typename Gemm::ElementC;
|
||||
using ElementAccumulator = typename Gemm::ElementAccumulator;
|
||||
using ElementT = typename Gemm::GemmKernel::Epilogue::ElementTensor;
|
||||
|
||||
|
||||
@@ -57,6 +57,9 @@ namespace device {
|
||||
template <typename Gemm, int InterleavedK>
|
||||
struct InterleavedTestbed {
|
||||
|
||||
using ElementA = typename Gemm::ElementA;
|
||||
using ElementB = typename Gemm::ElementB;
|
||||
using ElementC = typename Gemm::ElementC;
|
||||
using ElementAccumulator = typename Gemm::ElementAccumulator;
|
||||
using ElementCompute = typename Gemm::GemmKernel::Epilogue::OutputOp::ElementCompute;
|
||||
|
||||
|
||||
@@ -64,6 +64,9 @@ namespace device {
|
||||
template <typename Rank2K>
|
||||
struct TestbedRank2KUniversal {
|
||||
|
||||
using ElementA = typename Rank2K::ElementA;
|
||||
using ElementB = typename Rank2K::ElementB;
|
||||
using ElementC = typename Rank2K::ElementC;
|
||||
using ElementAccumulator = typename Rank2K::ElementAccumulator;
|
||||
using ElementCompute = typename Rank2K::Rank2Kkernel::Epilogue::OutputOp::ElementCompute;
|
||||
|
||||
@@ -301,7 +304,6 @@ struct TestbedRank2KUniversal {
|
||||
if (properties.sharedMemPerBlockOptin < smem_size) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
@@ -63,6 +63,8 @@ namespace device {
|
||||
template <typename RankK>
|
||||
struct TestbedRank2KUniversal {
|
||||
|
||||
using ElementA = typename RankK::ElementA;
|
||||
using ElementC = typename RankK::ElementC;
|
||||
using ElementAccumulator = typename RankK::ElementAccumulator;
|
||||
using ElementCompute = typename RankK::RankKkernel::Epilogue::OutputOp::ElementCompute;
|
||||
|
||||
|
||||
@@ -64,6 +64,9 @@ namespace device {
|
||||
template <typename Gemm>
|
||||
struct SparseTestbed {
|
||||
|
||||
using ElementA = typename Gemm::ElementA;
|
||||
using ElementB = typename Gemm::ElementB;
|
||||
using ElementC = typename Gemm::ElementC;
|
||||
using ElementAccumulator = typename Gemm::ElementAccumulator;
|
||||
using ElementCompute = typename Gemm::GemmKernel::Epilogue::OutputOp::ElementCompute;
|
||||
|
||||
|
||||
@@ -64,6 +64,9 @@ namespace device {
|
||||
template <typename Symm>
|
||||
struct TestbedSymmUniversal {
|
||||
|
||||
using ElementA = typename Symm::ElementA;
|
||||
using ElementB = typename Symm::ElementB;
|
||||
using ElementC = typename Symm::ElementC;
|
||||
using ElementAccumulator = typename Symm::ElementAccumulator;
|
||||
using ElementCompute = typename Symm::SymmKernel::Epilogue::OutputOp::ElementCompute;
|
||||
|
||||
|
||||
@@ -66,6 +66,9 @@ namespace device {
|
||||
template <typename Trmm>
|
||||
struct TestbedTrmmUniversal {
|
||||
|
||||
using ElementA = typename Trmm::ElementA;
|
||||
using ElementB = typename Trmm::ElementB;
|
||||
using ElementC = typename Trmm::ElementC;
|
||||
using ElementAccumulator = typename Trmm::ElementAccumulator;
|
||||
using ElementCompute = typename Trmm::TrmmKernel::Epilogue::OutputOp::ElementCompute;
|
||||
|
||||
|
||||
@@ -61,6 +61,9 @@ namespace device {
|
||||
template <typename Gemm, bool Relu = false>
|
||||
struct TestbedUniversal {
|
||||
|
||||
using ElementA = typename Gemm::ElementA;
|
||||
using ElementB = typename Gemm::ElementB;
|
||||
using ElementC = typename Gemm::ElementC;
|
||||
using ElementAccumulator = typename Gemm::ElementAccumulator;
|
||||
using ElementCompute = typename Gemm::GemmKernel::Epilogue::OutputOp::ElementCompute;
|
||||
|
||||
|
||||
@@ -48,7 +48,7 @@
|
||||
|
||||
#include "testbed_trmm_universal.h"
|
||||
|
||||
#if defined(CUTLASS_ARCH_MMA_SM90_SUPPORTED)
|
||||
#if defined(CUTLASS_ARCH_MMA_SM90_F64_MMA_ENABLED)
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
@@ -134,4 +134,4 @@ TEST(SM90_Device_Trmm_cf64h_cf64n_cf64t_ls_u_nu_tensor_op_f64, 64x64x16_32x32x16
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#endif // #if defined(CUTLASS_ARCH_MMA_SM90_SUPPORTED)
|
||||
#endif // #if defined(CUTLASS_ARCH_MMA_SM90_F64_MMA_ENABLED)
|
||||
|
||||
@@ -48,7 +48,7 @@
|
||||
|
||||
#include "testbed_trmm_universal.h"
|
||||
|
||||
#if defined(CUTLASS_ARCH_MMA_SM90_SUPPORTED)
|
||||
#if defined(CUTLASS_ARCH_MMA_SM90_F64_MMA_ENABLED)
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
@@ -124,4 +124,4 @@ TEST(SM90_Device_Trmm_f64t_f64t_f64n_rs_l_nu_tensor_op_f64, 64x64x16_32x32x16) {
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#endif // #if defined(CUTLASS_ARCH_MMA_SM90_SUPPORTED)
|
||||
#endif // #if defined(CUTLASS_ARCH_MMA_SM90_F64_MMA_ENABLED)
|
||||
|
||||
@@ -241,8 +241,6 @@ struct SparseTestbed {
|
||||
// Determine SMEM requirements and waive if not satisfied
|
||||
//
|
||||
|
||||
int smem_size = int(sizeof(typename Mma::SharedStorage));
|
||||
|
||||
cudaDeviceProp properties;
|
||||
int device_idx;
|
||||
cudaError_t result = cudaGetDevice(&device_idx);
|
||||
@@ -257,10 +255,6 @@ struct SparseTestbed {
|
||||
throw std::runtime_error("cudaGetDeviceProperties() failed");
|
||||
}
|
||||
|
||||
if (properties.sharedMemPerBlockOptin < smem_size) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -415,7 +409,12 @@ struct SparseTestbed {
|
||||
bool passed = cutlass::reference::host::TensorEquals(
|
||||
matrix_C_computed.host_view(), matrix_C_reference.host_view());
|
||||
|
||||
EXPECT_TRUE(passed)
|
||||
EXPECT_TRUE(passed);
|
||||
|
||||
if (!passed && CUTLASS_TEST_UNIT_ENABLE_WARNINGS) {
|
||||
|
||||
std::cout
|
||||
<< __FILE__ << ":" << __LINE__ << " "
|
||||
<< "A:\n" << matrix_A.host_view() << "\n"
|
||||
<< "B:\n" << matrix_B.host_view() << "\n"
|
||||
<< "E:\n" << matrix_E.host_view() << "\n"
|
||||
@@ -423,6 +422,7 @@ struct SparseTestbed {
|
||||
<< matrix_C_reference.host_view() << "\n"
|
||||
<< "Computed:\n"
|
||||
<< matrix_C_computed.host_view() << "\n";
|
||||
}
|
||||
|
||||
EXPECT_GT(cutlass::reference::host::TensorNorm(matrix_C_reference.host_view()), 0);
|
||||
EXPECT_GT(cutlass::reference::host::TensorNorm(matrix_C_computed.host_view()), 0);
|
||||
|
||||
@@ -193,11 +193,40 @@ struct Testbed {
|
||||
matrix_C_reference.reset(cutlass::make_Coord(m, n), false);
|
||||
}
|
||||
|
||||
/// Returns true if the CUDA device is sufficient to execute the kernel.
|
||||
bool sufficient() const {
|
||||
|
||||
//
|
||||
// Determine SMEM requirements and waive if not satisfied
|
||||
//
|
||||
|
||||
cudaDeviceProp properties;
|
||||
int device_idx;
|
||||
cudaError_t result = cudaGetDevice(&device_idx);
|
||||
|
||||
if (result != cudaSuccess) {
|
||||
throw std::runtime_error("cudaGetDevice() API call failed.");
|
||||
}
|
||||
|
||||
result = cudaGetDeviceProperties(&properties, device_idx);
|
||||
|
||||
if (result != cudaSuccess) {
|
||||
throw std::runtime_error("cudaGetDeviceProperties() failed");
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/// Runs the test
|
||||
bool run(
|
||||
dim3 grid, dim3 block,
|
||||
cutlass::Distribution::Kind init_A = cutlass::Distribution::Uniform,
|
||||
cutlass::Distribution::Kind init_B = cutlass::Distribution::Uniform) {
|
||||
|
||||
if (!sufficient()) {
|
||||
return true;
|
||||
}
|
||||
|
||||
//
|
||||
// initialize device memory
|
||||
//
|
||||
@@ -318,13 +347,18 @@ struct Testbed {
|
||||
bool passed = cutlass::reference::host::TensorEquals(
|
||||
matrix_C_computed.host_view(), matrix_C_reference.host_view());
|
||||
|
||||
EXPECT_TRUE(passed)
|
||||
EXPECT_TRUE(passed);
|
||||
|
||||
if (!passed && CUTLASS_TEST_UNIT_ENABLE_WARNINGS) {
|
||||
std::cout
|
||||
<< __FILE__ << ":" << __LINE__ << " "
|
||||
<< "A:\n" << matrix_A.host_view() << "\n"
|
||||
<< "B:\n" << matrix_B.host_view() << "\n"
|
||||
<< "Reference:\n"
|
||||
<< matrix_C_reference.host_view() << "\n"
|
||||
<< "Computed:\n"
|
||||
<< matrix_C_computed.host_view() << "\n";
|
||||
}
|
||||
|
||||
EXPECT_GT(cutlass::reference::host::TensorNorm(matrix_C_reference.host_view()), 0);
|
||||
EXPECT_GT(cutlass::reference::host::TensorNorm(matrix_C_computed.host_view()), 0);
|
||||
|
||||
@@ -217,11 +217,25 @@ struct Testbed {
|
||||
matrix_C_reference.reset(cutlass::make_Coord(m, n), false);
|
||||
}
|
||||
|
||||
bool sufficient() {
|
||||
return true;
|
||||
}
|
||||
|
||||
/// Runs the test
|
||||
bool run(
|
||||
dim3 grid, dim3 block,
|
||||
cutlass::Distribution::Kind init_A = cutlass::Distribution::Uniform,
|
||||
cutlass::Distribution::Kind init_B = cutlass::Distribution::Uniform) {
|
||||
|
||||
// Waive test if insufficient CUDA device
|
||||
if (!sufficient()) {
|
||||
if (CUTLASS_TEST_UNIT_ENABLE_WARNINGS) {
|
||||
std::cerr << "Test waived due to insufficient CUDA device." << std::endl;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
//
|
||||
// initialize device memory
|
||||
//
|
||||
@@ -300,7 +314,7 @@ struct Testbed {
|
||||
|
||||
cudaError_t result = cudaDeviceSynchronize();
|
||||
EXPECT_EQ(result, cudaSuccess)
|
||||
<< " kernel error: " << cudaGetErrorString(result);
|
||||
<< " kernel error: " << cudaGetErrorString(result) << " on device " << GetCudaDevice();
|
||||
|
||||
matrix_C_computed.sync_host();
|
||||
|
||||
@@ -316,7 +330,7 @@ struct Testbed {
|
||||
bool passed = cutlass::reference::host::TensorEquals(
|
||||
matrix_C_computed.host_view(), matrix_C_reference.host_view());
|
||||
|
||||
EXPECT_TRUE(passed);
|
||||
EXPECT_TRUE(passed) << "Failed on device " << GetCudaDevice();
|
||||
|
||||
if (!passed) {
|
||||
std::ofstream output("mma_pipelined_testbed_errors.txt");
|
||||
|
||||
@@ -50,7 +50,7 @@
|
||||
|
||||
#include "testbed.h"
|
||||
|
||||
#if defined(CUTLASS_ARCH_MMA_SM90_SUPPORTED)
|
||||
#if defined(CUTLASS_ARCH_MMA_SM90_F64_MMA_ENABLED)
|
||||
|
||||
TEST(SM90_warp_gemm_complex_tensor_op_f64, 16x8x4_16x8x4_nt) {
|
||||
|
||||
@@ -331,4 +331,4 @@ TEST(SM90_warp_gemm_complex_tensor_op_f64, 64x64x4_16x8x4_tn) {
|
||||
test::gemm::warp::TestbedComplex<MmaTensorOp, Shape>().run();
|
||||
}
|
||||
|
||||
#endif // if defined(CUTLASS_ARCH_MMA_SM90_SUPPORTED)
|
||||
#endif // if defined(CUTLASS_ARCH_MMA_SM90_F64_MMA_ENABLED)
|
||||
|
||||
@@ -50,7 +50,7 @@
|
||||
|
||||
#include "testbed.h"
|
||||
|
||||
#if defined(CUTLASS_ARCH_MMA_SM90_SUPPORTED)
|
||||
#if defined(CUTLASS_ARCH_MMA_SM90_F64_MMA_ENABLED)
|
||||
|
||||
TEST(SM90_warp_gemm_tensor_op_congruous_f64, 16x16x4_16x16x4_16x8x4) {
|
||||
using Shape = cutlass::gemm::GemmShape<16, 16, 4>;
|
||||
@@ -203,4 +203,4 @@ TEST(SM90_warp_gemm_tensor_op_crosswise_f64, 32x64x16_32x64x16_16x8x4) {
|
||||
}
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#endif // if defined(CUTLASS_ARCH_MMA_SM90_SUPPORTED)
|
||||
#endif // if defined(CUTLASS_ARCH_MMA_SM90_F64_MMA_ENABLED)
|
||||
|
||||
@@ -191,10 +191,47 @@ struct Testbed {
|
||||
tensor_D_reference.reset(cutlass::make_Coord(Shape::kM, Shape::kN), false);
|
||||
}
|
||||
|
||||
/// Returns true if the CUDA device is sufficient to execute the kernel.
|
||||
bool sufficient() const {
|
||||
|
||||
cudaDeviceProp properties;
|
||||
int device_idx;
|
||||
cudaError_t result = cudaGetDevice(&device_idx);
|
||||
|
||||
if (result != cudaSuccess) {
|
||||
throw std::runtime_error("cudaGetDevice() API call failed.");
|
||||
}
|
||||
|
||||
result = cudaGetDeviceProperties(&properties, device_idx);
|
||||
|
||||
if (result != cudaSuccess) {
|
||||
throw std::runtime_error("cudaGetDeviceProperties() failed");
|
||||
}
|
||||
|
||||
if (properties.major == 9) {
|
||||
// NVIDIA Hopper drops support for several data types
|
||||
if (
|
||||
cutlass::sizeof_bits<ElementA>::value < 8 ||
|
||||
cutlass::sizeof_bits<ElementB>::value < 8 ||
|
||||
cutlass::sizeof_bits<ElementC>::value < 8) {
|
||||
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
/// Runs the test
|
||||
bool run(
|
||||
cutlass::Distribution::Kind init_A = cutlass::Distribution::Uniform,
|
||||
cutlass::Distribution::Kind init_B = cutlass::Distribution::Uniform) {
|
||||
|
||||
if (!sufficient()) {
|
||||
return true;
|
||||
}
|
||||
|
||||
//
|
||||
// initialize device memory
|
||||
//
|
||||
@@ -401,10 +438,46 @@ struct TestbedComplex {
|
||||
tensor_D_reference.reset(cutlass::make_Coord(Shape::kM, Shape::kN), false);
|
||||
}
|
||||
|
||||
/// Returns true if the CUDA device is sufficient to execute the kernel.
|
||||
bool sufficient() const {
|
||||
|
||||
cudaDeviceProp properties;
|
||||
int device_idx;
|
||||
cudaError_t result = cudaGetDevice(&device_idx);
|
||||
|
||||
if (result != cudaSuccess) {
|
||||
throw std::runtime_error("cudaGetDevice() API call failed.");
|
||||
}
|
||||
|
||||
result = cudaGetDeviceProperties(&properties, device_idx);
|
||||
|
||||
if (result != cudaSuccess) {
|
||||
throw std::runtime_error("cudaGetDeviceProperties() failed");
|
||||
}
|
||||
|
||||
if (properties.major == 9) {
|
||||
// NVIDIA Hopper drops support for several data types
|
||||
if (
|
||||
cutlass::sizeof_bits<ElementA>::value < 8 ||
|
||||
cutlass::sizeof_bits<ElementB>::value < 8 ||
|
||||
cutlass::sizeof_bits<ElementC>::value < 8) {
|
||||
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/// Runs the test
|
||||
bool run(
|
||||
cutlass::Distribution::Kind init_A = cutlass::Distribution::Uniform,
|
||||
cutlass::Distribution::Kind init_B = cutlass::Distribution::Uniform) {
|
||||
|
||||
if (!sufficient()) {
|
||||
return true;
|
||||
}
|
||||
|
||||
//
|
||||
// initialize device memory
|
||||
//
|
||||
@@ -676,10 +749,46 @@ struct TransformTestbed {
|
||||
tensor_D_reference.reset(cutlass::make_Coord(Shape::kM, Shape::kN), false);
|
||||
}
|
||||
|
||||
/// Returns true if the CUDA device is sufficient to execute the kernel.
|
||||
bool sufficient() const {
|
||||
|
||||
cudaDeviceProp properties;
|
||||
int device_idx;
|
||||
cudaError_t result = cudaGetDevice(&device_idx);
|
||||
|
||||
if (result != cudaSuccess) {
|
||||
throw std::runtime_error("cudaGetDevice() API call failed.");
|
||||
}
|
||||
|
||||
result = cudaGetDeviceProperties(&properties, device_idx);
|
||||
|
||||
if (result != cudaSuccess) {
|
||||
throw std::runtime_error("cudaGetDeviceProperties() failed");
|
||||
}
|
||||
|
||||
if (properties.major == 9) {
|
||||
// NVIDIA Hopper drops support for several data types
|
||||
if (
|
||||
cutlass::sizeof_bits<ElementA>::value < 8 ||
|
||||
cutlass::sizeof_bits<ElementB>::value < 8 ||
|
||||
cutlass::sizeof_bits<ElementC>::value < 8) {
|
||||
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/// Runs the test
|
||||
bool run(
|
||||
cutlass::Distribution::Kind init_A = cutlass::Distribution::Uniform,
|
||||
cutlass::Distribution::Kind init_B = cutlass::Distribution::Uniform) {
|
||||
|
||||
if (!sufficient()) {
|
||||
return true;
|
||||
}
|
||||
|
||||
//
|
||||
// initialize device memory
|
||||
//
|
||||
@@ -878,10 +987,46 @@ struct TransformedTestbedComplex {
|
||||
tensor_D_reference.reset(cutlass::make_Coord(Shape::kM, Shape::kN), false);
|
||||
}
|
||||
|
||||
/// Returns true if the CUDA device is sufficient to execute the kernel.
|
||||
bool sufficient() const {
|
||||
|
||||
cudaDeviceProp properties;
|
||||
int device_idx;
|
||||
cudaError_t result = cudaGetDevice(&device_idx);
|
||||
|
||||
if (result != cudaSuccess) {
|
||||
throw std::runtime_error("cudaGetDevice() API call failed.");
|
||||
}
|
||||
|
||||
result = cudaGetDeviceProperties(&properties, device_idx);
|
||||
|
||||
if (result != cudaSuccess) {
|
||||
throw std::runtime_error("cudaGetDeviceProperties() failed");
|
||||
}
|
||||
|
||||
if (properties.major == 9) {
|
||||
// NVIDIA Hopper drops support for several data types
|
||||
if (
|
||||
cutlass::sizeof_bits<ElementA>::value < 8 ||
|
||||
cutlass::sizeof_bits<ElementB>::value < 8 ||
|
||||
cutlass::sizeof_bits<ElementC>::value < 8) {
|
||||
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/// Runs the test
|
||||
bool run(
|
||||
cutlass::Distribution::Kind init_A = cutlass::Distribution::Uniform,
|
||||
cutlass::Distribution::Kind init_B = cutlass::Distribution::Uniform) {
|
||||
|
||||
if (!sufficient()) {
|
||||
return true;
|
||||
}
|
||||
|
||||
//
|
||||
// initialize device memory
|
||||
//
|
||||
@@ -1199,12 +1344,47 @@ struct SparseTestbed {
|
||||
Shape::kM, Shape::kK / Sparse / ElementsPerElementE));
|
||||
}
|
||||
|
||||
/// Returns true if the CUDA device is sufficient to execute the kernel.
|
||||
bool sufficient() const {
|
||||
|
||||
cudaDeviceProp properties;
|
||||
int device_idx;
|
||||
cudaError_t result = cudaGetDevice(&device_idx);
|
||||
|
||||
if (result != cudaSuccess) {
|
||||
throw std::runtime_error("cudaGetDevice() API call failed.");
|
||||
}
|
||||
|
||||
result = cudaGetDeviceProperties(&properties, device_idx);
|
||||
|
||||
if (result != cudaSuccess) {
|
||||
throw std::runtime_error("cudaGetDeviceProperties() failed");
|
||||
}
|
||||
|
||||
if (properties.major == 9) {
|
||||
// NVIDIA Hopper drops support for several data types
|
||||
if (
|
||||
cutlass::sizeof_bits<ElementA>::value < 8 ||
|
||||
cutlass::sizeof_bits<ElementB>::value < 8 ||
|
||||
cutlass::sizeof_bits<ElementC>::value < 8) {
|
||||
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/// Runs the test
|
||||
bool run(
|
||||
cutlass::Distribution::Kind init_A = cutlass::Distribution::Uniform,
|
||||
cutlass::Distribution::Kind init_B = cutlass::Distribution::Uniform,
|
||||
cutlass::Distribution::Kind init_E = cutlass::Distribution::Uniform) {
|
||||
|
||||
if (!sufficient()) {
|
||||
return true;
|
||||
}
|
||||
|
||||
//
|
||||
// initialize device memory
|
||||
//
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
# Copyright (c) 2017 - 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: BSD-3-Clause
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions are met:
|
||||
#
|
||||
# 1. Redistributions of source code must retain the above copyright notice, this
|
||||
# list of conditions and the following disclaimer.
|
||||
#
|
||||
# 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
# this list of conditions and the following disclaimer in the documentation
|
||||
# and/or other materials provided with the distribution.
|
||||
#
|
||||
# 3. Neither the name of the copyright holder nor the names of its
|
||||
# contributors may be used to endorse or promote products derived from
|
||||
# this software without specific prior written permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
cutlass_test_unit_add_executable(
|
||||
cutlass_test_unit_pipeline
|
||||
pipeline_tma_async.cu
|
||||
pipeline_tma_async_warp_specialized.cu
|
||||
pipeline_tma_async_warp_specialized_persistent.cu
|
||||
pipeline_async.cu
|
||||
sequence_barrier.cu
|
||||
)
|
||||
@@ -0,0 +1,468 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017 - 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
/*! \file
|
||||
\brief Unit test for the PipelineAsync class
|
||||
*/
|
||||
|
||||
#define KERNEL_DBG_TRACE false
|
||||
|
||||
#include "../common/cutlass_unit_test.h"
|
||||
#include <thrust/host_vector.h>
|
||||
#include <thrust/device_vector.h>
|
||||
|
||||
#include <cute/tensor.hpp>
|
||||
#include <cute/arch/cluster_sm90.hpp>
|
||||
|
||||
#include <cutlass/util/reference/host/gemm.h>
|
||||
#include <cutlass/cluster_launch.hpp>
|
||||
|
||||
#include "cutlass/core_io.h"
|
||||
|
||||
#include "cutlass/util/print_error.hpp"
|
||||
#include "cutlass/util/GPU_Clock.hpp"
|
||||
|
||||
#include "testbed.h"
|
||||
#include "cutlass/pipeline.hpp"
|
||||
#include "cutlass/arch/barrier.h"
|
||||
#include "cute/arch/cluster_sm90.hpp"
|
||||
|
||||
using namespace cute;
|
||||
|
||||
//////////////////// KERNEL /////////////////////////
|
||||
|
||||
template <uint32_t Stages>
|
||||
struct SharedStorage
|
||||
{
|
||||
typename cutlass::PipelineAsync<Stages>::SharedStorage storage;
|
||||
};
|
||||
|
||||
// Goal of this kernel is to complete deadlock-free
|
||||
// Simple 1 producer warp, one consumer warp scenario
|
||||
template <class ClusterShape, uint32_t NumStages>
|
||||
__global__ static
|
||||
void pipeline_async_basic_device(uint32_t const num_iterations)
|
||||
{
|
||||
|
||||
extern __shared__ char shared_memory[];
|
||||
using MainloopPipeline = typename cutlass::PipelineAsync<NumStages>;
|
||||
using PipelineState = typename cutlass::PipelineState<NumStages>;
|
||||
|
||||
using SharedStorage = SharedStorage<NumStages>;
|
||||
SharedStorage& shared_storage = *reinterpret_cast<SharedStorage*>(shared_memory);
|
||||
|
||||
|
||||
auto cta_layout = Layout<ClusterShape>{}; // (m,n) -> cta_id
|
||||
|
||||
int warp_idx = __shfl_sync(0xffffffff, threadIdx.x / 32, 0);
|
||||
int lane_predicate = cute::elect_one_sync();
|
||||
dim3 block_id_in_cluster = cute::block_id_in_cluster();
|
||||
auto cluster_shape = ClusterShape{};
|
||||
|
||||
// This example showcases 2 producer 1 consumer example
|
||||
typename MainloopPipeline::Params params;
|
||||
params.producer_arv_count = 2;
|
||||
params.consumer_arv_count = 1;
|
||||
MainloopPipeline pipeline(shared_storage.storage, params);
|
||||
|
||||
// Ensure All CTAs in Cluster have completed init before issuing commits
|
||||
cute::cluster_arrive_relaxed();
|
||||
cute::cluster_wait();
|
||||
__syncthreads();
|
||||
|
||||
if (lane_predicate) {
|
||||
// Producer Warps
|
||||
if (warp_idx==0 || warp_idx==1) {
|
||||
|
||||
int prologue_iterations = min(NumStages, num_iterations);
|
||||
for ( int i = 0; i < prologue_iterations; ++i) {
|
||||
// Can also specify stage to commit directly
|
||||
pipeline.producer_commit(i);
|
||||
}
|
||||
|
||||
int mainloop_iterations = num_iterations - prologue_iterations;
|
||||
|
||||
// Only the mainloop needs a PipelineState because this is where we start "waiting" (acquiring)
|
||||
PipelineState smem_pipe_write;
|
||||
|
||||
for ( ; mainloop_iterations > 0; --mainloop_iterations) {
|
||||
pipeline.producer_acquire(smem_pipe_write);
|
||||
pipeline.producer_commit(smem_pipe_write);
|
||||
++smem_pipe_write;
|
||||
}
|
||||
}
|
||||
else {
|
||||
PipelineState smem_pipe_read;
|
||||
for (int iter=0 ; iter < num_iterations; ++iter) {
|
||||
pipeline.consumer_wait(smem_pipe_read);
|
||||
pipeline.consumer_release(smem_pipe_read.index());
|
||||
++smem_pipe_read;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// To make sure remote SMEM doesn't get destroyed
|
||||
cute::cluster_arrive();
|
||||
cute::cluster_wait();
|
||||
}
|
||||
/////////////////////////////////////////////////////
|
||||
|
||||
template<uint32_t Stages_, typename ClusterShape_>
|
||||
struct PipelineTest {
|
||||
|
||||
//
|
||||
// Data members
|
||||
//
|
||||
static constexpr uint32_t Stages = Stages_;
|
||||
static constexpr uint32_t kBlockSize = 96;
|
||||
using ClusterShape = ClusterShape_;
|
||||
|
||||
//
|
||||
// Methods
|
||||
//
|
||||
|
||||
// Ctor
|
||||
PipelineTest() = default;
|
||||
|
||||
|
||||
// Run CuTe GEMM kernel
|
||||
cudaError_t run(uint32_t const kNumIters,
|
||||
cudaStream_t stream = nullptr) {
|
||||
|
||||
// Pipeline (multistage pipeline)
|
||||
auto num_stages = Int<Stages>{};
|
||||
|
||||
auto cluster_shape = Shape<Int<ClusterShape::kM>, Int<ClusterShape::kN>, _1>{};
|
||||
|
||||
//
|
||||
// Configure and launch
|
||||
//
|
||||
int iterations = 2;
|
||||
cudaError_t result;
|
||||
|
||||
for (int iter = 0; iter < iterations; ++iter) {
|
||||
|
||||
// Define the tiled MMA layout (static, 4warps)
|
||||
using MainloopPipeline = typename cutlass::PipelineAsync<Stages>;
|
||||
|
||||
int smem_size = int(sizeof(SharedStorage<Stages>));
|
||||
|
||||
result = cudaFuncSetAttribute(
|
||||
pipeline_async_basic_device<decltype(cluster_shape), Stages>,
|
||||
cudaFuncAttributeMaxDynamicSharedMemorySize,
|
||||
smem_size);
|
||||
|
||||
// Launch a single Cluster, with 128 thread per CTA
|
||||
dim3 dimCluster(size<0>(cluster_shape), size<1>(cluster_shape), 1);
|
||||
dim3 dimGrid(size<0>(cluster_shape), size<1>(cluster_shape), 1);
|
||||
dim3 dimBlock(kBlockSize,1,1);
|
||||
|
||||
const void* kernel = (const void*)pipeline_async_basic_device<decltype(cluster_shape), Stages>;
|
||||
int iters = kNumIters;
|
||||
void* kernel_params[] = {reinterpret_cast<void*>(&iters)};
|
||||
cutlass::ClusterLauncher::launch(dimGrid, dimCluster, dimBlock, smem_size, stream, kernel, kernel_params);
|
||||
|
||||
} // profiling loop ends
|
||||
|
||||
result = cudaDeviceSynchronize();
|
||||
|
||||
if (result != cudaSuccess) {
|
||||
std::cerr << "Error: cudaDeviceSynchronize() failed" << std::endl;
|
||||
return result;
|
||||
}
|
||||
|
||||
return cudaSuccess;
|
||||
}
|
||||
|
||||
};
|
||||
|
||||
#if CUDA_12_0_SM90_FEATURES_SUPPORTED
|
||||
TEST(SM90_Verify_PipelineAsync, Cluster1x1_Stage2) {
|
||||
Options options;
|
||||
using ClusterShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
static constexpr uint32_t Stages = 2;
|
||||
using Test = PipelineTest<Stages, ClusterShape>;
|
||||
Testbed<Test> testbed(options);
|
||||
EXPECT_TRUE(testbed.verification());
|
||||
}
|
||||
|
||||
TEST(SM90_Verify_PipelineAsync, Cluster1x1_Stage5) {
|
||||
Options options;
|
||||
using ClusterShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
static constexpr uint32_t Stages = 5;
|
||||
using Test = PipelineTest<Stages, ClusterShape>;
|
||||
Testbed<Test> testbed(options);
|
||||
EXPECT_TRUE(testbed.verification());
|
||||
}
|
||||
|
||||
TEST(SM90_Verify_PipelineAsync, Cluster1x1_Stage10) {
|
||||
Options options;
|
||||
using ClusterShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
static constexpr uint32_t Stages = 10;
|
||||
using Test = PipelineTest<Stages, ClusterShape>;
|
||||
Testbed<Test> testbed(options);
|
||||
EXPECT_TRUE(testbed.verification());
|
||||
}
|
||||
|
||||
TEST(SM90_Verify_PipelineAsync, Cluster2x2_Stage2) {
|
||||
Options options;
|
||||
using ClusterShape = cutlass::gemm::GemmShape<2, 2, 1>;
|
||||
static constexpr uint32_t Stages = 2;
|
||||
using Test = PipelineTest<Stages, ClusterShape>;
|
||||
Testbed<Test> testbed(options);
|
||||
EXPECT_TRUE(testbed.verification());
|
||||
}
|
||||
|
||||
TEST(SM90_Verify_PipelineAsync, Cluster2x2_Stage5) {
|
||||
Options options;
|
||||
using ClusterShape = cutlass::gemm::GemmShape<2, 2, 1>;
|
||||
static constexpr uint32_t Stages = 5;
|
||||
using Test = PipelineTest<Stages, ClusterShape>;
|
||||
Testbed<Test> testbed(options);
|
||||
EXPECT_TRUE(testbed.verification());
|
||||
}
|
||||
|
||||
TEST(SM90_Verify_PipelineAsync, Cluster2x2_Stage10) {
|
||||
Options options;
|
||||
using ClusterShape = cutlass::gemm::GemmShape<2, 2, 1>;
|
||||
static constexpr uint32_t Stages = 10;
|
||||
using Test = PipelineTest<Stages, ClusterShape>;
|
||||
Testbed<Test> testbed(options);
|
||||
EXPECT_TRUE(testbed.verification());
|
||||
}
|
||||
|
||||
TEST(SM90_Verify_PipelineAsync, Cluster1x2_Stage2) {
|
||||
Options options;
|
||||
using ClusterShape = cutlass::gemm::GemmShape<1, 2, 1>;
|
||||
static constexpr uint32_t Stages = 2;
|
||||
using Test = PipelineTest<Stages, ClusterShape>;
|
||||
Testbed<Test> testbed(options);
|
||||
EXPECT_TRUE(testbed.verification());
|
||||
}
|
||||
|
||||
TEST(SM90_Verify_PipelineAsync, Cluster1x2_Stage7) {
|
||||
Options options;
|
||||
using ClusterShape = cutlass::gemm::GemmShape<1, 2, 1>;
|
||||
static constexpr uint32_t Stages = 7;
|
||||
using Test = PipelineTest<Stages, ClusterShape>;
|
||||
Testbed<Test> testbed(options);
|
||||
EXPECT_TRUE(testbed.verification());
|
||||
}
|
||||
|
||||
TEST(SM90_Verify_PipelineAsync, Cluster1x2_Stage10) {
|
||||
Options options;
|
||||
using ClusterShape = cutlass::gemm::GemmShape<1, 2, 1>;
|
||||
static constexpr uint32_t Stages = 10;
|
||||
using Test = PipelineTest<Stages, ClusterShape>;
|
||||
Testbed<Test> testbed(options);
|
||||
EXPECT_TRUE(testbed.verification());
|
||||
}
|
||||
|
||||
TEST(SM90_Verify_PipelineAsync, Cluster2x1_Stage2) {
|
||||
Options options;
|
||||
using ClusterShape = cutlass::gemm::GemmShape<2, 1, 1>;
|
||||
static constexpr uint32_t Stages = 2;
|
||||
using Test = PipelineTest<Stages, ClusterShape>;
|
||||
Testbed<Test> testbed(options);
|
||||
EXPECT_TRUE(testbed.verification());
|
||||
}
|
||||
|
||||
TEST(SM90_Verify_PipelineAsync, Cluster2x1_Stage7) {
|
||||
Options options;
|
||||
using ClusterShape = cutlass::gemm::GemmShape<2, 1, 1>;
|
||||
static constexpr uint32_t Stages = 7;
|
||||
using Test = PipelineTest<Stages, ClusterShape>;
|
||||
Testbed<Test> testbed(options);
|
||||
EXPECT_TRUE(testbed.verification());
|
||||
}
|
||||
|
||||
TEST(SM90_Verify_PipelineAsync, Cluster4x1_Stage2) {
|
||||
Options options;
|
||||
using ClusterShape = cutlass::gemm::GemmShape<4, 1, 1>;
|
||||
static constexpr uint32_t Stages = 2;
|
||||
using Test = PipelineTest<Stages, ClusterShape>;
|
||||
Testbed<Test> testbed(options);
|
||||
EXPECT_TRUE(testbed.verification());
|
||||
}
|
||||
|
||||
TEST(SM90_Verify_PipelineAsync, Cluster4x1_Stage7) {
|
||||
Options options;
|
||||
using ClusterShape = cutlass::gemm::GemmShape<4, 1, 1>;
|
||||
static constexpr uint32_t Stages = 7;
|
||||
using Test = PipelineTest<Stages, ClusterShape>;
|
||||
Testbed<Test> testbed(options);
|
||||
EXPECT_TRUE(testbed.verification());
|
||||
}
|
||||
|
||||
TEST(SM90_Verify_PipelineAsync, Cluster1x4_Stage2) {
|
||||
Options options;
|
||||
using ClusterShape = cutlass::gemm::GemmShape<1, 4, 1>;
|
||||
static constexpr uint32_t Stages = 2;
|
||||
using Test = PipelineTest<Stages, ClusterShape>;
|
||||
Testbed<Test> testbed(options);
|
||||
EXPECT_TRUE(testbed.verification());
|
||||
}
|
||||
|
||||
TEST(SM90_Verify_PipelineAsync, Cluster1x4_Stage7) {
|
||||
Options options;
|
||||
using ClusterShape = cutlass::gemm::GemmShape<1, 4, 1>;
|
||||
static constexpr uint32_t Stages = 7;
|
||||
using Test = PipelineTest<Stages, ClusterShape>;
|
||||
Testbed<Test> testbed(options);
|
||||
EXPECT_TRUE(testbed.verification());
|
||||
}
|
||||
|
||||
TEST(SM90_Verify_PipelineAsync, Cluster2x4_Stage2) {
|
||||
Options options;
|
||||
using ClusterShape = cutlass::gemm::GemmShape<2, 4, 1>;
|
||||
static constexpr uint32_t Stages = 2;
|
||||
using Test = PipelineTest<Stages, ClusterShape>;
|
||||
Testbed<Test> testbed(options);
|
||||
EXPECT_TRUE(testbed.verification());
|
||||
}
|
||||
|
||||
TEST(SM90_Verify_PipelineAsync, Cluster2x4_Stage7) {
|
||||
Options options;
|
||||
using ClusterShape = cutlass::gemm::GemmShape<2, 4, 1>;
|
||||
static constexpr uint32_t Stages = 7;
|
||||
using Test = PipelineTest<Stages, ClusterShape>;
|
||||
Testbed<Test> testbed(options);
|
||||
EXPECT_TRUE(testbed.verification());
|
||||
}
|
||||
|
||||
TEST(SM90_Verify_PipelineAsync, Cluster4x2_Stage2) {
|
||||
Options options;
|
||||
using ClusterShape = cutlass::gemm::GemmShape<4, 2, 1>;
|
||||
static constexpr uint32_t Stages = 2;
|
||||
using Test = PipelineTest<Stages, ClusterShape>;
|
||||
Testbed<Test> testbed(options);
|
||||
EXPECT_TRUE(testbed.verification());
|
||||
}
|
||||
|
||||
TEST(SM90_Verify_PipelineAsync, Cluster4x2_Stage7) {
|
||||
Options options;
|
||||
using ClusterShape = cutlass::gemm::GemmShape<4, 2, 1>;
|
||||
static constexpr uint32_t Stages = 7;
|
||||
using Test = PipelineTest<Stages, ClusterShape>;
|
||||
Testbed<Test> testbed(options);
|
||||
EXPECT_TRUE(testbed.verification());
|
||||
}
|
||||
|
||||
TEST(SM90_Verify_PipelineAsync, Cluster4x4_Stage2) {
|
||||
Options options;
|
||||
using ClusterShape = cutlass::gemm::GemmShape<4, 4, 1>;
|
||||
static constexpr uint32_t Stages = 2;
|
||||
using Test = PipelineTest<Stages, ClusterShape>;
|
||||
Testbed<Test> testbed(options);
|
||||
EXPECT_TRUE(testbed.verification());
|
||||
}
|
||||
|
||||
TEST(SM90_Verify_PipelineAsync, Cluster4x4_Stage3) {
|
||||
Options options;
|
||||
using ClusterShape = cutlass::gemm::GemmShape<4, 4, 1>;
|
||||
static constexpr uint32_t Stages = 3;
|
||||
using Test = PipelineTest<Stages, ClusterShape>;
|
||||
Testbed<Test> testbed(options);
|
||||
EXPECT_TRUE(testbed.verification());
|
||||
}
|
||||
|
||||
TEST(SM90_Verify_PipelineAsync, Cluster4x4_Stage4) {
|
||||
Options options;
|
||||
using ClusterShape = cutlass::gemm::GemmShape<4, 4, 1>;
|
||||
static constexpr uint32_t Stages = 4;
|
||||
using Test = PipelineTest<Stages, ClusterShape>;
|
||||
Testbed<Test> testbed(options);
|
||||
EXPECT_TRUE(testbed.verification());
|
||||
}
|
||||
|
||||
TEST(SM90_Verify_PipelineAsync, Cluster4x4_Stage5) {
|
||||
Options options;
|
||||
using ClusterShape = cutlass::gemm::GemmShape<4, 4, 1>;
|
||||
static constexpr uint32_t Stages = 5;
|
||||
using Test = PipelineTest<Stages, ClusterShape>;
|
||||
Testbed<Test> testbed(options);
|
||||
EXPECT_TRUE(testbed.verification());
|
||||
}
|
||||
|
||||
TEST(SM90_Verify_PipelineAsync, Cluster4x4_Stage6) {
|
||||
Options options;
|
||||
using ClusterShape = cutlass::gemm::GemmShape<4, 4, 1>;
|
||||
static constexpr uint32_t Stages = 6;
|
||||
using Test = PipelineTest<Stages, ClusterShape>;
|
||||
Testbed<Test> testbed(options);
|
||||
EXPECT_TRUE(testbed.verification());
|
||||
}
|
||||
|
||||
TEST(SM90_Verify_PipelineAsync, Cluster4x4_Stage7) {
|
||||
Options options;
|
||||
using ClusterShape = cutlass::gemm::GemmShape<4, 4, 1>;
|
||||
static constexpr uint32_t Stages = 7;
|
||||
using Test = PipelineTest<Stages, ClusterShape>;
|
||||
Testbed<Test> testbed(options);
|
||||
EXPECT_TRUE(testbed.verification());
|
||||
}
|
||||
|
||||
TEST(SM90_Verify_PipelineAsync, Cluster4x4_Stage8) {
|
||||
Options options;
|
||||
using ClusterShape = cutlass::gemm::GemmShape<4, 4, 1>;
|
||||
static constexpr uint32_t Stages = 8;
|
||||
using Test = PipelineTest<Stages, ClusterShape>;
|
||||
Testbed<Test> testbed(options);
|
||||
EXPECT_TRUE(testbed.verification());
|
||||
}
|
||||
|
||||
TEST(SM90_Verify_PipelineAsync, Cluster4x4_Stage9) {
|
||||
Options options;
|
||||
using ClusterShape = cutlass::gemm::GemmShape<4, 4, 1>;
|
||||
static constexpr uint32_t Stages = 9;
|
||||
using Test = PipelineTest<Stages, ClusterShape>;
|
||||
Testbed<Test> testbed(options);
|
||||
EXPECT_TRUE(testbed.verification());
|
||||
}
|
||||
|
||||
TEST(SM90_Verify_PipelineAsync, Cluster4x4_Stage10) {
|
||||
Options options;
|
||||
using ClusterShape = cutlass::gemm::GemmShape<4, 4, 1>;
|
||||
static constexpr uint32_t Stages = 10;
|
||||
using Test = PipelineTest<Stages, ClusterShape>;
|
||||
Testbed<Test> testbed(options);
|
||||
EXPECT_TRUE(testbed.verification());
|
||||
}
|
||||
|
||||
TEST(SM90_Verify_PipelineAsync, Cluster4x4_Stage11) {
|
||||
Options options;
|
||||
using ClusterShape = cutlass::gemm::GemmShape<4, 4, 1>;
|
||||
static constexpr uint32_t Stages = 11;
|
||||
using Test = PipelineTest<Stages, ClusterShape>;
|
||||
Testbed<Test> testbed(options);
|
||||
EXPECT_TRUE(testbed.verification());
|
||||
}
|
||||
#endif
|
||||
@@ -0,0 +1,469 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017 - 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
/*! \file
|
||||
\brief Unit test for the PipelineTmaAsync class
|
||||
*/
|
||||
|
||||
|
||||
#define KERNEL_DBG_TRACE false
|
||||
|
||||
#include "../common/cutlass_unit_test.h"
|
||||
#include <thrust/host_vector.h>
|
||||
#include <thrust/device_vector.h>
|
||||
|
||||
#include <cute/tensor.hpp>
|
||||
#include <cute/arch/cluster_sm90.hpp>
|
||||
|
||||
#include <cutlass/util/reference/host/gemm.h>
|
||||
#include <cutlass/cluster_launch.hpp>
|
||||
|
||||
#include "cutlass/core_io.h"
|
||||
|
||||
#include "cutlass/util/print_error.hpp"
|
||||
#include "cutlass/util/GPU_Clock.hpp"
|
||||
|
||||
#include "testbed.h"
|
||||
#include "cutlass/pipeline.hpp"
|
||||
#include "cutlass/arch/barrier.h"
|
||||
#include "cute/arch/cluster_sm90.hpp"
|
||||
|
||||
using namespace cute;
|
||||
|
||||
//////////////////// KERNEL /////////////////////////
|
||||
|
||||
template <uint32_t Stages, typename ClusterShape>
|
||||
struct SharedStorage
|
||||
{
|
||||
typename cutlass::PipelineTmaAsync<Stages, ClusterShape>::SharedStorage storage;
|
||||
};
|
||||
|
||||
// Goal of this kernel is to complete deadlock-free
|
||||
template <class ClusterShape, uint32_t NumStages>
|
||||
__global__ static
|
||||
void pipeline_device(uint32_t const NumIterations)
|
||||
{
|
||||
|
||||
extern __shared__ char shared_memory[];
|
||||
using DispatchPolicy = cutlass::gemm::MainloopSm90TmaGmma<NumStages, ClusterShape>;
|
||||
using MainloopPipeline = cutlass::PipelineTmaAsync<NumStages, ClusterShape>;
|
||||
using PipelineState = cutlass::PipelineState<NumStages>;
|
||||
|
||||
using SharedStorage = SharedStorage<NumStages, ClusterShape>;
|
||||
SharedStorage& shared_storage = *reinterpret_cast<SharedStorage*>(shared_memory);
|
||||
|
||||
auto cta_layout = Layout<ClusterShape>{}; // (m,n) -> cta_id
|
||||
int warp_idx = __shfl_sync(0xffffffff, threadIdx.x / 32, 0);
|
||||
int warp_group_thread_idx = threadIdx.x % 128;
|
||||
dim3 block_id_in_cluster = cute::block_id_in_cluster();
|
||||
|
||||
auto cluster_shape = ClusterShape{};
|
||||
|
||||
// #Producers = #RowsInCluster + #ColsInCluster - 1
|
||||
uint32_t const NumProducers = cute::size<0>(cluster_shape) + cute::size<1>(cluster_shape) - 1;
|
||||
uint32_t const TmaTransactionBytes = sizeof(uint32_t) * NumProducers;
|
||||
uint32_t const per_cta_bytes = sizeof(uint32_t);
|
||||
|
||||
// mbarrier.init
|
||||
typename MainloopPipeline::Params params;
|
||||
params.transaction_bytes = TmaTransactionBytes;
|
||||
params.role = MainloopPipeline::ThreadCategory::ProducerConsumer;
|
||||
params.is_leader = warp_group_thread_idx == 0;
|
||||
params.num_consumers = 128;
|
||||
|
||||
MainloopPipeline pipeline(shared_storage.storage, params);
|
||||
|
||||
__syncthreads();
|
||||
|
||||
// Ensure All CTAs in Cluster have completed init before issuing commits
|
||||
cute::cluster_arrive_relaxed();
|
||||
cute::cluster_wait();
|
||||
|
||||
// Total number of gemm_k_iterations
|
||||
auto mma_k_iterations = NumIterations;
|
||||
auto tma_k_iterations = NumIterations;
|
||||
|
||||
PipelineState smem_pipe_read;
|
||||
// For the DMA (prologue) - we start with an opposite phase - since we skip all waits
|
||||
// i.e., we know that the buffer is indeed empty
|
||||
PipelineState smem_pipe_write = cutlass::make_producer_start_state<MainloopPipeline>();
|
||||
PipelineState smem_pipe_release;
|
||||
int K_TILE_MMAS = 1;
|
||||
|
||||
int lane_predicate = cute::elect_one_sync();
|
||||
int k_pipe_tma_prologue = min(NumStages, tma_k_iterations);
|
||||
|
||||
// DMA Prologue (Loads)
|
||||
CUTLASS_PRAGMA_UNROLL
|
||||
for(int i = 0; i < k_pipe_tma_prologue; ++i) {
|
||||
pipeline.producer_acquire(smem_pipe_write);
|
||||
// cp.async.bulk.tensor would typically happen here
|
||||
pipeline.producer_commit(smem_pipe_write.index(), per_cta_bytes);
|
||||
++smem_pipe_write;
|
||||
}
|
||||
tma_k_iterations -= k_pipe_tma_prologue;
|
||||
|
||||
// MMA Prologue (Compute) - modeling inflight MMAs
|
||||
for (int iter = 0; iter < K_TILE_MMAS; ++iter)
|
||||
{
|
||||
pipeline.consumer_wait(smem_pipe_read);
|
||||
warpgroup_arrive();
|
||||
// GMMA would typically happen here
|
||||
|
||||
++smem_pipe_read;
|
||||
}
|
||||
|
||||
mma_k_iterations -= K_TILE_MMAS;
|
||||
|
||||
CUTLASS_PRAGMA_NO_UNROLL
|
||||
for (int iter = 0; iter < mma_k_iterations; ++iter)
|
||||
{
|
||||
pipeline.consumer_wait(smem_pipe_read);
|
||||
|
||||
warpgroup_arrive();
|
||||
// GMMA would typically happen here
|
||||
|
||||
pipeline.consumer_release(smem_pipe_release);
|
||||
|
||||
if (lane_predicate && (warp_idx == 0) && (tma_k_iterations > 0)) {
|
||||
pipeline.producer_acquire(smem_pipe_write);
|
||||
// cp.async.bulk.tensor would typically happen here
|
||||
pipeline.producer_commit(smem_pipe_write.index(), per_cta_bytes);
|
||||
++smem_pipe_write;
|
||||
--tma_k_iterations;
|
||||
}
|
||||
|
||||
// next read stage
|
||||
++smem_pipe_read;
|
||||
++smem_pipe_release;
|
||||
}
|
||||
|
||||
// To make sure remote SMEM doesn't get destoryed
|
||||
cute::cluster_arrive();
|
||||
cute::cluster_wait();
|
||||
}
|
||||
/////////////////////////////////////////////////////
|
||||
|
||||
/// Device NT GMMA + TMA specialized
|
||||
template<uint32_t Stages_, typename ClusterShape_>
|
||||
struct PipelineTest {
|
||||
|
||||
//
|
||||
// Data members
|
||||
//
|
||||
static constexpr uint32_t Stages = Stages_;
|
||||
static constexpr uint32_t kBlockSize = 128;
|
||||
using ClusterShape = ClusterShape_;
|
||||
|
||||
//
|
||||
// Methods
|
||||
//
|
||||
|
||||
// Ctor
|
||||
PipelineTest(){};
|
||||
|
||||
|
||||
// Run CuTe GEMM kernel
|
||||
cudaError_t run(uint32_t const kNumIters,
|
||||
cudaStream_t stream = 0) {
|
||||
|
||||
float elapsed_ms = 0.0f;
|
||||
// Pipeline (multistage pipeline)
|
||||
auto num_stages = Int<Stages>{};
|
||||
|
||||
auto cluster_shape = Shape<Int<ClusterShape::kM>, Int<ClusterShape::kN>, _1>{};
|
||||
|
||||
//
|
||||
// Configure and launch
|
||||
//
|
||||
int iterations = 1;
|
||||
cudaEvent_t events[2];
|
||||
cudaError_t result;
|
||||
|
||||
for (cudaEvent_t & event : events) {
|
||||
result = cudaEventCreate(&event);
|
||||
if (result != cudaSuccess) {
|
||||
std::cerr << "Error: Failed to create event.";
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
result = cudaEventRecord(events[0]);
|
||||
|
||||
if (result != cudaSuccess) {
|
||||
std::cerr << "Error: Failed to record start event.";
|
||||
return result;
|
||||
}
|
||||
|
||||
for (int iter = 0; iter < iterations; ++iter) {
|
||||
|
||||
// Define the tiled MMA layout (static, 4warps)
|
||||
using DispatchPolicy = cutlass::gemm::MainloopSm90TmaGmma<Stages, decltype(cluster_shape)>;
|
||||
using MainloopPipeline = typename cutlass::PipelineTmaAsync<Stages, decltype(cluster_shape)>;
|
||||
|
||||
int smem_size = int(sizeof(SharedStorage<Stages, decltype(cluster_shape)>));
|
||||
|
||||
result = cudaFuncSetAttribute(
|
||||
pipeline_device<decltype(cluster_shape), Stages>,
|
||||
cudaFuncAttributeMaxDynamicSharedMemorySize,
|
||||
smem_size);
|
||||
|
||||
// Launch a single Cluster, with 128 thread per CTA
|
||||
dim3 dimCluster(size<0>(cluster_shape), size<1>(cluster_shape), 1);
|
||||
dim3 dimGrid(size<0>(cluster_shape), size<1>(cluster_shape), 1);
|
||||
dim3 dimBlock(kBlockSize,1,1);
|
||||
|
||||
const void* kernel = (const void*)pipeline_device<decltype(cluster_shape), Stages>;
|
||||
int iters = kNumIters;
|
||||
void* kernel_params[] = {reinterpret_cast<void*>(&iters)};
|
||||
cutlass::ClusterLauncher::launch(dimGrid, dimCluster, dimBlock, smem_size, stream, kernel, kernel_params);
|
||||
|
||||
} // profiling loop ends
|
||||
|
||||
result = cudaEventRecord(events[1]);
|
||||
|
||||
if (result != cudaSuccess) {
|
||||
std::cerr << "Error: Failed to record stop event.";
|
||||
return result;
|
||||
}
|
||||
|
||||
result = cudaDeviceSynchronize();
|
||||
|
||||
if (result != cudaSuccess) {
|
||||
std::cerr << "Error: cudaDeviceSynchronize() failed" << std::endl;
|
||||
return result;
|
||||
}
|
||||
|
||||
result = cudaEventElapsedTime(&elapsed_ms, events[0], events[1]);
|
||||
|
||||
if (result != cudaSuccess) {
|
||||
std::cerr << "Failed to create event.";
|
||||
return result;
|
||||
}
|
||||
|
||||
for (cudaEvent_t & event : events) {
|
||||
(void)cudaEventDestroy(event);
|
||||
}
|
||||
|
||||
return cudaSuccess;
|
||||
}
|
||||
};
|
||||
|
||||
#if CUDA_12_0_SM90_FEATURES_SUPPORTED
|
||||
TEST(SM90_Verify_PipelineTmaAsync, Cluster1x1_Stage2) {
|
||||
Options options;
|
||||
using ClusterShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
static constexpr uint32_t Stages = 2;
|
||||
using Test = PipelineTest<Stages, ClusterShape>;
|
||||
Testbed<Test> testbed(options);
|
||||
EXPECT_TRUE(testbed.verification());
|
||||
}
|
||||
|
||||
TEST(SM90_Verify_PipelineTmaAsync, Cluster1x1_Stage5) {
|
||||
Options options;
|
||||
using ClusterShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
static constexpr uint32_t Stages = 5;
|
||||
using Test = PipelineTest<Stages, ClusterShape>;
|
||||
Testbed<Test> testbed(options);
|
||||
EXPECT_TRUE(testbed.verification());
|
||||
}
|
||||
|
||||
TEST(SM90_Verify_PipelineTmaAsync, Cluster1x1_Stage10) {
|
||||
Options options;
|
||||
using ClusterShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
static constexpr uint32_t Stages = 10;
|
||||
using Test = PipelineTest<Stages, ClusterShape>;
|
||||
Testbed<Test> testbed(options);
|
||||
EXPECT_TRUE(testbed.verification());
|
||||
}
|
||||
|
||||
TEST(SM90_Verify_PipelineTmaAsync, Cluster2x2_Stage2) {
|
||||
Options options;
|
||||
using ClusterShape = cutlass::gemm::GemmShape<2, 2, 1>;
|
||||
static constexpr uint32_t Stages = 2;
|
||||
using Test = PipelineTest<Stages, ClusterShape>;
|
||||
Testbed<Test> testbed(options);
|
||||
EXPECT_TRUE(testbed.verification());
|
||||
}
|
||||
|
||||
TEST(SM90_Verify_PipelineTmaAsync, Cluster2x2_Stage5) {
|
||||
Options options;
|
||||
using ClusterShape = cutlass::gemm::GemmShape<2, 2, 1>;
|
||||
static constexpr uint32_t Stages = 5;
|
||||
using Test = PipelineTest<Stages, ClusterShape>;
|
||||
Testbed<Test> testbed(options);
|
||||
EXPECT_TRUE(testbed.verification());
|
||||
}
|
||||
|
||||
TEST(SM90_Verify_PipelineTmaAsync, Cluster2x2_Stage10) {
|
||||
Options options;
|
||||
using ClusterShape = cutlass::gemm::GemmShape<2, 2, 1>;
|
||||
static constexpr uint32_t Stages = 10;
|
||||
using Test = PipelineTest<Stages, ClusterShape>;
|
||||
Testbed<Test> testbed(options);
|
||||
EXPECT_TRUE(testbed.verification());
|
||||
}
|
||||
|
||||
TEST(SM90_Verify_PipelineTmaAsync, Cluster4x4_Stage2) {
|
||||
Options options;
|
||||
using ClusterShape = cutlass::gemm::GemmShape<4, 4, 1>;
|
||||
static constexpr uint32_t Stages = 2;
|
||||
using Test = PipelineTest<Stages, ClusterShape>;
|
||||
Testbed<Test> testbed(options);
|
||||
EXPECT_TRUE(testbed.verification());
|
||||
}
|
||||
|
||||
TEST(SM90_Verify_PipelineTmaAsync, Cluster4x4_Stage10) {
|
||||
Options options;
|
||||
using ClusterShape = cutlass::gemm::GemmShape<4, 4, 1>;
|
||||
static constexpr uint32_t Stages = 10;
|
||||
using Test = PipelineTest<Stages, ClusterShape>;
|
||||
Testbed<Test> testbed(options);
|
||||
EXPECT_TRUE(testbed.verification());
|
||||
}
|
||||
|
||||
TEST(SM90_Verify_PipelineTmaAsync, Cluster1x2_Stage2) {
|
||||
Options options;
|
||||
using ClusterShape = cutlass::gemm::GemmShape<1, 2, 1>;
|
||||
static constexpr uint32_t Stages = 2;
|
||||
using Test = PipelineTest<Stages, ClusterShape>;
|
||||
Testbed<Test> testbed(options);
|
||||
EXPECT_TRUE(testbed.verification());
|
||||
}
|
||||
|
||||
TEST(SM90_Verify_PipelineTmaAsync, Cluster1x2_Stage7) {
|
||||
Options options;
|
||||
using ClusterShape = cutlass::gemm::GemmShape<1, 2, 1>;
|
||||
static constexpr uint32_t Stages = 7;
|
||||
using Test = PipelineTest<Stages, ClusterShape>;
|
||||
Testbed<Test> testbed(options);
|
||||
EXPECT_TRUE(testbed.verification());
|
||||
}
|
||||
|
||||
TEST(SM90_Verify_PipelineTmaAsync, Cluster1x2_Stage10) {
|
||||
Options options;
|
||||
using ClusterShape = cutlass::gemm::GemmShape<1, 2, 1>;
|
||||
static constexpr uint32_t Stages = 10;
|
||||
using Test = PipelineTest<Stages, ClusterShape>;
|
||||
Testbed<Test> testbed(options);
|
||||
EXPECT_TRUE(testbed.verification());
|
||||
}
|
||||
|
||||
TEST(SM90_Verify_PipelineTmaAsync, Cluster2x1_Stage2) {
|
||||
Options options;
|
||||
using ClusterShape = cutlass::gemm::GemmShape<2, 1, 1>;
|
||||
static constexpr uint32_t Stages = 2;
|
||||
using Test = PipelineTest<Stages, ClusterShape>;
|
||||
Testbed<Test> testbed(options);
|
||||
EXPECT_TRUE(testbed.verification());
|
||||
}
|
||||
|
||||
TEST(SM90_Verify_PipelineTmaAsync, Cluster2x1_Stage7) {
|
||||
Options options;
|
||||
using ClusterShape = cutlass::gemm::GemmShape<2, 1, 1>;
|
||||
static constexpr uint32_t Stages = 7;
|
||||
using Test = PipelineTest<Stages, ClusterShape>;
|
||||
Testbed<Test> testbed(options);
|
||||
EXPECT_TRUE(testbed.verification());
|
||||
}
|
||||
|
||||
TEST(SM90_Verify_PipelineTmaAsync, Cluster4x1_Stage2) {
|
||||
Options options;
|
||||
using ClusterShape = cutlass::gemm::GemmShape<4, 1, 1>;
|
||||
static constexpr uint32_t Stages = 2;
|
||||
using Test = PipelineTest<Stages, ClusterShape>;
|
||||
Testbed<Test> testbed(options);
|
||||
EXPECT_TRUE(testbed.verification());
|
||||
}
|
||||
|
||||
TEST(SM90_Verify_PipelineTmaAsync, Cluster4x1_Stage7) {
|
||||
Options options;
|
||||
using ClusterShape = cutlass::gemm::GemmShape<4, 1, 1>;
|
||||
static constexpr uint32_t Stages = 7;
|
||||
using Test = PipelineTest<Stages, ClusterShape>;
|
||||
Testbed<Test> testbed(options);
|
||||
EXPECT_TRUE(testbed.verification());
|
||||
}
|
||||
|
||||
TEST(SM90_Verify_PipelineTmaAsync, Cluster1x4_Stage2) {
|
||||
Options options;
|
||||
using ClusterShape = cutlass::gemm::GemmShape<1, 4, 1>;
|
||||
static constexpr uint32_t Stages = 2;
|
||||
using Test = PipelineTest<Stages, ClusterShape>;
|
||||
Testbed<Test> testbed(options);
|
||||
EXPECT_TRUE(testbed.verification());
|
||||
}
|
||||
|
||||
TEST(SM90_Verify_PipelineTmaAsync, Cluster1x4_Stage7) {
|
||||
Options options;
|
||||
using ClusterShape = cutlass::gemm::GemmShape<1, 4, 1>;
|
||||
static constexpr uint32_t Stages = 7;
|
||||
using Test = PipelineTest<Stages, ClusterShape>;
|
||||
Testbed<Test> testbed(options);
|
||||
EXPECT_TRUE(testbed.verification());
|
||||
}
|
||||
|
||||
TEST(SM90_Verify_PipelineTmaAsync, Cluster2x4_Stage2) {
|
||||
Options options;
|
||||
using ClusterShape = cutlass::gemm::GemmShape<2, 4, 1>;
|
||||
static constexpr uint32_t Stages = 2;
|
||||
using Test = PipelineTest<Stages, ClusterShape>;
|
||||
Testbed<Test> testbed(options);
|
||||
EXPECT_TRUE(testbed.verification());
|
||||
}
|
||||
|
||||
TEST(SM90_Verify_PipelineTmaAsync, Cluster2x4_Stage7) {
|
||||
Options options;
|
||||
using ClusterShape = cutlass::gemm::GemmShape<2, 4, 1>;
|
||||
static constexpr uint32_t Stages = 7;
|
||||
using Test = PipelineTest<Stages, ClusterShape>;
|
||||
Testbed<Test> testbed(options);
|
||||
EXPECT_TRUE(testbed.verification());
|
||||
}
|
||||
|
||||
TEST(SM90_Verify_PipelineTmaAsync, Cluster4x2_Stage2) {
|
||||
Options options;
|
||||
using ClusterShape = cutlass::gemm::GemmShape<4, 2, 1>;
|
||||
static constexpr uint32_t Stages = 2;
|
||||
using Test = PipelineTest<Stages, ClusterShape>;
|
||||
Testbed<Test> testbed(options);
|
||||
EXPECT_TRUE(testbed.verification());
|
||||
}
|
||||
|
||||
TEST(SM90_Verify_PipelineTmaAsync, Cluster4x2_Stage7) {
|
||||
Options options;
|
||||
using ClusterShape = cutlass::gemm::GemmShape<4, 2, 1>;
|
||||
static constexpr uint32_t Stages = 7;
|
||||
using Test = PipelineTest<Stages, ClusterShape>;
|
||||
Testbed<Test> testbed(options);
|
||||
EXPECT_TRUE(testbed.verification());
|
||||
}
|
||||
#endif
|
||||
@@ -0,0 +1,525 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017 - 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
/*! \file
|
||||
\brief Unit test for the PipelineTmaAsync class as it would be used in a Warp specialized loop
|
||||
*/
|
||||
|
||||
#define KERNEL_DBG_TRACE false
|
||||
|
||||
#include "../common/cutlass_unit_test.h"
|
||||
#include <thrust/host_vector.h>
|
||||
#include <thrust/device_vector.h>
|
||||
|
||||
#include <cute/tensor.hpp>
|
||||
#include <cute/arch/cluster_sm90.hpp>
|
||||
|
||||
#include <cutlass/util/reference/host/gemm.h>
|
||||
#include <cutlass/cluster_launch.hpp>
|
||||
|
||||
#include "cutlass/core_io.h"
|
||||
#include "cutlass/util/print_error.hpp"
|
||||
#include "cutlass/util/GPU_Clock.hpp"
|
||||
|
||||
#include "testbed.h"
|
||||
#include "cutlass/pipeline.hpp"
|
||||
#include "cutlass/arch/barrier.h"
|
||||
#include "cute/arch/cluster_sm90.hpp"
|
||||
#include "cutlass/arch/barrier.h"
|
||||
#include "cutlass/arch/reg_reconfig.h"
|
||||
|
||||
|
||||
using namespace cute;
|
||||
using namespace cutlass;
|
||||
|
||||
//////////////////// KERNEL /////////////////////////
|
||||
|
||||
template <uint32_t Stages, typename ClusterShape>
|
||||
struct SharedStorage
|
||||
{
|
||||
typename cutlass::PipelineTmaAsync<Stages, ClusterShape>::SharedStorage storage ;
|
||||
};
|
||||
|
||||
struct KernelParams
|
||||
{
|
||||
uint32_t num_iterations;
|
||||
int* data_ptr;
|
||||
};
|
||||
|
||||
// Goal of this kernel is to complete deadlock-free
|
||||
template <typename ClusterShape, uint32_t Stages>
|
||||
__launch_bounds__(384, 1)
|
||||
__global__ static
|
||||
void pipeline_device(KernelParams const kernel_params)
|
||||
{
|
||||
extern __shared__ char shared_memory[];
|
||||
using MainloopPipeline = typename cutlass::PipelineTmaAsync<Stages, ClusterShape>;
|
||||
using PipelineState = typename cutlass::PipelineState<Stages>;
|
||||
|
||||
using SharedStorage = SharedStorage<Stages, ClusterShape>;
|
||||
SharedStorage& shared_storage = *reinterpret_cast<SharedStorage*>(shared_memory);
|
||||
|
||||
auto cta_layout = Layout<ClusterShape>{}; // (m,n) -> cta_id
|
||||
int warp_group_idx = __shfl_sync(0xffffffff, threadIdx.x / 128, 0);
|
||||
int warp_idx_in_warpgroup = __shfl_sync(0xffffffff, (threadIdx.x / 32) % 4, 0);
|
||||
int warp_group_thread_idx = threadIdx.x % 128;
|
||||
dim3 block_id_in_cluster = cute::block_id_in_cluster();
|
||||
|
||||
auto cluster_shape = ClusterShape{};
|
||||
|
||||
// #Producers = #RowsInCluster + #ColsInCluster - 1
|
||||
uint32_t const NumProducers = cute::size<0>(cluster_shape) + cute::size<1>(cluster_shape) - 1;
|
||||
uint32_t const TmaTransactionBytes = static_cast<uint32_t>(sizeof(uint32_t) * NumProducers);
|
||||
uint32_t const per_cta_bytes = sizeof(uint32_t);
|
||||
|
||||
// mbarrier.init
|
||||
typename MainloopPipeline::Params params;
|
||||
params.transaction_bytes = TmaTransactionBytes;
|
||||
if (warp_group_idx == 0) {
|
||||
params.role = MainloopPipeline::ThreadCategory::Producer;
|
||||
}
|
||||
else {
|
||||
params.role = MainloopPipeline::ThreadCategory::Consumer;
|
||||
}
|
||||
params.is_leader = warp_group_thread_idx == 0;
|
||||
params.num_consumers = 128;
|
||||
|
||||
MainloopPipeline pipeline(shared_storage.storage, params);
|
||||
|
||||
__syncthreads();
|
||||
|
||||
// Ensure All CTAs in Cluster have completed init before issuing commits
|
||||
cute::cluster_arrive_relaxed();
|
||||
cute::cluster_wait();
|
||||
|
||||
|
||||
// Producer WarpGroup
|
||||
if (warp_group_idx == 0) {
|
||||
cutlass::arch::warpgroup_reg_alloc<232>();
|
||||
|
||||
int lane_predicate = cute::elect_one_sync();
|
||||
if (warp_idx_in_warpgroup == 0 && lane_predicate) {
|
||||
|
||||
int tma_k_prologue = min(Stages, kernel_params.num_iterations);
|
||||
|
||||
// Simulating Prologue TMA Loads
|
||||
// For the DMA (prologue) - we start with an opposite phase - since we skip all waits
|
||||
// i.e., we know that the buffer is indeed empty
|
||||
PipelineState smem_pipe_write = make_producer_start_state<MainloopPipeline>();
|
||||
CUTLASS_PRAGMA_UNROLL
|
||||
for(int i = 0; i < tma_k_prologue; ++i) {
|
||||
pipeline.producer_acquire(smem_pipe_write);
|
||||
// Simulating cp.async.bulk.tensor behavior
|
||||
pipeline.producer_commit(smem_pipe_write.index(), per_cta_bytes);
|
||||
++smem_pipe_write;
|
||||
}
|
||||
int tma_k_iter = kernel_params.num_iterations - tma_k_prologue;
|
||||
|
||||
// Simulating Mainloop TMA Loads
|
||||
CUTE_NO_UNROLL
|
||||
for ( ; tma_k_iter > 0; --tma_k_iter) {
|
||||
|
||||
pipeline.producer_acquire(smem_pipe_write);
|
||||
|
||||
// Simulating cp.async.bulk.tensor behavior
|
||||
pipeline.producer_commit(smem_pipe_write.index(), per_cta_bytes);
|
||||
|
||||
// Advance write stage
|
||||
++smem_pipe_write;
|
||||
}
|
||||
|
||||
// Tail Loop
|
||||
// Handles the case where we never enter the mainloop
|
||||
PipelineState tail = tma_k_prologue == Stages ? smem_pipe_write : PipelineState{};
|
||||
for ( int i = 0; i < tma_k_prologue; ++i) {
|
||||
pipeline.producer_acquire(tail);
|
||||
++tail;
|
||||
}
|
||||
}
|
||||
// Consumer WarpGroup
|
||||
} else if(warp_group_idx == 1) {
|
||||
cutlass::arch::warpgroup_reg_alloc<232>();
|
||||
|
||||
PipelineState smem_pipe_read;
|
||||
PipelineState smem_pipe_release;
|
||||
|
||||
// simulates accumulators + extra reg. pressure
|
||||
int arr[168];
|
||||
|
||||
// Init Shared Memory read stages & PhaseBit
|
||||
static constexpr uint32_t K_PIPE_MMAS = 1;
|
||||
static_assert( K_PIPE_MMAS < Stages, "ERROR : Too many MMAs in flight");
|
||||
|
||||
// Total number of gemm iterations
|
||||
auto gemm_k_iterations = kernel_params.num_iterations;
|
||||
|
||||
// Simulating Prologue MMAs
|
||||
int mma_k_prologue = min(K_PIPE_MMAS, gemm_k_iterations);
|
||||
CUTLASS_PRAGMA_UNROLL
|
||||
for (int iter = 0; iter < mma_k_prologue; ++iter) {
|
||||
pipeline.consumer_wait(smem_pipe_read);
|
||||
|
||||
warpgroup_arrive();
|
||||
// GMMA would typically happen here
|
||||
|
||||
++smem_pipe_read;
|
||||
}
|
||||
gemm_k_iterations -= mma_k_prologue;
|
||||
|
||||
// Simulating Mainloop MMAs
|
||||
CUTLASS_PRAGMA_NO_UNROLL
|
||||
for ( ; gemm_k_iterations > 0; --gemm_k_iterations) {
|
||||
|
||||
/// Wait on the smem_pipe_read stage / phase
|
||||
pipeline.consumer_wait(smem_pipe_read);
|
||||
|
||||
warpgroup_arrive();
|
||||
// GMMA would typically happen here
|
||||
|
||||
// Dummy op - which will never happen
|
||||
// But simulates high register usage.
|
||||
CUTE_UNROLL
|
||||
for(int i = 0; i < 168; ++i){
|
||||
if (threadIdx.x > 256){
|
||||
arr[i] += kernel_params.data_ptr[i];
|
||||
}
|
||||
}
|
||||
|
||||
pipeline.consumer_release(smem_pipe_release);
|
||||
|
||||
// Advance stages
|
||||
++smem_pipe_read;
|
||||
++smem_pipe_release;
|
||||
}
|
||||
|
||||
// Dummy op - which will never happen
|
||||
CUTE_UNROLL
|
||||
for(int i = 0; i < 168; ++i){
|
||||
if (threadIdx.x > 256){
|
||||
kernel_params.data_ptr[i] = arr[i];
|
||||
}
|
||||
}
|
||||
|
||||
// Tail Loop
|
||||
for (int i = 0; i < K_PIPE_MMAS; ++i){
|
||||
pipeline.consumer_release(smem_pipe_release);
|
||||
++smem_pipe_release;
|
||||
}
|
||||
|
||||
// Warp-Group #2
|
||||
} else {
|
||||
cutlass::arch::warpgroup_reg_dealloc<40>();
|
||||
}
|
||||
}
|
||||
/////////////////////////////////////////////////////
|
||||
|
||||
/// Device NT GMMA + TMA specialized
|
||||
template<uint32_t Stages_, typename ClusterShape_>
|
||||
struct PipelineTest {
|
||||
|
||||
//
|
||||
// Data members
|
||||
//
|
||||
static constexpr uint32_t Stages = Stages_;
|
||||
static constexpr uint32_t kBlockSize = 128 * 3;
|
||||
using ClusterShape = ClusterShape_;
|
||||
|
||||
//
|
||||
// Methods
|
||||
//
|
||||
|
||||
// Ctor
|
||||
PipelineTest(){};
|
||||
|
||||
// Run CuTe GEMM kernel
|
||||
cudaError_t run(uint32_t const kNumIters,
|
||||
cudaStream_t stream = 0) {
|
||||
|
||||
float elapsed_ms = 0.0f;
|
||||
// Pipeline (multistage pipeline)
|
||||
auto num_stages = Int<Stages>{};
|
||||
auto cluster_shape = Shape<Int<ClusterShape::kM>, Int<ClusterShape::kN>, _1>{};
|
||||
|
||||
//
|
||||
// Configure and launch
|
||||
//
|
||||
int iterations = 1;
|
||||
cudaEvent_t events[2];
|
||||
cudaError_t result;
|
||||
|
||||
for (cudaEvent_t & event : events) {
|
||||
result = cudaEventCreate(&event);
|
||||
if (result != cudaSuccess) {
|
||||
std::cerr << "Error: Failed to create event.";
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
result = cudaEventRecord(events[0]);
|
||||
|
||||
if (result != cudaSuccess) {
|
||||
std::cerr << "Error: Failed to record start event.";
|
||||
return result;
|
||||
}
|
||||
|
||||
for (int iter = 0; iter < iterations; ++iter) {
|
||||
|
||||
using MainloopPipeline = typename cutlass::PipelineTmaAsync<Stages, decltype(cluster_shape)>;
|
||||
|
||||
int smem_size = int(sizeof(SharedStorage<Stages, decltype(cluster_shape)>));
|
||||
|
||||
result = cudaFuncSetAttribute(
|
||||
pipeline_device<decltype(cluster_shape), Stages>,
|
||||
cudaFuncAttributeMaxDynamicSharedMemorySize,
|
||||
smem_size);
|
||||
|
||||
// Launch a single Cluster, with kBlockSize threads per CTA
|
||||
dim3 dimCluster(size<0>(cluster_shape), size<1>(cluster_shape), 1);
|
||||
dim3 dimGrid(size<0>(cluster_shape), size<1>(cluster_shape), 1);
|
||||
dim3 dimBlock(kBlockSize,1,1);
|
||||
|
||||
const void* kernel = (const void*)pipeline_device<decltype(cluster_shape), Stages>;
|
||||
KernelParams params{kNumIters, nullptr};
|
||||
void* kernel_params[] = {reinterpret_cast<void*>(¶ms)};
|
||||
cutlass::ClusterLauncher::launch(dimGrid, dimCluster, dimBlock, smem_size, stream, kernel, kernel_params);
|
||||
|
||||
}
|
||||
|
||||
result = cudaEventRecord(events[1]);
|
||||
|
||||
if (result != cudaSuccess) {
|
||||
std::cerr << "Error: Failed to record stop event.";
|
||||
return result;
|
||||
}
|
||||
|
||||
result = cudaDeviceSynchronize();
|
||||
|
||||
if (result != cudaSuccess) {
|
||||
std::cerr << "Error: cudaDeviceSynchronize() failed" << std::endl;
|
||||
return result;
|
||||
}
|
||||
|
||||
result = cudaEventElapsedTime(&elapsed_ms, events[0], events[1]);
|
||||
|
||||
if (result != cudaSuccess) {
|
||||
std::cerr << "Failed to create event.";
|
||||
return result;
|
||||
}
|
||||
|
||||
for (cudaEvent_t & event : events) {
|
||||
(void)cudaEventDestroy(event);
|
||||
}
|
||||
|
||||
return cudaSuccess;
|
||||
}
|
||||
};
|
||||
|
||||
#if CUDA_12_0_SM90_FEATURES_SUPPORTED
|
||||
TEST(SM90_Verify_PipelineTmaAsync_WS, Cluster1x1_Stage2) {
|
||||
Options options;
|
||||
using ClusterShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
static constexpr uint32_t Stages = 2;
|
||||
using Test = PipelineTest<Stages, ClusterShape>;
|
||||
Testbed<Test> testbed(options);
|
||||
EXPECT_TRUE(testbed.verification());
|
||||
}
|
||||
|
||||
TEST(SM90_Verify_PipelineTmaAsync_WS, Cluster1x1_Stage5) {
|
||||
Options options;
|
||||
using ClusterShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
static constexpr uint32_t Stages = 5;
|
||||
using Test = PipelineTest<Stages, ClusterShape>;
|
||||
Testbed<Test> testbed(options);
|
||||
EXPECT_TRUE(testbed.verification());
|
||||
}
|
||||
|
||||
TEST(SM90_Verify_PipelineTmaAsync_WS, Cluster1x1_Stage10) {
|
||||
Options options;
|
||||
using ClusterShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
static constexpr uint32_t Stages = 10;
|
||||
using Test = PipelineTest<Stages, ClusterShape>;
|
||||
Testbed<Test> testbed(options);
|
||||
EXPECT_TRUE(testbed.verification());
|
||||
}
|
||||
|
||||
TEST(SM90_Verify_PipelineTmaAsync_WS, Cluster2x2_Stage2) {
|
||||
Options options;
|
||||
using ClusterShape = cutlass::gemm::GemmShape<2, 2, 1>;
|
||||
static constexpr uint32_t Stages = 2;
|
||||
using Test = PipelineTest<Stages, ClusterShape>;
|
||||
Testbed<Test> testbed(options);
|
||||
EXPECT_TRUE(testbed.verification());
|
||||
}
|
||||
|
||||
TEST(SM90_Verify_PipelineTmaAsync_WS, Cluster2x2_Stage5) {
|
||||
Options options;
|
||||
using ClusterShape = cutlass::gemm::GemmShape<2, 2, 1>;
|
||||
static constexpr uint32_t Stages = 5;
|
||||
using Test = PipelineTest<Stages, ClusterShape>;
|
||||
Testbed<Test> testbed(options);
|
||||
EXPECT_TRUE(testbed.verification());
|
||||
}
|
||||
|
||||
TEST(SM90_Verify_PipelineTmaAsync_WS, Cluster2x2_Stage7) {
|
||||
Options options;
|
||||
using ClusterShape = cutlass::gemm::GemmShape<2, 2, 1>;
|
||||
static constexpr uint32_t Stages = 7;
|
||||
using Test = PipelineTest<Stages, ClusterShape>;
|
||||
Testbed<Test> testbed(options);
|
||||
EXPECT_TRUE(testbed.verification());
|
||||
}
|
||||
|
||||
TEST(SM90_Verify_PipelineTmaAsync_WS, Cluster4x4_Stage2) {
|
||||
Options options;
|
||||
using ClusterShape = cutlass::gemm::GemmShape<4, 4, 1>;
|
||||
static constexpr uint32_t Stages = 2;
|
||||
using Test = PipelineTest<Stages, ClusterShape>;
|
||||
Testbed<Test> testbed(options);
|
||||
EXPECT_TRUE(testbed.verification());
|
||||
}
|
||||
|
||||
TEST(SM90_Verify_PipelineTmaAsync_WS, Cluster4x4_Stage7) {
|
||||
Options options;
|
||||
using ClusterShape = cutlass::gemm::GemmShape<4, 4, 1>;
|
||||
static constexpr uint32_t Stages = 7;
|
||||
using Test = PipelineTest<Stages, ClusterShape>;
|
||||
Testbed<Test> testbed(options);
|
||||
EXPECT_TRUE(testbed.verification());
|
||||
}
|
||||
|
||||
TEST(SM90_Verify_PipelineTmaAsync_WS, Cluster2x1_Stage2) {
|
||||
Options options;
|
||||
using ClusterShape = cutlass::gemm::GemmShape<2, 1, 1>;
|
||||
static constexpr uint32_t Stages = 2;
|
||||
using Test = PipelineTest<Stages, ClusterShape>;
|
||||
Testbed<Test> testbed(options);
|
||||
EXPECT_TRUE(testbed.verification());
|
||||
}
|
||||
|
||||
TEST(SM90_Verify_PipelineTmaAsync_WS, Cluster2x1_Stage7) {
|
||||
Options options;
|
||||
using ClusterShape = cutlass::gemm::GemmShape<2, 1, 1>;
|
||||
static constexpr uint32_t Stages = 7;
|
||||
using Test = PipelineTest<Stages, ClusterShape>;
|
||||
Testbed<Test> testbed(options);
|
||||
EXPECT_TRUE(testbed.verification());
|
||||
}
|
||||
|
||||
TEST(SM90_Verify_PipelineTmaAsync_WS, Cluster1x2_Stage2) {
|
||||
Options options;
|
||||
using ClusterShape = cutlass::gemm::GemmShape<1, 2, 1>;
|
||||
static constexpr uint32_t Stages = 2;
|
||||
using Test = PipelineTest<Stages, ClusterShape>;
|
||||
Testbed<Test> testbed(options);
|
||||
EXPECT_TRUE(testbed.verification());
|
||||
}
|
||||
|
||||
TEST(SM90_Verify_PipelineTmaAsync_WS, Cluster1x2_Stage7) {
|
||||
Options options;
|
||||
using ClusterShape = cutlass::gemm::GemmShape<1, 2, 1>;
|
||||
static constexpr uint32_t Stages = 7;
|
||||
using Test = PipelineTest<Stages, ClusterShape>;
|
||||
Testbed<Test> testbed(options);
|
||||
EXPECT_TRUE(testbed.verification());
|
||||
}
|
||||
|
||||
TEST(SM90_Verify_PipelineTmaAsync_WS, Cluster4x1_Stage2) {
|
||||
Options options;
|
||||
using ClusterShape = cutlass::gemm::GemmShape<4, 1, 1>;
|
||||
static constexpr uint32_t Stages = 2;
|
||||
using Test = PipelineTest<Stages, ClusterShape>;
|
||||
Testbed<Test> testbed(options);
|
||||
EXPECT_TRUE(testbed.verification());
|
||||
}
|
||||
|
||||
TEST(SM90_Verify_PipelineTmaAsync_WS, Cluster4x1_Stage7) {
|
||||
Options options;
|
||||
using ClusterShape = cutlass::gemm::GemmShape<4, 1, 1>;
|
||||
static constexpr uint32_t Stages = 7;
|
||||
using Test = PipelineTest<Stages, ClusterShape>;
|
||||
Testbed<Test> testbed(options);
|
||||
EXPECT_TRUE(testbed.verification());
|
||||
}
|
||||
|
||||
TEST(SM90_Verify_PipelineTmaAsync_WS, Cluster1x4_Stage2) {
|
||||
Options options;
|
||||
using ClusterShape = cutlass::gemm::GemmShape<1, 4, 1>;
|
||||
static constexpr uint32_t Stages = 2;
|
||||
using Test = PipelineTest<Stages, ClusterShape>;
|
||||
Testbed<Test> testbed(options);
|
||||
EXPECT_TRUE(testbed.verification());
|
||||
}
|
||||
|
||||
TEST(SM90_Verify_PipelineTmaAsync_WS, Cluster1x4_Stage7) {
|
||||
Options options;
|
||||
using ClusterShape = cutlass::gemm::GemmShape<1, 4, 1>;
|
||||
static constexpr uint32_t Stages = 7;
|
||||
using Test = PipelineTest<Stages, ClusterShape>;
|
||||
Testbed<Test> testbed(options);
|
||||
EXPECT_TRUE(testbed.verification());
|
||||
}
|
||||
|
||||
TEST(SM90_Verify_PipelineTmaAsync_WS, Cluster2x4_Stage2) {
|
||||
Options options;
|
||||
using ClusterShape = cutlass::gemm::GemmShape<2, 4, 1>;
|
||||
static constexpr uint32_t Stages = 2;
|
||||
using Test = PipelineTest<Stages, ClusterShape>;
|
||||
Testbed<Test> testbed(options);
|
||||
EXPECT_TRUE(testbed.verification());
|
||||
}
|
||||
|
||||
TEST(SM90_Verify_PipelineTmaAsync_WS, Cluster2x4_Stage7) {
|
||||
Options options;
|
||||
using ClusterShape = cutlass::gemm::GemmShape<2, 4, 1>;
|
||||
static constexpr uint32_t Stages = 7;
|
||||
using Test = PipelineTest<Stages, ClusterShape>;
|
||||
Testbed<Test> testbed(options);
|
||||
EXPECT_TRUE(testbed.verification());
|
||||
}
|
||||
|
||||
TEST(SM90_Verify_PipelineTmaAsync_WS, Cluster4x2_Stage2) {
|
||||
Options options;
|
||||
using ClusterShape = cutlass::gemm::GemmShape<4, 2, 1>;
|
||||
static constexpr uint32_t Stages = 2;
|
||||
using Test = PipelineTest<Stages, ClusterShape>;
|
||||
Testbed<Test> testbed(options);
|
||||
EXPECT_TRUE(testbed.verification());
|
||||
}
|
||||
|
||||
TEST(SM90_Verify_PipelineTmaAsync_WS, Cluster4x2_Stage7) {
|
||||
Options options;
|
||||
using ClusterShape = cutlass::gemm::GemmShape<4, 2, 1>;
|
||||
static constexpr uint32_t Stages = 7;
|
||||
using Test = PipelineTest<Stages, ClusterShape>;
|
||||
Testbed<Test> testbed(options);
|
||||
EXPECT_TRUE(testbed.verification());
|
||||
}
|
||||
#endif
|
||||
@@ -0,0 +1,585 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017 - 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
/*! \file
|
||||
\brief Unit test for the PipelineTmaAsync class used in a WarpSpecialized Persistent loop
|
||||
*/
|
||||
|
||||
#define KERNEL_DBG_TRACE false
|
||||
|
||||
#include "../common/cutlass_unit_test.h"
|
||||
#include <thrust/host_vector.h>
|
||||
#include <thrust/device_vector.h>
|
||||
|
||||
#include <cute/tensor.hpp>
|
||||
#include <cute/arch/cluster_sm90.hpp>
|
||||
|
||||
#include <cutlass/util/reference/host/gemm.h>
|
||||
#include <cutlass/cluster_launch.hpp>
|
||||
|
||||
#include "cutlass/core_io.h"
|
||||
#include "cutlass/util/print_error.hpp"
|
||||
#include "cutlass/util/GPU_Clock.hpp"
|
||||
|
||||
#include "testbed.h"
|
||||
#include "cutlass/pipeline.hpp"
|
||||
#include "cutlass/arch/barrier.h"
|
||||
#include "cute/arch/cluster_sm90.hpp"
|
||||
#include "cutlass/arch/barrier.h"
|
||||
#include "cutlass/arch/reg_reconfig.h"
|
||||
|
||||
|
||||
using namespace cute;
|
||||
using namespace cutlass;
|
||||
|
||||
//////////////////// KERNEL /////////////////////////
|
||||
|
||||
template <uint32_t Stages, typename ClusterShape, typename PingPongBarrier>
|
||||
struct SharedStorage
|
||||
{
|
||||
typename cutlass::PipelineTmaAsync<Stages, ClusterShape>::SharedStorage pipeline_storage;
|
||||
typename PingPongBarrier::SharedStorage pingpong_storage;
|
||||
};
|
||||
|
||||
template <typename ClusterShape, uint32_t Stages>
|
||||
struct CollectiveSimulation {
|
||||
using MainloopPipeline = typename cutlass::PipelineTmaAsync<Stages, ClusterShape>;
|
||||
using PipelineState = typename cutlass::PipelineState<Stages>;
|
||||
|
||||
CUTLASS_DEVICE
|
||||
static void
|
||||
dma_wg_simulation(MainloopPipeline pipeline, PipelineState tile_start_state_pipe,
|
||||
uint32_t const num_iterations) {
|
||||
uint32_t const per_cta_bytes = sizeof(uint32_t);
|
||||
int warp_idx_in_warpgroup = __shfl_sync(0xffffffff, (threadIdx.x / 32) % 4, 0);
|
||||
int lane_predicate = cute::elect_one_sync();
|
||||
if (warp_idx_in_warpgroup==0 && lane_predicate) {
|
||||
|
||||
int tma_k_prologue = min(Stages, num_iterations);
|
||||
|
||||
// Simulating Prologue TMA Loads
|
||||
CUTLASS_PRAGMA_UNROLL
|
||||
for(int i = 0; i < tma_k_prologue; ++i) {
|
||||
pipeline.producer_acquire(tile_start_state_pipe);
|
||||
// Simulating cp.async.bulk.tensor behavior
|
||||
pipeline.producer_commit(tile_start_state_pipe.index(), per_cta_bytes);
|
||||
++tile_start_state_pipe;
|
||||
}
|
||||
int tma_k_iter = num_iterations - tma_k_prologue;
|
||||
|
||||
PipelineState wr_pipe = tile_start_state_pipe;
|
||||
// Simulating Mainloop TMA Loads
|
||||
CUTE_NO_UNROLL
|
||||
for ( ; tma_k_iter > 0; --tma_k_iter){
|
||||
|
||||
pipeline.producer_acquire(wr_pipe);
|
||||
|
||||
// Simulating cp.async.bulk.tensor behavior
|
||||
pipeline.producer_commit(wr_pipe.index(), per_cta_bytes);
|
||||
|
||||
// Advance write stage
|
||||
++wr_pipe;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
CUTLASS_DEVICE
|
||||
static void
|
||||
math_wg_simulation(MainloopPipeline pipeline, PipelineState tile_start_state_pipe,
|
||||
uint32_t const num_iterations, int* data_ptr) {
|
||||
PipelineState rd_pipe = tile_start_state_pipe;
|
||||
PipelineState release_pipe = rd_pipe;
|
||||
|
||||
// simulates accumulators + extra reg. pressure
|
||||
int arr[168];
|
||||
|
||||
// Init Shared Memory read stages & PhaseBit
|
||||
static constexpr uint32_t K_PIPE_MMAS = 1;
|
||||
static_assert( K_PIPE_MMAS < Stages, "ERROR : Too many MMAs in flight");
|
||||
|
||||
// Total number of gemm iterations
|
||||
auto gemm_k_iterations = num_iterations;
|
||||
|
||||
// Simulating Prologue MMAs
|
||||
int mma_k_prologue = min(K_PIPE_MMAS, gemm_k_iterations);
|
||||
CUTLASS_PRAGMA_UNROLL
|
||||
for (int iter = 0; iter < mma_k_prologue; ++iter) {
|
||||
pipeline.consumer_wait(rd_pipe);
|
||||
|
||||
warpgroup_arrive();
|
||||
// GMMA would typically happen here
|
||||
|
||||
++rd_pipe;
|
||||
}
|
||||
gemm_k_iterations -= mma_k_prologue;
|
||||
|
||||
// Simulating Mainloop MMAs
|
||||
CUTLASS_PRAGMA_NO_UNROLL
|
||||
for ( ; gemm_k_iterations > 0; --gemm_k_iterations) {
|
||||
|
||||
/// Wait on the rd_pipe stage / phase
|
||||
pipeline.consumer_wait(rd_pipe);
|
||||
|
||||
warpgroup_arrive();
|
||||
// GMMA would typically happen here
|
||||
|
||||
// Dummy op - which will never happen
|
||||
// But simulates high register usage.
|
||||
CUTE_UNROLL
|
||||
for(int i = 0; i < 168; ++i){
|
||||
if (threadIdx.x > 384){
|
||||
arr[i] += data_ptr[i];
|
||||
}
|
||||
}
|
||||
|
||||
pipeline.consumer_release(release_pipe);
|
||||
|
||||
// Advance stages
|
||||
++rd_pipe;
|
||||
++release_pipe;
|
||||
}
|
||||
|
||||
// Dummy op - which will never happen
|
||||
CUTE_UNROLL
|
||||
for(int i = 0; i < 168; ++i){
|
||||
if (threadIdx.x > 384){
|
||||
data_ptr[i] = arr[i];
|
||||
}
|
||||
}
|
||||
|
||||
// Tail Loop
|
||||
for (int i = 0; i < K_PIPE_MMAS; ++i){
|
||||
pipeline.consumer_release(release_pipe);
|
||||
++release_pipe;
|
||||
}
|
||||
|
||||
}
|
||||
};
|
||||
|
||||
struct KernelParams
|
||||
{
|
||||
uint32_t num_iterations;
|
||||
int tiles_per_cluster;
|
||||
int* data_ptr;
|
||||
};
|
||||
|
||||
// Goal of this kernel is to complete deadlock-free
|
||||
template <typename ClusterShape, uint32_t Stages>
|
||||
__launch_bounds__(384, 1)
|
||||
__global__ static
|
||||
void pipeline_device(KernelParams params)
|
||||
{
|
||||
extern __shared__ char shared_memory[];
|
||||
using DispatchPolicy = cutlass::gemm::MainloopSm90TmaGmmaWarpSpecialized<Stages,
|
||||
ClusterShape,
|
||||
cutlass::gemm::KernelTmaWarpSpecializedPersistent>;
|
||||
using MainloopPipeline = typename cutlass::PipelineTmaAsync<Stages, ClusterShape>;
|
||||
using PipelineState = typename cutlass::PipelineState<Stages>;
|
||||
|
||||
/* One for Mainloop and one for Epilogue */
|
||||
constexpr int StagesPerMathWarpGroup = 2;
|
||||
constexpr int MathWarpGroupCountPersistent = 2;
|
||||
using PingPongBarrier = typename cutlass::OrderedSequenceBarrier<StagesPerMathWarpGroup, MathWarpGroupCountPersistent>;
|
||||
|
||||
using SharedStorage = SharedStorage<Stages, ClusterShape, PingPongBarrier>;
|
||||
SharedStorage& shared_storage = *reinterpret_cast<SharedStorage*>(shared_memory);
|
||||
|
||||
auto cta_layout = Layout<ClusterShape>{}; // (m,n) -> cta_id
|
||||
int warp_group_idx = __shfl_sync(0xffffffff, threadIdx.x / NumThreadsPerWarpGroup, 0);
|
||||
int warp_group_thread_idx = threadIdx.x % NumThreadsPerWarpGroup;
|
||||
dim3 block_id_in_cluster = cute::block_id_in_cluster();
|
||||
|
||||
auto cluster_shape = ClusterShape{};
|
||||
|
||||
// #Producers = #RowsInCluster + #ColsInCluster - 1
|
||||
uint32_t const NumProducers = cute::size<0>(cluster_shape) + cute::size<1>(cluster_shape) - 1;
|
||||
uint32_t const TmaTransactionBytes = static_cast<uint32_t>(sizeof(uint32_t) * NumProducers);
|
||||
|
||||
// mbarrier.init
|
||||
typename MainloopPipeline::Params pipeline_params;
|
||||
pipeline_params.transaction_bytes = TmaTransactionBytes;
|
||||
if (warp_group_idx == 0) {
|
||||
pipeline_params.role = MainloopPipeline::ThreadCategory::Producer;
|
||||
}
|
||||
else {
|
||||
pipeline_params.role = MainloopPipeline::ThreadCategory::Consumer;
|
||||
}
|
||||
pipeline_params.is_leader = warp_group_thread_idx == 0;
|
||||
pipeline_params.num_consumers = NumThreadsPerWarpGroup;
|
||||
|
||||
MainloopPipeline pipeline(shared_storage.pipeline_storage, pipeline_params);
|
||||
PipelineState tile_start_state_pipe;
|
||||
|
||||
int tiles_per_cluster = params.tiles_per_cluster;
|
||||
|
||||
/* Offset pipeline start state for Math WG 2 */
|
||||
if (warp_group_idx == 2) {
|
||||
// Update pipeline state for next persistent tile
|
||||
tile_start_state_pipe.advance(params.num_iterations);
|
||||
tiles_per_cluster--;
|
||||
}
|
||||
|
||||
typename PingPongBarrier::Params pingpong_params;
|
||||
pingpong_params.group_id = warp_group_idx - 1; // Since DMA Warp Group Idx 0 will not participate
|
||||
pingpong_params.group_size = NumThreadsPerWarpGroup; // Number of threads / participants in a group
|
||||
PingPongBarrier math_wg_barrier(shared_storage.pingpong_storage, pingpong_params);
|
||||
|
||||
__syncthreads();
|
||||
|
||||
// Ensure All CTAs in Cluster have completed init before issuing commits
|
||||
cute::cluster_arrive_relaxed();
|
||||
cute::cluster_wait();
|
||||
|
||||
// Producer/DMA WarpGroup
|
||||
if (warp_group_idx == 0) {
|
||||
cutlass::arch::warpgroup_reg_dealloc<40>();
|
||||
// For the DMA (prologue) - we start with an opposite phase - since we skip all waits
|
||||
// i.e., we know that the buffer is indeed empty
|
||||
PipelineState tile_prologue_state_pipe = make_producer_start_state<MainloopPipeline>();
|
||||
while (tiles_per_cluster > 0) {
|
||||
CollectiveSimulation<ClusterShape,Stages>::dma_wg_simulation(pipeline, tile_prologue_state_pipe, params.num_iterations);
|
||||
// Update pipeline state for next persistent tile
|
||||
tile_prologue_state_pipe.advance(params.num_iterations);
|
||||
tiles_per_cluster--;
|
||||
}
|
||||
}
|
||||
// Math WarpGropups
|
||||
if(warp_group_idx == 1 || warp_group_idx == 2) {
|
||||
cutlass::arch::warpgroup_reg_alloc<232>();
|
||||
while (tiles_per_cluster > 0) {
|
||||
// MMA
|
||||
math_wg_barrier.wait();
|
||||
CollectiveSimulation<ClusterShape,Stages>::math_wg_simulation(pipeline, tile_start_state_pipe, params.num_iterations, params.data_ptr);
|
||||
math_wg_barrier.arrive();
|
||||
// Epilogue
|
||||
math_wg_barrier.wait();
|
||||
// Simulates long running stage
|
||||
#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 700)
|
||||
__nanosleep(100000);
|
||||
#endif
|
||||
math_wg_barrier.arrive();
|
||||
// Update pipeline state for next persistent tile
|
||||
tile_start_state_pipe.advance(params.num_iterations * 2);
|
||||
tiles_per_cluster -= 2;
|
||||
}
|
||||
}
|
||||
|
||||
// Makes sure remote SMEM doesn't get destroyed
|
||||
cute::cluster_arrive_relaxed();
|
||||
cute::cluster_wait();
|
||||
}
|
||||
/////////////////////////////////////////////////////
|
||||
|
||||
/// Device NT GMMA + TMA specialized
|
||||
template<uint32_t Stages_, typename ClusterShape_>
|
||||
struct PipelineTest {
|
||||
|
||||
//
|
||||
// Data members
|
||||
//
|
||||
static constexpr uint32_t Stages = Stages_;
|
||||
static constexpr uint32_t kBlockSize = 128 * 3;
|
||||
using ClusterShape = ClusterShape_;
|
||||
|
||||
//
|
||||
// Methods
|
||||
//
|
||||
|
||||
// Run CuTe GEMM kernel
|
||||
cudaError_t run(uint32_t const kNumIters,
|
||||
cudaStream_t stream = 0) {
|
||||
|
||||
float elapsed_ms = 0.0f;
|
||||
// Pipeline (multistage pipeline)
|
||||
auto num_stages = Int<Stages>{};
|
||||
auto cluster_shape = Shape<Int<ClusterShape::kM>, Int<ClusterShape::kN>, _1>{};
|
||||
|
||||
//
|
||||
// Configure and launch
|
||||
//
|
||||
int iterations = 1;
|
||||
cudaEvent_t events[2];
|
||||
cudaError_t result;
|
||||
|
||||
for (cudaEvent_t & event : events) {
|
||||
result = cudaEventCreate(&event);
|
||||
if (result != cudaSuccess) {
|
||||
std::cerr << "Error: Failed to create event.";
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
result = cudaEventRecord(events[0]);
|
||||
|
||||
if (result != cudaSuccess) {
|
||||
std::cerr << "Error: Failed to record start event.";
|
||||
return result;
|
||||
}
|
||||
|
||||
for (int iter = 0; iter < iterations; ++iter) {
|
||||
|
||||
using MainloopPipeline = typename cutlass::PipelineTmaAsync<Stages, decltype(cluster_shape)>;
|
||||
|
||||
constexpr int StagesPerMathWarpGroup = 2;
|
||||
constexpr int MathWarpGroupCountPersistent = 2;
|
||||
int smem_size = int(sizeof(SharedStorage<Stages, decltype(cluster_shape),
|
||||
typename cutlass::OrderedSequenceBarrier<StagesPerMathWarpGroup, MathWarpGroupCountPersistent>>));
|
||||
|
||||
result = cudaFuncSetAttribute(
|
||||
pipeline_device<decltype(cluster_shape), Stages>,
|
||||
cudaFuncAttributeMaxDynamicSharedMemorySize,
|
||||
smem_size);
|
||||
|
||||
// Launch a single Cluster, with kBlockSize threads per CTA
|
||||
dim3 dimCluster(size<0>(cluster_shape), size<1>(cluster_shape), 1);
|
||||
dim3 dimGrid(size<0>(cluster_shape), size<1>(cluster_shape), 1);
|
||||
dim3 dimBlock(kBlockSize,1,1);
|
||||
|
||||
int tiles_per_cluster = (kNumIters % 10) + 1;
|
||||
printf("Persistent version: Tiles per Cluster = %d\n", tiles_per_cluster);
|
||||
|
||||
const void* kernel = (const void*)pipeline_device<decltype(cluster_shape), Stages>;
|
||||
KernelParams params{kNumIters, tiles_per_cluster, nullptr};
|
||||
void *kernel_params[] = {¶ms};
|
||||
cutlass::ClusterLauncher::launch(dimGrid, dimCluster, dimBlock, smem_size, stream, kernel, kernel_params);
|
||||
|
||||
}
|
||||
|
||||
result = cudaEventRecord(events[1]);
|
||||
|
||||
if (result != cudaSuccess) {
|
||||
std::cerr << "Error: Failed to record stop event.";
|
||||
return result;
|
||||
}
|
||||
|
||||
result = cudaDeviceSynchronize();
|
||||
|
||||
if (result != cudaSuccess) {
|
||||
std::cerr << "Error: cudaDeviceSynchronize() failed" << std::endl;
|
||||
return result;
|
||||
}
|
||||
|
||||
result = cudaEventElapsedTime(&elapsed_ms, events[0], events[1]);
|
||||
|
||||
if (result != cudaSuccess) {
|
||||
std::cerr << "Failed to create event.";
|
||||
return result;
|
||||
}
|
||||
|
||||
for (cudaEvent_t & event : events) {
|
||||
(void)cudaEventDestroy(event);
|
||||
}
|
||||
|
||||
return cudaSuccess;
|
||||
}
|
||||
};
|
||||
|
||||
#if CUDA_12_0_SM90_FEATURES_SUPPORTED
|
||||
TEST(SM90_Verify_PipelineTmaAsync_WS_Persistent, Cluster1x1_Stage2) {
|
||||
Options options;
|
||||
using ClusterShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
static constexpr uint32_t Stages = 2;
|
||||
using Test = PipelineTest<Stages, ClusterShape>;
|
||||
Testbed<Test> testbed(options);
|
||||
EXPECT_TRUE(testbed.verification());
|
||||
}
|
||||
|
||||
TEST(SM90_Verify_PipelineTmaAsync_WS_Persistent, Cluster1x1_Stage5) {
|
||||
Options options;
|
||||
using ClusterShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
static constexpr uint32_t Stages = 5;
|
||||
using Test = PipelineTest<Stages, ClusterShape>;
|
||||
Testbed<Test> testbed(options);
|
||||
EXPECT_TRUE(testbed.verification());
|
||||
}
|
||||
|
||||
TEST(SM90_Verify_PipelineTmaAsync_WS_Persistent, Cluster1x1_Stage10) {
|
||||
Options options;
|
||||
using ClusterShape = cutlass::gemm::GemmShape<1, 1, 1>;
|
||||
static constexpr uint32_t Stages = 10;
|
||||
using Test = PipelineTest<Stages, ClusterShape>;
|
||||
Testbed<Test> testbed(options);
|
||||
EXPECT_TRUE(testbed.verification());
|
||||
}
|
||||
|
||||
TEST(SM90_Verify_PipelineTmaAsync_WS_Persistent, Cluster2x2_Stage2) {
|
||||
Options options;
|
||||
using ClusterShape = cutlass::gemm::GemmShape<2, 2, 1>;
|
||||
static constexpr uint32_t Stages = 2;
|
||||
using Test = PipelineTest<Stages, ClusterShape>;
|
||||
Testbed<Test> testbed(options);
|
||||
EXPECT_TRUE(testbed.verification());
|
||||
}
|
||||
|
||||
TEST(SM90_Verify_PipelineTmaAsync_WS_Persistent, Cluster2x2_Stage5) {
|
||||
Options options;
|
||||
using ClusterShape = cutlass::gemm::GemmShape<2, 2, 1>;
|
||||
static constexpr uint32_t Stages = 5;
|
||||
using Test = PipelineTest<Stages, ClusterShape>;
|
||||
Testbed<Test> testbed(options);
|
||||
EXPECT_TRUE(testbed.verification());
|
||||
}
|
||||
|
||||
TEST(SM90_Verify_PipelineTmaAsync_WS_Persistent, Cluster2x2_Stage7) {
|
||||
Options options;
|
||||
using ClusterShape = cutlass::gemm::GemmShape<2, 2, 1>;
|
||||
static constexpr uint32_t Stages = 7;
|
||||
using Test = PipelineTest<Stages, ClusterShape>;
|
||||
Testbed<Test> testbed(options);
|
||||
EXPECT_TRUE(testbed.verification());
|
||||
}
|
||||
|
||||
TEST(SM90_Verify_PipelineTmaAsync_WS_Persistent, Cluster4x4_Stage2) {
|
||||
Options options;
|
||||
using ClusterShape = cutlass::gemm::GemmShape<4, 4, 1>;
|
||||
static constexpr uint32_t Stages = 2;
|
||||
using Test = PipelineTest<Stages, ClusterShape>;
|
||||
Testbed<Test> testbed(options);
|
||||
EXPECT_TRUE(testbed.verification());
|
||||
}
|
||||
|
||||
TEST(SM90_Verify_PipelineTmaAsync_WS_Persistent, Cluster4x4_Stage7) {
|
||||
Options options;
|
||||
using ClusterShape = cutlass::gemm::GemmShape<4, 4, 1>;
|
||||
static constexpr uint32_t Stages = 7;
|
||||
using Test = PipelineTest<Stages, ClusterShape>;
|
||||
Testbed<Test> testbed(options);
|
||||
EXPECT_TRUE(testbed.verification());
|
||||
}
|
||||
|
||||
TEST(SM90_Verify_PipelineTmaAsync_WS_Persistent, Cluster2x1_Stage2) {
|
||||
Options options;
|
||||
using ClusterShape = cutlass::gemm::GemmShape<2, 1, 1>;
|
||||
static constexpr uint32_t Stages = 2;
|
||||
using Test = PipelineTest<Stages, ClusterShape>;
|
||||
Testbed<Test> testbed(options);
|
||||
EXPECT_TRUE(testbed.verification());
|
||||
}
|
||||
|
||||
TEST(SM90_Verify_PipelineTmaAsync_WS_Persistent, Cluster2x1_Stage7) {
|
||||
Options options;
|
||||
using ClusterShape = cutlass::gemm::GemmShape<2, 1, 1>;
|
||||
static constexpr uint32_t Stages = 7;
|
||||
using Test = PipelineTest<Stages, ClusterShape>;
|
||||
Testbed<Test> testbed(options);
|
||||
EXPECT_TRUE(testbed.verification());
|
||||
}
|
||||
|
||||
TEST(SM90_Verify_PipelineTmaAsync_WS_Persistent, Cluster1x2_Stage2) {
|
||||
Options options;
|
||||
using ClusterShape = cutlass::gemm::GemmShape<1, 2, 1>;
|
||||
static constexpr uint32_t Stages = 2;
|
||||
using Test = PipelineTest<Stages, ClusterShape>;
|
||||
Testbed<Test> testbed(options);
|
||||
EXPECT_TRUE(testbed.verification());
|
||||
}
|
||||
|
||||
TEST(SM90_Verify_PipelineTmaAsync_WS_Persistent, Cluster1x2_Stage7) {
|
||||
Options options;
|
||||
using ClusterShape = cutlass::gemm::GemmShape<1, 2, 1>;
|
||||
static constexpr uint32_t Stages = 7;
|
||||
using Test = PipelineTest<Stages, ClusterShape>;
|
||||
Testbed<Test> testbed(options);
|
||||
EXPECT_TRUE(testbed.verification());
|
||||
}
|
||||
|
||||
TEST(SM90_Verify_PipelineTmaAsync_WS_Persistent, Cluster4x1_Stage2) {
|
||||
Options options;
|
||||
using ClusterShape = cutlass::gemm::GemmShape<4, 1, 1>;
|
||||
static constexpr uint32_t Stages = 2;
|
||||
using Test = PipelineTest<Stages, ClusterShape>;
|
||||
Testbed<Test> testbed(options);
|
||||
EXPECT_TRUE(testbed.verification());
|
||||
}
|
||||
|
||||
TEST(SM90_Verify_PipelineTmaAsync_WS_Persistent, Cluster4x1_Stage7) {
|
||||
Options options;
|
||||
using ClusterShape = cutlass::gemm::GemmShape<4, 1, 1>;
|
||||
static constexpr uint32_t Stages = 7;
|
||||
using Test = PipelineTest<Stages, ClusterShape>;
|
||||
Testbed<Test> testbed(options);
|
||||
EXPECT_TRUE(testbed.verification());
|
||||
}
|
||||
|
||||
TEST(SM90_Verify_PipelineTmaAsync_WS_Persistent, Cluster1x4_Stage2) {
|
||||
Options options;
|
||||
using ClusterShape = cutlass::gemm::GemmShape<1, 4, 1>;
|
||||
static constexpr uint32_t Stages = 2;
|
||||
using Test = PipelineTest<Stages, ClusterShape>;
|
||||
Testbed<Test> testbed(options);
|
||||
EXPECT_TRUE(testbed.verification());
|
||||
}
|
||||
|
||||
TEST(SM90_Verify_PipelineTmaAsync_WS_Persistent, Cluster1x4_Stage7) {
|
||||
Options options;
|
||||
using ClusterShape = cutlass::gemm::GemmShape<1, 4, 1>;
|
||||
static constexpr uint32_t Stages = 7;
|
||||
using Test = PipelineTest<Stages, ClusterShape>;
|
||||
Testbed<Test> testbed(options);
|
||||
EXPECT_TRUE(testbed.verification());
|
||||
}
|
||||
|
||||
TEST(SM90_Verify_PipelineTmaAsync_WS_Persistent, Cluster2x4_Stage2) {
|
||||
Options options;
|
||||
using ClusterShape = cutlass::gemm::GemmShape<2, 4, 1>;
|
||||
static constexpr uint32_t Stages = 2;
|
||||
using Test = PipelineTest<Stages, ClusterShape>;
|
||||
Testbed<Test> testbed(options);
|
||||
EXPECT_TRUE(testbed.verification());
|
||||
}
|
||||
|
||||
TEST(SM90_Verify_PipelineTmaAsync_WS_Persistent, Cluster2x4_Stage7) {
|
||||
Options options;
|
||||
using ClusterShape = cutlass::gemm::GemmShape<2, 4, 1>;
|
||||
static constexpr uint32_t Stages = 7;
|
||||
using Test = PipelineTest<Stages, ClusterShape>;
|
||||
Testbed<Test> testbed(options);
|
||||
EXPECT_TRUE(testbed.verification());
|
||||
}
|
||||
|
||||
TEST(SM90_Verify_PipelineTmaAsync_WS_Persistent, Cluster4x2_Stage2) {
|
||||
Options options;
|
||||
using ClusterShape = cutlass::gemm::GemmShape<4, 2, 1>;
|
||||
static constexpr uint32_t Stages = 2;
|
||||
using Test = PipelineTest<Stages, ClusterShape>;
|
||||
Testbed<Test> testbed(options);
|
||||
EXPECT_TRUE(testbed.verification());
|
||||
}
|
||||
|
||||
TEST(SM90_Verify_PipelineTmaAsync_WS_Persistent, Cluster4x2_Stage7) {
|
||||
Options options;
|
||||
using ClusterShape = cutlass::gemm::GemmShape<4, 2, 1>;
|
||||
static constexpr uint32_t Stages = 7;
|
||||
using Test = PipelineTest<Stages, ClusterShape>;
|
||||
Testbed<Test> testbed(options);
|
||||
EXPECT_TRUE(testbed.verification());
|
||||
}
|
||||
#endif
|
||||
@@ -0,0 +1,226 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017 - 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
/*! \file
|
||||
\brief Unit test for the OrderedSequenceBarrier class
|
||||
*/
|
||||
|
||||
#include "../common/cutlass_unit_test.h"
|
||||
#include <thrust/host_vector.h>
|
||||
#include <thrust/device_vector.h>
|
||||
|
||||
#include <cute/tensor.hpp>
|
||||
#include <cute/arch/cluster_sm90.hpp>
|
||||
|
||||
#include <cutlass/util/reference/host/gemm.h>
|
||||
#include <cutlass/cluster_launch.hpp>
|
||||
|
||||
#include "cutlass/core_io.h"
|
||||
|
||||
#include "cutlass/util/print_error.hpp"
|
||||
#include "cutlass/util/GPU_Clock.hpp"
|
||||
|
||||
#include "testbed.h"
|
||||
#include "cutlass/pipeline.hpp"
|
||||
#include "cutlass/arch/barrier.h"
|
||||
#include "cute/arch/cluster_sm90.hpp"
|
||||
|
||||
using namespace cute;
|
||||
|
||||
//////////////////// KERNEL /////////////////////////
|
||||
|
||||
template<typename OrderedSequencer>
|
||||
struct SharedStorage
|
||||
{
|
||||
typename OrderedSequencer::SharedStorage storage;
|
||||
};
|
||||
|
||||
// Goal of this kernel is to complete deadlock-free
|
||||
template<int Stages, int GroupCount, int ThreadsPerGroup>
|
||||
__global__ static
|
||||
void ordered_sequence_device(uint32_t const num_iterations)
|
||||
{
|
||||
|
||||
extern __shared__ char shared_memory[];
|
||||
using SequenceBarrier = typename cutlass::OrderedSequenceBarrier<Stages, GroupCount>;
|
||||
using SmemStorage = SharedStorage<SequenceBarrier>;
|
||||
|
||||
SmemStorage& shared_storage = *reinterpret_cast<SmemStorage*>(shared_memory);
|
||||
|
||||
int group_idx = threadIdx.x / ThreadsPerGroup;
|
||||
|
||||
typename SequenceBarrier::Params params;
|
||||
params.group_id = group_idx; // sequence ID
|
||||
params.group_size = ThreadsPerGroup; // Number of threads / participants in a group
|
||||
|
||||
SequenceBarrier barrier(shared_storage.storage, params);
|
||||
|
||||
// Ensure All CTAs in Cluster have completed init before issuing commits
|
||||
__syncthreads();
|
||||
cute::cluster_arrive_relaxed();
|
||||
cute::cluster_wait();
|
||||
|
||||
CUTLASS_PRAGMA_NO_UNROLL
|
||||
for (int i = 0; i < num_iterations; ++i){
|
||||
|
||||
barrier.wait();
|
||||
// STAGE 1 CODE...
|
||||
#ifndef NDEBUG
|
||||
int thread_idx_in_group = threadIdx.x % ThreadsPerGroup;
|
||||
if (thread_idx_in_group == 0) {
|
||||
printf("STAGE 0 : Group_IDX : %d, id = %d, iter = %d, tidx = %d\n", group_idx, params.id, i, threadIdx.x);
|
||||
}
|
||||
#endif
|
||||
// Simulates long running stage
|
||||
#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 700)
|
||||
__nanosleep(100000);
|
||||
#endif
|
||||
barrier.arrive();
|
||||
|
||||
barrier.wait();
|
||||
// STAGE 2 CODE...
|
||||
#ifndef NDEBUG
|
||||
if (thread_idx_in_group == 0) {
|
||||
printf("STAGE 1 : Group_IDX : %d, id = %d, iter = %d, tidx = %d\n", group_idx, params.id, i, threadIdx.x);
|
||||
}
|
||||
#endif
|
||||
// Simulates long running stage
|
||||
#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 700)
|
||||
__nanosleep(100000);
|
||||
#endif
|
||||
barrier.arrive();
|
||||
}
|
||||
|
||||
// To make sure remote SMEM doesn't get destroyed
|
||||
cute::cluster_arrive();
|
||||
cute::cluster_wait();
|
||||
}
|
||||
/////////////////////////////////////////////////////
|
||||
|
||||
template<uint32_t Stages_, uint32_t GroupCount_>
|
||||
struct PipelineTest {
|
||||
|
||||
//
|
||||
// Data members
|
||||
//
|
||||
static constexpr uint32_t ThreadsPerGroup = 128;
|
||||
static constexpr uint32_t BlockSize = GroupCount_ * ThreadsPerGroup;
|
||||
static constexpr uint32_t Stages = Stages_;
|
||||
static constexpr uint32_t GroupCount = GroupCount_;
|
||||
using SequenceBarrier = typename cutlass::OrderedSequenceBarrier<Stages, GroupCount>;
|
||||
using SmemStorage = SharedStorage<SequenceBarrier>;
|
||||
|
||||
//
|
||||
// Methods
|
||||
//
|
||||
|
||||
// Run CuTe GEMM kernel
|
||||
cudaError_t run(uint32_t const kNumIters,
|
||||
cudaStream_t stream = nullptr) {
|
||||
|
||||
// Pipeline (multistage pipeline)
|
||||
auto cluster_shape = Shape<_1, _1, _1>{};
|
||||
|
||||
//
|
||||
// Configure and launch
|
||||
//
|
||||
int iterations = 1;
|
||||
cudaError_t result;
|
||||
|
||||
for (int iter = 0; iter < iterations; ++iter) {
|
||||
|
||||
int smem_size = int(sizeof(SmemStorage));
|
||||
|
||||
result = cudaFuncSetAttribute(
|
||||
ordered_sequence_device<Stages, GroupCount, ThreadsPerGroup>,
|
||||
cudaFuncAttributeMaxDynamicSharedMemorySize,
|
||||
smem_size);
|
||||
|
||||
// Launch a single Cluster, with 128 thread per CTA
|
||||
dim3 dimCluster(size<0>(cluster_shape), size<1>(cluster_shape), size<2>(cluster_shape));
|
||||
dim3 dimGrid(size<0>(cluster_shape), size<1>(cluster_shape), 1);
|
||||
dim3 dimBlock(BlockSize,1,1);
|
||||
|
||||
const void* kernel = (const void*)ordered_sequence_device<Stages, GroupCount, ThreadsPerGroup>;
|
||||
int iters = kNumIters;
|
||||
void* kernel_params[] = {reinterpret_cast<void*>(&iters)};
|
||||
cutlass::ClusterLauncher::launch(dimGrid, dimCluster, dimBlock, smem_size, stream, kernel, kernel_params);
|
||||
|
||||
} // profiling loop ends
|
||||
|
||||
result = cudaDeviceSynchronize();
|
||||
|
||||
if (result != cudaSuccess) {
|
||||
std::cerr << "Error: cudaDeviceSynchronize() failed" << std::endl;
|
||||
return result;
|
||||
}
|
||||
|
||||
return cudaSuccess;
|
||||
}
|
||||
};
|
||||
|
||||
#if CUDA_12_0_SM90_FEATURES_SUPPORTED
|
||||
TEST(SM90_Verify_OrderedSequence, Depth_2_Length_2) {
|
||||
Options options;
|
||||
static constexpr uint32_t GroupCount = 2;
|
||||
static constexpr uint32_t Stages = 2;
|
||||
using Test = PipelineTest<Stages, GroupCount>;
|
||||
Testbed<Test> testbed(options);
|
||||
EXPECT_TRUE(testbed.verification());
|
||||
}
|
||||
|
||||
TEST(SM90_Verify_OrderedSequence, Depth_2_Length_3) {
|
||||
Options options;
|
||||
static constexpr uint32_t GroupCount = 3;
|
||||
static constexpr uint32_t Stages = 2;
|
||||
using Test = PipelineTest<Stages, GroupCount>;
|
||||
Testbed<Test> testbed(options);
|
||||
EXPECT_TRUE(testbed.verification());
|
||||
}
|
||||
|
||||
TEST(SM90_Verify_OrderedSequence, Depth_2_Length_4) {
|
||||
Options options;
|
||||
static constexpr uint32_t GroupCount = 4;
|
||||
static constexpr uint32_t Stages = 2;
|
||||
using Test = PipelineTest<Stages, GroupCount>;
|
||||
Testbed<Test> testbed(options);
|
||||
EXPECT_TRUE(testbed.verification());
|
||||
}
|
||||
|
||||
TEST(SM90_Verify_OrderedSequence, Depth_2_Length_5) {
|
||||
Options options;
|
||||
static constexpr uint32_t GroupCount = 5;
|
||||
static constexpr uint32_t Stages = 2;
|
||||
using Test = PipelineTest<Stages, GroupCount>;
|
||||
Testbed<Test> testbed(options);
|
||||
EXPECT_TRUE(testbed.verification());
|
||||
}
|
||||
#endif
|
||||
@@ -0,0 +1,145 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017 - 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
* SPDX-License-Identifier: BSD-3-Clause
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions are met:
|
||||
*
|
||||
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||
* list of conditions and the following disclaimer.
|
||||
*
|
||||
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
* this list of conditions and the following disclaimer in the documentation
|
||||
* and/or other materials provided with the distribution.
|
||||
*
|
||||
* 3. Neither the name of the copyright holder nor the names of its
|
||||
* contributors may be used to endorse or promote products derived from
|
||||
* this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
* SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
* CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
* OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
/*! \file
|
||||
\brief Common Testbed file shared by Pipeline unit tests
|
||||
*/
|
||||
|
||||
#include <cstdlib>
|
||||
#include <cstdio>
|
||||
#include <cassert>
|
||||
#include <cutlass/gemm/gemm.h>
|
||||
|
||||
#include "cutlass/util/command_line.h"
|
||||
#include "../common/cutlass_unit_test.h"
|
||||
|
||||
#if CUDA_12_0_SM90_FEATURES_SUPPORTED
|
||||
#define CUTLASS_UNIT_TEST_PIPELINE true
|
||||
#else
|
||||
#define CUTLASS_UNIT_TEST_PIPELINE false
|
||||
#endif
|
||||
|
||||
// Command line test options
|
||||
struct Options {
|
||||
//
|
||||
// Data Members
|
||||
//
|
||||
bool help;
|
||||
bool verification_enabled;
|
||||
int SM_count;
|
||||
int clock_MHz;
|
||||
|
||||
//
|
||||
// Methods
|
||||
//
|
||||
Options():
|
||||
help(false),
|
||||
verification_enabled(true),
|
||||
SM_count(116),
|
||||
clock_MHz(1477)
|
||||
{ }
|
||||
|
||||
void parse(int argc, char const **args) {
|
||||
cutlass::CommandLine cmd(argc, args);
|
||||
|
||||
if (cmd.check_cmd_line_flag("help")) {
|
||||
help = true;
|
||||
}
|
||||
|
||||
cmd.get_cmd_line_argument("verification-enabled", verification_enabled, true);
|
||||
cmd.get_cmd_line_argument("sm-count", SM_count, 116);
|
||||
cmd.get_cmd_line_argument("clock", clock_MHz, 1477);
|
||||
}
|
||||
|
||||
/// Prints the usage statement.
|
||||
std::ostream & print_usage(std::ostream &out) const {
|
||||
|
||||
out << "Options:\n\n"
|
||||
<< " --help If specified, displays this usage statement.\n\n"
|
||||
<< " --verification-enabled=<bool> Enable/Disable verification\n"
|
||||
<< " --sm-count=<int> Number of SMs on the chip\n"
|
||||
<< " --clock=<int> Locked clock value in Mhz\n";
|
||||
|
||||
return out;
|
||||
}
|
||||
};
|
||||
|
||||
//
|
||||
// Testbed
|
||||
//
|
||||
|
||||
template<typename Pipeline>
|
||||
struct Testbed {
|
||||
private:
|
||||
// Commandline options
|
||||
Options options;
|
||||
|
||||
void run_test(uint32_t const kNumIters) {
|
||||
|
||||
// Run CuTe Gemm
|
||||
Pipeline pipeline;
|
||||
|
||||
cudaError_t result = pipeline.run(kNumIters);
|
||||
|
||||
CUTE_CHECK_LAST();
|
||||
}
|
||||
|
||||
|
||||
public:
|
||||
Testbed(Options const &options_) : options(options_) {
|
||||
int device_id = 0;
|
||||
cudaDeviceProp device_prop;
|
||||
CUTE_CHECK_ERROR(cudaSetDevice(device_id));
|
||||
CUTE_CHECK_ERROR(cudaGetDeviceProperties(&device_prop, device_id));
|
||||
|
||||
if (device_prop.major < 1) {
|
||||
fprintf(stderr, "Device does not support CUDA.\n");
|
||||
exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
/// Run verification Gemm problem sizes
|
||||
bool verification() {
|
||||
|
||||
std::array<uint32_t, 5> kNumIters;
|
||||
|
||||
for (int i = 0; i < kNumIters.size(); ++i) {
|
||||
kNumIters[i] = (rand() % 1000) + 1;
|
||||
}
|
||||
|
||||
for (int n : kNumIters) {
|
||||
std::cout << "Stages = " << Pipeline::Stages << " kNumIters = " << n << "\n";
|
||||
run_test(n);
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
};
|
||||
@@ -29,9 +29,5 @@
|
||||
cutlass_test_unit_add_executable(
|
||||
cutlass_test_unit_util
|
||||
tensor_reduce.cu
|
||||
)
|
||||
|
||||
cutlass_test_unit_add_executable(
|
||||
cutlass_test_unit_levels
|
||||
cutlass_test_levels.cu
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user