leavelet/cutlass
1
0
Fork 0
Code Issues Pull Requests Actions 28 Packages Projects Releases Wiki Activity

CUTLASS 2.8 (#363)

Browse Source 
CUTLASS 2.8
This commit is contained in:
Manish Gupta
2021-11-19 13:26:35 -08:00
committed by GitHub
parent 6fc5008803
commit 808c25337a
127 changed files with 18555 additions and 1338 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/unit/gemm/device/CMakeLists.txt
+10
View File
@@ -192,6 +192,7 @@ cutlass_test_unit_add_executable(
gemm_f16t_f16t_f32t_tensor_op_f32_sm80.cu
gemm_bf16n_bf16n_f32t_tensor_op_f32_sm80.cu
gemm_bf16t_bf16t_bf16t_tensor_op_f32_sm80.cu
gemm_f16n_f16n_f16n_direct_store_tensor_op_f32_sm80.cu
)
cutlass_test_unit_add_executable(
@@ -319,6 +320,15 @@ cutlass_test_unit_add_executable(
gemm_planar_complex_f16_f16_f32_tensor_op_sm80.cu
)
cutlass_test_unit_add_executable(
cutlass_test_unit_gemm_device_grouped
BATCH_SOURCES ON
BATCH_SIZE 4
gemm_grouped_sm80.cu
)
cutlass_test_unit_add_executable(
cutlass_test_unit_gemm_device_sparse_tensorop_sm80
test/unit/gemm/device/gemm_f16n_f16n_f16n_direct_store_tensor_op_f32_sm80.cu
+108
View File
@@ -0,0 +1,108 @@
/***************************************************************************************************
* Copyright (c) 2017-2021, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
* * Redistributions of source code must retain the above copyright notice, this list of
* conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright notice, this list of
* conditions and the following disclaimer in the documentation and/or other materials
* provided with the distribution.
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
* to endorse or promote products derived from this software without specific prior written
* permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/*! \file
\brief Tests for device-wide GEMM interface
*/
#include <iostream>
#include "../../common/cutlass_unit_test.h"
#include "cutlass/cutlass.h"
#include "cutlass/gemm/kernel/gemm_universal.h"
#include "cutlass/gemm/device/gemm_universal.h"
#include "cutlass/gemm/device/gemm_universal_adapter.h"
#include "cutlass/util/host_tensor.h"
#include "cutlass/util/reference/host/gemm.h"
#include "cutlass/util/reference/host/tensor_compare.h"
#include "cutlass/util/reference/host/tensor_copy.h"
#include "cutlass/util/reference/host/tensor_fill.h"
#include "cutlass/util/tensor_view_io.h"
#include "testbed_universal.h"
////////////////////////////////////////////////////////////////////////////////
#include "cutlass/epilogue/threadblock/epilogue_direct_store.h"
#include "cutlass/epilogue/threadblock/default_epilogue_direct_store.h"
////////////////////////////////////////////////////////////////////////////////
#if defined(CUTLASS_ARCH_MMA_SM80_SUPPORTED)
////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_GemmUniversal_DirectStore_f16n_f16t_f32n_tensor_op_f32, 128x128x32_64x64x32) {
using ElementOutput = float;
using ElementAccumulator = float;
// Define the GEMM kernel
using GemmBase = cutlass::gemm::device::GemmUniversal<
cutlass::half_t,
cutlass::layout::ColumnMajor,
cutlass::half_t,
cutlass::layout::RowMajor,
ElementOutput, cutlass::layout::ColumnMajor,
ElementAccumulator, cutlass::arch::OpClassTensorOp, cutlass::arch::Sm80,
cutlass::gemm::GemmShape<128, 128, 32>,
cutlass::gemm::GemmShape<64, 64, 32>,
cutlass::gemm::GemmShape<16, 8, 16>,
cutlass::epilogue::thread::LinearCombination<
ElementOutput,
4, // This is the vector size of the epilogue.
ElementAccumulator,
ElementAccumulator>,
cutlass::gemm::threadblock::GemmBatchedIdentityThreadblockSwizzle,
3,
8,
8
>;
// Define the direct store epilogue
using EpilogueDirectStore = typename cutlass::epilogue::threadblock::DefaultEpilogueDirectStore<
typename GemmBase::GemmKernel::Epilogue
>::Epilogue;
// Define a new kernel
using Kernel = cutlass::gemm::kernel::GemmUniversal<
typename GemmBase::GemmKernel::Mma,
EpilogueDirectStore,
typename GemmBase::GemmKernel::ThreadblockSwizzle
>;
// Define the adaptor
using Gemm = cutlass::gemm::device::GemmUniversalAdapter<Kernel>;
EXPECT_TRUE(test::gemm::device::TestAllGemmUniversal<Gemm>());
}
////////////////////////////////////////////////////////////////////////////////
#endif // #if defined(CUTLASS_ARCH_MMA_SM80_SUPPORTED)
////////////////////////////////////////////////////////////////////////////////
test/unit/gemm/device/gemm_f16n_f16t_f16t_tensor_op_f32_sm80.cu
-2
View File
@@ -50,7 +50,6 @@
TEST(SM80_Device_GemmUniversal_f16n_f16t_f32t_tensor_op_f32, 64x64x32_32x32x32) {
/*
using ElementOutput = cutlass::half_t;
using ElementAccumulator = cutlass::half_t;
@@ -66,7 +65,6 @@ TEST(SM80_Device_GemmUniversal_f16n_f16t_f32t_tensor_op_f32, 64x64x32_32x32x32)
cutlass::gemm::threadblock::GemmIdentityThreadblockSwizzle, 10>;
EXPECT_TRUE(test::gemm::device::TestAllGemm<Gemm>());
*/
}
////////////////////////////////////////////////////////////////////////////////
test/unit/gemm/device/gemm_grouped_sm80.cu
+598
View File
@@ -0,0 +1,598 @@
/***************************************************************************************************
* Copyright (c) 2017-2021, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
* * Redistributions of source code must retain the above copyright notice, this list of
* conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright notice, this list of
* conditions and the following disclaimer in the documentation and/or other materials
* provided with the distribution.
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
* to endorse or promote products derived from this software without specific prior written
* permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/*! \file
\brief Tests for device-wide GEMM interface
*/
#include <iostream>
#include "../../common/cutlass_unit_test.h"
#include "cutlass/cutlass.h"
#include "cutlass/gemm/gemm.h"
#include "cutlass/gemm/kernel/gemm_grouped.h"
#include "cutlass/gemm/kernel/default_gemm_grouped.h"
#include "cutlass/gemm/device/gemm_grouped.h"
#include "cutlass/util/host_tensor.h"
#include "cutlass/util/reference/host/gemm.h"
#include "cutlass/util/reference/host/tensor_compare.h"
#include "cutlass/util/reference/host/tensor_copy.h"
#include "cutlass/util/reference/host/tensor_fill.h"
#include "cutlass/util/tensor_view_io.h"
#include "testbed_grouped.h"
/////////////////////////////////////////////////////////////////////////////////////////////////
#if defined(CUTLASS_ARCH_MMA_SM80_SUPPORTED)
/////////////////////////////////////////////////////////////////////////////////////////////////
/// Visitor class to abstract away the algorithm for iterating over tiles.
//
// This is the prototype. We will delete this when the efficient kernel is
// available.
struct GemmGroupedProblemVisitor {
struct Params {
cutlass::gemm::GemmCoord const *problem_sizes;
int32_t problem_count;
int64_t const *tile_count;
};
struct SharedStorage {
//
// Nothing for now. As an optimization step, we could consider parallel
// argmin or prefix sums across the block.
//
};
//
// Data members
//
SharedStorage &shared_storage;
Params const &params;
cutlass::MatrixCoord threadblock_shape;
int64_t tile_idx;
int64_t tile_count_sum;
int64_t problem_tile_start;
int32_t problem_idx;
//
// Methods
//
CUTLASS_DEVICE
GemmGroupedProblemVisitor(
SharedStorage &shared_storage_,
Params const &params_,
cutlass::MatrixCoord threadblock_shape_,
int32_t block_idx
):
shared_storage(shared_storage_),
params(params_),
threadblock_shape(threadblock_shape_),
tile_idx(block_idx),
tile_count_sum(0),
problem_idx(0)
{
cutlass::gemm::GemmCoord problem = params.problem_sizes[problem_idx];
cutlass::gemm::GemmCoord grid = grid_shape(problem);
problem_tile_start = 0;
tile_count_sum = grid.m() * grid.n();
}
/// Get the grid shape
CUTLASS_HOST_DEVICE
static cutlass::gemm::GemmCoord grid_shape(
cutlass::gemm::GemmCoord const &problem,
cutlass::MatrixCoord const & block_shape) {
return cutlass::gemm::GemmCoord(
((problem.m() - 1 + block_shape.row()) / block_shape.row()),
((problem.n() - 1 + block_shape.column()) / block_shape.column()),
1);
}
/// Get the grid shape
CUTLASS_DEVICE
cutlass::gemm::GemmCoord grid_shape(cutlass::gemm::GemmCoord const &problem) const {
return grid_shape(problem, threadblock_shape);
}
/// Returns true if there is a tile to compute
CUTLASS_DEVICE
bool next_tile() {
if (tile_idx < tile_count_sum) {
return true;
}
do {
++problem_idx;
if (problem_idx >= params.problem_count) {
return false;
}
cutlass::gemm::GemmCoord problem = params.problem_sizes[problem_idx];
cutlass::gemm::GemmCoord grid = grid_shape(problem);
int64_t tile_count = grid.m() * grid.n();
problem_tile_start = tile_count_sum;
tile_count_sum += tile_count;
} while (tile_count_sum <= tile_idx);
return true;
}
/// Gets the global tile index
CUTLASS_HOST_DEVICE
int64_t tile_index() const {
return tile_idx;
}
/// Gets the index of the problem
CUTLASS_HOST_DEVICE
int32_t problem_index() const {
return problem_idx;
}
/// Returns the problem size for the current problem
CUTLASS_HOST_DEVICE
cutlass::gemm::GemmCoord problem_size() const {
return params.problem_sizes[problem_idx];
}
CUTLASS_HOST_DEVICE
int64_t threadblock_index() const {
return tile_idx - problem_tile_start;
}
CUTLASS_DEVICE
void advance(int32_t grid_size) {
tile_idx += grid_size;
}
};
/////////////////////////////////////////////////////////////////////////////////////////////////
template <int CtaShapeM, int CtaShapeN>
__global__ void GroupedBatchedKernel(GemmGroupedProblemVisitor::Params params) {
__shared__ GemmGroupedProblemVisitor::SharedStorage shared_storage;
GemmGroupedProblemVisitor problem_visitor(
shared_storage,
params,
{CtaShapeM, CtaShapeN},
blockIdx.x);
while (problem_visitor.next_tile()) {
cutlass::gemm::GemmCoord problem_size = problem_visitor.problem_size();
int64_t cta_idx = problem_visitor.threadblock_index();
cutlass::gemm::GemmCoord grid_shape = problem_visitor.grid_shape(problem_size);
int cta_tile_m_idx = int(cta_idx / grid_shape.n());
int cta_tile_n_idx = int(cta_idx % grid_shape.n());
//
// Do the MMA
//
if (threadIdx.x == 0) {
#if 0
printf("Block %d - tile: %lld, problem %d, cta_idx: %lld, cta(m: %d, n: %d)\n",
blockIdx.x,
problem_visitor.tile_index(),
problem_visitor.problem_index(),
cta_idx,
cta_tile_m_idx,
cta_tile_n_idx);
#endif
}
// Next tile
problem_visitor.advance(gridDim.x);
}
}
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_GemmGrouped_scheduler, 64x64x32_32x32x32) {
int32_t problem_count = 16;
int const kCtaShapeM = 64;
int const kCtaShapeN = 64;
std::vector<cutlass::gemm::GemmCoord> problem_sizes(problem_count);
std::vector<int64_t> tile_counts(problem_count);
// construct a few problems of random sizes
srand(1921);
for (int32_t i = 0; i < problem_count; ++i) {
problem_sizes.at(i) = cutlass::gemm::GemmCoord(
8 * (rand() % 48) + 64,
8 * (rand() % 48) + 64,
8 * (rand() % 48) + 64);
}
// compute prefix sum
int64_t tile_count = 0;
for (int32_t i = 0; i < problem_count; ++i) {
cutlass::gemm::GemmCoord grid_shape = GemmGroupedProblemVisitor::grid_shape(
problem_sizes.at(i), {kCtaShapeM, kCtaShapeN});
int32_t problem_tile_count = (grid_shape.m() * grid_shape.n());
int64_t tile_start = tile_count;
tile_count += problem_tile_count;
tile_counts.at(i) = tile_count;
if (false) {
std::cout << "Problem " << i << " size("
<< problem_sizes.at(i).m() << "-by-" << problem_sizes.at(i).n()
<< ") - tiles: " << problem_tile_count << ", grid(" << grid_shape.m() << ", " << grid_shape.n()
<< "), tiles[" << tile_start << ", " << tile_count << ")" << std::endl;
}
}
// Copy to device memory
cutlass::DeviceAllocation<cutlass::gemm::GemmCoord> problem_sizes_device(problem_count);
cutlass::DeviceAllocation<int64_t> tile_counts_device(problem_count);
problem_sizes_device.copy_from_host(problem_sizes.data());
tile_counts_device.copy_from_host(tile_counts.data());
GemmGroupedProblemVisitor::Params params;
params.problem_sizes = problem_sizes_device.get();
params.problem_count = problem_count;
params.tile_count = tile_counts_device.get();
// Launch the kernel
dim3 grid(108, 1, 1);
dim3 block(128, 1, 1);
GroupedBatchedKernel<kCtaShapeM, kCtaShapeN><<< grid, block >>>(params);
// wait
cudaDeviceSynchronize();
}
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_GemmGrouped_f16n_f16t_f32n_tensor_op_f32, 128x128x32_64x64x32) {
using ElementOutput = float;
using ElementAccumulator = float;
using GemmKernel = typename cutlass::gemm::kernel::DefaultGemmGrouped<
cutlass::half_t,
cutlass::layout::ColumnMajor,
cutlass::ComplexTransform::kNone,
8,
cutlass::half_t,
cutlass::layout::ColumnMajor,
cutlass::ComplexTransform::kNone,
8,
ElementOutput, cutlass::layout::ColumnMajor,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<128, 128, 32>,
cutlass::gemm::GemmShape<64, 64, 32>,
cutlass::gemm::GemmShape<16, 8, 16>,
cutlass::epilogue::thread::LinearCombination<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmBatchedIdentityThreadblockSwizzle,
3>::GemmKernel;
using Gemm = cutlass::gemm::device::GemmGrouped<GemmKernel>;
//
// Test
//
test::gemm::device::TestbedGrouped<Gemm> testbed;
bool passed = testbed.run(24);
EXPECT_TRUE(passed);
}
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_GemmGrouped_f16t_f16n_f32n_tensor_op_f32, 128x64x32_64x32x32) {
using ElementOutput = cutlass::half_t;
using ElementAccumulator = float;
using GemmKernel = typename cutlass::gemm::kernel::DefaultGemmGrouped<
cutlass::half_t,
cutlass::layout::RowMajor,
cutlass::ComplexTransform::kNone,
8,
cutlass::half_t,
cutlass::layout::ColumnMajor,
cutlass::ComplexTransform::kNone,
8,
ElementOutput, cutlass::layout::RowMajor,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<128, 64, 32>,
cutlass::gemm::GemmShape<64, 32, 32>,
cutlass::gemm::GemmShape<16, 8, 16>,
cutlass::epilogue::thread::LinearCombination<
ElementOutput, 128 / cutlass::sizeof_bits<ElementOutput>::value,
ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmBatchedIdentityThreadblockSwizzle,
4>::GemmKernel;
using Gemm = cutlass::gemm::device::GemmGrouped<GemmKernel>;
//
// Test
//
test::gemm::device::TestbedGrouped<Gemm> testbed;
bool passed = testbed.run(27);
EXPECT_TRUE(passed);
}
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_GemmGrouped_f64t_f64t_f64n_tensor_op_f64, 64x64x16_32x32x16) {
using ElementInput = double;
using ElementOutput = double;
using ElementAccumulator = double;
using GemmKernel = typename cutlass::gemm::kernel::DefaultGemmGrouped<
ElementInput,
cutlass::layout::RowMajor,
cutlass::ComplexTransform::kNone,
1,
ElementInput,
cutlass::layout::RowMajor,
cutlass::ComplexTransform::kNone,
1,
ElementOutput, cutlass::layout::ColumnMajor,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<64, 64, 16>,
cutlass::gemm::GemmShape<32, 32, 16>,
cutlass::gemm::GemmShape<8, 8, 4>,
cutlass::epilogue::thread::LinearCombination<
ElementOutput, 1,
ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmBatchedIdentityThreadblockSwizzle,
4>::GemmKernel;
using Gemm = cutlass::gemm::device::GemmGrouped<GemmKernel>;
//
// Test
//
test::gemm::device::TestbedGrouped<Gemm> testbed;
bool passed = testbed.run(27);
EXPECT_TRUE(passed);
}
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_GemmGrouped_f32t_f32t_f32n_simt_f32, 128x128x8_64x32x1) {
using ElementInput = float;
using ElementOutput = float;
using ElementAccumulator = float;
using GemmKernel = typename cutlass::gemm::kernel::DefaultGemmGrouped<
ElementInput,
cutlass::layout::RowMajor,
cutlass::ComplexTransform::kNone,
1,
ElementInput,
cutlass::layout::RowMajor,
cutlass::ComplexTransform::kNone,
1,
ElementOutput, cutlass::layout::ColumnMajor,
ElementAccumulator,
cutlass::arch::OpClassSimt,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<128, 128, 8>,
cutlass::gemm::GemmShape<64, 32, 8>,
cutlass::gemm::GemmShape<1, 1, 1>,
cutlass::epilogue::thread::LinearCombination<
ElementOutput, 1,
ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmBatchedIdentityThreadblockSwizzle,
3>::GemmKernel;
using Gemm = cutlass::gemm::device::GemmGrouped<GemmKernel>;
//
// Test
//
test::gemm::device::TestbedGrouped<Gemm> testbed;
bool passed = testbed.run(27);
EXPECT_TRUE(passed);
}
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_GemmGrouped_cf32n_cf32n_cf32n_tensorop_f32, 64x64x16_32x32x16) {
using ElementInput = cutlass::complex<float>;
using ElementOutput = cutlass::complex<float>;
using ElementAccumulator = cutlass::complex<float>;
using GemmKernel = typename cutlass::gemm::kernel::DefaultGemmGrouped<
ElementInput,
cutlass::layout::ColumnMajor,
cutlass::ComplexTransform::kNone,
1,
ElementInput,
cutlass::layout::ColumnMajor,
cutlass::ComplexTransform::kNone,
1,
ElementOutput, cutlass::layout::ColumnMajor,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<64, 64, 16>,
cutlass::gemm::GemmShape<32, 32, 16>,
cutlass::gemm::GemmShape<16, 8, 8>,
cutlass::epilogue::thread::LinearCombination<
ElementOutput, 1,
ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmBatchedIdentityThreadblockSwizzle,
3,
cutlass::arch::OpMultiplyAddComplex>::GemmKernel;
using Gemm = cutlass::gemm::device::GemmGrouped<GemmKernel>;
//
// Test
//
test::gemm::device::TestbedGrouped<Gemm> testbed;
bool passed = testbed.run(27);
EXPECT_TRUE(passed);
}
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_GemmGrouped_cf32c_cf32t_cf32c_tensorop_f32, 64x64x16_32x32x16) {
using ElementInput = cutlass::complex<float>;
using ElementOutput = cutlass::complex<float>;
using ElementAccumulator = cutlass::complex<float>;
using GemmKernel = typename cutlass::gemm::kernel::DefaultGemmGrouped<
ElementInput,
cutlass::layout::ColumnMajor,
cutlass::ComplexTransform::kConjugate,
1,
ElementInput,
cutlass::layout::ColumnMajor,
cutlass::ComplexTransform::kConjugate,
1,
ElementOutput, cutlass::layout::ColumnMajor,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<64, 64, 16>,
cutlass::gemm::GemmShape<32, 32, 16>,
cutlass::gemm::GemmShape<16, 8, 8>,
cutlass::epilogue::thread::LinearCombination<
ElementOutput, 1,
ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmBatchedIdentityThreadblockSwizzle,
3,
cutlass::arch::OpMultiplyAddComplex>::GemmKernel;
using Gemm = cutlass::gemm::device::GemmGrouped<GemmKernel>;
//
// Test
//
test::gemm::device::TestbedGrouped<Gemm> testbed;
bool passed = testbed.run(27);
EXPECT_TRUE(passed);
}
/////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SM80_Device_GemmGrouped_cf32t_cf32h_cf32n_tensorop_f32, 64x64x16_16x16x16) {
using ElementInput = cutlass::complex<double>;
using ElementOutput = cutlass::complex<double>;
using ElementAccumulator = cutlass::complex<double>;
using GemmKernel = typename cutlass::gemm::kernel::DefaultGemmGrouped<
ElementInput,
cutlass::layout::RowMajor,
cutlass::ComplexTransform::kNone,
1,
ElementInput,
cutlass::layout::RowMajor,
cutlass::ComplexTransform::kConjugate,
1,
ElementOutput, cutlass::layout::ColumnMajor,
ElementAccumulator,
cutlass::arch::OpClassTensorOp,
cutlass::arch::Sm80,
cutlass::gemm::GemmShape<32, 32, 16>,
cutlass::gemm::GemmShape<16, 16, 16>,
cutlass::gemm::GemmShape<8, 8, 4>,
cutlass::epilogue::thread::LinearCombination<
ElementOutput, 1,
ElementAccumulator, ElementAccumulator>,
cutlass::gemm::threadblock::GemmBatchedIdentityThreadblockSwizzle,
3,
cutlass::arch::OpMultiplyAddComplex>::GemmKernel;
using Gemm = cutlass::gemm::device::GemmGrouped<GemmKernel>;
//
// Test
//
test::gemm::device::TestbedGrouped<Gemm> testbed;
bool passed = testbed.run(27);
EXPECT_TRUE(passed);
}
/////////////////////////////////////////////////////////////////////////////////////////////////
#endif // #if defined(CUTLASS_ARCH_MMA_SM80_SUPPORTED)
/////////////////////////////////////////////////////////////////////////////////////////////////
test/unit/gemm/device/testbed_complex.h
+30 -30
View File
@@ -102,33 +102,33 @@ struct TestbedComplex : public Testbed<Gemm> {
}
/// Returns true if the CUDA device is sufficient to execute the kernel.
bool sufficient() const {
//
// Determine SMEM requirements and waive if not satisfied
//
int smem_size = int(sizeof(typename Gemm::GemmKernel::SharedStorage));
cudaDeviceProp properties;
int device_idx;
cudaError_t result = cudaGetDevice(&device_idx);
if (result != cudaSuccess) {
throw std::runtime_error("cudaGetDevice() API call failed.");
}
result = cudaGetDeviceProperties(&properties, device_idx);
if (result != cudaSuccess) {
throw std::runtime_error("cudaGetDeviceProperties() failed");
}
if (properties.sharedMemPerMultiprocessor < smem_size) {
return false;
}
return true;
}
bool sufficient() const {
//
// Determine SMEM requirements and waive if not satisfied
//
int smem_size = int(sizeof(typename Gemm::GemmKernel::SharedStorage));
cudaDeviceProp properties;
int device_idx;
cudaError_t result = cudaGetDevice(&device_idx);
if (result != cudaSuccess) {
throw std::runtime_error("cudaGetDevice() API call failed.");
}
result = cudaGetDeviceProperties(&properties, device_idx);
if (result != cudaSuccess) {
throw std::runtime_error("cudaGetDeviceProperties() failed");
}
if (properties.sharedMemPerMultiprocessor < smem_size) {
return false;
}
return true;
}
/// Executes one test
bool run(
@@ -145,9 +145,9 @@ struct TestbedComplex : public Testbed<Gemm> {
return true;
}
//
// Initialize workspace
//
//
// Initialize workspace
//
this->initialize(problem_size);
test/unit/gemm/device/testbed_grouped.h
+515
View File
@@ -0,0 +1,515 @@
/***************************************************************************************************
* Copyright (c) 2017-2021, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
* * Redistributions of source code must retain the above copyright notice, this list of
* conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright notice, this list of
* conditions and the following disclaimer in the documentation and/or other materials
* provided with the distribution.
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
* to endorse or promote products derived from this software without specific prior written
* permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/*! \file
\brief Tests for device-wide GEMM interface
*/
#pragma once
#include <iostream>
#include "../../common/cutlass_unit_test.h"
#include "cutlass/cutlass.h"
#include "cutlass/gemm/gemm.h"
#include "cutlass/gemm/kernel/gemm_grouped.h"
#include "cutlass/gemm/kernel/default_gemm_grouped.h"
#include "cutlass/gemm/device/gemm_grouped.h"
#include "cutlass/util/host_tensor.h"
#include "cutlass/util/reference/host/gemm_complex.h"
#include "cutlass/util/reference/host/tensor_compare.h"
#include "cutlass/util/reference/host/tensor_copy.h"
#include "cutlass/util/reference/host/tensor_fill.h"
#include "cutlass/util/reference/host/tensor_norm.h"
#include "cutlass/util/tensor_view_io.h"
/////////////////////////////////////////////////////////////////////////////////////////////////
namespace test {
namespace gemm {
namespace device {
/////////////////////////////////////////////////////////////////////////////////////////////////
template <typename Gemm>
struct TestbedGrouped {
//
// Type definitions
//
using ElementA = typename Gemm::ElementA;
using ElementB = typename Gemm::ElementB;
using ElementC = typename Gemm::ElementC;
using ElementAccumulator = typename Gemm::ElementAccumulator;
using EpilogueOutputOp = typename Gemm::GemmKernel::Epilogue::OutputOp;
using ElementCompute = typename EpilogueOutputOp::ElementCompute;
using LayoutA = typename Gemm::LayoutA;
using LayoutB = typename Gemm::LayoutB;
using LayoutC = typename Gemm::LayoutC;
using MatrixCoord = typename LayoutC::TensorCoord;
//
// Data members
//
/// Initialization
cutlass::Distribution::Kind init_A;
cutlass::Distribution::Kind init_B;
cutlass::Distribution::Kind init_C;
uint32_t seed;
int problem_count;
std::vector<cutlass::gemm::GemmCoord> problem_sizes_host;
cutlass::DeviceAllocation<cutlass::gemm::GemmCoord> problem_sizes_device;
std::vector<int64_t> offset_A;
std::vector<int64_t> offset_B;
std::vector<int64_t> offset_C;
std::vector<int64_t> offset_D;
std::vector<int64_t> lda_host;
std::vector<int64_t> ldb_host;
std::vector<int64_t> ldc_host;
std::vector<int64_t> ldd_host;
cutlass::DeviceAllocation<int64_t> lda;
cutlass::DeviceAllocation<int64_t> ldb;
cutlass::DeviceAllocation<int64_t> ldc;
cutlass::DeviceAllocation<int64_t> ldd;
cutlass::DeviceAllocation<ElementA> block_A;
cutlass::DeviceAllocation<ElementB> block_B;
cutlass::DeviceAllocation<ElementC> block_C;
cutlass::DeviceAllocation<ElementC> block_D;
cutlass::DeviceAllocation<ElementA *> ptr_A;
cutlass::DeviceAllocation<ElementB *> ptr_B;
cutlass::DeviceAllocation<ElementC *> ptr_C;
cutlass::DeviceAllocation<ElementC *> ptr_D;
//
// Methods
//
TestbedGrouped(
cutlass::Distribution::Kind init_A_ = cutlass::Distribution::Uniform,
cutlass::Distribution::Kind init_B_ = cutlass::Distribution::Uniform,
cutlass::Distribution::Kind init_C_ = cutlass::Distribution::Uniform,
uint32_t seed_ = 3080
):
init_A(init_A_), init_B(init_B_), init_C(init_C_), seed(seed_) { }
/// Helper to initialize a tensor view
template <typename Element, typename Layout>
bool initialize_tensor(
cutlass::TensorView<Element, Layout> view,
cutlass::Distribution::Kind dist_kind,
uint32_t seed) {
if (dist_kind == cutlass::Distribution::Uniform) {
double scope_max, scope_min;
int bits_input = cutlass::sizeof_bits<Element>::value;
int bits_output = cutlass::sizeof_bits<typename Gemm::ElementC>::value;
if (bits_input == 1) {
scope_max = 2;
scope_min = 0;
} else if (bits_input <= 8) {
scope_max = 2;
scope_min = -2;
} else if (bits_output == 16) {
if (cutlass::sizeof_bits<ElementAccumulator>::value <= 16) {
scope_max = 5;
scope_min = -5;
}
else {
scope_max = 8;
scope_min = -8;
}
} else {
scope_max = 8;
scope_min = -8;
}
cutlass::reference::host::TensorFillRandomUniform(
view, seed, scope_max, scope_min, 0);
}
else if (dist_kind == cutlass::Distribution::Identity) {
cutlass::reference::host::TensorFillIdentity(view);
}
else if (dist_kind == cutlass::Distribution::Gaussian) {
cutlass::reference::host::TensorFillRandomGaussian(view, seed, 0, 0.5);
}
else if (dist_kind == cutlass::Distribution::Sequential) {
cutlass::reference::host::BlockFillSequential(
view.data(), view.capacity());
}
else {
// no fill - remain zero
}
return true;
}
/// Initializes data structures
void initialize() {
//
// Choose random problem sizes
//
// construct a few problems of random sizes
srand(seed);
int64_t total_elements_A = 0;
int64_t total_elements_B = 0;
int64_t total_elements_C = 0;
int64_t total_elements_D = 0;
lda_host.resize(problem_count);
ldb_host.resize(problem_count);
ldc_host.resize(problem_count);
ldd_host.resize(problem_count);
problem_sizes_host.clear();
problem_sizes_host.resize(problem_count);
for (int32_t i = 0; i < problem_count; ++i) {
cutlass::gemm::GemmCoord problem(
8 * (rand() % 64) + 24,
8 * (rand() % 64) + 24,
8 * (rand() % 64) + 24);
if (!i) {
problem = cutlass::gemm::GemmCoord(48, 16, 8);
}
problem_sizes_host.at(i) = problem;
// std::cout << "Problem[" << i << "]: " << problem << std::endl;
lda_host.at(i) = LayoutA::packed({problem.m(), problem.k()}).stride(0);
ldb_host.at(i) = LayoutB::packed({problem.k(), problem.n()}).stride(0);
ldc_host.at(i) = LayoutC::packed({problem.m(), problem.n()}).stride(0);
ldd_host.at(i) = LayoutC::packed({problem.m(), problem.n()}).stride(0);
offset_A.push_back(total_elements_A);
offset_B.push_back(total_elements_B);
offset_C.push_back(total_elements_C);
offset_D.push_back(total_elements_D);
int64_t elements_A = problem.m() * problem.k();
int64_t elements_B = problem.k() * problem.n();
int64_t elements_C = problem.m() * problem.n();
int64_t elements_D = problem.m() * problem.n();
total_elements_A += elements_A;
total_elements_B += elements_B;
total_elements_C += elements_C;
total_elements_D += elements_D;
// Random strides between problems?
}
problem_sizes_device.reset(problem_count);
problem_sizes_device.copy_from_host(problem_sizes_host.data());
lda.reset(problem_count);
ldb.reset(problem_count);
ldc.reset(problem_count);
ldd.reset(problem_count);
lda.copy_from_host(lda_host.data());
ldb.copy_from_host(ldb_host.data());
ldc.copy_from_host(ldc_host.data());
ldd.copy_from_host(ldd_host.data());
//
// Assign pointers
//
block_A.reset(total_elements_A);
block_B.reset(total_elements_B);
block_C.reset(total_elements_C);
block_D.reset(total_elements_D);
std::vector<ElementA *> ptr_A_host(problem_count);
std::vector<ElementB *> ptr_B_host(problem_count);
std::vector<ElementC *> ptr_C_host(problem_count);
std::vector<ElementC *> ptr_D_host(problem_count);
for (int32_t i = 0; i < problem_count; ++i) {
ptr_A_host.at(i) = block_A.get() + offset_A.at(i);
ptr_B_host.at(i) = block_B.get() + offset_B.at(i);
ptr_C_host.at(i) = block_C.get() + offset_C.at(i);
ptr_D_host.at(i) = block_D.get() + offset_D.at(i);
}
ptr_A.reset(problem_count);
ptr_A.copy_from_host(ptr_A_host.data());
ptr_B.reset(problem_count);
ptr_B.copy_from_host(ptr_B_host.data());
ptr_C.reset(problem_count);
ptr_C.copy_from_host(ptr_C_host.data());
ptr_D.reset(problem_count);
ptr_D.copy_from_host(ptr_D_host.data());
//
// Initialize the problems of the workspace
//
for (int32_t i = 0; i < problem_count; ++i) {
cutlass::gemm::GemmCoord problem = problem_sizes_host.at(i);
LayoutA layout_A(lda_host.at(i));
LayoutB layout_B(ldb_host.at(i));
LayoutC layout_C(ldc_host.at(i));
LayoutC layout_D(ldd_host.at(i));
MatrixCoord extent_A{problem.m(), problem.k()};
MatrixCoord extent_B{problem.k(), problem.n()};
MatrixCoord extent_C{problem.m(), problem.n()};
std::vector<ElementA> matrix_A(layout_A.capacity(extent_A));
std::vector<ElementB> matrix_B(layout_B.capacity(extent_B));
std::vector<ElementC> matrix_C(layout_C.capacity(extent_C));
std::vector<ElementC> matrix_D(layout_D.capacity(extent_C));
initialize_tensor(cutlass::TensorView<ElementA, LayoutA>(matrix_A.data(), layout_A, extent_A), init_A, seed * 2021);
initialize_tensor(cutlass::TensorView<ElementB, LayoutB>(matrix_B.data(), layout_B, extent_B), init_B, seed * 2022);
initialize_tensor(cutlass::TensorView<ElementC, LayoutC>(matrix_C.data(), layout_C, extent_C), init_C, seed * 2023);
cutlass::device_memory::copy_to_device(ptr_A_host.at(i), matrix_A.data(), matrix_A.size());
cutlass::device_memory::copy_to_device(ptr_B_host.at(i), matrix_B.data(), matrix_B.size());
cutlass::device_memory::copy_to_device(ptr_C_host.at(i), matrix_C.data(), matrix_C.size());
cutlass::device_memory::copy_to_device(ptr_D_host.at(i), matrix_D.data(), matrix_D.size());
}
}
/// Verifies the result is a GEMM
bool verify(
ElementCompute alpha,
ElementCompute beta) {
bool passed = true;
for (int32_t i = 0; i < problem_count; ++i) {
cutlass::gemm::GemmCoord problem = problem_sizes_host.at(i);
LayoutA layout_A(lda_host.at(i));
LayoutB layout_B(ldb_host.at(i));
LayoutC layout_C(ldc_host.at(i));
LayoutC layout_D(ldd_host.at(i));
MatrixCoord extent_A{problem.m(), problem.k()};
MatrixCoord extent_B{problem.k(), problem.n()};
MatrixCoord extent_C{problem.m(), problem.n()};
std::vector<ElementA> matrix_A(layout_A.capacity(extent_A));
std::vector<ElementB> matrix_B(layout_B.capacity(extent_B));
std::vector<ElementC> matrix_C(layout_C.capacity(extent_C));
std::vector<ElementC> matrix_D(layout_D.capacity(extent_C));
std::vector<ElementC> matrix_Ref(layout_D.capacity(extent_C));
cutlass::device_memory::copy_to_host(matrix_A.data(), block_A.get() + offset_A.at(i), matrix_A.size());
cutlass::device_memory::copy_to_host(matrix_B.data(), block_B.get() + offset_B.at(i), matrix_B.size());
cutlass::device_memory::copy_to_host(matrix_C.data(), block_C.get() + offset_C.at(i), matrix_C.size());
cutlass::device_memory::copy_to_host(matrix_D.data(), block_D.get() + offset_D.at(i), matrix_D.size());
cutlass::TensorView<ElementA, LayoutA> view_A(matrix_A.data(), layout_A, extent_A);
cutlass::TensorView<ElementB, LayoutB> view_B(matrix_B.data(), layout_B, extent_B);
cutlass::TensorView<ElementC, LayoutC> view_C(matrix_C.data(), layout_C, extent_C);
cutlass::TensorView<ElementC, LayoutC> view_D(matrix_D.data(), layout_D, extent_C);
cutlass::TensorView<ElementC, LayoutC> view_Ref(matrix_Ref.data(), layout_D, extent_C);
// Reference GEMM
cutlass::reference::host::GemmComplex<
ElementA, LayoutA,
ElementB, LayoutB,
ElementC, LayoutC,
ElementCompute, ElementAccumulator
>(
problem,
alpha,
view_A,
Gemm::kTransformA,
view_B,
Gemm::kTransformB,
beta,
view_C,
view_Ref,
ElementAccumulator(0)
);
// Ensure that no input or output is entirely zero
EXPECT_GT(cutlass::reference::host::TensorNorm(view_A), 0);
EXPECT_GT(cutlass::reference::host::TensorNorm(view_B), 0);
EXPECT_GT(cutlass::reference::host::TensorNorm(view_C), 0);
EXPECT_GT(cutlass::reference::host::TensorNorm(view_D), 0);
EXPECT_GT(cutlass::reference::host::TensorNorm(view_Ref), 0);
// Compare against reference
passed = cutlass::reference::host::TensorEquals(view_D, view_Ref);
if (!passed) {
std::ofstream file("testbed_grouped_errors.txt");
file
<< "problem: " << problem << " [group: " << i << "]\n"
<< ", alpha: " << alpha << ", beta: " << beta << "\n\n";
file
<< "A =\n" << view_A
<< "\nB =\n" << view_B
<< "\nC =\n" << view_C
<< "\n\nReference =\n" << view_Ref
<< "\nComputed =\n" << view_D;
return passed;
}
}
return passed;
}
/// Returns the number of threadblocks to launch if the kernel can run on the target
/// device. Otherwise, returns zero.
int sufficient() const {
//
// Determine SMEM requirements and waive if not satisfied
//
int smem_size = int(sizeof(typename Gemm::GemmKernel::SharedStorage));
cudaDeviceProp properties;
int device_idx;
cudaError_t result = cudaGetDevice(&device_idx);
if (result != cudaSuccess) {
throw std::runtime_error("cudaGetDevice() API call failed.");
}
result = cudaGetDeviceProperties(&properties, device_idx);
if (result != cudaSuccess) {
throw std::runtime_error("cudaGetDeviceProperties() failed");
}
int occupancy = std::min(2, int(properties.sharedMemPerMultiprocessor / smem_size));
return properties.multiProcessorCount * occupancy;
}
/// Executes one test
bool run(
int problem_count,
ElementCompute alpha = ElementCompute(1),
ElementCompute beta = ElementCompute(0)) {
int threadblock_count = sufficient();
// Early exit
if (!threadblock_count) {
return false;
}
this->problem_count = problem_count;
// Initialize the problem
initialize();
// Configure the GEMM arguments
typename EpilogueOutputOp::Params epilogue_op(alpha, beta);
// Configure GEMM arguments
typename Gemm::Arguments args(
problem_sizes_device.get(),
problem_count,
threadblock_count,
epilogue_op,
ptr_A.get(),
ptr_B.get(),
ptr_C.get(),
ptr_D.get(),
lda.get(),
ldb.get(),
ldc.get(),
ldd.get()
);
// Initialize the GEMM object
Gemm gemm;
cutlass::Status status = gemm.initialize(args);
if (status != cutlass::Status::kSuccess) {
return false;
}
// Run the GEMM object
status = gemm.run();
if (status != cutlass::Status::kSuccess) {
return false;
}
// Wait for completion
cudaError_t result = cudaDeviceSynchronize();
EXPECT_EQ(result, cudaSuccess)
<< "Kernel execution error: " << cudaGetErrorString(result);
if (result != cudaSuccess) {
return false;
}
// Verify correctness
return verify(alpha, beta);
}
};
/////////////////////////////////////////////////////////////////////////////////////////////////
} // device
} // gemm
} // test
/////////////////////////////////////////////////////////////////////////////////////////////////
test/unit/gemm/warp/gemm_complex_sm80.cu
-2
View File
@@ -689,6 +689,4 @@ TEST(SM80_warp_gemm_complex_tensor_op_f64, 32x32x8_8x8x4_nt) {
////////////////////////////////////////////////////////////////////////////////////////////////
#endif // #if defined(CUTLASS_ARCH_MMA_SM80_SUPPORTED)
test/unit/gemm/warp/gemm_sm80.cu
+2 -2
View File
@@ -635,7 +635,7 @@ TEST(SM80_warp_gemm_tensor_op_congruous_tf32, 128x128x32_32x32x32_16x8x8) {
}
////////////////////////////////////////////////////////////////////////////////
TEST(SM80_warp_gemm_tensor_op_tn, tf32_round_128x128x32_16x16x32_16x8x8) {
TEST(SM80_warp_gemm_tensor_op_tn, tf32_round_128x128x32_64x64x32_16x8x8) {
using Shape = cutlass::gemm::GemmShape<64, 64, 32>;
using InstructionShape = cutlass::gemm::GemmShape<16, 8, 8>;
@@ -657,7 +657,7 @@ TEST(SM80_warp_gemm_tensor_op_tn, tf32_round_128x128x32_16x16x32_16x8x8) {
.run();
}
TEST(SM80_warp_gemm_tensor_op_nt, tf32_round_128x128x32_16x16x32_16x8x8) {
TEST(SM80_warp_gemm_tensor_op_nt, tf32_round_128x128x32_64x64x32_16x8x8) {
using Shape = cutlass::gemm::GemmShape<64, 64, 32>;
using InstructionShape = cutlass::gemm::GemmShape<16, 8, 8>;