Checkpointing CUTLASS 1.1 release.
This commit is contained in:
akerr
@@ -37,14 +37,23 @@ link_libraries(
|
||||
set(CUTLASS_UNIT_TEST_HEADERS
|
||||
cutlass_unit_test.h
|
||||
core/layout_verification.h
|
||||
gemm/run_gemm.h
|
||||
gemm/gemm_testbed.h
|
||||
)
|
||||
|
||||
set(CUTLASS_UNIT_TEST_SOURCES
|
||||
cutlass_unit_test.cpp
|
||||
core/tensor_ref.cu
|
||||
core/tensor_view.cu
|
||||
util/unique_ptr.cu
|
||||
util/host_tensor.cu
|
||||
util/complex.cu
|
||||
util/tensor_foreach.cu
|
||||
util/tensor_elementwise.cu
|
||||
core/layout_verification.cu
|
||||
core/predicate_vector.cu
|
||||
core/tile_iterator.cu
|
||||
core/zip_tile_iterator.cu
|
||||
gemm/dgemm.cu
|
||||
gemm/hgemm_128x128x8.cu
|
||||
gemm/hgemm_128x128x16.cu
|
||||
@@ -68,7 +77,19 @@ set(CUTLASS_UNIT_TEST_SOURCES
|
||||
gemm/sgemm_64x64x16.cu
|
||||
gemm/sgemm_64x32x8.cu
|
||||
gemm/sgemm_64x32x16.cu
|
||||
gemm/fp16_sgemm_fp32_128x128x16.cu
|
||||
gemm/fp16_sgemm_fp16_128x128x16.cu
|
||||
gemm/wmma_gemm.cu
|
||||
gemm/wmma_binary_gemm.cu
|
||||
gemm/wmma_integer_gemm.cu
|
||||
gemm/sgemm_threadblock_swizzle_nn.cu
|
||||
gemm/sgemm_threadblock_swizzle_nt.cu
|
||||
gemm/sgemm_threadblock_swizzle_tn.cu
|
||||
gemm/sgemm_threadblock_swizzle_tt.cu
|
||||
gemm/batched_strided_sgemm_128x128x8.cu
|
||||
gemm/batched_strided_dgemm_128x128x8.cu
|
||||
gemm/batched_strided_hgemm_128x128x8.cu
|
||||
gemm/epilogue_functor.cu
|
||||
)
|
||||
|
||||
if (CUTLASS_NVRTC_ENABLE)
|
||||
@@ -101,4 +122,6 @@ if (CUTLASS_NVRTC_ENABLE)
|
||||
endif()
|
||||
endif()
|
||||
|
||||
CUDA_ADD_CUBLAS_TO_TARGET(cutlass_unit_test)
|
||||
target_link_libraries(cutlass_unit_test ${CUBLAS_LIBRARY})
|
||||
|
||||
|
||||
|
||||
@@ -22,9 +22,9 @@
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
#include <cutlass_unit_test.h>
|
||||
#include "cutlass_unit_test.h"
|
||||
#include <algorithm>
|
||||
#include <tools/test/unit/core/layout_verification.h>
|
||||
#include "tools/test/unit/core/layout_verification.h"
|
||||
|
||||
|
||||
namespace test {
|
||||
|
||||
@@ -29,12 +29,12 @@
|
||||
#include <iostream>
|
||||
#include <vector>
|
||||
|
||||
#include <cutlass/tensor_view.h>
|
||||
#include "cutlass/tensor_view.h"
|
||||
|
||||
#include <tools/util/half.h>
|
||||
#include <tools/util/host_tensor_view.h>
|
||||
#include <tools/util/tensor_view_io.h>
|
||||
#include <tools/util/type_traits.h>
|
||||
#include "tools/util/half.h"
|
||||
#include "tools/util/host_tensor_view.h"
|
||||
#include "tools/util/tensor_view_io.h"
|
||||
#include "tools/util/type_traits.h"
|
||||
|
||||
namespace test {
|
||||
|
||||
@@ -275,6 +275,8 @@ class VerifyLayout {
|
||||
for (int index = 0; index < count; ++index) {
|
||||
SourceBits element = hash(layout(index));
|
||||
|
||||
// std::cout << " " << index << ": 0x" << std::hex << element << std::dec << std::endl;
|
||||
|
||||
data[index] = element;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -26,9 +26,9 @@
|
||||
#include <cublas_v2.h>
|
||||
#include <cstring>
|
||||
|
||||
#include <cutlass_unit_test.h>
|
||||
#include <cutlass/predicate_vector.h>
|
||||
#include <tools/util/host_tensor.h>
|
||||
#include "cutlass_unit_test.h"
|
||||
#include "cutlass/predicate_vector.h"
|
||||
#include "tools/util/host_tensor.h"
|
||||
|
||||
namespace test {
|
||||
|
||||
@@ -118,3 +118,126 @@ TEST(PredicateVector, Basic) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
TEST(PredicateVector, Count) {
|
||||
|
||||
{
|
||||
typedef cutlass::PredicateVector<4, 8> PredicateVector;
|
||||
EXPECT_EQ(int(PredicateVector::kWordCount), 1)
|
||||
<< "PredicateVector<4, 8> word count: " << PredicateVector::kWordCount;
|
||||
}
|
||||
|
||||
{
|
||||
typedef cutlass::PredicateVector<4, 4> PredicateVector;
|
||||
EXPECT_EQ(int(PredicateVector::kWordCount), 1)
|
||||
<< "PredicateVector<4, 4> word count: " << PredicateVector::kWordCount;
|
||||
}
|
||||
|
||||
{
|
||||
typedef cutlass::PredicateVector<4, 2> PredicateVector;
|
||||
EXPECT_EQ(int(PredicateVector::kWordCount), 1)
|
||||
<< "PredicateVector<4, 2> word count: " << PredicateVector::kWordCount;
|
||||
}
|
||||
|
||||
{
|
||||
typedef cutlass::PredicateVector<4, 1> PredicateVector;
|
||||
EXPECT_EQ(int(PredicateVector::kWordCount), 1)
|
||||
<< "PredicateVector<4, 1> word count: " << PredicateVector::kWordCount;
|
||||
}
|
||||
|
||||
{
|
||||
typedef cutlass::PredicateVector<8, 8> PredicateVector;
|
||||
EXPECT_EQ(int(PredicateVector::kWordCount), 1)
|
||||
<< "PredicateVector<8, 8> word count: " << PredicateVector::kWordCount;
|
||||
}
|
||||
|
||||
{
|
||||
typedef cutlass::PredicateVector<8, 4> PredicateVector;
|
||||
EXPECT_EQ(int(PredicateVector::kWordCount), 1)
|
||||
<< "PredicateVector<8, 4> word count: " << PredicateVector::kWordCount;
|
||||
}
|
||||
|
||||
{
|
||||
typedef cutlass::PredicateVector<8, 2> PredicateVector;
|
||||
EXPECT_EQ(int(PredicateVector::kWordCount), 1)
|
||||
<< "PredicateVector<8, 2> word count: " << PredicateVector::kWordCount;
|
||||
}
|
||||
|
||||
{
|
||||
typedef cutlass::PredicateVector<8, 1> PredicateVector;
|
||||
EXPECT_EQ(int(PredicateVector::kWordCount), 2)
|
||||
<< "PredicateVector<8, 1> word count: " << PredicateVector::kWordCount;
|
||||
}
|
||||
|
||||
{
|
||||
typedef cutlass::PredicateVector<16, 8> PredicateVector;
|
||||
EXPECT_EQ(int(PredicateVector::kWordCount), 1)
|
||||
<< "PredicateVector<16, 8> word count: " << PredicateVector::kWordCount;
|
||||
}
|
||||
|
||||
{
|
||||
typedef cutlass::PredicateVector<16, 4> PredicateVector;
|
||||
EXPECT_EQ(int(PredicateVector::kWordCount), 1)
|
||||
<< "PredicateVector<16, 4> word count: " << PredicateVector::kWordCount;
|
||||
}
|
||||
|
||||
{
|
||||
typedef cutlass::PredicateVector<16, 2> PredicateVector;
|
||||
EXPECT_EQ(int(PredicateVector::kWordCount), 2)
|
||||
<< "PredicateVector<16, 2> word count: " << PredicateVector::kWordCount;
|
||||
}
|
||||
|
||||
{
|
||||
typedef cutlass::PredicateVector<16, 1> PredicateVector;
|
||||
EXPECT_EQ(int(PredicateVector::kWordCount), 4)
|
||||
<< "PredicateVector<16, 1> word count: " << PredicateVector::kWordCount;
|
||||
}
|
||||
|
||||
{
|
||||
typedef cutlass::PredicateVector<32, 8> PredicateVector;
|
||||
EXPECT_EQ(int(PredicateVector::kWordCount), 1)
|
||||
<< "PredicateVector<32, 8> word count: " << PredicateVector::kWordCount;
|
||||
}
|
||||
|
||||
{
|
||||
typedef cutlass::PredicateVector<32, 4> PredicateVector;
|
||||
EXPECT_EQ(int(PredicateVector::kWordCount), 2)
|
||||
<< "PredicateVector<32, 4> word count: " << PredicateVector::kWordCount;
|
||||
}
|
||||
|
||||
{
|
||||
typedef cutlass::PredicateVector<32, 2> PredicateVector;
|
||||
EXPECT_EQ(int(PredicateVector::kWordCount), 4)
|
||||
<< "PredicateVector<32, 2> word count: " << PredicateVector::kWordCount;
|
||||
}
|
||||
|
||||
{
|
||||
typedef cutlass::PredicateVector<32, 1> PredicateVector;
|
||||
EXPECT_EQ(int(PredicateVector::kWordCount), 8)
|
||||
<< "PredicateVector<32, 1> word count: " << PredicateVector::kWordCount;
|
||||
}
|
||||
|
||||
{
|
||||
typedef cutlass::PredicateVector<64, 8> PredicateVector;
|
||||
EXPECT_EQ(int(PredicateVector::kWordCount), 2)
|
||||
<< "PredicateVector<64, 8> word count: " << PredicateVector::kWordCount;
|
||||
}
|
||||
|
||||
{
|
||||
typedef cutlass::PredicateVector<64, 4> PredicateVector;
|
||||
EXPECT_EQ(int(PredicateVector::kWordCount), 4)
|
||||
<< "PredicateVector<64, 4> word count: " << PredicateVector::kWordCount;
|
||||
}
|
||||
|
||||
{
|
||||
typedef cutlass::PredicateVector<64, 2> PredicateVector;
|
||||
EXPECT_EQ(int(PredicateVector::kWordCount), 8)
|
||||
<< "PredicateVector<64, 2> word count: " << PredicateVector::kWordCount;
|
||||
}
|
||||
|
||||
{
|
||||
typedef cutlass::PredicateVector<64, 1> PredicateVector;
|
||||
EXPECT_EQ(int(PredicateVector::kWordCount), 16)
|
||||
<< "PredicateVector<64, 1> word count: " << PredicateVector::kWordCount;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,220 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
#include "cutlass_unit_test.h"
|
||||
|
||||
#include "cutlass/tensor_ref.h"
|
||||
#include "cutlass/matrix_traits.h"
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(TensorRef, basic_rank2) {
|
||||
int const M = 8;
|
||||
int const N = 16;
|
||||
|
||||
int matrix_data[M * N] = {0};
|
||||
|
||||
cutlass::TensorRef<int, 2> matrix_ref(matrix_data, cutlass::make_Coord(N, 1));
|
||||
for (int m = 0; m < M; ++m) {
|
||||
for (int n = 0; n < N; ++n) {
|
||||
matrix_ref.at(cutlass::make_Coord(m, n)) = m * N + n;
|
||||
}
|
||||
}
|
||||
|
||||
for (int m = 0; m < M; ++m) {
|
||||
for (int n = 0; n < N; ++n) {
|
||||
EXPECT_EQ(matrix_data[m * N + n], int(m * N + n));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(TensorRef, rank2_column_major) {
|
||||
int const M = 8;
|
||||
int const N = 8;
|
||||
|
||||
int matrix_data[M * N];
|
||||
|
||||
cutlass::TensorRef<int, 2, cutlass::MatrixLayout::ColumnMajor> ref(matrix_data, M);
|
||||
|
||||
for (int m = 0; m < M; ++m) {
|
||||
for (int n = 0; n < N; ++n) {
|
||||
ref.at(cutlass::make_Coord(m, n)) = m * N + n;
|
||||
}
|
||||
}
|
||||
|
||||
for (int m = 0; m < M; ++m) {
|
||||
for (int n = 0; n < N; ++n) {
|
||||
EXPECT_EQ(matrix_data[m + n * M], int(m * N + n));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(TensorRef, rank2_row_major) {
|
||||
int const M = 8;
|
||||
int const N = 16;
|
||||
|
||||
int matrix_data[M * N] = { 0 };
|
||||
|
||||
cutlass::TensorRef<int, 2, cutlass::MatrixLayout::RowMajor> ref(matrix_data, N);
|
||||
|
||||
for (int m = 0; m < M; ++m) {
|
||||
for (int n = 0; n < N; ++n) {
|
||||
ref.at(cutlass::make_Coord(m, n)) = m * N + n;
|
||||
}
|
||||
}
|
||||
|
||||
for (int m = 0; m < M; ++m) {
|
||||
for (int n = 0; n < N; ++n) {
|
||||
EXPECT_EQ(matrix_data[m * N + n], int(m * N + n));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(TensorRef, rank2_contiguous_dynamic) {
|
||||
int const M = 8;
|
||||
int const N = 16;
|
||||
|
||||
typedef cutlass::TensorRef<int, 2, cutlass::MatrixLayout::ContiguousLayout> ContiguousTensorRef;
|
||||
|
||||
cutlass::MatrixLayout::Kind layouts[] = {
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor
|
||||
};
|
||||
|
||||
for (int i = 0; i < 2; ++i) {
|
||||
|
||||
int matrix_data[M * N] = { 0 };
|
||||
|
||||
int ldm;
|
||||
int row_stride;
|
||||
int col_stride;
|
||||
|
||||
if (layouts[i] == cutlass::MatrixLayout::kColumnMajor) {
|
||||
row_stride = 1;
|
||||
col_stride = M;
|
||||
ldm = col_stride;
|
||||
}
|
||||
else {
|
||||
row_stride = N;
|
||||
col_stride = 1;
|
||||
ldm = row_stride;
|
||||
}
|
||||
|
||||
// Use helper to determine stride vector from leading dimension
|
||||
ContiguousTensorRef ref(
|
||||
matrix_data,
|
||||
cutlass::MatrixLayout::ContiguousLayout::stride(layouts[i], ldm));
|
||||
|
||||
for (int m = 0; m < M; ++m) {
|
||||
for (int n = 0; n < N; ++n) {
|
||||
ref.at(cutlass::make_Coord(m, n)) = m * N + n;
|
||||
}
|
||||
}
|
||||
|
||||
for (int m = 0; m < M; ++m) {
|
||||
for (int n = 0; n < N; ++n) {
|
||||
EXPECT_EQ(matrix_data[m * row_stride + n * col_stride], int(m * N + n));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(TensorRef, rank2_column_major_interleaved) {
|
||||
int const M = 16;
|
||||
int const N = 16;
|
||||
int const kInterleave = 4;
|
||||
|
||||
int matrix_data[M * N] = {0};
|
||||
|
||||
// Define the TensorRefMapFunc for a column-major interleaved matrix format
|
||||
typedef cutlass::MatrixLayout::ColumnMajorInterleaved<kInterleave> TensorRefMapFunc;
|
||||
|
||||
// Construct a TensorRef
|
||||
cutlass::TensorRef<
|
||||
int,
|
||||
2,
|
||||
TensorRefMapFunc> ref(matrix_data, TensorRefMapFunc::stride(M));
|
||||
|
||||
for (int m = 0; m < M; ++m) {
|
||||
for (int n = 0; n < N; ++n) {
|
||||
ref.at(cutlass::make_Coord(m, n)) = m + n * M;
|
||||
}
|
||||
}
|
||||
|
||||
// Verify
|
||||
for (int m = 0; m < M; ++m) {
|
||||
for (int n = 0; n < N; n += kInterleave) {
|
||||
for (int i = 0; i < kInterleave; ++i) {
|
||||
EXPECT_EQ(matrix_data[m * kInterleave + n * M + i], int(m + (n + i) * M));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(TensorRef, rank2_row_major_interleaved) {
|
||||
int const M = 16;
|
||||
int const N = 16;
|
||||
int const kInterleave = 4;
|
||||
|
||||
int matrix_data[M * N] = {0};
|
||||
|
||||
// Define the TensorRefMapFunc for a row-major interleaved matrix format
|
||||
typedef cutlass::MatrixLayout::RowMajorInterleaved<kInterleave> TensorRefMapFunc;
|
||||
|
||||
// Construct a TensorRef
|
||||
cutlass::TensorRef<
|
||||
int,
|
||||
2,
|
||||
TensorRefMapFunc> ref(matrix_data, TensorRefMapFunc::stride(N));
|
||||
|
||||
for (int m = 0; m < M; ++m) {
|
||||
for (int n = 0; n < N; ++n) {
|
||||
ref.at(cutlass::make_Coord(m, n)) = m + n * M;
|
||||
}
|
||||
}
|
||||
|
||||
// Verify
|
||||
for (int m = 0; m < M; m += kInterleave) {
|
||||
for (int n = 0; n < N; ++n) {
|
||||
for (int i = 0; i < kInterleave; ++i) {
|
||||
EXPECT_EQ(matrix_data[m * N + i + n * kInterleave], int((m + i) + n * M));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
@@ -0,0 +1,235 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
#include "cutlass_unit_test.h"
|
||||
|
||||
#include "cutlass/tensor_view.h"
|
||||
#include "cutlass/matrix_traits.h"
|
||||
|
||||
#include "tools/util/tensor_view_io.h"
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(TensorView, rank2_contiguous_dynamic) {
|
||||
int const M = 8;
|
||||
int const N = 16;
|
||||
|
||||
typedef cutlass::TensorView<int, 2, cutlass::MatrixLayout::ContiguousLayout> ContiguousTensorView;
|
||||
|
||||
cutlass::MatrixLayout::Kind layouts[] = {
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor
|
||||
};
|
||||
|
||||
cutlass::Coord<2> bounds = cutlass::make_Coord(M - 2, N - 2);
|
||||
|
||||
for (int i = 0; i < 2; ++i) {
|
||||
|
||||
int matrix_data[M * N] = { 0 };
|
||||
|
||||
int ldm;
|
||||
int row_stride;
|
||||
int col_stride;
|
||||
|
||||
if (layouts[i] == cutlass::MatrixLayout::kColumnMajor) {
|
||||
row_stride = 1;
|
||||
col_stride = M;
|
||||
ldm = col_stride;
|
||||
}
|
||||
else {
|
||||
row_stride = N;
|
||||
col_stride = 1;
|
||||
ldm = row_stride;
|
||||
}
|
||||
|
||||
// Use helper to determine stride vector from leading dimension
|
||||
ContiguousTensorView view(
|
||||
matrix_data,
|
||||
cutlass::MatrixLayout::ContiguousLayout::stride(layouts[i], ldm),
|
||||
bounds);
|
||||
|
||||
ASSERT_TRUE(view.good());
|
||||
|
||||
for (int m = 0; m < M; ++m) {
|
||||
for (int n = 0; n < N; ++n) {
|
||||
cutlass::Coord<2> coord = cutlass::make_Coord(m, n);
|
||||
if (view.contains(coord)) {
|
||||
view.at(coord) = m * N + n;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (int m = 0; m < M; ++m) {
|
||||
for (int n = 0; n < N; ++n) {
|
||||
int expected = 0;
|
||||
if (m < bounds[0] && n < bounds[1]) {
|
||||
expected = int(m * N + n);
|
||||
}
|
||||
EXPECT_EQ(matrix_data[m * row_stride + n * col_stride], expected);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
//
|
||||
// Uncomment the following line to observe output from printing TensorView objects
|
||||
//
|
||||
|
||||
// #define OBSERVE_TENSORVIEW_IO // uncomment to enable printing
|
||||
|
||||
#ifdef OBSERVE_TENSORVIEW_IO
|
||||
|
||||
// This test construct a TensorView of rank=2 with matrix layouts known at runtime. This
|
||||
// uses TensorRefMapFunc classes defined in cutlass/matrix_traits.h to define the mapping
|
||||
// from logical tensor indices to storage in memory.
|
||||
//
|
||||
// Helpers in tools/util/tensor_view_io.h print both the logical TensorView and the
|
||||
// linear memory of the tensor.
|
||||
TEST(TensorView, contiguous) {
|
||||
|
||||
int const M = 8;
|
||||
int const N = 16;
|
||||
|
||||
typedef cutlass::TensorView<
|
||||
int32_t,
|
||||
2,
|
||||
cutlass::MatrixLayout::ContiguousLayout> ContiguousTensorView;
|
||||
|
||||
cutlass::MatrixLayout::Kind layouts[] = {
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor
|
||||
};
|
||||
|
||||
cutlass::Coord<2> bounds = cutlass::make_Coord(M, N);
|
||||
|
||||
for (int i = 0; i < 2; ++i) {
|
||||
|
||||
int matrix_data[M * N] = { 0 };
|
||||
|
||||
int ldm;
|
||||
int row_stride;
|
||||
int col_stride;
|
||||
|
||||
if (layouts[i] == cutlass::MatrixLayout::kColumnMajor) {
|
||||
row_stride = 1;
|
||||
col_stride = M;
|
||||
ldm = col_stride;
|
||||
}
|
||||
else {
|
||||
row_stride = N;
|
||||
col_stride = 1;
|
||||
ldm = row_stride;
|
||||
}
|
||||
|
||||
// Use helper to determine stride vector from leading dimension
|
||||
ContiguousTensorView view(
|
||||
matrix_data,
|
||||
cutlass::MatrixLayout::ContiguousLayout::stride(layouts[i], ldm),
|
||||
bounds);
|
||||
|
||||
for (int m = 0; m < M; ++m) {
|
||||
for (int n = 0; n < N; ++n) {
|
||||
cutlass::Coord<2> coord = cutlass::make_Coord(m, n);
|
||||
if (view.contains(coord)) {
|
||||
view.at(coord) = m * N + n;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
std::cout << "---------\n";
|
||||
std::cout << (layouts[i] == cutlass::MatrixLayout::kColumnMajor ?
|
||||
"Column-major:" : "Row-major:") << "\n\n";
|
||||
|
||||
std::cout << "Logical view:\n";
|
||||
std::cout.width(4);
|
||||
std::cout << view << "\n" << std::endl; // Print TensorView object.
|
||||
|
||||
std::cout << "Linear memory:";
|
||||
for (int idx = 0; idx < view.capacity(); ++idx) {
|
||||
if (!(idx % (layouts[i] == cutlass::MatrixLayout::kColumnMajor ? M : N))) {
|
||||
std::cout << std::endl;
|
||||
}
|
||||
std::cout << std::setw(4) << view.at(idx) << " ";
|
||||
}
|
||||
|
||||
std::cout << "\n" << std::endl;
|
||||
}
|
||||
}
|
||||
|
||||
// This test is similar to the previous except it uses a column-major, interleaved data
|
||||
// layout. The test prints both the logical representation (a typical column-major matrix)
|
||||
// and a representation of linear memory.
|
||||
//
|
||||
// Note, the interleave=4 structure implies that every four consecutive elements in the
|
||||
// same row shall be adjacent in memory followed by the next row.
|
||||
TEST(TensorView, rank2_column_major_interleaved) {
|
||||
int const M = 16;
|
||||
int const N = 16;
|
||||
int const kInterleave = 4;
|
||||
|
||||
int matrix_data[M * N] = {0};
|
||||
|
||||
cutlass::Coord<2> bounds = cutlass::make_Coord(M, N);
|
||||
|
||||
// Define the TensorRefMapFunc for a column-major interleaved matrix format
|
||||
typedef cutlass::MatrixLayout::ColumnMajorInterleaved<kInterleave> TensorRefMapFunc;
|
||||
|
||||
// Define a TensorView of rank=2 using the column-major interleaved mapping function
|
||||
typedef cutlass::TensorView<
|
||||
int,
|
||||
2,
|
||||
TensorRefMapFunc> InterleavedTensorView;
|
||||
|
||||
InterleavedTensorView view(
|
||||
matrix_data,
|
||||
TensorRefMapFunc::stride(M),
|
||||
bounds);
|
||||
|
||||
// Initialize
|
||||
for (int m = 0; m < M; ++m) {
|
||||
for (int n = 0; n < N; ++n) {
|
||||
view.at(cutlass::make_Coord(m, n)) = m + n * M;
|
||||
}
|
||||
}
|
||||
|
||||
// Print logical view
|
||||
std::cout << "Column-major, interleave=" << kInterleave << " (logical view):\n";
|
||||
|
||||
std::cout << std::setw(4) << view << "\n" << std::endl;
|
||||
|
||||
// Now define a linear view of the same data in memory
|
||||
typedef cutlass::TensorView<int, 2, cutlass::MatrixLayout::RowMajor> LinearTensorView;
|
||||
|
||||
LinearTensorView linear_view(matrix_data, cutlass::make_Coord(N), bounds);
|
||||
|
||||
std::cout << "Linear view in memory:\n";
|
||||
std::cout << std::setw(4) << linear_view << std::endl;
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
|
||||
@@ -3,14 +3,14 @@
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
@@ -22,29 +22,37 @@
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
#include <cutlass_unit_test.h>
|
||||
#include <tools/util/host_tensor.h>
|
||||
#include <tools/util/tensor_view_io.h>
|
||||
#include <cutlass/shape.h>
|
||||
#include <cutlass/predicate_vector.h>
|
||||
#include <cutlass/tile_iterator.h>
|
||||
#include <cutlass/tile_traits_standard.h>
|
||||
#include <cutlass/iterator_access.h>
|
||||
#include "cutlass_unit_test.h"
|
||||
#include "tools/util/host_matrix.h"
|
||||
#include "tools/util/tensor_view_io.h"
|
||||
#include "cutlass/shape.h"
|
||||
#include "cutlass/predicate_vector.h"
|
||||
#include "cutlass/tile_iterator.h"
|
||||
#include "cutlass/tile_traits_standard.h"
|
||||
#include "cutlass/iterator_access.h"
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
namespace test {
|
||||
|
||||
template <typename Traits, typename Scalar>
|
||||
__global__ void load_store_global(
|
||||
typename cutlass::TileLoadIterator<Traits, Scalar, cutlass::IteratorAdvance::kH,
|
||||
cutlass::MemorySpace::kGlobal>::Scalar const *input,
|
||||
typename cutlass::TileStoreIterator<Traits, Scalar, cutlass::IteratorAdvance::kH,
|
||||
cutlass::MemorySpace::kGlobal>::Scalar *output
|
||||
) {
|
||||
template <typename Traits, typename Scalar>
|
||||
__global__ void load_store_global(
|
||||
typename cutlass::TileLoadIterator<Traits, Scalar, cutlass::IteratorAdvance::kH,
|
||||
cutlass::MemorySpace::kGlobal>::Scalar const *input,
|
||||
typename cutlass::TileStoreIterator<Traits, Scalar, cutlass::IteratorAdvance::kH,
|
||||
cutlass::MemorySpace::kGlobal>::Scalar *output,
|
||||
int kW,
|
||||
int kH,
|
||||
typename cutlass::TileStoreIterator<Traits, Scalar, cutlass::IteratorAdvance::kH,
|
||||
cutlass::MemorySpace::kGlobal>::Scalar identity = 0
|
||||
) {
|
||||
|
||||
/// Load iterator
|
||||
typedef cutlass::TileLoadIterator<Traits, Scalar, cutlass::IteratorAdvance::kH, cutlass::MemorySpace::kGlobal> LoadIterator;
|
||||
/// Store iterator
|
||||
typedef cutlass::TileStoreIterator<Traits, Scalar, cutlass::IteratorAdvance::kH, cutlass::MemorySpace::kGlobal> StoreIterator;
|
||||
/// Predicate vector
|
||||
typedef typename LoadIterator::PredicateVector PredicateVector;
|
||||
|
||||
typename LoadIterator::Params load_params;
|
||||
typename StoreIterator::Params store_params;
|
||||
@@ -56,98 +64,144 @@ namespace test {
|
||||
|
||||
LoadIterator load_iterator(load_params);
|
||||
StoreIterator store_iterator(store_params);
|
||||
PredicateVector predicates;
|
||||
|
||||
load_iterator.initialize_predicates(predicates.begin(), cutlass::make_Coord(1, kH, kW));
|
||||
|
||||
typename LoadIterator::Fragment fragment;
|
||||
|
||||
load_iterator.load(fragment);
|
||||
store_iterator.store(fragment);
|
||||
load_iterator.load_post_increment(fragment, predicates.begin());
|
||||
store_iterator.store_post_increment(fragment);
|
||||
}
|
||||
|
||||
/// Launches the load_store_global test
|
||||
template <typename Scalar, typename Tile, int kThreadsPerThreadBlock>
|
||||
void run_load_store_global(int kW, int kH) {
|
||||
|
||||
typedef cutlass::TileTraitsStandard<Tile, kThreadsPerThreadBlock> Traits;
|
||||
|
||||
typedef typename cutlass::TypeTraits<Scalar>::device_type ScalarDevice;
|
||||
|
||||
cutlass::HostMatrix<Scalar> input;
|
||||
cutlass::HostMatrix<Scalar> output;
|
||||
|
||||
input.resize(cutlass::make_Coord(Tile::kW, Tile::kH), cutlass::MatrixLayout::kColumnMajor);
|
||||
output.resize(cutlass::make_Coord(Tile::kW, Tile::kH), cutlass::MatrixLayout::kColumnMajor);
|
||||
|
||||
input.fill_linear(cutlass::make_Coord(1, Tile::kW));
|
||||
output.fill(0);
|
||||
|
||||
test::load_store_global<Traits, ScalarDevice> <<<
|
||||
dim3(1, 1, 1),
|
||||
dim3(kThreadsPerThreadBlock, 1)
|
||||
>>>(input.device_data(), output.device_data(), kW, kH);
|
||||
|
||||
cudaError_t result = cudaDeviceSynchronize();
|
||||
|
||||
ASSERT_EQ(result, cudaSuccess) << "\nCUDA kernel launch error: " << cudaGetErrorString(result)
|
||||
<< "\n";
|
||||
output.sync_host();
|
||||
|
||||
bool passed = true;
|
||||
for(int i = 0; i < Tile::kW; ++i) {
|
||||
for(int j = 0; j < Tile::kH; ++j) {
|
||||
if(i < kW && j < kH && output.at(cutlass::make_Coord(i, j)) != Scalar(Tile::kW*j+i)){
|
||||
std::cout << "FAILED: (" << i << ", " << j
|
||||
<< ") -- expected: " << (Tile::kW*j+i)
|
||||
<< ", actual: " << output.at(cutlass::make_Coord(i, j))
|
||||
<< std::endl;
|
||||
passed = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
EXPECT_TRUE(passed);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(TileIterator, tile_128x8_contiguous) {
|
||||
|
||||
static int const M = 128;
|
||||
static int const N = 1;
|
||||
static int const K = 8;
|
||||
|
||||
static int const kThreads = M;
|
||||
|
||||
typedef cutlass::Shape<K, N, M> ThreadBlockTile;
|
||||
|
||||
typedef cutlass::TileTraitsStandard<cutlass::Shape<N, K, M>, kThreads> Traits;
|
||||
|
||||
cutlass::HostTensor<float> input;
|
||||
cutlass::HostTensor<float> output;
|
||||
|
||||
input.resize_matrix(ThreadBlockTile::kW, ThreadBlockTile::kD,
|
||||
cutlass::MatrixLayout::kColumnMajor);
|
||||
|
||||
output.resize_matrix(ThreadBlockTile::kW, ThreadBlockTile::kD,
|
||||
cutlass::MatrixLayout::kColumnMajor);
|
||||
|
||||
input.fill_linear(cutlass::make_Coord(1, 1, ThreadBlockTile::kW, 1));
|
||||
output.fill(0);
|
||||
|
||||
test::load_store_global< Traits, float ><<<
|
||||
dim3(1,1,1),
|
||||
dim3(kThreads, 1)
|
||||
>>>(
|
||||
input.device_data(),
|
||||
output.device_data()
|
||||
);
|
||||
|
||||
cudaError_t result = cudaDeviceSynchronize();
|
||||
ASSERT_EQ(result, cudaSuccess) << "\nCUDA kernel launch error: " << cudaGetErrorString(result)
|
||||
<< "\n";
|
||||
output.sync_host();
|
||||
|
||||
EXPECT_TRUE(input.bit_equals(output));
|
||||
|
||||
run_load_store_global<float, cutlass::Shape<1, 8, 128>, 128>(128, 8);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(TileIterator, tile_128x8_rake) {
|
||||
|
||||
static int const M = 128;
|
||||
static int const N = 1;
|
||||
static int const K = 8;
|
||||
|
||||
static int const kThreads = 32;
|
||||
|
||||
typedef cutlass::Shape<K, N, M> ThreadBlockTile;
|
||||
|
||||
typedef cutlass::TileTraitsStandard<cutlass::Shape<N, K, M>, kThreads> Traits;
|
||||
|
||||
cutlass::HostTensor<float> input;
|
||||
cutlass::HostTensor<float> output;
|
||||
|
||||
input.resize_matrix(ThreadBlockTile::kW, ThreadBlockTile::kD,
|
||||
cutlass::MatrixLayout::kColumnMajor);
|
||||
|
||||
output.resize_matrix(ThreadBlockTile::kW, ThreadBlockTile::kD,
|
||||
cutlass::MatrixLayout::kColumnMajor);
|
||||
|
||||
input.fill_linear(cutlass::make_Coord(1, 1, ThreadBlockTile::kW, 1));
|
||||
output.fill(0);
|
||||
|
||||
test::load_store_global< Traits, float ><<<
|
||||
dim3(1,1,1),
|
||||
dim3(kThreads, 1)
|
||||
>>>(
|
||||
input.device_data(),
|
||||
output.device_data()
|
||||
);
|
||||
|
||||
cudaError_t result = cudaDeviceSynchronize();
|
||||
ASSERT_EQ(result, cudaSuccess) << "\nCUDA kernel launch error: " << cudaGetErrorString(result)
|
||||
<< "\n";
|
||||
|
||||
output.sync_host();
|
||||
|
||||
EXPECT_TRUE(input.bit_equals(output));
|
||||
|
||||
run_load_store_global<float, cutlass::Shape<1, 8, 128>, 32>(128, 8);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(TileIterator, tile_127x8_contiguous) {
|
||||
run_load_store_global<float, cutlass::Shape<1, 8, 128>, 128>(127, 8);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(TileIterator, tile_129x8_contiguous) {
|
||||
run_load_store_global<float, cutlass::Shape<1, 8, 128>, 128>(129, 8);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(TileIterator, tile_112x8_contiguous) {
|
||||
run_load_store_global<float, cutlass::Shape<1, 8, 128>, 128>(112, 8);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(TileIterator, tile_67x8_contiguous) {
|
||||
run_load_store_global<float, cutlass::Shape<1, 8, 128>, 128>(67, 8);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(TileIterator, tile_113x7_contiguous) {
|
||||
run_load_store_global<float, cutlass::Shape<1, 8, 128>, 128>(113, 7);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(TileIterator, tile_113x10_contiguous) {
|
||||
run_load_store_global<float, cutlass::Shape<1, 8, 128>, 128>(113, 10);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(TileIterator, tile_131x7_contiguous) {
|
||||
run_load_store_global<float, cutlass::Shape<1, 8, 128>, 128>(131, 7);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(TileIterator, tile_131x9_contiguous) {
|
||||
run_load_store_global<float, cutlass::Shape<1, 8, 128>, 128>(131, 9);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// Half
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(TileIterator, tile_128x8_contiguous_f16) {
|
||||
run_load_store_global<cutlass::half_t, cutlass::Shape<1, 8, 128>, 128>(128, 8);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// Double
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(TileIterator, tile_128x8_contiguous_f64) {
|
||||
run_load_store_global<double, cutlass::Shape<1, 8, 128>, 128>(128, 8);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// Int
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(TileIterator, tile_128x8_contiguous_s32) {
|
||||
run_load_store_global<int, cutlass::Shape<1, 8, 128>, 128>(128, 8);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
} // namespace test
|
||||
|
||||
@@ -0,0 +1,173 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
#include "cutlass_unit_test.h"
|
||||
#include "tools/util/host_matrix.h"
|
||||
#include "tools/util/tensor_view_io.h"
|
||||
#include "cutlass/shape.h"
|
||||
#include "cutlass/predicate_vector.h"
|
||||
#include "cutlass/tile_iterator.h"
|
||||
#include "cutlass/tile_traits_standard.h"
|
||||
|
||||
#include "cutlass/zip_tile_iterator.h"
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
namespace test {
|
||||
|
||||
/// Kernel which can use tile iterators and zip iterators
|
||||
template <typename LoadIterator, typename StoreIterator>
|
||||
__global__ void zip_iterator_kernel(
|
||||
typename LoadIterator::Params load_params,
|
||||
typename StoreIterator::Params store_params) {
|
||||
|
||||
LoadIterator load_iterator(load_params);
|
||||
StoreIterator store_iterator(store_params);
|
||||
|
||||
typename LoadIterator::Fragment fragment;
|
||||
|
||||
load_iterator.load_post_increment(fragment);
|
||||
store_iterator.store_post_increment(fragment);
|
||||
}
|
||||
|
||||
} // namespace test
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Test framework
|
||||
template <typename Scalar, typename Shape>
|
||||
struct ZipIteratorTest {
|
||||
|
||||
//
|
||||
// Type definitions
|
||||
//
|
||||
|
||||
static int const kThreadCount = 128;
|
||||
|
||||
typedef cutlass::TileTraitsStandard<Shape, kThreadCount> TileTraits;
|
||||
|
||||
typedef cutlass::TileLoadIterator<TileTraits, Scalar> ScalarLoadIterator;
|
||||
typedef cutlass::TileStoreIterator<TileTraits, Scalar> ScalarStoreIterator;
|
||||
|
||||
typedef cutlass::ZipTileIterator<ScalarLoadIterator, ScalarLoadIterator> ZipLoadIterator;
|
||||
typedef cutlass::ZipTileIterator<ScalarStoreIterator, ScalarStoreIterator> ZipStoreIterator;
|
||||
|
||||
//
|
||||
// Data members
|
||||
//
|
||||
|
||||
cutlass::HostMatrix<Scalar> tensor_source_real;
|
||||
cutlass::HostMatrix<Scalar> tensor_source_imag;
|
||||
|
||||
cutlass::HostMatrix<Scalar> tensor_dest_real;
|
||||
cutlass::HostMatrix<Scalar> tensor_dest_imag;
|
||||
|
||||
//
|
||||
// Methods
|
||||
//
|
||||
|
||||
/// Ctor
|
||||
ZipIteratorTest() {
|
||||
|
||||
tensor_source_real.resize(cutlass::make_Coord(Shape::kH, Shape::kW), cutlass::MatrixLayout::kRowMajor);
|
||||
tensor_source_imag.resize(cutlass::make_Coord(Shape::kH, Shape::kW), cutlass::MatrixLayout::kRowMajor);
|
||||
tensor_dest_real.resize(cutlass::make_Coord(Shape::kH, Shape::kW), cutlass::MatrixLayout::kRowMajor);
|
||||
tensor_dest_imag.resize(cutlass::make_Coord(Shape::kH, Shape::kW), cutlass::MatrixLayout::kRowMajor);
|
||||
}
|
||||
|
||||
/// Runs test
|
||||
void run() {
|
||||
|
||||
tensor_source_real.fill_sequential();
|
||||
tensor_source_imag.fill_sequential();
|
||||
|
||||
tensor_dest_real.fill(0);
|
||||
tensor_dest_imag.fill(0);
|
||||
|
||||
tensor_source_real.sync_device();
|
||||
tensor_source_imag.sync_device();
|
||||
tensor_dest_real.sync_device();
|
||||
tensor_dest_imag.sync_device();
|
||||
|
||||
|
||||
typename ZipLoadIterator::Params load_params;
|
||||
typename ZipStoreIterator::Params store_params;
|
||||
|
||||
load_params.first.initialize(
|
||||
tensor_source_real.device_data(),
|
||||
0,
|
||||
tensor_source_real.leading_dim(),
|
||||
1
|
||||
);
|
||||
|
||||
load_params.second.initialize(
|
||||
tensor_source_imag.device_data(),
|
||||
0,
|
||||
tensor_source_real.leading_dim(),
|
||||
1
|
||||
);
|
||||
|
||||
store_params.first.initialize(
|
||||
tensor_dest_real.device_data(),
|
||||
0,
|
||||
tensor_source_real.leading_dim(),
|
||||
1
|
||||
);
|
||||
|
||||
store_params.second.initialize(
|
||||
tensor_dest_imag.device_data(),
|
||||
0,
|
||||
tensor_source_real.leading_dim(),
|
||||
1
|
||||
);
|
||||
|
||||
/// Launch kernel
|
||||
test::zip_iterator_kernel<ZipLoadIterator, ZipStoreIterator><<<
|
||||
dim3(1,1),
|
||||
dim3(kThreadCount, 1)
|
||||
>>>(
|
||||
load_params,
|
||||
store_params
|
||||
);
|
||||
|
||||
cudaError_t result = cudaGetLastError();
|
||||
EXPECT_EQ(result, cudaSuccess) << "Error on kernel launch: " << cudaGetErrorString(result);
|
||||
|
||||
tensor_dest_real.sync_host();
|
||||
tensor_dest_imag.sync_host();
|
||||
|
||||
// Verify equivalence
|
||||
EXPECT_TRUE(tensor_dest_real.bit_equals(tensor_source_real));
|
||||
EXPECT_TRUE(tensor_dest_imag.bit_equals(tensor_source_imag));
|
||||
}
|
||||
};
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(ZipTileIterator, tile_128x8) {
|
||||
ZipIteratorTest<int, cutlass::Shape<1, 8, 128> >().run();
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
@@ -29,19 +29,65 @@
|
||||
#include <cuda_runtime_api.h>
|
||||
#include <gtest/gtest.h>
|
||||
|
||||
/// Sets flags for Unit test
|
||||
void set_gtest_flag() {
|
||||
// Default flags can be overwritten by --gtest_filter from commandline
|
||||
cudaError_t err;
|
||||
|
||||
int cudaDeviceId;
|
||||
err = cudaGetDevice(&cudaDeviceId);
|
||||
if (cudaSuccess != err) {
|
||||
std::cerr << "*** Error: Could not detect active GPU device ID"
|
||||
<< " [" << cudaGetErrorString(err) << "]" << std::endl;
|
||||
exit(1);
|
||||
}
|
||||
|
||||
cudaDeviceProp deviceProperties;
|
||||
cudaGetDeviceProperties(&deviceProperties, 0);
|
||||
err = cudaGetDeviceProperties(&deviceProperties, cudaDeviceId);
|
||||
if (cudaSuccess != err) {
|
||||
std::cerr << "*** Error: Could not get device properties for GPU " << cudaDeviceId << " ["
|
||||
<< cudaGetErrorString(err) << "]" << std::endl;
|
||||
exit(1);
|
||||
}
|
||||
|
||||
int deviceMajorMinor = deviceProperties.major * 10 + deviceProperties.minor;
|
||||
|
||||
if (deviceMajorMinor < 53)
|
||||
::testing::GTEST_FLAG(filter) = "-*Igemm*:*Hgemm*:*mma*";
|
||||
else if (deviceMajorMinor < 61)
|
||||
::testing::GTEST_FLAG(filter) = "-*Igemm*:*mma*";
|
||||
else if (deviceMajorMinor < 70)
|
||||
::testing::GTEST_FLAG(filter) = "-*mma*";
|
||||
// Defines text filters for each GEMM kernel based on minimum supported compute capability
|
||||
struct {
|
||||
|
||||
/// Unit test filter string
|
||||
char const *filter;
|
||||
|
||||
/// Minimum compute capability for the kernels in the named test
|
||||
int compute_capability;
|
||||
|
||||
/// If true, the tests are enabled strictly for one compute capability
|
||||
bool experimental;
|
||||
} test_filters[] = {
|
||||
{ "Sgemm*", 50, false },
|
||||
{ "Dgemm*", 60, false },
|
||||
{ "Fp16_sgemm*", 60, false },
|
||||
{ "Hgemm*", 60, false },
|
||||
{ "Igemm*", 61, false },
|
||||
{ "WmmaGemm*", 70, false },
|
||||
{ "WmmaInt8*", 72, false },
|
||||
{ "WmmaInt4*", 75, true },
|
||||
{ "WmmaBinary*", 75, true },
|
||||
{ 0, 0, false }
|
||||
};
|
||||
|
||||
// Set negative test filters
|
||||
std::stringstream ss;
|
||||
ss << "-";
|
||||
for (int i = 0, j = 0; test_filters[i].filter; ++i) {
|
||||
if (deviceMajorMinor < test_filters[i].compute_capability ||
|
||||
(test_filters[i].experimental && deviceMajorMinor != test_filters[i].compute_capability)) {
|
||||
|
||||
ss << (j++ ? ":" : "") << test_filters[i].filter;
|
||||
}
|
||||
}
|
||||
|
||||
::testing::GTEST_FLAG(filter) = ss.str();
|
||||
}
|
||||
|
||||
int main(int argc, char* arg[]) {
|
||||
|
||||
@@ -28,3 +28,4 @@
|
||||
#pragma diag_suppress boolean_controlling_expr_is_constant
|
||||
#include <gtest/gtest.h>
|
||||
#pragma diag_warning boolean_controlling_expr_is_constant
|
||||
#pragma warning( disable : 4503)
|
||||
|
||||
@@ -0,0 +1,103 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
#include "cutlass_unit_test.h"
|
||||
#include "cutlass/gemm/gemm.h"
|
||||
#include "cutlass/gemm/dgemm_traits.h"
|
||||
#include "tools/test/unit/gemm/gemm_testbed.h"
|
||||
#include "tools/test/unit/gemm/run_gemm.h"
|
||||
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(dgemm_strided_batched_128x128x8, dgemm_256x384x64x3_nn) {
|
||||
typedef cutlass::gemm::DgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor, cutlass::Shape<8, 128, 128> >
|
||||
DgemmTraits;
|
||||
//think about using run_gemm directly
|
||||
run_batched_strided_gemm<DgemmTraits>(256/*m*/, 384/*n*/, 64/*k*/, 3 /*batch_size*/);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(dgemm_strided_batched_128x128x8, sgemm_128x384x192x2_nn) {
|
||||
typedef cutlass::gemm::DgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor, cutlass::Shape<8, 128, 128> >
|
||||
DgemmTraits;
|
||||
//think about using run_gemm directly
|
||||
run_batched_strided_gemm<DgemmTraits>(128/*m*/, 384/*n*/, 192/*k*/, 2 /*batch_size*/);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(dgemm_strided_batched_128x128x8, dgemm_256x384x64x3_nt) {
|
||||
typedef cutlass::gemm::DgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor, cutlass::Shape<8, 128, 128> >
|
||||
DgemmTraits;
|
||||
//think about using run_gemm directly
|
||||
run_batched_strided_gemm<DgemmTraits>(256/*m*/, 384/*n*/, 64/*k*/, 3 /*batch_size*/);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(dgemm_strided_batched_128x128x8, sgemm_128x384x192x2_nt) {
|
||||
typedef cutlass::gemm::DgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor, cutlass::Shape<8, 128, 128> >
|
||||
DgemmTraits;
|
||||
//think about using run_gemm directly
|
||||
run_batched_strided_gemm<DgemmTraits>(128/*m*/, 384/*n*/, 192/*k*/, 2 /*batch_size*/);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(dgemm_strided_batched_128x128x8, dgemm_256x384x64x3_tn) {
|
||||
typedef cutlass::gemm::DgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor, cutlass::Shape<8, 128, 128> >
|
||||
DgemmTraits;
|
||||
//think about using run_gemm directly
|
||||
run_batched_strided_gemm<DgemmTraits>(256/*m*/, 384/*n*/, 64/*k*/, 3 /*batch_size*/);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(dgemm_strided_batched_128x128x8, sgemm_128x384x192x2_tn) {
|
||||
typedef cutlass::gemm::DgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor, cutlass::Shape<8, 128, 128> >
|
||||
DgemmTraits;
|
||||
//think about using run_gemm directly
|
||||
run_batched_strided_gemm<DgemmTraits>(128/*m*/, 384/*n*/, 192/*k*/, 2 /*batch_size*/);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(dgemm_strided_batched_128x128x8, dgemm_256x384x64x3_tt) {
|
||||
typedef cutlass::gemm::DgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor, cutlass::Shape<8, 128, 128> >
|
||||
DgemmTraits;
|
||||
//think about using run_gemm directly
|
||||
run_batched_strided_gemm<DgemmTraits>(256/*m*/, 384/*n*/, 64/*k*/, 3 /*batch_size*/);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
@@ -0,0 +1,112 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
#include "cutlass_unit_test.h"
|
||||
#include "cutlass/gemm/gemm.h"
|
||||
#include "cutlass/gemm/hgemm_traits.h"
|
||||
#include "tools/test/unit/gemm/gemm_testbed.h"
|
||||
#include "tools/test/unit/gemm/run_gemm.h"
|
||||
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Hgemm_strided_batched_128x128x8, hgemm_256x384x64x3_nn) {
|
||||
typedef cutlass::gemm::HgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor, cutlass::Shape<8, 128, 128> >
|
||||
HgemmTraits;
|
||||
//think about using run_gemm directly
|
||||
run_batched_strided_gemm<HgemmTraits>(256/*m*/, 384/*n*/, 64/*k*/, 3 /*batch_size*/);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Hgemm_strided_batched_128x128x8, hgemm_128x384x192x2_nn) {
|
||||
typedef cutlass::gemm::HgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor, cutlass::Shape<8, 128, 128> >
|
||||
HgemmTraits;
|
||||
//think about using run_gemm directly
|
||||
run_batched_strided_gemm<HgemmTraits>(128/*m*/, 384/*n*/, 192/*k*/, 2 /*batch_size*/);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Hgemm_strided_batched_128x128x8, hgemm_256x384x64x3_nt) {
|
||||
typedef cutlass::gemm::HgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor, cutlass::Shape<8, 128, 128> >
|
||||
HgemmTraits;
|
||||
//think about using run_gemm directly
|
||||
run_batched_strided_gemm<HgemmTraits>(256/*m*/, 384/*n*/, 64/*k*/, 3 /*batch_size*/);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Hgemm_strided_batched_128x128x8, hgemm_128x384x192x2_nt) {
|
||||
typedef cutlass::gemm::HgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor, cutlass::Shape<8, 128, 128> >
|
||||
HgemmTraits;
|
||||
//think about using run_gemm directly
|
||||
run_batched_strided_gemm<HgemmTraits>(128/*m*/, 384/*n*/, 192/*k*/, 2 /*batch_size*/);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Hgemm_strided_batched_128x128x8, hgemm_256x384x64x3_tn) {
|
||||
typedef cutlass::gemm::HgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor, cutlass::Shape<8, 128, 128> >
|
||||
HgemmTraits;
|
||||
//think about using run_gemm directly
|
||||
run_batched_strided_gemm<HgemmTraits>(256/*m*/, 384/*n*/, 64/*k*/, 3 /*batch_size*/);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Hgemm_strided_batched_128x128x8, hgemm_128x384x192x2_tn) {
|
||||
typedef cutlass::gemm::HgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor, cutlass::Shape<8, 128, 128> >
|
||||
HgemmTraits;
|
||||
//think about using run_gemm directly
|
||||
run_batched_strided_gemm<HgemmTraits>(128/*m*/, 384/*n*/, 192/*k*/, 2 /*batch_size*/);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Hgemm_strided_batched_128x128x8, hgemm_256x384x64x3_tt) {
|
||||
typedef cutlass::gemm::HgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor, cutlass::Shape<8, 128, 128> >
|
||||
HgemmTraits;
|
||||
//think about using run_gemm directly
|
||||
run_batched_strided_gemm<HgemmTraits>(256/*m*/, 384/*n*/, 64/*k*/, 3 /*batch_size*/);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Hgemm_strided_batched_128x128x8, hgemm_128x384x192x2_tt) {
|
||||
typedef cutlass::gemm::HgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor, cutlass::Shape<8, 128, 128> >
|
||||
HgemmTraits;
|
||||
//think about using run_gemm directly
|
||||
run_batched_strided_gemm<HgemmTraits>(128/*m*/, 384/*n*/, 192/*k*/, 2 /*batch_size*/);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
@@ -0,0 +1,135 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
#include "cutlass_unit_test.h"
|
||||
#include "cutlass/gemm/gemm.h"
|
||||
#include "cutlass/gemm/sgemm_traits.h"
|
||||
#include "tools/test/unit/gemm/gemm_testbed.h"
|
||||
#include "tools/test/unit/gemm/run_gemm.h"
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_strided_batched_128x128x8, sgemm_256x384x64x3_nn) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor, cutlass::Shape<8, 128, 128> >
|
||||
SgemmTraits;
|
||||
//think about using run_gemm directly
|
||||
run_batched_strided_gemm<SgemmTraits>(256/*m*/, 384/*n*/, 64/*k*/, 3 /*batch_size*/);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_strided_batched_128x128x8, sgemm_128x384x192x2_nn) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor, cutlass::Shape<8, 128, 128> >
|
||||
SgemmTraits;
|
||||
//think about using run_gemm directly
|
||||
run_batched_strided_gemm<SgemmTraits>(128/*m*/, 384/*n*/, 192/*k*/, 2 /*batch_size*/);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_strided_batched_128x128x8, sgemm_127x384x192x2_nn) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor, cutlass::Shape<8, 128, 128> >
|
||||
SgemmTraits;
|
||||
//think about using run_gemm directly
|
||||
run_batched_strided_gemm<SgemmTraits>(127/*m*/, 384/*n*/, 192/*k*/, 2 /*batch_size*/);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_strided_batched_128x128x8, sgemm_127x388x190x2_nn) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor, cutlass::Shape<8, 128, 128> >
|
||||
SgemmTraits;
|
||||
//think about using run_gemm directly
|
||||
run_batched_strided_gemm<SgemmTraits>(127/*m*/, 388/*n*/, 190/*k*/, 2 /*batch_size*/);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_strided_batched_128x128x8, sgemm_256x384x64x3_nt) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor, cutlass::Shape<8, 128, 128> >
|
||||
SgemmTraits;
|
||||
//think about using run_gemm directly
|
||||
run_batched_strided_gemm<SgemmTraits>(256/*m*/, 384/*n*/, 64/*k*/, 3 /*batch_size*/);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_strided_batched_128x128x8, sgemm_128x384x192x2_nt) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor, cutlass::Shape<8, 128, 128> >
|
||||
SgemmTraits;
|
||||
//think about using run_gemm directly
|
||||
run_batched_strided_gemm<SgemmTraits>(128/*m*/, 384/*n*/, 192/*k*/, 2 /*batch_size*/);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_strided_batched_128x128x8, sgemm_256x384x64x3_tn) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor, cutlass::Shape<8, 128, 128> >
|
||||
SgemmTraits;
|
||||
//think about using run_gemm directly
|
||||
run_batched_strided_gemm<SgemmTraits>(256/*m*/, 384/*n*/, 64/*k*/, 3 /*batch_size*/);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_strided_batched_128x128x8, sgemm_128x384x192x2_tn) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor, cutlass::Shape<8, 128, 128> >
|
||||
SgemmTraits;
|
||||
//think about using run_gemm directly
|
||||
run_batched_strided_gemm<SgemmTraits>(128/*m*/, 384/*n*/, 192/*k*/, 2 /*batch_size*/);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_strided_batched_128x128x8, sgemm_256x384x64x3_tt) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor, cutlass::Shape<8, 128, 128> >
|
||||
SgemmTraits;
|
||||
//think about using run_gemm directly
|
||||
run_batched_strided_gemm<SgemmTraits>(256/*m*/, 384/*n*/, 64/*k*/, 3 /*batch_size*/);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_strided_batched_128x128x8, sgemm_128x384x192x2_tt) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor, cutlass::Shape<8, 128, 128> >
|
||||
SgemmTraits;
|
||||
//think about using run_gemm directly
|
||||
run_batched_strided_gemm<SgemmTraits>(128/*m*/, 384/*n*/, 192/*k*/, 2 /*batch_size*/);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
@@ -0,0 +1,77 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
|
||||
template <typename GemmTraits_>
|
||||
static void run_binary_gemm(int m, int n, int k, int alpha = 1, int beta = 1) {
|
||||
typedef cutlass::gemm::Gemm<GemmTraits_> Gemm;
|
||||
typename Gemm::Params params;
|
||||
|
||||
test::GemmTestbed<cutlass::Vector<cutlass::bin1_t, 32>, // AType
|
||||
cutlass::Vector<cutlass::bin1_t, 32>, // BType
|
||||
int32_t, // CType
|
||||
int32_t, // Accumulator
|
||||
int // Scalar
|
||||
>
|
||||
testbed(m,
|
||||
n,
|
||||
k / 32,
|
||||
test::convert(GemmTraits_::kLayoutA),
|
||||
test::convert(GemmTraits_::kLayoutB),
|
||||
alpha,
|
||||
beta);
|
||||
|
||||
// Initializes the input vectors for computation
|
||||
testbed.initialize_binary();
|
||||
|
||||
// Compute the reference result on the host (CPU)
|
||||
testbed.compute_host();
|
||||
|
||||
params.initialize(testbed.M(),
|
||||
testbed.N(),
|
||||
testbed.K() * 32,
|
||||
testbed.alpha,
|
||||
testbed.ptr_A(),
|
||||
testbed.lda(),
|
||||
testbed.ptr_B(),
|
||||
testbed.ldb(),
|
||||
testbed.beta,
|
||||
testbed.ptr_C_initial(),
|
||||
testbed.ldc(),
|
||||
testbed.ptr_computed(),
|
||||
testbed.ldc());
|
||||
|
||||
Gemm::launch(params);
|
||||
|
||||
cudaError_t result = cudaDeviceSynchronize();
|
||||
ASSERT_EQ(result, cudaSuccess) << "\nCUDA kernel launch error: " << cudaGetErrorString(result)
|
||||
<< "\n";
|
||||
|
||||
testbed.computed.sync_host();
|
||||
|
||||
// Check the results
|
||||
ASSERT_TRUE(testbed.computed.bit_equals(testbed.ref_host));
|
||||
}
|
||||
@@ -24,11 +24,11 @@
|
||||
**************************************************************************************************/
|
||||
#include <cublas_v2.h>
|
||||
#include <cstring>
|
||||
#include <cutlass_unit_test.h>
|
||||
#include <cutlass/gemm/gemm.h>
|
||||
#include <cutlass/gemm/dgemm_traits.h>
|
||||
#include <tools/test/unit/gemm/gemm_testbed.h>
|
||||
#include <tools/test/unit/gemm/gemm.h>
|
||||
#include "cutlass_unit_test.h"
|
||||
#include "cutlass/gemm/gemm.h"
|
||||
#include "cutlass/gemm/dgemm_traits.h"
|
||||
#include "tools/test/unit/gemm/gemm_testbed.h"
|
||||
#include "tools/test/unit/gemm/run_gemm.h"
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
@@ -40,6 +40,7 @@ TEST(Dgemm_64x32x8, dgemm_64x32x8_nt) {
|
||||
run_gemm<GemmTraits>(64, 32, 8);
|
||||
}
|
||||
|
||||
|
||||
TEST(Dgemm_64x32x8, dgemm_256x128x64_nt) {
|
||||
|
||||
typedef cutlass::gemm::DgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
@@ -569,5 +570,3 @@ TEST(Dgemm_128x32x16, dgemm_256x64x64_tt) {
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,121 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
#include "cutlass_unit_test.h"
|
||||
#include "cutlass/gemm/gemm.h"
|
||||
#include "cutlass/gemm/linear_scaling_device_ptr.h"
|
||||
#include "cutlass/gemm/sgemm_traits.h"
|
||||
|
||||
#include "tools/test/unit/gemm/gemm_testbed.h"
|
||||
#include "tools/test/unit/gemm/run_gemm.h"
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
// This example defines an SGEMM with a linear scaling functor that supports optionally passing
|
||||
// alpha and beta via device-side pointers as in cuBLAS.
|
||||
TEST(Sgemm_epilogue_functor, device_ptr_mode_sgemm_1024x512x128_nt) {
|
||||
|
||||
typedef cutlass::gemm::SgemmTraits<
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<8, 128, 128>,
|
||||
cutlass::gemm::LinearScalingDevicePtr<float>
|
||||
>
|
||||
SgemmTraits;
|
||||
|
||||
// Define a GEMM problem size
|
||||
int const m = 1025;
|
||||
int const n = 512;
|
||||
int const k = 128;
|
||||
|
||||
// Define scalars
|
||||
float alpha_host = 3;
|
||||
float beta_host = 2;
|
||||
|
||||
// Define a device-backed tensor to contain the scalars
|
||||
cutlass::HostTensor<float, 1> device_scalars(2);
|
||||
|
||||
// Copy scalar values to device memory for device-ptr mode
|
||||
device_scalars.at(0) = alpha_host;
|
||||
device_scalars.at(1) = beta_host;
|
||||
device_scalars.sync_device();
|
||||
|
||||
// Construct a GemmTestbed instance
|
||||
test::GemmTestbed<
|
||||
float, // AType
|
||||
float, // BType
|
||||
float, // CType
|
||||
float, // Accumulator
|
||||
float // Scalar
|
||||
>
|
||||
testbed(m,
|
||||
n,
|
||||
k,
|
||||
test::convert(SgemmTraits::kLayoutA),
|
||||
test::convert(SgemmTraits::kLayoutB),
|
||||
alpha_host,
|
||||
beta_host);
|
||||
|
||||
testbed.initialize();
|
||||
|
||||
//
|
||||
// Construct a CUTLASS GEMM and initialize parameters
|
||||
//
|
||||
typedef typename SgemmTraits::KernelClass Gemm;
|
||||
typename Gemm::Params params;
|
||||
|
||||
params.initialize(testbed.M(),
|
||||
testbed.N(),
|
||||
testbed.K(),
|
||||
0, // alpha ignored
|
||||
testbed.ptr_A(),
|
||||
testbed.lda(),
|
||||
testbed.ptr_B(),
|
||||
testbed.ldb(),
|
||||
0, // beta ignored
|
||||
testbed.ptr_C_initial(),
|
||||
testbed.ldc(),
|
||||
testbed.ptr_computed(),
|
||||
testbed.ldc());
|
||||
|
||||
// Explicitly call the epilogue functor's initialize method to pass additional arguments
|
||||
params.epilogue.functor.initialize(
|
||||
device_scalars.device_data() + 0, // pointer to alpha in device memory
|
||||
device_scalars.device_data() + 1); // pointer to beta in device memory
|
||||
|
||||
// Launch the CUTLASS SGEMM kernel
|
||||
Gemm::launch(params);
|
||||
|
||||
// Report any errors
|
||||
cudaError_t result = cudaDeviceSynchronize();
|
||||
ASSERT_EQ(result, cudaSuccess)
|
||||
<< "\nCUDA kernel launch error: " << cudaGetErrorString(result)
|
||||
<< "\n";
|
||||
|
||||
// Verify result
|
||||
ASSERT_TRUE(testbed.verify_with_cublas());
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
@@ -0,0 +1,321 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
#include "cutlass_unit_test.h"
|
||||
#include "cutlass/gemm/gemm.h"
|
||||
#include "cutlass/gemm/fp16_sgemm_traits.h"
|
||||
#include "tools/test/unit/gemm/gemm_testbed.h"
|
||||
#include "tools/test/unit/gemm/run_gemm.h"
|
||||
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Fp16_sgemm_alphaFp16_fp16_128x128x16, fp16_sgemm_fp16_128x128x16_nn) {
|
||||
/*A, B, C, D are half typed, accumulator is always float for sgemm
|
||||
alpha and beta are both fp16*/
|
||||
typedef cutlass::gemm::Fp16SgemmSgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::Shape<16, 128, 128>,
|
||||
half, /*A type*/
|
||||
half, /*B type*/
|
||||
half, /*C type*/
|
||||
half, /*D type*/
|
||||
half /*alpha, beta type*/
|
||||
>
|
||||
SgemmTraits;
|
||||
run_gemm<SgemmTraits>(128, 128, 16);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Fp16_sgemm_alphaFp16_fp16_128x128x16, fp16_sgemm_fp16_128x128x16_nt) {
|
||||
/*A, B, C, D are half typed, accumulator is always float for sgemm
|
||||
alpha and beta are both fp16*/
|
||||
typedef cutlass::gemm::Fp16SgemmSgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<16, 128, 128>,
|
||||
half, /*A type*/
|
||||
half, /*B type*/
|
||||
half, /*C type*/
|
||||
half, /*D type*/
|
||||
half /*alpha, beta type*/
|
||||
>
|
||||
SgemmTraits;
|
||||
run_gemm<SgemmTraits>(128, 128, 16);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Fp16_sgemm_alphaFp16_fp16_128x128x16, fp16_sgemm_fp16_128x128x16_tn) {
|
||||
/*A, B, C, D are half typed, accumulator is always float for sgemm
|
||||
alpha and beta are both fp16*/
|
||||
typedef cutlass::gemm::Fp16SgemmSgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::Shape<16, 128, 128>,
|
||||
half, /*A type*/
|
||||
half, /*B type*/
|
||||
half, /*C type*/
|
||||
half, /*D type*/
|
||||
half /*alpha, beta type*/
|
||||
>
|
||||
SgemmTraits;
|
||||
run_gemm<SgemmTraits>(128, 128, 16);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Fp16_sgemm_alphaFp16_fp16_128x128x16, fp16_sgemm_fp16_128x128x16_tt) {
|
||||
/*A, B, C, D are half typed, accumulator is always float for sgemm
|
||||
alpha and beta are both fp16*/
|
||||
typedef cutlass::gemm::Fp16SgemmSgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<16, 128, 128>,
|
||||
half, /*A type*/
|
||||
half, /*B type*/
|
||||
half, /*C type*/
|
||||
half, /*D type*/
|
||||
half /*alpha, beta type*/
|
||||
>
|
||||
SgemmTraits;
|
||||
run_gemm<SgemmTraits>(128, 128, 16);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Fp16_sgemm_alphaFp16_fp16_128x128x16, fp16_sgemm_fp16_128x112x17_nn) {
|
||||
/*A, B, C, D are half typed, accumulator is always float for sgemm
|
||||
alpha and beta are both fp16*/
|
||||
typedef cutlass::gemm::Fp16SgemmSgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::Shape<16, 128, 128>,
|
||||
half, /*A type*/
|
||||
half, /*B type*/
|
||||
half, /*C type*/
|
||||
half, /*D type*/
|
||||
half /*alpha, beta type*/
|
||||
>
|
||||
SgemmTraits;
|
||||
run_gemm<SgemmTraits>(128, 112, 17);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Fp16_sgemm_alphaFp16_fp16_128x128x16, fp16_sgemm_fp16_128x112x17_nt) {
|
||||
/*A, B, C, D are half typed, accumulator is always float for sgemm
|
||||
alpha and beta are both fp16*/
|
||||
typedef cutlass::gemm::Fp16SgemmSgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<16, 128, 128>,
|
||||
half, /*A type*/
|
||||
half, /*B type*/
|
||||
half, /*C type*/
|
||||
half, /*D type*/
|
||||
half /*alpha, beta type*/
|
||||
>
|
||||
SgemmTraits;
|
||||
run_gemm<SgemmTraits>(128, 112, 17);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Fp16_sgemm_alphaFp16_fp16_128x128x16, fp16_sgemm_fp16_128x112x17_tn) {
|
||||
/*A, B, C, D are half typed, accumulator is always float for sgemm
|
||||
alpha and beta are both fp16*/
|
||||
typedef cutlass::gemm::Fp16SgemmSgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::Shape<16, 128, 128>,
|
||||
half, /*A type*/
|
||||
half, /*B type*/
|
||||
half, /*C type*/
|
||||
half, /*D type*/
|
||||
half /*alpha, beta type*/
|
||||
>
|
||||
SgemmTraits;
|
||||
run_gemm<SgemmTraits>(128, 112, 17);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Fp16_sgemm_alphaFp16_fp16_128x128x16, fp16_sgemm_fp16_128x112x17_tt) {
|
||||
/*A, B, C, D are half typed, accumulator is always float for sgemm
|
||||
alpha and beta are both fp16*/
|
||||
typedef cutlass::gemm::Fp16SgemmSgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<16, 128, 128>,
|
||||
half, /*A type*/
|
||||
half, /*B type*/
|
||||
half, /*C type*/
|
||||
half, /*D type*/
|
||||
half /*alpha, beta type*/
|
||||
>
|
||||
SgemmTraits;
|
||||
run_gemm<SgemmTraits>(128, 112, 17);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
//alpha and beta are both fp32
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Fp16_sgemm_alphaFp32_fp16_128x128x16, fp16_sgemm_fp16_128x128x16_nn) {
|
||||
/*A, B, C, D are half typed, accumulator is always float for sgemm
|
||||
alpha and beta are both fp32*/
|
||||
typedef cutlass::gemm::Fp16SgemmSgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::Shape<16, 128, 128>,
|
||||
half, /*A type*/
|
||||
half, /*B type*/
|
||||
half, /*C type*/
|
||||
half, /*D type*/
|
||||
float /*alpha, beta type*/
|
||||
>
|
||||
SgemmTraits;
|
||||
run_gemm<SgemmTraits>(128, 128, 16);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Fp16_sgemm_alphaFp32_fp16_128x128x16, fp16_sgemm_fp16_128x128x16_nt) {
|
||||
/*A, B, C, D are half typed, accumulator is always float for sgemm
|
||||
alpha and beta are both fp32*/
|
||||
typedef cutlass::gemm::Fp16SgemmSgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<16, 128, 128>,
|
||||
half, /*A type*/
|
||||
half, /*B type*/
|
||||
half, /*C type*/
|
||||
half, /*D type*/
|
||||
float /*alpha, beta type*/
|
||||
>
|
||||
SgemmTraits;
|
||||
run_gemm<SgemmTraits>(128, 128, 16);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Fp16_sgemm_alphaFp32_fp16_128x128x16, fp16_sgemm_fp16_128x128x16_tn) {
|
||||
/*A, B, C, D are half typed, accumulator is always float for sgemm
|
||||
alpha and beta are both fp32*/
|
||||
typedef cutlass::gemm::Fp16SgemmSgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::Shape<16, 128, 128>,
|
||||
half, /*A type*/
|
||||
half, /*B type*/
|
||||
half, /*C type*/
|
||||
half, /*D type*/
|
||||
float /*alpha, beta type*/
|
||||
>
|
||||
SgemmTraits;
|
||||
run_gemm<SgemmTraits>(128, 128, 16);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Fp16_sgemm_alphaFp32_fp16_128x128x16, fp16_sgemm_fp16_128x128x16_tt) {
|
||||
/*A, B, C, D are half typed, accumulator is always float for sgemm
|
||||
alpha and beta are both fp32*/
|
||||
typedef cutlass::gemm::Fp16SgemmSgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<16, 128, 128>,
|
||||
half, /*A type*/
|
||||
half, /*B type*/
|
||||
half, /*C type*/
|
||||
half, /*D type*/
|
||||
float /*alpha, beta type*/
|
||||
>
|
||||
SgemmTraits;
|
||||
run_gemm<SgemmTraits>(128, 128, 16);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Fp16_sgemm_alphaFp32_fp16_128x128x16, fp16_sgemm_fp16_128x112x17_nn) {
|
||||
/*A, B, C, D are half typed, accumulator is always float for sgemm
|
||||
alpha and beta are both fp32*/
|
||||
typedef cutlass::gemm::Fp16SgemmSgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::Shape<16, 128, 128>,
|
||||
half, /*A type*/
|
||||
half, /*B type*/
|
||||
half, /*C type*/
|
||||
half, /*D type*/
|
||||
float /*alpha, beta type*/
|
||||
>
|
||||
SgemmTraits;
|
||||
run_gemm<SgemmTraits>(128, 112, 17);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Fp16_sgemm_alphaFp32_fp16_128x128x16, fp16_sgemm_fp16_128x112x17_nt) {
|
||||
/*A, B, C, D are half typed, accumulator is always float for sgemm
|
||||
alpha and beta are both fp32*/
|
||||
typedef cutlass::gemm::Fp16SgemmSgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<16, 128, 128>,
|
||||
half, /*A type*/
|
||||
half, /*B type*/
|
||||
half, /*C type*/
|
||||
half, /*D type*/
|
||||
float /*alpha, beta type*/
|
||||
>
|
||||
SgemmTraits;
|
||||
run_gemm<SgemmTraits>(128, 112, 17);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Fp16_sgemm_alphaFp32_fp16_128x128x16, fp16_sgemm_fp16_128x112x17_tn) {
|
||||
/*A, B, C, D are half typed, accumulator is always float for sgemm
|
||||
alpha and beta are both fp32*/
|
||||
typedef cutlass::gemm::Fp16SgemmSgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::Shape<16, 128, 128>,
|
||||
half, /*A type*/
|
||||
half, /*B type*/
|
||||
half, /*C type*/
|
||||
half, /*D type*/
|
||||
float /*alpha, beta type*/
|
||||
>
|
||||
SgemmTraits;
|
||||
run_gemm<SgemmTraits>(128, 112, 17);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Fp16_sgemm_alphaFp32_fp16_128x128x16, fp16_sgemm_fp16_128x112x17_tt) {
|
||||
/*A, B, C, D are half typed, accumulator is always float for sgemm
|
||||
alpha and beta are both fp32*/
|
||||
typedef cutlass::gemm::Fp16SgemmSgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<16, 128, 128>,
|
||||
half, /*A type*/
|
||||
half, /*B type*/
|
||||
half, /*C type*/
|
||||
half, /*D type*/
|
||||
float /*alpha, beta type*/
|
||||
>
|
||||
SgemmTraits;
|
||||
run_gemm<SgemmTraits>(128, 112, 17);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,174 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
#include "cutlass_unit_test.h"
|
||||
#include "cutlass/gemm/gemm.h"
|
||||
#include "cutlass/gemm/fp16_sgemm_traits.h"
|
||||
#include "tools/test/unit/gemm/gemm_testbed.h"
|
||||
#include "tools/test/unit/gemm/run_gemm.h"
|
||||
|
||||
|
||||
|
||||
|
||||
TEST(Fp16_sgemm_alphaFp32_fp32_128x128x16, fp16_sgemm_fp32_128x128x16_nn) {
|
||||
/*A, B are half typed, accumulator is always float for sgemm, C, D are float typed
|
||||
alpha and beta are both fp32*/
|
||||
typedef cutlass::gemm::Fp16SgemmSgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::Shape<16, 128, 128>,
|
||||
half, /*A type*/
|
||||
half, /*B type*/
|
||||
float, /*C type*/
|
||||
float, /*D type*/
|
||||
float /*alpha, beta type*/
|
||||
>
|
||||
SgemmTraits;
|
||||
run_gemm<SgemmTraits>(128, 128, 16);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Fp16_sgemm_alphaFp32_fp32_128x128x16, fp16_sgemm_fp32_128x128x16_nt) {
|
||||
/*A, B are half typed, accumulator is always float for sgemm, C, D are float typed
|
||||
alpha and beta are both fp32*/
|
||||
typedef cutlass::gemm::Fp16SgemmSgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<16, 128, 128>,
|
||||
half, /*A type*/
|
||||
half, /*B type*/
|
||||
float, /*C type*/
|
||||
float, /*D type*/
|
||||
float /*alpha, beta type*/
|
||||
>
|
||||
SgemmTraits;
|
||||
run_gemm<SgemmTraits>(128, 128, 16);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Fp16_sgemm_alphaFp32_fp32_128x128x16, fp16_sgemm_fp32_128x128x16_tn) {
|
||||
/*A, B are half typed, accumulator is always float for sgemm, C, D are float typed
|
||||
alpha and beta are both fp32*/
|
||||
typedef cutlass::gemm::Fp16SgemmSgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::Shape<16, 128, 128>,
|
||||
half, /*A type*/
|
||||
half, /*B type*/
|
||||
float, /*C type*/
|
||||
float, /*D type*/
|
||||
float /*alpha, beta type*/
|
||||
>
|
||||
SgemmTraits;
|
||||
run_gemm<SgemmTraits>(128, 128, 16);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Fp16_sgemm_alphaFp32_fp32_128x128x16, fp16_sgemm_fp32_128x128x16_tt) {
|
||||
/*A, B are half typed, accumulator is always float for sgemm, C, D are float typed
|
||||
alpha and beta are both fp32*/
|
||||
typedef cutlass::gemm::Fp16SgemmSgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<16, 128, 128>,
|
||||
half, /*A type*/
|
||||
half, /*B type*/
|
||||
float, /*C type*/
|
||||
float, /*D type*/
|
||||
float /*alpha, beta type*/
|
||||
>
|
||||
SgemmTraits;
|
||||
run_gemm<SgemmTraits>(128, 128, 16);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Fp16_sgemm_alphaFp32_fp32_128x128x16, fp16_sgemm_fp32_128x112x17_nn) {
|
||||
/*A, B are half typed, accumulator is always float for sgemm, C, D are float typed
|
||||
alpha and beta are both fp32*/
|
||||
typedef cutlass::gemm::Fp16SgemmSgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<16, 128, 128>,
|
||||
half, /*A type*/
|
||||
half, /*B type*/
|
||||
float, /*C type*/
|
||||
float, /*D type*/
|
||||
float /*alpha, beta type*/
|
||||
>
|
||||
SgemmTraits;
|
||||
run_gemm<SgemmTraits>(128, 112, 17);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Fp16_sgemm_alphaFp32_fp32_128x128x16, fp16_sgemm_fp32_128x112x17_nt) {
|
||||
/*A, B are half typed, accumulator is always float for sgemm, C, D are float typed
|
||||
alpha and beta are both fp32*/
|
||||
typedef cutlass::gemm::Fp16SgemmSgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::Shape<16, 128, 128>,
|
||||
half, /*A type*/
|
||||
half, /*B type*/
|
||||
float, /*C type*/
|
||||
float, /*D type*/
|
||||
float /*alpha, beta type*/
|
||||
>
|
||||
SgemmTraits;
|
||||
run_gemm<SgemmTraits>(128, 112, 17);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Fp16_sgemm_alphaFp32_fp32_128x128x16, fp16_sgemm_fp32_128x112x17_tn) {
|
||||
/*A, B are half typed, accumulator is always float for sgemm, C, D are float typed
|
||||
alpha and beta are both fp32*/
|
||||
typedef cutlass::gemm::Fp16SgemmSgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<16, 128, 128>,
|
||||
half, /*A type*/
|
||||
half, /*B type*/
|
||||
float, /*C type*/
|
||||
float, /*D type*/
|
||||
float /*alpha, beta type*/
|
||||
>
|
||||
SgemmTraits;
|
||||
run_gemm<SgemmTraits>(128, 112, 17);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Fp16_sgemm_alphaFp32_fp32_128x128x16, fp16_sgemm_fp32_128x112x17_tt) {
|
||||
/*A, B are half typed, accumulator is always float for sgemm, C, D are float typed
|
||||
alpha and beta are both fp32*/
|
||||
typedef cutlass::gemm::Fp16SgemmSgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::Shape<16, 128, 128>,
|
||||
half, /*A type*/
|
||||
half, /*B type*/
|
||||
float, /*C type*/
|
||||
float, /*D type*/
|
||||
float /*alpha, beta type*/
|
||||
>
|
||||
SgemmTraits;
|
||||
run_gemm<SgemmTraits>(128, 112, 17);
|
||||
}
|
||||
@@ -22,13 +22,13 @@
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
#include <cutlass_unit_test.h>
|
||||
#include <cutlass/gemm/gemm.h>
|
||||
#include <cutlass/gemm/dgemm_traits.h>
|
||||
#include <cutlass/gemm/igemm_traits.h>
|
||||
#include <cutlass/gemm/sgemm_traits.h>
|
||||
#include <tools/test/unit/gemm/gemm_testbed.h>
|
||||
#include <tools/test/unit/gemm/gemm_nvrtc.h>
|
||||
#include "cutlass_unit_test.h"
|
||||
#include "cutlass/gemm/gemm.h"
|
||||
#include "cutlass/gemm/dgemm_traits.h"
|
||||
#include "cutlass/gemm/igemm_traits.h"
|
||||
#include "cutlass/gemm/sgemm_traits.h"
|
||||
#include "tools/test/unit/gemm/gemm_testbed.h"
|
||||
#include "tools/test/unit/gemm/gemm_nvrtc.h"
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
|
||||
@@ -25,10 +25,10 @@
|
||||
|
||||
#define NVRTC_GET_TYPE_NAME 1
|
||||
|
||||
#include <cutlass/cutlass.h>
|
||||
#include "cutlass/cutlass.h"
|
||||
|
||||
#include <nvrtc.h>
|
||||
#include <tools/nvrtc/cutlass/nvrtc/environment.h>
|
||||
#include "tools/nvrtc/cutlass/nvrtc/environment.h"
|
||||
#include <string>
|
||||
|
||||
static inline bool check_nvrtc_error(nvrtcResult error) {
|
||||
@@ -76,13 +76,13 @@ static __host__ void run_gemm_nvrtc(
|
||||
nvrtcResult result_nvrtc;
|
||||
nvrtcProgram program;
|
||||
static char const *src =
|
||||
"#include <cutlass/gemm/gemm.h>\n"
|
||||
"#include <cutlass/gemm/sgemm_traits.h>\n"
|
||||
"#include <cutlass/gemm/dgemm_traits.h>\n"
|
||||
"#include <cutlass/gemm/igemm_traits.h>\n"
|
||||
"#include "cutlass/gemm/gemm.h"\n"
|
||||
"#include "cutlass/gemm/sgemm_traits.h"\n"
|
||||
"#include "cutlass/gemm/dgemm_traits.h"\n"
|
||||
"#include "cutlass/gemm/igemm_traits.h"\n"
|
||||
#if defined(CUTLASS_NVRTC_HAS_FP16)
|
||||
"#include <cutlass/gemm/hgemm_traits.h>\n"
|
||||
"#include <cutlass/gemm/wmma_gemm_traits.h>\n"
|
||||
"#include "cutlass/gemm/hgemm_traits.h"\n"
|
||||
"#include "cutlass/gemm/wmma_gemm_traits.h"\n"
|
||||
#endif
|
||||
;
|
||||
|
||||
|
||||
@@ -1,621 +0,0 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
#include <cutlass_unit_tests.h>
|
||||
#include <tools/util/host_tensor.h>
|
||||
#include <tools/util/tensor_view_io.h>
|
||||
#include <cutlass/gemm/gemm.h>
|
||||
#include <cutlass/shape.h>
|
||||
#include <cutlass/gemm/sgemm_traits.h>
|
||||
#include <cutlass/gemm/dgemm_traits.h>
|
||||
#include <cutlass/gemm/hgemm_traits.h>
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
namespace test {
|
||||
|
||||
// M/N/K struct.
|
||||
struct GemmDesc {
|
||||
int m, n, k;
|
||||
inline __host__ __device__ GemmDesc(int m_, int n_, int k_) : m(m_), n(n_), k(k_) {}
|
||||
};
|
||||
|
||||
/// Simple test to load from global memory and store to shared memory
|
||||
|
||||
// Loading from global memory and storing to shared memory for A
|
||||
template <typename Traits>
|
||||
__global__ void gemm_load_global_store_shared_a(
|
||||
typename Traits::GlobalLoadStreamA::Scalar *output,
|
||||
typename Traits::GlobalLoadStreamA::Scalar const *input,
|
||||
int M,
|
||||
int N,
|
||||
int K,
|
||||
int ldm,
|
||||
int skew) {
|
||||
|
||||
//Create shared memory.
|
||||
__shared__ typename Traits::SharedStorage shared_storage;
|
||||
|
||||
// Create those iterators.
|
||||
typedef typename Traits::GlobalLoadStreamA GlobalLoadStreamA;
|
||||
|
||||
typename GlobalLoadStreamA::Params global_load_params;
|
||||
GemmDesc desc(M, N, K);
|
||||
global_load_params.initialize(desc, input, ldm);
|
||||
|
||||
GlobalLoadStreamA stream_a(global_load_params, shared_storage.main_loop.stream_a.global, M, N, K, cutlass::make_Coord(0, 0, 0));
|
||||
stream_a.copy();
|
||||
stream_a.commit();
|
||||
|
||||
// store barrier
|
||||
__syncthreads();
|
||||
|
||||
// one thread writes everything out
|
||||
if (threadIdx.x == 0) {
|
||||
for (int i = 0; i < (M+skew)*K; ++i) {
|
||||
output[i] = shared_storage.main_loop.stream_a.shared.scalars[i];
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// Loading from global memory and storing to shared memory for B
|
||||
template <typename Traits>
|
||||
__global__ void gemm_load_global_store_shared_b(
|
||||
typename Traits::GlobalLoadStreamB::Scalar *output,
|
||||
typename Traits::GlobalLoadStreamB::Scalar const *input,
|
||||
int M,
|
||||
int N,
|
||||
int K,
|
||||
int ldm,
|
||||
int skew) {
|
||||
|
||||
//Create shared memory.
|
||||
__shared__ typename Traits::SharedStorage shared_storage;
|
||||
|
||||
// Create those iterators.
|
||||
typedef typename Traits::GlobalLoadStreamB GlobalLoadStreamB;
|
||||
typename GlobalLoadStreamB::Params global_load_params;
|
||||
GemmDesc desc(M, N, K);
|
||||
global_load_params.initialize(desc, input, ldm);
|
||||
|
||||
GlobalLoadStreamB stream_b(global_load_params, shared_storage.main_loop.stream_b.global, M, N, K, cutlass::make_Coord(0, 0, 0));
|
||||
stream_b.copy();
|
||||
stream_b.commit();
|
||||
|
||||
// store barrier
|
||||
__syncthreads();
|
||||
|
||||
// one thread writes everything out
|
||||
if (threadIdx.x == 0) {
|
||||
for (int i = 0; i < (N+skew)*K; ++i) {
|
||||
output[i] = shared_storage.main_loop.stream_b.shared.scalars[i];
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
TEST(GemmSharedMemLayout, A_float_contiguous) {
|
||||
|
||||
static int const M = 64;
|
||||
static int const N = 64;
|
||||
static int const K = 8;
|
||||
|
||||
typedef cutlass::Shape<K, N, M> ThreadBlockTile;
|
||||
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor, cutlass::MatrixLayout::kRowMajor, ThreadBlockTile >
|
||||
SgemmTraits;
|
||||
|
||||
|
||||
cutlass::HostTensor<float> input;
|
||||
cutlass::HostTensor<float> output;
|
||||
int skew = 0;
|
||||
|
||||
input.resize_matrix(ThreadBlockTile::kW, ThreadBlockTile::kD,
|
||||
cutlass::MatrixLayout::kColumnMajor);
|
||||
output.resize_matrix(ThreadBlockTile::kW, ThreadBlockTile::kD,
|
||||
cutlass::MatrixLayout::kColumnMajor);
|
||||
|
||||
|
||||
input.fill_linear(cutlass::make_Coord(1, 1, ThreadBlockTile::kW, 1));
|
||||
|
||||
output.fill(0);
|
||||
|
||||
test::gemm_load_global_store_shared_a< SgemmTraits ><<<
|
||||
dim3(1,1,1),
|
||||
dim3(SgemmTraits::kThreads, 1)
|
||||
>>>(
|
||||
output.device_data(),
|
||||
input.device_data(),
|
||||
M,
|
||||
N,
|
||||
K,
|
||||
M,
|
||||
skew
|
||||
);
|
||||
|
||||
cudaError_t result = cudaDeviceSynchronize();
|
||||
ASSERT_EQ(result, cudaSuccess) << "\nCUDA kernel launch error: " << cudaGetErrorString(result)
|
||||
<< "\n";
|
||||
|
||||
|
||||
output.sync_host();
|
||||
|
||||
EXPECT_TRUE(input.bit_equals(output));
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
TEST(GemmSharedMemLayout, A_float_crosswise) {
|
||||
|
||||
static int const M = 64;
|
||||
static int const N = 64;
|
||||
static int const K = 8;
|
||||
|
||||
typedef cutlass::Shape<K, N, M> ThreadBlockTile;
|
||||
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kRowMajor, cutlass::MatrixLayout::kRowMajor, ThreadBlockTile >
|
||||
SgemmTraits;
|
||||
|
||||
|
||||
cutlass::HostTensor<float> input;
|
||||
cutlass::HostTensor<float> output;
|
||||
int skew = 4;
|
||||
|
||||
input.resize_matrix(ThreadBlockTile::kW, ThreadBlockTile::kD,
|
||||
cutlass::MatrixLayout::kRowMajor);
|
||||
output.resize_matrix(ThreadBlockTile::kW + skew, ThreadBlockTile::kD,
|
||||
cutlass::MatrixLayout::kColumnMajor);
|
||||
|
||||
|
||||
input.fill_linear(cutlass::make_Coord(1, ThreadBlockTile::kD, 1, 1));
|
||||
|
||||
output.fill(0);
|
||||
|
||||
test::gemm_load_global_store_shared_a< SgemmTraits ><<<
|
||||
dim3(1,1,1),
|
||||
dim3(SgemmTraits::kThreads, 1)
|
||||
>>>(
|
||||
output.device_data(),
|
||||
input.device_data(),
|
||||
M,
|
||||
N,
|
||||
K,
|
||||
K,
|
||||
skew
|
||||
);
|
||||
|
||||
cudaError_t result = cudaDeviceSynchronize();
|
||||
ASSERT_EQ(result, cudaSuccess) << "\nCUDA kernel launch error: " << cudaGetErrorString(result)
|
||||
<< "\n";
|
||||
|
||||
output.sync_host();
|
||||
|
||||
EXPECT_TRUE(input.bit_equals(output));
|
||||
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
TEST(GemmSharedMemLayout, B_float_contiguous) {
|
||||
|
||||
static int const M = 64;
|
||||
static int const N = 64;
|
||||
static int const K = 8;
|
||||
|
||||
typedef cutlass::Shape<K, N, M> ThreadBlockTile;
|
||||
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor, cutlass::MatrixLayout::kRowMajor, ThreadBlockTile >
|
||||
SgemmTraits;
|
||||
|
||||
|
||||
cutlass::HostTensor<float> input;
|
||||
cutlass::HostTensor<float> output;
|
||||
int skew = 0;
|
||||
|
||||
input.resize_matrix(ThreadBlockTile::kD, ThreadBlockTile::kH,
|
||||
cutlass::MatrixLayout::kRowMajor);
|
||||
output.resize_matrix(ThreadBlockTile::kD, ThreadBlockTile::kH,
|
||||
cutlass::MatrixLayout::kRowMajor);
|
||||
|
||||
|
||||
input.fill_linear(cutlass::make_Coord(1, ThreadBlockTile::kH, 1, 1));
|
||||
|
||||
output.fill(0);
|
||||
|
||||
test::gemm_load_global_store_shared_b< SgemmTraits ><<<
|
||||
dim3(1,1,1),
|
||||
dim3(SgemmTraits::kThreads, 1)
|
||||
>>>(
|
||||
output.device_data(),
|
||||
input.device_data(),
|
||||
M,
|
||||
N,
|
||||
K,
|
||||
N,
|
||||
skew
|
||||
);
|
||||
|
||||
cudaError_t result = cudaDeviceSynchronize();
|
||||
ASSERT_EQ(result, cudaSuccess) << "\nCUDA kernel launch error: " << cudaGetErrorString(result)
|
||||
<< "\n";
|
||||
|
||||
|
||||
output.sync_host();
|
||||
|
||||
EXPECT_TRUE(input.bit_equals(output));
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
TEST(GemmSharedMemLayout, B_float_crosswise) {
|
||||
|
||||
static int const M = 64;
|
||||
static int const N = 64;
|
||||
static int const K = 8;
|
||||
|
||||
typedef cutlass::Shape<K, N, M> ThreadBlockTile;
|
||||
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor, ThreadBlockTile >
|
||||
SgemmTraits;
|
||||
|
||||
|
||||
cutlass::HostTensor<float> input;
|
||||
cutlass::HostTensor<float> output;
|
||||
int skew = 4;
|
||||
|
||||
input.resize_matrix(ThreadBlockTile::kD, ThreadBlockTile::kH,
|
||||
cutlass::MatrixLayout::kColumnMajor);
|
||||
output.resize_matrix(ThreadBlockTile::kD + skew, ThreadBlockTile::kH,
|
||||
cutlass::MatrixLayout::kRowMajor);
|
||||
|
||||
|
||||
input.fill_linear(cutlass::make_Coord(1, 1, ThreadBlockTile::kD, 1));
|
||||
|
||||
output.fill(0);
|
||||
|
||||
test::gemm_load_global_store_shared_b< SgemmTraits ><<<
|
||||
dim3(1,1,1),
|
||||
dim3(SgemmTraits::kThreads, 1)
|
||||
>>>(
|
||||
output.device_data(),
|
||||
input.device_data(),
|
||||
M,
|
||||
N,
|
||||
K,
|
||||
K,
|
||||
skew
|
||||
);
|
||||
|
||||
cudaError_t result = cudaDeviceSynchronize();
|
||||
ASSERT_EQ(result, cudaSuccess) << "\nCUDA kernel launch error: " << cudaGetErrorString(result)
|
||||
<< "\n";
|
||||
|
||||
output.sync_host();
|
||||
|
||||
EXPECT_TRUE(input.bit_equals(output));
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
TEST(GemmSharedMemLayout, A_double_contiguous) {
|
||||
|
||||
static int const M = 64;
|
||||
static int const N = 64;
|
||||
static int const K = 8;
|
||||
|
||||
typedef cutlass::Shape<K, N, M> ThreadBlockTile;
|
||||
|
||||
typedef cutlass::gemm::DgemmTraits<cutlass::MatrixLayout::kColumnMajor, cutlass::MatrixLayout::kRowMajor, ThreadBlockTile >
|
||||
DgemmTraits;
|
||||
|
||||
|
||||
cutlass::HostTensor<double> input;
|
||||
cutlass::HostTensor<double> output;
|
||||
int skew = 0;
|
||||
|
||||
input.resize_matrix(ThreadBlockTile::kW, ThreadBlockTile::kD,
|
||||
cutlass::MatrixLayout::kColumnMajor);
|
||||
output.resize_matrix(ThreadBlockTile::kW, ThreadBlockTile::kD,
|
||||
cutlass::MatrixLayout::kColumnMajor);
|
||||
|
||||
|
||||
input.fill_linear(cutlass::make_Coord(1, 1, ThreadBlockTile::kW, 1));
|
||||
|
||||
output.fill(0);
|
||||
|
||||
test::gemm_load_global_store_shared_a< DgemmTraits ><<<
|
||||
dim3(1,1,1),
|
||||
dim3(DgemmTraits::kThreads, 1)
|
||||
>>>(
|
||||
output.device_data(),
|
||||
input.device_data(),
|
||||
M,
|
||||
N,
|
||||
K,
|
||||
M,
|
||||
skew
|
||||
);
|
||||
|
||||
cudaError_t result = cudaDeviceSynchronize();
|
||||
ASSERT_EQ(result, cudaSuccess) << "\nCUDA kernel launch error: " << cudaGetErrorString(result)
|
||||
<< "\n";
|
||||
|
||||
|
||||
output.sync_host();
|
||||
|
||||
EXPECT_TRUE(input.bit_equals(output));
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
TEST(GemmSharedMemLayout, A_double_crosswise) {
|
||||
|
||||
static int const M = 64;
|
||||
static int const N = 64;
|
||||
static int const K = 8;
|
||||
|
||||
typedef cutlass::Shape<K, N, M> ThreadBlockTile;
|
||||
|
||||
typedef cutlass::gemm::DgemmTraits<cutlass::MatrixLayout::kRowMajor, cutlass::MatrixLayout::kRowMajor, ThreadBlockTile >
|
||||
DgemmTraits;
|
||||
|
||||
|
||||
cutlass::HostTensor<double> input;
|
||||
cutlass::HostTensor<double> output;
|
||||
int skew = 2;
|
||||
|
||||
input.resize_matrix(ThreadBlockTile::kW, ThreadBlockTile::kD,
|
||||
cutlass::MatrixLayout::kRowMajor);
|
||||
output.resize_matrix(ThreadBlockTile::kW + skew, ThreadBlockTile::kD,
|
||||
cutlass::MatrixLayout::kColumnMajor);
|
||||
|
||||
|
||||
input.fill_linear(cutlass::make_Coord(1, ThreadBlockTile::kD, 1, 1));
|
||||
|
||||
output.fill(0);
|
||||
|
||||
test::gemm_load_global_store_shared_a< DgemmTraits ><<<
|
||||
dim3(1,1,1),
|
||||
dim3(DgemmTraits::kThreads, 1)
|
||||
>>>(
|
||||
output.device_data(),
|
||||
input.device_data(),
|
||||
M,
|
||||
N,
|
||||
K,
|
||||
K,
|
||||
skew
|
||||
);
|
||||
|
||||
cudaError_t result = cudaDeviceSynchronize();
|
||||
ASSERT_EQ(result, cudaSuccess) << "\nCUDA kernel launch error: " << cudaGetErrorString(result)
|
||||
<< "\n";
|
||||
|
||||
output.sync_host();
|
||||
|
||||
EXPECT_TRUE(input.bit_equals(output));
|
||||
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
TEST(GemmSharedMemLayout, B_double_contiguous) {
|
||||
|
||||
static int const M = 64;
|
||||
static int const N = 64;
|
||||
static int const K = 8;
|
||||
|
||||
typedef cutlass::Shape<K, N, M> ThreadBlockTile;
|
||||
|
||||
typedef cutlass::gemm::DgemmTraits<cutlass::MatrixLayout::kColumnMajor, cutlass::MatrixLayout::kRowMajor, ThreadBlockTile >
|
||||
DgemmTraits;
|
||||
|
||||
|
||||
cutlass::HostTensor<double> input;
|
||||
cutlass::HostTensor<double> output;
|
||||
int skew = 0;
|
||||
|
||||
input.resize_matrix(ThreadBlockTile::kD, ThreadBlockTile::kH,
|
||||
cutlass::MatrixLayout::kRowMajor);
|
||||
output.resize_matrix(ThreadBlockTile::kD, ThreadBlockTile::kH,
|
||||
cutlass::MatrixLayout::kRowMajor);
|
||||
|
||||
|
||||
input.fill_linear(cutlass::make_Coord(1, ThreadBlockTile::kH, 1, 1));
|
||||
|
||||
output.fill(0);
|
||||
|
||||
test::gemm_load_global_store_shared_b< DgemmTraits ><<<
|
||||
dim3(1,1,1),
|
||||
dim3(DgemmTraits::kThreads, 1)
|
||||
>>>(
|
||||
output.device_data(),
|
||||
input.device_data(),
|
||||
M,
|
||||
N,
|
||||
K,
|
||||
N,
|
||||
skew
|
||||
);
|
||||
|
||||
cudaError_t result = cudaDeviceSynchronize();
|
||||
ASSERT_EQ(result, cudaSuccess) << "\nCUDA kernel launch error: " << cudaGetErrorString(result)
|
||||
<< "\n";
|
||||
|
||||
|
||||
output.sync_host();
|
||||
|
||||
EXPECT_TRUE(input.bit_equals(output));
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
TEST(GemmSharedMemLayout, B_double_crosswise) {
|
||||
|
||||
static int const M = 64;
|
||||
static int const N = 64;
|
||||
static int const K = 8;
|
||||
|
||||
typedef cutlass::Shape<K, N, M> ThreadBlockTile;
|
||||
|
||||
typedef cutlass::gemm::DgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor, ThreadBlockTile >
|
||||
DgemmTraits;
|
||||
|
||||
|
||||
cutlass::HostTensor<double> input;
|
||||
cutlass::HostTensor<double> output;
|
||||
int skew = 2;
|
||||
|
||||
input.resize_matrix(ThreadBlockTile::kD, ThreadBlockTile::kH,
|
||||
cutlass::MatrixLayout::kColumnMajor);
|
||||
output.resize_matrix(ThreadBlockTile::kD + skew, ThreadBlockTile::kH,
|
||||
cutlass::MatrixLayout::kRowMajor);
|
||||
|
||||
|
||||
input.fill_linear(cutlass::make_Coord(1, 1, ThreadBlockTile::kD, 1));
|
||||
|
||||
output.fill(0);
|
||||
|
||||
test::gemm_load_global_store_shared_b< DgemmTraits ><<<
|
||||
dim3(1,1,1),
|
||||
dim3(DgemmTraits::kThreads, 1)
|
||||
>>>(
|
||||
output.device_data(),
|
||||
input.device_data(),
|
||||
M,
|
||||
N,
|
||||
K,
|
||||
K,
|
||||
skew
|
||||
);
|
||||
|
||||
cudaError_t result = cudaDeviceSynchronize();
|
||||
ASSERT_EQ(result, cudaSuccess) << "\nCUDA kernel launch error: " << cudaGetErrorString(result)
|
||||
<< "\n";
|
||||
|
||||
output.sync_host();
|
||||
|
||||
EXPECT_TRUE(input.bit_equals(output));
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
TEST(GemmSharedMemLayout, A_half_crosswise) {
|
||||
|
||||
static int const M = 128;
|
||||
static int const N = 128;
|
||||
static int const K = 8;
|
||||
|
||||
typedef cutlass::Shape<K, N, M> ThreadBlockTile;
|
||||
|
||||
typedef cutlass::gemm::HgemmTraits<cutlass::MatrixLayout::kRowMajor, cutlass::MatrixLayout::kRowMajor, ThreadBlockTile >
|
||||
HgemmTraits;
|
||||
|
||||
|
||||
cutlass::HostTensor<cutlass::half_t> input;
|
||||
cutlass::HostTensor<cutlass::half_t> output;
|
||||
int skew = 8;
|
||||
|
||||
input.resize_matrix(ThreadBlockTile::kW, ThreadBlockTile::kD,
|
||||
cutlass::MatrixLayout::kRowMajor);
|
||||
output.resize_matrix(ThreadBlockTile::kW + skew, ThreadBlockTile::kD,
|
||||
cutlass::MatrixLayout::kColumnMajor);
|
||||
|
||||
|
||||
input.fill_linear(cutlass::make_Coord(1, ThreadBlockTile::kD, 1, 1));
|
||||
|
||||
output.fill(0);
|
||||
|
||||
test::gemm_load_global_store_shared_a< HgemmTraits ><<<
|
||||
dim3(1,1,1),
|
||||
dim3(HgemmTraits::kThreads, 1)
|
||||
>>>(
|
||||
output.device_data(),
|
||||
input.device_data(),
|
||||
M,
|
||||
N,
|
||||
K,
|
||||
K,
|
||||
skew
|
||||
);
|
||||
|
||||
cudaError_t result = cudaDeviceSynchronize();
|
||||
ASSERT_EQ(result, cudaSuccess) << "\nCUDA kernel launch error: " << cudaGetErrorString(result)
|
||||
<< "\n";
|
||||
|
||||
output.sync_host();
|
||||
|
||||
EXPECT_TRUE(input.bit_equals(output));
|
||||
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
TEST(GemmSharedMemLayout, B_half_crosswise) {
|
||||
|
||||
static int const M = 128;
|
||||
static int const N = 128;
|
||||
static int const K = 8;
|
||||
|
||||
typedef cutlass::Shape<K, N, M> ThreadBlockTile;
|
||||
|
||||
typedef cutlass::gemm::HgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor, ThreadBlockTile >
|
||||
HgemmTraits;
|
||||
|
||||
|
||||
cutlass::HostTensor<cutlass::half_t> input;
|
||||
cutlass::HostTensor<cutlass::half_t> output;
|
||||
int skew = 8;
|
||||
|
||||
input.resize_matrix(ThreadBlockTile::kD, ThreadBlockTile::kH,
|
||||
cutlass::MatrixLayout::kColumnMajor);
|
||||
output.resize_matrix(ThreadBlockTile::kD + skew, ThreadBlockTile::kH,
|
||||
cutlass::MatrixLayout::kRowMajor);
|
||||
|
||||
|
||||
input.fill_linear(cutlass::make_Coord(1, 1, ThreadBlockTile::kD, 1));
|
||||
|
||||
output.fill(0);
|
||||
|
||||
test::gemm_load_global_store_shared_b< HgemmTraits ><<<
|
||||
dim3(1,1,1),
|
||||
dim3(HgemmTraits::kThreads, 1)
|
||||
>>>(
|
||||
output.device_data(),
|
||||
input.device_data(),
|
||||
M,
|
||||
N,
|
||||
K,
|
||||
K,
|
||||
skew
|
||||
);
|
||||
|
||||
cudaError_t result = cudaDeviceSynchronize();
|
||||
ASSERT_EQ(result, cudaSuccess) << "\nCUDA kernel launch error: " << cudaGetErrorString(result)
|
||||
<< "\n";
|
||||
|
||||
output.sync_host();
|
||||
|
||||
EXPECT_TRUE(input.bit_equals(output));
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
}
|
||||
|
||||
+510
-105
@@ -32,30 +32,39 @@
|
||||
#include <iomanip>
|
||||
#include <sstream>
|
||||
#include <string>
|
||||
#include <algorithm>
|
||||
|
||||
#include <cublas_v2.h>
|
||||
|
||||
#include <cutlass/matrix_traits.h>
|
||||
#include <cutlass/util/platform.h>
|
||||
#include "cutlass/matrix_traits.h"
|
||||
#include "cutlass/util/platform.h"
|
||||
#include "cutlass/gemm/gemm_coord.h"
|
||||
|
||||
#include <tools/util/host_tensor.h>
|
||||
#include <tools/util/tensor_view_io.h>
|
||||
#include <tools/util/type_traits.h>
|
||||
#include "tools/util/host_matrix.h"
|
||||
#include "tools/util/host_matrix_view.h"
|
||||
#include "tools/util/tensor_view_io.h"
|
||||
#include "tools/util/type_traits.h"
|
||||
|
||||
#include "tools/util/reference/host/gemm.h"
|
||||
#include "tools/util/reference/host/tensor_elementwise.h"
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
namespace cutlass {
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
template <cutlass::GemmOperand::Kind kOperand_,
|
||||
cutlass::MatrixLayout::Kind kLayout_,
|
||||
typename Scalar_,
|
||||
typename WmmaShape_>
|
||||
struct WmmaMatrix;
|
||||
}
|
||||
|
||||
} // namespace cutlass
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
namespace test {
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
//////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
template <typename T>
|
||||
struct GemmTestbedTraits : public cutlass::TypeTraits<T> {};
|
||||
@@ -66,14 +75,39 @@ template <cutlass::GemmOperand::Kind kOperand_,
|
||||
typename WmmaShape_>
|
||||
struct GemmTestbedTraits<cutlass::WmmaMatrix<kOperand_, kLayout_, Scalar_, WmmaShape_> > {
|
||||
static cudaDataType_t const cublas_type = cutlass::TypeTraits<Scalar_>::cublas_type;
|
||||
typedef Scalar_ host_type;
|
||||
typedef Scalar_ device_type;
|
||||
typedef typename cutlass::TypeTraits<Scalar_>::host_type host_type;
|
||||
typedef typename cutlass::TypeTraits<Scalar_>::device_type device_type;
|
||||
static inline double remove_negative_zero(double x) { return x == -0.0 ? 0.0 : x; }
|
||||
static inline double to_print(double x) { return x; }
|
||||
};
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
inline cublasOperation_t convert(cutlass::MatrixLayout::Kind layout) {
|
||||
switch (layout) {
|
||||
case cutlass::MatrixLayout::kRowMajor:
|
||||
return CUBLAS_OP_T;
|
||||
case cutlass::MatrixLayout::kColumnMajor:
|
||||
return CUBLAS_OP_N;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
return CUBLAS_OP_N;
|
||||
}
|
||||
|
||||
inline cutlass::MatrixLayout::Kind convert(cublasOperation_t transform) {
|
||||
switch (transform) {
|
||||
case CUBLAS_OP_T:
|
||||
return cutlass::MatrixLayout::kRowMajor;
|
||||
case CUBLAS_OP_N:
|
||||
return cutlass::MatrixLayout::kColumnMajor;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
return cutlass::MatrixLayout::kColumnMajor;
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Testbed for evaluating real-valued GEMMs
|
||||
template <typename AType, typename BType, typename CType, typename Accumulator, typename Scalar>
|
||||
struct GemmTestbed {
|
||||
//
|
||||
@@ -81,13 +115,13 @@ struct GemmTestbed {
|
||||
//
|
||||
|
||||
/// Host tensor for operand A
|
||||
typedef cutlass::HostTensor<AType> HostTensorA;
|
||||
typedef cutlass::HostMatrix<AType> HostMatrixA;
|
||||
|
||||
/// Host tensor for operand B
|
||||
typedef cutlass::HostTensor<BType> HostTensorB;
|
||||
typedef cutlass::HostMatrix<BType> HostMatrixB;
|
||||
|
||||
/// Host tensor for operand C
|
||||
typedef cutlass::HostTensor<CType> HostTensorC;
|
||||
typedef cutlass::HostMatrix<CType> HostMatrixC;
|
||||
|
||||
/// Functor to print errors
|
||||
struct PrintErrors {
|
||||
@@ -98,18 +132,18 @@ struct GemmTestbed {
|
||||
std::ostream& out;
|
||||
|
||||
/// Reference tensor view
|
||||
cutlass::HostTensorView<CType> const& reference;
|
||||
HostMatrixC const& reference;
|
||||
|
||||
/// Computed tensor view
|
||||
cutlass::HostTensorView<CType> const& experimental;
|
||||
HostMatrixC const& experimental;
|
||||
|
||||
/// Errors greater than or this amount result in printing
|
||||
integer_t ulps_threshold;
|
||||
|
||||
///
|
||||
PrintErrors(std::ostream& _out,
|
||||
cutlass::HostTensorView<CType> const& _reference,
|
||||
cutlass::HostTensorView<CType> const& _experimental,
|
||||
HostMatrixC const& _reference,
|
||||
HostMatrixC const& _experimental,
|
||||
integer_t _ulps_threshold = 1)
|
||||
: out(_out),
|
||||
reference(_reference),
|
||||
@@ -117,7 +151,7 @@ struct GemmTestbed {
|
||||
ulps_threshold(_ulps_threshold) {}
|
||||
|
||||
/// Compares one element
|
||||
void operator()(CType const& element, typename HostTensorC::Coord_t coord) {
|
||||
void operator()(CType const& element, typename HostMatrixC::TensorCoord coord) {
|
||||
CType exp = experimental.at(coord);
|
||||
CType ref = reference.at(coord);
|
||||
|
||||
@@ -165,6 +199,20 @@ struct GemmTestbed {
|
||||
bool only_ones;
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
struct RandomBitGenerator {
|
||||
RandomBitGenerator(int seed = -1) { srand(seed); }
|
||||
|
||||
T operator()() {
|
||||
uint32_t val = 0;
|
||||
for (int i = 0; i < 32; i++) {
|
||||
val |= rand() % 2;
|
||||
val <<= 1;
|
||||
}
|
||||
return T(val);
|
||||
}
|
||||
};
|
||||
|
||||
//
|
||||
// Data members
|
||||
//
|
||||
@@ -178,29 +226,32 @@ struct GemmTestbed {
|
||||
/// cuBLAS GEMM algorithm selector
|
||||
cublasGemmAlgo_t algorithm;
|
||||
|
||||
/// Problem size as a GemmCoord
|
||||
cutlass::gemm::GemmCoord problem_size;
|
||||
|
||||
/// A matrix operand
|
||||
HostTensorA A;
|
||||
HostMatrixA A;
|
||||
|
||||
/// Layout of A matrix
|
||||
cublasOperation_t layout_A;
|
||||
|
||||
/// B matrix operand
|
||||
HostTensorB B;
|
||||
HostMatrixB B;
|
||||
|
||||
/// Layout of B matrix
|
||||
cublasOperation_t layout_B;
|
||||
|
||||
/// C matrix operand
|
||||
HostTensorC C_initial;
|
||||
HostMatrixC C_initial;
|
||||
|
||||
/// Reference result computed on the host
|
||||
cutlass::HostTensor<CType, false> ref_host;
|
||||
HostMatrixC ref_host;
|
||||
|
||||
/// Reference result computed with cublas
|
||||
HostTensorC ref_cublas;
|
||||
HostMatrixC ref_cublas;
|
||||
|
||||
/// Computed result
|
||||
HostTensorC computed;
|
||||
HostMatrixC computed;
|
||||
|
||||
/// Linear scalaring factor
|
||||
Scalar alpha;
|
||||
@@ -208,36 +259,105 @@ struct GemmTestbed {
|
||||
/// Linear scaling factor
|
||||
Scalar beta;
|
||||
|
||||
/// batch count
|
||||
int batch_count;
|
||||
|
||||
/// distance between A[i] and A[i+1] for strided batched gemm
|
||||
long long int batch_stride_A;
|
||||
|
||||
/// distance between B[i] and B[i+1] for strided batched gemm
|
||||
long long int batch_stride_B;
|
||||
|
||||
/// distance between C[i] and C[i+1] for strided batched gemm
|
||||
long long int batch_stride_C;
|
||||
|
||||
//
|
||||
// Static helpers
|
||||
//
|
||||
|
||||
/// Helper to resize a matrix with a given size and layout
|
||||
template <typename T, bool DeviceBacked>
|
||||
static void resize(cutlass::HostTensor<T, DeviceBacked>& tensor,
|
||||
template <typename T>
|
||||
static void resize(cutlass::HostMatrix<T>& tensor,
|
||||
int rows,
|
||||
int columns,
|
||||
cublasOperation_t layout,
|
||||
int ldm = 0) {
|
||||
if (!ldm) {
|
||||
ldm = (layout == CUBLAS_OP_N ? rows : columns);
|
||||
}
|
||||
|
||||
typedef cutlass::Coord<cutlass::HostTensor<T>::Rank> Coord_t;
|
||||
|
||||
size_t matrix_stride = layout == CUBLAS_OP_N ? columns * ldm : rows * ldm;
|
||||
// TODO: Remove that (int) cast.
|
||||
Coord_t stride = cutlass::make_Coord(
|
||||
(int)matrix_stride, layout == CUBLAS_OP_N ? 1 : ldm, layout == CUBLAS_OP_N ? ldm : 1, 1);
|
||||
Coord_t size = cutlass::make_Coord(1, rows, columns, 1);
|
||||
tensor.reset(stride, size);
|
||||
tensor.resize(cutlass::make_Coord(rows, columns), convert(layout), ldm);
|
||||
}
|
||||
|
||||
//
|
||||
// Methods
|
||||
//
|
||||
|
||||
/// Constructs a workspace for verifying GEMM.
|
||||
/// Constructs a workspace for verifying GEMM, assumes
|
||||
/// dense packing.
|
||||
GemmTestbed(int M_,
|
||||
int N_,
|
||||
int K_,
|
||||
cublasOperation_t layout_a,
|
||||
cublasOperation_t layout_b,
|
||||
Scalar alpha_ = Scalar(1),
|
||||
Scalar beta_ = Scalar(0),
|
||||
cublasGemmAlgo_t algorithm_ = CUBLAS_GEMM_DEFAULT,
|
||||
cublasOperation_t layout_c = CUBLAS_OP_N)
|
||||
: problem_size(K_, N_, M_, 1),
|
||||
layout_A(layout_a),
|
||||
layout_B(layout_b),
|
||||
alpha(alpha_),
|
||||
beta(beta_),
|
||||
algorithm(algorithm_),
|
||||
batch_count(1),
|
||||
batch_stride_A(static_cast<long long int>(0)),
|
||||
batch_stride_B(static_cast<long long int>(0)),
|
||||
batch_stride_C(static_cast<long long int>(0)) {
|
||||
status = cublasCreate(&handle);
|
||||
if (status != CUBLAS_STATUS_SUCCESS) {
|
||||
throw cutlass::cuda_exception("Failed to create CUBLAS handle");
|
||||
}
|
||||
|
||||
resize(A, M_, K_, layout_a);
|
||||
resize(B, K_, N_, layout_b);
|
||||
resize(C_initial, M_, N_, layout_c);
|
||||
resize(ref_host, M_, N_, layout_c);
|
||||
resize(ref_cublas, M_, N_, layout_c);
|
||||
resize(computed, M_, N_, layout_c);
|
||||
}
|
||||
|
||||
/// Constructs a workspace for verifying GEMM, assumes
|
||||
/// dense packing.
|
||||
GemmTestbed(cublasHandle_t handle_,
|
||||
int M_,
|
||||
int N_,
|
||||
int K_,
|
||||
cublasOperation_t layout_a,
|
||||
cublasOperation_t layout_b,
|
||||
Scalar alpha_ = Scalar(1),
|
||||
Scalar beta_ = Scalar(0),
|
||||
cublasGemmAlgo_t algorithm_ = CUBLAS_GEMM_DEFAULT,
|
||||
cublasOperation_t layout_c = CUBLAS_OP_N)
|
||||
: status(CUBLAS_STATUS_SUCCESS),
|
||||
handle(handle_),
|
||||
problem_size(K_, N_, M_, 1),
|
||||
layout_A(layout_a),
|
||||
layout_B(layout_b),
|
||||
alpha(alpha_),
|
||||
beta(beta_),
|
||||
algorithm(algorithm_),
|
||||
batch_count(1),
|
||||
batch_stride_A(static_cast<long long int>(0)),
|
||||
batch_stride_B(static_cast<long long int>(0)),
|
||||
batch_stride_C(static_cast<long long int>(0)) {
|
||||
|
||||
resize(A, M_, K_ * batch_count, layout_a);
|
||||
resize(B, K_ * batch_count, N_, layout_b);
|
||||
resize(C_initial, M_, N_ * batch_count, layout_c);
|
||||
resize(ref_host, M_, N_ * batch_count, layout_c);
|
||||
resize(ref_cublas, M_, N_ * batch_count, layout_c);
|
||||
resize(computed, M_, N_ * batch_count, layout_c);
|
||||
}
|
||||
|
||||
/// Constructs a workspace for verifying GEMM with arbitrary strides
|
||||
GemmTestbed(int M_,
|
||||
int N_,
|
||||
int K_,
|
||||
@@ -250,7 +370,16 @@ struct GemmTestbed {
|
||||
Scalar beta_ = Scalar(0),
|
||||
cublasGemmAlgo_t algorithm_ = CUBLAS_GEMM_DEFAULT,
|
||||
cublasOperation_t layout_c = CUBLAS_OP_N)
|
||||
: layout_A(layout_a), layout_B(layout_b), alpha(alpha_), beta(beta_), algorithm(algorithm_) {
|
||||
: problem_size(K_, N_, M_, 1),
|
||||
layout_A(layout_a),
|
||||
layout_B(layout_b),
|
||||
alpha(alpha_),
|
||||
beta(beta_),
|
||||
algorithm(algorithm_),
|
||||
batch_count(1),
|
||||
batch_stride_A(static_cast<long long int>(0)),
|
||||
batch_stride_B(static_cast<long long int>(0)),
|
||||
batch_stride_C(static_cast<long long int>(0)) {
|
||||
status = cublasCreate(&handle);
|
||||
if (status != CUBLAS_STATUS_SUCCESS) {
|
||||
throw cutlass::cuda_exception("Failed to create CUBLAS handle");
|
||||
@@ -264,39 +393,119 @@ struct GemmTestbed {
|
||||
resize(computed, M_, N_, layout_c, ldc);
|
||||
}
|
||||
|
||||
~GemmTestbed() { status = cublasDestroy(handle); }
|
||||
/// Constructs a workspace for verifying GEMM with arbitrary strides
|
||||
GemmTestbed(cublasHandle_t handle_,
|
||||
int M_,
|
||||
int N_,
|
||||
int K_,
|
||||
int ldc,
|
||||
cublasOperation_t layout_a,
|
||||
int lda,
|
||||
cublasOperation_t layout_b,
|
||||
int ldb,
|
||||
Scalar alpha_ = Scalar(1),
|
||||
Scalar beta_ = Scalar(0),
|
||||
cublasGemmAlgo_t algorithm_ = CUBLAS_GEMM_DEFAULT,
|
||||
cublasOperation_t layout_c = CUBLAS_OP_N)
|
||||
: status(CUBLAS_STATUS_SUCCESS),
|
||||
handle(handle_),
|
||||
problem_size(K_, N_, M_, 1),
|
||||
alpha(alpha_),
|
||||
beta(beta_),
|
||||
algorithm(algorithm_),
|
||||
batch_count(1),
|
||||
batch_stride_A(static_cast<long long int>(0)),
|
||||
batch_stride_B(static_cast<long long int>(0)),
|
||||
batch_stride_C(static_cast<long long int>(0)) {
|
||||
|
||||
resize(A, M_, K_ * batch_count, layout_a);
|
||||
resize(B, K_ * batch_count, N_, layout_b);
|
||||
resize(C_initial, M_, N_ * batch_count, layout_c);
|
||||
resize(ref_host, M_, N_ * batch_count, layout_c);
|
||||
resize(ref_cublas, M_, N_ * batch_count, layout_c);
|
||||
resize(computed, M_, N_ * batch_count, layout_c);
|
||||
}
|
||||
|
||||
/// Constructs a workspace for verifying strided batched GEMM, assumes
|
||||
/// dense packing.
|
||||
/// batches are "concated" along K for matrix A and matrix B, and along N for matrix C
|
||||
/// a full implementation of strided batched GEMM should handle other corner cases
|
||||
GemmTestbed(int M_,
|
||||
int N_,
|
||||
int K_,
|
||||
int batch_count_,
|
||||
cublasOperation_t layout_a,
|
||||
cublasOperation_t layout_b,
|
||||
Scalar alpha_ = Scalar(1),
|
||||
Scalar beta_ = Scalar(0),
|
||||
cublasGemmAlgo_t algorithm_ = CUBLAS_GEMM_DEFAULT,
|
||||
cublasOperation_t layout_c = CUBLAS_OP_N)
|
||||
: problem_size(K_, N_, M_, batch_count_),
|
||||
layout_A(layout_a),
|
||||
layout_B(layout_b),
|
||||
alpha(alpha_),
|
||||
beta(beta_),
|
||||
algorithm(algorithm_),
|
||||
batch_count(batch_count_) {
|
||||
|
||||
status = cublasCreate(&handle);
|
||||
if (status != CUBLAS_STATUS_SUCCESS) {
|
||||
throw cutlass::cuda_exception("Failed to create CUBLAS handle");
|
||||
}
|
||||
|
||||
resize(A, M_, K_ * batch_count, layout_a);
|
||||
resize(B, K_ * batch_count, N_, layout_b);
|
||||
resize(C_initial, M_, N_ * batch_count, layout_c);
|
||||
resize(ref_host, M_, N_ * batch_count, layout_c);
|
||||
resize(ref_cublas, M_, N_ * batch_count, layout_c);
|
||||
resize(computed, M_, N_ * batch_count, layout_c);
|
||||
|
||||
batch_stride_A = (layout_a == CUBLAS_OP_N) ? M_ * K_ : K_;
|
||||
batch_stride_B = (layout_b == CUBLAS_OP_N) ? K_ : K_ * N_;
|
||||
batch_stride_C = M_ * N_;
|
||||
}
|
||||
|
||||
/// Destructs the GEMM testbed
|
||||
~GemmTestbed() {
|
||||
if (status != CUBLAS_STATUS_NOT_INITIALIZED) {
|
||||
status = cublasDestroy(handle);
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns true if the last CUBLAS call returned successfully
|
||||
bool good() const { return status == CUBLAS_STATUS_SUCCESS; }
|
||||
|
||||
/// Returns a pointer to the A operand
|
||||
typename HostTensorA::DeviceType* ptr_A() const { return A.device_data(); }
|
||||
typename HostMatrixA::DeviceType* ptr_A() const { return A.device_data(); }
|
||||
|
||||
/// Stride of A matrix
|
||||
int lda() const { return std::max(A.stride(HostTensorA::Dim_H), A.stride(HostTensorA::Dim_W)); }
|
||||
int lda() const { return A.leading_dim(); }
|
||||
|
||||
/// Returns a pointer to the B operand
|
||||
typename HostTensorB::DeviceType* ptr_B() const { return B.device_data(); }
|
||||
typename HostMatrixB::DeviceType* ptr_B() const { return B.device_data(); }
|
||||
|
||||
/// Stride of B matrix
|
||||
int ldb() const { return std::max(B.stride(HostTensorB::Dim_H), B.stride(HostTensorB::Dim_W)); }
|
||||
int ldb() const { return B.leading_dim(); }
|
||||
|
||||
/// Returns a pointer to the initial state of the result tensor in device memory
|
||||
typename HostTensorC::DeviceType* ptr_C_initial() const { return C_initial.device_data(); }
|
||||
typename HostMatrixC::DeviceType* ptr_C_initial() const { return C_initial.device_data(); }
|
||||
|
||||
/// Returns a pointer to the result tensor in device memory
|
||||
typename HostTensorC::DeviceType* ptr_computed() const { return computed.device_data(); }
|
||||
typename HostMatrixC::DeviceType* ptr_computed() const { return computed.device_data(); }
|
||||
|
||||
/// Returns a pointer to the result tensor in device memory
|
||||
typename HostTensorC::DeviceType* ptr_cublas() const { return ref_cublas.device_data(); }
|
||||
typename HostMatrixC::DeviceType* ptr_cublas() const { return ref_cublas.device_data(); }
|
||||
|
||||
/// Stride of C matrix
|
||||
int ldc() const {
|
||||
return std::max(C_initial.stride(HostTensorC::Dim_H), C_initial.stride(HostTensorC::Dim_W));
|
||||
//return std::max(C_initial.stride(HostTensorC::Dim_H), C_initial.stride(HostTensorC::Dim_W));
|
||||
return C_initial.leading_dim();
|
||||
}
|
||||
|
||||
/// Returns the number of flops implied by the computation (1 multiply-accumulate = 2 flops)
|
||||
uint64_t flops() const { return uint64_t(M()) * uint64_t(N()) * uint64_t(K()) * 2ULL; }
|
||||
uint64_t flops() const {
|
||||
return uint64_t(batch_count) * uint64_t(M()) * uint64_t(N()) * uint64_t(K()) * 2ULL;
|
||||
}
|
||||
|
||||
/// Computes the speed of the computation in GFLOPs/s
|
||||
double GFLOPs_per_sec(double runtime_ms) const { return double(flops()) / runtime_ms / 1.0e6; }
|
||||
@@ -307,53 +516,151 @@ struct GemmTestbed {
|
||||
/// Matrix layout of B
|
||||
cublasOperation_t layout_b() const { return layout_B; }
|
||||
|
||||
/// Number of rows of problem
|
||||
int M() const { return C_initial.size(HostTensorC::Dim_H); }
|
||||
/// Number of rows of problem, per batch; assumptions made here that we concat C by adding columns
|
||||
int M() const {
|
||||
return problem_size.m();
|
||||
}
|
||||
|
||||
/// Number of columns of problem
|
||||
int N() const { return C_initial.size(HostTensorC::Dim_W); }
|
||||
/// Number of columns of problem, per batch; assumptions made here that we concat C by adding
|
||||
/// columns
|
||||
int N() const {
|
||||
return problem_size.n();
|
||||
}
|
||||
|
||||
/// Number of columns of problem
|
||||
int K() const { return A.size(HostTensorA::Dim_W); }
|
||||
/// Number of columns of problem, per batch; assumptions made here that we concat A by adding
|
||||
/// columns
|
||||
int K() const {
|
||||
return problem_size.k();
|
||||
}
|
||||
|
||||
/// Number of batches
|
||||
int get_batch_count() const {
|
||||
return problem_size.batch();
|
||||
}
|
||||
|
||||
///
|
||||
long long int get_batch_stride_A() const { return batch_stride_A; }
|
||||
|
||||
///
|
||||
long long int get_batch_stride_B() const { return batch_stride_B; }
|
||||
|
||||
///
|
||||
long long int get_batch_stride_C() const { return batch_stride_C; }
|
||||
|
||||
///
|
||||
|
||||
/// Initializes data, randomly
|
||||
void initialize(int seed = -1) {
|
||||
A.fill_random(RandomGenerator<AType>(seed));
|
||||
B.fill_random(RandomGenerator<BType>(seed + 11));
|
||||
C_initial.fill_random(RandomGenerator<CType>(seed + 13));
|
||||
|
||||
// Initialize the source matrix with a uniform distribution
|
||||
cutlass::Distribution dist;
|
||||
dist.set_uniform(-8, 8);
|
||||
|
||||
cutlass::reference::host::TensorInitialize(A.host_view(), seed, dist);
|
||||
cutlass::reference::host::TensorInitialize(B.host_view(), seed + 11, dist);
|
||||
cutlass::reference::host::TensorInitialize(C_initial.host_view(), seed + 13, dist);
|
||||
|
||||
A.sync_device();
|
||||
B.sync_device();
|
||||
C_initial.sync_device();
|
||||
}
|
||||
|
||||
/// Initializes binary data
|
||||
void initialize_binary(int seed = -1) {
|
||||
//A.fill_random(RandomBitGenerator<AType>(seed));
|
||||
//B.fill_random(RandomBitGenerator<BType>(seed + 11));
|
||||
//C_initial.fill_random(RandomGenerator<CType>(seed + 13));
|
||||
A.fill_sequential();
|
||||
B.fill_sequential();
|
||||
C_initial.fill(0);
|
||||
}
|
||||
|
||||
/// Initializes integer data (sequential for now)
|
||||
void initialize_integer(int seed =-1) {
|
||||
A.fill_sequential();
|
||||
B.fill_sequential();
|
||||
C_initial.fill(0);
|
||||
}
|
||||
|
||||
/// Computes the matrix product on the host
|
||||
void compute_host() {
|
||||
ref_host.fill(C_initial);
|
||||
ref_host.template gemm<AType, BType, Accumulator, Scalar>(A, B, alpha, beta);
|
||||
|
||||
cutlass::reference::host::Gemm(problem_size, alpha, A.host_ref(), B.host_ref(), beta, ref_host.host_ref(), Accumulator(0));
|
||||
}
|
||||
|
||||
/// Excutes an equivalent GEMM using cuBLAS
|
||||
bool execute_cublas() {
|
||||
status = cublasGemmEx(handle,
|
||||
layout_a(),
|
||||
layout_b(),
|
||||
M(),
|
||||
N(),
|
||||
K(),
|
||||
&alpha,
|
||||
ptr_A(),
|
||||
cutlass::TypeTraits<AType>::cublas_type,
|
||||
lda(),
|
||||
ptr_B(),
|
||||
cutlass::TypeTraits<BType>::cublas_type,
|
||||
ldb(),
|
||||
&beta,
|
||||
ref_cublas.device_data(),
|
||||
cutlass::TypeTraits<CType>::cublas_type,
|
||||
ldc(),
|
||||
cutlass::TypeTraits<Accumulator>::cublas_type,
|
||||
algorithm);
|
||||
if (batch_count == 1) {
|
||||
status = cublasGemmEx(handle,
|
||||
layout_a(),
|
||||
layout_b(),
|
||||
M(),
|
||||
N(),
|
||||
K(),
|
||||
&alpha,
|
||||
ptr_A(),
|
||||
cutlass::TypeTraits<AType>::cublas_type,
|
||||
lda(),
|
||||
ptr_B(),
|
||||
cutlass::TypeTraits<BType>::cublas_type,
|
||||
ldb(),
|
||||
&beta,
|
||||
ref_cublas.device_data(),
|
||||
cutlass::TypeTraits<CType>::cublas_type,
|
||||
ldc(),
|
||||
cutlass::TypeTraits<Accumulator>::cublas_type,
|
||||
algorithm);
|
||||
|
||||
return status == CUBLAS_STATUS_SUCCESS;
|
||||
return status == CUBLAS_STATUS_SUCCESS;
|
||||
} else {
|
||||
// call strided batched gemm
|
||||
status = cublasGemmStridedBatchedTemplate(handle,
|
||||
layout_a(),
|
||||
layout_b(),
|
||||
M(),
|
||||
N(),
|
||||
K(),
|
||||
&alpha,
|
||||
ptr_A(),
|
||||
lda(),
|
||||
batch_stride_A,
|
||||
ptr_B(),
|
||||
ldb(),
|
||||
batch_stride_B,
|
||||
&beta,
|
||||
ref_cublas.device_data(),
|
||||
ldc(),
|
||||
batch_stride_C,
|
||||
batch_count);
|
||||
|
||||
return status == CUBLAS_STATUS_SUCCESS;
|
||||
}
|
||||
}
|
||||
|
||||
/// Helper function to use cublasGemmStridedBatched
|
||||
cublasStatus_t cublasGemmStridedBatchedTemplate(cublasHandle_t handle,
|
||||
cublasOperation_t transa,
|
||||
cublasOperation_t transb,
|
||||
int M,
|
||||
int N,
|
||||
int K,
|
||||
const Scalar *alpha,
|
||||
const typename HostMatrixA::DeviceType *ptr_A,
|
||||
int lda,
|
||||
long long int stride_A,
|
||||
const typename HostMatrixB::DeviceType *ptr_B,
|
||||
int ldb,
|
||||
long long int stride_B,
|
||||
const Scalar *beta,
|
||||
typename HostMatrixC::DeviceType *ptr_C,
|
||||
int ldc,
|
||||
long long int stride_C,
|
||||
int batchCount) {
|
||||
return CUBLAS_STATUS_NOT_SUPPORTED;
|
||||
}
|
||||
|
||||
|
||||
/// Computes the matrix product using cuBLAS
|
||||
void compute_cublas() {
|
||||
ref_cublas.fill(C_initial);
|
||||
@@ -374,8 +681,11 @@ struct GemmTestbed {
|
||||
<< (layout_b() == CUBLAS_OP_N ? "n" : "t") << "_" << typeid(AType).name() << "_"
|
||||
<< typeid(BType).name() << "_" << typeid(CType).name() << "_" << typeid(Accumulator).name()
|
||||
<< "_" << typeid(Scalar).name() << "_" << M() << "x" << N() << "x" << K();
|
||||
|
||||
return ss.str();
|
||||
//make sure there is no space in the ss
|
||||
std::string thisString = ss.str();
|
||||
std::replace(thisString.begin(), thisString.end(), ' ', '_');
|
||||
std::replace(thisString.begin(), thisString.end(), ':', '_');
|
||||
return thisString;
|
||||
}
|
||||
|
||||
/// Writes the workspace to an ostream
|
||||
@@ -389,8 +699,8 @@ struct GemmTestbed {
|
||||
|
||||
/// Outputs each mismatching element
|
||||
std::ostream& write_errors(std::ostream& out,
|
||||
cutlass::HostTensorView<CType> const& experimental,
|
||||
cutlass::HostTensorView<CType> const& ref) const {
|
||||
HostMatrixC const& experimental,
|
||||
HostMatrixC const& ref) const {
|
||||
PrintErrors printer(out, ref, experimental);
|
||||
|
||||
computed.visit(printer);
|
||||
@@ -419,8 +729,8 @@ struct GemmTestbed {
|
||||
}
|
||||
|
||||
/// Saves the workspace to files
|
||||
void save_workspace(cutlass::HostTensorView<CType> const& experimental,
|
||||
cutlass::HostTensorView<CType> const& ref) {
|
||||
void save_workspace(HostMatrixC const& experimental,
|
||||
HostMatrixC const& ref) {
|
||||
std::string name = workspace_name();
|
||||
|
||||
std::string results_name = name + "_results.txt";
|
||||
@@ -453,6 +763,7 @@ struct GemmTestbed {
|
||||
ref_cublas.sync_host();
|
||||
computed.sync_host();
|
||||
|
||||
|
||||
bool passed = computed.bit_equals(ref_cublas);
|
||||
|
||||
if ((!passed && save_on_error) || always_print) {
|
||||
@@ -494,22 +805,116 @@ struct GemmTestbed {
|
||||
bool has_cublas_support() const { return cutlass::platform::is_same<Accumulator, Scalar>::value; }
|
||||
};
|
||||
|
||||
//
|
||||
//specialization for cublasGemmStridedBatchedTemplate
|
||||
template<> inline cublasStatus_t GemmTestbed<float, float, float, float, float>::cublasGemmStridedBatchedTemplate(cublasHandle_t handle,
|
||||
cublasOperation_t transa,
|
||||
cublasOperation_t transb,
|
||||
int M,
|
||||
int N,
|
||||
int K,
|
||||
const float *alpha,
|
||||
const float *ptr_A,
|
||||
int lda,
|
||||
long long int stride_A,
|
||||
const float *ptr_B,
|
||||
int ldb,
|
||||
long long int stride_B,
|
||||
const float *beta,
|
||||
float *ptr_C,
|
||||
int ldc,
|
||||
long long int stride_C,
|
||||
int batchCount) {
|
||||
return cublasSgemmStridedBatched(handle,
|
||||
transa,
|
||||
transb,
|
||||
M, N, K,
|
||||
alpha,
|
||||
ptr_A,
|
||||
lda,
|
||||
stride_A,
|
||||
ptr_B,
|
||||
ldb,
|
||||
stride_B,
|
||||
beta,
|
||||
ptr_C,
|
||||
ldc,
|
||||
stride_C,
|
||||
batchCount);
|
||||
}
|
||||
|
||||
template<> inline cublasStatus_t GemmTestbed<double, double, double, double, double>::cublasGemmStridedBatchedTemplate(cublasHandle_t handle,
|
||||
cublasOperation_t transa,
|
||||
cublasOperation_t transb,
|
||||
int M,
|
||||
int N,
|
||||
int K,
|
||||
const double *alpha,
|
||||
const double *ptr_A,
|
||||
int lda,
|
||||
long long int stride_A,
|
||||
const double *ptr_B,
|
||||
int ldb,
|
||||
long long int stride_B,
|
||||
const double *beta,
|
||||
double *ptr_C,
|
||||
int ldc,
|
||||
long long int stride_C,
|
||||
int batchCount) {
|
||||
return cublasDgemmStridedBatched(handle,
|
||||
transa,
|
||||
transb,
|
||||
M, N, K,
|
||||
alpha,
|
||||
ptr_A,
|
||||
lda,
|
||||
stride_A,
|
||||
ptr_B,
|
||||
ldb,
|
||||
stride_B,
|
||||
beta,
|
||||
ptr_C,
|
||||
ldc,
|
||||
stride_C,
|
||||
batchCount);
|
||||
}
|
||||
|
||||
template<> inline cublasStatus_t GemmTestbed<cutlass::half_t, cutlass::half_t, cutlass::half_t, cutlass::half_t, cutlass::half_t>::cublasGemmStridedBatchedTemplate(cublasHandle_t handle,
|
||||
cublasOperation_t transa,
|
||||
cublasOperation_t transb,
|
||||
int M,
|
||||
int N,
|
||||
int K,
|
||||
const cutlass::half_t *alpha,
|
||||
const half *ptr_A,
|
||||
int lda,
|
||||
long long int stride_A,
|
||||
const half *ptr_B,
|
||||
int ldb,
|
||||
long long int stride_B,
|
||||
const cutlass::half_t *beta,
|
||||
half *ptr_C,
|
||||
int ldc,
|
||||
long long int stride_C,
|
||||
int batchCount) {
|
||||
half temp_alpha = alpha->operator half();
|
||||
half temp_beta = beta->operator half();
|
||||
return cublasHgemmStridedBatched(handle,
|
||||
transa,
|
||||
transb,
|
||||
M, N, K,
|
||||
&temp_alpha,
|
||||
ptr_A,
|
||||
lda,
|
||||
stride_A,
|
||||
ptr_B,
|
||||
ldb,
|
||||
stride_B,
|
||||
&temp_beta,
|
||||
ptr_C,
|
||||
ldc,
|
||||
stride_C,
|
||||
batchCount);
|
||||
}
|
||||
|
||||
} // namespace test
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
namespace cutlass {
|
||||
inline cublasOperation_t convert(cutlass::MatrixLayout::Kind layout) {
|
||||
switch (layout) {
|
||||
case cutlass::MatrixLayout::kRowMajor:
|
||||
return CUBLAS_OP_T;
|
||||
case cutlass::MatrixLayout::kColumnMajor:
|
||||
return CUBLAS_OP_N;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
return CUBLAS_OP_N;
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
}
|
||||
|
||||
@@ -22,30 +22,12 @@
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
#include <cutlass_unit_test.h>
|
||||
#include <tools/util/half.h>
|
||||
#include <cutlass/gemm/gemm.h>
|
||||
#include <cutlass/gemm/hgemm_traits.h>
|
||||
#include <tools/test/unit/gemm/gemm_testbed.h>
|
||||
#include <tools/test/unit/gemm/gemm.h>
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Hgemm_128x128x16, hgemm_2x2x2_nt) {
|
||||
typedef cutlass::gemm::HgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor, cutlass::Shape<16, 128, 128> >
|
||||
HgemmTraits;
|
||||
run_gemm<HgemmTraits>(2, 2, 2);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Hgemm_128x128x16, hgemm_128x128x8_nt) {
|
||||
typedef cutlass::gemm::HgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor, cutlass::Shape<16, 128, 128> >
|
||||
HgemmTraits;
|
||||
run_gemm<HgemmTraits>(128, 128, 8);
|
||||
}
|
||||
#include "cutlass_unit_test.h"
|
||||
#include "tools/util/half.h"
|
||||
#include "cutlass/gemm/gemm.h"
|
||||
#include "cutlass/gemm/hgemm_traits.h"
|
||||
#include "tools/test/unit/gemm/gemm_testbed.h"
|
||||
#include "tools/test/unit/gemm/run_gemm.h"
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
|
||||
@@ -22,12 +22,12 @@
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
#include <cutlass_unit_test.h>
|
||||
#include <tools/util/half.h>
|
||||
#include <cutlass/gemm/gemm.h>
|
||||
#include <cutlass/gemm/hgemm_traits.h>
|
||||
#include <tools/test/unit/gemm/gemm_testbed.h>
|
||||
#include <tools/test/unit/gemm/gemm.h>
|
||||
#include "cutlass_unit_test.h"
|
||||
#include "tools/util/half.h"
|
||||
#include "cutlass/gemm/gemm.h"
|
||||
#include "cutlass/gemm/hgemm_traits.h"
|
||||
#include "tools/test/unit/gemm/gemm_testbed.h"
|
||||
#include "tools/test/unit/gemm/run_gemm.h"
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
|
||||
@@ -22,11 +22,11 @@
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
#include <cutlass_unit_test.h>
|
||||
#include <cutlass/gemm/gemm.h>
|
||||
#include <cutlass/gemm/hgemm_traits.h>
|
||||
#include <tools/test/unit/gemm/gemm_testbed.h>
|
||||
#include <tools/test/unit/gemm/gemm.h>
|
||||
#include "cutlass_unit_test.h"
|
||||
#include "cutlass/gemm/gemm.h"
|
||||
#include "cutlass/gemm/hgemm_traits.h"
|
||||
#include "tools/test/unit/gemm/gemm_testbed.h"
|
||||
#include "tools/test/unit/gemm/run_gemm.h"
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
|
||||
@@ -22,11 +22,11 @@
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
#include <cutlass_unit_test.h>
|
||||
#include <cutlass/gemm/gemm.h>
|
||||
#include <cutlass/gemm/hgemm_traits.h>
|
||||
#include <tools/test/unit/gemm/gemm_testbed.h>
|
||||
#include <tools/test/unit/gemm/gemm.h>
|
||||
#include "cutlass_unit_test.h"
|
||||
#include "cutlass/gemm/gemm.h"
|
||||
#include "cutlass/gemm/hgemm_traits.h"
|
||||
#include "tools/test/unit/gemm/gemm_testbed.h"
|
||||
#include "tools/test/unit/gemm/run_gemm.h"
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
|
||||
@@ -22,16 +22,17 @@
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
#include <cutlass_unit_test.h>
|
||||
#include <cutlass/gemm/gemm.h>
|
||||
#include <cutlass/gemm/igemm_traits.h>
|
||||
#include <tools/test/unit/gemm/gemm.h>
|
||||
#include "cutlass_unit_test.h"
|
||||
#include "cutlass/gemm/gemm.h"
|
||||
#include "cutlass/gemm/igemm_traits.h"
|
||||
#include "tools/test/unit/gemm/gemm_testbed.h"
|
||||
#include "tools/test/unit/gemm/run_gemm.h"
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Igemm_128x128x32, igemm_128x128x4_nt) {
|
||||
typedef cutlass::gemm::IgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<32, 128, 128>, int, cutlass::gemm::LinearScaling<int> >
|
||||
IgemmTraits;
|
||||
run_gemm<IgemmTraits>(128, 128, 4);
|
||||
@@ -41,7 +42,7 @@ TEST(Igemm_128x128x32, igemm_128x128x4_nt) {
|
||||
|
||||
TEST(Igemm_128x128x32, igemm_128x128x32_nt) {
|
||||
typedef cutlass::gemm::IgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<32, 128, 128>, int, cutlass::gemm::LinearScaling<int> >
|
||||
IgemmTraits;
|
||||
run_gemm<IgemmTraits>(128, 128, 32);
|
||||
@@ -51,7 +52,7 @@ TEST(Igemm_128x128x32, igemm_128x128x32_nt) {
|
||||
|
||||
TEST(Igemm_128x128x32, igemm_128x128x36_nt) {
|
||||
typedef cutlass::gemm::IgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<32, 128, 128>, int, cutlass::gemm::LinearScaling<int> >
|
||||
IgemmTraits;
|
||||
run_gemm<IgemmTraits>(128, 128, 36);
|
||||
@@ -61,7 +62,7 @@ TEST(Igemm_128x128x32, igemm_128x128x36_nt) {
|
||||
|
||||
TEST(Igemm_128x128x32, igemm_128x128x64_nt) {
|
||||
typedef cutlass::gemm::IgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<32, 128, 128>, int, cutlass::gemm::LinearScaling<int> >
|
||||
IgemmTraits;
|
||||
run_gemm<IgemmTraits>(128, 128, 64);
|
||||
@@ -71,7 +72,7 @@ TEST(Igemm_128x128x32, igemm_128x128x64_nt) {
|
||||
|
||||
TEST(Igemm_128x128x32, igemm_128x128x256_nt) {
|
||||
typedef cutlass::gemm::IgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<32, 128, 128>, int, cutlass::gemm::LinearScaling<int> >
|
||||
IgemmTraits;
|
||||
run_gemm<IgemmTraits>(128, 128, 256);
|
||||
@@ -81,7 +82,7 @@ TEST(Igemm_128x128x32, igemm_128x128x256_nt) {
|
||||
|
||||
TEST(Igemm_128x128x32, igemm_256x128x64_nt) {
|
||||
typedef cutlass::gemm::IgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<32, 128, 128>, int, cutlass::gemm::LinearScaling<int> >
|
||||
IgemmTraits;
|
||||
run_gemm<IgemmTraits>(256, 128, 64);
|
||||
@@ -91,7 +92,7 @@ TEST(Igemm_128x128x32, igemm_256x128x64_nt) {
|
||||
|
||||
TEST(Igemm_128x128x32, igemm_128x256x64_nt) {
|
||||
typedef cutlass::gemm::IgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<32, 128, 128>, int, cutlass::gemm::LinearScaling<int> >
|
||||
IgemmTraits;
|
||||
run_gemm<IgemmTraits>(128, 256, 64);
|
||||
@@ -101,7 +102,7 @@ TEST(Igemm_128x128x32, igemm_128x256x64_nt) {
|
||||
|
||||
TEST(Igemm_128x128x32, igemm_256x256x64_nt) {
|
||||
typedef cutlass::gemm::IgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<32, 128, 128>, int, cutlass::gemm::LinearScaling<int> >
|
||||
IgemmTraits;
|
||||
run_gemm<IgemmTraits>(256, 256, 64);
|
||||
@@ -189,6 +190,12 @@ TEST(Igemm_128x128x32, igemm_256x256x64_nn) {
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
// NB: I have removed tests in which k=1. These result in the test environment definining matrices
|
||||
// in which ld{a,b} = 1 which cannot be launched by cuBLAS.
|
||||
//
|
||||
// This problem size remains untested. --akerr
|
||||
//
|
||||
|
||||
TEST(Igemm_128x128x32, igemm_128x128x4_tn) {
|
||||
typedef cutlass::gemm::IgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
@@ -281,7 +288,7 @@ TEST(Igemm_128x128x32, igemm_128x128x4_tt) {
|
||||
|
||||
TEST(Igemm_128x128x32, igemm_128x128x32_tt) {
|
||||
typedef cutlass::gemm::IgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<32, 128, 128>, int, cutlass::gemm::LinearScaling<int> >
|
||||
IgemmTraits;
|
||||
run_gemm<IgemmTraits>(128, 128, 32);
|
||||
@@ -291,7 +298,7 @@ TEST(Igemm_128x128x32, igemm_128x128x32_tt) {
|
||||
|
||||
TEST(Igemm_128x128x32, igemm_128x128x36_tt) {
|
||||
typedef cutlass::gemm::IgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<32, 128, 128>, int, cutlass::gemm::LinearScaling<int> >
|
||||
IgemmTraits;
|
||||
run_gemm<IgemmTraits>(128, 128, 36);
|
||||
@@ -301,7 +308,7 @@ TEST(Igemm_128x128x32, igemm_128x128x36_tt) {
|
||||
|
||||
TEST(Igemm_128x128x32, igemm_128x128x64_tt) {
|
||||
typedef cutlass::gemm::IgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<32, 128, 128>, int, cutlass::gemm::LinearScaling<int> >
|
||||
IgemmTraits;
|
||||
run_gemm<IgemmTraits>(128, 128, 64);
|
||||
@@ -311,7 +318,7 @@ TEST(Igemm_128x128x32, igemm_128x128x64_tt) {
|
||||
|
||||
TEST(Igemm_128x128x32, igemm_128x128x256_tt) {
|
||||
typedef cutlass::gemm::IgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<32, 128, 128>, int, cutlass::gemm::LinearScaling<int> >
|
||||
IgemmTraits;
|
||||
run_gemm<IgemmTraits>(128, 128, 256);
|
||||
@@ -321,7 +328,7 @@ TEST(Igemm_128x128x32, igemm_128x128x256_tt) {
|
||||
|
||||
TEST(Igemm_128x128x32, igemm_256x128x64_tt) {
|
||||
typedef cutlass::gemm::IgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<32, 128, 128>, int, cutlass::gemm::LinearScaling<int> >
|
||||
IgemmTraits;
|
||||
run_gemm<IgemmTraits>(256, 128, 64);
|
||||
@@ -331,7 +338,7 @@ TEST(Igemm_128x128x32, igemm_256x128x64_tt) {
|
||||
|
||||
TEST(Igemm_128x128x32, igemm_128x256x64_tt) {
|
||||
typedef cutlass::gemm::IgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<32, 128, 128>, int, cutlass::gemm::LinearScaling<int> >
|
||||
IgemmTraits;
|
||||
run_gemm<IgemmTraits>(128, 256, 64);
|
||||
@@ -341,7 +348,7 @@ TEST(Igemm_128x128x32, igemm_128x256x64_tt) {
|
||||
|
||||
TEST(Igemm_128x128x32, igemm_256x256x64_tt) {
|
||||
typedef cutlass::gemm::IgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<32, 128, 128>, int, cutlass::gemm::LinearScaling<int> >
|
||||
IgemmTraits;
|
||||
run_gemm<IgemmTraits>(256, 256, 64);
|
||||
|
||||
@@ -22,17 +22,17 @@
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
#include <cutlass_unit_test.h>
|
||||
#include <cutlass/gemm/gemm.h>
|
||||
#include <cutlass/gemm/igemm_traits.h>
|
||||
#include <tools/test/unit/gemm/gemm_testbed.h>
|
||||
#include <tools/test/unit/gemm/gemm.h>
|
||||
#include "cutlass_unit_test.h"
|
||||
#include "cutlass/gemm/gemm.h"
|
||||
#include "cutlass/gemm/igemm_traits.h"
|
||||
#include "tools/test/unit/gemm/gemm_testbed.h"
|
||||
#include "tools/test/unit/gemm/run_gemm.h"
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Igemm_128x128x32_float, igemm_128x128x4_nt) {
|
||||
typedef cutlass::gemm::IgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<32, 128, 128>, float>
|
||||
IgemmTraits;
|
||||
run_gemm<IgemmTraits>(128, 128, 4);
|
||||
@@ -42,7 +42,7 @@ TEST(Igemm_128x128x32_float, igemm_128x128x4_nt) {
|
||||
|
||||
TEST(Igemm_128x128x32_float, igemm_128x128x32_nt) {
|
||||
typedef cutlass::gemm::IgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<32, 128, 128>, float>
|
||||
IgemmTraits;
|
||||
run_gemm<IgemmTraits>(128, 128, 32);
|
||||
@@ -52,7 +52,7 @@ TEST(Igemm_128x128x32_float, igemm_128x128x32_nt) {
|
||||
|
||||
TEST(Igemm_128x128x32_float, igemm_128x128x36_nt) {
|
||||
typedef cutlass::gemm::IgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<32, 128, 128>, float>
|
||||
IgemmTraits;
|
||||
run_gemm<IgemmTraits>(128, 128, 36);
|
||||
@@ -62,7 +62,7 @@ TEST(Igemm_128x128x32_float, igemm_128x128x36_nt) {
|
||||
|
||||
TEST(Igemm_128x128x32_float, igemm_128x128x64_nt) {
|
||||
typedef cutlass::gemm::IgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<32, 128, 128>, float>
|
||||
IgemmTraits;
|
||||
run_gemm<IgemmTraits>(128, 128, 64);
|
||||
@@ -72,7 +72,7 @@ TEST(Igemm_128x128x32_float, igemm_128x128x64_nt) {
|
||||
|
||||
TEST(Igemm_128x128x32_float, igemm_128x128x256_nt) {
|
||||
typedef cutlass::gemm::IgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<32, 128, 128>, float>
|
||||
IgemmTraits;
|
||||
run_gemm<IgemmTraits>(128, 128, 256);
|
||||
@@ -82,7 +82,7 @@ TEST(Igemm_128x128x32_float, igemm_128x128x256_nt) {
|
||||
|
||||
TEST(Igemm_128x128x32_float, igemm_256x128x64_nt) {
|
||||
typedef cutlass::gemm::IgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<32, 128, 128>, float>
|
||||
IgemmTraits;
|
||||
run_gemm<IgemmTraits>(256, 128, 64);
|
||||
@@ -92,7 +92,7 @@ TEST(Igemm_128x128x32_float, igemm_256x128x64_nt) {
|
||||
|
||||
TEST(Igemm_128x128x32_float, igemm_128x256x64_nt) {
|
||||
typedef cutlass::gemm::IgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<32, 128, 128>, float>
|
||||
IgemmTraits;
|
||||
run_gemm<IgemmTraits>(128, 256, 64);
|
||||
@@ -102,7 +102,7 @@ TEST(Igemm_128x128x32_float, igemm_128x256x64_nt) {
|
||||
|
||||
TEST(Igemm_128x128x32_float, igemm_256x256x64_nt) {
|
||||
typedef cutlass::gemm::IgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<32, 128, 128>, float>
|
||||
IgemmTraits;
|
||||
run_gemm<IgemmTraits>(256, 256, 64);
|
||||
@@ -190,6 +190,12 @@ TEST(Igemm_128x128x32_float, igemm_256x256x64_nn) {
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
// NB: I have removed tests in which k=1. These result in the test environment definining matrices
|
||||
// in which ld{a,b} = 1 which cannot be launched by cuBLAS.
|
||||
//
|
||||
// This problem size remains untested. --akerr
|
||||
//
|
||||
|
||||
TEST(Igemm_128x128x32_float, igemm_128x128x4_tn) {
|
||||
typedef cutlass::gemm::IgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
@@ -282,7 +288,7 @@ TEST(Igemm_128x128x32_float, igemm_128x128x4_tt) {
|
||||
|
||||
TEST(Igemm_128x128x32_float, igemm_128x128x32_tt) {
|
||||
typedef cutlass::gemm::IgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<32, 128, 128>, float>
|
||||
IgemmTraits;
|
||||
run_gemm<IgemmTraits>(128, 128, 32);
|
||||
@@ -292,7 +298,7 @@ TEST(Igemm_128x128x32_float, igemm_128x128x32_tt) {
|
||||
|
||||
TEST(Igemm_128x128x32_float, igemm_128x128x36_tt) {
|
||||
typedef cutlass::gemm::IgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<32, 128, 128>, float>
|
||||
IgemmTraits;
|
||||
run_gemm<IgemmTraits>(128, 128, 36);
|
||||
@@ -302,7 +308,7 @@ TEST(Igemm_128x128x32_float, igemm_128x128x36_tt) {
|
||||
|
||||
TEST(Igemm_128x128x32_float, igemm_128x128x64_tt) {
|
||||
typedef cutlass::gemm::IgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<32, 128, 128>, float>
|
||||
IgemmTraits;
|
||||
run_gemm<IgemmTraits>(128, 128, 64);
|
||||
@@ -312,7 +318,7 @@ TEST(Igemm_128x128x32_float, igemm_128x128x64_tt) {
|
||||
|
||||
TEST(Igemm_128x128x32_float, igemm_128x128x256_tt) {
|
||||
typedef cutlass::gemm::IgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<32, 128, 128>, float>
|
||||
IgemmTraits;
|
||||
run_gemm<IgemmTraits>(128, 128, 256);
|
||||
@@ -322,7 +328,7 @@ TEST(Igemm_128x128x32_float, igemm_128x128x256_tt) {
|
||||
|
||||
TEST(Igemm_128x128x32_float, igemm_256x128x64_tt) {
|
||||
typedef cutlass::gemm::IgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<32, 128, 128>, float>
|
||||
IgemmTraits;
|
||||
run_gemm<IgemmTraits>(256, 128, 64);
|
||||
@@ -332,7 +338,7 @@ TEST(Igemm_128x128x32_float, igemm_256x128x64_tt) {
|
||||
|
||||
TEST(Igemm_128x128x32_float, igemm_128x256x64_tt) {
|
||||
typedef cutlass::gemm::IgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<32, 128, 128>, float>
|
||||
IgemmTraits;
|
||||
run_gemm<IgemmTraits>(128, 256, 64);
|
||||
@@ -342,7 +348,7 @@ TEST(Igemm_128x128x32_float, igemm_128x256x64_tt) {
|
||||
|
||||
TEST(Igemm_128x128x32_float, igemm_256x256x64_tt) {
|
||||
typedef cutlass::gemm::IgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<32, 128, 128>, float>
|
||||
IgemmTraits;
|
||||
|
||||
|
||||
@@ -22,17 +22,17 @@
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
#include <cutlass_unit_test.h>
|
||||
#include <cutlass/gemm/gemm.h>
|
||||
#include <cutlass/gemm/igemm_traits.h>
|
||||
#include <tools/test/unit/gemm/gemm_testbed.h>
|
||||
#include <tools/test/unit/gemm/gemm.h>
|
||||
#include "cutlass_unit_test.h"
|
||||
#include "cutlass/gemm/gemm.h"
|
||||
#include "cutlass/gemm/igemm_traits.h"
|
||||
#include "tools/test/unit/gemm/gemm_testbed.h"
|
||||
#include "tools/test/unit/gemm/run_gemm.h"
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Igemm_128x128x32_int8, igemm_128x128x4_nt) {
|
||||
typedef cutlass::gemm::IgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<32, 128, 128>, int8_t>
|
||||
IgemmTraits;
|
||||
run_gemm<IgemmTraits>(128, 128, 4);
|
||||
@@ -42,7 +42,7 @@ TEST(Igemm_128x128x32_int8, igemm_128x128x4_nt) {
|
||||
|
||||
TEST(Igemm_128x128x32_int8, igemm_128x128x32_nt) {
|
||||
typedef cutlass::gemm::IgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<32, 128, 128>, int8_t>
|
||||
IgemmTraits;
|
||||
run_gemm<IgemmTraits>(128, 128, 32);
|
||||
@@ -52,7 +52,7 @@ TEST(Igemm_128x128x32_int8, igemm_128x128x32_nt) {
|
||||
|
||||
TEST(Igemm_128x128x32_int8, igemm_128x128x36_nt) {
|
||||
typedef cutlass::gemm::IgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<32, 128, 128>, int8_t>
|
||||
IgemmTraits;
|
||||
run_gemm<IgemmTraits>(128, 128, 36);
|
||||
@@ -62,7 +62,7 @@ TEST(Igemm_128x128x32_int8, igemm_128x128x36_nt) {
|
||||
|
||||
TEST(Igemm_128x128x32_int8, igemm_128x128x64_nt) {
|
||||
typedef cutlass::gemm::IgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<32, 128, 128>, int8_t>
|
||||
IgemmTraits;
|
||||
run_gemm<IgemmTraits>(128, 128, 64);
|
||||
@@ -72,7 +72,7 @@ TEST(Igemm_128x128x32_int8, igemm_128x128x64_nt) {
|
||||
|
||||
TEST(Igemm_128x128x32_int8, igemm_128x128x256_nt) {
|
||||
typedef cutlass::gemm::IgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<32, 128, 128>, int8_t>
|
||||
IgemmTraits;
|
||||
run_gemm<IgemmTraits>(128, 128, 256);
|
||||
@@ -82,7 +82,7 @@ TEST(Igemm_128x128x32_int8, igemm_128x128x256_nt) {
|
||||
|
||||
TEST(Igemm_128x128x32_int8, igemm_256x128x64_nt) {
|
||||
typedef cutlass::gemm::IgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<32, 128, 128>, int8_t>
|
||||
IgemmTraits;
|
||||
run_gemm<IgemmTraits>(256, 128, 64);
|
||||
@@ -92,7 +92,7 @@ TEST(Igemm_128x128x32_int8, igemm_256x128x64_nt) {
|
||||
|
||||
TEST(Igemm_128x128x32_int8, igemm_128x256x64_nt) {
|
||||
typedef cutlass::gemm::IgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<32, 128, 128>, int8_t>
|
||||
IgemmTraits;
|
||||
run_gemm<IgemmTraits>(128, 256, 64);
|
||||
@@ -102,7 +102,7 @@ TEST(Igemm_128x128x32_int8, igemm_128x256x64_nt) {
|
||||
|
||||
TEST(Igemm_128x128x32_int8, igemm_256x256x64_nt) {
|
||||
typedef cutlass::gemm::IgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<32, 128, 128>, int8_t>
|
||||
IgemmTraits;
|
||||
run_gemm<IgemmTraits>(256, 256, 64);
|
||||
@@ -190,6 +190,14 @@ TEST(Igemm_128x128x32_int8, igemm_256x256x64_nn) {
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
// NB: I have removed tests in which k=1. These result in the test environment definining matrices
|
||||
// in which ld{a,b} = 1 which cannot be launched by cuBLAS.
|
||||
//
|
||||
// This problem size remains untested. --akerr
|
||||
//
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Igemm_128x128x32_int8, igemm_128x128x4_tn) {
|
||||
typedef cutlass::gemm::IgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
@@ -282,7 +290,7 @@ TEST(Igemm_128x128x32_int8, igemm_128x128x4_tt) {
|
||||
|
||||
TEST(Igemm_128x128x32_int8, igemm_128x128x32_tt) {
|
||||
typedef cutlass::gemm::IgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<32, 128, 128>, int8_t>
|
||||
IgemmTraits;
|
||||
run_gemm<IgemmTraits>(128, 128, 32);
|
||||
@@ -292,7 +300,7 @@ TEST(Igemm_128x128x32_int8, igemm_128x128x32_tt) {
|
||||
|
||||
TEST(Igemm_128x128x32_int8, igemm_128x128x36_tt) {
|
||||
typedef cutlass::gemm::IgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<32, 128, 128>, int8_t>
|
||||
IgemmTraits;
|
||||
run_gemm<IgemmTraits>(128, 128, 36);
|
||||
@@ -302,7 +310,7 @@ TEST(Igemm_128x128x32_int8, igemm_128x128x36_tt) {
|
||||
|
||||
TEST(Igemm_128x128x32_int8, igemm_128x128x64_tt) {
|
||||
typedef cutlass::gemm::IgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<32, 128, 128>, int8_t>
|
||||
IgemmTraits;
|
||||
run_gemm<IgemmTraits>(128, 128, 64);
|
||||
@@ -312,7 +320,7 @@ TEST(Igemm_128x128x32_int8, igemm_128x128x64_tt) {
|
||||
|
||||
TEST(Igemm_128x128x32_int8, igemm_128x128x256_tt) {
|
||||
typedef cutlass::gemm::IgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<32, 128, 128>, int8_t>
|
||||
IgemmTraits;
|
||||
run_gemm<IgemmTraits>(128, 128, 256);
|
||||
@@ -322,7 +330,7 @@ TEST(Igemm_128x128x32_int8, igemm_128x128x256_tt) {
|
||||
|
||||
TEST(Igemm_128x128x32_int8, igemm_256x128x64_tt) {
|
||||
typedef cutlass::gemm::IgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<32, 128, 128>, int8_t>
|
||||
IgemmTraits;
|
||||
run_gemm<IgemmTraits>(256, 128, 64);
|
||||
@@ -332,7 +340,7 @@ TEST(Igemm_128x128x32_int8, igemm_256x128x64_tt) {
|
||||
|
||||
TEST(Igemm_128x128x32_int8, igemm_128x256x64_tt) {
|
||||
typedef cutlass::gemm::IgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<32, 128, 128>, int8_t>
|
||||
IgemmTraits;
|
||||
run_gemm<IgemmTraits>(128, 256, 64);
|
||||
@@ -342,7 +350,7 @@ TEST(Igemm_128x128x32_int8, igemm_128x256x64_tt) {
|
||||
|
||||
TEST(Igemm_128x128x32_int8, igemm_256x256x64_tt) {
|
||||
typedef cutlass::gemm::IgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<32, 128, 128>, int8_t>
|
||||
IgemmTraits;
|
||||
run_gemm<IgemmTraits>(256, 256, 64);
|
||||
|
||||
@@ -22,17 +22,17 @@
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
#include <cutlass_unit_test.h>
|
||||
#include <cutlass/gemm/gemm.h>
|
||||
#include <cutlass/gemm/igemm_traits.h>
|
||||
#include <tools/test/unit/gemm/gemm_testbed.h>
|
||||
#include <tools/test/unit/gemm/gemm.h>
|
||||
#include "cutlass_unit_test.h"
|
||||
#include "cutlass/gemm/gemm.h"
|
||||
#include "cutlass/gemm/igemm_traits.h"
|
||||
#include "tools/test/unit/gemm/gemm_testbed.h"
|
||||
#include "tools/test/unit/gemm/run_gemm.h"
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Igemm_128x32x32, igemm_128x32x32x4_nt) {
|
||||
typedef cutlass::gemm::IgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<32, 32, 128>, int>
|
||||
IgemmTraits;
|
||||
run_gemm<IgemmTraits>(128, 32, 4);
|
||||
@@ -42,7 +42,7 @@ TEST(Igemm_128x32x32, igemm_128x32x32x4_nt) {
|
||||
|
||||
TEST(Igemm_128x32x32, igemm_128x32x32_nt) {
|
||||
typedef cutlass::gemm::IgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<32, 32, 128>, int>
|
||||
IgemmTraits;
|
||||
run_gemm<IgemmTraits>(128, 32, 20);
|
||||
@@ -52,7 +52,7 @@ TEST(Igemm_128x32x32, igemm_128x32x32_nt) {
|
||||
|
||||
TEST(Igemm_128x32x32, igemm_128x32x36_nt) {
|
||||
typedef cutlass::gemm::IgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<32, 32, 128>, int>
|
||||
IgemmTraits;
|
||||
run_gemm<IgemmTraits>(128, 32, 36);
|
||||
@@ -62,7 +62,7 @@ TEST(Igemm_128x32x32, igemm_128x32x36_nt) {
|
||||
|
||||
TEST(Igemm_128x32x32, igemm_128x32x64_nt) {
|
||||
typedef cutlass::gemm::IgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<32, 32, 128>, int>
|
||||
IgemmTraits;
|
||||
run_gemm<IgemmTraits>(128, 32, 64);
|
||||
@@ -72,7 +72,7 @@ TEST(Igemm_128x32x32, igemm_128x32x64_nt) {
|
||||
|
||||
TEST(Igemm_128x32x32, igemm_128x32x256_nt) {
|
||||
typedef cutlass::gemm::IgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<32, 32, 128>, int>
|
||||
IgemmTraits;
|
||||
run_gemm<IgemmTraits>(128, 32, 256);
|
||||
@@ -82,7 +82,7 @@ TEST(Igemm_128x32x32, igemm_128x32x256_nt) {
|
||||
|
||||
TEST(Igemm_128x32x32, igemm_256x32x64_nt) {
|
||||
typedef cutlass::gemm::IgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<32, 32, 128>, int>
|
||||
IgemmTraits;
|
||||
run_gemm<IgemmTraits>(256, 32, 64);
|
||||
@@ -92,7 +92,7 @@ TEST(Igemm_128x32x32, igemm_256x32x64_nt) {
|
||||
|
||||
TEST(Igemm_128x32x32, igemm_128x128x32_nt) {
|
||||
typedef cutlass::gemm::IgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<32, 32, 128>, int>
|
||||
IgemmTraits;
|
||||
run_gemm<IgemmTraits>(128, 128, 32);
|
||||
@@ -102,7 +102,7 @@ TEST(Igemm_128x32x32, igemm_128x128x32_nt) {
|
||||
|
||||
TEST(Igemm_128x32x32, igemm_256x128x32_nt) {
|
||||
typedef cutlass::gemm::IgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<32, 32, 128>, int>
|
||||
IgemmTraits;
|
||||
run_gemm<IgemmTraits>(256, 128, 32);
|
||||
@@ -190,6 +190,15 @@ TEST(Igemm_128x32x32, igemm_256x128x32_nn) {
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
// FIXME: This test fails in cuBLAS. I don't know why!!!
|
||||
// TEST(Igemm_128x32x32, igemm_128x32x1_tn) {
|
||||
// typedef cutlass::gemm::IgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
// cutlass::MatrixLayout::kColumnMajor, cutlass::Shape<32, 32, 128> > IgemmTraits;
|
||||
// run_gemm<IgemmTraits>(128, 32, 1);
|
||||
// }
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Igemm_128x32x32, igemm_128x32x4_tn) {
|
||||
typedef cutlass::gemm::IgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
@@ -282,7 +291,7 @@ TEST(Igemm_128x32x32, igemm_128x32x4_tt) {
|
||||
|
||||
TEST(Igemm_128x32x32, igemm_128x32x32_tt) {
|
||||
typedef cutlass::gemm::IgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<32, 32, 128>, int>
|
||||
IgemmTraits;
|
||||
run_gemm<IgemmTraits>(128, 32, 32);
|
||||
@@ -292,7 +301,7 @@ TEST(Igemm_128x32x32, igemm_128x32x32_tt) {
|
||||
|
||||
TEST(Igemm_128x32x32, igemm_128x32x36_tt) {
|
||||
typedef cutlass::gemm::IgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<32, 32, 128>, int>
|
||||
IgemmTraits;
|
||||
run_gemm<IgemmTraits>(128, 32, 36);
|
||||
@@ -302,7 +311,7 @@ TEST(Igemm_128x32x32, igemm_128x32x36_tt) {
|
||||
|
||||
TEST(Igemm_128x32x32, igemm_128x32x64_tt) {
|
||||
typedef cutlass::gemm::IgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<32, 32, 128>, int>
|
||||
IgemmTraits;
|
||||
run_gemm<IgemmTraits>(128, 32, 64);
|
||||
@@ -312,7 +321,7 @@ TEST(Igemm_128x32x32, igemm_128x32x64_tt) {
|
||||
|
||||
TEST(Igemm_128x32x32, igemm_128x32x256_tt) {
|
||||
typedef cutlass::gemm::IgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<32, 32, 128>, int>
|
||||
IgemmTraits;
|
||||
run_gemm<IgemmTraits>(128, 32, 256);
|
||||
@@ -322,7 +331,7 @@ TEST(Igemm_128x32x32, igemm_128x32x256_tt) {
|
||||
|
||||
TEST(Igemm_128x32x32, igemm_256x32x64_tt) {
|
||||
typedef cutlass::gemm::IgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<32, 32, 128>, int>
|
||||
IgemmTraits;
|
||||
run_gemm<IgemmTraits>(256, 32, 64);
|
||||
@@ -332,7 +341,7 @@ TEST(Igemm_128x32x32, igemm_256x32x64_tt) {
|
||||
|
||||
TEST(Igemm_128x32x32, igemm_128x128x32_tt) {
|
||||
typedef cutlass::gemm::IgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<32, 32, 128>, int>
|
||||
IgemmTraits;
|
||||
run_gemm<IgemmTraits>(128, 128, 32);
|
||||
@@ -342,7 +351,7 @@ TEST(Igemm_128x32x32, igemm_128x128x32_tt) {
|
||||
|
||||
TEST(Igemm_128x32x32, igemm_256x128x32_tt) {
|
||||
typedef cutlass::gemm::IgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<32, 32, 128>, int>
|
||||
IgemmTraits;
|
||||
run_gemm<IgemmTraits>(256, 128, 32);
|
||||
|
||||
@@ -22,11 +22,11 @@
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
#include <cutlass_unit_test.h>
|
||||
#include <cutlass/gemm/gemm.h>
|
||||
#include <cutlass/gemm/igemm_traits.h>
|
||||
#include <tools/test/unit/gemm/gemm_testbed.h>
|
||||
#include <tools/test/unit/gemm/gemm.h>
|
||||
#include "cutlass_unit_test.h"
|
||||
#include "cutlass/gemm/gemm.h"
|
||||
#include "cutlass/gemm/igemm_traits.h"
|
||||
#include "tools/test/unit/gemm/gemm_testbed.h"
|
||||
#include "tools/test/unit/gemm/run_gemm.h"
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
@@ -190,6 +190,15 @@ TEST(Igemm_128x64x32, igemm_256x128x64_nn) {
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
// FIXME: This test fails in cuBLAS. I don't know why!!!
|
||||
// TEST(Igemm_128x64x32, igemm_128x64x1_tn) {
|
||||
// typedef cutlass::gemm::IgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
// cutlass::MatrixLayout::kColumnMajor, cutlass::Shape<32, 64, 128> > IgemmTraits;
|
||||
// run_gemm<IgemmTraits>(128, 64, 1);
|
||||
// }
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Igemm_128x64x32, igemm_128x64x4_tn) {
|
||||
typedef cutlass::gemm::IgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
|
||||
@@ -22,10 +22,10 @@
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
#include <cutlass_unit_test.h>
|
||||
#include <cutlass/gemm/gemm.h>
|
||||
#include <cutlass/gemm/igemm_traits.h>
|
||||
#include <tools/test/unit/gemm/gemm.h>
|
||||
#include "cutlass_unit_test.h"
|
||||
#include "cutlass/gemm/gemm.h"
|
||||
#include "cutlass/gemm/igemm_traits.h"
|
||||
#include "tools/test/unit/gemm/run_gemm.h"
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
|
||||
@@ -23,57 +23,54 @@
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
#include <cutlass/cutlass.h>
|
||||
#include <tools/test/unit/gemm/gemm_testbed.h>
|
||||
#include "cutlass/cutlass.h"
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Helper Function to get the number of elements in the scalar.
|
||||
template <typename T>
|
||||
unsigned getElementsPerScalar() { return 1; }
|
||||
|
||||
template<>
|
||||
unsigned getElementsPerScalar<cutlass::Vector<cutlass::int4_t, 8> >() { return 8; }
|
||||
|
||||
template<>
|
||||
unsigned getElementsPerScalar<cutlass::Vector<cutlass::uint4_t, 8> >() { return 8; }
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Function to run GEMM for integer operands
|
||||
template <typename GemmTraits_>
|
||||
static void run_gemm(
|
||||
int m,
|
||||
int n,
|
||||
int k,
|
||||
int lda,
|
||||
int ldb,
|
||||
int ldc,
|
||||
typename test::GemmTestbedTraits<typename GemmTraits_::Epilogue::Scalar>::host_type alpha =
|
||||
typename test::GemmTestbedTraits<typename GemmTraits_::Epilogue::Scalar>::host_type(1),
|
||||
typename test::GemmTestbedTraits<typename GemmTraits_::Epilogue::Scalar>::host_type beta =
|
||||
typename test::GemmTestbedTraits<typename GemmTraits_::Epilogue::Scalar>::host_type(0)) {
|
||||
static void run_integer_gemm(int m, int n, int k, int alpha = 1, int beta = 1) {
|
||||
typedef cutlass::gemm::Gemm<GemmTraits_> Gemm;
|
||||
typename Gemm::Params params;
|
||||
|
||||
test::GemmTestbed<
|
||||
typename test::GemmTestbedTraits<
|
||||
typename GemmTraits_::GemmConfig::ScalarA>::host_type, // AType
|
||||
typename test::GemmTestbedTraits<
|
||||
typename GemmTraits_::GemmConfig::ScalarB>::host_type, // BType
|
||||
typename test::GemmTestbedTraits<
|
||||
typename GemmTraits_::Epilogue::ScalarC>::host_type, // CType
|
||||
typename test::GemmTestbedTraits<
|
||||
typename GemmTraits_::Epilogue::Accumulators::Element>::host_type, // Accumulator
|
||||
typename test::GemmTestbedTraits<typename GemmTraits_::Epilogue::Scalar>::host_type // Scalar
|
||||
>
|
||||
unsigned const elementsPerScalar =
|
||||
getElementsPerScalar<typename GemmTraits_::GemmConfig::ScalarA>();
|
||||
|
||||
test::GemmTestbed<typename GemmTraits_::GemmConfig::ScalarA, // AType
|
||||
typename GemmTraits_::GemmConfig::ScalarB, // BType
|
||||
int, // CType
|
||||
int, // Accumulator
|
||||
int // Scalar
|
||||
>
|
||||
testbed(m,
|
||||
n,
|
||||
k,
|
||||
lda,
|
||||
ldb,
|
||||
ldc,
|
||||
cutlass::convert(GemmTraits_::kLayoutA),
|
||||
cutlass::convert(GemmTraits_::kLayoutB),
|
||||
k / elementsPerScalar,
|
||||
test::convert(GemmTraits_::kLayoutA),
|
||||
test::convert(GemmTraits_::kLayoutB),
|
||||
alpha,
|
||||
beta);
|
||||
|
||||
testbed.initialize();
|
||||
// Initializes the input vectors for computation FIXME
|
||||
testbed.initialize_integer();
|
||||
|
||||
if (testbed.has_cublas_support()) {
|
||||
EXPECT_TRUE(testbed.verify_host_with_cublas());
|
||||
}
|
||||
// Compute the reference result on the host (CPU)
|
||||
testbed.compute_host();
|
||||
|
||||
params.initialize(testbed.M(),
|
||||
testbed.N(),
|
||||
testbed.K(),
|
||||
testbed.K() * elementsPerScalar,
|
||||
testbed.alpha,
|
||||
testbed.ptr_A(),
|
||||
testbed.lda(),
|
||||
@@ -91,28 +88,8 @@ static void run_gemm(
|
||||
ASSERT_EQ(result, cudaSuccess) << "\nCUDA kernel launch error: " << cudaGetErrorString(result)
|
||||
<< "\n";
|
||||
|
||||
if (testbed.has_cublas_support()) {
|
||||
ASSERT_TRUE(testbed.verify_with_cublas());
|
||||
} else {
|
||||
ASSERT_TRUE(testbed.verify_with_host());
|
||||
}
|
||||
testbed.computed.sync_host();
|
||||
|
||||
// Check the results
|
||||
ASSERT_TRUE(testbed.computed.bit_equals(testbed.ref_host));
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
template <typename GemmTraits_>
|
||||
static void run_gemm(
|
||||
int m,
|
||||
int n,
|
||||
int k,
|
||||
typename test::GemmTestbedTraits<typename GemmTraits_::Epilogue::Scalar>::host_type alpha =
|
||||
typename test::GemmTestbedTraits<typename GemmTraits_::Epilogue::Scalar>::host_type(1),
|
||||
typename test::GemmTestbedTraits<typename GemmTraits_::Epilogue::Scalar>::host_type beta =
|
||||
typename test::GemmTestbedTraits<typename GemmTraits_::Epilogue::Scalar>::host_type(0)) {
|
||||
int lda = GemmTraits_::kLayoutA == cutlass::MatrixLayout::kColumnMajor ? m : k;
|
||||
int ldb = GemmTraits_::kLayoutB == cutlass::MatrixLayout::kColumnMajor ? k : n;
|
||||
|
||||
run_gemm<GemmTraits_>(m, n, k, lda, ldb, m, alpha, beta);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
@@ -0,0 +1,244 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "tools/test/unit/gemm/gemm_testbed.h"
|
||||
template <typename GemmTraits_>
|
||||
static void run_gemm(
|
||||
int m,
|
||||
int n,
|
||||
int k,
|
||||
int lda,
|
||||
int ldb,
|
||||
int ldc,
|
||||
typename test::GemmTestbedTraits<typename GemmTraits_::Epilogue::Scalar>::host_type alpha =
|
||||
typename test::GemmTestbedTraits<typename GemmTraits_::Epilogue::Scalar>::host_type(1),
|
||||
typename test::GemmTestbedTraits<typename GemmTraits_::Epilogue::Scalar>::host_type beta =
|
||||
typename test::GemmTestbedTraits<typename GemmTraits_::Epilogue::Scalar>::host_type(0)) {
|
||||
|
||||
typedef typename GemmTraits_::KernelClass Gemm;
|
||||
typename Gemm::Params params;
|
||||
|
||||
test::GemmTestbed<
|
||||
typename test::GemmTestbedTraits<
|
||||
typename GemmTraits_::GemmConfig::ScalarA>::host_type, // AType
|
||||
typename test::GemmTestbedTraits<
|
||||
typename GemmTraits_::GemmConfig::ScalarB>::host_type, // BType
|
||||
typename test::GemmTestbedTraits<
|
||||
typename GemmTraits_::Epilogue::ScalarC>::host_type, // CType
|
||||
typename test::GemmTestbedTraits<
|
||||
typename GemmTraits_::Epilogue::Accumulators::Element>::host_type, // Accumulator
|
||||
typename test::GemmTestbedTraits<typename GemmTraits_::Epilogue::Scalar>::host_type // Scalar
|
||||
>
|
||||
testbed(m,
|
||||
n,
|
||||
k,
|
||||
lda,
|
||||
ldb,
|
||||
ldc,
|
||||
test::convert(GemmTraits_::kLayoutA),
|
||||
test::convert(GemmTraits_::kLayoutB),
|
||||
alpha,
|
||||
beta);
|
||||
|
||||
testbed.initialize();
|
||||
|
||||
if (testbed.has_cublas_support()) {
|
||||
EXPECT_TRUE(testbed.verify_host_with_cublas());
|
||||
}
|
||||
|
||||
params.initialize(testbed.M(),
|
||||
testbed.N(),
|
||||
testbed.K(),
|
||||
testbed.alpha,
|
||||
testbed.ptr_A(),
|
||||
testbed.lda(),
|
||||
testbed.ptr_B(),
|
||||
testbed.ldb(),
|
||||
testbed.beta,
|
||||
testbed.ptr_C_initial(),
|
||||
testbed.ldc(),
|
||||
testbed.ptr_computed(),
|
||||
testbed.ldc());
|
||||
|
||||
Gemm::launch(params);
|
||||
|
||||
cudaError_t result = cudaDeviceSynchronize();
|
||||
ASSERT_EQ(result, cudaSuccess) << "\nCUDA kernel launch error: " << cudaGetErrorString(result)
|
||||
<< "\n";
|
||||
|
||||
if (testbed.has_cublas_support()) {
|
||||
ASSERT_TRUE(testbed.verify_with_cublas());
|
||||
} else {
|
||||
ASSERT_TRUE(testbed.verify_with_host());
|
||||
}
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
template <typename GemmTraits_>
|
||||
static void run_gemm(
|
||||
int m,
|
||||
int n,
|
||||
int k,
|
||||
typename test::GemmTestbedTraits<typename GemmTraits_::Epilogue::Scalar>::host_type alpha =
|
||||
typename test::GemmTestbedTraits<typename GemmTraits_::Epilogue::Scalar>::host_type(1.0f),
|
||||
typename test::GemmTestbedTraits<typename GemmTraits_::Epilogue::Scalar>::host_type beta =
|
||||
typename test::GemmTestbedTraits<typename GemmTraits_::Epilogue::Scalar>::host_type(0.0f)) {
|
||||
//typedef cutlass::gemm::Gemm<GemmTraits_> Gemm;
|
||||
typedef typename GemmTraits_::KernelClass Gemm;
|
||||
typename Gemm::Params params;
|
||||
|
||||
typedef test::GemmTestbed<
|
||||
typename test::GemmTestbedTraits<
|
||||
typename GemmTraits_::GemmConfig::ScalarA>::host_type, // AType
|
||||
typename test::GemmTestbedTraits<
|
||||
typename GemmTraits_::GemmConfig::ScalarB>::host_type, // BType
|
||||
typename test::GemmTestbedTraits<
|
||||
typename GemmTraits_::Epilogue::ScalarC>::host_type, // CType
|
||||
typename test::GemmTestbedTraits<
|
||||
typename GemmTraits_::Epilogue::Accumulators::Element>::host_type, // Accumulator
|
||||
typename test::GemmTestbedTraits<typename GemmTraits_::Epilogue::Scalar>::host_type // Scalar
|
||||
> GemmTestbed;
|
||||
|
||||
GemmTestbed testbed(m,
|
||||
n,
|
||||
k,
|
||||
test::convert(GemmTraits_::kLayoutA),
|
||||
test::convert(GemmTraits_::kLayoutB),
|
||||
alpha,
|
||||
beta);
|
||||
|
||||
testbed.initialize();
|
||||
|
||||
if (testbed.has_cublas_support()) {
|
||||
EXPECT_TRUE(testbed.verify_host_with_cublas());
|
||||
}
|
||||
|
||||
params.initialize(testbed.M(),
|
||||
testbed.N(),
|
||||
testbed.K(),
|
||||
testbed.alpha,
|
||||
testbed.ptr_A(),
|
||||
testbed.lda(),
|
||||
testbed.ptr_B(),
|
||||
testbed.ldb(),
|
||||
testbed.beta,
|
||||
testbed.ptr_C_initial(),
|
||||
testbed.ldc(),
|
||||
testbed.ptr_computed(),
|
||||
testbed.ldc());
|
||||
|
||||
Gemm::launch(params);
|
||||
|
||||
cudaError_t result = cudaDeviceSynchronize();
|
||||
ASSERT_EQ(result, cudaSuccess) << "\nCUDA kernel launch error: " << cudaGetErrorString(result)
|
||||
<< "\n";
|
||||
|
||||
if (testbed.has_cublas_support()) {
|
||||
ASSERT_TRUE(testbed.verify_with_cublas());
|
||||
} else {
|
||||
ASSERT_TRUE(testbed.verify_with_host());
|
||||
}
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
template <typename GemmTraits_>
|
||||
static void run_batched_strided_gemm(
|
||||
int m,
|
||||
int n,
|
||||
int k,
|
||||
int batch_count,
|
||||
typename test::GemmTestbedTraits<typename GemmTraits_::Epilogue::Scalar>::host_type alpha =
|
||||
typename test::GemmTestbedTraits<typename GemmTraits_::Epilogue::Scalar>::host_type(1),
|
||||
typename test::GemmTestbedTraits<typename GemmTraits_::Epilogue::Scalar>::host_type beta =
|
||||
typename test::GemmTestbedTraits<typename GemmTraits_::Epilogue::Scalar>::host_type(0)) {
|
||||
//typedef cutlass::gemm::Gemm<GemmTraits_> Gemm;
|
||||
typedef typename GemmTraits_::KernelClass Gemm;
|
||||
typename Gemm::Params params;
|
||||
test::GemmTestbed<
|
||||
typename test::GemmTestbedTraits<
|
||||
typename GemmTraits_::GemmConfig::ScalarA>::host_type, // AType
|
||||
typename test::GemmTestbedTraits<
|
||||
typename GemmTraits_::GemmConfig::ScalarB>::host_type, // BType
|
||||
typename test::GemmTestbedTraits<
|
||||
typename GemmTraits_::Epilogue::ScalarC>::host_type, // CType
|
||||
typename test::GemmTestbedTraits<
|
||||
typename GemmTraits_::Epilogue::Accumulators::Element>::host_type, // Accumulator
|
||||
typename test::GemmTestbedTraits<typename GemmTraits_::Epilogue::Scalar>::host_type // Scalar
|
||||
>
|
||||
testbed(m,
|
||||
n,
|
||||
k,
|
||||
batch_count,
|
||||
test::convert(GemmTraits_::kLayoutA),
|
||||
test::convert(GemmTraits_::kLayoutB),
|
||||
alpha,
|
||||
beta);
|
||||
|
||||
testbed.initialize();
|
||||
|
||||
// host support is not implemented for strided batched gemm
|
||||
// if (testbed.has_cublas_support()) {
|
||||
// EXPECT_TRUE(testbed.verify_host_with_cublas());
|
||||
//}
|
||||
|
||||
params.initialize(testbed.M(),
|
||||
testbed.N(),
|
||||
testbed.K(),
|
||||
testbed.alpha,
|
||||
testbed.ptr_A(),
|
||||
testbed.lda(),
|
||||
testbed.get_batch_stride_A(),
|
||||
testbed.ptr_B(),
|
||||
testbed.ldb(),
|
||||
testbed.get_batch_stride_B(),
|
||||
testbed.beta,
|
||||
testbed.ptr_C_initial(),
|
||||
testbed.ldc(),
|
||||
testbed.get_batch_stride_C(),
|
||||
testbed.ptr_computed(),
|
||||
testbed.ldc(),
|
||||
testbed.get_batch_stride_C(),
|
||||
testbed.get_batch_count());
|
||||
|
||||
Gemm::launch(params);
|
||||
|
||||
cudaError_t result = cudaDeviceSynchronize();
|
||||
ASSERT_EQ(result, cudaSuccess) << "\nCUDA kernel launch error: " << cudaGetErrorString(result)
|
||||
<< "\n";
|
||||
|
||||
if (testbed.has_cublas_support()) {
|
||||
ASSERT_TRUE(testbed.verify_with_cublas());
|
||||
} else {
|
||||
// ASSERT_TRUE(testbed.verify_with_host());
|
||||
ASSERT_TRUE(false) << "host support is not implemented for strided batched gemm" << std::endl;
|
||||
}
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
@@ -22,16 +22,16 @@
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
#include <cutlass_unit_test.h>
|
||||
#include <cutlass/gemm/gemm.h>
|
||||
#include <cutlass/gemm/sgemm_traits.h>
|
||||
#include <tools/test/unit/gemm/gemm_testbed.h>
|
||||
#include <tools/test/unit/gemm/gemm.h>
|
||||
#include "cutlass_unit_test.h"
|
||||
#include "cutlass/gemm/gemm.h"
|
||||
#include "cutlass/gemm/sgemm_traits.h"
|
||||
#include "tools/test/unit/gemm/gemm_testbed.h"
|
||||
#include "tools/test/unit/gemm/run_gemm.h"
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_128x128x16, sgemm_128x128x16_nt) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
typedef cutlass::gemm::SgemmLBTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor, cutlass::Shape<16, 128, 128> >
|
||||
SgemmTraits;
|
||||
run_gemm<SgemmTraits>(128, 128, 16);
|
||||
@@ -40,7 +40,7 @@ TEST(Sgemm_128x128x16, sgemm_128x128x16_nt) {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_128x128x16, sgemm_128x81x1_nt) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
typedef cutlass::gemm::SgemmLBTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor, cutlass::Shape<16, 128, 128> >
|
||||
SgemmTraits;
|
||||
run_gemm<SgemmTraits>(128, 81, 1);
|
||||
@@ -49,7 +49,7 @@ TEST(Sgemm_128x128x16, sgemm_128x81x1_nt) {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_128x128x16, sgemm_128x112x16_nt) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
typedef cutlass::gemm::SgemmLBTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor, cutlass::Shape<16, 128, 128> >
|
||||
SgemmTraits;
|
||||
run_gemm<SgemmTraits>(128, 112, 16);
|
||||
@@ -58,7 +58,7 @@ TEST(Sgemm_128x128x16, sgemm_128x112x16_nt) {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_128x128x16, sgemm_128x112x17_nt) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
typedef cutlass::gemm::SgemmLBTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor, cutlass::Shape<16, 128, 128> >
|
||||
SgemmTraits;
|
||||
run_gemm<SgemmTraits>(128, 112, 17);
|
||||
@@ -67,7 +67,7 @@ TEST(Sgemm_128x128x16, sgemm_128x112x17_nt) {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_128x128x16, sgemm_128x73x16_nt) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
typedef cutlass::gemm::SgemmLBTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor, cutlass::Shape<16, 128, 128> >
|
||||
SgemmTraits;
|
||||
run_gemm<SgemmTraits>(128, 73, 16);
|
||||
@@ -76,7 +76,7 @@ TEST(Sgemm_128x128x16, sgemm_128x73x16_nt) {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_128x128x16, sgemm_97x112x64_nt) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
typedef cutlass::gemm::SgemmLBTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor, cutlass::Shape<16, 128, 128> >
|
||||
SgemmTraits;
|
||||
run_gemm<SgemmTraits>(97, 112, 64);
|
||||
@@ -85,7 +85,7 @@ TEST(Sgemm_128x128x16, sgemm_97x112x64_nt) {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_128x128x16, sgemm_256x112x16_nt) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
typedef cutlass::gemm::SgemmLBTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor, cutlass::Shape<16, 128, 128> >
|
||||
SgemmTraits;
|
||||
run_gemm<SgemmTraits>(256, 112, 16);
|
||||
@@ -94,7 +94,7 @@ TEST(Sgemm_128x128x16, sgemm_256x112x16_nt) {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_128x128x16, sgemm_128x240x16_nt) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
typedef cutlass::gemm::SgemmLBTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor, cutlass::Shape<16, 128, 128> >
|
||||
SgemmTraits;
|
||||
run_gemm<SgemmTraits>(128, 240, 16);
|
||||
@@ -103,7 +103,7 @@ TEST(Sgemm_128x128x16, sgemm_128x240x16_nt) {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_128x128x16, sgemm_256x240x16_nt) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
typedef cutlass::gemm::SgemmLBTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor, cutlass::Shape<16, 128, 128> >
|
||||
SgemmTraits;
|
||||
run_gemm<SgemmTraits>(256, 240, 16);
|
||||
@@ -112,7 +112,7 @@ TEST(Sgemm_128x128x16, sgemm_256x240x16_nt) {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_128x128x16, sgemm_128x128x16_nn) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
typedef cutlass::gemm::SgemmLBTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor, cutlass::Shape<16, 128, 128> >
|
||||
SgemmTraits;
|
||||
run_gemm<SgemmTraits>(128, 128, 16);
|
||||
@@ -121,7 +121,7 @@ TEST(Sgemm_128x128x16, sgemm_128x128x16_nn) {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_128x128x16, sgemm_128x112x1_nn) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
typedef cutlass::gemm::SgemmLBTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::Shape<16, 128, 128> >
|
||||
SgemmTraits;
|
||||
@@ -131,7 +131,7 @@ TEST(Sgemm_128x128x16, sgemm_128x112x1_nn) {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_128x128x16, sgemm_79x112x16_nn) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
typedef cutlass::gemm::SgemmLBTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::Shape<16, 128, 128> >
|
||||
SgemmTraits;
|
||||
@@ -141,7 +141,7 @@ TEST(Sgemm_128x128x16, sgemm_79x112x16_nn) {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_128x128x16, sgemm_128x81x17_nn) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
typedef cutlass::gemm::SgemmLBTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::Shape<16, 128, 128> >
|
||||
SgemmTraits;
|
||||
@@ -151,7 +151,7 @@ TEST(Sgemm_128x128x16, sgemm_128x81x17_nn) {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_128x128x16, sgemm_128x112x16_nn) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
typedef cutlass::gemm::SgemmLBTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::Shape<16, 128, 128> >
|
||||
SgemmTraits;
|
||||
@@ -161,7 +161,7 @@ TEST(Sgemm_128x128x16, sgemm_128x112x16_nn) {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_128x128x16, sgemm_128x73x64_nn) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
typedef cutlass::gemm::SgemmLBTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::Shape<16, 128, 128> >
|
||||
SgemmTraits;
|
||||
@@ -171,7 +171,7 @@ TEST(Sgemm_128x128x16, sgemm_128x73x64_nn) {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_128x128x16, sgemm_256x112x16_nn) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
typedef cutlass::gemm::SgemmLBTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::Shape<16, 128, 128> >
|
||||
SgemmTraits;
|
||||
@@ -181,7 +181,7 @@ TEST(Sgemm_128x128x16, sgemm_256x112x16_nn) {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_128x128x16, sgemm_128x256x16_nn) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
typedef cutlass::gemm::SgemmLBTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::Shape<16, 128, 128> >
|
||||
SgemmTraits;
|
||||
@@ -191,7 +191,7 @@ TEST(Sgemm_128x128x16, sgemm_128x256x16_nn) {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_128x128x16, sgemm_256x256x16_nn) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
typedef cutlass::gemm::SgemmLBTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::Shape<16, 128, 128> >
|
||||
SgemmTraits;
|
||||
@@ -201,7 +201,7 @@ TEST(Sgemm_128x128x16, sgemm_256x256x16_nn) {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_128x128x16, sgemm_128x128x16_tn) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
typedef cutlass::gemm::SgemmLBTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor, cutlass::Shape<16, 128, 128> >
|
||||
SgemmTraits;
|
||||
run_gemm<SgemmTraits>(128, 128, 16);
|
||||
@@ -210,7 +210,7 @@ TEST(Sgemm_128x128x16, sgemm_128x128x16_tn) {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_128x128x16, sgemm_128x128x1_tn) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
typedef cutlass::gemm::SgemmLBTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor, cutlass::Shape<16, 128, 128> > SgemmTraits;
|
||||
run_gemm<SgemmTraits>(128, 128, 1);
|
||||
}
|
||||
@@ -218,7 +218,7 @@ TEST(Sgemm_128x128x16, sgemm_128x128x1_tn) {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_128x128x16, sgemm_127x112x16_tn) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
typedef cutlass::gemm::SgemmLBTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::Shape<16, 128, 128> >
|
||||
SgemmTraits;
|
||||
@@ -228,7 +228,7 @@ TEST(Sgemm_128x128x16, sgemm_127x112x16_tn) {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_128x128x16, sgemm_21x112x17_tn) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
typedef cutlass::gemm::SgemmLBTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::Shape<16, 128, 128> >
|
||||
SgemmTraits;
|
||||
@@ -238,7 +238,7 @@ TEST(Sgemm_128x128x16, sgemm_21x112x17_tn) {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_128x128x16, sgemm_128x73x16_tn) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
typedef cutlass::gemm::SgemmLBTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::Shape<16, 128, 128> >
|
||||
SgemmTraits;
|
||||
@@ -248,7 +248,7 @@ TEST(Sgemm_128x128x16, sgemm_128x73x16_tn) {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_128x128x16, sgemm_128x81x64_tn) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
typedef cutlass::gemm::SgemmLBTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::Shape<16, 128, 128> >
|
||||
SgemmTraits;
|
||||
@@ -258,7 +258,7 @@ TEST(Sgemm_128x128x16, sgemm_128x81x64_tn) {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_128x128x16, sgemm_256x112x16_tn) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
typedef cutlass::gemm::SgemmLBTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::Shape<16, 128, 128> >
|
||||
SgemmTraits;
|
||||
@@ -268,7 +268,7 @@ TEST(Sgemm_128x128x16, sgemm_256x112x16_tn) {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_128x128x16, sgemm_47x256x16_tn) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
typedef cutlass::gemm::SgemmLBTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::Shape<16, 128, 128> >
|
||||
SgemmTraits;
|
||||
@@ -278,7 +278,7 @@ TEST(Sgemm_128x128x16, sgemm_47x256x16_tn) {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_128x128x16, sgemm_211x256x16_tn) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
typedef cutlass::gemm::SgemmLBTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::Shape<16, 128, 128> >
|
||||
SgemmTraits;
|
||||
@@ -288,7 +288,7 @@ TEST(Sgemm_128x128x16, sgemm_211x256x16_tn) {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_128x128x16, sgemm_128x128x16_tt) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
typedef cutlass::gemm::SgemmLBTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor, cutlass::Shape<16, 128, 128> >
|
||||
SgemmTraits;
|
||||
run_gemm<SgemmTraits>(128, 128, 16);
|
||||
@@ -297,7 +297,7 @@ TEST(Sgemm_128x128x16, sgemm_128x128x16_tt) {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_128x128x16, sgemm_128x128x1_tt) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
typedef cutlass::gemm::SgemmLBTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor, cutlass::Shape<16, 128, 128> > SgemmTraits;
|
||||
run_gemm<SgemmTraits>(128, 128, 1);
|
||||
}
|
||||
@@ -305,7 +305,7 @@ TEST(Sgemm_128x128x16, sgemm_128x128x1_tt) {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_128x128x16, sgemm_109x112x16_tt) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
typedef cutlass::gemm::SgemmLBTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor, cutlass::Shape<16, 128, 128> >
|
||||
SgemmTraits;
|
||||
run_gemm<SgemmTraits>(109, 112, 16);
|
||||
@@ -314,7 +314,7 @@ TEST(Sgemm_128x128x16, sgemm_109x112x16_tt) {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_128x128x16, sgemm_128x112x17_tt) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
typedef cutlass::gemm::SgemmLBTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor, cutlass::Shape<16, 128, 128> >
|
||||
SgemmTraits;
|
||||
run_gemm<SgemmTraits>(128, 112, 17);
|
||||
@@ -323,7 +323,7 @@ TEST(Sgemm_128x128x16, sgemm_128x112x17_tt) {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_128x128x16, sgemm_128x112x16_tt) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
typedef cutlass::gemm::SgemmLBTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor, cutlass::Shape<16, 128, 128> >
|
||||
SgemmTraits;
|
||||
run_gemm<SgemmTraits>(128, 112, 16);
|
||||
@@ -332,7 +332,7 @@ TEST(Sgemm_128x128x16, sgemm_128x112x16_tt) {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_128x128x16, sgemm_123x112x64_tt) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
typedef cutlass::gemm::SgemmLBTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor, cutlass::Shape<16, 128, 128> >
|
||||
SgemmTraits;
|
||||
run_gemm<SgemmTraits>(123, 112, 64);
|
||||
@@ -341,7 +341,7 @@ TEST(Sgemm_128x128x16, sgemm_123x112x64_tt) {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_128x128x16, sgemm_256x112x16_tt) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
typedef cutlass::gemm::SgemmLBTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor, cutlass::Shape<16, 128, 128> >
|
||||
SgemmTraits;
|
||||
run_gemm<SgemmTraits>(256, 112, 16);
|
||||
@@ -350,7 +350,7 @@ TEST(Sgemm_128x128x16, sgemm_256x112x16_tt) {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_128x128x16, sgemm_128x256x16_tt) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
typedef cutlass::gemm::SgemmLBTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor, cutlass::Shape<16, 128, 128> >
|
||||
SgemmTraits;
|
||||
run_gemm<SgemmTraits>(128, 256, 16);
|
||||
@@ -359,7 +359,7 @@ TEST(Sgemm_128x128x16, sgemm_128x256x16_tt) {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_128x128x16, sgemm_256x256x16_tt) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
typedef cutlass::gemm::SgemmLBTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor, cutlass::Shape<16, 128, 128> >
|
||||
SgemmTraits;
|
||||
run_gemm<SgemmTraits>(256, 256, 16);
|
||||
@@ -369,13 +369,12 @@ TEST(Sgemm_128x128x16, sgemm_256x256x16_tt) {
|
||||
|
||||
TEST(Sgemm_128x128x16, sgemm_120x112x64_ldg4_nt) {
|
||||
// Load 4 floats per LDG for A/B.
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
typedef cutlass::gemm::SgemmLBTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<16, 128, 128>,
|
||||
cutlass::gemm::LinearScaling<float>,
|
||||
cutlass::Shape<8, 8, 8>,
|
||||
4,
|
||||
4>
|
||||
4, 4>
|
||||
SgemmTraits;
|
||||
run_gemm<SgemmTraits>(120, 112, 64);
|
||||
}
|
||||
@@ -383,7 +382,7 @@ TEST(Sgemm_128x128x16, sgemm_120x112x64_ldg4_nt) {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_128x128x16, sgemm_128x128x16_alpha2_nt) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
typedef cutlass::gemm::SgemmLBTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor, cutlass::Shape<16, 128, 128> >
|
||||
SgemmTraits;
|
||||
run_gemm<SgemmTraits>(128, 128, 16, 2.f, 0.f);
|
||||
@@ -392,7 +391,7 @@ TEST(Sgemm_128x128x16, sgemm_128x128x16_alpha2_nt) {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_128x128x16, sgemm_128x112x16_beta1_nt) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
typedef cutlass::gemm::SgemmLBTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor, cutlass::Shape<16, 128, 128> >
|
||||
SgemmTraits;
|
||||
run_gemm<SgemmTraits>(128, 112, 16, 1.f, 1.f);
|
||||
@@ -401,7 +400,7 @@ TEST(Sgemm_128x128x16, sgemm_128x112x16_beta1_nt) {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_128x128x16, sgemm_128x112x16_alpha2_beta1_nt) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
typedef cutlass::gemm::SgemmLBTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor, cutlass::Shape<16, 128, 128> >
|
||||
SgemmTraits;
|
||||
run_gemm<SgemmTraits>(128, 112, 16, 2.f, 1.f);
|
||||
|
||||
@@ -22,11 +22,20 @@
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
#include <cutlass_unit_test.h>
|
||||
#include <cutlass/gemm/gemm.h>
|
||||
#include <cutlass/gemm/sgemm_traits.h>
|
||||
#include <tools/test/unit/gemm/gemm_testbed.h>
|
||||
#include <tools/test/unit/gemm/gemm.h>
|
||||
#include "cutlass_unit_test.h"
|
||||
#include "cutlass/gemm/gemm.h"
|
||||
#include "cutlass/gemm/sgemm_traits.h"
|
||||
#include "tools/test/unit/gemm/gemm_testbed.h"
|
||||
#include "tools/test/unit/gemm/run_gemm.h"
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_128x128x8, sgemm_1024x512x8_nt) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor, cutlass::Shape<8, 128, 128> >
|
||||
SgemmTraits;
|
||||
run_gemm<SgemmTraits>(1024, 512, 8);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
|
||||
@@ -22,16 +22,16 @@
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
#include <cutlass_unit_test.h>
|
||||
#include <cutlass/gemm/gemm.h>
|
||||
#include <cutlass/gemm/sgemm_traits.h>
|
||||
#include <tools/test/unit/gemm/gemm_testbed.h>
|
||||
#include <tools/test/unit/gemm/gemm.h>
|
||||
#include "cutlass_unit_test.h"
|
||||
#include "cutlass/gemm/gemm.h"
|
||||
#include "cutlass/gemm/sgemm_traits.h"
|
||||
#include "tools/test/unit/gemm/gemm_testbed.h"
|
||||
#include "tools/test/unit/gemm/run_gemm.h"
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_128x32x16, sgemm_128x32x1_nt) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
typedef cutlass::gemm::SgemmLBTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor, cutlass::Shape<16, 32, 128> >
|
||||
SgemmTraits;
|
||||
run_gemm<SgemmTraits>(128, 32, 1);
|
||||
@@ -40,7 +40,7 @@ TEST(Sgemm_128x32x16, sgemm_128x32x1_nt) {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_128x32x16, sgemm_128x32x16_nt) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
typedef cutlass::gemm::SgemmLBTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor, cutlass::Shape<16, 32, 128> >
|
||||
SgemmTraits;
|
||||
run_gemm<SgemmTraits>(128, 32, 16);
|
||||
@@ -49,7 +49,7 @@ TEST(Sgemm_128x32x16, sgemm_128x32x16_nt) {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_128x32x16, sgemm_128x32x17_nt) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
typedef cutlass::gemm::SgemmLBTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor, cutlass::Shape<16, 32, 128> >
|
||||
SgemmTraits;
|
||||
run_gemm<SgemmTraits>(128, 32, 17);
|
||||
@@ -58,7 +58,7 @@ TEST(Sgemm_128x32x16, sgemm_128x32x17_nt) {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_128x32x16, sgemm_128x32x32_nt) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
typedef cutlass::gemm::SgemmLBTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor, cutlass::Shape<16, 32, 128> >
|
||||
SgemmTraits;
|
||||
run_gemm<SgemmTraits>(128, 32, 32);
|
||||
@@ -67,7 +67,7 @@ TEST(Sgemm_128x32x16, sgemm_128x32x32_nt) {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_128x32x16, sgemm_256x32x16_nt) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
typedef cutlass::gemm::SgemmLBTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor, cutlass::Shape<16, 32, 128> >
|
||||
SgemmTraits;
|
||||
run_gemm<SgemmTraits>(256, 32, 16);
|
||||
@@ -76,7 +76,7 @@ TEST(Sgemm_128x32x16, sgemm_256x32x16_nt) {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_128x32x16, sgemm_128x64x16_nt) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
typedef cutlass::gemm::SgemmLBTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor, cutlass::Shape<16, 32, 128> >
|
||||
SgemmTraits;
|
||||
run_gemm<SgemmTraits>(128, 64, 16);
|
||||
@@ -85,7 +85,7 @@ TEST(Sgemm_128x32x16, sgemm_128x64x16_nt) {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_128x32x16, sgemm_256x64x16_nt) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
typedef cutlass::gemm::SgemmLBTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor, cutlass::Shape<16, 32, 128> >
|
||||
SgemmTraits;
|
||||
run_gemm<SgemmTraits>(256, 64, 16);
|
||||
@@ -94,7 +94,7 @@ TEST(Sgemm_128x32x16, sgemm_256x64x16_nt) {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_128x32x16, sgemm_128x32x1_nn) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
typedef cutlass::gemm::SgemmLBTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::Shape<16, 32, 128> >
|
||||
SgemmTraits;
|
||||
@@ -104,7 +104,7 @@ TEST(Sgemm_128x32x16, sgemm_128x32x1_nn) {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_128x32x16, sgemm_128x32x16_nn) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
typedef cutlass::gemm::SgemmLBTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::Shape<16, 32, 128> >
|
||||
SgemmTraits;
|
||||
@@ -114,7 +114,7 @@ TEST(Sgemm_128x32x16, sgemm_128x32x16_nn) {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_128x32x16, sgemm_128x32x17_nn) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
typedef cutlass::gemm::SgemmLBTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::Shape<16, 32, 128> >
|
||||
SgemmTraits;
|
||||
@@ -124,7 +124,7 @@ TEST(Sgemm_128x32x16, sgemm_128x32x17_nn) {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_128x32x16, sgemm_128x32x32_nn) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
typedef cutlass::gemm::SgemmLBTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::Shape<16, 32, 128> >
|
||||
SgemmTraits;
|
||||
@@ -134,7 +134,7 @@ TEST(Sgemm_128x32x16, sgemm_128x32x32_nn) {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_128x32x16, sgemm_256x32x16_nn) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
typedef cutlass::gemm::SgemmLBTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::Shape<16, 32, 128> >
|
||||
SgemmTraits;
|
||||
@@ -144,7 +144,7 @@ TEST(Sgemm_128x32x16, sgemm_256x32x16_nn) {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_128x32x16, sgemm_128x64x16_nn) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
typedef cutlass::gemm::SgemmLBTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::Shape<16, 32, 128> >
|
||||
SgemmTraits;
|
||||
@@ -154,7 +154,7 @@ TEST(Sgemm_128x32x16, sgemm_128x64x16_nn) {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_128x32x16, sgemm_256x64x16_nn) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
typedef cutlass::gemm::SgemmLBTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::Shape<16, 32, 128> >
|
||||
SgemmTraits;
|
||||
@@ -164,7 +164,7 @@ TEST(Sgemm_128x32x16, sgemm_256x64x16_nn) {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_128x32x16, sgemm_128x32x1_tn) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
typedef cutlass::gemm::SgemmLBTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor, cutlass::Shape<16, 128, 128> > SgemmTraits;
|
||||
run_gemm<SgemmTraits>(128, 128, 1);
|
||||
}
|
||||
@@ -172,7 +172,7 @@ TEST(Sgemm_128x32x16, sgemm_128x32x1_tn) {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_128x32x16, sgemm_128x32x16_tn) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
typedef cutlass::gemm::SgemmLBTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::Shape<16, 32, 128> >
|
||||
SgemmTraits;
|
||||
@@ -182,7 +182,7 @@ TEST(Sgemm_128x32x16, sgemm_128x32x16_tn) {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_128x32x16, sgemm_128x32x17_tn) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
typedef cutlass::gemm::SgemmLBTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::Shape<16, 32, 128> >
|
||||
SgemmTraits;
|
||||
@@ -192,7 +192,7 @@ TEST(Sgemm_128x32x16, sgemm_128x32x17_tn) {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_128x32x16, sgemm_128x32x32_tn) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
typedef cutlass::gemm::SgemmLBTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::Shape<16, 32, 128> >
|
||||
SgemmTraits;
|
||||
@@ -202,7 +202,7 @@ TEST(Sgemm_128x32x16, sgemm_128x32x32_tn) {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_128x32x16, sgemm_256x32x16_tn) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
typedef cutlass::gemm::SgemmLBTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::Shape<16, 32, 128> >
|
||||
SgemmTraits;
|
||||
@@ -212,7 +212,7 @@ TEST(Sgemm_128x32x16, sgemm_256x32x16_tn) {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_128x32x16, sgemm_128x64x16_tn) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
typedef cutlass::gemm::SgemmLBTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::Shape<16, 32, 128> >
|
||||
SgemmTraits;
|
||||
@@ -222,7 +222,7 @@ TEST(Sgemm_128x32x16, sgemm_128x64x16_tn) {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_128x32x16, sgemm_256x64x16_tn) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
typedef cutlass::gemm::SgemmLBTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::Shape<16, 32, 128> >
|
||||
SgemmTraits;
|
||||
@@ -232,7 +232,7 @@ TEST(Sgemm_128x32x16, sgemm_256x64x16_tn) {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_128x32x16, sgemm_128x32x1_tt) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
typedef cutlass::gemm::SgemmLBTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor, cutlass::Shape<16, 128, 128> > SgemmTraits;
|
||||
run_gemm<SgemmTraits>(128, 128, 1);
|
||||
}
|
||||
@@ -240,7 +240,7 @@ TEST(Sgemm_128x32x16, sgemm_128x32x1_tt) {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_128x32x16, sgemm_128x32x16_tt) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
typedef cutlass::gemm::SgemmLBTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor, cutlass::Shape<16, 32, 128> >
|
||||
SgemmTraits;
|
||||
run_gemm<SgemmTraits>(128, 32, 16);
|
||||
@@ -249,7 +249,7 @@ TEST(Sgemm_128x32x16, sgemm_128x32x16_tt) {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_128x32x16, sgemm_128x32x17_tt) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
typedef cutlass::gemm::SgemmLBTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor, cutlass::Shape<16, 32, 128> >
|
||||
SgemmTraits;
|
||||
run_gemm<SgemmTraits>(128, 32, 17);
|
||||
@@ -258,7 +258,7 @@ TEST(Sgemm_128x32x16, sgemm_128x32x17_tt) {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_128x32x16, sgemm_128x32x32_tt) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
typedef cutlass::gemm::SgemmLBTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor, cutlass::Shape<16, 32, 128> >
|
||||
SgemmTraits;
|
||||
run_gemm<SgemmTraits>(128, 32, 32);
|
||||
@@ -267,7 +267,7 @@ TEST(Sgemm_128x32x16, sgemm_128x32x32_tt) {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_128x32x16, sgemm_256x32x16_tt) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
typedef cutlass::gemm::SgemmLBTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor, cutlass::Shape<16, 32, 128> >
|
||||
SgemmTraits;
|
||||
run_gemm<SgemmTraits>(256, 32, 16);
|
||||
@@ -276,7 +276,7 @@ TEST(Sgemm_128x32x16, sgemm_256x32x16_tt) {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_128x32x16, sgemm_128x64x16_tt) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
typedef cutlass::gemm::SgemmLBTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor, cutlass::Shape<16, 32, 128> >
|
||||
SgemmTraits;
|
||||
run_gemm<SgemmTraits>(128, 64, 16);
|
||||
@@ -285,7 +285,7 @@ TEST(Sgemm_128x32x16, sgemm_128x64x16_tt) {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_128x32x16, sgemm_256x64x16_tt) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
typedef cutlass::gemm::SgemmLBTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor, cutlass::Shape<16, 32, 128> >
|
||||
SgemmTraits;
|
||||
run_gemm<SgemmTraits>(256, 64, 16);
|
||||
|
||||
@@ -22,11 +22,11 @@
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
#include <cutlass_unit_test.h>
|
||||
#include <cutlass/gemm/gemm.h>
|
||||
#include <cutlass/gemm/sgemm_traits.h>
|
||||
#include <tools/test/unit/gemm/gemm_testbed.h>
|
||||
#include <tools/test/unit/gemm/gemm.h>
|
||||
#include "cutlass_unit_test.h"
|
||||
#include "cutlass/gemm/gemm.h"
|
||||
#include "cutlass/gemm/sgemm_traits.h"
|
||||
#include "tools/test/unit/gemm/gemm_testbed.h"
|
||||
#include "tools/test/unit/gemm/run_gemm.h"
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
|
||||
@@ -22,16 +22,16 @@
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
#include <cutlass_unit_test.h>
|
||||
#include <cutlass/gemm/gemm.h>
|
||||
#include <cutlass/gemm/sgemm_traits.h>
|
||||
#include <tools/test/unit/gemm/gemm_testbed.h>
|
||||
#include <tools/test/unit/gemm/gemm.h>
|
||||
#include "cutlass_unit_test.h"
|
||||
#include "cutlass/gemm/gemm.h"
|
||||
#include "cutlass/gemm/sgemm_traits.h"
|
||||
#include "tools/test/unit/gemm/gemm_testbed.h"
|
||||
#include "tools/test/unit/gemm/run_gemm.h"
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_128x64x16, sgemm_128x64x1_nt) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
typedef cutlass::gemm::SgemmLBTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor, cutlass::Shape<16, 64, 128> >
|
||||
SgemmTraits;
|
||||
run_gemm<SgemmTraits>(128, 64, 1);
|
||||
@@ -40,7 +40,7 @@ TEST(Sgemm_128x64x16, sgemm_128x64x1_nt) {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_128x64x16, sgemm_128x64x16_nt) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
typedef cutlass::gemm::SgemmLBTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor, cutlass::Shape<16, 64, 128> >
|
||||
SgemmTraits;
|
||||
run_gemm<SgemmTraits>(128, 64, 16);
|
||||
@@ -49,7 +49,7 @@ TEST(Sgemm_128x64x16, sgemm_128x64x16_nt) {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_128x64x16, sgemm_128x64x17_nt) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
typedef cutlass::gemm::SgemmLBTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor, cutlass::Shape<16, 64, 128> >
|
||||
SgemmTraits;
|
||||
run_gemm<SgemmTraits>(128, 64, 17);
|
||||
@@ -58,7 +58,7 @@ TEST(Sgemm_128x64x16, sgemm_128x64x17_nt) {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_128x64x16, sgemm_128x64x64_nt) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
typedef cutlass::gemm::SgemmLBTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor, cutlass::Shape<16, 64, 128> >
|
||||
SgemmTraits;
|
||||
run_gemm<SgemmTraits>(128, 64, 64);
|
||||
@@ -67,7 +67,7 @@ TEST(Sgemm_128x64x16, sgemm_128x64x64_nt) {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_128x64x16, sgemm_256x64x16_nt) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
typedef cutlass::gemm::SgemmLBTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor, cutlass::Shape<16, 64, 128> >
|
||||
SgemmTraits;
|
||||
run_gemm<SgemmTraits>(256, 64, 16);
|
||||
@@ -76,7 +76,7 @@ TEST(Sgemm_128x64x16, sgemm_256x64x16_nt) {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_128x64x16, sgemm_128x128x16_nt) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
typedef cutlass::gemm::SgemmLBTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor, cutlass::Shape<16, 64, 128> >
|
||||
SgemmTraits;
|
||||
run_gemm<SgemmTraits>(128, 128, 16);
|
||||
@@ -85,7 +85,7 @@ TEST(Sgemm_128x64x16, sgemm_128x128x16_nt) {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_128x64x16, sgemm_256x128x16_nt) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
typedef cutlass::gemm::SgemmLBTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor, cutlass::Shape<16, 64, 128> >
|
||||
SgemmTraits;
|
||||
run_gemm<SgemmTraits>(256, 128, 16);
|
||||
@@ -94,7 +94,7 @@ TEST(Sgemm_128x64x16, sgemm_256x128x16_nt) {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_128x64x16, sgemm_128x64x1_nn) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
typedef cutlass::gemm::SgemmLBTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::Shape<16, 64, 128> >
|
||||
SgemmTraits;
|
||||
@@ -104,7 +104,7 @@ TEST(Sgemm_128x64x16, sgemm_128x64x1_nn) {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_128x64x16, sgemm_128x64x8_nn) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
typedef cutlass::gemm::SgemmLBTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::Shape<16, 64, 128> >
|
||||
SgemmTraits;
|
||||
@@ -114,7 +114,7 @@ TEST(Sgemm_128x64x16, sgemm_128x64x8_nn) {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_128x64x16, sgemm_128x64x17_nn) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
typedef cutlass::gemm::SgemmLBTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::Shape<16, 64, 128> >
|
||||
SgemmTraits;
|
||||
@@ -124,7 +124,7 @@ TEST(Sgemm_128x64x16, sgemm_128x64x17_nn) {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_128x64x16, sgemm_128x64x64_nn) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
typedef cutlass::gemm::SgemmLBTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::Shape<16, 64, 128> >
|
||||
SgemmTraits;
|
||||
@@ -134,7 +134,7 @@ TEST(Sgemm_128x64x16, sgemm_128x64x64_nn) {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_128x64x16, sgemm_256x64x16_nn) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
typedef cutlass::gemm::SgemmLBTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::Shape<16, 64, 128> >
|
||||
SgemmTraits;
|
||||
@@ -144,7 +144,7 @@ TEST(Sgemm_128x64x16, sgemm_256x64x16_nn) {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_128x64x16, sgemm_128x128x16_nn) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
typedef cutlass::gemm::SgemmLBTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::Shape<16, 64, 128> >
|
||||
SgemmTraits;
|
||||
@@ -154,7 +154,7 @@ TEST(Sgemm_128x64x16, sgemm_128x128x16_nn) {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_128x64x16, sgemm_256x128x16_nn) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
typedef cutlass::gemm::SgemmLBTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::Shape<16, 64, 128> >
|
||||
SgemmTraits;
|
||||
@@ -164,7 +164,7 @@ TEST(Sgemm_128x64x16, sgemm_256x128x16_nn) {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_128x64x16, sgemm_128x64x1_tn) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
typedef cutlass::gemm::SgemmLBTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor, cutlass::Shape<16, 128, 128> > SgemmTraits;
|
||||
run_gemm<SgemmTraits>(128, 128, 1);
|
||||
}
|
||||
@@ -172,7 +172,7 @@ TEST(Sgemm_128x64x16, sgemm_128x64x1_tn) {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_128x64x16, sgemm_128x64x16_tn) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
typedef cutlass::gemm::SgemmLBTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::Shape<16, 64, 128> >
|
||||
SgemmTraits;
|
||||
@@ -182,7 +182,7 @@ TEST(Sgemm_128x64x16, sgemm_128x64x16_tn) {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_128x64x16, sgemm_128x64x17_tn) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
typedef cutlass::gemm::SgemmLBTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::Shape<16, 64, 128> >
|
||||
SgemmTraits;
|
||||
@@ -192,7 +192,7 @@ TEST(Sgemm_128x64x16, sgemm_128x64x17_tn) {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_128x64x16, sgemm_128x64x64_tn) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
typedef cutlass::gemm::SgemmLBTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::Shape<16, 64, 128> >
|
||||
SgemmTraits;
|
||||
@@ -202,7 +202,7 @@ TEST(Sgemm_128x64x16, sgemm_128x64x64_tn) {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_128x64x16, sgemm_256x64x16_tn) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
typedef cutlass::gemm::SgemmLBTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::Shape<16, 64, 128> >
|
||||
SgemmTraits;
|
||||
@@ -212,7 +212,7 @@ TEST(Sgemm_128x64x16, sgemm_256x64x16_tn) {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_128x64x16, sgemm_128x128x16_tn) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
typedef cutlass::gemm::SgemmLBTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::Shape<16, 64, 128> >
|
||||
SgemmTraits;
|
||||
@@ -222,7 +222,7 @@ TEST(Sgemm_128x64x16, sgemm_128x128x16_tn) {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_128x64x16, sgemm_256x128x16_tn) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
typedef cutlass::gemm::SgemmLBTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::Shape<16, 64, 128> >
|
||||
SgemmTraits;
|
||||
@@ -232,7 +232,7 @@ TEST(Sgemm_128x64x16, sgemm_256x128x16_tn) {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_128x64x16, sgemm_128x64x1_tt) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
typedef cutlass::gemm::SgemmLBTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor, cutlass::Shape<16, 128, 128> > SgemmTraits;
|
||||
run_gemm<SgemmTraits>(128, 128, 1);
|
||||
}
|
||||
@@ -240,7 +240,7 @@ TEST(Sgemm_128x64x16, sgemm_128x64x1_tt) {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_128x64x16, sgemm_128x64x16_tt) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
typedef cutlass::gemm::SgemmLBTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor, cutlass::Shape<16, 64, 128> >
|
||||
SgemmTraits;
|
||||
run_gemm<SgemmTraits>(128, 64, 16);
|
||||
@@ -249,7 +249,7 @@ TEST(Sgemm_128x64x16, sgemm_128x64x16_tt) {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_128x64x16, sgemm_128x64x17_tt) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
typedef cutlass::gemm::SgemmLBTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor, cutlass::Shape<16, 64, 128> >
|
||||
SgemmTraits;
|
||||
run_gemm<SgemmTraits>(128, 64, 17);
|
||||
@@ -258,7 +258,7 @@ TEST(Sgemm_128x64x16, sgemm_128x64x17_tt) {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_128x64x16, sgemm_128x64x64_tt) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
typedef cutlass::gemm::SgemmLBTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor, cutlass::Shape<16, 64, 128> >
|
||||
SgemmTraits;
|
||||
run_gemm<SgemmTraits>(128, 64, 64);
|
||||
@@ -267,7 +267,7 @@ TEST(Sgemm_128x64x16, sgemm_128x64x64_tt) {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_128x64x16, sgemm_128x128x16_tt) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
typedef cutlass::gemm::SgemmLBTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor, cutlass::Shape<16, 64, 128> >
|
||||
SgemmTraits;
|
||||
run_gemm<SgemmTraits>(128, 128, 16);
|
||||
@@ -276,7 +276,7 @@ TEST(Sgemm_128x64x16, sgemm_128x128x16_tt) {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_128x64x16, sgemm_256x128x16_tt) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
typedef cutlass::gemm::SgemmLBTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor, cutlass::Shape<16, 64, 128> >
|
||||
SgemmTraits;
|
||||
run_gemm<SgemmTraits>(256, 128, 16);
|
||||
|
||||
@@ -22,11 +22,11 @@
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
#include <cutlass_unit_test.h>
|
||||
#include <cutlass/gemm/gemm.h>
|
||||
#include <cutlass/gemm/sgemm_traits.h>
|
||||
#include <tools/test/unit/gemm/gemm_testbed.h>
|
||||
#include <tools/test/unit/gemm/gemm.h>
|
||||
#include "cutlass_unit_test.h"
|
||||
#include "cutlass/gemm/gemm.h"
|
||||
#include "cutlass/gemm/sgemm_traits.h"
|
||||
#include "tools/test/unit/gemm/gemm_testbed.h"
|
||||
#include "tools/test/unit/gemm/run_gemm.h"
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
@@ -348,7 +348,7 @@ TEST(Sgemm_128x64x8, sgemm_128x64x64_4x8_accumulators_nt) {
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<8, 64, 128>,
|
||||
cutlass::gemm::LinearScaling<float>,
|
||||
cutlass::Shape<8, 8, 4> >
|
||||
cutlass::Shape<8, 8, 8> >
|
||||
SgemmTraits;
|
||||
run_gemm<SgemmTraits>(128, 64, 64);
|
||||
}
|
||||
|
||||
@@ -22,16 +22,16 @@
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
#include <cutlass_unit_test.h>
|
||||
#include <cutlass/gemm/gemm.h>
|
||||
#include <cutlass/gemm/sgemm_traits.h>
|
||||
#include <tools/test/unit/gemm/gemm_testbed.h>
|
||||
#include <tools/test/unit/gemm/gemm.h>
|
||||
#include "cutlass_unit_test.h"
|
||||
#include "cutlass/gemm/gemm.h"
|
||||
#include "cutlass/gemm/sgemm_traits.h"
|
||||
#include "tools/test/unit/gemm/gemm_testbed.h"
|
||||
#include "tools/test/unit/gemm/run_gemm.h"
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Sgemm_64x128x16, sgemm_64x128x64_4x8_accumulators_nt) {
|
||||
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
typedef cutlass::gemm::SgemmLBTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<16, 128, 64>,
|
||||
cutlass::gemm::LinearScaling<float>,
|
||||
|
||||
@@ -22,11 +22,11 @@
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
#include <cutlass_unit_test.h>
|
||||
#include <cutlass/gemm/gemm.h>
|
||||
#include <cutlass/gemm/sgemm_traits.h>
|
||||
#include <tools/test/unit/gemm/gemm_testbed.h>
|
||||
#include <tools/test/unit/gemm/gemm.h>
|
||||
#include "cutlass_unit_test.h"
|
||||
#include "cutlass/gemm/gemm.h"
|
||||
#include "cutlass/gemm/sgemm_traits.h"
|
||||
#include "tools/test/unit/gemm/gemm_testbed.h"
|
||||
#include "tools/test/unit/gemm/run_gemm.h"
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
|
||||
@@ -22,11 +22,11 @@
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
#include <cutlass_unit_test.h>
|
||||
#include <cutlass/gemm/gemm.h>
|
||||
#include <cutlass/gemm/sgemm_traits.h>
|
||||
#include <tools/test/unit/gemm/gemm_testbed.h>
|
||||
#include <tools/test/unit/gemm/gemm.h>
|
||||
#include "cutlass_unit_test.h"
|
||||
#include "cutlass/gemm/gemm.h"
|
||||
#include "cutlass/gemm/sgemm_traits.h"
|
||||
#include "tools/test/unit/gemm/gemm_testbed.h"
|
||||
#include "tools/test/unit/gemm/run_gemm.h"
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
|
||||
@@ -22,11 +22,11 @@
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
#include <cutlass_unit_test.h>
|
||||
#include <cutlass/gemm/gemm.h>
|
||||
#include <cutlass/gemm/sgemm_traits.h>
|
||||
#include <tools/test/unit/gemm/gemm_testbed.h>
|
||||
#include <tools/test/unit/gemm/gemm.h>
|
||||
#include "cutlass_unit_test.h"
|
||||
#include "cutlass/gemm/gemm.h"
|
||||
#include "cutlass/gemm/sgemm_traits.h"
|
||||
#include "tools/test/unit/gemm/gemm_testbed.h"
|
||||
#include "tools/test/unit/gemm/run_gemm.h"
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
|
||||
@@ -22,11 +22,11 @@
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
#include <cutlass_unit_test.h>
|
||||
#include <cutlass/gemm/gemm.h>
|
||||
#include <cutlass/gemm/sgemm_traits.h>
|
||||
#include <tools/test/unit/gemm/gemm_testbed.h>
|
||||
#include <tools/test/unit/gemm/gemm.h>
|
||||
#include "cutlass_unit_test.h"
|
||||
#include "cutlass/gemm/gemm.h"
|
||||
#include "cutlass/gemm/sgemm_traits.h"
|
||||
#include "tools/test/unit/gemm/gemm_testbed.h"
|
||||
#include "tools/test/unit/gemm/run_gemm.h"
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
|
||||
@@ -22,11 +22,11 @@
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
#include <cutlass_unit_test.h>
|
||||
#include <cutlass/gemm/gemm.h>
|
||||
#include <cutlass/gemm/sgemm_traits.h>
|
||||
#include <tools/test/unit/gemm/gemm_testbed.h>
|
||||
#include <tools/test/unit/gemm/gemm.h>
|
||||
#include "cutlass_unit_test.h"
|
||||
#include "cutlass/gemm/gemm.h"
|
||||
#include "cutlass/gemm/sgemm_traits.h"
|
||||
#include "tools/test/unit/gemm/gemm_testbed.h"
|
||||
#include "tools/test/unit/gemm/run_gemm.h"
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,276 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
// Guard conditions around the entire file.
|
||||
#if !defined(__CUDA_ARCH__) || __CUDA_ARCH__ >= 700
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#include "cutlass_unit_tests.h"
|
||||
#include "tools/util/half.h"
|
||||
#include "tools/test/unit/gemm/gemm_testbed.h"
|
||||
#include "cutlass/gemm/warp_multiply_add_nvcuda.h"
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
//
|
||||
// Tests for Warp-level Multiply Add operation using NvCuda API
|
||||
//
|
||||
|
||||
namespace test {
|
||||
|
||||
///
|
||||
template <typename WarpMultiplyAdd>
|
||||
__global__ void warp_multiply_add(
|
||||
typename WarpMultiplyAdd::ScalarA const *A,
|
||||
int lda,
|
||||
typename WarpMultiplyAdd::ScalarB const *B,
|
||||
int ldb,
|
||||
typename WarpMultiplyAdd::ScalarC *C, int ldc) {
|
||||
|
||||
typename WarpMultiplyAdd::LoadIteratorA iteratorA(A, lda);
|
||||
typename WarpMultiplyAdd::LoadIteratorB iteratorB(B, ldb);
|
||||
typename WarpMultiplyAdd::StoreIteratorC iteratorC(C, ldc);
|
||||
|
||||
typename WarpMultiplyAdd::FragmentA fragmentA;
|
||||
typename WarpMultiplyAdd::FragmentB fragmentB;
|
||||
typename WarpMultiplyAdd::FragmentC fragmentC;
|
||||
|
||||
iteratorA.load(fragmentA);
|
||||
iteratorB.load(fragmentB);
|
||||
|
||||
fragmentC.clear();
|
||||
|
||||
WarpMultiplyAdd::multiply_add(fragmentC, fragmentA, fragmentB, fragmentC);
|
||||
|
||||
iteratorC.store(fragmentC);
|
||||
}
|
||||
|
||||
/// Test environment for Warp Multiply Add operation
|
||||
template <
|
||||
cutlass::MatrixLayout::Kind LayoutA,
|
||||
cutlass::MatrixLayout::Kind LayoutB,
|
||||
cutlass::MatrixLayout::Kind LayoutC,
|
||||
typename ScalarC,
|
||||
typename WarpTile,
|
||||
typename WmmaTile
|
||||
>
|
||||
struct TestWarpMultiplyAdd {
|
||||
|
||||
typedef cutlass::gemm::WarpMultiplyAddNvcuda<
|
||||
LayoutA,
|
||||
LayoutB,
|
||||
LayoutC,
|
||||
half,
|
||||
half,
|
||||
ScalarC,
|
||||
WarpTile,
|
||||
cutlass::Shape<1, 1, 1, 1>,
|
||||
WmmaTile
|
||||
> WarpMultiplyAdd;
|
||||
|
||||
/// Testbed type
|
||||
typedef test::GemmTestbed<
|
||||
cutlass::half_t,
|
||||
cutlass::half_t,
|
||||
ScalarC,
|
||||
ScalarC,
|
||||
ScalarC
|
||||
> GemmTestbed;
|
||||
|
||||
//
|
||||
// Data members
|
||||
//
|
||||
|
||||
GemmTestbed testbed;
|
||||
|
||||
//
|
||||
// Methods
|
||||
//
|
||||
|
||||
TestWarpMultiplyAdd(): testbed(
|
||||
WarpTile::kW, // M
|
||||
WarpTile::kH, // N
|
||||
WarpTile::kD, // K
|
||||
cutlass::convert(LayoutA),
|
||||
cutlass::convert(LayoutB),
|
||||
1,
|
||||
0,
|
||||
CUBLAS_GEMM_DEFAULT_TENSOR_OP,
|
||||
cutlass::convert(LayoutC))
|
||||
{
|
||||
|
||||
}
|
||||
|
||||
/// Run
|
||||
void run() {
|
||||
testbed.initialize();
|
||||
|
||||
// launch
|
||||
warp_multiply_add<WarpMultiplyAdd><<<
|
||||
dim3(1,1,1), dim3(32, 1, 1)
|
||||
>>>(
|
||||
testbed.ptr_A(),
|
||||
testbed.lda(),
|
||||
testbed.ptr_B(),
|
||||
testbed.ldb(),
|
||||
testbed.ptr_computed(),
|
||||
testbed.ldc()
|
||||
);
|
||||
|
||||
// verify
|
||||
ASSERT_TRUE(testbed.verify_with_host());
|
||||
}
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
template <
|
||||
typename ScalarC,
|
||||
typename WarpTile,
|
||||
typename WmmaTile
|
||||
>
|
||||
struct TestWarpMultiplyAddForAllLayouts {
|
||||
|
||||
void run() {
|
||||
|
||||
test::TestWarpMultiplyAdd<
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
ScalarC,
|
||||
WarpTile,
|
||||
WmmaTile
|
||||
>().run();
|
||||
|
||||
test::TestWarpMultiplyAdd<
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
ScalarC,
|
||||
WarpTile,
|
||||
WmmaTile
|
||||
>().run();
|
||||
|
||||
test::TestWarpMultiplyAdd<
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
ScalarC,
|
||||
WarpTile,
|
||||
WmmaTile
|
||||
>().run();
|
||||
|
||||
test::TestWarpMultiplyAdd<
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
ScalarC,
|
||||
WarpTile,
|
||||
WmmaTile
|
||||
>().run();
|
||||
|
||||
|
||||
test::TestWarpMultiplyAdd<
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
ScalarC,
|
||||
WarpTile,
|
||||
WmmaTile
|
||||
>().run();
|
||||
|
||||
test::TestWarpMultiplyAdd<
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
ScalarC,
|
||||
WarpTile,
|
||||
WmmaTile
|
||||
>().run();
|
||||
|
||||
test::TestWarpMultiplyAdd<
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
ScalarC,
|
||||
WarpTile,
|
||||
WmmaTile
|
||||
>().run();
|
||||
|
||||
test::TestWarpMultiplyAdd<
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
ScalarC,
|
||||
WarpTile,
|
||||
WmmaTile
|
||||
>().run();
|
||||
}
|
||||
};
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
//
|
||||
// 16x16x16 WMMA Tile Shape with F32 accumulation
|
||||
//
|
||||
|
||||
TEST(WmmaGemm, WarpMultiplyAdd_f32_16x16x16_16x16x16) {
|
||||
TestWarpMultiplyAddForAllLayouts<
|
||||
float,
|
||||
cutlass::Shape<16, 16, 16>,
|
||||
cutlass::Shape<16, 16, 16>
|
||||
>().run();
|
||||
}
|
||||
|
||||
TEST(WmmaGemm, WarpMultiplyAdd_f32_16x16x32_16x16x16) {
|
||||
TestWarpMultiplyAddForAllLayouts<
|
||||
float,
|
||||
cutlass::Shape<16, 16, 32>,
|
||||
cutlass::Shape<16, 16, 16>
|
||||
>().run();
|
||||
}
|
||||
|
||||
TEST(WmmaGemm, WarpMultiplyAdd_f32_16x32x32_16x16x16) {
|
||||
TestWarpMultiplyAddForAllLayouts<
|
||||
float,
|
||||
cutlass::Shape<16, 32, 32>,
|
||||
cutlass::Shape<16, 16, 16>
|
||||
>().run();
|
||||
}
|
||||
|
||||
TEST(WmmaGemm, WarpMultiplyAdd_f32_16x32x64_16x16x16) {
|
||||
TestWarpMultiplyAddForAllLayouts<
|
||||
float,
|
||||
cutlass::Shape<16, 32, 64>,
|
||||
cutlass::Shape<16, 16, 16>
|
||||
>().run();
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,236 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
#include "cutlass/wmma_matrix.h"
|
||||
#ifdef CUTLASS_USE_SUBBYTE_WMMA
|
||||
|
||||
#include "cutlass_unit_test.h"
|
||||
#include "cutlass/gemm/gemm.h"
|
||||
#include "cutlass/gemm/wmma_gemm_traits.h"
|
||||
#include "tools/test/unit/gemm/gemm_testbed.h"
|
||||
#include "tools/test/unit/gemm/binary_gemm.h"
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(WmmaBinaryGemm_32x32x256, wmma_binary_gemm_32x32x256) {
|
||||
|
||||
typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::Shape<256, 32, 32>,
|
||||
cutlass::Vector<cutlass::bin1_t, 32>,
|
||||
cutlass::Vector<cutlass::bin1_t, 32>,
|
||||
int,
|
||||
cutlass::gemm::LinearScaling<int>,
|
||||
int,
|
||||
cutlass::Shape<256, 32, 32>,
|
||||
cutlass::Shape<128, 8, 8>,
|
||||
32,
|
||||
32>
|
||||
WmmaGemmTraits;
|
||||
run_binary_gemm<WmmaGemmTraits>(32, 32, 256);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(WmmaBinaryGemm_32x32x512, wmma_binary_gemm_32x32x512) {
|
||||
|
||||
typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::Shape<512, 32, 32>,
|
||||
cutlass::Vector<cutlass::bin1_t, 32>,
|
||||
cutlass::Vector<cutlass::bin1_t, 32>,
|
||||
int,
|
||||
cutlass::gemm::LinearScaling<int>,
|
||||
int,
|
||||
cutlass::Shape<512, 32, 32>,
|
||||
cutlass::Shape<128, 8, 8>,
|
||||
32,
|
||||
32>
|
||||
WmmaGemmTraits;
|
||||
run_binary_gemm<WmmaGemmTraits>(32, 32, 512);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(WmmaBinaryGemm_64x64x256, wmma_binary_gemm_64x64x256) {
|
||||
|
||||
typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::Shape<256, 64, 64>,
|
||||
cutlass::Vector<cutlass::bin1_t, 32>,
|
||||
cutlass::Vector<cutlass::bin1_t, 32>,
|
||||
int,
|
||||
cutlass::gemm::LinearScaling<int>,
|
||||
int,
|
||||
cutlass::Shape<256, 32, 32>,
|
||||
cutlass::Shape<128, 8, 8>,
|
||||
32,
|
||||
32>
|
||||
WmmaGemmTraits;
|
||||
run_binary_gemm<WmmaGemmTraits>(64, 64, 256);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(WmmaBinaryGemm_64x32x256, wmma_binary_gemm_64x32x256) {
|
||||
|
||||
typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::Shape<256, 32, 64>,
|
||||
cutlass::Vector<cutlass::bin1_t, 32>,
|
||||
cutlass::Vector<cutlass::bin1_t, 32>,
|
||||
int,
|
||||
cutlass::gemm::LinearScaling<int>,
|
||||
int,
|
||||
cutlass::Shape<256, 32, 32>,
|
||||
cutlass::Shape<128, 8, 8>,
|
||||
32,
|
||||
32>
|
||||
WmmaGemmTraits;
|
||||
run_binary_gemm<WmmaGemmTraits>(64, 32, 256);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(WmmaBinaryGemm_32x64x256, wmma_binary_gemm_32x64x256) {
|
||||
|
||||
typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::Shape<256, 64, 32>,
|
||||
cutlass::Vector<cutlass::bin1_t, 32>,
|
||||
cutlass::Vector<cutlass::bin1_t, 32>,
|
||||
int,
|
||||
cutlass::gemm::LinearScaling<int>,
|
||||
int,
|
||||
cutlass::Shape<256, 32, 32>,
|
||||
cutlass::Shape<128, 8, 8>,
|
||||
32,
|
||||
32>
|
||||
WmmaGemmTraits;
|
||||
run_binary_gemm<WmmaGemmTraits>(32, 64, 256);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(WmmaBinaryGemm_128x128x256, wmma_binary_gemm_128x128x256) {
|
||||
|
||||
typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::Shape<256, 128, 128>,
|
||||
cutlass::Vector<cutlass::bin1_t, 32>,
|
||||
cutlass::Vector<cutlass::bin1_t, 32>,
|
||||
int,
|
||||
cutlass::gemm::LinearScaling<int>,
|
||||
int,
|
||||
cutlass::Shape<256, 64, 64>,
|
||||
cutlass::Shape<128, 8, 8>,
|
||||
128,
|
||||
128>
|
||||
WmmaGemmTraits;
|
||||
run_binary_gemm<WmmaGemmTraits>(128, 128, 256);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(WmmaBinaryGemm_128x128x256, wmma_binary_gemm_512x512x256) {
|
||||
|
||||
typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::Shape<256, 128, 128>,
|
||||
cutlass::Vector<cutlass::bin1_t, 32>,
|
||||
cutlass::Vector<cutlass::bin1_t, 32>,
|
||||
int,
|
||||
cutlass::gemm::LinearScaling<int>,
|
||||
int,
|
||||
cutlass::Shape<256, 64, 64>,
|
||||
cutlass::Shape<128, 8, 8>,
|
||||
32,
|
||||
32>
|
||||
WmmaGemmTraits;
|
||||
run_binary_gemm<WmmaGemmTraits>(512, 512, 256);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(WmmaBinaryGemm_32x32x256, wmma_binary_gemm_32x32x512) {
|
||||
|
||||
typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::Shape<256, 32, 32>,
|
||||
cutlass::Vector<cutlass::bin1_t, 32>,
|
||||
cutlass::Vector<cutlass::bin1_t, 32>,
|
||||
int,
|
||||
cutlass::gemm::LinearScaling<int>,
|
||||
int,
|
||||
cutlass::Shape<256, 32, 32>,
|
||||
cutlass::Shape<128, 8, 8>,
|
||||
32,
|
||||
32>
|
||||
WmmaGemmTraits;
|
||||
run_binary_gemm<WmmaGemmTraits>(32, 32, 512);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(WmmaBinaryGemm_32x32x1024, wmma_binary_gemm_128x128x1024) {
|
||||
|
||||
typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::Shape<1024, 128, 128>,
|
||||
cutlass::Vector<cutlass::bin1_t, 32>,
|
||||
cutlass::Vector<cutlass::bin1_t, 32>,
|
||||
int,
|
||||
cutlass::gemm::LinearScaling<int>,
|
||||
int,
|
||||
cutlass::Shape<1024, 32, 32>,
|
||||
cutlass::Shape<128, 8, 8>,
|
||||
128,
|
||||
128>
|
||||
WmmaGemmTraits;
|
||||
run_binary_gemm<WmmaGemmTraits>(128, 128, 1024);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(WmmaBinaryGemm_64x32x1024, wmma_binary_gemm_128x128x1024) {
|
||||
|
||||
typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::Shape<1024, 128, 128>,
|
||||
cutlass::Vector<cutlass::bin1_t, 32>,
|
||||
cutlass::Vector<cutlass::bin1_t, 32>,
|
||||
int,
|
||||
cutlass::gemm::LinearScaling<int>,
|
||||
int,
|
||||
cutlass::Shape<1024, 32, 64>,
|
||||
cutlass::Shape<128, 8, 8>,
|
||||
128,
|
||||
128>
|
||||
WmmaGemmTraits;
|
||||
run_binary_gemm<WmmaGemmTraits>(128, 128, 1024);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#endif // ifdef CUTLASS_USE_SUBBYTE_WMMA
|
||||
@@ -22,20 +22,81 @@
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
#include <cutlass/wmma_matrix.h>
|
||||
#include "cutlass/wmma_matrix.h"
|
||||
#if defined(CUTLASS_USE_WMMA_API)
|
||||
|
||||
#include <cutlass_unit_test.h>
|
||||
#include <cutlass/gemm/gemm.h>
|
||||
#include <cutlass/gemm/wmma_gemm_traits.h>
|
||||
#include <tools/test/unit/gemm/gemm_testbed.h>
|
||||
#include <tools/test/unit/gemm/gemm.h>
|
||||
#include "cutlass_unit_test.h"
|
||||
#include "cutlass/gemm/gemm.h"
|
||||
#include "cutlass/gemm/wmma_gemm_traits.h"
|
||||
#include "tools/test/unit/gemm/gemm_testbed.h"
|
||||
#include "tools/test/unit/gemm/run_gemm.h"
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
//
|
||||
// FP16 accumulation
|
||||
//
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(WmmaGemm_16x16x32_f16, wmma_gemm_16x16x16_nn) {
|
||||
typedef cutlass::gemm::WmmaGemmTraits<
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::Shape<32, 16, 16>,
|
||||
half,
|
||||
half,
|
||||
half,
|
||||
cutlass::gemm::LinearScaling<half>,
|
||||
half
|
||||
>
|
||||
WmmaGemmTraits;
|
||||
|
||||
run_gemm<WmmaGemmTraits>(16, 16, 16);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(WmmaGemm_16x16x32_f16, wmma_gemm_16x16x32_nn) {
|
||||
|
||||
typedef cutlass::gemm::WmmaGemmTraits<
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::Shape<32, 16, 16>,
|
||||
half,
|
||||
half,
|
||||
half,
|
||||
cutlass::gemm::LinearScaling<half>,
|
||||
half
|
||||
>
|
||||
WmmaGemmTraits;
|
||||
run_gemm<WmmaGemmTraits>(16, 16, 32);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(WmmaGemm_128x128x32_f16, wmma_16x16x16_gemm_256x256x128_nn) {
|
||||
typedef cutlass::gemm::WmmaGemmTraits<
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::Shape<32, 128, 128>,
|
||||
half,
|
||||
half,
|
||||
half,
|
||||
cutlass::gemm::LinearScaling<half>,
|
||||
half
|
||||
>
|
||||
WmmaGemmTraits;
|
||||
run_gemm<WmmaGemmTraits>(256, 256, 128);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
//
|
||||
// FP32 accumulation
|
||||
//
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(WmmaGemm_16x16x32, wmma_gemm_16x16x16_nt) {
|
||||
typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<32, 16, 16> >
|
||||
WmmaGemmTraits;
|
||||
run_gemm<WmmaGemmTraits>(16, 16, 16);
|
||||
@@ -45,7 +106,7 @@ TEST(WmmaGemm_16x16x32, wmma_gemm_16x16x16_nt) {
|
||||
|
||||
TEST(WmmaGemm_16x16x32, wmma_gemm_16x16x32_nt) {
|
||||
typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<32, 16, 16> >
|
||||
WmmaGemmTraits;
|
||||
run_gemm<WmmaGemmTraits>(16, 16, 32);
|
||||
@@ -53,9 +114,9 @@ TEST(WmmaGemm_16x16x32, wmma_gemm_16x16x32_nt) {
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(WmmaGemm_128x128x32, wmma_16x16x16_gemm_256x256x128_nt) {
|
||||
TEST(WmmaGemm_128x128x32, wmma_16x16x16_gemm_256x256x128_nt) {
|
||||
typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<32, 128, 128> >
|
||||
WmmaGemmTraits;
|
||||
run_gemm<WmmaGemmTraits>(256, 256, 128);
|
||||
@@ -64,10 +125,12 @@ TEST(WmmaGemm_128x128x32, wmma_16x16x16_gemm_256x256x128_nt) {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#if defined(CUDA_VERSION) && CUDA_VERSION >= 9100
|
||||
TEST(WmmaGemm_128x128x32, wmma_8x32x16_gemm_256x256x128_nt) {
|
||||
TEST(WmmaGemm_128x128x32, wmma_8x32x16_gemm_256x256x128_nt) {
|
||||
typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<32, 128, 128>,
|
||||
half,
|
||||
half,
|
||||
float,
|
||||
cutlass::gemm::LinearScaling<float>,
|
||||
float,
|
||||
@@ -82,10 +145,12 @@ TEST(WmmaGemm_128x128x32, wmma_8x32x16_gemm_256x256x128_nt) {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#if defined(CUDA_VERSION) && CUDA_VERSION >= 9100
|
||||
TEST(WmmaGemm_128x128x32, wmma_32x8x16_gemm_256x256x128_nt) {
|
||||
TEST(WmmaGemm_128x128x32, wmma_32x8x16_gemm_256x256x128_nt) {
|
||||
typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<32, 128, 128>,
|
||||
half,
|
||||
half,
|
||||
float,
|
||||
cutlass::gemm::LinearScaling<float>,
|
||||
float,
|
||||
@@ -100,7 +165,7 @@ TEST(WmmaGemm_128x128x32, wmma_32x8x16_gemm_256x256x128_nt) {
|
||||
|
||||
TEST(WmmaGemm_16x16x32, wmma_gemm_16x16x16_nn) {
|
||||
typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::Shape<32, 16, 16> >
|
||||
WmmaGemmTraits;
|
||||
run_gemm<WmmaGemmTraits>(16, 16, 16);
|
||||
@@ -110,7 +175,7 @@ TEST(WmmaGemm_16x16x32, wmma_gemm_16x16x16_nn) {
|
||||
|
||||
TEST(WmmaGemm_16x16x32, wmma_gemm_16x16x32_nn) {
|
||||
typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::Shape<32, 16, 16> >
|
||||
WmmaGemmTraits;
|
||||
run_gemm<WmmaGemmTraits>(16, 16, 32);
|
||||
@@ -118,9 +183,9 @@ TEST(WmmaGemm_16x16x32, wmma_gemm_16x16x32_nn) {
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(WmmaGemm_128x128x32, wmma_16x16x16_gemm_256x256x128_nn) {
|
||||
TEST(WmmaGemm_128x128x32, wmma_16x16x16_gemm_256x256x128_nn) {
|
||||
typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::Shape<32, 128, 128> >
|
||||
WmmaGemmTraits;
|
||||
run_gemm<WmmaGemmTraits>(256, 256, 128);
|
||||
@@ -129,10 +194,12 @@ TEST(WmmaGemm_128x128x32, wmma_16x16x16_gemm_256x256x128_nn) {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#if defined(CUDA_VERSION) && CUDA_VERSION >= 9100
|
||||
TEST(WmmaGemm_128x128x32, wmma_8x32x16_gemm_256x256x128_nn) {
|
||||
TEST(WmmaGemm_128x128x32, wmma_8x32x16_gemm_256x256x128_nn) {
|
||||
typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::Shape<32, 128, 128>,
|
||||
half,
|
||||
half,
|
||||
float,
|
||||
cutlass::gemm::LinearScaling<float>,
|
||||
float,
|
||||
@@ -147,10 +214,12 @@ TEST(WmmaGemm_128x128x32, wmma_8x32x16_gemm_256x256x128_nn) {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#if defined(CUDA_VERSION) && CUDA_VERSION >= 9100
|
||||
TEST(WmmaGemm_128x128x32, wmma_32x8x16_gemm_256x256x128_nn) {
|
||||
TEST(WmmaGemm_128x128x32, wmma_32x8x16_gemm_256x256x128_nn) {
|
||||
typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::Shape<32, 128, 128>,
|
||||
half,
|
||||
half,
|
||||
float,
|
||||
cutlass::gemm::LinearScaling<float>,
|
||||
float,
|
||||
@@ -165,7 +234,7 @@ TEST(WmmaGemm_128x128x32, wmma_32x8x16_gemm_256x256x128_nn) {
|
||||
|
||||
TEST(WmmaGemm_16x16x32, wmma_gemm_16x16x16_tt) {
|
||||
typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<32, 16, 16> >
|
||||
WmmaGemmTraits;
|
||||
run_gemm<WmmaGemmTraits>(16, 16, 16);
|
||||
@@ -175,7 +244,7 @@ TEST(WmmaGemm_16x16x32, wmma_gemm_16x16x16_tt) {
|
||||
|
||||
TEST(WmmaGemm_16x16x32, wmma_gemm_16x16x32_tt) {
|
||||
typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<32, 16, 16> >
|
||||
WmmaGemmTraits;
|
||||
run_gemm<WmmaGemmTraits>(16, 16, 32);
|
||||
@@ -183,9 +252,9 @@ TEST(WmmaGemm_16x16x32, wmma_gemm_16x16x32_tt) {
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(WmmaGemm_128x128x32, wmma_16x16x16_gemm_256x256x128_tt) {
|
||||
TEST(WmmaGemm_128x128x32, wmma_16x16x16_gemm_256x256x128_tt) {
|
||||
typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<32, 128, 128> >
|
||||
WmmaGemmTraits;
|
||||
run_gemm<WmmaGemmTraits>(256, 256, 128);
|
||||
@@ -194,10 +263,12 @@ TEST(WmmaGemm_128x128x32, wmma_16x16x16_gemm_256x256x128_tt) {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#if defined(CUDA_VERSION) && CUDA_VERSION >= 9100
|
||||
TEST(WmmaGemm_128x128x32, wmma_8x32x16_gemm_256x256x128_tt) {
|
||||
TEST(WmmaGemm_128x128x32, wmma_8x32x16_gemm_256x256x128_tt) {
|
||||
typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<32, 128, 128>,
|
||||
half,
|
||||
half,
|
||||
float,
|
||||
cutlass::gemm::LinearScaling<float>,
|
||||
float,
|
||||
@@ -212,10 +283,12 @@ TEST(WmmaGemm_128x128x32, wmma_8x32x16_gemm_256x256x128_tt) {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#if defined(CUDA_VERSION) && CUDA_VERSION >= 9100
|
||||
TEST(WmmaGemm_128x128x32, wmma_32x8x16_gemm_256x256x128_tt) {
|
||||
TEST(WmmaGemm_128x128x32, wmma_32x8x16_gemm_256x256x128_tt) {
|
||||
typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<32, 128, 128>,
|
||||
half,
|
||||
half,
|
||||
float,
|
||||
cutlass::gemm::LinearScaling<float>,
|
||||
float,
|
||||
@@ -230,7 +303,7 @@ TEST(WmmaGemm_128x128x32, wmma_32x8x16_gemm_256x256x128_tt) {
|
||||
|
||||
TEST(WmmaGemm_16x16x32, wmma_gemm_16x16x16_tn) {
|
||||
typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::Shape<32, 16, 16> >
|
||||
WmmaGemmTraits;
|
||||
run_gemm<WmmaGemmTraits>(16, 16, 16);
|
||||
@@ -240,7 +313,7 @@ TEST(WmmaGemm_16x16x32, wmma_gemm_16x16x16_tn) {
|
||||
|
||||
TEST(WmmaGemm_16x16x32, wmma_gemm_16x16x32_tn) {
|
||||
typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::Shape<32, 16, 16> >
|
||||
WmmaGemmTraits;
|
||||
run_gemm<WmmaGemmTraits>(16, 16, 32);
|
||||
@@ -248,9 +321,9 @@ TEST(WmmaGemm_16x16x32, wmma_gemm_16x16x32_tn) {
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(WmmaGemm_128x128x32, wmma_16x16x16_gemm_256x256x128_tn) {
|
||||
TEST(WmmaGemm_128x128x32, wmma_16x16x16_gemm_256x256x128_tn) {
|
||||
typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::Shape<32, 128, 128> >
|
||||
WmmaGemmTraits;
|
||||
run_gemm<WmmaGemmTraits>(256, 256, 128);
|
||||
@@ -259,10 +332,12 @@ TEST(WmmaGemm_128x128x32, wmma_16x16x16_gemm_256x256x128_tn) {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#if defined(CUDA_VERSION) && CUDA_VERSION >= 9100
|
||||
TEST(WmmaGemm_128x128x32, wmma_8x32x16_gemm_256x256x128_tn) {
|
||||
TEST(WmmaGemm_128x128x32, wmma_8x32x16_gemm_256x256x128_tn) {
|
||||
typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::Shape<32, 128, 128>,
|
||||
half,
|
||||
half,
|
||||
float,
|
||||
cutlass::gemm::LinearScaling<float>,
|
||||
float,
|
||||
@@ -277,10 +352,12 @@ TEST(WmmaGemm_128x128x32, wmma_8x32x16_gemm_256x256x128_tn) {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#if defined(CUDA_VERSION) && CUDA_VERSION >= 9100
|
||||
TEST(WmmaGemm_128x128x32, wmma_32x8x16_gemm_256x256x128_tn) {
|
||||
TEST(WmmaGemm_128x128x32, wmma_32x8x16_gemm_256x256x128_tn) {
|
||||
typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::Shape<32, 128, 128>,
|
||||
half,
|
||||
half,
|
||||
float,
|
||||
cutlass::gemm::LinearScaling<float>,
|
||||
float,
|
||||
|
||||
@@ -0,0 +1,446 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
// Guard conditions around the entire file.
|
||||
#if !defined(__CUDA_ARCH__) || __CUDA_ARCH__ >= 700
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#include "cutlass_unit_tests.h"
|
||||
#include "tools/util/half.h"
|
||||
#include "tools/test/unit/gemm/gemm_testbed.h"
|
||||
#include "cutlass/gemm/wmma_gemm_traits.h"
|
||||
#include "cutlass/gemm/wmma_gemm_epilogue.h"
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
template <typename GemmTraits, typename EpilogueTraits, typename LoadAccumulatorIterator>
|
||||
__global__ void test_epilogue_kernel(
|
||||
typename EpilogueTraits::Params params,
|
||||
cutlass::Coord<3> problem,
|
||||
typename EpilogueTraits::AccumulatorScalar *accum_ptr,
|
||||
int ldm) {
|
||||
|
||||
// Shared memory allocation
|
||||
__shared__ typename EpilogueTraits::SharedStorage shared_storage;
|
||||
|
||||
//
|
||||
// Load accumulators from memory - normally, a GEMM would compute these
|
||||
//
|
||||
|
||||
// Traits class defines tiling
|
||||
GemmTraits traits;
|
||||
|
||||
int warp_id = (threadIdx.x / 32);
|
||||
cutlass::Coord<3> warp_offset = traits(warp_id);
|
||||
|
||||
// Accumulator fragment
|
||||
typename EpilogueTraits::AccumulatorFragment accumulator;
|
||||
|
||||
// Construct an out-of-band LoadIterator for accumulators to initialize them
|
||||
|
||||
LoadAccumulatorIterator load_accum_iterator(accum_ptr, ldm, warp_offset);
|
||||
load_accum_iterator.load(accumulator);
|
||||
|
||||
__syncthreads();
|
||||
|
||||
//
|
||||
// Test the epilogue itself
|
||||
//
|
||||
|
||||
typedef cutlass::gemm::WmmaGemmEpilogue<EpilogueTraits> Epilogue;
|
||||
|
||||
Epilogue epilogue(params, problem, warp_offset);
|
||||
|
||||
// Perform the epilogue operation
|
||||
epilogue.update(shared_storage, accumulator);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
template <
|
||||
typename ThreadBlockTile,
|
||||
typename WarpTile,
|
||||
typename WmmaTile,
|
||||
typename EpilogueTile,
|
||||
typename StreamTile,
|
||||
typename AccumulatorType,
|
||||
typename ScalarC
|
||||
>
|
||||
struct TestWmmaGemmEpilogue {
|
||||
|
||||
typedef cutlass::gemm::WmmaGemmTraits<
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
ThreadBlockTile,
|
||||
WarpTile,
|
||||
WmmaTile,
|
||||
AccumulatorType,
|
||||
AccumulatorType,
|
||||
1,
|
||||
AccumulatorType,
|
||||
EpilogueTile,
|
||||
StreamTile
|
||||
> Traits;
|
||||
|
||||
// Construct an actual epilogue
|
||||
typedef cutlass::gemm::EpilogueLinearScaling<ScalarC, ScalarC, ScalarC, ScalarC> EpilogueLinearScaling;
|
||||
|
||||
/// Define some traits
|
||||
typedef cutlass::gemm::WmmaGemmEpilogueTraitsBasic<
|
||||
ScalarC,
|
||||
typename Traits::WarpMultiplyAdd::StoreIteratorC,
|
||||
ScalarC,
|
||||
ThreadBlockTile,
|
||||
32 * Traits::Warps::kCount,
|
||||
WarpTile,
|
||||
WmmaTile,
|
||||
EpilogueTile,
|
||||
StreamTile,
|
||||
EpilogueLinearScaling
|
||||
> WmmaGemmEpilogueTraits;
|
||||
|
||||
/// Type alias for EpilogueTraits type
|
||||
typedef typename WmmaGemmEpilogueTraits::Traits EpilogueTraits;
|
||||
|
||||
TestWmmaGemmEpilogue() {
|
||||
|
||||
}
|
||||
|
||||
void run(cutlass::Coord<3> problem) {
|
||||
//
|
||||
// Prepare accumulator tile
|
||||
//
|
||||
cutlass::HostTensor<ScalarC> accumulator_matrix;
|
||||
cutlass::HostTensor<ScalarC> source_matrix;
|
||||
cutlass::HostTensor<ScalarC> destination_matrix;
|
||||
|
||||
accumulator_matrix.resize_matrix(
|
||||
ThreadBlockTile::kW,
|
||||
ThreadBlockTile::kH,
|
||||
cutlass::MatrixLayout::kColumnMajor);
|
||||
|
||||
source_matrix.resize_matrix(
|
||||
problem[2],
|
||||
problem[1],
|
||||
cutlass::MatrixLayout::kColumnMajor);
|
||||
|
||||
destination_matrix.resize_matrix(
|
||||
problem[2],
|
||||
problem[1],
|
||||
cutlass::MatrixLayout::kColumnMajor);
|
||||
|
||||
accumulator_matrix.fill_sequential();
|
||||
|
||||
source_matrix.fill_sequential();
|
||||
|
||||
int value = 0;
|
||||
for (int row = 0; row < ThreadBlockTile::kW; ++row) {
|
||||
for (int col = 0; col < ThreadBlockTile::kH; ++col, ++value) {
|
||||
if (row < problem[2] && col < problem[1]) {
|
||||
source_matrix.at(cutlass::make_Coord(0, row, col, 0)) = ScalarC(value);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
destination_matrix.fill(0);
|
||||
|
||||
//
|
||||
// Launch test kernel
|
||||
//
|
||||
dim3 grid(1,1);
|
||||
dim3 block(32 * Traits::Warps::kCount, 1, 1);
|
||||
|
||||
EpilogueLinearScaling functor;
|
||||
functor.initialize(1, 0);
|
||||
|
||||
typename EpilogueTraits::Params params;
|
||||
|
||||
params.initialize(
|
||||
functor,
|
||||
source_matrix.device_data(),
|
||||
source_matrix.leading_dim(),
|
||||
destination_matrix.device_data(),
|
||||
destination_matrix.leading_dim()
|
||||
);
|
||||
|
||||
test_epilogue_kernel<
|
||||
Traits,
|
||||
EpilogueTraits,
|
||||
typename Traits::WarpMultiplyAdd::LoadIteratorC
|
||||
><<< grid, block >>>(
|
||||
params,
|
||||
problem,
|
||||
accumulator_matrix.device_data(),
|
||||
accumulator_matrix.leading_dim()
|
||||
);
|
||||
|
||||
destination_matrix.sync_host();
|
||||
|
||||
EXPECT_TRUE(accumulator_matrix.bit_equals(destination_matrix))
|
||||
<< "Accumulators:\n" << accumulator_matrix << "\nDestination:\n" << destination_matrix;
|
||||
}
|
||||
|
||||
void run() {
|
||||
run(cutlass::make_Coord(ThreadBlockTile::kD, ThreadBlockTile::kH, ThreadBlockTile::kW));
|
||||
}
|
||||
};
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Small epilogue
|
||||
TEST(WmmaGemm_16x16x16, wmma_epilogue_basic) {
|
||||
|
||||
// GEMM threadblock structure
|
||||
typedef cutlass::Shape<16, 16, 16> ThreadBlockTile;
|
||||
typedef cutlass::Shape<16, 16, 16> WarpTile;
|
||||
typedef cutlass::Shape<16, 16, 16> WmmaTile;
|
||||
|
||||
// Epilogue shapes
|
||||
typedef cutlass::Shape<1, 16, 16> EpilogueTile;
|
||||
typedef cutlass::Shape<1, 16, 16> StreamTile;
|
||||
|
||||
typedef float AccumulatorType;
|
||||
typedef float ScalarC;
|
||||
|
||||
TestWmmaGemmEpilogue<
|
||||
ThreadBlockTile,
|
||||
WarpTile,
|
||||
WmmaTile,
|
||||
EpilogueTile,
|
||||
StreamTile,
|
||||
AccumulatorType,
|
||||
ScalarC
|
||||
>().run();
|
||||
}
|
||||
|
||||
TEST(WmmaGemm_16x16x16, wmma_epilogue_ragged) {
|
||||
|
||||
// GEMM threadblock structure
|
||||
typedef cutlass::Shape<16, 16, 16> ThreadBlockTile;
|
||||
typedef cutlass::Shape<16, 16, 16> WarpTile;
|
||||
typedef cutlass::Shape<16, 16, 16> WmmaTile;
|
||||
|
||||
// Epilogue shapes
|
||||
typedef cutlass::Shape<1, 16, 16> EpilogueTile;
|
||||
typedef cutlass::Shape<1, 16, 16> StreamTile;
|
||||
|
||||
typedef float AccumulatorType;
|
||||
typedef float ScalarC;
|
||||
|
||||
TestWmmaGemmEpilogue<
|
||||
ThreadBlockTile,
|
||||
WarpTile,
|
||||
WmmaTile,
|
||||
EpilogueTile,
|
||||
StreamTile,
|
||||
AccumulatorType,
|
||||
ScalarC
|
||||
>().run(cutlass::make_Coord(0, 15, 15));
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Small epilogue
|
||||
TEST(WmmaGemm_32x32x16, wmma_epilogue_basic_32x32_32x32) {
|
||||
|
||||
// GEMM threadblock structure
|
||||
typedef cutlass::Shape<16, 32, 32> ThreadBlockTile;
|
||||
typedef cutlass::Shape<16, 32, 32> WarpTile;
|
||||
typedef cutlass::Shape<16, 16, 16> WmmaTile;
|
||||
|
||||
// Epilogue shapes
|
||||
typedef cutlass::Shape<1, 32, 32> EpilogueTile;
|
||||
typedef cutlass::Shape<1, 4, 32> StreamTile;
|
||||
|
||||
typedef float AccumulatorType;
|
||||
typedef float ScalarC;
|
||||
|
||||
TestWmmaGemmEpilogue<
|
||||
ThreadBlockTile,
|
||||
WarpTile,
|
||||
WmmaTile,
|
||||
EpilogueTile,
|
||||
StreamTile,
|
||||
AccumulatorType,
|
||||
ScalarC
|
||||
>().run();
|
||||
}
|
||||
|
||||
/// Small epilogue
|
||||
TEST(WmmaGemm_32x32x16, wmma_epilogue_basic_32x32_32x32_ragged) {
|
||||
|
||||
// GEMM threadblock structure
|
||||
typedef cutlass::Shape<16, 32, 32> ThreadBlockTile;
|
||||
typedef cutlass::Shape<16, 32, 32> WarpTile;
|
||||
typedef cutlass::Shape<16, 16, 16> WmmaTile;
|
||||
|
||||
// Epilogue shapes
|
||||
typedef cutlass::Shape<1, 32, 32> EpilogueTile;
|
||||
typedef cutlass::Shape<1, 4, 32> StreamTile;
|
||||
|
||||
typedef float AccumulatorType;
|
||||
typedef float ScalarC;
|
||||
|
||||
TestWmmaGemmEpilogue<
|
||||
ThreadBlockTile,
|
||||
WarpTile,
|
||||
WmmaTile,
|
||||
EpilogueTile,
|
||||
StreamTile,
|
||||
AccumulatorType,
|
||||
ScalarC
|
||||
>().run(cutlass::make_Coord(0, 14, 17));
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Small epilogue
|
||||
TEST(WmmaGemm_32x32x16, wmma_epilogue_basic_32x32_16x16) {
|
||||
|
||||
// GEMM threadblock structure
|
||||
typedef cutlass::Shape<16, 32, 32> ThreadBlockTile;
|
||||
typedef cutlass::Shape<16, 16, 16> WarpTile;
|
||||
typedef cutlass::Shape<16, 16, 16> WmmaTile;
|
||||
|
||||
// Epilogue shapes
|
||||
typedef cutlass::Shape<1, 32, 32> EpilogueTile;
|
||||
typedef cutlass::Shape<1, 4, 32> StreamTile;
|
||||
|
||||
typedef float AccumulatorType;
|
||||
typedef float ScalarC;
|
||||
|
||||
TestWmmaGemmEpilogue<
|
||||
ThreadBlockTile,
|
||||
WarpTile,
|
||||
WmmaTile,
|
||||
EpilogueTile,
|
||||
StreamTile,
|
||||
AccumulatorType,
|
||||
ScalarC
|
||||
>().run();
|
||||
}
|
||||
|
||||
/// Small epilogue
|
||||
TEST(WmmaGemm_32x32x16, wmma_epilogue_basic_32x32_16x16_ragged) {
|
||||
|
||||
// GEMM threadblock structure
|
||||
typedef cutlass::Shape<16, 32, 32> ThreadBlockTile;
|
||||
typedef cutlass::Shape<16, 16, 16> WarpTile;
|
||||
typedef cutlass::Shape<16, 16, 16> WmmaTile;
|
||||
|
||||
// Epilogue shapes
|
||||
typedef cutlass::Shape<1, 32, 32> EpilogueTile;
|
||||
typedef cutlass::Shape<1, 4, 32> StreamTile;
|
||||
|
||||
typedef float AccumulatorType;
|
||||
typedef float ScalarC;
|
||||
|
||||
TestWmmaGemmEpilogue<
|
||||
ThreadBlockTile,
|
||||
WarpTile,
|
||||
WmmaTile,
|
||||
EpilogueTile,
|
||||
StreamTile,
|
||||
AccumulatorType,
|
||||
ScalarC
|
||||
>().run(cutlass::make_Coord(0, 23, 19));
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Large epilogue
|
||||
TEST(WmmaGemm_128x128x16, wmma_epilogue_basic_32x32_16x16) {
|
||||
|
||||
// GEMM threadblock structure
|
||||
typedef cutlass::Shape<16, 128, 128> ThreadBlockTile;
|
||||
typedef cutlass::Shape<16, 32, 64> WarpTile;
|
||||
typedef cutlass::Shape<16, 16, 16> WmmaTile;
|
||||
|
||||
// Epilogue shapes
|
||||
typedef cutlass::Shape<1, 64, 64> EpilogueTile;
|
||||
typedef cutlass::Shape<1, 4, 64> StreamTile;
|
||||
|
||||
typedef float AccumulatorType;
|
||||
typedef float ScalarC;
|
||||
|
||||
typedef cutlass::gemm::WmmaGemmEpilogueStructure<
|
||||
ThreadBlockTile,
|
||||
EpilogueTile,
|
||||
StreamTile,
|
||||
WarpTile,
|
||||
WmmaTile
|
||||
> Structure;
|
||||
|
||||
TestWmmaGemmEpilogue<
|
||||
ThreadBlockTile,
|
||||
WarpTile,
|
||||
WmmaTile,
|
||||
EpilogueTile,
|
||||
StreamTile,
|
||||
AccumulatorType,
|
||||
ScalarC
|
||||
>().run();
|
||||
}
|
||||
|
||||
/// Large epilogue
|
||||
TEST(WmmaGemm_128x128x16, wmma_epilogue_basic_32x32_16x16_ragged) {
|
||||
|
||||
// GEMM threadblock structure
|
||||
typedef cutlass::Shape<16, 128, 128> ThreadBlockTile;
|
||||
typedef cutlass::Shape<16, 32, 64> WarpTile;
|
||||
typedef cutlass::Shape<16, 16, 16> WmmaTile;
|
||||
|
||||
// Epilogue shapes
|
||||
typedef cutlass::Shape<1, 64, 64> EpilogueTile;
|
||||
typedef cutlass::Shape<1, 4, 64> StreamTile;
|
||||
|
||||
typedef float AccumulatorType;
|
||||
typedef float ScalarC;
|
||||
|
||||
typedef cutlass::gemm::WmmaGemmEpilogueStructure<
|
||||
ThreadBlockTile,
|
||||
EpilogueTile,
|
||||
StreamTile,
|
||||
WarpTile,
|
||||
WmmaTile
|
||||
> Structure;
|
||||
|
||||
TestWmmaGemmEpilogue<
|
||||
ThreadBlockTile,
|
||||
WarpTile,
|
||||
WmmaTile,
|
||||
EpilogueTile,
|
||||
StreamTile,
|
||||
AccumulatorType,
|
||||
ScalarC
|
||||
>().run(cutlass::make_Coord(0, 119, 101));
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#endif // end guard conditional on SM70
|
||||
@@ -0,0 +1,504 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
// Guard conditions around the entire file.
|
||||
#if !defined(__CUDA_ARCH__) || __CUDA_ARCH__ >= 700
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#include "cutlass_unit_tests.h"
|
||||
|
||||
#include "tools/util/half.h"
|
||||
#include "tools/util/tensor_view_io.h"
|
||||
#include "tools/util/host_tensor.h"
|
||||
|
||||
#include "tools/test/unit/gemm/gemm_testbed.h"
|
||||
|
||||
#include "cutlass/gemm/gemm_fragment_stream.h"
|
||||
#include "cutlass/gemm/warp_multiply_add_nvcuda.h"
|
||||
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
namespace test {
|
||||
|
||||
template <typename FragmentStream>
|
||||
__global__ void fragment_stream(typename FragmentStream::Params params, half *output) {
|
||||
|
||||
__shared__ typename FragmentStream::Storage storage;
|
||||
|
||||
params.store_params.initialize(storage);
|
||||
FragmentStream stream(
|
||||
params,
|
||||
cutlass::make_Coord(16, 256, 256)
|
||||
);
|
||||
|
||||
// load
|
||||
stream.load();
|
||||
|
||||
// store
|
||||
stream.commit();
|
||||
|
||||
__syncthreads();
|
||||
|
||||
// one thread writes it all out
|
||||
if (threadIdx.x == 0) {
|
||||
|
||||
half const *ptr = reinterpret_cast<half const *>(storage.data());
|
||||
|
||||
CUTLASS_PRAGMA_NO_UNROLL
|
||||
for (int i = 0; i < FragmentStream::Storage::Shape::kCount; ++i) {
|
||||
output[i] = ptr[i];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
struct TestGemmDesc {
|
||||
int m, n, k;
|
||||
inline __host__ __device__ TestGemmDesc() : m(0), n(0), k(0) {}
|
||||
};
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
template <
|
||||
typename ThreadBlockTile,
|
||||
cutlass::MatrixLayout::Kind LayoutA,
|
||||
cutlass::MatrixLayout::Kind LayoutB,
|
||||
int Threads,
|
||||
int ScalarsPerAccess
|
||||
>
|
||||
struct TestGemmFragmentStream {
|
||||
|
||||
/// TileStream for Operand A
|
||||
typedef cutlass::gemm::GemmFragmentStreamTraits<
|
||||
cutlass::gemm::GemmOperand::kA,
|
||||
uint16_t,
|
||||
LayoutA,
|
||||
ThreadBlockTile,
|
||||
Threads,
|
||||
ScalarsPerAccess
|
||||
> FragmentStreamTraitsA;
|
||||
|
||||
/// Defines fragment stream for A operand
|
||||
typedef typename cutlass::gemm::GemmFragmentStream<FragmentStreamTraitsA> FragmentStreamA;
|
||||
|
||||
/// TileStream for Operand B
|
||||
typedef typename cutlass::gemm::GemmFragmentStreamTraits<
|
||||
cutlass::gemm::GemmOperand::kB,
|
||||
uint16_t,
|
||||
LayoutB,
|
||||
ThreadBlockTile,
|
||||
Threads,
|
||||
ScalarsPerAccess
|
||||
> FragmentStreamTraitsB;
|
||||
|
||||
/// Defines fragment stream for A operand
|
||||
typedef typename cutlass::gemm::GemmFragmentStream<FragmentStreamTraitsB> FragmentStreamB;
|
||||
|
||||
//
|
||||
// Data members
|
||||
//
|
||||
|
||||
cutlass::HostTensor<cutlass::half_t> tensor_A_in;
|
||||
cutlass::HostTensor<cutlass::half_t> tensor_A_out;
|
||||
|
||||
cutlass::HostTensor<cutlass::half_t> tensor_B_in;
|
||||
cutlass::HostTensor<cutlass::half_t> tensor_B_out;
|
||||
|
||||
//
|
||||
// Methods
|
||||
//
|
||||
|
||||
/// Constructor
|
||||
TestGemmFragmentStream() {
|
||||
tensor_A_in.resize_matrix(ThreadBlockTile::kW, ThreadBlockTile::kD, LayoutA);
|
||||
tensor_A_out.resize_matrix(ThreadBlockTile::kW, ThreadBlockTile::kD, LayoutA);
|
||||
|
||||
tensor_B_in.resize_matrix(ThreadBlockTile::kD, ThreadBlockTile::kH, LayoutB);
|
||||
tensor_B_out.resize_matrix(ThreadBlockTile::kD, ThreadBlockTile::kH, LayoutB);
|
||||
}
|
||||
|
||||
/// Writes details about TileStream
|
||||
template <typename TileStream>
|
||||
std::ostream & write(std::ostream &out, typename TileStream::Params const ¶ms) {
|
||||
|
||||
out << "TileStream::LoadIterator\n"
|
||||
<< " Tile(" << TileStream::LoadIterator::Tile::kH << ", "
|
||||
<< TileStream::LoadIterator::Tile::kW << ")\n"
|
||||
<< " Delta(" << TileStream::LoadIterator::Steps::kH << ", "
|
||||
<< TileStream::LoadIterator::Steps::kW << ")\n"
|
||||
<< " Iterations(" << TileStream::LoadIterator::Iterations::kH << ", "
|
||||
<< TileStream::LoadIterator::Iterations::kW << ")\n";
|
||||
|
||||
out
|
||||
<< " stride_h: " << params.load_params.stride_h << "\n"
|
||||
<< " stride_w: " << params.load_params.stride_w << "\n"
|
||||
<< " inc_d: " << params.load_params.inc_d << "\n"
|
||||
<< " inc_h: " << params.load_params.inc_h << "\n"
|
||||
<< " inc_w: " << params.load_params.inc_w << std::endl;
|
||||
|
||||
out << "output elements: " << TileStream::Storage::Shape::kCount << std::endl;
|
||||
|
||||
return out;
|
||||
}
|
||||
|
||||
/// Runs test
|
||||
void run() {
|
||||
|
||||
tensor_A_in.fill_linear(
|
||||
LayoutA == cutlass::MatrixLayout::kColumnMajor ?
|
||||
cutlass::make_Coord(1, 1, ThreadBlockTile::kW, 1) :
|
||||
cutlass::make_Coord(1, ThreadBlockTile::kD, 1, 1));
|
||||
|
||||
tensor_A_out.fill(0);
|
||||
|
||||
tensor_A_in.sync_device();
|
||||
tensor_A_out.sync_device();
|
||||
|
||||
tensor_B_in.fill_linear(
|
||||
LayoutB == cutlass::MatrixLayout::kColumnMajor ?
|
||||
cutlass::make_Coord(1, 1, ThreadBlockTile::kD, 1) :
|
||||
cutlass::make_Coord(1, ThreadBlockTile::kH, 1, 1));
|
||||
|
||||
tensor_B_out.fill(0);
|
||||
|
||||
tensor_B_in.sync_device();
|
||||
tensor_B_out.sync_device();
|
||||
|
||||
|
||||
typename FragmentStreamA::Params params_A;
|
||||
typename FragmentStreamB::Params params_B;
|
||||
|
||||
TestGemmDesc desc;
|
||||
params_A.initialize(
|
||||
desc,
|
||||
reinterpret_cast<uint16_t const *>(tensor_A_in.device_ref().data()),
|
||||
tensor_A_in.leading_dim()
|
||||
);
|
||||
|
||||
params_B.initialize(
|
||||
desc,
|
||||
reinterpret_cast<uint16_t const *>(tensor_A_in.device_ref().data()),
|
||||
tensor_B_in.leading_dim()
|
||||
);
|
||||
|
||||
test::fragment_stream<FragmentStreamA><<< dim3(1,1,1), dim3(Threads,1,1) >>>(
|
||||
params_A,
|
||||
tensor_A_out.device_data()
|
||||
);
|
||||
|
||||
test::fragment_stream<FragmentStreamB><<< dim3(1,1,1), dim3(Threads,1,1) >>>(
|
||||
params_B,
|
||||
tensor_B_out.device_data()
|
||||
);
|
||||
|
||||
tensor_A_out.sync_host();
|
||||
tensor_B_out.sync_host();
|
||||
|
||||
bool passed_A = tensor_A_in.bit_equals(tensor_A_out);
|
||||
bool passed_B = tensor_B_in.bit_equals(tensor_B_out);
|
||||
|
||||
EXPECT_TRUE(passed_A) << tensor_A_out;
|
||||
if (!passed_A) {
|
||||
this->template write<FragmentStreamA>(std::cout, params_A);
|
||||
}
|
||||
|
||||
EXPECT_TRUE(passed_B) << "In: " << tensor_B_in << "\n, Out:\n" << tensor_B_out;
|
||||
if (!passed_B) {
|
||||
this->template write<FragmentStreamB>(std::cout, params_B);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(WmmaGemmFragmentStream, half_32x32x16_col_row) {
|
||||
|
||||
TestGemmFragmentStream<
|
||||
cutlass::Shape<16, 32, 32>,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
32,
|
||||
2
|
||||
>().run();
|
||||
}
|
||||
|
||||
TEST(WmmaGemmFragmentStream, half_128x64x16_col_row) {
|
||||
|
||||
TestGemmFragmentStream<
|
||||
cutlass::Shape<16, 64, 128>,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
32,
|
||||
2
|
||||
>().run();
|
||||
}
|
||||
|
||||
TEST(WmmaGemmFragmentStream, half_256x128x16_col_row) {
|
||||
|
||||
TestGemmFragmentStream<
|
||||
cutlass::Shape<16, 128, 256>,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
32,
|
||||
1
|
||||
>().run();
|
||||
|
||||
TestGemmFragmentStream<
|
||||
cutlass::Shape<16, 128, 256>,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
64,
|
||||
2
|
||||
>().run();
|
||||
|
||||
TestGemmFragmentStream<
|
||||
cutlass::Shape<16, 128, 256>,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
128,
|
||||
4
|
||||
>().run();
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(WmmaGemmFragmentStream, half_32x32x16_col_col) {
|
||||
|
||||
TestGemmFragmentStream<
|
||||
cutlass::Shape<16, 32, 32>,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
32,
|
||||
2
|
||||
>().run();
|
||||
}
|
||||
|
||||
TEST(WmmaGemmFragmentStream, half_128x64x16_col_col) {
|
||||
|
||||
TestGemmFragmentStream<
|
||||
cutlass::Shape<16, 64, 128>,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
32,
|
||||
2
|
||||
>().run();
|
||||
}
|
||||
|
||||
TEST(WmmaGemmFragmentStream, half_256x128x16_col_col) {
|
||||
|
||||
TestGemmFragmentStream<
|
||||
cutlass::Shape<16, 128, 256>,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
32,
|
||||
1
|
||||
>().run();
|
||||
|
||||
TestGemmFragmentStream<
|
||||
cutlass::Shape<16, 128, 256>,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
64,
|
||||
2
|
||||
>().run();
|
||||
|
||||
TestGemmFragmentStream<
|
||||
cutlass::Shape<16, 128, 256>,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
128,
|
||||
4
|
||||
>().run();
|
||||
|
||||
TestGemmFragmentStream<
|
||||
cutlass::Shape<16, 128, 256>,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
128,
|
||||
8
|
||||
>().run();
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(WmmaGemmFragmentStream, half_32x32x16_row_col) {
|
||||
|
||||
TestGemmFragmentStream<
|
||||
cutlass::Shape<16, 32, 32>,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
32,
|
||||
2
|
||||
>().run();
|
||||
}
|
||||
|
||||
TEST(WmmaGemmFragmentStream, half_128x64x16_row_col) {
|
||||
|
||||
TestGemmFragmentStream<
|
||||
cutlass::Shape<16, 64, 128>,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
32,
|
||||
2
|
||||
>().run();
|
||||
}
|
||||
|
||||
TEST(WmmaGemmFragmentStream, half_256x128x16_row_col) {
|
||||
|
||||
TestGemmFragmentStream<
|
||||
cutlass::Shape<16, 128, 256>,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
32,
|
||||
2
|
||||
>().run();
|
||||
|
||||
TestGemmFragmentStream<
|
||||
cutlass::Shape<16, 128, 256>,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
64,
|
||||
4
|
||||
>().run();
|
||||
|
||||
TestGemmFragmentStream<
|
||||
cutlass::Shape<16, 128, 256>,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
128,
|
||||
8
|
||||
>().run();
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(WmmaGemmFragmentStream, half_32x32x16_row_row) {
|
||||
|
||||
TestGemmFragmentStream<
|
||||
cutlass::Shape<16, 32, 32>,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
32,
|
||||
2
|
||||
>().run();
|
||||
}
|
||||
|
||||
TEST(WmmaGemmFragmentStream, half_128x64x16_row_row) {
|
||||
|
||||
TestGemmFragmentStream<
|
||||
cutlass::Shape<16, 64, 128>,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
32,
|
||||
2
|
||||
>().run();
|
||||
}
|
||||
|
||||
TEST(WmmaGemmFragmentStream, half_256x128x16_row_row) {
|
||||
|
||||
TestGemmFragmentStream<
|
||||
cutlass::Shape<16, 128, 256>,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
32,
|
||||
2
|
||||
>().run();
|
||||
|
||||
TestGemmFragmentStream<
|
||||
cutlass::Shape<16, 128, 256>,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
64,
|
||||
4
|
||||
>().run();
|
||||
|
||||
TestGemmFragmentStream<
|
||||
cutlass::Shape<16, 128, 256>,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
128,
|
||||
8
|
||||
>().run();
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(WmmaGemmFragmentStream, half4_32x32x16_row_row) {
|
||||
|
||||
TestGemmFragmentStream<
|
||||
cutlass::Shape<16, 32, 32>,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
32,
|
||||
4
|
||||
>().run();
|
||||
}
|
||||
|
||||
TEST(WmmaGemmFragmentStream, half4_128x64x16_row_row) {
|
||||
|
||||
TestGemmFragmentStream<
|
||||
cutlass::Shape<16, 64, 128>,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
32,
|
||||
4
|
||||
>().run();
|
||||
}
|
||||
|
||||
TEST(WmmaGemmFragmentStream, half4_256x128x16_row_row) {
|
||||
|
||||
TestGemmFragmentStream<
|
||||
cutlass::Shape<16, 128, 256>,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
32,
|
||||
4
|
||||
>().run();
|
||||
|
||||
TestGemmFragmentStream<
|
||||
cutlass::Shape<16, 128, 256>,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
64,
|
||||
4
|
||||
>().run();
|
||||
|
||||
TestGemmFragmentStream<
|
||||
cutlass::Shape<16, 128, 256>,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
128,
|
||||
8
|
||||
>().run();
|
||||
}
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,629 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
|
||||
#include "cutlass/wmma_matrix.h"
|
||||
|
||||
#ifdef CUTLASS_USE_WMMA_API
|
||||
|
||||
#include "cutlass_unit_tests.h"
|
||||
#include "tools/test/unit/gemm/gemm_testbed.h"
|
||||
#include "tools/util/half.h"
|
||||
|
||||
#include "cutlass/gemm/gemm_global_stream.h"
|
||||
#include "cutlass/gemm/gemm_shared_stream.h"
|
||||
#include "cutlass/gemm/wmma_gemm_multiply_add.h"
|
||||
#include "cutlass/gemm/wmma_gemm_global_tile.h"
|
||||
#include "cutlass/gemm/wmma_gemm_shared_tile.h"
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
struct ProblemDesc {
|
||||
int m, n, k;
|
||||
inline __device__ ProblemDesc(int m_, int n_, int k_) : m(m_), n(n_), k(k_) {}
|
||||
};
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
template <typename StoreIterator_, typename LoadIterator_>
|
||||
union SharedStorage {
|
||||
// Storage to store the data.
|
||||
typename StoreIterator_::SharedStorage store;
|
||||
// Storage to load the data.
|
||||
typename LoadIterator_::SharedStorage load;
|
||||
};
|
||||
|
||||
template <class> struct Debug {};
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
template <typename Threads_, int kW_, bool = (Threads_::kW > kW_)>
|
||||
struct ReshapeThreadsA {
|
||||
typedef cutlass::Shape<Threads_::kD, Threads_::kH, Threads_::kW> Threads;
|
||||
};
|
||||
|
||||
template <typename Threads_, int kW_>
|
||||
struct ReshapeThreadsA<Threads_, kW_, true> {
|
||||
typedef cutlass::Shape<Threads_::kD, Threads_::kH * Threads_::kW / kW_, kW_> Threads;
|
||||
};
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
template <typename Threads_, int kH_, bool = (Threads_::kW > kH_)>
|
||||
struct ReshapeThreadsB {
|
||||
typedef cutlass::Shape<Threads_::kD, Threads_::kH, Threads_::kW> Threads;
|
||||
};
|
||||
|
||||
template <typename Threads_, int kH_>
|
||||
struct ReshapeThreadsB<Threads_, kH_, true> {
|
||||
typedef cutlass::Shape<Threads_::kD, Threads_::kH * Threads_::kW / kH_, kH_> Threads;
|
||||
};
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#if 1
|
||||
template <typename Traits_>
|
||||
static __global__ void kernel_nt(half const *d_a, int lda, half const *d_b, int ldb, float *d_c,
|
||||
int ldc) {
|
||||
#if 0
|
||||
// The default configuration of threads.
|
||||
typedef cutlass::Shape<1, Warps_::kCount, 32> Threads_;
|
||||
// The threads.
|
||||
typedef typename ReshapeThreadsA<Threads_, OutputTile_::kW>::Threads ThreadsA;
|
||||
// The threads.
|
||||
typedef typename ReshapeThreadsB<Threads_, OutputTile_::kH>::Threads ThreadsB;
|
||||
// The number of elements loaded per LDG.
|
||||
int const kScalarsPerLdg = 1;
|
||||
// The tile for A.
|
||||
typedef cutlass::Shape<1, OutputTile_::kD, OutputTile_::kW> TileA;
|
||||
// The tile for B.
|
||||
typedef cutlass::Shape<1, OutputTile_::kD, OutputTile_::kH> TileB;
|
||||
// The tile for C.
|
||||
typedef cutlass::Shape<1, Warps_::kH*WmmaShape_::kH, OutputTile_::kW> TileC;
|
||||
#endif
|
||||
|
||||
// The problem descriptor.
|
||||
ProblemDesc desc(Traits_::OutputTile::kW, Traits_::OutputTile::kH, Traits::OutputTile::kD);
|
||||
|
||||
// The elements computed by a single warp.
|
||||
typedef typename cutlass::ShapeDiv<OutputTile_, Warps_>::Shape AccumulatorsPerWarp;
|
||||
|
||||
// Global memory load for A.
|
||||
typedef cutlass::gemm::GemmGlobalIteratorAb<
|
||||
cutlass::gemm::GemmGlobalIteratorTraits<
|
||||
cutlass::GemmOperand::kA, cutlass::MatrixLayout::kColumnMajor, half const, TileA, ThreadsA, kScalarsPerLdg>
|
||||
>
|
||||
GlobalLoadIteratorA;
|
||||
|
||||
// Shared store iterator for A.
|
||||
typedef cutlass::gemm::GemmSharedStoreIteratorAb<
|
||||
cutlass::gemm::GemmSharedStoreIteratorAbTraits<
|
||||
half, TileA, ThreadsA, kScalarsPerLdg>
|
||||
>
|
||||
SharedStoreIteratorA;
|
||||
|
||||
// The global stream for A.
|
||||
typedef cutlass::gemm::GlobalLoadStream<
|
||||
GlobalLoadIteratorA,
|
||||
cutlass::Copy<typename GlobalLoadIteratorA::Fragment>,
|
||||
SharedStoreIteratorA>
|
||||
GlobalLoadStreamA;
|
||||
|
||||
// Shared load iterator for A.
|
||||
typedef cutlass::gemm::WmmaGemmSharedLoadIteratorA<
|
||||
cutlass::gemm::WmmaGemmSharedLoadIteratorAbTraits<
|
||||
cutlass::GemmOperand::kA,
|
||||
cutlass::MatrixLayout::kColumnMajor, half, OutputTile_, Warps_, WmmaShape_> >
|
||||
SharedLoadIteratorA;
|
||||
|
||||
// Global memory load for B.
|
||||
typedef cutlass::gemm::GemmGlobalIteratorAb<
|
||||
cutlass::gemm::GemmGlobalIteratorTraits<
|
||||
cutlass::GemmOperand::kB, cutlass::MatrixLayout::kRowMajor, half const, TileB, ThreadsB, kScalarsPerLdg> >
|
||||
GlobalLoadIteratorB;
|
||||
|
||||
// Shared store iterator for B.
|
||||
typedef cutlass::gemm::GemmSharedStoreIteratorAb<
|
||||
cutlass::gemm::GemmSharedStoreIteratorAbTraits<
|
||||
half, TileB, ThreadsB, kScalarsPerLdg> >
|
||||
SharedStoreIteratorB;
|
||||
|
||||
// The global stream for B.
|
||||
typedef cutlass::gemm::GlobalLoadStream<GlobalLoadIteratorB,
|
||||
cutlass::Copy<typename GlobalLoadIteratorB::Fragment>,
|
||||
SharedStoreIteratorB>
|
||||
GlobalLoadStreamB;
|
||||
|
||||
// Shared load iterator for B.
|
||||
typedef cutlass::gemm::WmmaGemmSharedLoadIteratorB<
|
||||
cutlass::gemm::WmmaGemmSharedLoadIteratorAbTraits<
|
||||
cutlass::GemmOperand::kB,
|
||||
cutlass::MatrixLayout::kRowMajor, half, OutputTile_, Warps_, WmmaShape_> >
|
||||
SharedLoadIteratorB;
|
||||
|
||||
// Share memory to exchange data for A.
|
||||
__shared__ SharedStorage<GlobalLoadStreamA, SharedLoadIteratorA> shared_storage_a;
|
||||
|
||||
// Share memory to exchange data for B.
|
||||
__shared__ SharedStorage<GlobalLoadStreamB, SharedLoadIteratorB> shared_storage_b;
|
||||
|
||||
// Iterator to load A.
|
||||
typename GlobalLoadStreamA::Params global_params_a;
|
||||
global_params_a.initialize(desc, d_a, lda);
|
||||
GlobalLoadStreamA global_load_a(global_params_a, shared_storage_a.store, desc.m, desc.n, desc.k,
|
||||
cutlass::make_Coord(0, 0, 0));
|
||||
|
||||
// Iterator to load B.
|
||||
typename GlobalLoadStreamB::Params global_params_b;
|
||||
global_params_b.initialize(desc, d_b, ldb);
|
||||
GlobalLoadStreamB global_load_b(global_params_b, shared_storage_b.store, desc.m, desc.n, desc.k,
|
||||
cutlass::make_Coord(0, 0, 0));
|
||||
|
||||
// Load A/B.
|
||||
global_load_a.copy();
|
||||
global_load_b.copy();
|
||||
|
||||
// Copy to shared memory.
|
||||
global_load_a.commit();
|
||||
global_load_b.commit();
|
||||
|
||||
// Make sure the data is in shared memory.
|
||||
__syncthreads();
|
||||
|
||||
// Load iterator A.
|
||||
typename SharedLoadIteratorA::Params shared_params_a;
|
||||
shared_params_a.initialize(desc);
|
||||
SharedLoadIteratorA shared_load_a(shared_params_a, shared_storage_a.load);
|
||||
|
||||
// Load iterator B.
|
||||
typename SharedLoadIteratorB::Params shared_params_b;
|
||||
shared_params_b.initialize(desc);
|
||||
SharedLoadIteratorB shared_load_b(shared_params_b, shared_storage_b.load);
|
||||
|
||||
// Copy A from shared memory.
|
||||
typename SharedLoadIteratorA::Fragment fragment_a;
|
||||
cutlass::gemm::load_shared(shared_load_a, fragment_a);
|
||||
|
||||
// Copy B from shared memory.
|
||||
typename SharedLoadIteratorB::Fragment fragment_b;
|
||||
cutlass::gemm::load_shared(shared_load_b, fragment_b);
|
||||
|
||||
// The functor to do WMMA.
|
||||
typedef cutlass::gemm::WmmaGemmMultiplyAdd<
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
float,
|
||||
AccumulatorsPerWarp,
|
||||
WmmaShape_> WmmaGemmMultiplyAdd;
|
||||
|
||||
// The output fragment.
|
||||
typename WmmaGemmMultiplyAdd::Accumulators fragment_c;
|
||||
fragment_c.clear();
|
||||
|
||||
// Do the WMMA.
|
||||
WmmaGemmMultiplyAdd multiply_add;
|
||||
multiply_add.multiply_add(fragment_a, fragment_b, fragment_c, fragment_c);
|
||||
|
||||
// Global memory stream to store D.
|
||||
typedef cutlass::gemm::WmmaGemmGlobalIteratorCd<
|
||||
cutlass::gemm::WmmaGemmGlobalIteratorCdTraits<
|
||||
float, TileC, ThreadsA, 1>
|
||||
>
|
||||
GlobalStoreIteratorD;
|
||||
typedef cutlass::gemm::GlobalStoreStream<GlobalStoreIteratorD> GlobalStoreStreamD;
|
||||
|
||||
// The shared memory to store D.
|
||||
__shared__ typename GlobalStoreStreamD::SharedStorage shared_storage_stream_d;
|
||||
|
||||
// Iterator to store C.
|
||||
typename GlobalStoreStreamD::Params global_params_d;
|
||||
global_params_d.initialize(desc, d_c, ldc);
|
||||
GlobalStoreStreamD global_store_d(global_params_d, shared_storage_stream_d, desc.m, desc.n, desc.k,
|
||||
cutlass::make_Coord(0, 0, 0));
|
||||
|
||||
// Shared store iterator/stream for C.
|
||||
typedef cutlass::gemm::WmmaGemmSharedStoreIteratorD<
|
||||
cutlass::gemm::WmmaGemmSharedStoreIteratorDTraits<
|
||||
cutlass::MatrixLayout::kColumnMajor, float, OutputTile_, Warps_, WmmaShape_> >
|
||||
SharedStoreIteratorD;
|
||||
typedef cutlass::gemm::SharedStoreStream<SharedStoreIteratorD> SharedStoreStreamD;
|
||||
|
||||
// Shared load iterator/stream for D.
|
||||
typedef cutlass::gemm::WmmaGemmSharedLoadIteratorD<
|
||||
cutlass::gemm::WmmaGemmSharedLoadIteratorDTraits<
|
||||
float, typename SharedStoreIteratorD::Tile, ThreadsA, 1> >
|
||||
SharedLoadIteratorD;
|
||||
typedef cutlass::gemm::SharedLoadStream<SharedLoadIteratorD> SharedLoadStreamD;
|
||||
|
||||
// The shared memory structure to swizzle D.
|
||||
union SharedStorageD {
|
||||
typename SharedStoreStreamD::SharedStorage store;
|
||||
typename SharedLoadStreamD::SharedStorage load;
|
||||
};
|
||||
|
||||
// The shared memory for D.
|
||||
__shared__ SharedStorageD shared_storage_d;
|
||||
|
||||
// Store iterator D.
|
||||
typename SharedStoreStreamD::Params shared_store_params_d;
|
||||
shared_store_params_d.initialize();
|
||||
|
||||
// Store iterator D.
|
||||
typename SharedLoadStreamD::Params shared_load_params_d;
|
||||
shared_load_params_d.initialize();
|
||||
|
||||
// The number of WMMA in the tile H/W dimension (N/M in GEMM).
|
||||
int const kWmmaPerH = OutputTile_::kH / Warps_::kH / WmmaShape_::kH;
|
||||
int const kWmmaPerW = OutputTile_::kW / Warps_::kW / WmmaShape_::kW;
|
||||
|
||||
// Iterate over the data.
|
||||
for (int i = 0; i < kWmmaPerH; ++i) {
|
||||
// Make sure the shared memory can be written to.
|
||||
__syncthreads();
|
||||
|
||||
// Create the iterator to store to SMEM.
|
||||
SharedStoreStreamD shared_store_d(shared_store_params_d,
|
||||
shared_storage_d.store,
|
||||
fragment_c,
|
||||
i*kWmmaPerW);
|
||||
shared_store_d.copy();
|
||||
shared_store_d.commit();
|
||||
|
||||
// Make sure the shared memory was written.
|
||||
__syncthreads();
|
||||
|
||||
// Create the iterator to load from SMEM.
|
||||
SharedLoadStreamD shared_load_d(shared_load_params_d, shared_storage_d.load);
|
||||
shared_load_d.copy();
|
||||
shared_load_d.commit();
|
||||
|
||||
// Copy the data.
|
||||
cutlass::Copy<typename SharedLoadStreamD::Fragment> copy;
|
||||
copy.transform(shared_load_d.fragment(), global_store_d.fragment());
|
||||
|
||||
// Copy the data to global memory.
|
||||
global_store_d.copy();
|
||||
global_store_d.commit();
|
||||
}
|
||||
}
|
||||
#else
|
||||
template <typename OutputTile_, typename Warps_, typename WmmaShape_>
|
||||
static __global__ void kernel_nt(half const *d_a, int lda, half const *d_b, int ldb, float *d_c,
|
||||
int ldc) {
|
||||
// The default configuration of threads.
|
||||
typedef cutlass::Shape<1, Warps_::kCount, 32> Threads_;
|
||||
// The threads.
|
||||
typedef typename ReshapeThreadsA<Threads_, OutputTile_::kW>::Threads ThreadsA;
|
||||
// The threads.
|
||||
typedef typename ReshapeThreadsB<Threads_, OutputTile_::kH>::Threads ThreadsB;
|
||||
// The number of elements loaded per LDG.
|
||||
int const kScalarsPerLdg = 1;
|
||||
// The tile for A.
|
||||
typedef cutlass::Shape<1, OutputTile_::kD, OutputTile_::kW> TileA;
|
||||
// The tile for B.
|
||||
typedef cutlass::Shape<1, OutputTile_::kD, OutputTile_::kH> TileB;
|
||||
// The tile for C.
|
||||
typedef cutlass::Shape<1, Warps_::kH*WmmaShape_::kH, OutputTile_::kW> TileC;
|
||||
|
||||
// The problem descriptor.
|
||||
ProblemDesc desc(OutputTile_::kW, OutputTile_::kH, OutputTile_::kD);
|
||||
|
||||
// The elements computed by a single warp.
|
||||
typedef typename cutlass::ShapeDiv<OutputTile_, Warps_>::Shape AccumulatorsPerWarp;
|
||||
|
||||
// Global memory load for A.
|
||||
typedef cutlass::gemm::GemmGlobalIteratorAb<
|
||||
cutlass::gemm::GemmGlobalIteratorTraits<
|
||||
cutlass::GemmOperand::kA, cutlass::MatrixLayout::kColumnMajor, half const, TileA, ThreadsA, kScalarsPerLdg>
|
||||
>
|
||||
GlobalLoadIteratorA;
|
||||
|
||||
// Shared store iterator for A.
|
||||
typedef cutlass::gemm::GemmSharedStoreIteratorAb<
|
||||
cutlass::gemm::GemmSharedStoreIteratorAbTraits<
|
||||
half, TileA, ThreadsA, kScalarsPerLdg>
|
||||
>
|
||||
SharedStoreIteratorA;
|
||||
|
||||
// The global stream for A.
|
||||
typedef cutlass::gemm::GlobalLoadStream<
|
||||
GlobalLoadIteratorA,
|
||||
cutlass::Copy<typename GlobalLoadIteratorA::Fragment>,
|
||||
SharedStoreIteratorA>
|
||||
GlobalLoadStreamA;
|
||||
|
||||
// Shared load iterator for A.
|
||||
typedef cutlass::gemm::WmmaGemmSharedLoadIteratorA<
|
||||
cutlass::gemm::WmmaGemmSharedLoadIteratorAbTraits<
|
||||
cutlass::GemmOperand::kA,
|
||||
cutlass::MatrixLayout::kColumnMajor, half, OutputTile_, Warps_, WmmaShape_> >
|
||||
SharedLoadIteratorA;
|
||||
|
||||
// Global memory load for B.
|
||||
typedef cutlass::gemm::GemmGlobalIteratorAb<
|
||||
cutlass::gemm::GemmGlobalIteratorTraits<
|
||||
cutlass::GemmOperand::kB, cutlass::MatrixLayout::kRowMajor, half const, TileB, ThreadsB, kScalarsPerLdg> >
|
||||
GlobalLoadIteratorB;
|
||||
|
||||
// Shared store iterator for B.
|
||||
typedef cutlass::gemm::GemmSharedStoreIteratorAb<
|
||||
cutlass::gemm::GemmSharedStoreIteratorAbTraits<
|
||||
half, TileB, ThreadsB, kScalarsPerLdg> >
|
||||
SharedStoreIteratorB;
|
||||
|
||||
// The global stream for B.
|
||||
typedef cutlass::gemm::GlobalLoadStream<GlobalLoadIteratorB,
|
||||
cutlass::Copy<typename GlobalLoadIteratorB::Fragment>,
|
||||
SharedStoreIteratorB>
|
||||
GlobalLoadStreamB;
|
||||
|
||||
// Shared load iterator for B.
|
||||
typedef cutlass::gemm::WmmaGemmSharedLoadIteratorB<
|
||||
cutlass::gemm::WmmaGemmSharedLoadIteratorAbTraits<
|
||||
cutlass::GemmOperand::kB,
|
||||
cutlass::MatrixLayout::kRowMajor, half, OutputTile_, Warps_, WmmaShape_> >
|
||||
SharedLoadIteratorB;
|
||||
|
||||
// Share memory to exchange data for A.
|
||||
__shared__ SharedStorage<GlobalLoadStreamA, SharedLoadIteratorA> shared_storage_a;
|
||||
|
||||
// Share memory to exchange data for B.
|
||||
__shared__ SharedStorage<GlobalLoadStreamB, SharedLoadIteratorB> shared_storage_b;
|
||||
|
||||
// Iterator to load A.
|
||||
typename GlobalLoadStreamA::Params global_params_a;
|
||||
global_params_a.initialize(desc, d_a, lda);
|
||||
GlobalLoadStreamA global_load_a(global_params_a, shared_storage_a.store, desc.m, desc.n, desc.k,
|
||||
cutlass::make_Coord(0, 0, 0));
|
||||
|
||||
// Iterator to load B.
|
||||
typename GlobalLoadStreamB::Params global_params_b;
|
||||
global_params_b.initialize(desc, d_b, ldb);
|
||||
GlobalLoadStreamB global_load_b(global_params_b, shared_storage_b.store, desc.m, desc.n, desc.k,
|
||||
cutlass::make_Coord(0, 0, 0));
|
||||
|
||||
// Load A/B.
|
||||
global_load_a.copy();
|
||||
global_load_b.copy();
|
||||
|
||||
// Copy to shared memory.
|
||||
global_load_a.commit();
|
||||
global_load_b.commit();
|
||||
|
||||
// Make sure the data is in shared memory.
|
||||
__syncthreads();
|
||||
|
||||
// Load iterator A.
|
||||
typename SharedLoadIteratorA::Params shared_params_a;
|
||||
shared_params_a.initialize(desc);
|
||||
SharedLoadIteratorA shared_load_a(shared_params_a, shared_storage_a.load);
|
||||
|
||||
// Load iterator B.
|
||||
typename SharedLoadIteratorB::Params shared_params_b;
|
||||
shared_params_b.initialize(desc);
|
||||
SharedLoadIteratorB shared_load_b(shared_params_b, shared_storage_b.load);
|
||||
|
||||
// Copy A from shared memory.
|
||||
typename SharedLoadIteratorA::Fragment fragment_a;
|
||||
cutlass::gemm::load_shared(shared_load_a, fragment_a);
|
||||
|
||||
// Copy B from shared memory.
|
||||
typename SharedLoadIteratorB::Fragment fragment_b;
|
||||
cutlass::gemm::load_shared(shared_load_b, fragment_b);
|
||||
|
||||
// The functor to do WMMA.
|
||||
typedef cutlass::gemm::WmmaGemmMultiplyAdd<
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
float,
|
||||
AccumulatorsPerWarp,
|
||||
WmmaShape_> WmmaGemmMultiplyAdd;
|
||||
|
||||
// The output fragment.
|
||||
typename WmmaGemmMultiplyAdd::Accumulators fragment_c;
|
||||
fragment_c.clear();
|
||||
|
||||
// Do the WMMA.
|
||||
WmmaGemmMultiplyAdd multiply_add;
|
||||
multiply_add.multiply_add(fragment_a, fragment_b, fragment_c, fragment_c);
|
||||
|
||||
// Global memory stream to store D.
|
||||
typedef cutlass::gemm::WmmaGemmGlobalIteratorCd<
|
||||
cutlass::gemm::WmmaGemmGlobalIteratorCdTraits<
|
||||
float, TileC, ThreadsA, 1>
|
||||
>
|
||||
GlobalStoreIteratorD;
|
||||
typedef cutlass::gemm::GlobalStoreStream<GlobalStoreIteratorD> GlobalStoreStreamD;
|
||||
|
||||
// The shared memory to store D.
|
||||
__shared__ typename GlobalStoreStreamD::SharedStorage shared_storage_stream_d;
|
||||
|
||||
// Iterator to store C.
|
||||
typename GlobalStoreStreamD::Params global_params_d;
|
||||
global_params_d.initialize(desc, d_c, ldc);
|
||||
GlobalStoreStreamD global_store_d(global_params_d, shared_storage_stream_d, desc.m, desc.n, desc.k,
|
||||
cutlass::make_Coord(0, 0, 0));
|
||||
|
||||
// Shared store iterator/stream for C.
|
||||
typedef cutlass::gemm::WmmaGemmSharedStoreIteratorD<
|
||||
cutlass::gemm::WmmaGemmSharedStoreIteratorDTraits<
|
||||
cutlass::MatrixLayout::kColumnMajor, float, OutputTile_, Warps_, WmmaShape_> >
|
||||
SharedStoreIteratorD;
|
||||
typedef cutlass::gemm::SharedStoreStream<SharedStoreIteratorD> SharedStoreStreamD;
|
||||
|
||||
// Shared load iterator/stream for D.
|
||||
typedef cutlass::gemm::WmmaGemmSharedLoadIteratorD<
|
||||
cutlass::gemm::WmmaGemmSharedLoadIteratorDTraits<
|
||||
float, typename SharedStoreIteratorD::Tile, ThreadsA, 1> >
|
||||
SharedLoadIteratorD;
|
||||
typedef cutlass::gemm::SharedLoadStream<SharedLoadIteratorD> SharedLoadStreamD;
|
||||
|
||||
// The shared memory structure to swizzle D.
|
||||
union SharedStorageD {
|
||||
typename SharedStoreStreamD::SharedStorage store;
|
||||
typename SharedLoadStreamD::SharedStorage load;
|
||||
};
|
||||
|
||||
// The shared memory for D.
|
||||
__shared__ SharedStorageD shared_storage_d;
|
||||
|
||||
// Store iterator D.
|
||||
typename SharedStoreStreamD::Params shared_store_params_d;
|
||||
shared_store_params_d.initialize();
|
||||
|
||||
// Store iterator D.
|
||||
typename SharedLoadStreamD::Params shared_load_params_d;
|
||||
shared_load_params_d.initialize();
|
||||
|
||||
// The number of WMMA in the tile H/W dimension (N/M in GEMM).
|
||||
int const kWmmaPerH = OutputTile_::kH / Warps_::kH / WmmaShape_::kH;
|
||||
int const kWmmaPerW = OutputTile_::kW / Warps_::kW / WmmaShape_::kW;
|
||||
|
||||
// Iterate over the data.
|
||||
for (int i = 0; i < kWmmaPerH; ++i) {
|
||||
// Make sure the shared memory can be written to.
|
||||
__syncthreads();
|
||||
|
||||
// Create the iterator to store to SMEM.
|
||||
SharedStoreStreamD shared_store_d(shared_store_params_d,
|
||||
shared_storage_d.store,
|
||||
fragment_c,
|
||||
i*kWmmaPerW);
|
||||
shared_store_d.copy();
|
||||
shared_store_d.commit();
|
||||
|
||||
// Make sure the shared memory was written.
|
||||
__syncthreads();
|
||||
|
||||
// Create the iterator to load from SMEM.
|
||||
SharedLoadStreamD shared_load_d(shared_load_params_d, shared_storage_d.load);
|
||||
shared_load_d.copy();
|
||||
shared_load_d.commit();
|
||||
|
||||
// Copy the data.
|
||||
cutlass::Copy<typename SharedLoadStreamD::Fragment> copy;
|
||||
copy.transform(shared_load_d.fragment(), global_store_d.fragment());
|
||||
|
||||
// Copy the data to global memory.
|
||||
global_store_d.copy();
|
||||
global_store_d.commit();
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
template <typename OutputTile_, typename Warps_, typename WmmaShape_>
|
||||
void run() {
|
||||
/// Testbed type.
|
||||
typedef test::GemmTestbed<cutlass::half_t, cutlass::half_t, float, float, float> GemmTestbed;
|
||||
|
||||
// Create the testbed.
|
||||
GemmTestbed testbed(OutputTile_::kW, // M
|
||||
OutputTile_::kH, // N
|
||||
OutputTile_::kD, // K
|
||||
cutlass::convert(cutlass::MatrixLayout::kColumnMajor),
|
||||
cutlass::convert(cutlass::MatrixLayout::kRowMajor), 1, 0,
|
||||
CUBLAS_GEMM_DEFAULT_TENSOR_OP,
|
||||
cutlass::convert(cutlass::MatrixLayout::kColumnMajor));
|
||||
|
||||
// Initialize.
|
||||
testbed.initialize();
|
||||
|
||||
// Launch the kernel.
|
||||
kernel_nt<OutputTile_, Warps_, WmmaShape_><<<1, 32*Warps_::kCount>>>(
|
||||
testbed.ptr_A(), testbed.lda(),
|
||||
testbed.ptr_B(), testbed.ldb(),
|
||||
testbed.ptr_computed(), testbed.ldc());
|
||||
ASSERT_EQ(cudaSuccess, cudaGetLastError());
|
||||
|
||||
// Make sure it worked as expected.
|
||||
ASSERT_TRUE(testbed.verify_with_host());
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(WmmaGemm, multiply_add_f32_16x16x16_16x16x16) {
|
||||
run<cutlass::Shape<16, 16, 16>, cutlass::Shape<1, 1, 1>, cutlass::Shape<16, 16, 16> >();
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(WmmaGemm, multiply_add_f32_16x32x16_16x16x16) {
|
||||
run<cutlass::Shape<16, 32, 16>, cutlass::Shape<1, 1, 1>, cutlass::Shape<16, 16, 16> >();
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(WmmaGemm, multiply_add_f32_32x16x16_16x16x16) {
|
||||
run<cutlass::Shape<16, 16, 32>, cutlass::Shape<1, 1, 1>, cutlass::Shape<16, 16, 16> >();
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(WmmaGemm, multiply_add_f32_64x16x16_16x16x16) {
|
||||
run<cutlass::Shape<16, 16, 64>, cutlass::Shape<1, 1, 1>, cutlass::Shape<16, 16, 16> >();
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(WmmaGemm, multiply_add_f32_64x64x16_16x16x16) {
|
||||
run<cutlass::Shape<16, 64, 64>, cutlass::Shape<1, 1, 1>, cutlass::Shape<16, 16, 16> >();
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(WmmaGemm, multiply_add_f32_128x128x16_16x16x16) {
|
||||
run<cutlass::Shape<16, 128, 128>, cutlass::Shape<1, 2, 2>, cutlass::Shape<16, 16, 16> >();
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(WmmaGemm, multiply_add_f32_32x8x16_32x8x16) {
|
||||
run<cutlass::Shape<16, 8, 32>, cutlass::Shape<1, 1, 1>, cutlass::Shape<16, 8, 32> >();
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(WmmaGemm, multiply_add_f32_128x128x16_32x8x16) {
|
||||
run<cutlass::Shape<16, 128, 128>, cutlass::Shape<1, 2, 2>, cutlass::Shape<16, 8, 32> >();
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(WmmaGemm, multiply_add_f32_8x32x16_8x32x16) {
|
||||
run<cutlass::Shape<16, 32, 8>, cutlass::Shape<1, 1, 1>, cutlass::Shape<16, 32, 8> >();
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(WmmaGemm, multiply_add_f32_128x128x16_8x32x16) {
|
||||
run<cutlass::Shape<16, 128, 128>, cutlass::Shape<1, 2, 2>, cutlass::Shape<16, 32, 8> >();
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#endif // defined CUTLASS_USE_WMMA_API
|
||||
@@ -0,0 +1,630 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
#include "cutlass/wmma_matrix.h"
|
||||
#ifdef CUTLASS_USE_SUBBYTE_WMMA
|
||||
|
||||
#include "cutlass_unit_test.h"
|
||||
#include "cutlass/gemm/gemm.h"
|
||||
#include "cutlass/gemm/wmma_gemm_traits.h"
|
||||
|
||||
#include "tools/test/unit/gemm/gemm_testbed.h"
|
||||
#include "tools/test/unit/gemm/integer_gemm.h"
|
||||
|
||||
/*
|
||||
TEST(TestGroup, TestName)
|
||||
|
||||
- TestGroup should follow this template:
|
||||
WmmaIntegerGemm_<CTAShape>_<InstructionShape>_<datatype>_<layout>
|
||||
|
||||
- TestName should follow this template
|
||||
wmma_integer_gemm_<ProblemShape>_{optional additional specifier(s)}
|
||||
|
||||
- Shapes should be specified as MxNxK (opposite to the Shape<> definition which is KxNxM)
|
||||
*/
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
//
|
||||
// S4 Integer GEMM Unit Tests
|
||||
//
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(WmmaInt4Gemm_32x32x64_8x8x32_s4, wmma_integer_gemm_32x32x64) {
|
||||
|
||||
typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::Shape<64, 32, 32>,
|
||||
cutlass::Vector<cutlass::int4_t, 8>,
|
||||
cutlass::Vector<cutlass::int4_t, 8>,
|
||||
int,
|
||||
cutlass::gemm::LinearScaling<int>,
|
||||
int,
|
||||
cutlass::Shape<64, 32, 32>,
|
||||
cutlass::Shape<32, 8, 8>,
|
||||
8,
|
||||
8>
|
||||
WmmaGemmTraits;
|
||||
run_integer_gemm<WmmaGemmTraits>(32, 32, 64);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(WmmaInt4Gemm_32x32x256_8x8x32_s4, wmma_integer_gemm_128x128x256) {
|
||||
|
||||
typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::Shape<256, 128, 128>,
|
||||
cutlass::Vector<cutlass::int4_t, 8>,
|
||||
cutlass::Vector<cutlass::int4_t, 8>,
|
||||
int,
|
||||
cutlass::gemm::LinearScaling<int>,
|
||||
int,
|
||||
cutlass::Shape<256, 32, 32>,
|
||||
cutlass::Shape<32, 8, 8>,
|
||||
32,
|
||||
32>
|
||||
WmmaGemmTraits;
|
||||
run_integer_gemm<WmmaGemmTraits>(128, 128, 256);
|
||||
}
|
||||
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
//
|
||||
// U4 Integer GEMM Unit Tests
|
||||
//
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(WmmaInt4Gemm_32x32x64_8x8x32_u4, wmma_integer_gemm_32x32x64) {
|
||||
|
||||
typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::Shape<64, 32, 32>,
|
||||
cutlass::Vector<cutlass::uint4_t, 8>,
|
||||
cutlass::Vector<cutlass::uint4_t, 8>,
|
||||
int,
|
||||
cutlass::gemm::LinearScaling<int>,
|
||||
int,
|
||||
cutlass::Shape<64, 32, 32>,
|
||||
cutlass::Shape<32, 8, 8>,
|
||||
8,
|
||||
8>
|
||||
WmmaGemmTraits;
|
||||
run_integer_gemm<WmmaGemmTraits>(32, 32, 64);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
//
|
||||
// S8 Integer GEMM Unit Tests
|
||||
//
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
//
|
||||
// 16x16x16
|
||||
//
|
||||
|
||||
TEST(WmmaInt8Gemm_32x32x32_16x16x16_s8_tn, wmma_integer_gemm_32x32x32) {
|
||||
|
||||
typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::Shape<32, 32, 32>,
|
||||
signed char,
|
||||
signed char,
|
||||
int,
|
||||
cutlass::gemm::LinearScaling<int>,
|
||||
int,
|
||||
cutlass::Shape<32, 32, 32>,
|
||||
cutlass::Shape<16, 16, 16>,
|
||||
4,
|
||||
4>
|
||||
WmmaGemmTraits;
|
||||
run_integer_gemm<WmmaGemmTraits>(32, 32, 32);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(WmmaInt8Gemm_32x32x32_16x16x16_s8_tt, wmma_integer_gemm_32x32x32) {
|
||||
|
||||
typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<32, 32, 32>,
|
||||
signed char,
|
||||
signed char,
|
||||
int,
|
||||
cutlass::gemm::LinearScaling<int>,
|
||||
int,
|
||||
cutlass::Shape<32, 32, 32>,
|
||||
cutlass::Shape<16, 16, 16>,
|
||||
4,
|
||||
4>
|
||||
WmmaGemmTraits;
|
||||
run_integer_gemm<WmmaGemmTraits>(32, 32, 32);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(WmmaInt8Gemm_32x32x32_16x16x16_s8_nt, wmma_integer_gemm_32x32x32) {
|
||||
|
||||
typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<32, 32, 32>,
|
||||
signed char,
|
||||
signed char,
|
||||
int,
|
||||
cutlass::gemm::LinearScaling<int>,
|
||||
int,
|
||||
cutlass::Shape<32, 32, 32>,
|
||||
cutlass::Shape<16, 16, 16>,
|
||||
4,
|
||||
4>
|
||||
WmmaGemmTraits;
|
||||
run_integer_gemm<WmmaGemmTraits>(32, 32, 32);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(WmmaInt8Gemm_32x32x32_16x16x16_s8_nn, wmma_integer_gemm_32x32x32) {
|
||||
|
||||
typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::Shape<32, 32, 32>,
|
||||
signed char,
|
||||
signed char,
|
||||
int,
|
||||
cutlass::gemm::LinearScaling<int>,
|
||||
int,
|
||||
cutlass::Shape<32, 32, 32>,
|
||||
cutlass::Shape<16, 16, 16>,
|
||||
4,
|
||||
4>
|
||||
WmmaGemmTraits;
|
||||
run_integer_gemm<WmmaGemmTraits>(32, 32, 32);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
//
|
||||
// 32x8x16
|
||||
//
|
||||
|
||||
TEST(WmmaInt8Gemm_32x32x32_32x8x16_s8_tn, wmma_integer_gemm_32x32x32) {
|
||||
|
||||
typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::Shape<32, 32, 32>,
|
||||
signed char,
|
||||
signed char,
|
||||
int,
|
||||
cutlass::gemm::LinearScaling<int>,
|
||||
int,
|
||||
cutlass::Shape<32, 32, 32>,
|
||||
cutlass::Shape<16, 8, 32>,
|
||||
4,
|
||||
4>
|
||||
WmmaGemmTraits;
|
||||
run_integer_gemm<WmmaGemmTraits>(32, 32, 32);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(WmmaInt8Gemm_32x32x32_32x8x16_s8_tt, wmma_integer_gemm_32x32x32) {
|
||||
|
||||
typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<32, 32, 32>,
|
||||
signed char,
|
||||
signed char,
|
||||
int,
|
||||
cutlass::gemm::LinearScaling<int>,
|
||||
int,
|
||||
cutlass::Shape<32, 32, 32>,
|
||||
cutlass::Shape<16, 8, 32>,
|
||||
4,
|
||||
4>
|
||||
WmmaGemmTraits;
|
||||
run_integer_gemm<WmmaGemmTraits>(32, 32, 32);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(WmmaInt8Gemm_32x32x32_32x8x16_s8_nt, wmma_integer_gemm_32x32x32) {
|
||||
|
||||
typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<32, 32, 32>,
|
||||
signed char,
|
||||
signed char,
|
||||
int,
|
||||
cutlass::gemm::LinearScaling<int>,
|
||||
int,
|
||||
cutlass::Shape<32, 32, 32>,
|
||||
cutlass::Shape<16, 8, 32>,
|
||||
4,
|
||||
4>
|
||||
WmmaGemmTraits;
|
||||
run_integer_gemm<WmmaGemmTraits>(32, 32, 32);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(WmmaInt8Gemm_32x32x32_32x8x16_s8_nn, wmma_integer_gemm_32x32x32) {
|
||||
|
||||
typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::Shape<32, 32, 32>,
|
||||
signed char,
|
||||
signed char,
|
||||
int,
|
||||
cutlass::gemm::LinearScaling<int>,
|
||||
int,
|
||||
cutlass::Shape<32, 32, 32>,
|
||||
cutlass::Shape<16, 8, 32>,
|
||||
4,
|
||||
4>
|
||||
WmmaGemmTraits;
|
||||
run_integer_gemm<WmmaGemmTraits>(32, 32, 32);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
//
|
||||
// 8x32x16
|
||||
//
|
||||
|
||||
TEST(WmmaInt8Gemm_32x32x32_8x32x16_s8_tn, wmma_integer_gemm_32x32x32) {
|
||||
|
||||
typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::Shape<32, 32, 32>,
|
||||
signed char,
|
||||
signed char,
|
||||
int,
|
||||
cutlass::gemm::LinearScaling<int>,
|
||||
int,
|
||||
cutlass::Shape<32, 32, 32>,
|
||||
cutlass::Shape<16, 32, 8>,
|
||||
4,
|
||||
4>
|
||||
WmmaGemmTraits;
|
||||
run_integer_gemm<WmmaGemmTraits>(32, 32, 32);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(WmmaInt8Gemm_32x32x32_8x32x16_s8_tt, wmma_integer_gemm_32x32x32) {
|
||||
|
||||
typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<32, 32, 32>,
|
||||
signed char,
|
||||
signed char,
|
||||
int,
|
||||
cutlass::gemm::LinearScaling<int>,
|
||||
int,
|
||||
cutlass::Shape<32, 32, 32>,
|
||||
cutlass::Shape<16, 32, 8>,
|
||||
4,
|
||||
4>
|
||||
WmmaGemmTraits;
|
||||
run_integer_gemm<WmmaGemmTraits>(32, 32, 32);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(WmmaInt8Gemm_32x32x32_8x32x16_s8_nt, wmma_integer_gemm_32x32x32) {
|
||||
|
||||
typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<32, 32, 32>,
|
||||
signed char,
|
||||
signed char,
|
||||
int,
|
||||
cutlass::gemm::LinearScaling<int>,
|
||||
int,
|
||||
cutlass::Shape<32, 32, 32>,
|
||||
cutlass::Shape<16, 32, 8>,
|
||||
4,
|
||||
4>
|
||||
WmmaGemmTraits;
|
||||
run_integer_gemm<WmmaGemmTraits>(32, 32, 32);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(WmmaInt8Gemm_32x32x32_8x32x16_s8_nn, wmma_integer_gemm_32x32x32) {
|
||||
|
||||
typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::Shape<32, 32, 32>,
|
||||
signed char,
|
||||
signed char,
|
||||
int,
|
||||
cutlass::gemm::LinearScaling<int>,
|
||||
int,
|
||||
cutlass::Shape<32, 32, 32>,
|
||||
cutlass::Shape<16, 32, 8>,
|
||||
4,
|
||||
4>
|
||||
WmmaGemmTraits;
|
||||
run_integer_gemm<WmmaGemmTraits>(32, 32, 32);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
//
|
||||
// U8 Integer GEMM Unit Tests
|
||||
//
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
//
|
||||
// 16x16x16
|
||||
//
|
||||
|
||||
TEST(WmmaInt8Gemm_32x32x32_16x16x16_u8_tn, wmma_integer_gemm_32x32x32) {
|
||||
|
||||
typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::Shape<32, 32, 32>,
|
||||
unsigned char,
|
||||
unsigned char,
|
||||
int,
|
||||
cutlass::gemm::LinearScaling<int>,
|
||||
int,
|
||||
cutlass::Shape<32, 32, 32>,
|
||||
cutlass::Shape<16, 16, 16>,
|
||||
4,
|
||||
4>
|
||||
WmmaGemmTraits;
|
||||
run_integer_gemm<WmmaGemmTraits>(32, 32, 32);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(WmmaInt8Gemm_32x32x32_16x16x16_u8_tt, wmma_integer_gemm_32x32x32) {
|
||||
|
||||
typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<32, 32, 32>,
|
||||
unsigned char,
|
||||
unsigned char,
|
||||
int,
|
||||
cutlass::gemm::LinearScaling<int>,
|
||||
int,
|
||||
cutlass::Shape<32, 32, 32>,
|
||||
cutlass::Shape<16, 16, 16>,
|
||||
4,
|
||||
4>
|
||||
WmmaGemmTraits;
|
||||
run_integer_gemm<WmmaGemmTraits>(32, 32, 32);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(WmmaInt8Gemm_32x32x32_16x16x16_u8_nt, wmma_integer_gemm_32x32x32) {
|
||||
|
||||
typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<32, 32, 32>,
|
||||
unsigned char,
|
||||
unsigned char,
|
||||
int,
|
||||
cutlass::gemm::LinearScaling<int>,
|
||||
int,
|
||||
cutlass::Shape<32, 32, 32>,
|
||||
cutlass::Shape<16, 16, 16>,
|
||||
4,
|
||||
4>
|
||||
WmmaGemmTraits;
|
||||
run_integer_gemm<WmmaGemmTraits>(32, 32, 32);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(WmmaInt8Gemm_32x32x32_16x16x16_u8_nn, wmma_integer_gemm_32x32x32) {
|
||||
|
||||
typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::Shape<32, 32, 32>,
|
||||
unsigned char,
|
||||
unsigned char,
|
||||
int,
|
||||
cutlass::gemm::LinearScaling<int>,
|
||||
int,
|
||||
cutlass::Shape<32, 32, 32>,
|
||||
cutlass::Shape<16, 16, 16>,
|
||||
4,
|
||||
4>
|
||||
WmmaGemmTraits;
|
||||
run_integer_gemm<WmmaGemmTraits>(32, 32, 32);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
//
|
||||
// 32x8x16
|
||||
//
|
||||
|
||||
TEST(WmmaInt8Gemm_32x32x32_32x8x16_u8_tn, wmma_integer_gemm_32x32x32) {
|
||||
|
||||
typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::Shape<32, 32, 32>,
|
||||
unsigned char,
|
||||
unsigned char,
|
||||
int,
|
||||
cutlass::gemm::LinearScaling<int>,
|
||||
int,
|
||||
cutlass::Shape<32, 32, 32>,
|
||||
cutlass::Shape<16, 8, 32>,
|
||||
4,
|
||||
4>
|
||||
WmmaGemmTraits;
|
||||
run_integer_gemm<WmmaGemmTraits>(32, 32, 32);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(WmmaInt8Gemm_32x32x32_32x8x16_u8_tt, wmma_integer_gemm_32x32x32) {
|
||||
|
||||
typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<32, 32, 32>,
|
||||
unsigned char,
|
||||
unsigned char,
|
||||
int,
|
||||
cutlass::gemm::LinearScaling<int>,
|
||||
int,
|
||||
cutlass::Shape<32, 32, 32>,
|
||||
cutlass::Shape<16, 8, 32>,
|
||||
4,
|
||||
4>
|
||||
WmmaGemmTraits;
|
||||
run_integer_gemm<WmmaGemmTraits>(32, 32, 32);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(WmmaInt8Gemm_32x32x32_32x8x16_u8_nt, wmma_integer_gemm_32x32x32) {
|
||||
|
||||
typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<32, 32, 32>,
|
||||
unsigned char,
|
||||
unsigned char,
|
||||
int,
|
||||
cutlass::gemm::LinearScaling<int>,
|
||||
int,
|
||||
cutlass::Shape<32, 32, 32>,
|
||||
cutlass::Shape<16, 8, 32>,
|
||||
4,
|
||||
4>
|
||||
WmmaGemmTraits;
|
||||
run_integer_gemm<WmmaGemmTraits>(32, 32, 32);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(WmmaInt8Gemm_32x32x32_32x8x16_u8_nn, wmma_integer_gemm_32x32x32) {
|
||||
|
||||
typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::Shape<32, 32, 32>,
|
||||
unsigned char,
|
||||
unsigned char,
|
||||
int,
|
||||
cutlass::gemm::LinearScaling<int>,
|
||||
int,
|
||||
cutlass::Shape<32, 32, 32>,
|
||||
cutlass::Shape<16, 8, 32>,
|
||||
4,
|
||||
4>
|
||||
WmmaGemmTraits;
|
||||
run_integer_gemm<WmmaGemmTraits>(32, 32, 32);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
//
|
||||
// 8x32x16
|
||||
//
|
||||
|
||||
TEST(WmmaInt8Gemm_32x32x32_8x32x16_u8_tn, wmma_integer_gemm_32x32x32) {
|
||||
|
||||
typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::Shape<32, 32, 32>,
|
||||
unsigned char,
|
||||
unsigned char,
|
||||
int,
|
||||
cutlass::gemm::LinearScaling<int>,
|
||||
int,
|
||||
cutlass::Shape<32, 32, 32>,
|
||||
cutlass::Shape<16, 32, 8>,
|
||||
4,
|
||||
4>
|
||||
WmmaGemmTraits;
|
||||
run_integer_gemm<WmmaGemmTraits>(32, 32, 32);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(WmmaInt8Gemm_32x32x32_8x32x16_u8_tt, wmma_integer_gemm_32x32x32) {
|
||||
|
||||
typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<32, 32, 32>,
|
||||
unsigned char,
|
||||
unsigned char,
|
||||
int,
|
||||
cutlass::gemm::LinearScaling<int>,
|
||||
int,
|
||||
cutlass::Shape<32, 32, 32>,
|
||||
cutlass::Shape<16, 32, 8>,
|
||||
4,
|
||||
4>
|
||||
WmmaGemmTraits;
|
||||
run_integer_gemm<WmmaGemmTraits>(32, 32, 32);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(WmmaInt8Gemm_32x32x32_8x32x16_u8_nt, wmma_integer_gemm_32x32x32) {
|
||||
|
||||
typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kRowMajor,
|
||||
cutlass::Shape<32, 32, 32>,
|
||||
unsigned char,
|
||||
unsigned char,
|
||||
int,
|
||||
cutlass::gemm::LinearScaling<int>,
|
||||
int,
|
||||
cutlass::Shape<32, 32, 32>,
|
||||
cutlass::Shape<16, 32, 8>,
|
||||
4,
|
||||
4>
|
||||
WmmaGemmTraits;
|
||||
run_integer_gemm<WmmaGemmTraits>(32, 32, 32);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(WmmaInt8Gemm_32x32x32_8x32x16_u8_nn, wmma_integer_gemm_32x32x32) {
|
||||
|
||||
typedef cutlass::gemm::WmmaGemmTraits<cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::MatrixLayout::kColumnMajor,
|
||||
cutlass::Shape<32, 32, 32>,
|
||||
unsigned char,
|
||||
unsigned char,
|
||||
int,
|
||||
cutlass::gemm::LinearScaling<int>,
|
||||
int,
|
||||
cutlass::Shape<32, 32, 32>,
|
||||
cutlass::Shape<16, 32, 8>,
|
||||
4,
|
||||
4>
|
||||
WmmaGemmTraits;
|
||||
run_integer_gemm<WmmaGemmTraits>(32, 32, 32);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#endif // ifdef CUTLASS_USE_SUBBYTE_WMMA
|
||||
@@ -0,0 +1,102 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
#include <complex>
|
||||
|
||||
#include "cutlass_unit_test.h"
|
||||
#include "cutlass/util/complex.h"
|
||||
#include "tools/util/half.h"
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
namespace test {
|
||||
|
||||
/// Thorough testing for basic complex math operators. Uses std::complex as a reference.
|
||||
template <typename T, int N, int M>
|
||||
struct ComplexOperators {
|
||||
ComplexOperators() {
|
||||
for (int ar = -N; ar <= N; ++ar) {
|
||||
for (int ai = -N; ai <= N; ++ai) {
|
||||
for (int br = -N; br <= N; ++br) {
|
||||
for (int bi = -N; bi <= N; ++bi) {
|
||||
|
||||
cutlass::platform::complex<T> Ae(T(ar) / T(M), T(ai) / T(M));
|
||||
cutlass::platform::complex<T> Be(T(br) / T(M), T(bi) / T(M));
|
||||
|
||||
std::complex<T> Ar(T(ar) / T(M), T(ai) / T(M));
|
||||
std::complex<T> Br(T(br) / T(M), T(bi) / T(M));
|
||||
|
||||
cutlass::platform::complex<T> add_e = Ae + Be;
|
||||
cutlass::platform::complex<T> sub_e = Ae - Be;
|
||||
cutlass::platform::complex<T> mul_e = Ae * Be;
|
||||
|
||||
std::complex<T> add_r = (Ar + Br);
|
||||
std::complex<T> sub_r = (Ar - Br);
|
||||
std::complex<T> mul_r = (Ar * Br);
|
||||
|
||||
EXPECT_EQ(real(add_e), real(add_r));
|
||||
EXPECT_EQ(imag(add_e), imag(add_r));
|
||||
|
||||
EXPECT_EQ(real(sub_e), real(sub_r));
|
||||
EXPECT_EQ(imag(sub_e), imag(sub_r));
|
||||
|
||||
EXPECT_EQ(real(mul_e), real(mul_r));
|
||||
EXPECT_EQ(imag(mul_e), imag(mul_r));
|
||||
|
||||
if (!(br == 0 && bi == 0)) {
|
||||
|
||||
cutlass::platform::complex<T> div_e = Ae * Be;
|
||||
std::complex<T> div_r = Ar * Br;
|
||||
|
||||
EXPECT_EQ(real(div_e), real(div_r));
|
||||
EXPECT_EQ(imag(div_e), imag(div_r));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Complex, host_float) {
|
||||
test::ComplexOperators<float, 32, 8> test;
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Complex, host_double) {
|
||||
test::ComplexOperators<double, 32, 8> test;
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Complex, host_half) {
|
||||
// Fewer test cases since half_t is emulated
|
||||
test::ComplexOperators<cutlass::half_t, 14, 4> test;
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
@@ -1,66 +1,342 @@
|
||||
/******************************************************************************
|
||||
* Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are not permitted.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
|
||||
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
|
||||
* DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
|
||||
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
|
||||
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
|
||||
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
||||
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
******************************************************************************/
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/* \file
|
||||
|
||||
/*! \file
|
||||
\brief Tests for Host_tensor, Host_tensor_view, and Tensor_view
|
||||
\brief Defines unit tests for HostTensor and HostMatrix.
|
||||
|
||||
HostTensor is a utility class for allocating memory on the host and on the selected CUDA device
|
||||
and presenting a TensorView of this memory.
|
||||
|
||||
HostMatrix is new in CUTLASS 1.1 that offers a matrix-like interface to a HostTensor with rank 2.
|
||||
Several examples are shown in this source file.
|
||||
*/
|
||||
|
||||
//#include <gtest/gtest.h>
|
||||
#include <cutlass_unit_test.h>
|
||||
#include <tools/util/host_tensor.h>
|
||||
#include <tools/util/tensor_view_io.h>
|
||||
#include "cutlass_unit_test.h"
|
||||
|
||||
/// Random number generator
|
||||
struct RandomGenerator {
|
||||
RandomGenerator(int seed = 17) {
|
||||
srand(seed);
|
||||
}
|
||||
#include "cutlass/matrix_traits.h"
|
||||
|
||||
float operator()() {
|
||||
return float(rand() % 64) / 8.0f;
|
||||
}
|
||||
};
|
||||
#include "tools/util/tensor_view_io.h"
|
||||
#include "tools/util/host_tensor.h"
|
||||
#include "tools/util/host_matrix.h"
|
||||
|
||||
TEST(HostTensor, gemm) {
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
int const M = 16;
|
||||
int const N = 16;
|
||||
int const K = 16;
|
||||
namespace test {
|
||||
|
||||
typedef cutlass::HostTensor<float, false> HostTensor;
|
||||
/// Kernel to compute a thread's unique coordinate within a CUDA kernel grid and write a value
|
||||
/// using a CUTLASS TensorView.
|
||||
template <typename TensorView>
|
||||
__global__ void fill_sequential(TensorView view) {
|
||||
|
||||
// allocate a host tensor
|
||||
HostTensor A(
|
||||
cutlass::make_Coord(1, K, M, 1)
|
||||
);
|
||||
// Compute the thread's coordinate in the 2D CUDA kernel grid
|
||||
cutlass::Coord<2> coord = cutlass::make_Coord(
|
||||
blockIdx.x * blockDim.x + threadIdx.x,
|
||||
blockIdx.y * blockDim.y + threadIdx.y
|
||||
);
|
||||
|
||||
HostTensor B(
|
||||
cutlass::make_Coord(1, N, K, 1)
|
||||
);
|
||||
|
||||
HostTensor C(
|
||||
cutlass::make_Coord(1, N, M, 1)
|
||||
);
|
||||
|
||||
A.fill_random(RandomGenerator());
|
||||
B.fill_random(RandomGenerator());
|
||||
|
||||
C.gemm<float, float, float, float>(A, B, 1.0f, 0.0f);
|
||||
// Write a value into the view
|
||||
if (view.contains(coord)) {
|
||||
view.at(coord) = coord[0] + view.size(0) * coord[1];
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace test
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
// This test constructs a CUTLASS HostTensor with column-major layout.
|
||||
TEST(HostTensor, fill_sequential_column_major) {
|
||||
|
||||
int const M = 16;
|
||||
int const N = 32;
|
||||
|
||||
cutlass::Coord<2> bounds = cutlass::make_Coord(M, N);
|
||||
|
||||
// Construct a rank=2 host tensor of size M-by-N with leading dimension M
|
||||
cutlass::HostTensor<
|
||||
int,
|
||||
2,
|
||||
cutlass::MatrixLayout::ColumnMajor> host_tensor(cutlass::make_Coord(M, 1), bounds);
|
||||
|
||||
// Fill it with zeros and synchronize device
|
||||
host_tensor.fill(0);
|
||||
host_tensor.sync_device();
|
||||
|
||||
// Launch a CUDA kernel by obtaining a TensorView of the device memory
|
||||
dim3 block(16, 16);
|
||||
dim3 grid((M + block.x - 1) / block.x, (N + block.y - 1) / block.y);
|
||||
|
||||
test::fill_sequential<<< grid, block >>>(host_tensor.device_view());
|
||||
|
||||
ASSERT_EQ(cudaDeviceSynchronize(), cudaSuccess);
|
||||
|
||||
// Synchronize the host data
|
||||
host_tensor.sync_host();
|
||||
|
||||
// Verify host_tensor contains sequential elements
|
||||
int errors = 0;
|
||||
for (int n = 0; n < N; ++n) {
|
||||
for (int m = 0; m < M; ++m) {
|
||||
int expected = m + n * M;
|
||||
int got = host_tensor.at(cutlass::make_Coord(m, n));
|
||||
if (expected != got) {
|
||||
++errors;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
EXPECT_EQ(errors, 0) << std::setw(4) << host_tensor << std::endl;
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
// This test constructs a CUTLASS HostTensor with column-major interleaved layout
|
||||
TEST(HostTensor, fill_sequential_column_major_interleaved) {
|
||||
|
||||
int const M = 16;
|
||||
int const N = 16;
|
||||
int const kInterleave = 4;
|
||||
|
||||
cutlass::Coord<2> bounds = cutlass::make_Coord(M, N);
|
||||
|
||||
// Define a mapping function for column-major interleaved layout
|
||||
typedef cutlass::MatrixLayout::ColumnMajorInterleaved<kInterleave> TensorRefMapFunc;
|
||||
|
||||
// Construct a rank=2 host tensor of size M-by-N
|
||||
cutlass::HostTensor<
|
||||
int,
|
||||
2,
|
||||
TensorRefMapFunc > host_tensor(TensorRefMapFunc::stride(M), bounds);
|
||||
|
||||
// Fill it with zeros and synchronize device
|
||||
host_tensor.fill(0);
|
||||
host_tensor.sync_device();
|
||||
|
||||
// Launch a CUDA kernel by obtaining a TensorView of the device memory
|
||||
dim3 block(16, 16);
|
||||
dim3 grid((M + block.x - 1) / block.x, (N + block.y - 1) / block.y);
|
||||
|
||||
test::fill_sequential<<< grid, block >>>(host_tensor.device_view());
|
||||
|
||||
ASSERT_EQ(cudaDeviceSynchronize(), cudaSuccess);
|
||||
|
||||
// Synchronize the host data
|
||||
host_tensor.sync_host();
|
||||
|
||||
// Verify host_tensor contains sequential elements
|
||||
int errors = 0;
|
||||
for (int n = 0; n < N; ++n) {
|
||||
for (int m = 0; m < M; ++m) {
|
||||
int expected = m + n * M;
|
||||
int got = host_tensor.at(cutlass::make_Coord(m, n));
|
||||
if (got != expected) {
|
||||
++errors;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
EXPECT_EQ(errors, 0) << std::setw(4) << host_tensor << std::endl;
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
//
|
||||
// cutlass::HostMatrix extends cutlass::HostTensor of rank=2 to facilitate allocate and operating
|
||||
// on matrices in device memory.
|
||||
//
|
||||
// cutlass::HostMatrix<T> accommodates both row-major and column-major matrices with a single
|
||||
// leading dimension.
|
||||
//
|
||||
// The first test demonstrates use of HostMatrix<> in the same circumstances as HostTensor but with
|
||||
// simplifcations to the calling interface.
|
||||
//
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
// This test constructs a CUTLASS cutlass::HostMatrix with column-major layout.
|
||||
TEST(HostMatrix, fill_sequential_column_major) {
|
||||
|
||||
int const M = 16;
|
||||
int const N = 32;
|
||||
int const ldm = M + 2; // define leading dimension with padding
|
||||
|
||||
cutlass::Coord<2> bounds = cutlass::make_Coord(M, N);
|
||||
|
||||
// Construct a HostMatrix of size M-by-N with leading dimension ldm
|
||||
cutlass::HostMatrix<int> host_matrix(bounds, cutlass::MatrixLayout::kColumnMajor, ldm);
|
||||
|
||||
// Fill it with zeros and synchronize device
|
||||
host_matrix.fill(0);
|
||||
host_matrix.sync_device();
|
||||
|
||||
// Launch a CUDA kernel by obtaining a TensorView of the device memory
|
||||
dim3 block(16, 16);
|
||||
dim3 grid((M + block.x - 1) / block.x, (N + block.y - 1) / block.y);
|
||||
|
||||
test::fill_sequential<<< grid, block >>>(host_matrix.device_view());
|
||||
|
||||
ASSERT_EQ(cudaDeviceSynchronize(), cudaSuccess);
|
||||
|
||||
// Synchronize the host data
|
||||
host_matrix.sync_host();
|
||||
|
||||
// Verify host_matrix contains sequential elements
|
||||
int errors = 0;
|
||||
for (int n = 0; n < N; ++n) {
|
||||
for (int m = 0; m < M; ++m) {
|
||||
int expected = m + n * M;
|
||||
int got = host_matrix.at(cutlass::make_Coord(m, n));
|
||||
if (expected != got) {
|
||||
++errors;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
EXPECT_EQ(errors, 0) << std::setw(4) << host_matrix << std::endl;
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
//
|
||||
// Previously, cutlass::HostTensorView<> offered a gemm() method defined for the H and W dimensions.
|
||||
// The other dimensions were ignored.
|
||||
//
|
||||
// To improve the interface, we We have moved this into the HostMatrixView<> and HostMatrix<>
|
||||
// classes which require rank=2. To accommodate matrix operands of differing layout, we have extracted
|
||||
// the host-side GEMM implementation into cutlass::reference::host::Gemm() which can compute the
|
||||
// general matrix product of matrices with arbitrary layout.
|
||||
//
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
// This test constructs a CUTLASS cutlass::HostMatrix with column-major layout.
|
||||
TEST(HostMatrix, gemm) {
|
||||
|
||||
// Problem size intentionally small, as reference check has complexity O(MNK).
|
||||
int const M = 32;
|
||||
int const N = 16;
|
||||
int const K = 4;
|
||||
|
||||
int const lda = M;
|
||||
int const ldb = N;
|
||||
int const ldc = M;
|
||||
|
||||
// Construct matrix operands
|
||||
cutlass::HostMatrix<int> A(cutlass::make_Coord(M, K), cutlass::MatrixLayout::kColumnMajor, lda);
|
||||
cutlass::HostMatrix<int> B(cutlass::make_Coord(K, N), cutlass::MatrixLayout::kRowMajor, ldb);
|
||||
cutlass::HostMatrix<int> C(cutlass::make_Coord(M, N), cutlass::MatrixLayout::kColumnMajor, ldc);
|
||||
|
||||
A.fill_sequential();
|
||||
B.fill_sequential();
|
||||
C.fill(0);
|
||||
|
||||
int alpha = 1;
|
||||
|
||||
// Compute host-side GEMM reference
|
||||
cutlass::reference::host::Gemm(
|
||||
cutlass::gemm::GemmCoord(K, N, M),
|
||||
alpha,
|
||||
A.host_ref(),
|
||||
B.host_ref(),
|
||||
int(0), // beta
|
||||
C.host_ref());
|
||||
|
||||
// Verify result
|
||||
int errors = 0;
|
||||
|
||||
// Primitive reference implementation for matrix product
|
||||
for (int i = 0; i < M; ++i) {
|
||||
for (int j = 0; j < N; ++j) {
|
||||
int result = 0;
|
||||
for (int k = 0; k < K; ++k) {
|
||||
result += A.at(cutlass::make_Coord(i, k)) * B.at(cutlass::make_Coord(k, j));
|
||||
}
|
||||
if (C.at(cutlass::make_Coord(i, j)) != alpha * result) {
|
||||
++errors;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
EXPECT_EQ(errors, 0) << "GEMM error\n"
|
||||
<< "A =\n" << A << "\nB = \n" << B << "\nC =\n" << C << "\n";
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
// When layout is known at compile time, we may be use the corresponding helper classes to smplify
|
||||
// matrix instantiation. The matrix layout becomes part of the type which reduces the StorageRank
|
||||
// of the internal stride vector.
|
||||
//
|
||||
// Apart from specifying the matrix layout at compile time, this test is functionally identical to
|
||||
// HostMatrix.gemm.
|
||||
//
|
||||
TEST(HostMatrix, gemm_compile_time_layout) {
|
||||
|
||||
// Problem size intentionally small, as reference check has complexity O(MNK).
|
||||
int const M = 32;
|
||||
int const N = 16;
|
||||
int const K = 4;
|
||||
|
||||
int const lda = M;
|
||||
int const ldb = N;
|
||||
int const ldc = M;
|
||||
|
||||
// Construct matrix operands
|
||||
cutlass::HostMatrixColumnMajor<int> A(cutlass::make_Coord(M, K), lda);
|
||||
cutlass::HostMatrixRowMajor<int> B(cutlass::make_Coord(K, N), ldb);
|
||||
cutlass::HostMatrixColumnMajor<int> C(cutlass::make_Coord(M, N), ldc);
|
||||
|
||||
A.fill_sequential();
|
||||
B.fill_sequential();
|
||||
C.fill(0);
|
||||
|
||||
int alpha = 1;
|
||||
|
||||
// Compute host-side GEMM reference
|
||||
cutlass::reference::host::Gemm(
|
||||
cutlass::gemm::GemmCoord(K, N, M),
|
||||
alpha,
|
||||
A.host_ref(),
|
||||
B.host_ref(),
|
||||
int(0), // beta
|
||||
C.host_ref());
|
||||
|
||||
// Verify result
|
||||
int errors = 0;
|
||||
|
||||
// Primitive reference implementation for matrix product
|
||||
for (int i = 0; i < M; ++i) {
|
||||
for (int j = 0; j < N; ++j) {
|
||||
int result = 0;
|
||||
for (int k = 0; k < K; ++k) {
|
||||
result += A.at(cutlass::make_Coord(i, k)) * B.at(cutlass::make_Coord(k, j));
|
||||
}
|
||||
if (C.at(cutlass::make_Coord(i, j)) != alpha * result) {
|
||||
++errors;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
EXPECT_EQ(errors, 0) << "GEMM error\n"
|
||||
<< "A =\n" << A << "\nB = \n" << B << "\nC =\n" << C << "\n";
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
@@ -0,0 +1,324 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/* \file
|
||||
|
||||
\brief
|
||||
|
||||
These tests initialize host- and device-side tensors according to several random distributions.
|
||||
*/
|
||||
|
||||
#include "cutlass_unit_test.h"
|
||||
|
||||
#include "cutlass/matrix_traits.h"
|
||||
|
||||
#include "tools/util/tensor_view_io.h"
|
||||
#include "tools/util/host_tensor.h"
|
||||
#include "tools/util/host_matrix.h"
|
||||
|
||||
#include "tools/util/reference/device/tensor_foreach.h"
|
||||
#include "tools/util/reference/device/tensor_elementwise.h"
|
||||
|
||||
#include "tools/util/reference/host/tensor_foreach.h"
|
||||
#include "tools/util/reference/host/tensor_elementwise.h"
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#define ENABLE_OUTPUT 0 // Supress output by default.
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(TensorInitialize, uniform_device) {
|
||||
|
||||
// Define the problem size
|
||||
int const M = 517;
|
||||
int const N = 117;
|
||||
|
||||
// Define HostMatrix type
|
||||
typedef cutlass::HostMatrix<float> HostMatrix;
|
||||
|
||||
// Construct the host matrix
|
||||
HostMatrix source(cutlass::MatrixCoord(M, N), cutlass::MatrixLayout::kRowMajor);
|
||||
source.fill(0);
|
||||
|
||||
// Initialize the source matrix with a uniform distribution
|
||||
cutlass::Distribution dist;
|
||||
dist.set_uniform(0, 128, -1);
|
||||
|
||||
// RNG seed is hard-coded for determinism in the test.
|
||||
unsigned seed = 2080;
|
||||
|
||||
cutlass::reference::device::TensorInitialize(source.device_view(), seed, dist);
|
||||
|
||||
source.sync_host();
|
||||
|
||||
if (ENABLE_OUTPUT) {
|
||||
std::ofstream result("TensorInitialize_uniform_device.csv");
|
||||
|
||||
for (int i = 0; i < M; ++i) {
|
||||
for (int j = 0; j < N; ++j) {
|
||||
result << source.at(cutlass::make_Coord(i, j)) << "\n";
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
TEST(TensorInitialize, uniform_host) {
|
||||
|
||||
// Define the problem size
|
||||
int const M = 517;
|
||||
int const N = 117;
|
||||
|
||||
bool const kDeviceBacked = false;
|
||||
|
||||
// Define HostMatrix type
|
||||
typedef cutlass::HostMatrix<float> HostMatrix;
|
||||
|
||||
// Construct the host matrix
|
||||
HostMatrix source(cutlass::MatrixCoord(M, N), cutlass::MatrixLayout::kRowMajor, kDeviceBacked);
|
||||
source.fill(0);
|
||||
|
||||
// Initialize the source matrix with a uniform distribution
|
||||
cutlass::Distribution dist;
|
||||
dist.set_uniform(0, 128, -1);
|
||||
|
||||
// RNG seed is hard-coded for determinism in the test.
|
||||
unsigned seed = 2080;
|
||||
|
||||
cutlass::reference::host::TensorInitialize(source.host_view(), seed, dist);
|
||||
|
||||
if (ENABLE_OUTPUT) {
|
||||
std::ofstream result("TensorInitialize_uniform_host.csv");
|
||||
|
||||
for (int i = 0; i < M; ++i) {
|
||||
for (int j = 0; j < N; ++j) {
|
||||
result << source.at(cutlass::make_Coord(i, j)) << "\n";
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
TEST(TensorInitialize, gaussian_device) {
|
||||
|
||||
// Define the problem size
|
||||
int const M = 517;
|
||||
int const N = 117;
|
||||
|
||||
|
||||
// Define HostMatrix type
|
||||
typedef cutlass::HostMatrix<float> HostMatrix;
|
||||
|
||||
// Construct the host matrix
|
||||
HostMatrix source(cutlass::MatrixCoord(M, N), cutlass::MatrixLayout::kRowMajor);
|
||||
source.fill(0);
|
||||
|
||||
// Initialize the source matrix with a uniform distribution
|
||||
cutlass::Distribution dist;
|
||||
dist.set_gaussian(1, 2, -1);
|
||||
|
||||
// RNG seed is hard-coded for determinism in the test.
|
||||
unsigned seed = 2080;
|
||||
|
||||
cutlass::reference::device::TensorInitialize(source.device_view(), seed, dist);
|
||||
|
||||
source.sync_host();
|
||||
|
||||
if (ENABLE_OUTPUT) {
|
||||
std::ofstream result("TensorInitialize_gaussian_device.csv");
|
||||
|
||||
for (int i = 0; i < M; ++i) {
|
||||
for (int j = 0; j < N; ++j) {
|
||||
result << source.at(cutlass::make_Coord(i, j)) << "\n";
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
TEST(TensorInitialize, gaussian_host) {
|
||||
// Define the problem size
|
||||
int const M = 517;
|
||||
int const N = 117;
|
||||
|
||||
bool const kDeviceBacked = false;
|
||||
|
||||
// Define HostMatrix type
|
||||
typedef cutlass::HostMatrix<float> HostMatrix;
|
||||
|
||||
// Construct the host matrix
|
||||
HostMatrix source(cutlass::MatrixCoord(M, N), cutlass::MatrixLayout::kRowMajor, kDeviceBacked);
|
||||
source.fill(0);
|
||||
|
||||
// Initialize the source matrix with a uniform distribution
|
||||
cutlass::Distribution dist;
|
||||
dist.set_gaussian(1, 2, -1);
|
||||
|
||||
// RNG seed is hard-coded for determinism in the test.
|
||||
unsigned seed = 2080;
|
||||
|
||||
cutlass::reference::host::TensorInitialize(source.host_view(), seed, dist);
|
||||
|
||||
if (ENABLE_OUTPUT) {
|
||||
std::ofstream result("TensorInitialize_gaussian_host.csv");
|
||||
|
||||
for (int i = 0; i < M; ++i) {
|
||||
for (int j = 0; j < N; ++j) {
|
||||
result << source.at(cutlass::make_Coord(i, j)) << "\n";
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
//
|
||||
// Interleaved matrix layouts
|
||||
//
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(TensorInitialize, interleaved_gaussian_device) {
|
||||
|
||||
// Define the problem size
|
||||
int const M = 512;
|
||||
int const N = 128;
|
||||
|
||||
// Define a mapping function for column-major interleaved layout
|
||||
int const kInterleave = 4;
|
||||
typedef cutlass::MatrixLayout::ColumnMajorInterleaved<kInterleave> TensorRefMapFunc;
|
||||
|
||||
// Construct a rank=2 host tensor of size M-by-N
|
||||
cutlass::HostTensor<
|
||||
float,
|
||||
2,
|
||||
TensorRefMapFunc > source(TensorRefMapFunc::stride(M), cutlass::make_Coord(M, N));
|
||||
|
||||
source.fill(0);
|
||||
|
||||
// Initialize the source matrix with a uniform distribution
|
||||
cutlass::Distribution dist;
|
||||
dist.set_gaussian(1, 2, -1);
|
||||
|
||||
// RNG seed is hard-coded for determinism in the test.
|
||||
unsigned seed = 2080;
|
||||
|
||||
cutlass::reference::device::TensorInitialize(source.device_view(), seed, dist);
|
||||
|
||||
source.sync_host();
|
||||
|
||||
if (ENABLE_OUTPUT) {
|
||||
std::ofstream result("TensorInitialize_interleaved_gaussian_device.csv");
|
||||
|
||||
for (int i = 0; i < M; ++i) {
|
||||
for (int j = 0; j < N; ++j) {
|
||||
result << source.at(cutlass::make_Coord(i, j)) << "\n";
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
TEST(TensorInitialize, interleaved_gaussian_host) {
|
||||
// Define the problem size
|
||||
int const M = 512;
|
||||
int const N = 128;
|
||||
|
||||
bool const kDeviceBacked = false;
|
||||
|
||||
// Define a mapping function for column-major interleaved layout
|
||||
int const kInterleave = 4;
|
||||
typedef cutlass::MatrixLayout::ColumnMajorInterleaved<kInterleave> TensorRefMapFunc;
|
||||
|
||||
// Construct a rank=2 host tensor of size M-by-N
|
||||
cutlass::HostTensor<
|
||||
float,
|
||||
2,
|
||||
TensorRefMapFunc > source(TensorRefMapFunc::stride(M), cutlass::make_Coord(M, N), kDeviceBacked);
|
||||
|
||||
// Construct the host matrix
|
||||
source.fill(0);
|
||||
|
||||
// Initialize the source matrix with a uniform distribution
|
||||
cutlass::Distribution dist;
|
||||
dist.set_gaussian(1, 2, -1);
|
||||
|
||||
// RNG seed is hard-coded for determinism in the test.
|
||||
unsigned seed = 2080;
|
||||
|
||||
cutlass::reference::host::TensorInitialize(source.host_view(), seed, dist);
|
||||
|
||||
if (ENABLE_OUTPUT) {
|
||||
std::ofstream result("TensorInitialize_interleaved_gaussian_host.csv");
|
||||
|
||||
for (int i = 0; i < M; ++i) {
|
||||
for (int j = 0; j < N; ++j) {
|
||||
result << source.at(cutlass::make_Coord(i, j)) << "\n";
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
//
|
||||
// Comparison operator
|
||||
//
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(TensorEquals, interleaved_device) {
|
||||
|
||||
// Define the problem size
|
||||
int const M = 512;
|
||||
int const N = 128;
|
||||
|
||||
// Define a mapping function for column-major interleaved layout
|
||||
int const kInterleave = 4;
|
||||
typedef cutlass::MatrixLayout::ColumnMajorInterleaved<kInterleave> TensorRefMapFunc;
|
||||
|
||||
// Construct two rank=2 host tensor of size M-by-N
|
||||
cutlass::HostTensor<
|
||||
float,
|
||||
2,
|
||||
TensorRefMapFunc > left(TensorRefMapFunc::stride(M), cutlass::make_Coord(M, N));
|
||||
|
||||
cutlass::HostTensor<
|
||||
float,
|
||||
2,
|
||||
TensorRefMapFunc > right(TensorRefMapFunc::stride(M), cutlass::make_Coord(M, N));
|
||||
|
||||
// Initialize
|
||||
left.fill_sequential();
|
||||
right.fill_sequential();
|
||||
|
||||
// Assert equality
|
||||
EXPECT_TRUE(cutlass::reference::device::TensorEquals(left.device_view(), right.device_view()));
|
||||
|
||||
// Overwrite one with an unexpected element
|
||||
left.at(cutlass::make_Coord(24, 17)) = -1;
|
||||
left.sync_device();
|
||||
|
||||
// Assert inequality
|
||||
EXPECT_FALSE(cutlass::reference::device::TensorEquals(left.device_view(), right.device_view()));
|
||||
}
|
||||
|
||||
TEST(TensorEquals, interleaved_host) {
|
||||
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
@@ -0,0 +1,217 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/* \file
|
||||
|
||||
\brief
|
||||
|
||||
These tests are intended to demonstrate the CUTLASS reference implementation for basic for-each
|
||||
operators on the index space of TensorView objects. They instantiate a HostMatrix, initialize
|
||||
its elements with random data according to specified random distributions, and clamp the
|
||||
elements using a TensorForEach() operation.
|
||||
|
||||
Both device-side and host-side reference implementations are called.
|
||||
*/
|
||||
|
||||
#include "cutlass_unit_test.h"
|
||||
|
||||
#include "cutlass/matrix_traits.h"
|
||||
|
||||
#include "tools/util/tensor_view_io.h"
|
||||
#include "tools/util/host_tensor.h"
|
||||
#include "tools/util/host_matrix.h"
|
||||
|
||||
#include "tools/util/reference/device/tensor_foreach.h"
|
||||
#include "tools/util/reference/device/tensor_elementwise.h"
|
||||
|
||||
#include "tools/util/reference/host/tensor_foreach.h"
|
||||
#include "tools/util/reference/host/tensor_elementwise.h"
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
namespace test {
|
||||
|
||||
/// Define a functor that computes the ReLu operation on a tensor.
|
||||
template <typename View>
|
||||
struct ReLuFunc {
|
||||
|
||||
/// Coordinate of index space
|
||||
typedef typename View::TensorCoord TensorCoord;
|
||||
|
||||
/// Scalar type
|
||||
typedef typename View::Storage T;
|
||||
|
||||
//
|
||||
// Data members
|
||||
//
|
||||
|
||||
/// Tensor view
|
||||
View view;
|
||||
|
||||
/// ReLu threshold
|
||||
T threshold;
|
||||
|
||||
//
|
||||
// Methods
|
||||
//
|
||||
|
||||
/// Constructor
|
||||
CUTLASS_HOST_DEVICE
|
||||
ReLuFunc(View const &view, T threshold): view(view), threshold(threshold) { }
|
||||
|
||||
/// ReLu function
|
||||
CUTLASS_HOST_DEVICE
|
||||
void operator()(TensorCoord const &coord) {
|
||||
T value = view.at(coord);
|
||||
|
||||
if (value < threshold) {
|
||||
value = threshold;
|
||||
}
|
||||
|
||||
view.at(coord) = value;
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace test
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// This tests models the computation of ReLu using reference utility code.
|
||||
TEST(TensorForEach, ReLu_device) {
|
||||
|
||||
// Define HostMatrix type
|
||||
typedef cutlass::HostMatrix<float> HostMatrix;
|
||||
typedef typename HostMatrix::DeviceTensorView View;
|
||||
|
||||
// Define the problem size
|
||||
int const M = 517;
|
||||
int const N = 117;
|
||||
|
||||
float threshold = 0;
|
||||
|
||||
// Construct the host matrix
|
||||
HostMatrix source(cutlass::MatrixCoord(M, N), cutlass::MatrixLayout::kRowMajor);
|
||||
source.fill(0);
|
||||
|
||||
// Initialize the source matrix with a uniform distribution
|
||||
cutlass::Distribution dist;
|
||||
dist.set_uniform(-16, 16);
|
||||
|
||||
// RNG seed is hard-coded for determinism in the test.
|
||||
int64_t seed = 2080;
|
||||
|
||||
cutlass::reference::device::TensorInitialize(source.device_view(), seed, dist);
|
||||
|
||||
// Define a functor called by TensorForEach<>
|
||||
typedef test::ReLuFunc<View> ReLuFunc;
|
||||
|
||||
// Instantiate on host with TensorView and threshold value
|
||||
ReLuFunc relu_func(source.device_view(), threshold);
|
||||
|
||||
// Launch kernel that applies the element-wise operator over the tensor's index space.
|
||||
cutlass::reference::device::TensorForEach<
|
||||
ReLuFunc,
|
||||
View::kRank,
|
||||
ReLuFunc>(source.size(), relu_func);
|
||||
|
||||
// Verify no element is less than the ReLu threshold.
|
||||
source.sync_host();
|
||||
|
||||
int errors = 0;
|
||||
for (cutlass::MatrixCoord coord(0, 0); coord.row() < M; ++coord.row()) {
|
||||
for (coord.column() = 0; coord.column() < N; ++coord.column()) {
|
||||
if (source.at(coord) < threshold) {
|
||||
++errors;
|
||||
if (errors < 10) {
|
||||
std::cout << "Error - source(" << coord << ") = "
|
||||
<< source.at(coord) << " is less than threshold " << threshold << std::endl;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
EXPECT_EQ(errors, 0)
|
||||
<< "Result: " << source;
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Test to apply the ReLu operation using host-side utilities
|
||||
TEST(TensorForEach, ReLu_host) {
|
||||
|
||||
// Define HostMatrix type
|
||||
typedef cutlass::HostMatrix<float> HostMatrix;
|
||||
typedef typename HostMatrix::HostTensorView View;
|
||||
|
||||
// Define the problem size
|
||||
int const M = 517;
|
||||
int const N = 117;
|
||||
|
||||
float threshold = 0;
|
||||
|
||||
bool const kDeviceBacked = false;
|
||||
|
||||
// Construct the host matrix
|
||||
HostMatrix source(cutlass::MatrixCoord(M, N), cutlass::MatrixLayout::kRowMajor, kDeviceBacked);
|
||||
source.fill(0);
|
||||
|
||||
// Initialize the source matrix with a uniform distribution
|
||||
cutlass::Distribution dist;
|
||||
dist.set_gaussian(-1, 4);
|
||||
|
||||
// RNG seed is hard-coded for determinism in the test.
|
||||
unsigned seed = 2080;
|
||||
|
||||
cutlass::reference::host::TensorInitialize(source.host_view(), seed, dist);
|
||||
|
||||
// Define a functor called by TensorForEach<>
|
||||
typedef test::ReLuFunc<View> ReLuFunc;
|
||||
|
||||
// Instantiate on host with TensorView and threshold value
|
||||
ReLuFunc relu_func(source.host_view(), threshold);
|
||||
|
||||
// Invoke host-side for-each computation on the tensor
|
||||
cutlass::reference::host::TensorForEach<
|
||||
ReLuFunc,
|
||||
View::kRank,
|
||||
ReLuFunc>(source.size(), relu_func);
|
||||
|
||||
int errors = 0;
|
||||
for (cutlass::MatrixCoord coord(0, 0); coord.row() < M; ++coord.row()) {
|
||||
for (coord.column() = 0; coord.column() < N; ++coord.column()) {
|
||||
if (source.at(coord) < threshold) {
|
||||
++errors;
|
||||
if (errors < 10) {
|
||||
std::cout << "Error - source(" << coord << ") = "
|
||||
<< source.at(coord) << " is less than threshold " << threshold << std::endl;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
EXPECT_EQ(errors, 0)
|
||||
<< "Result: " << source;
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
@@ -0,0 +1,25 @@
|
||||
/******************************************************************************
|
||||
* Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are not permitted.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
|
||||
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
* DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY
|
||||
* DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
|
||||
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
|
||||
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
|
||||
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
|
||||
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
******************************************************************************/
|
||||
|
||||
#include "cutlass_unit_test.h"
|
||||
#include "cutlass/util/platform.h"
|
||||
|
||||
TEST(unique_ptr, basic) {
|
||||
cutlass::platform::unique_ptr<int> ptr(new int);
|
||||
}
|
||||
Reference in New Issue
Block a user