leavelet/cutlass
1
0
Fork 0
Code Issues Pull Requests Actions 26 Packages Projects Releases Wiki Activity

CUTLASS 1.2

Browse Source
This commit is contained in:
akerr
2018-10-26 14:38:46 -07:00
parent 2332df492e
commit 74df0331f2
97 changed files with 11301 additions and 632 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/test/unit/gemm/batched_strided_fp16_wmma_gemm_fp16.cu
+385
View File
@@ -0,0 +1,385 @@
/***************************************************************************************************
* Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
* * Redistributions of source code must retain the above copyright notice, this list of
* conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright notice, this list of
* conditions and the following disclaimer in the documentation and/or other materials
* provided with the distribution.
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
* to endorse or promote products derived from this software without specific prior written
* permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
#include "cutlass/wmma_matrix.h"
#if defined(CUTLASS_USE_WMMA_API)
#include "cutlass_unit_test.h"
#include "cutlass/gemm/gemm.h"
#include "cutlass/gemm/wmma_gemm_traits.h"
#include "tools/test/unit/gemm/gemm_testbed.h"
#include "tools/test/unit/gemm/run_gemm.h"
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(WmmaGemm_strided_batched_16x16x32_f32, fp16_wmma_gemm_fp16_32x32x16_nn) {
typedef float accumu_type;
typedef half c_type;
typedef cutlass::gemm::WmmaGemmTraits<
cutlass::MatrixLayout::kColumnMajor,
cutlass::MatrixLayout::kColumnMajor,
cutlass::Shape<32, 16, 16>,
half,
half,
c_type,
cutlass::gemm::LinearScaling<accumu_type>,
accumu_type,
typename cutlass::gemm::WmmaGemmAccumulatorsPerWarp<typename cutlass::Shape<32, 16, 16> >::Shape,
typename cutlass::Shape<16, 16, 16>,
8, /*kScalarsPerLdgA_*/
8, /*kScalarsPerLdgB_*/
8, /*KScalarsPerLdsA_*/
8, /*KScalarsPerLdsB_*/
16 / sizeof(c_type), /*kScalarsPerLdgCAndStgD_*/
16 / sizeof(accumu_type), /*kScalarsPerStsD_*/
16 / sizeof(accumu_type) /*kScalarsPerLdsD_*/
>
WmmaGemmTraits;
run_batched_strided_gemm<WmmaGemmTraits>(32, 32, 64, 3);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(WmmaGemm_strided_batched_16x16x32_f32, fp16_wmma_gemm_fp16_32x32x16_nt) {
typedef float accumu_type;
typedef half c_type;
typedef cutlass::gemm::WmmaGemmTraits<
cutlass::MatrixLayout::kColumnMajor,
cutlass::MatrixLayout::kRowMajor,
cutlass::Shape<32, 16, 16>,
half,
half,
c_type,
cutlass::gemm::LinearScaling<accumu_type>,
accumu_type,
typename cutlass::gemm::WmmaGemmAccumulatorsPerWarp<typename cutlass::Shape<32, 16, 16> >::Shape,
typename cutlass::Shape<16, 16, 16>,
8, /*kScalarsPerLdgA_*/
8, /*kScalarsPerLdgB_*/
8, /*KScalarsPerLdsA_*/
8, /*KScalarsPerLdsB_*/
16 / sizeof(c_type), /*kScalarsPerLdgCAndStgD_*/
16 / sizeof(accumu_type), /*kScalarsPerStsD_*/
16 / sizeof(accumu_type) /*kScalarsPerLdsD_*/
>
WmmaGemmTraits;
run_batched_strided_gemm<WmmaGemmTraits>(32, 32, 64, 3);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(WmmaGemm_strided_batched_16x16x32_f32, fp16_wmma_gemm_fp16_32x32x16_tn) {
typedef float accumu_type;
typedef half c_type;
typedef cutlass::gemm::WmmaGemmTraits<
cutlass::MatrixLayout::kRowMajor,
cutlass::MatrixLayout::kColumnMajor,
cutlass::Shape<32, 16, 16>,
half,
half,
c_type,
cutlass::gemm::LinearScaling<accumu_type>,
accumu_type,
typename cutlass::gemm::WmmaGemmAccumulatorsPerWarp<typename cutlass::Shape<32, 16, 16> >::Shape,
typename cutlass::Shape<16, 16, 16>,
8, /*kScalarsPerLdgA_*/
8, /*kScalarsPerLdgB_*/
8, /*KScalarsPerLdsA_*/
8, /*KScalarsPerLdsB_*/
16 / sizeof(c_type), /*kScalarsPerLdgCAndStgD_*/
16 / sizeof(accumu_type), /*kScalarsPerStsD_*/
16 / sizeof(accumu_type) /*kScalarsPerLdsD_*/
>
WmmaGemmTraits;
run_batched_strided_gemm<WmmaGemmTraits>(32, 32, 64, 3);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(WmmaGemm_strided_batched_16x16x32_f32, fp16_wmma_gemm_fp16_32x32x16_tt) {
typedef float accumu_type;
typedef half c_type;
typedef cutlass::gemm::WmmaGemmTraits<
cutlass::MatrixLayout::kRowMajor,
cutlass::MatrixLayout::kRowMajor,
cutlass::Shape<32, 16, 16>,
half,
half,
c_type,
cutlass::gemm::LinearScaling<accumu_type>,
accumu_type,
typename cutlass::gemm::WmmaGemmAccumulatorsPerWarp<typename cutlass::Shape<32, 16, 16> >::Shape,
typename cutlass::Shape<16, 16, 16>,
8, /*kScalarsPerLdgA_*/
8, /*kScalarsPerLdgB_*/
8, /*KScalarsPerLdsA_*/
8, /*KScalarsPerLdsB_*/
16 / sizeof(c_type), /*kScalarsPerLdgCAndStgD_*/
16 / sizeof(accumu_type), /*kScalarsPerStsD_*/
16 / sizeof(accumu_type) /*kScalarsPerLdsD_*/
>
WmmaGemmTraits;
run_batched_strided_gemm<WmmaGemmTraits>(32, 32, 64, 3);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
//mulitple of 4
TEST(WmmaGemm_strided_batched_16x16x32_f32, fp16_wmma_gemm_fp16_36x36x16_nn) {
typedef float accumu_type;
typedef half c_type;
typedef cutlass::gemm::WmmaGemmTraits<
cutlass::MatrixLayout::kColumnMajor,
cutlass::MatrixLayout::kColumnMajor,
cutlass::Shape<32, 16, 16>,
half,
half,
c_type,
cutlass::gemm::LinearScaling<accumu_type>,
accumu_type,
typename cutlass::gemm::WmmaGemmAccumulatorsPerWarp<typename cutlass::Shape<32, 16, 16> >::Shape,
typename cutlass::Shape<16, 16, 16>,
4, /*kScalarsPerLdgA_*/
4, /*kScalarsPerLdgB_*/
4, /*KScalarsPerLdsA_*/
4, /*KScalarsPerLdsB_*/
8 / sizeof(c_type), /*kScalarsPerLdgCAndStgD_*/
8 / sizeof(accumu_type), /*kScalarsPerStsD_*/
8 / sizeof(accumu_type) /*kScalarsPerLdsD_*/
>
WmmaGemmTraits;
run_batched_strided_gemm<WmmaGemmTraits>(36, 36, 64, 3);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(WmmaGemm_strided_batched_16x16x32_f32, fp16_wmma_gemm_fp16_36x36x16_nt) {
typedef float accumu_type;
typedef half c_type;
typedef cutlass::gemm::WmmaGemmTraits<
cutlass::MatrixLayout::kColumnMajor,
cutlass::MatrixLayout::kRowMajor,
cutlass::Shape<32, 16, 16>,
half,
half,
c_type,
cutlass::gemm::LinearScaling<accumu_type>,
accumu_type,
typename cutlass::gemm::WmmaGemmAccumulatorsPerWarp<typename cutlass::Shape<32, 16, 16> >::Shape,
typename cutlass::Shape<16, 16, 16>,
4, /*kScalarsPerLdgA_*/
4, /*kScalarsPerLdgB_*/
4, /*KScalarsPerLdsA_*/
4, /*KScalarsPerLdsB_*/
8 / sizeof(c_type), /*kScalarsPerLdgCAndStgD_*/
8 / sizeof(accumu_type), /*kScalarsPerStsD_*/
8 / sizeof(accumu_type) /*kScalarsPerLdsD_*/
>
WmmaGemmTraits;
run_batched_strided_gemm<WmmaGemmTraits>(36, 36, 64, 3);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(WmmaGemm_strided_batched_16x16x32_f32, fp16_wmma_gemm_fp16_36x36x16_tn) {
typedef float accumu_type;
typedef half c_type;
typedef cutlass::gemm::WmmaGemmTraits<
cutlass::MatrixLayout::kRowMajor,
cutlass::MatrixLayout::kColumnMajor,
cutlass::Shape<32, 16, 16>,
half,
half,
c_type,
cutlass::gemm::LinearScaling<accumu_type>,
accumu_type,
typename cutlass::gemm::WmmaGemmAccumulatorsPerWarp<typename cutlass::Shape<32, 16, 16> >::Shape,
typename cutlass::Shape<16, 16, 16>,
4, /*kScalarsPerLdgA_*/
4, /*kScalarsPerLdgB_*/
4, /*KScalarsPerLdsA_*/
4, /*KScalarsPerLdsB_*/
8 / sizeof(c_type), /*kScalarsPerLdgCAndStgD_*/
8 / sizeof(accumu_type), /*kScalarsPerStsD_*/
8 / sizeof(accumu_type) /*kScalarsPerLdsD_*/
>
WmmaGemmTraits;
run_batched_strided_gemm<WmmaGemmTraits>(36, 36, 64, 3);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(WmmaGemm_strided_batched_16x16x32_f32, fp16_wmma_gemm_fp16_36x36x16_tt) {
typedef float accumu_type;
typedef half c_type;
typedef cutlass::gemm::WmmaGemmTraits<
cutlass::MatrixLayout::kRowMajor,
cutlass::MatrixLayout::kRowMajor,
cutlass::Shape<32, 16, 16>,
half,
half,
c_type,
cutlass::gemm::LinearScaling<accumu_type>,
accumu_type,
typename cutlass::gemm::WmmaGemmAccumulatorsPerWarp<typename cutlass::Shape<32, 16, 16> >::Shape,
typename cutlass::Shape<16, 16, 16>,
4, /*kScalarsPerLdgA_*/
4, /*kScalarsPerLdgB_*/
4, /*KScalarsPerLdsA_*/
4, /*KScalarsPerLdsB_*/
8 / sizeof(c_type), /*kScalarsPerLdgCAndStgD_*/
8 / sizeof(accumu_type), /*kScalarsPerStsD_*/
8 / sizeof(accumu_type) /*kScalarsPerLdsD_*/
>
WmmaGemmTraits;
run_batched_strided_gemm<WmmaGemmTraits>(36, 36, 64, 3);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
//mulitple of 2
TEST(WmmaGemm_strided_batched_16x16x32_f32, fp16_wmma_gemm_fp16_34x34x16_nn) {
typedef float accumu_type;
typedef half c_type;
typedef cutlass::gemm::WmmaGemmTraits<
cutlass::MatrixLayout::kColumnMajor,
cutlass::MatrixLayout::kColumnMajor,
cutlass::Shape<32, 16, 16>,
half,
half,
c_type,
cutlass::gemm::LinearScaling<accumu_type>,
accumu_type,
typename cutlass::gemm::WmmaGemmAccumulatorsPerWarp<typename cutlass::Shape<32, 16, 16> >::Shape,
typename cutlass::Shape<16, 16, 16>,
2, /*kScalarsPerLdgA_*/
2, /*kScalarsPerLdgB_*/
2, /*KScalarsPerLdsA_*/
2, /*KScalarsPerLdsB_*/
4 / sizeof(c_type), /*kScalarsPerLdgCAndStgD_*/
4 / sizeof(accumu_type), /*kScalarsPerStsD_*/
4 / sizeof(accumu_type) /*kScalarsPerLdsD_*/
>
WmmaGemmTraits;
run_batched_strided_gemm<WmmaGemmTraits>(34, 34, 64, 3);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(WmmaGemm_strided_batched_16x16x32_f32, fp16_wmma_gemm_fp16_34x34x16_nt) {
typedef float accumu_type;
typedef half c_type;
typedef cutlass::gemm::WmmaGemmTraits<
cutlass::MatrixLayout::kColumnMajor,
cutlass::MatrixLayout::kRowMajor,
cutlass::Shape<32, 16, 16>,
half,
half,
c_type,
cutlass::gemm::LinearScaling<accumu_type>,
accumu_type,
typename cutlass::gemm::WmmaGemmAccumulatorsPerWarp<typename cutlass::Shape<32, 16, 16> >::Shape,
typename cutlass::Shape<16, 16, 16>,
2, /*kScalarsPerLdgA_*/
2, /*kScalarsPerLdgB_*/
2, /*KScalarsPerLdsA_*/
2, /*KScalarsPerLdsB_*/
4 / sizeof(c_type), /*kScalarsPerLdgCAndStgD_*/
4 / sizeof(accumu_type), /*kScalarsPerStsD_*/
4 / sizeof(accumu_type) /*kScalarsPerLdsD_*/
>
WmmaGemmTraits;
run_batched_strided_gemm<WmmaGemmTraits>(34, 34, 64, 3);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(WmmaGemm_strided_batched_16x16x32_f32, fp16_wmma_gemm_fp16_34x34x16_tn) {
typedef float accumu_type;
typedef half c_type;
typedef cutlass::gemm::WmmaGemmTraits<
cutlass::MatrixLayout::kRowMajor,
cutlass::MatrixLayout::kColumnMajor,
cutlass::Shape<32, 16, 16>,
half,
half,
c_type,
cutlass::gemm::LinearScaling<accumu_type>,
accumu_type,
typename cutlass::gemm::WmmaGemmAccumulatorsPerWarp<typename cutlass::Shape<32, 16, 16> >::Shape,
typename cutlass::Shape<16, 16, 16>,
2, /*kScalarsPerLdgA_*/
2, /*kScalarsPerLdgB_*/
2, /*KScalarsPerLdsA_*/
2, /*KScalarsPerLdsB_*/
4 / sizeof(c_type), /*kScalarsPerLdgCAndStgD_*/
4 / sizeof(accumu_type), /*kScalarsPerStsD_*/
4 / sizeof(accumu_type) /*kScalarsPerLdsD_*/
>
WmmaGemmTraits;
run_batched_strided_gemm<WmmaGemmTraits>(34, 34, 64, 3);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(WmmaGemm_strided_batched_16x16x32_f32, fp16_wmma_gemm_fp16_34x34x16_tt) {
typedef float accumu_type;
typedef half c_type;
typedef cutlass::gemm::WmmaGemmTraits<
cutlass::MatrixLayout::kRowMajor,
cutlass::MatrixLayout::kRowMajor,
cutlass::Shape<32, 16, 16>,
half,
half,
c_type,
cutlass::gemm::LinearScaling<accumu_type>,
accumu_type,
typename cutlass::gemm::WmmaGemmAccumulatorsPerWarp<typename cutlass::Shape<32, 16, 16> >::Shape,
typename cutlass::Shape<16, 16, 16>,
2, /*kScalarsPerLdgA_*/
2, /*kScalarsPerLdgB_*/
2, /*KScalarsPerLdsA_*/
2, /*KScalarsPerLdsB_*/
4 / sizeof(c_type), /*kScalarsPerLdgCAndStgD_*/
4 / sizeof(accumu_type), /*kScalarsPerStsD_*/
4 / sizeof(accumu_type) /*kScalarsPerLdsD_*/
>
WmmaGemmTraits;
run_batched_strided_gemm<WmmaGemmTraits>(34, 34, 64, 3);
}
#endif
tools/test/unit/gemm/batched_strided_sgemm_128x128x8.cu
+10 -1
View File
@@ -34,6 +34,7 @@ TEST(Sgemm_strided_batched_128x128x8, sgemm_256x384x64x3_nn) {
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor,
cutlass::MatrixLayout::kColumnMajor, cutlass::Shape<8, 128, 128> >
SgemmTraits;
//think about using run_gemm directly
run_batched_strided_gemm<SgemmTraits>(256/*m*/, 384/*n*/, 64/*k*/, 3 /*batch_size*/);
}
@@ -43,6 +44,7 @@ TEST(Sgemm_strided_batched_128x128x8, sgemm_128x384x192x2_nn) {
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor,
cutlass::MatrixLayout::kColumnMajor, cutlass::Shape<8, 128, 128> >
SgemmTraits;
//think about using run_gemm directly
run_batched_strided_gemm<SgemmTraits>(128/*m*/, 384/*n*/, 192/*k*/, 2 /*batch_size*/);
}
@@ -52,6 +54,7 @@ TEST(Sgemm_strided_batched_128x128x8, sgemm_127x384x192x2_nn) {
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor,
cutlass::MatrixLayout::kColumnMajor, cutlass::Shape<8, 128, 128> >
SgemmTraits;
//think about using run_gemm directly
run_batched_strided_gemm<SgemmTraits>(127/*m*/, 384/*n*/, 192/*k*/, 2 /*batch_size*/);
}
@@ -61,6 +64,7 @@ TEST(Sgemm_strided_batched_128x128x8, sgemm_127x388x190x2_nn) {
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor,
cutlass::MatrixLayout::kColumnMajor, cutlass::Shape<8, 128, 128> >
SgemmTraits;
//think about using run_gemm directly
run_batched_strided_gemm<SgemmTraits>(127/*m*/, 388/*n*/, 190/*k*/, 2 /*batch_size*/);
}
@@ -70,6 +74,7 @@ TEST(Sgemm_strided_batched_128x128x8, sgemm_256x384x64x3_nt) {
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor,
cutlass::MatrixLayout::kRowMajor, cutlass::Shape<8, 128, 128> >
SgemmTraits;
//think about using run_gemm directly
run_batched_strided_gemm<SgemmTraits>(256/*m*/, 384/*n*/, 64/*k*/, 3 /*batch_size*/);
}
@@ -79,6 +84,7 @@ TEST(Sgemm_strided_batched_128x128x8, sgemm_128x384x192x2_nt) {
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor,
cutlass::MatrixLayout::kRowMajor, cutlass::Shape<8, 128, 128> >
SgemmTraits;
//think about using run_gemm directly
run_batched_strided_gemm<SgemmTraits>(128/*m*/, 384/*n*/, 192/*k*/, 2 /*batch_size*/);
}
@@ -90,6 +96,7 @@ TEST(Sgemm_strided_batched_128x128x8, sgemm_256x384x64x3_tn) {
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kRowMajor,
cutlass::MatrixLayout::kColumnMajor, cutlass::Shape<8, 128, 128> >
SgemmTraits;
//think about using run_gemm directly
run_batched_strided_gemm<SgemmTraits>(256/*m*/, 384/*n*/, 64/*k*/, 3 /*batch_size*/);
}
@@ -99,6 +106,7 @@ TEST(Sgemm_strided_batched_128x128x8, sgemm_128x384x192x2_tn) {
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kRowMajor,
cutlass::MatrixLayout::kColumnMajor, cutlass::Shape<8, 128, 128> >
SgemmTraits;
//think about using run_gemm directly
run_batched_strided_gemm<SgemmTraits>(128/*m*/, 384/*n*/, 192/*k*/, 2 /*batch_size*/);
}
@@ -110,6 +118,7 @@ TEST(Sgemm_strided_batched_128x128x8, sgemm_256x384x64x3_tt) {
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kRowMajor,
cutlass::MatrixLayout::kRowMajor, cutlass::Shape<8, 128, 128> >
SgemmTraits;
//think about using run_gemm directly
run_batched_strided_gemm<SgemmTraits>(256/*m*/, 384/*n*/, 64/*k*/, 3 /*batch_size*/);
}
@@ -119,8 +128,8 @@ TEST(Sgemm_strided_batched_128x128x8, sgemm_128x384x192x2_tt) {
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kRowMajor,
cutlass::MatrixLayout::kRowMajor, cutlass::Shape<8, 128, 128> >
SgemmTraits;
//think about using run_gemm directly
run_batched_strided_gemm<SgemmTraits>(128/*m*/, 384/*n*/, 192/*k*/, 2 /*batch_size*/);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
tools/test/unit/gemm/batched_strided_wmma_gemm.cu
+240
View File
@@ -0,0 +1,240 @@
/***************************************************************************************************
* Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
* * Redistributions of source code must retain the above copyright notice, this list of
* conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright notice, this list of
* conditions and the following disclaimer in the documentation and/or other materials
* provided with the distribution.
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
* to endorse or promote products derived from this software without specific prior written
* permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
#include "cutlass/wmma_matrix.h"
#if defined(CUTLASS_USE_WMMA_API)
#include "cutlass_unit_test.h"
#include "cutlass/gemm/gemm.h"
#include "cutlass/gemm/wmma_gemm_traits.h"
#include "tools/test/unit/gemm/gemm_testbed.h"
#include "tools/test/unit/gemm/run_gemm.h"
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(WmmaGemm_strided_batched_16x16x32_f16, wmma_gemm_32x32x16_nn) {
/*
this wmmaTraits requires leading dim to be divisible by 4
*/
typedef cutlass::gemm::WmmaGemmTraits<
cutlass::MatrixLayout::kColumnMajor,
cutlass::MatrixLayout::kColumnMajor,
cutlass::Shape<32, 16, 16>,
half,
half,
half,
cutlass::gemm::LinearScaling<half>,
half
>
WmmaGemmTraits;
run_batched_strided_gemm<WmmaGemmTraits>(32, 32, 64, 3);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(WmmaGemm_strided_batched_16x16x32_f16, wmma_gemm_32x32x16_nt) {
/*
this wmmaTraits requires leading dim to be divisible by 4
*/
typedef cutlass::gemm::WmmaGemmTraits<
cutlass::MatrixLayout::kColumnMajor,
cutlass::MatrixLayout::kRowMajor,
cutlass::Shape<32, 16, 16>,
half,
half,
half,
cutlass::gemm::LinearScaling<half>,
half
>
WmmaGemmTraits;
run_batched_strided_gemm<WmmaGemmTraits>(32, 32, 64, 3);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(WmmaGemm_strided_batched_16x16x32_f16, wmma_gemm_32x32x16_tn) {
/*
this wmmaTraits requires leading dim to be divisible by 4
*/
typedef cutlass::gemm::WmmaGemmTraits<
cutlass::MatrixLayout::kRowMajor,
cutlass::MatrixLayout::kColumnMajor,
cutlass::Shape<32, 16, 16>,
half,
half,
half,
cutlass::gemm::LinearScaling<half>,
half
>
WmmaGemmTraits;
run_batched_strided_gemm<WmmaGemmTraits>(32, 32, 64, 3);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(WmmaGemm_strided_batched_16x16x32_f16, wmma_gemm_32x32x16_tt) {
/*
this wmmaTraits requires leading dim to be divisible by 4
*/
typedef cutlass::gemm::WmmaGemmTraits<
cutlass::MatrixLayout::kRowMajor,
cutlass::MatrixLayout::kRowMajor,
cutlass::Shape<32, 16, 16>,
half,
half,
half,
cutlass::gemm::LinearScaling<half>,
half
>
WmmaGemmTraits;
run_batched_strided_gemm<WmmaGemmTraits>(32, 32, 64, 3);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
//non multiple of 16
TEST(WmmaGemm_strided_batched_16x16x32_f16, wmma_gemm_36x36x16_nn) {
/*
this wmmaTraits requires leading dim to be divisible by 4
*/
typedef cutlass::gemm::WmmaGemmTraits<
cutlass::MatrixLayout::kColumnMajor,
cutlass::MatrixLayout::kColumnMajor,
cutlass::Shape<32, 16, 16>,
half,
half,
half,
cutlass::gemm::LinearScaling<half>,
half,
typename cutlass::gemm::WmmaGemmAccumulatorsPerWarp<typename cutlass::Shape<32, 16, 16> >::Shape,
typename cutlass::Shape<16, 16, 16>,
4, /*kScalarsPerLdgA_*/
4, /*kScalarsPerLdgB_*/
4, /*KScalarsPerLdsA_*/
4, /*KScalarsPerLdsB_*/
4 / sizeof(half), /*kScalarsPerLdgCAndStgD_*/
4 / sizeof(half), /*kScalarsPerStsD_*/
4 / sizeof(half) /*kScalarsPerLdsD_*/
>
WmmaGemmTraits;
run_batched_strided_gemm<WmmaGemmTraits>(36, 36, 64, 3);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(WmmaGemm_strided_batched_16x16x32_f16, wmma_gemm_36x36x16_nt) {
/*
this wmmaTraits requires leading dim to be divisible by 4
*/
typedef cutlass::gemm::WmmaGemmTraits<
cutlass::MatrixLayout::kColumnMajor,
cutlass::MatrixLayout::kRowMajor,
cutlass::Shape<32, 16, 16>,
half,
half,
half,
cutlass::gemm::LinearScaling<half>,
half,
typename cutlass::gemm::WmmaGemmAccumulatorsPerWarp<typename cutlass::Shape<32, 16, 16> >::Shape,
typename cutlass::Shape<16, 16, 16>,
4, /*kScalarsPerLdgA_*/
4, /*kScalarsPerLdgB_*/
4, /*KScalarsPerLdsA_*/
4, /*KScalarsPerLdsB_*/
4 / sizeof(half), /*kScalarsPerLdgCAndStgD_*/
4 / sizeof(half), /*kScalarsPerStsD_*/
4 / sizeof(half) /*kScalarsPerLdsD_*/
>
WmmaGemmTraits;
run_batched_strided_gemm<WmmaGemmTraits>(36, 36, 64, 3);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(WmmaGemm_strided_batched_16x16x32_f16, wmma_gemm_36x36x16_tn) {
/*
this wmmaTraits requires leading dim to be divisible by 4
*/
typedef cutlass::gemm::WmmaGemmTraits<
cutlass::MatrixLayout::kRowMajor,
cutlass::MatrixLayout::kColumnMajor,
cutlass::Shape<32, 16, 16>,
half,
half,
half,
cutlass::gemm::LinearScaling<half>,
half,
typename cutlass::gemm::WmmaGemmAccumulatorsPerWarp<typename cutlass::Shape<32, 16, 16> >::Shape,
typename cutlass::Shape<16, 16, 16>,
4, /*kScalarsPerLdgA_*/
4, /*kScalarsPerLdgB_*/
4, /*KScalarsPerLdsA_*/
4, /*KScalarsPerLdsB_*/
4 / sizeof(half), /*kScalarsPerLdgCAndStgD_*/
4 / sizeof(half), /*kScalarsPerStsD_*/
4 / sizeof(half) /*kScalarsPerLdsD_*/
>
WmmaGemmTraits;
run_batched_strided_gemm<WmmaGemmTraits>(36, 36, 64, 3);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(WmmaGemm_strided_batched_16x16x32_f16, wmma_gemm_36x36x16_tt) {
/*
this wmmaTraits requires leading dim to be divisible by 4
*/
typedef cutlass::gemm::WmmaGemmTraits<
cutlass::MatrixLayout::kRowMajor,
cutlass::MatrixLayout::kRowMajor,
cutlass::Shape<32, 16, 16>,
half,
half,
half,
cutlass::gemm::LinearScaling<half>,
half,
typename cutlass::gemm::WmmaGemmAccumulatorsPerWarp<typename cutlass::Shape<32, 16, 16> >::Shape,
typename cutlass::Shape<16, 16, 16>,
4, /*kScalarsPerLdgA_*/
4, /*kScalarsPerLdgB_*/
4, /*KScalarsPerLdsA_*/
4, /*KScalarsPerLdsB_*/
4 / sizeof(half), /*kScalarsPerLdgCAndStgD_*/
4 / sizeof(half), /*kScalarsPerStsD_*/
4 / sizeof(half) /*kScalarsPerLdsD_*/
>
WmmaGemmTraits;
run_batched_strided_gemm<WmmaGemmTraits>(36, 36, 64, 3);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
#endif
tools/test/unit/gemm/fp16_sgemm_fp16_128x128x16.cu
+3 -1
View File
@@ -28,7 +28,7 @@
#include "tools/test/unit/gemm/gemm_testbed.h"
#include "tools/test/unit/gemm/run_gemm.h"
#if !defined(__CUDA_ARCH__) || __CUDA_ARCH__ >= 530
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(Fp16_sgemm_alphaFp16_fp16_128x128x16, fp16_sgemm_fp16_128x128x16_nn) {
@@ -319,3 +319,5 @@ TEST(Fp16_sgemm_alphaFp32_fp16_128x128x16, fp16_sgemm_fp16_128x112x17_tt) {
run_gemm<SgemmTraits>(128, 112, 17);
}
#endif
tools/test/unit/gemm/fp16_sgemm_fp32_128x128x16.cu
+4 -1
View File
@@ -28,7 +28,7 @@
#include "tools/test/unit/gemm/gemm_testbed.h"
#include "tools/test/unit/gemm/run_gemm.h"
#if !defined(__CUDA_ARCH__) || __CUDA_ARCH__ >= 530
TEST(Fp16_sgemm_alphaFp32_fp32_128x128x16, fp16_sgemm_fp32_128x128x16_nn) {
@@ -172,3 +172,6 @@ TEST(Fp16_sgemm_alphaFp32_fp32_128x128x16, fp16_sgemm_fp32_128x112x17_tt) {
SgemmTraits;
run_gemm<SgemmTraits>(128, 112, 17);
}
#endif
tools/test/unit/gemm/fp16_wmma_gemm_fp16.cu
+381
View File
@@ -0,0 +1,381 @@
/***************************************************************************************************
* Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
* * Redistributions of source code must retain the above copyright notice, this list of
* conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright notice, this list of
* conditions and the following disclaimer in the documentation and/or other materials
* provided with the distribution.
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
* to endorse or promote products derived from this software without specific prior written
* permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
#include "cutlass/wmma_matrix.h"
#if defined(CUTLASS_USE_WMMA_API)
#include "cutlass_unit_test.h"
#include "cutlass/gemm/gemm.h"
#include "cutlass/gemm/wmma_gemm_traits.h"
#include "tools/test/unit/gemm/gemm_testbed.h"
#include "tools/test/unit/gemm/run_gemm.h"
TEST(WmmaGemm_16x16x32_fp32, fp16_wmma_gemm_fp16_16x16x16_nn) {
typedef float accumu_type;
typedef half c_type;
typedef cutlass::gemm::WmmaGemmTraits<
cutlass::MatrixLayout::kColumnMajor,
cutlass::MatrixLayout::kColumnMajor,
cutlass::Shape<32, 16, 16>,
half,
half,
c_type,
cutlass::gemm::LinearScaling<accumu_type>,
accumu_type,
typename cutlass::gemm::WmmaGemmAccumulatorsPerWarp<typename cutlass::Shape<32, 16, 16> >::Shape,
typename cutlass::Shape<16, 16, 16>,
8, /*kScalarsPerLdgA_*/
8, /*kScalarsPerLdgB_*/
8, /*KScalarsPerLdsA_*/
8, /*KScalarsPerLdsB_*/
16 / sizeof(c_type), /*kScalarsPerLdgCAndStgD_*/
16 / sizeof(accumu_type), /*kScalarsPerStsD_*/
16 / sizeof(accumu_type) /*kScalarsPerLdsD_*/
>
WmmaGemmTraits;
run_gemm<WmmaGemmTraits>(16, 16, 16);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(WmmaGemm_16x16x32_fp32, fp16_wmma_gemm_fp16_8x8x16_nn) {
typedef float accumu_type;
typedef half c_type;
typedef cutlass::gemm::WmmaGemmTraits<
cutlass::MatrixLayout::kColumnMajor,
cutlass::MatrixLayout::kColumnMajor,
cutlass::Shape<32, 16, 16>,
half,
half,
c_type,
cutlass::gemm::LinearScaling<accumu_type>,
accumu_type,
typename cutlass::gemm::WmmaGemmAccumulatorsPerWarp<typename cutlass::Shape<32, 16, 16> >::Shape,
typename cutlass::Shape<16, 16, 16>,
8, /*kScalarsPerLdgA_*/
8, /*kScalarsPerLdgB_*/
8, /*KScalarsPerLdsA_*/
8, /*KScalarsPerLdsB_*/
16 / sizeof(c_type), /*kScalarsPerLdgCAndStgD_*/
16 / sizeof(accumu_type), /*kScalarsPerStsD_*/
16 / sizeof(accumu_type) /*kScalarsPerLdsD_*/
>
WmmaGemmTraits;
run_gemm<WmmaGemmTraits>(8, 8, 16);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(WmmaGemm_16x16x32_fp32, fp16_wmma_gemm_fp16_256x256x64_nn) {
typedef float accumu_type;
typedef half c_type;
typedef cutlass::gemm::WmmaGemmTraits<
cutlass::MatrixLayout::kColumnMajor,
cutlass::MatrixLayout::kColumnMajor,
cutlass::Shape<32, 16, 16>,
half,
half,
c_type,
cutlass::gemm::LinearScaling<accumu_type>,
accumu_type,
typename cutlass::gemm::WmmaGemmAccumulatorsPerWarp<typename cutlass::Shape<32, 16, 16> >::Shape,
typename cutlass::Shape<16, 16, 16>,
8, /*kScalarsPerLdgA_*/
8, /*kScalarsPerLdgB_*/
8, /*KScalarsPerLdsA_*/
8, /*KScalarsPerLdsB_*/
16 / sizeof(c_type), /*kScalarsPerLdgCAndStgD_*/
16 / sizeof(accumu_type), /*kScalarsPerStsD_*/
16 / sizeof(accumu_type) /*kScalarsPerLdsD_*/
>
WmmaGemmTraits;
run_gemm<WmmaGemmTraits>(256, 256, 64);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(WmmaGemm_16x16x32_fp32, fp16_wmma_gemm_fp16_16x16x16_nt) {
typedef float accumu_type;
typedef half c_type;
typedef cutlass::gemm::WmmaGemmTraits<
cutlass::MatrixLayout::kColumnMajor,
cutlass::MatrixLayout::kRowMajor,
cutlass::Shape<32, 16, 16>,
half,
half,
c_type,
cutlass::gemm::LinearScaling<accumu_type>,
accumu_type,
typename cutlass::gemm::WmmaGemmAccumulatorsPerWarp<typename cutlass::Shape<32, 16, 16> >::Shape,
typename cutlass::Shape<16, 16, 16>,
8, /*kScalarsPerLdgA_*/
8, /*kScalarsPerLdgB_*/
8, /*KScalarsPerLdsA_*/
8, /*KScalarsPerLdsB_*/
16 / sizeof(c_type), /*kScalarsPerLdgCAndStgD_*/
16 / sizeof(accumu_type), /*kScalarsPerStsD_*/
16 / sizeof(accumu_type) /*kScalarsPerLdsD_*/
>
WmmaGemmTraits;
run_gemm<WmmaGemmTraits>(16, 16, 16);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(WmmaGemm_16x16x32_fp32, fp16_wmma_gemm_fp16_8x8x16_nt) {
typedef float accumu_type;
typedef half c_type;
typedef cutlass::gemm::WmmaGemmTraits<
cutlass::MatrixLayout::kColumnMajor,
cutlass::MatrixLayout::kRowMajor,
cutlass::Shape<32, 16, 16>,
half,
half,
c_type,
cutlass::gemm::LinearScaling<accumu_type>,
accumu_type,
typename cutlass::gemm::WmmaGemmAccumulatorsPerWarp<typename cutlass::Shape<32, 16, 16> >::Shape,
typename cutlass::Shape<16, 16, 16>,
8, /*kScalarsPerLdgA_*/
8, /*kScalarsPerLdgB_*/
8, /*KScalarsPerLdsA_*/
8, /*KScalarsPerLdsB_*/
16 / sizeof(c_type), /*kScalarsPerLdgCAndStgD_*/
16 / sizeof(accumu_type), /*kScalarsPerStsD_*/
16 / sizeof(accumu_type) /*kScalarsPerLdsD_*/
>
WmmaGemmTraits;
run_gemm<WmmaGemmTraits>(8, 8, 16);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(WmmaGemm_16x16x32_fp32, fp16_wmma_gemm_fp16_256x256x64_nt) {
typedef float accumu_type;
typedef half c_type;
typedef cutlass::gemm::WmmaGemmTraits<
cutlass::MatrixLayout::kColumnMajor,
cutlass::MatrixLayout::kRowMajor,
cutlass::Shape<32, 16, 16>,
half,
half,
c_type,
cutlass::gemm::LinearScaling<accumu_type>,
accumu_type,
typename cutlass::gemm::WmmaGemmAccumulatorsPerWarp<typename cutlass::Shape<32, 16, 16> >::Shape,
typename cutlass::Shape<16, 16, 16>,
8, /*kScalarsPerLdgA_*/
8, /*kScalarsPerLdgB_*/
8, /*KScalarsPerLdsA_*/
8, /*KScalarsPerLdsB_*/
16 / sizeof(c_type), /*kScalarsPerLdgCAndStgD_*/
16 / sizeof(accumu_type), /*kScalarsPerStsD_*/
16 / sizeof(accumu_type) /*kScalarsPerLdsD_*/
>
WmmaGemmTraits;
run_gemm<WmmaGemmTraits>(256, 256, 64);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(WmmaGemm_16x16x32_fp32, fp16_wmma_gemm_fp16_16x16x16_tn) {
typedef float accumu_type;
typedef half c_type;
typedef cutlass::gemm::WmmaGemmTraits<
cutlass::MatrixLayout::kRowMajor,
cutlass::MatrixLayout::kColumnMajor,
cutlass::Shape<32, 16, 16>,
half,
half,
c_type,
cutlass::gemm::LinearScaling<accumu_type>,
accumu_type,
typename cutlass::gemm::WmmaGemmAccumulatorsPerWarp<typename cutlass::Shape<32, 16, 16> >::Shape,
typename cutlass::Shape<16, 16, 16>,
8, /*kScalarsPerLdgA_*/
8, /*kScalarsPerLdgB_*/
8, /*KScalarsPerLdsA_*/
8, /*KScalarsPerLdsB_*/
16 / sizeof(c_type), /*kScalarsPerLdgCAndStgD_*/
16 / sizeof(accumu_type), /*kScalarsPerStsD_*/
16 / sizeof(accumu_type) /*kScalarsPerLdsD_*/
>
WmmaGemmTraits;
run_gemm<WmmaGemmTraits>(16, 16, 16);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(WmmaGemm_16x16x32_fp32, fp16_wmma_gemm_fp16_8x8x16_tn) {
typedef float accumu_type;
typedef half c_type;
typedef cutlass::gemm::WmmaGemmTraits<
cutlass::MatrixLayout::kRowMajor,
cutlass::MatrixLayout::kColumnMajor,
cutlass::Shape<32, 16, 16>,
half,
half,
c_type,
cutlass::gemm::LinearScaling<accumu_type>,
accumu_type,
typename cutlass::gemm::WmmaGemmAccumulatorsPerWarp<typename cutlass::Shape<32, 16, 16> >::Shape,
typename cutlass::Shape<16, 16, 16>,
8, /*kScalarsPerLdgA_*/
8, /*kScalarsPerLdgB_*/
8, /*KScalarsPerLdsA_*/
8, /*KScalarsPerLdsB_*/
16 / sizeof(c_type), /*kScalarsPerLdgCAndStgD_*/
16 / sizeof(accumu_type), /*kScalarsPerStsD_*/
16 / sizeof(accumu_type) /*kScalarsPerLdsD_*/
>
WmmaGemmTraits;
run_gemm<WmmaGemmTraits>(8, 8, 16);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(WmmaGemm_16x16x32_fp32, fp16_wmma_gemm_fp16_256x256x64_tn) {
typedef float accumu_type;
typedef half c_type;
typedef cutlass::gemm::WmmaGemmTraits<
cutlass::MatrixLayout::kRowMajor,
cutlass::MatrixLayout::kColumnMajor,
cutlass::Shape<32, 16, 16>,
half,
half,
c_type,
cutlass::gemm::LinearScaling<accumu_type>,
accumu_type,
typename cutlass::gemm::WmmaGemmAccumulatorsPerWarp<typename cutlass::Shape<32, 16, 16> >::Shape,
typename cutlass::Shape<16, 16, 16>,
8, /*kScalarsPerLdgA_*/
8, /*kScalarsPerLdgB_*/
8, /*KScalarsPerLdsA_*/
8, /*KScalarsPerLdsB_*/
16 / sizeof(c_type), /*kScalarsPerLdgCAndStgD_*/
16 / sizeof(accumu_type), /*kScalarsPerStsD_*/
16 / sizeof(accumu_type) /*kScalarsPerLdsD_*/
>
WmmaGemmTraits;
run_gemm<WmmaGemmTraits>(256, 256, 64);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(WmmaGemm_16x16x32_fp32, fp16_wmma_gemm_fp16_16x16x16_tt) {
typedef float accumu_type;
typedef half c_type;
typedef cutlass::gemm::WmmaGemmTraits<
cutlass::MatrixLayout::kRowMajor,
cutlass::MatrixLayout::kRowMajor,
cutlass::Shape<32, 16, 16>,
half,
half,
c_type,
cutlass::gemm::LinearScaling<accumu_type>,
accumu_type,
typename cutlass::gemm::WmmaGemmAccumulatorsPerWarp<typename cutlass::Shape<32, 16, 16> >::Shape,
typename cutlass::Shape<16, 16, 16>,
8, /*kScalarsPerLdgA_*/
8, /*kScalarsPerLdgB_*/
8, /*KScalarsPerLdsA_*/
8, /*KScalarsPerLdsB_*/
16 / sizeof(c_type), /*kScalarsPerLdgCAndStgD_*/
16 / sizeof(accumu_type), /*kScalarsPerStsD_*/
16 / sizeof(accumu_type) /*kScalarsPerLdsD_*/
>
WmmaGemmTraits;
run_gemm<WmmaGemmTraits>(16, 16, 16);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(WmmaGemm_16x16x32_fp32, fp16_wmma_gemm_fp16_8x8x16_tt) {
typedef float accumu_type;
typedef half c_type;
typedef cutlass::gemm::WmmaGemmTraits<
cutlass::MatrixLayout::kRowMajor,
cutlass::MatrixLayout::kRowMajor,
cutlass::Shape<32, 16, 16>,
half,
half,
c_type,
cutlass::gemm::LinearScaling<accumu_type>,
accumu_type,
typename cutlass::gemm::WmmaGemmAccumulatorsPerWarp<typename cutlass::Shape<32, 16, 16> >::Shape,
typename cutlass::Shape<16, 16, 16>,
8, /*kScalarsPerLdgA_*/
8, /*kScalarsPerLdgB_*/
8, /*KScalarsPerLdsA_*/
8, /*KScalarsPerLdsB_*/
16 / sizeof(c_type), /*kScalarsPerLdgCAndStgD_*/
16 / sizeof(accumu_type), /*kScalarsPerStsD_*/
16 / sizeof(accumu_type) /*kScalarsPerLdsD_*/
>
WmmaGemmTraits;
run_gemm<WmmaGemmTraits>(8, 8, 16);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(WmmaGemm_16x16x32_fp32, fp16_wmma_gemm_fp16_256x256x64_tt) {
typedef float accumu_type;
typedef half c_type;
typedef cutlass::gemm::WmmaGemmTraits<
cutlass::MatrixLayout::kRowMajor,
cutlass::MatrixLayout::kRowMajor,
cutlass::Shape<32, 16, 16>,
half,
half,
c_type,
cutlass::gemm::LinearScaling<accumu_type>,
accumu_type,
typename cutlass::gemm::WmmaGemmAccumulatorsPerWarp<typename cutlass::Shape<32, 16, 16> >::Shape,
typename cutlass::Shape<16, 16, 16>,
8, /*kScalarsPerLdgA_*/
8, /*kScalarsPerLdgB_*/
8, /*KScalarsPerLdsA_*/
8, /*KScalarsPerLdsB_*/
16 / sizeof(c_type), /*kScalarsPerLdgCAndStgD_*/
16 / sizeof(accumu_type), /*kScalarsPerStsD_*/
16 / sizeof(accumu_type) /*kScalarsPerLdsD_*/
>
WmmaGemmTraits;
run_gemm<WmmaGemmTraits>(256, 256, 64);
}
#endif //#if defined(CUTLASS_USE_WMMA_API)
tools/test/unit/gemm/fp16_wmma_gemm_fp16_non_multiple16.cu
+273
View File
@@ -0,0 +1,273 @@
/***************************************************************************************************
* Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
* * Redistributions of source code must retain the above copyright notice, this list of
* conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright notice, this list of
* conditions and the following disclaimer in the documentation and/or other materials
* provided with the distribution.
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
* to endorse or promote products derived from this software without specific prior written
* permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
#include "cutlass/wmma_matrix.h"
#if defined(CUTLASS_USE_WMMA_API)
#include "cutlass_unit_test.h"
#include "cutlass/gemm/gemm.h"
#include "cutlass/gemm/wmma_gemm_traits.h"
#include "tools/test/unit/gemm/gemm_testbed.h"
#include "tools/test/unit/gemm/run_gemm.h"
////////////////////////////////////////////////////////////////////////////////////////////////////
/* mulitple of 4*/
TEST(WmmaGemm_16x16x32_fp32, fp16_wmma_gemm_fp16_36x36x64_nn) {
typedef float accumu_type;
typedef half c_type;
typedef cutlass::gemm::WmmaGemmTraits<
cutlass::MatrixLayout::kColumnMajor,
cutlass::MatrixLayout::kColumnMajor,
cutlass::Shape<32, 16, 16>,
half,
half,
c_type,
cutlass::gemm::LinearScaling<accumu_type>,
accumu_type,
typename cutlass::gemm::WmmaGemmAccumulatorsPerWarp<typename cutlass::Shape<32, 16, 16> >::Shape,
typename cutlass::Shape<16, 16, 16>,
4, /*kScalarsPerLdgA_*/
4, /*kScalarsPerLdgB_*/
4, /*KScalarsPerLdsA_*/
4, /*KScalarsPerLdsB_*/
8 / sizeof(c_type), /*kScalarsPerLdgCAndStgD_*/
8 / sizeof(accumu_type), /*kScalarsPerStsD_*/
8 / sizeof(accumu_type) /*kScalarsPerLdsD_*/
>
WmmaGemmTraits;
run_gemm<WmmaGemmTraits>(36, 36, 64);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(WmmaGemm_16x16x32_fp32, fp16_wmma_gemm_fp16_36x36x64_nt) {
typedef float accumu_type;
typedef half c_type;
typedef cutlass::gemm::WmmaGemmTraits<
cutlass::MatrixLayout::kColumnMajor,
cutlass::MatrixLayout::kRowMajor,
cutlass::Shape<32, 16, 16>,
half,
half,
c_type,
cutlass::gemm::LinearScaling<accumu_type>,
accumu_type,
typename cutlass::gemm::WmmaGemmAccumulatorsPerWarp<typename cutlass::Shape<32, 16, 16> >::Shape,
typename cutlass::Shape<16, 16, 16>,
4, /*kScalarsPerLdgA_*/
4, /*kScalarsPerLdgB_*/
4, /*KScalarsPerLdsA_*/
4, /*KScalarsPerLdsB_*/
8 / sizeof(c_type), /*kScalarsPerLdgCAndStgD_*/
8 / sizeof(accumu_type), /*kScalarsPerStsD_*/
8 / sizeof(accumu_type) /*kScalarsPerLdsD_*/
>
WmmaGemmTraits;
run_gemm<WmmaGemmTraits>(36, 36, 64);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(WmmaGemm_16x16x32_fp32, fp16_wmma_gemm_fp16_36x36x64_tn) {
typedef float accumu_type;
typedef half c_type;
typedef cutlass::gemm::WmmaGemmTraits<
cutlass::MatrixLayout::kRowMajor,
cutlass::MatrixLayout::kColumnMajor,
cutlass::Shape<32, 16, 16>,
half,
half,
c_type,
cutlass::gemm::LinearScaling<accumu_type>,
accumu_type,
typename cutlass::gemm::WmmaGemmAccumulatorsPerWarp<typename cutlass::Shape<32, 16, 16> >::Shape,
typename cutlass::Shape<16, 16, 16>,
4, /*kScalarsPerLdgA_*/
4, /*kScalarsPerLdgB_*/
4, /*KScalarsPerLdsA_*/
4, /*KScalarsPerLdsB_*/
8 / sizeof(c_type), /*kScalarsPerLdgCAndStgD_*/
8 / sizeof(accumu_type), /*kScalarsPerStsD_*/
8 / sizeof(accumu_type) /*kScalarsPerLdsD_*/
>
WmmaGemmTraits;
run_gemm<WmmaGemmTraits>(36, 36, 64);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(WmmaGemm_16x16x32_fp32, fp16_wmma_gemm_fp16_36x36x64_tt) {
typedef float accumu_type;
typedef half c_type;
typedef cutlass::gemm::WmmaGemmTraits<
cutlass::MatrixLayout::kRowMajor,
cutlass::MatrixLayout::kRowMajor,
cutlass::Shape<32, 16, 16>,
half,
half,
c_type,
cutlass::gemm::LinearScaling<accumu_type>,
accumu_type,
typename cutlass::gemm::WmmaGemmAccumulatorsPerWarp<typename cutlass::Shape<32, 16, 16> >::Shape,
typename cutlass::Shape<16, 16, 16>,
4, /*kScalarsPerLdgA_*/
4, /*kScalarsPerLdgB_*/
4, /*KScalarsPerLdsA_*/
4, /*KScalarsPerLdsB_*/
8 / sizeof(c_type), /*kScalarsPerLdgCAndStgD_*/
8 / sizeof(accumu_type), /*kScalarsPerStsD_*/
8 / sizeof(accumu_type) /*kScalarsPerLdsD_*/
>
WmmaGemmTraits;
run_gemm<WmmaGemmTraits>(36, 36, 64);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
/* mulitple of 2*/
TEST(WmmaGemm_16x16x32_fp32, fp16_wmma_gemm_fp16_34x34x64_nn) {
typedef float accumu_type;
typedef half c_type;
typedef cutlass::gemm::WmmaGemmTraits<
cutlass::MatrixLayout::kColumnMajor,
cutlass::MatrixLayout::kColumnMajor,
cutlass::Shape<32, 16, 16>,
half,
half,
c_type,
cutlass::gemm::LinearScaling<accumu_type>,
accumu_type,
typename cutlass::gemm::WmmaGemmAccumulatorsPerWarp<typename cutlass::Shape<32, 16, 16> >::Shape,
typename cutlass::Shape<16, 16, 16>,
2, /*kScalarsPerLdgA_*/
2, /*kScalarsPerLdgB_*/
2, /*KScalarsPerLdsA_*/
2, /*KScalarsPerLdsB_*/
4 / sizeof(c_type), /*kScalarsPerLdgCAndStgD_*/
4 / sizeof(accumu_type), /*kScalarsPerStsD_*/
4 / sizeof(accumu_type) /*kScalarsPerLdsD_*/
>
WmmaGemmTraits;
run_gemm<WmmaGemmTraits>(34, 34, 64);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
/* mulitple of 2*/
TEST(WmmaGemm_16x16x32_fp32, fp16_wmma_gemm_fp16_34x34x64_nt) {
typedef float accumu_type;
typedef half c_type;
typedef cutlass::gemm::WmmaGemmTraits<
cutlass::MatrixLayout::kColumnMajor,
cutlass::MatrixLayout::kRowMajor,
cutlass::Shape<32, 16, 16>,
half,
half,
c_type,
cutlass::gemm::LinearScaling<accumu_type>,
accumu_type,
typename cutlass::gemm::WmmaGemmAccumulatorsPerWarp<typename cutlass::Shape<32, 16, 16> >::Shape,
typename cutlass::Shape<16, 16, 16>,
2, /*kScalarsPerLdgA_*/
2, /*kScalarsPerLdgB_*/
2, /*KScalarsPerLdsA_*/
2, /*KScalarsPerLdsB_*/
4 / sizeof(c_type), /*kScalarsPerLdgCAndStgD_*/
4 / sizeof(accumu_type), /*kScalarsPerStsD_*/
4 / sizeof(accumu_type) /*kScalarsPerLdsD_*/
>
WmmaGemmTraits;
run_gemm<WmmaGemmTraits>(34, 34, 64);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
/* mulitple of 2*/
TEST(WmmaGemm_16x16x32_fp32, fp16_wmma_gemm_fp16_34x34x64_tn) {
typedef float accumu_type;
typedef half c_type;
typedef cutlass::gemm::WmmaGemmTraits<
cutlass::MatrixLayout::kRowMajor,
cutlass::MatrixLayout::kColumnMajor,
cutlass::Shape<32, 16, 16>,
half,
half,
c_type,
cutlass::gemm::LinearScaling<accumu_type>,
accumu_type,
typename cutlass::gemm::WmmaGemmAccumulatorsPerWarp<typename cutlass::Shape<32, 16, 16> >::Shape,
typename cutlass::Shape<16, 16, 16>,
2, /*kScalarsPerLdgA_*/
2, /*kScalarsPerLdgB_*/
2, /*KScalarsPerLdsA_*/
2, /*KScalarsPerLdsB_*/
4 / sizeof(c_type), /*kScalarsPerLdgCAndStgD_*/
4 / sizeof(accumu_type), /*kScalarsPerStsD_*/
4 / sizeof(accumu_type) /*kScalarsPerLdsD_*/
>
WmmaGemmTraits;
run_gemm<WmmaGemmTraits>(34, 34, 64);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
/* mulitple of 2*/
TEST(WmmaGemm_16x16x32_fp32, fp16_wmma_gemm_fp16_34x34x64_tt) {
typedef float accumu_type;
typedef half c_type;
typedef cutlass::gemm::WmmaGemmTraits<
cutlass::MatrixLayout::kRowMajor,
cutlass::MatrixLayout::kRowMajor,
cutlass::Shape<32, 16, 16>,
half,
half,
c_type,
cutlass::gemm::LinearScaling<accumu_type>,
accumu_type,
typename cutlass::gemm::WmmaGemmAccumulatorsPerWarp<typename cutlass::Shape<32, 16, 16> >::Shape,
typename cutlass::Shape<16, 16, 16>,
2, /*kScalarsPerLdgA_*/
2, /*kScalarsPerLdgB_*/
2, /*KScalarsPerLdsA_*/
2, /*KScalarsPerLdsB_*/
4 / sizeof(c_type), /*kScalarsPerLdgCAndStgD_*/
4 / sizeof(accumu_type), /*kScalarsPerStsD_*/
4 / sizeof(accumu_type) /*kScalarsPerLdsD_*/
>
WmmaGemmTraits;
run_gemm<WmmaGemmTraits>(34, 34, 64);
}
#endif
tools/test/unit/gemm/gemm_load_global_store_shared.cu
+382
View File
@@ -0,0 +1,382 @@
/***************************************************************************************************
* Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
* * Redistributions of source code must retain the above copyright notice, this list of
* conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright notice, this list of
* conditions and the following disclaimer in the documentation and/or other materials
* provided with the distribution.
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
* to endorse or promote products derived from this software without specific prior written
* permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
#include "cutlass_unit_tests.h"
#include "tools/util/host_tensor.h"
#include "tools/test/unit/core/layout_verification.h"
#include "tools/util/tensor_view_io.h"
#include "cutlass/gemm/gemm.h"
#include "cutlass/shape.h"
#include "cutlass/gemm/sgemm_traits.h"
#include "cutlass/gemm/dgemm_traits.h"
#include "cutlass/gemm/hgemm_traits.h"
////////////////////////////////////////////////////////////////////////////////////////////////////
namespace test {
// M/N/K struct.
struct GemmDesc {
int m, n, k;
CUTLASS_HOST_DEVICE GemmDesc(int m_, int n_, int k_) : m(m_), n(n_), k(k_) {}
};
/// Simple test to load from global memory and store to shared memory
// Loading from global memory and storing to shared memory for A
template <typename Traits>
__global__ void Gemm_load_global_store_shared_a(
typename Traits::GlobalLoadStreamA::Scalar *output,
typename Traits::GlobalLoadStreamA::Scalar const *input,
int M,
int N,
int K,
int ldm) {
//Create shared memory.
__shared__ typename Traits::SharedStorage shared_storage;
// Create those iterators.
typedef typename Traits::GlobalLoadStreamA GlobalLoadStreamA;
typename GlobalLoadStreamA::Params global_load_params;
GemmDesc desc(M, N, K);
global_load_params.initialize(desc, input, ldm);
GlobalLoadStreamA stream_a(global_load_params, shared_storage.main_loop.stream_a.global, M, N, K, cutlass::make_Coord(0, 0, 0));
stream_a.copy();
stream_a.commit();
// store barrier
__syncthreads();
// one thread writes everything out
if (threadIdx.x == 0) {
for (int i = 0; i < M*K; ++i) {
output[i] = shared_storage.main_loop.stream_a.shared[i];
}
}
}
// Loading from global memory and storing to shared memory for B
template <typename Traits>
__global__ void Gemm_load_global_store_shared_b(
typename Traits::GlobalLoadStreamB::Scalar *output,
typename Traits::GlobalLoadStreamB::Scalar const *input,
int M,
int N,
int K,
int ldm) {
//Create shared memory.
__shared__ typename Traits::SharedStorage shared_storage;
// Create those iterators.
typedef typename Traits::GlobalLoadStreamB GlobalLoadStreamB;
typename GlobalLoadStreamB::Params global_load_params;
GemmDesc desc(M, N, K);
global_load_params.initialize(desc, input, ldm);
GlobalLoadStreamB stream_b(global_load_params, shared_storage.main_loop.stream_b.global, M, N, K, cutlass::make_Coord(0, 0, 0));
stream_b.copy();
stream_b.commit();
// store barrier
__syncthreads();
// one thread writes everything out
if (threadIdx.x == 0) {
for (int i = 0; i < M*K; ++i) {
output[i] = shared_storage.main_loop.stream_b.shared[i];
}
}
}
////////////////////////////////////////////////////////////////////////////////////////////////////
template <
typename CtaTile, // concept: Shape
typename DestType, // raw data type
typename SourceType // raw data type
>
class VerifyDataMovement {
public:
/// Tensor to store the destination data
cutlass::HostTensor<DestType> destination;
/// Tensor to store the source data
cutlass::HostTensor<SourceType> source;
/// Verification utility
typedef test::VerifyLayout<
DestType,
test::CoordinatePack<DestType>,
SourceType,
test::CoordinatePack<SourceType> > VerifyLayout;
/// Verification object
VerifyLayout verify_layout;
public:
VerifyDataMovement() { }
VerifyDataMovement(test::Layout const &source_layout) {
// Actual layout here doesn't matter here, just the number of elements
destination.resize_matrix(CtaTile::kH, CtaTile::kW, cutlass::MatrixLayout::kRowMajor);
source.resize_matrix(CtaTile::kH, CtaTile::kW, cutlass::MatrixLayout::kRowMajor);
verify_layout.initialize(source, source_layout);
destination.fill(0);
destination.sync_device();
source.sync_device();
}
/// Verifies resulting layout
bool verify(test::Layout const & destination_layout) {
destination.sync_host();
typename VerifyLayout::VisitorVerbose visitor(std::cout);
bool passed = verify_layout.verify(
destination,
destination_layout,
visitor);
return passed;
}
};
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(Gemm_shared_tile, A_float_contiguous) {
static int const M = 64;
static int const N = 64;
static int const K = 8;
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor,
cutlass::MatrixLayout::kRowMajor, cutlass::Shape<K, N, M> >
SgemmTraits;
typedef test::Layout::Span Span;
test::Layout::SpanVector dst_layout;
test::Layout::SpanVector src_layout;
// define the source layout
src_layout.push_back(Span(0, K));
src_layout.push_back(Span(1, M));
typedef VerifyDataMovement<
cutlass::Shape<1, M, K, 1>,
float,
float
> VerifyDataMovement_t;
VerifyDataMovement_t testbed(src_layout);
test::Gemm_load_global_store_shared_a< SgemmTraits ><<<
dim3(1,1,1),
dim3(SgemmTraits::kThreads, 1)
>>>(
testbed.destination.device_data(),
testbed.source.device_data(),
M,
N,
K,
M
);
cudaError_t result = cudaDeviceSynchronize();
ASSERT_EQ(result, cudaSuccess) << "\nCUDA kernel launch error: " << cudaGetErrorString(result)
<< "\n";
// define the destination layout
dst_layout.push_back(Span(0, K));
dst_layout.push_back(Span(1, M));
EXPECT_TRUE(testbed.verify(dst_layout));
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(Gemm_shared_tile, A_double_contiguous) {
static int const M = 64;
static int const N = 64;
static int const K = 8;
typedef cutlass::gemm::DgemmTraits<cutlass::MatrixLayout::kColumnMajor,
cutlass::MatrixLayout::kRowMajor, cutlass::Shape<K, N, M> >
DgemmTraits;
typedef test::Layout::Span Span;
test::Layout::SpanVector dst_layout;
test::Layout::SpanVector src_layout;
// define the source layout
src_layout.push_back(Span(0, K));
src_layout.push_back(Span(1, M));
typedef VerifyDataMovement<
cutlass::Shape<1, M, K, 1>,
double,
double
> VerifyDataMovement_t;
VerifyDataMovement_t testbed(src_layout);
test::Gemm_load_global_store_shared_a< DgemmTraits ><<<
dim3(1,1,1),
dim3(DgemmTraits::kThreads, 1)
>>>(
testbed.destination.device_data(),
testbed.source.device_data(),
M,
N,
K,
M
);
cudaError_t result = cudaDeviceSynchronize();
ASSERT_EQ(result, cudaSuccess) << "\nCUDA kernel launch error: " << cudaGetErrorString(result)
<< "\n";
// define the destination layout
dst_layout.push_back(Span(0, K));
dst_layout.push_back(Span(1, M));
EXPECT_TRUE(testbed.verify(dst_layout));
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(Gemm_shared_tile, B_float_contiguous) {
static int const M = 64;
static int const N = 64;
static int const K = 8;
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor,
cutlass::MatrixLayout::kRowMajor, cutlass::Shape<K, N, M> >
SgemmTraits;
typedef test::Layout::Span Span;
test::Layout::SpanVector dst_layout;
test::Layout::SpanVector src_layout;
// define the source layout
src_layout.push_back(Span(0, K));
src_layout.push_back(Span(1, M));
typedef VerifyDataMovement<
cutlass::Shape<1, M, K, 1>,
float,
float
> VerifyDataMovement_t;
VerifyDataMovement_t testbed(src_layout);
test::Gemm_load_global_store_shared_b< SgemmTraits ><<<
dim3(1,1,1),
dim3(SgemmTraits::kThreads, 1)
>>>(
testbed.destination.device_data(),
testbed.source.device_data(),
M,
N,
K,
M
);
cudaError_t result = cudaDeviceSynchronize();
ASSERT_EQ(result, cudaSuccess) << "\nCUDA kernel launch error: " << cudaGetErrorString(result)
<< "\n";
// define the destination layout
dst_layout.push_back(Span(0, K));
dst_layout.push_back(Span(1, M));
EXPECT_TRUE(testbed.verify(dst_layout));
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(Gemm_shared_tile, B_double_contiguous) {
static int const M = 64;
static int const N = 64;
static int const K = 8;
typedef cutlass::gemm::DgemmTraits<cutlass::MatrixLayout::kColumnMajor,
cutlass::MatrixLayout::kRowMajor, cutlass::Shape<K, N, M> >
DgemmTraits;
typedef test::Layout::Span Span;
test::Layout::SpanVector dst_layout;
test::Layout::SpanVector src_layout;
// define the source layout
src_layout.push_back(Span(0, K));
src_layout.push_back(Span(1, M));
typedef VerifyDataMovement<
cutlass::Shape<1, M, K, 1>,
double,
double
> VerifyDataMovement_t;
VerifyDataMovement_t testbed(src_layout);
test::Gemm_load_global_store_shared_b< DgemmTraits ><<<
dim3(1,1,1),
dim3(DgemmTraits::kThreads, 1)
>>>(
testbed.destination.device_data(),
testbed.source.device_data(),
M,
N,
K,
M
);
cudaError_t result = cudaDeviceSynchronize();
ASSERT_EQ(result, cudaSuccess) << "\nCUDA kernel launch error: " << cudaGetErrorString(result)
<< "\n";
// define the destination layout
dst_layout.push_back(Span(0, K));
dst_layout.push_back(Span(1, M));
EXPECT_TRUE(testbed.verify(dst_layout));
}
////////////////////////////////////////////////////////////////////////////////////////////////////
}
tools/test/unit/gemm/gemm_testbed.h
+238 -46
View File
@@ -46,6 +46,7 @@
#include "tools/util/type_traits.h"
#include "tools/util/reference/host/gemm.h"
#include "tools/util/reference/device/gemm.h"
#include "tools/util/reference/host/tensor_elementwise.h"
//////////////////////////////////////////////////////////////////////////////////////////
@@ -247,6 +248,9 @@ struct GemmTestbed {
/// Reference result computed on the host
HostMatrixC ref_host;
/// Reference result computed on the device
HostMatrixC ref_device;
/// Reference result computed with cublas
HostMatrixC ref_cublas;
@@ -262,6 +266,9 @@ struct GemmTestbed {
/// batch count
int batch_count;
/// partitionK count
int partitionK_count;
/// distance between A[i] and A[i+1] for strided batched gemm
long long int batch_stride_A;
@@ -308,6 +315,7 @@ struct GemmTestbed {
beta(beta_),
algorithm(algorithm_),
batch_count(1),
partitionK_count(1),
batch_stride_A(static_cast<long long int>(0)),
batch_stride_B(static_cast<long long int>(0)),
batch_stride_C(static_cast<long long int>(0)) {
@@ -320,6 +328,7 @@ struct GemmTestbed {
resize(B, K_, N_, layout_b);
resize(C_initial, M_, N_, layout_c);
resize(ref_host, M_, N_, layout_c);
resize(ref_device, M_, N_, layout_c);
resize(ref_cublas, M_, N_, layout_c);
resize(computed, M_, N_, layout_c);
}
@@ -345,6 +354,7 @@ struct GemmTestbed {
beta(beta_),
algorithm(algorithm_),
batch_count(1),
partitionK_count(1),
batch_stride_A(static_cast<long long int>(0)),
batch_stride_B(static_cast<long long int>(0)),
batch_stride_C(static_cast<long long int>(0)) {
@@ -353,6 +363,7 @@ struct GemmTestbed {
resize(B, K_ * batch_count, N_, layout_b);
resize(C_initial, M_, N_ * batch_count, layout_c);
resize(ref_host, M_, N_ * batch_count, layout_c);
resize(ref_device, M_, N_ * batch_count, layout_c);
resize(ref_cublas, M_, N_ * batch_count, layout_c);
resize(computed, M_, N_ * batch_count, layout_c);
}
@@ -377,6 +388,7 @@ struct GemmTestbed {
beta(beta_),
algorithm(algorithm_),
batch_count(1),
partitionK_count(1),
batch_stride_A(static_cast<long long int>(0)),
batch_stride_B(static_cast<long long int>(0)),
batch_stride_C(static_cast<long long int>(0)) {
@@ -389,6 +401,7 @@ struct GemmTestbed {
resize(B, K_, N_, layout_b, ldb);
resize(C_initial, M_, N_, layout_c, ldc);
resize(ref_host, M_, N_, layout_c, ldc);
resize(ref_device, M_, N_, layout_c, ldc);
resize(ref_cublas, M_, N_, layout_c, ldc);
resize(computed, M_, N_, layout_c, ldc);
}
@@ -414,6 +427,7 @@ struct GemmTestbed {
beta(beta_),
algorithm(algorithm_),
batch_count(1),
partitionK_count(1),
batch_stride_A(static_cast<long long int>(0)),
batch_stride_B(static_cast<long long int>(0)),
batch_stride_C(static_cast<long long int>(0)) {
@@ -422,6 +436,7 @@ struct GemmTestbed {
resize(B, K_ * batch_count, N_, layout_b);
resize(C_initial, M_, N_ * batch_count, layout_c);
resize(ref_host, M_, N_ * batch_count, layout_c);
resize(ref_device, M_, N_ * batch_count, layout_c);
resize(ref_cublas, M_, N_ * batch_count, layout_c);
resize(computed, M_, N_ * batch_count, layout_c);
}
@@ -446,7 +461,8 @@ struct GemmTestbed {
alpha(alpha_),
beta(beta_),
algorithm(algorithm_),
batch_count(batch_count_) {
batch_count(batch_count_),
partitionK_count(1) {
status = cublasCreate(&handle);
if (status != CUBLAS_STATUS_SUCCESS) {
@@ -457,6 +473,7 @@ struct GemmTestbed {
resize(B, K_ * batch_count, N_, layout_b);
resize(C_initial, M_, N_ * batch_count, layout_c);
resize(ref_host, M_, N_ * batch_count, layout_c);
resize(ref_device, M_, N_ * batch_count, layout_c);
resize(ref_cublas, M_, N_ * batch_count, layout_c);
resize(computed, M_, N_ * batch_count, layout_c);
@@ -465,6 +482,50 @@ struct GemmTestbed {
batch_stride_C = M_ * N_;
}
/// Constructs a workspace for verifying partitionedK GEMM, assumes
/// dense packing.
/// in partitionedK GEMM, the K is partitioned by partitionK_size
/// each partition is of the same size, except for the last partition
/// each partition, except for the last one, is of size K / partitionK_count
/// if K is not divisible by partitionK_size, the last partitionK = K % partitionK_count + K / partitionK_count
GemmTestbed(int M_,
int N_,
std::pair<int, int> K_pair_, /*(k, partitionK_count)*/
cublasOperation_t layout_a,
cublasOperation_t layout_b,
Scalar alpha_ = Scalar(1),
Scalar beta_ = Scalar(0),
cublasGemmAlgo_t algorithm_ = CUBLAS_GEMM_DEFAULT,
cublasOperation_t layout_c = CUBLAS_OP_N)
: problem_size(K_pair_.first, N_, M_, 1),
layout_A(layout_a),
layout_B(layout_b),
alpha(alpha_),
beta(beta_),
algorithm(algorithm_),
batch_count(1),
partitionK_count(K_pair_.second) {
status = cublasCreate(&handle);
if (status != CUBLAS_STATUS_SUCCESS) {
throw cutlass::cuda_exception("Failed to create CUBLAS handle");
}
resize(A, M_, K_pair_.first, layout_a);
resize(B, K_pair_.first, N_, layout_b);
resize(C_initial, M_, N_ * partitionK_count, layout_c);
resize(ref_host, M_, N_ * partitionK_count, layout_c);
resize(ref_device, M_, N_ * partitionK_count, layout_c);
resize(ref_cublas, M_, N_ * partitionK_count, layout_c);
resize(computed, M_, N_ * partitionK_count, layout_c);
// we can use a combination of batched stried gemm and regular gemm
// to simulation partitionedK, which is what we will do for reference code
int partitionK_size = K() / partitionK_count;
batch_stride_A = (layout_a == CUBLAS_OP_N) ? M_ * partitionK_size : partitionK_size;
batch_stride_B = (layout_b == CUBLAS_OP_N) ? partitionK_size : partitionK_size * N_;
batch_stride_C = M_ * N_;
}
/// Destructs the GEMM testbed
~GemmTestbed() {
if (status != CUBLAS_STATUS_NOT_INITIALIZED) {
@@ -504,7 +565,14 @@ struct GemmTestbed {
/// Returns the number of flops implied by the computation (1 multiply-accumulate = 2 flops)
uint64_t flops() const {
return uint64_t(batch_count) * uint64_t(M()) * uint64_t(N()) * uint64_t(K()) * 2ULL;
if (partitionK_count == 1) {
return uint64_t(batch_count) * uint64_t(M()) * uint64_t(N()) * uint64_t(K()) * 2ULL;
}
else {
int partitionK_size = K() / partitionK_count;
return (uint64_t(partitionK_count - 1) * uint64_t(batch_count) * uint64_t(M()) * uint64_t(N()) * uint64_t(partitionK_size) * 2ULL)
+ (uint64_t(batch_count) * uint64_t(M()) * uint64_t(N()) * uint64_t(K() - partitionK_size * (partitionK_count - 1)) * 2ULL);
}
}
/// Computes the speed of the computation in GFLOPs/s
@@ -555,14 +623,15 @@ struct GemmTestbed {
// Initialize the source matrix with a uniform distribution
cutlass::Distribution dist;
dist.set_uniform(-8, 8);
cutlass::reference::host::TensorInitialize(A.host_view(), seed, dist);
cutlass::reference::host::TensorInitialize(B.host_view(), seed + 11, dist);
cutlass::reference::host::TensorInitialize(C_initial.host_view(), seed + 13, dist);
A.sync_device();
B.sync_device();
C_initial.sync_device();
}
/// Initializes binary data
@@ -585,56 +654,121 @@ struct GemmTestbed {
/// Computes the matrix product on the host
void compute_host() {
ref_host.fill(C_initial);
cutlass::reference::host::Gemm(problem_size, alpha, A.host_ref(), B.host_ref(), beta, ref_host.host_ref(), Accumulator(0));
}
/// Compute the matrix product using the device-side reference
void compute_device_reference() {
ref_device.fill(C_initial);
cutlass::reference::device::Gemm(
problem_size,
cutlass::TypeTraits<Scalar>::to_device(alpha),
A.device_ref(),
B.device_ref(),
cutlass::TypeTraits<Scalar>::to_device(beta),
ref_device.device_ref(),
cutlass::TypeTraits<Accumulator>::to_device(0)
);
}
/// Excutes an equivalent GEMM using cuBLAS
bool execute_cublas() {
if (batch_count == 1) {
status = cublasGemmEx(handle,
layout_a(),
layout_b(),
M(),
N(),
K(),
&alpha,
ptr_A(),
cutlass::TypeTraits<AType>::cublas_type,
lda(),
ptr_B(),
cutlass::TypeTraits<BType>::cublas_type,
ldb(),
&beta,
ref_cublas.device_data(),
cutlass::TypeTraits<CType>::cublas_type,
ldc(),
cutlass::TypeTraits<Accumulator>::cublas_type,
algorithm);
if (partitionK_count == 1) {
if (batch_count == 1) {
status = cublasGemmEx(handle,
layout_a(),
layout_b(),
M(),
N(),
K(),
&alpha,
ptr_A(),
cutlass::TypeTraits<AType>::cublas_type,
lda(),
ptr_B(),
cutlass::TypeTraits<BType>::cublas_type,
ldb(),
&beta,
ref_cublas.device_data(),
cutlass::TypeTraits<CType>::cublas_type,
ldc(),
cutlass::TypeTraits<Accumulator>::cublas_type,
algorithm);
return status == CUBLAS_STATUS_SUCCESS;
} else {
// call strided batched gemm
return status == CUBLAS_STATUS_SUCCESS;
}
else {
// call strided batched gemm
status = cublasGemmStridedBatchedTemplate(handle,
layout_a(),
layout_b(),
M(),
N(),
K(),
&alpha,
ptr_A(),
lda(),
batch_stride_A,
ptr_B(),
ldb(),
batch_stride_B,
&beta,
ref_cublas.device_data(),
ldc(),
batch_stride_C,
batch_count);
return status == CUBLAS_STATUS_SUCCESS;
}
}
else {
assert(batch_count == 1);
//the last batch is of a different K
//first call strided batched gemm
int partitionK_size = K() / partitionK_count;
//int lastK_size = (K() % partitionK_size) + partitionK_size;
int lastK_size = K() - partitionK_size * (partitionK_count - 1);
status = cublasGemmStridedBatchedTemplate(handle,
layout_a(),
layout_b(),
M(),
N(),
K(),
&alpha,
ptr_A(),
lda(),
batch_stride_A,
ptr_B(),
ldb(),
batch_stride_B,
&beta,
ref_cublas.device_data(),
ldc(),
batch_stride_C,
batch_count);
layout_a(),
layout_b(),
M(),
N(),
partitionK_size,
&alpha,
ptr_A(),
lda(),
batch_stride_A,
ptr_B(),
ldb(),
batch_stride_B,
&beta,
ref_cublas.device_data(),
ldc(),
batch_stride_C,
partitionK_count - 1);
//then call gemm for the last batch
status = cublasGemmEx(handle,
layout_a(),
layout_b(),
M(),
N(),
lastK_size,
&alpha,
ptr_A() + (partitionK_count - 1) * batch_stride_A,
cutlass::TypeTraits<AType>::cublas_type,
lda(),
ptr_B() + (partitionK_count - 1) * batch_stride_B,
cutlass::TypeTraits<BType>::cublas_type,
ldb(),
&beta,
ref_cublas.device_data() + (partitionK_count - 1) * batch_stride_C,
cutlass::TypeTraits<CType>::cublas_type,
ldc(),
cutlass::TypeTraits<Accumulator>::cublas_type,
algorithm);
return status == CUBLAS_STATUS_SUCCESS;
}
}
@@ -787,6 +921,24 @@ struct GemmTestbed {
return passed;
}
/// Verifies the reference implementation with cuBLAS
bool verify_reference_with_cublas(bool save_on_error = true, bool always_print = false) {
compute_device_reference();
ref_device.sync_host();
compute_cublas();
ref_cublas.sync_host();
bool passed = ref_device.bit_equals(ref_cublas);
if ((!passed && save_on_error) || always_print) {
save_workspace(ref_device, ref_cublas);
}
return passed;
}
/// Verifies with host-side and device-side computations
bool verify_with_all() {
bool passed = true;
@@ -917,4 +1069,44 @@ template<> inline cublasStatus_t GemmTestbed<cutlass::half_t, cutlass::half_t, c
batchCount);
}
template<> inline cublasStatus_t GemmTestbed<cutlass::half_t, cutlass::half_t, cutlass::half_t, float, float>::cublasGemmStridedBatchedTemplate(cublasHandle_t handle,
cublasOperation_t transa,
cublasOperation_t transb,
int M,
int N,
int K,
const float *alpha,
const half *ptr_A,
int lda,
long long int stride_A,
const half *ptr_B,
int ldb,
long long int stride_B,
const float *beta,
half *ptr_C,
int ldc,
long long int stride_C,
int batchCount) {
return cublasGemmStridedBatchedEx(handle,
transa,
transb,
M, N, K,
alpha,
ptr_A,
cutlass::TypeTraits<cutlass::half_t>::cublas_type,
lda,
stride_A,
ptr_B,
cutlass::TypeTraits<cutlass::half_t>::cublas_type,
ldb,
stride_B,
beta,
ptr_C,
cutlass::TypeTraits<cutlass::half_t>::cublas_type,
ldc,
stride_C,
batchCount,
cutlass::TypeTraits<float>::cublas_type,
CUBLAS_GEMM_DEFAULT);
}
} // namespace test
tools/test/unit/gemm/hgemm_128x128x16.cu
+3
View File
@@ -29,6 +29,8 @@
#include "tools/test/unit/gemm/gemm_testbed.h"
#include "tools/test/unit/gemm/run_gemm.h"
#if !defined(__CUDA_ARCH__) || __CUDA_ARCH__ >= 530
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(Hgemm_128x128x16, hgemm_128x128x16_nt) {
@@ -326,4 +328,5 @@ TEST(Hgemm_128x128x16, hgemm_124x126x32_ragged_alpha2_beta1_nt) {
run_gemm<HgemmTraits>(124, 126, 32, cutlass::half_t(2), cutlass::half_t(1));
}
////////////////////////////////////////////////////////////////////////////////////////////////////
#endif
tools/test/unit/gemm/hgemm_128x128x8.cu
+3 -1
View File
@@ -29,6 +29,8 @@
#include "tools/test/unit/gemm/gemm_testbed.h"
#include "tools/test/unit/gemm/run_gemm.h"
#if !defined(__CUDA_ARCH__) || __CUDA_ARCH__ >= 530
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(Hgemm_128x128x8, hgemm_128x128x1_nt) {
@@ -384,5 +386,5 @@ TEST(Hgemm_128x128x8, hgemm_124x126x32_ragged_alpha2_beta1_nt) {
}
////////////////////////////////////////////////////////////////////////////////////////////////////
#endif
tools/test/unit/gemm/hgemm_128x32x8.cu
+3
View File
@@ -28,6 +28,7 @@
#include "tools/test/unit/gemm/gemm_testbed.h"
#include "tools/test/unit/gemm/run_gemm.h"
#if !defined(__CUDA_ARCH__) || __CUDA_ARCH__ >= 530
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(Hgemm_128x32x8, hgemm_128x32x1_nt) {
@@ -312,3 +313,5 @@ TEST(Hgemm_128x32x8, hgemm_256x64x16_tt) {
run_gemm<HgemmTraits>(256, 64, 16);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
#endif
tools/test/unit/gemm/hgemm_128x64x8.cu
+3
View File
@@ -28,6 +28,7 @@
#include "tools/test/unit/gemm/gemm_testbed.h"
#include "tools/test/unit/gemm/run_gemm.h"
#if !defined(__CUDA_ARCH__) || __CUDA_ARCH__ >= 530
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(Hgemm_128x64x8, hgemm_128x64x1_nt) {
@@ -312,3 +313,5 @@ TEST(Hgemm_128x64x8, hgemm_256x128x16_tt) {
run_gemm<HgemmTraits>(256, 128, 16);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
#endif
tools/test/unit/gemm/partitionedK_sgemm_128x128x8.cu
+378
View File
@@ -0,0 +1,378 @@
/***************************************************************************************************
* Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
* * Redistributions of source code must retain the above copyright notice, this list of
* conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright notice, this list of
* conditions and the following disclaimer in the documentation and/or other materials
* provided with the distribution.
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
* to endorse or promote products derived from this software without specific prior written
* permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
#include "cutlass_unit_test.h"
#include "cutlass/gemm/gemm.h"
#include "cutlass/gemm/sgemm_traits.h"
#include "tools/test/unit/gemm/gemm_testbed.h"
#include "tools/test/unit/gemm/run_gemm.h"
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(Sgemm_partitionedK_128x128x8, sgemm_128x256x100x8_nn) {
/*
for example
partitionedK sgemm, m = 128, n = 256, overall_K = 100, partitionK_count = 8
for the first 7 partition k = overall_k / partitionK_count = 12
for the last partition last_k = overall_k - (partitionK_count - 1) * k = 16
*/
int m = 128;
int n = 256;
int overall_k = 100;
int partitionK_count = 8;
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor,
cutlass::MatrixLayout::kColumnMajor, cutlass::Shape<8, 128, 128> >
SgemmTraits;
run_partitioned_k_gemm<SgemmTraits>(m, n, overall_k, partitionK_count);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(Sgemm_partitionedK_128x128x8, sgemm_128x256x175x8_nn) {
int m = 128;
int n = 256;
int overall_k = 175;
int partitionK_count = 8;
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor,
cutlass::MatrixLayout::kColumnMajor, cutlass::Shape<8, 128, 128> >
SgemmTraits;
run_partitioned_k_gemm<SgemmTraits>(m, n, overall_k, partitionK_count);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(Sgemm_partitionedK_128x128x8, sgemm_10x12x20x3_nn) {
int m = 10;
int n = 12;
int overall_k = 20;
int partitionK_count = 3;
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor,
cutlass::MatrixLayout::kColumnMajor, cutlass::Shape<8, 128, 128> >
SgemmTraits;
run_partitioned_k_gemm<SgemmTraits>(m, n, overall_k, partitionK_count);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(Sgemm_partitionedK_128x128x8, sgemm_10x12x60x8_nn) {
int m = 10;
int n = 12;
int overall_k = 60;
int partitionK_count = 8;
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor,
cutlass::MatrixLayout::kColumnMajor, cutlass::Shape<8, 128, 128> >
SgemmTraits;
run_partitioned_k_gemm<SgemmTraits>(m, n, overall_k, partitionK_count);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(Sgemm_partitionedK_128x128x8, sgemm_128x256x100x4_nn) {
int m = 128;
int n = 256;
int overall_k = 100;
int partitionK_count = 4;
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor,
cutlass::MatrixLayout::kColumnMajor, cutlass::Shape<8, 128, 128> >
SgemmTraits;
run_partitioned_k_gemm<SgemmTraits>(m, n, overall_k, partitionK_count);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(Sgemm_partitionedK_128x128x8, sgemm_128x256x100x8_nt) {
/*
for example
partitionedK sgemm, m = 128, n = 256, overall_K = 100, partitionK_count = 8
for the first 7 partition k = overall_k / partitionK_count = 12
for the last partition last_k = overall_k - (partitionK_count - 1) * k = 16
*/
int m = 128;
int n = 256;
int overall_k = 100;
int partitionK_count = 8;
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor,
cutlass::MatrixLayout::kRowMajor, cutlass::Shape<8, 128, 128> >
SgemmTraits;
run_partitioned_k_gemm<SgemmTraits>(m, n, overall_k, partitionK_count);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(Sgemm_partitionedK_128x128x8, sgemm_128x256x175x8_nt) {
int m = 128;
int n = 256;
int overall_k = 175;
int partitionK_count = 8;
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor,
cutlass::MatrixLayout::kRowMajor, cutlass::Shape<8, 128, 128> >
SgemmTraits;
run_partitioned_k_gemm<SgemmTraits>(m, n, overall_k, partitionK_count);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(Sgemm_partitionedK_128x128x8, sgemm_10x12x20x3_nt) {
int m = 10;
int n = 12;
int overall_k = 20;
int partitionK_count = 3;
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor,
cutlass::MatrixLayout::kRowMajor, cutlass::Shape<8, 128, 128> >
SgemmTraits;
run_partitioned_k_gemm<SgemmTraits>(m, n, overall_k, partitionK_count);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(Sgemm_partitionedK_128x128x8, sgemm_10x12x60x8_nt) {
int m = 10;
int n = 12;
int overall_k = 60;
int partitionK_count = 8;
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor,
cutlass::MatrixLayout::kRowMajor, cutlass::Shape<8, 128, 128> >
SgemmTraits;
run_partitioned_k_gemm<SgemmTraits>(m, n, overall_k, partitionK_count);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(Sgemm_partitionedK_128x128x8, sgemm_128x256x100x4_nt) {
int m = 128;
int n = 256;
int overall_k = 100;
int partitionK_count = 4;
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor,
cutlass::MatrixLayout::kRowMajor, cutlass::Shape<8, 128, 128> >
SgemmTraits;
run_partitioned_k_gemm<SgemmTraits>(m, n, overall_k, partitionK_count);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(Sgemm_partitionedK_128x128x8, sgemm_128x256x100x8_tn) {
/*
for example
partitionedK sgemm, m = 128, n = 256, overall_K = 100, partitionK_count = 8
for the first 7 partition k = overall_k / partitionK_count = 12
for the last partition last_k = overall_k - (partitionK_count - 1) * k = 16
*/
int m = 128;
int n = 256;
int overall_k = 100;
int partitionK_count = 8;
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kRowMajor,
cutlass::MatrixLayout::kColumnMajor, cutlass::Shape<8, 128, 128> >
SgemmTraits;
run_partitioned_k_gemm<SgemmTraits>(m, n, overall_k, partitionK_count);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(Sgemm_partitionedK_128x128x8, sgemm_128x256x175x8_tn) {
int m = 128;
int n = 256;
int overall_k = 175;
int partitionK_count = 8;
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kRowMajor,
cutlass::MatrixLayout::kColumnMajor, cutlass::Shape<8, 128, 128> >
SgemmTraits;
run_partitioned_k_gemm<SgemmTraits>(m, n, overall_k, partitionK_count);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(Sgemm_partitionedK_128x128x8, sgemm_10x12x20x3_tn) {
int m = 10;
int n = 12;
int overall_k = 20;
int partitionK_count = 3;
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kRowMajor,
cutlass::MatrixLayout::kColumnMajor, cutlass::Shape<8, 128, 128> >
SgemmTraits;
run_partitioned_k_gemm<SgemmTraits>(m, n, overall_k, partitionK_count);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(Sgemm_partitionedK_128x128x8, sgemm_10x12x60x8_tn) {
int m = 10;
int n = 12;
int overall_k = 60;
int partitionK_count = 8;
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kRowMajor,
cutlass::MatrixLayout::kColumnMajor, cutlass::Shape<8, 128, 128> >
SgemmTraits;
run_partitioned_k_gemm<SgemmTraits>(m, n, overall_k, partitionK_count);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(Sgemm_partitionedK_128x128x8, sgemm_128x256x100x4_tn) {
int m = 128;
int n = 256;
int overall_k = 100;
int partitionK_count = 4;
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kRowMajor,
cutlass::MatrixLayout::kColumnMajor, cutlass::Shape<8, 128, 128> >
SgemmTraits;
run_partitioned_k_gemm<SgemmTraits>(m, n, overall_k, partitionK_count);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(Sgemm_partitionedK_128x128x8, sgemm_128x256x100x8_tt) {
/*
for example
partitionedK sgemm, m = 128, n = 256, overall_K = 100, partitionK_count = 8
for the first 7 partition k = overall_k / partitionK_count = 12
for the last partition last_k = overall_k - (partitionK_count - 1) * k = 16
*/
int m = 128;
int n = 256;
int overall_k = 100;
int partitionK_count = 8;
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kRowMajor,
cutlass::MatrixLayout::kRowMajor, cutlass::Shape<8, 128, 128> >
SgemmTraits;
run_partitioned_k_gemm<SgemmTraits>(m, n, overall_k, partitionK_count);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(Sgemm_partitionedK_128x128x8, sgemm_128x256x175x8_tt) {
int m = 128;
int n = 256;
int overall_k = 175;
int partitionK_count = 8;
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kRowMajor,
cutlass::MatrixLayout::kRowMajor, cutlass::Shape<8, 128, 128> >
SgemmTraits;
run_partitioned_k_gemm<SgemmTraits>(m, n, overall_k, partitionK_count);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(Sgemm_partitionedK_128x128x8, sgemm_10x12x20x3_tt) {
int m = 10;
int n = 12;
int overall_k = 20;
int partitionK_count = 3;
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kRowMajor,
cutlass::MatrixLayout::kRowMajor, cutlass::Shape<8, 128, 128> >
SgemmTraits;
run_partitioned_k_gemm<SgemmTraits>(m, n, overall_k, partitionK_count);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(Sgemm_partitionedK_128x128x8, sgemm_10x12x60x8_tt) {
int m = 10;
int n = 12;
int overall_k = 60;
int partitionK_count = 8;
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kRowMajor,
cutlass::MatrixLayout::kRowMajor, cutlass::Shape<8, 128, 128> >
SgemmTraits;
run_partitioned_k_gemm<SgemmTraits>(m, n, overall_k, partitionK_count);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(Sgemm_partitionedK_128x128x8, sgemm_128x256x100x4_tt) {
int m = 128;
int n = 256;
int overall_k = 100;
int partitionK_count = 4;
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kRowMajor,
cutlass::MatrixLayout::kRowMajor, cutlass::Shape<8, 128, 128> >
SgemmTraits;
run_partitioned_k_gemm<SgemmTraits>(m, n, overall_k, partitionK_count);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
tools/test/unit/gemm/run_gemm.h
+161 -4
View File
@@ -25,8 +25,12 @@
#pragma once
#include <utility>
#include "cutlass/cutlass.h"
#include "tools/test/unit/gemm/gemm_testbed.h"
#include "cutlass/gemm/device_gemm.h"
#include "cutlass/gemm/device_gemm_traits.h"
template <typename GemmTraits_>
static void run_gemm(
int m,
@@ -36,9 +40,9 @@ static void run_gemm(
int ldb,
int ldc,
typename test::GemmTestbedTraits<typename GemmTraits_::Epilogue::Scalar>::host_type alpha =
typename test::GemmTestbedTraits<typename GemmTraits_::Epilogue::Scalar>::host_type(1),
typename test::GemmTestbedTraits<typename GemmTraits_::Epilogue::Scalar>::host_type(1.0f),
typename test::GemmTestbedTraits<typename GemmTraits_::Epilogue::Scalar>::host_type beta =
typename test::GemmTestbedTraits<typename GemmTraits_::Epilogue::Scalar>::host_type(0)) {
typename test::GemmTestbedTraits<typename GemmTraits_::Epilogue::Scalar>::host_type(0.0f)) {
typedef typename GemmTraits_::KernelClass Gemm;
typename Gemm::Params params;
@@ -69,8 +73,10 @@ static void run_gemm(
if (testbed.has_cublas_support()) {
EXPECT_TRUE(testbed.verify_host_with_cublas());
EXPECT_TRUE(testbed.verify_reference_with_cublas());
}
params.initialize(testbed.M(),
testbed.N(),
testbed.K(),
@@ -137,6 +143,7 @@ static void run_gemm(
if (testbed.has_cublas_support()) {
EXPECT_TRUE(testbed.verify_host_with_cublas());
EXPECT_TRUE(testbed.verify_reference_with_cublas());
}
params.initialize(testbed.M(),
@@ -175,9 +182,9 @@ static void run_batched_strided_gemm(
int k,
int batch_count,
typename test::GemmTestbedTraits<typename GemmTraits_::Epilogue::Scalar>::host_type alpha =
typename test::GemmTestbedTraits<typename GemmTraits_::Epilogue::Scalar>::host_type(1),
typename test::GemmTestbedTraits<typename GemmTraits_::Epilogue::Scalar>::host_type(1.0f),
typename test::GemmTestbedTraits<typename GemmTraits_::Epilogue::Scalar>::host_type beta =
typename test::GemmTestbedTraits<typename GemmTraits_::Epilogue::Scalar>::host_type(0)) {
typename test::GemmTestbedTraits<typename GemmTraits_::Epilogue::Scalar>::host_type(0.0f)) {
//typedef cutlass::gemm::Gemm<GemmTraits_> Gemm;
typedef typename GemmTraits_::KernelClass Gemm;
typename Gemm::Params params;
@@ -242,3 +249,153 @@ static void run_batched_strided_gemm(
}
////////////////////////////////////////////////////////////////////////////////////////////////////
template <typename GemmTraits_, typename ReductionTraits_>
static void run_splitK_gemm(int m,
int n,
int k,
typename test::GemmTestbedTraits<typename ReductionTraits_::ScalarAlphaBeta>::host_type alpha =
typename test::GemmTestbedTraits<typename ReductionTraits_::ScalarAlphaBeta>::host_type(1.0f),
typename test::GemmTestbedTraits<typename ReductionTraits_::ScalarAlphaBeta>::host_type beta =
typename test::GemmTestbedTraits<typename ReductionTraits_::ScalarAlphaBeta>::host_type(0.0f),
bool use_host_reference = false){
test::GemmTestbed<
typename test::GemmTestbedTraits<
typename GemmTraits_::GemmConfig::ScalarA>::host_type, // AType
typename test::GemmTestbedTraits<
typename GemmTraits_::GemmConfig::ScalarB>::host_type, // BType
typename test::GemmTestbedTraits<
typename ReductionTraits_::ScalarC>::host_type, // CType
typename test::GemmTestbedTraits<
typename GemmTraits_::GemmConfig::ScalarD>::host_type, // Workspace Accumulator
typename test::GemmTestbedTraits<typename ReductionTraits_::ScalarAlphaBeta>::host_type // Scalar
>
testbed(m,
n,
k,
test::convert(GemmTraits_::kLayoutA),
test::convert(GemmTraits_::kLayoutB),
alpha,
beta);
testbed.initialize();
// create a device gemm
typedef cutlass::gemm::SplitkPIGemmTraits<GemmTraits_, ReductionTraits_> deviceGemmTraits;
typedef typename deviceGemmTraits::KernelClass deviceGemm;
typename deviceGemm::Params deviceGemmParams(testbed.M(), testbed.N(), testbed.K());
// query if workspace is needed
int workspace_size = deviceGemmParams.required_workspace_memory_in_byte();
typename test::GemmTestbedTraits<typename GemmTraits_::GemmConfig::ScalarD>::device_type
*workspace_ptr = 0;
if (workspace_size != 0) {
cudaError_t workspace_err = cudaMalloc(&workspace_ptr, workspace_size);
ASSERT_EQ(workspace_err, cudaSuccess) << "\nCUDA workspace malloc error: " << cudaGetErrorString(workspace_err)
<< "\n";
}
deviceGemmParams.initialize(testbed.alpha,
testbed.ptr_A(),
testbed.lda(),
testbed.ptr_B(),
testbed.ldb(),
testbed.beta,
testbed.ptr_C_initial(),
testbed.ldc(),
testbed.ptr_computed(),
testbed.ldc(),
workspace_ptr);
deviceGemm::launch(deviceGemmParams);
cudaError_t result = cudaDeviceSynchronize();
ASSERT_EQ(result, cudaSuccess) << "\nCUDA kernel launch error: " << cudaGetErrorString(result)
<< "\n";
if (workspace_size != 0) {
cudaError_t workspace_err = cudaFree(workspace_ptr);
ASSERT_EQ(workspace_err, cudaSuccess) << "\nCUDA workspace free error: " << cudaGetErrorString(workspace_err)
<< "\n";
}
if (use_host_reference == true || testbed.has_cublas_support() == false) {
ASSERT_TRUE(testbed.verify_with_host());
}
else {
ASSERT_TRUE(testbed.verify_with_cublas());
}
}
////////////////////////////////////////////////////////////////////////////////////////////////////
template <typename GemmTraits_>
static void run_partitioned_k_gemm(
int m,
int n,
int k,
int partitionK_count,
typename test::GemmTestbedTraits<typename GemmTraits_::Epilogue::Scalar>::host_type alpha =
typename test::GemmTestbedTraits<typename GemmTraits_::Epilogue::Scalar>::host_type(1.0f),
typename test::GemmTestbedTraits<typename GemmTraits_::Epilogue::Scalar>::host_type beta =
typename test::GemmTestbedTraits<typename GemmTraits_::Epilogue::Scalar>::host_type(0.0f)) {
//typedef cutlass::gemm::Gemm<GemmTraits_> Gemm;
typedef typename GemmTraits_::KernelClass Gemm;
typename Gemm::Params params;
test::GemmTestbed<
typename test::GemmTestbedTraits<
typename GemmTraits_::GemmConfig::ScalarA>::host_type, // AType
typename test::GemmTestbedTraits<
typename GemmTraits_::GemmConfig::ScalarB>::host_type, // BType
typename test::GemmTestbedTraits<
typename GemmTraits_::Epilogue::ScalarC>::host_type, // CType
typename test::GemmTestbedTraits<
typename GemmTraits_::Epilogue::Accumulators::Element>::host_type, // Accumulator
typename test::GemmTestbedTraits<typename GemmTraits_::Epilogue::Scalar>::host_type // Scalar
>
testbed(m,
n,
std::make_pair(k, partitionK_count),
test::convert(GemmTraits_::kLayoutA),
test::convert(GemmTraits_::kLayoutB),
alpha,
beta);
testbed.initialize();
// host support is not implemented for strided batched gemm
// if (testbed.has_cublas_support()) {
// EXPECT_TRUE(testbed.verify_host_with_cublas());
//}
params.initialize(testbed.M(),
testbed.N(),
testbed.K(),
testbed.alpha,
testbed.ptr_A(),
testbed.lda(),
testbed.ptr_B(),
testbed.ldb(),
testbed.beta,
testbed.ptr_C_initial(),
testbed.ldc(),
testbed.ptr_computed(),
testbed.ldc(),
partitionK_count);
Gemm::launch(params);
cudaError_t result = cudaDeviceSynchronize();
ASSERT_EQ(result, cudaSuccess) << "\nCUDA kernel launch error: " << cudaGetErrorString(result)
<< "\n";
if (testbed.has_cublas_support()) {
ASSERT_TRUE(testbed.verify_with_cublas());
}
else {
// ASSERT_TRUE(testbed.verify_with_host());
ASSERT_TRUE(false) << "host support is not implemented for strided batched gemm" << std::endl;
}
}
tools/test/unit/gemm/splitK_dgemm.cu
+247
View File
@@ -0,0 +1,247 @@
/***************************************************************************************************
* Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
* * Redistributions of source code must retain the above copyright notice, this list of
* conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright notice, this list of
* conditions and the following disclaimer in the documentation and/or other materials
* provided with the distribution.
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
* to endorse or promote products derived from this software without specific prior written
* permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
#include "cutlass_unit_test.h"
#include "cutlass/gemm/gemm.h"
#include "cutlass/gemm/dgemm_traits.h"
#include "cutlass/reduction/batched_reduction_traits.h"
#include "tools/test/unit/gemm/gemm_testbed.h"
#include "tools/test/unit/gemm/run_gemm.h"
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SplitK_dgemm_128x128x8_splits16, dgemm_128x256x512_nn) {
const int splits_count = 16;
const int m = 128;
const int n = 256;
const int k = 512;
/*batched sgemm traits*/
typedef cutlass::gemm::DgemmTraits<cutlass::MatrixLayout::kColumnMajor,
cutlass::MatrixLayout::kColumnMajor, cutlass::Shape<8, 128, 128> >
DgemmTraits;
/*batched reduction traits*/
typedef cutlass::reduction::BatchedReductionTraits<double,
double,
double,
double,
double, /*accumulation type*/
splits_count,
cutlass::Shape<1, 1, 128>,
cutlass::Shape<1, 1, 64>,
cutlass::Shape<1, 1, 2> >
BatchedReductionTraits;
run_splitK_gemm<DgemmTraits, BatchedReductionTraits>(m, n, k, 2.0f, 1.0f);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SplitK_dgemm_128x128x8_splits16, dgemm_128x256x512_nt) {
const int splits_count = 16;
const int m = 128;
const int n = 256;
const int k = 512;
/*batched sgemm traits*/
typedef cutlass::gemm::DgemmTraits<cutlass::MatrixLayout::kColumnMajor,
cutlass::MatrixLayout::kRowMajor, cutlass::Shape<8, 128, 128> >
DgemmTraits;
/*batched reduction traits*/
typedef cutlass::reduction::BatchedReductionTraits<double,
double,
double,
double,
double, /*accumulation type*/
splits_count,
cutlass::Shape<1, 1, 128>,
cutlass::Shape<1, 1, 64>,
cutlass::Shape<1, 1, 2> >
BatchedReductionTraits;
run_splitK_gemm<DgemmTraits, BatchedReductionTraits>(m, n, k, 2.0f, 1.0f);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SplitK_dgemm_128x128x8_splits16, dgemm_128x256x512_tn) {
const int splits_count = 16;
const int m = 128;
const int n = 256;
const int k = 512;
/*batched sgemm traits*/
typedef cutlass::gemm::DgemmTraits<cutlass::MatrixLayout::kRowMajor,
cutlass::MatrixLayout::kColumnMajor, cutlass::Shape<8, 128, 128> >
DgemmTraits;
/*batched reduction traits*/
typedef cutlass::reduction::BatchedReductionTraits<double,
double,
double,
double,
double, /*accumulation type*/
splits_count,
cutlass::Shape<1, 1, 128>,
cutlass::Shape<1, 1, 64>,
cutlass::Shape<1, 1, 2> >
BatchedReductionTraits;
run_splitK_gemm<DgemmTraits, BatchedReductionTraits>(m, n, k, 2.0f, 1.0f);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SplitK_dgemm_128x128x8_splits16, dgemm_128x256x512_tt) {
const int splits_count = 16;
const int m = 128;
const int n = 256;
const int k = 512;
/*batched sgemm traits*/
typedef cutlass::gemm::DgemmTraits<cutlass::MatrixLayout::kRowMajor,
cutlass::MatrixLayout::kRowMajor, cutlass::Shape<8, 128, 128> >
DgemmTraits;
/*batched reduction traits*/
typedef cutlass::reduction::BatchedReductionTraits<double,
double,
double,
double,
double, /*accumulation type*/
splits_count,
cutlass::Shape<1, 1, 128>,
cutlass::Shape<1, 1, 64>,
cutlass::Shape<1, 1, 2> >
BatchedReductionTraits;
run_splitK_gemm<DgemmTraits, BatchedReductionTraits>(m, n, k, 2.0f, 1.0f);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SplitK_dgemm_128x128x8_splits16, dgemm_128x256x500_nn) {
const int splits_count = 16;
const int m = 128;
const int n = 256;
const int k = 500;
/*batched sgemm traits*/
typedef cutlass::gemm::DgemmTraits<cutlass::MatrixLayout::kColumnMajor,
cutlass::MatrixLayout::kColumnMajor, cutlass::Shape<8, 128, 128> >
DgemmTraits;
/*batched reduction traits*/
typedef cutlass::reduction::BatchedReductionTraits<double,
double,
double,
double,
double, /*accumulation type*/
splits_count,
cutlass::Shape<1, 1, 128>,
cutlass::Shape<1, 1, 64>,
cutlass::Shape<1, 1, 2> >
BatchedReductionTraits;
run_splitK_gemm<DgemmTraits, BatchedReductionTraits>(m, n, k, 2.0f, 1.0f);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SplitK_dgemm_128x128x8_splits16, dgemm_128x256x500_nt) {
const int splits_count = 16;
const int m = 128;
const int n = 256;
const int k = 500;
/*batched sgemm traits*/
typedef cutlass::gemm::DgemmTraits<cutlass::MatrixLayout::kColumnMajor,
cutlass::MatrixLayout::kRowMajor, cutlass::Shape<8, 128, 128> >
DgemmTraits;
/*batched reduction traits*/
typedef cutlass::reduction::BatchedReductionTraits<double,
double,
double,
double,
double, /*accumulation type*/
splits_count,
cutlass::Shape<1, 1, 128>,
cutlass::Shape<1, 1, 64>,
cutlass::Shape<1, 1, 2> >
BatchedReductionTraits;
run_splitK_gemm<DgemmTraits, BatchedReductionTraits>(m, n, k, 2.0f, 1.0f);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SplitK_dgemm_128x128x8_splits16, dgemm_128x256x500_tn) {
const int splits_count = 16;
const int m = 128;
const int n = 256;
const int k = 500;
/*batched sgemm traits*/
typedef cutlass::gemm::DgemmTraits<cutlass::MatrixLayout::kRowMajor,
cutlass::MatrixLayout::kColumnMajor, cutlass::Shape<8, 128, 128> >
DgemmTraits;
/*batched reduction traits*/
typedef cutlass::reduction::BatchedReductionTraits<double,
double,
double,
double,
double, /*accumulation type*/
splits_count,
cutlass::Shape<1, 1, 128>,
cutlass::Shape<1, 1, 64>,
cutlass::Shape<1, 1, 2> >
BatchedReductionTraits;
run_splitK_gemm<DgemmTraits, BatchedReductionTraits>(m, n, k, 2.0f, 1.0f);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SplitK_dgemm_128x128x8_splits16, dgemm_128x256x500_tt) {
const int splits_count = 16;
const int m = 128;
const int n = 256;
const int k = 500;
/*batched sgemm traits*/
typedef cutlass::gemm::DgemmTraits<cutlass::MatrixLayout::kRowMajor,
cutlass::MatrixLayout::kRowMajor, cutlass::Shape<8, 128, 128> >
DgemmTraits;
/*batched reduction traits*/
typedef cutlass::reduction::BatchedReductionTraits<double,
double,
double,
double,
double, /*accumulation type*/
splits_count,
cutlass::Shape<1, 1, 128>,
cutlass::Shape<1, 1, 64>,
cutlass::Shape<1, 1, 2> >
BatchedReductionTraits;
run_splitK_gemm<DgemmTraits, BatchedReductionTraits>(m, n, k, 2.0f, 1.0f);
}
tools/test/unit/gemm/splitK_fp16_sgemm_fp16.cu
+579
View File
@@ -0,0 +1,579 @@
/***************************************************************************************************
* Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
* * Redistributions of source code must retain the above copyright notice, this list of
* conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright notice, this list of
* conditions and the following disclaimer in the documentation and/or other materials
* provided with the distribution.
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
* to endorse or promote products derived from this software without specific prior written
* permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
#include "cutlass_unit_test.h"
#include "cutlass/gemm/gemm.h"
#include "cutlass/gemm/fp16_sgemm_traits.h"
#include "cutlass/reduction/batched_reduction_traits.h"
#include "tools/test/unit/gemm/gemm_testbed.h"
#include "tools/test/unit/gemm/run_gemm.h"
////////////////////////////////////////////////////////////////////////////////////////////////////
/*
for fp16_sgemm_fp16 A, B, C and D are half typed. alpha and beta can be half or float typed.
Accumulation is float typed.
1. in batched gemm kernel, Ab and Bb are half typed, and pointing to A and B.
Cb and Db are float typed, since Db is actually pointing to the workspace memory
thus is of the same type with accumulation. Cb is generally ignored since beta is zero. alpha is one.
2. in the reduction kernel. Dr = alpha * Reduction(Ar) + beta * Cr. Ar is float typed and pointing to the same
workspace memory with Db. Cr is half typed and pointing to C. Dr is half typed and pointing to D.
ALPHAr is the same with alpha, BETAr is the same with beta.
*/
TEST(SplitK_fp16_sgemm_fp16_alphabetaFloat_128x128x8_splits16, sgemm_128x256x512_nn) {
const int splits_count = 16;
const int m = 128;
const int n = 256;
const int k = 512;
/*batched sgemm traits*/
typedef cutlass::gemm::Fp16SgemmSgemmTraits<cutlass::MatrixLayout::kColumnMajor,
cutlass::MatrixLayout::kColumnMajor,
cutlass::Shape<16, 128, 128>,
half, /*Ab type*/
half, /*Bb type*/
float, /*Cb type*/
float, /*Db type*/
float /*alpha, beta type*/
>
SgemmTraits;
/*batched reduction traits*/
typedef cutlass::reduction::BatchedReductionTraits<float, /*Ar type*/
half, /*Cr type*/
half, /*Dr type*/
float, /*alpha, beta type*/
float, /*accumulation type*/
splits_count,
cutlass::Shape<1, 1, 128>,
cutlass::Shape<1, 1, 64>,
cutlass::Shape<1, 1, 2> >
BatchedReductionTraits;
run_splitK_gemm<SgemmTraits, BatchedReductionTraits>(m, n, k, 2.0f, 1.0f, true/*use host reference*/);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SplitK_fp16_sgemm_fp16_alphabetaFloat_128x128x8_splits16, sgemm_128x256x512_nt) {
const int splits_count = 16;
const int m = 128;
const int n = 256;
const int k = 512;
/*batched sgemm traits*/
typedef cutlass::gemm::Fp16SgemmSgemmTraits<cutlass::MatrixLayout::kColumnMajor,
cutlass::MatrixLayout::kRowMajor,
cutlass::Shape<16, 128, 128>,
half, /*Ab type*/
half, /*Bb type*/
float, /*Cb type*/
float, /*Db type*/
float /*alpha, beta type*/
>
SgemmTraits;
/*batched reduction traits*/
typedef cutlass::reduction::BatchedReductionTraits<float, /*Ar type*/
half, /*Cr type*/
half, /*Dr type*/
float, /*alpha, beta type*/
float, /*accumulation type*/
splits_count,
cutlass::Shape<1, 1, 128>,
cutlass::Shape<1, 1, 64>,
cutlass::Shape<1, 1, 2> >
BatchedReductionTraits;
run_splitK_gemm<SgemmTraits, BatchedReductionTraits>(m, n, k, 2.0f, 1.0f, true/*use host reference*/);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SplitK_fp16_sgemm_fp16_alphabetaFloat_128x128x8_splits16, sgemm_128x256x512_tn) {
const int splits_count = 16;
const int m = 128;
const int n = 256;
const int k = 512;
/*batched sgemm traits*/
typedef cutlass::gemm::Fp16SgemmSgemmTraits<cutlass::MatrixLayout::kRowMajor,
cutlass::MatrixLayout::kColumnMajor,
cutlass::Shape<16, 128, 128>,
half, /*Ab type*/
half, /*Bb type*/
float, /*Cb type*/
float, /*Db type*/
float /*alpha, beta type*/
>
SgemmTraits;
/*batched reduction traits*/
typedef cutlass::reduction::BatchedReductionTraits<float, /*Ar type*/
half, /*Cr type*/
half, /*Dr type*/
float, /*alpha, beta type*/
float, /*accumulation type*/
splits_count,
cutlass::Shape<1, 1, 128>,
cutlass::Shape<1, 1, 64>,
cutlass::Shape<1, 1, 2> >
BatchedReductionTraits;
run_splitK_gemm<SgemmTraits, BatchedReductionTraits>(m, n, k, 2.0f, 1.0f, true/*use host reference*/);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SplitK_fp16_sgemm_fp16_alphabetaFloat_128x128x8_splits16, sgemm_128x256x512_tt) {
const int splits_count = 16;
const int m = 128;
const int n = 256;
const int k = 512;
/*batched sgemm traits*/
typedef cutlass::gemm::Fp16SgemmSgemmTraits<cutlass::MatrixLayout::kRowMajor,
cutlass::MatrixLayout::kRowMajor,
cutlass::Shape<16, 128, 128>,
half, /*Ab type*/
half, /*Bb type*/
float, /*Cb type*/
float, /*Db type*/
float /*alpha, beta type*/
>
SgemmTraits;
/*batched reduction traits*/
typedef cutlass::reduction::BatchedReductionTraits<float, /*Ar type*/
half, /*Cr type*/
half, /*Dr type*/
float, /*alpha, beta type*/
float, /*accumulation type*/
splits_count,
cutlass::Shape<1, 1, 128>,
cutlass::Shape<1, 1, 64>,
cutlass::Shape<1, 1, 2> >
BatchedReductionTraits;
run_splitK_gemm<SgemmTraits, BatchedReductionTraits>(m, n, k, 2.0f, 1.0f, true/*use host reference*/);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SplitK_fp16_sgemm_fp16_alphabetaFloat_128x128x8_splits16, sgemm_128x256x500_nn) {
const int splits_count = 16;
const int m = 128;
const int n = 256;
const int k = 500;
/*batched sgemm traits*/
typedef cutlass::gemm::Fp16SgemmSgemmTraits<cutlass::MatrixLayout::kColumnMajor,
cutlass::MatrixLayout::kColumnMajor,
cutlass::Shape<16, 128, 128>,
half, /*Ab type*/
half, /*Bb type*/
float, /*Cb type*/
float, /*Db type*/
float /*alpha, beta type*/
>
SgemmTraits;
/*batched reduction traits*/
typedef cutlass::reduction::BatchedReductionTraits<float, /*Ar type*/
half, /*Cr type*/
half, /*Dr type*/
float, /*alpha, beta type*/
float, /*accumulation type*/
splits_count,
cutlass::Shape<1, 1, 128>,
cutlass::Shape<1, 1, 64>,
cutlass::Shape<1, 1, 2> >
BatchedReductionTraits;
run_splitK_gemm<SgemmTraits, BatchedReductionTraits>(m, n, k, 2.0f, 1.0f, true/*use host reference*/);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SplitK_fp16_sgemm_fp16_alphabetaFloat_128x128x8_splits16, sgemm_128x256x500_nt) {
const int splits_count = 16;
const int m = 128;
const int n = 256;
const int k = 500;
/*batched sgemm traits*/
typedef cutlass::gemm::Fp16SgemmSgemmTraits<cutlass::MatrixLayout::kColumnMajor,
cutlass::MatrixLayout::kRowMajor,
cutlass::Shape<16, 128, 128>,
half, /*Ab type*/
half, /*Bb type*/
float, /*Cb type*/
float, /*Db type*/
float /*alpha, beta type*/
>
SgemmTraits;
/*batched reduction traits*/
typedef cutlass::reduction::BatchedReductionTraits<float, /*Ar type*/
half, /*Cr type*/
half, /*Dr type*/
float, /*alpha, beta type*/
float, /*accumulation type*/
splits_count,
cutlass::Shape<1, 1, 128>,
cutlass::Shape<1, 1, 64>,
cutlass::Shape<1, 1, 2> >
BatchedReductionTraits;
run_splitK_gemm<SgemmTraits, BatchedReductionTraits>(m, n, k, 2.0f, 1.0f, true/*use host reference*/);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SplitK_fp16_sgemm_fp16_alphabetaFloat_128x128x8_splits16, sgemm_128x256x500_tn) {
const int splits_count = 16;
const int m = 128;
const int n = 256;
const int k = 500;
/*batched sgemm traits*/
typedef cutlass::gemm::Fp16SgemmSgemmTraits<cutlass::MatrixLayout::kRowMajor,
cutlass::MatrixLayout::kColumnMajor,
cutlass::Shape<16, 128, 128>,
half, /*Ab type*/
half, /*Bb type*/
float, /*Cb type*/
float, /*Db type*/
float /*alpha, beta type*/
>
SgemmTraits;
/*batched reduction traits*/
typedef cutlass::reduction::BatchedReductionTraits<float, /*Ar type*/
half, /*Cr type*/
half, /*Dr type*/
float, /*alpha, beta type*/
float, /*accumulation type*/
splits_count,
cutlass::Shape<1, 1, 128>,
cutlass::Shape<1, 1, 64>,
cutlass::Shape<1, 1, 2> >
BatchedReductionTraits;
run_splitK_gemm<SgemmTraits, BatchedReductionTraits>(m, n, k, 2.0f, 1.0f, true/*use host reference*/);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SplitK_fp16_sgemm_fp16_alphabetaFloat_128x128x8_splits16, sgemm_128x256x500_tt) {
const int splits_count = 16;
const int m = 128;
const int n = 256;
const int k = 500;
/*batched sgemm traits*/
typedef cutlass::gemm::Fp16SgemmSgemmTraits<cutlass::MatrixLayout::kRowMajor,
cutlass::MatrixLayout::kRowMajor,
cutlass::Shape<16, 128, 128>,
half, /*Ab type*/
half, /*Bb type*/
float, /*Cb type*/
float, /*Db type*/
float /*alpha, beta type*/
>
SgemmTraits;
/*batched reduction traits*/
typedef cutlass::reduction::BatchedReductionTraits<float, /*Ar type*/
half, /*Cr type*/
half, /*Dr type*/
float, /*alpha, beta type*/
float, /*accumulation type*/
splits_count,
cutlass::Shape<1, 1, 128>,
cutlass::Shape<1, 1, 64>,
cutlass::Shape<1, 1, 2> >
BatchedReductionTraits;
run_splitK_gemm<SgemmTraits, BatchedReductionTraits>(m, n, k, 2.0f, 1.0f, true/*use host reference*/);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SplitK_fp16_sgemm_fp16_alphabetaFp16_128x128x8_splits16, sgemm_128x256x512_nn) {
const int splits_count = 16;
const int m = 128;
const int n = 256;
const int k = 512;
/*batched sgemm traits*/
typedef cutlass::gemm::Fp16SgemmSgemmTraits<cutlass::MatrixLayout::kColumnMajor,
cutlass::MatrixLayout::kColumnMajor,
cutlass::Shape<16, 128, 128>,
half, /*Ab type*/
half, /*Bb type*/
float, /*Cb type*/
float, /*Db type*/
float /*alpha, beta type*/
>
SgemmTraits;
/*batched reduction traits*/
typedef cutlass::reduction::BatchedReductionTraits<float, /*Ar type*/
half, /*Cr type*/
half, /*Dr type*/
half, /*alpha, beta type*/
float, /*accumulation type*/
splits_count,
cutlass::Shape<1, 1, 128>,
cutlass::Shape<1, 1, 64>,
cutlass::Shape<1, 1, 2> >
BatchedReductionTraits;
run_splitK_gemm<SgemmTraits, BatchedReductionTraits>(m, n, k, 2.0f, 1.0f, true/*use host reference*/);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SplitK_fp16_sgemm_fp16_alphabetaFp16_128x128x8_splits16, sgemm_128x256x512_nt) {
const int splits_count = 16;
const int m = 128;
const int n = 256;
const int k = 512;
/*batched sgemm traits*/
typedef cutlass::gemm::Fp16SgemmSgemmTraits<cutlass::MatrixLayout::kColumnMajor,
cutlass::MatrixLayout::kRowMajor,
cutlass::Shape<16, 128, 128>,
half, /*Ab type*/
half, /*Bb type*/
float, /*Cb type*/
float, /*Db type*/
float /*alpha, beta type*/
>
SgemmTraits;
/*batched reduction traits*/
typedef cutlass::reduction::BatchedReductionTraits<float, /*Ar type*/
half, /*Cr type*/
half, /*Dr type*/
half, /*alpha, beta type*/
float, /*accumulation type*/
splits_count,
cutlass::Shape<1, 1, 128>,
cutlass::Shape<1, 1, 64>,
cutlass::Shape<1, 1, 2> >
BatchedReductionTraits;
run_splitK_gemm<SgemmTraits, BatchedReductionTraits>(m, n, k, 2.0f, 1.0f, true/*use host reference*/);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SplitK_fp16_sgemm_fp16_alphabetaFp16_128x128x8_splits16, sgemm_128x256x512_tn) {
const int splits_count = 16;
const int m = 128;
const int n = 256;
const int k = 512;
/*batched sgemm traits*/
typedef cutlass::gemm::Fp16SgemmSgemmTraits<cutlass::MatrixLayout::kRowMajor,
cutlass::MatrixLayout::kColumnMajor,
cutlass::Shape<16, 128, 128>,
half, /*Ab type*/
half, /*Bb type*/
float, /*Cb type*/
float, /*Db type*/
float /*alpha, beta type*/
>
SgemmTraits;
/*batched reduction traits*/
typedef cutlass::reduction::BatchedReductionTraits<float, /*Ar type*/
half, /*Cr type*/
half, /*Dr type*/
half, /*alpha, beta type*/
float, /*accumulation type*/
splits_count,
cutlass::Shape<1, 1, 128>,
cutlass::Shape<1, 1, 64>,
cutlass::Shape<1, 1, 2> >
BatchedReductionTraits;
run_splitK_gemm<SgemmTraits, BatchedReductionTraits>(m, n, k, 2.0f, 1.0f, true/*use host reference*/);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SplitK_fp16_sgemm_fp16_alphabetaFp16_128x128x8_splits16, sgemm_128x256x512_tt) {
const int splits_count = 16;
const int m = 128;
const int n = 256;
const int k = 512;
/*batched sgemm traits*/
typedef cutlass::gemm::Fp16SgemmSgemmTraits<cutlass::MatrixLayout::kRowMajor,
cutlass::MatrixLayout::kRowMajor,
cutlass::Shape<16, 128, 128>,
half, /*Ab type*/
half, /*Bb type*/
float, /*Cb type*/
float, /*Db type*/
float /*alpha, beta type*/
>
SgemmTraits;
/*batched reduction traits*/
typedef cutlass::reduction::BatchedReductionTraits<float, /*Ar type*/
half, /*Cr type*/
half, /*Dr type*/
half, /*alpha, beta type*/
float, /*accumulation type*/
splits_count,
cutlass::Shape<1, 1, 128>,
cutlass::Shape<1, 1, 64>,
cutlass::Shape<1, 1, 2> >
BatchedReductionTraits;
run_splitK_gemm<SgemmTraits, BatchedReductionTraits>(m, n, k, 2.0f, 1.0f, true/*use host reference*/);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
//k = 500
TEST(SplitK_fp16_sgemm_fp16_alphabetaFp16_128x128x8_splits16, sgemm_128x256x500_nn) {
const int splits_count = 16;
const int m = 128;
const int n = 256;
const int k = 500;
/*batched sgemm traits*/
typedef cutlass::gemm::Fp16SgemmSgemmTraits<cutlass::MatrixLayout::kColumnMajor,
cutlass::MatrixLayout::kColumnMajor,
cutlass::Shape<16, 128, 128>,
half, /*Ab type*/
half, /*Bb type*/
float, /*Cb type*/
float, /*Db type*/
float /*alpha, beta type*/
>
SgemmTraits;
/*batched reduction traits*/
typedef cutlass::reduction::BatchedReductionTraits<float, /*Ar type*/
half, /*Cr type*/
half, /*Dr type*/
half, /*alpha, beta type*/
float, /*accumulation type*/
splits_count,
cutlass::Shape<1, 1, 128>,
cutlass::Shape<1, 1, 64>,
cutlass::Shape<1, 1, 2> >
BatchedReductionTraits;
run_splitK_gemm<SgemmTraits, BatchedReductionTraits>(m, n, k, 2.0f, 1.0f, true/*use host reference*/);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SplitK_fp16_sgemm_fp16_alphabetaFp16_128x128x8_splits16, sgemm_128x256x500_nt) {
const int splits_count = 16;
const int m = 128;
const int n = 256;
const int k = 500;
/*batched sgemm traits*/
typedef cutlass::gemm::Fp16SgemmSgemmTraits<cutlass::MatrixLayout::kColumnMajor,
cutlass::MatrixLayout::kRowMajor,
cutlass::Shape<16, 128, 128>,
half, /*Ab type*/
half, /*Bb type*/
float, /*Cb type*/
float, /*Db type*/
float /*alpha, beta type*/
>
SgemmTraits;
/*batched reduction traits*/
typedef cutlass::reduction::BatchedReductionTraits<float, /*Ar type*/
half, /*Cr type*/
half, /*Dr type*/
half, /*alpha, beta type*/
float, /*accumulation type*/
splits_count,
cutlass::Shape<1, 1, 128>,
cutlass::Shape<1, 1, 64>,
cutlass::Shape<1, 1, 2> >
BatchedReductionTraits;
run_splitK_gemm<SgemmTraits, BatchedReductionTraits>(m, n, k, 2.0f, 1.0f, true/*use host reference*/);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SplitK_fp16_sgemm_fp16_alphabetaFp16_128x128x8_splits16, sgemm_128x256x500_tn) {
const int splits_count = 16;
const int m = 128;
const int n = 256;
const int k = 500;
/*batched sgemm traits*/
typedef cutlass::gemm::Fp16SgemmSgemmTraits<cutlass::MatrixLayout::kRowMajor,
cutlass::MatrixLayout::kColumnMajor,
cutlass::Shape<16, 128, 128>,
half, /*Ab type*/
half, /*Bb type*/
float, /*Cb type*/
float, /*Db type*/
float /*alpha, beta type*/
>
SgemmTraits;
/*batched reduction traits*/
typedef cutlass::reduction::BatchedReductionTraits<float, /*Ar type*/
half, /*Cr type*/
half, /*Dr type*/
half, /*alpha, beta type*/
float, /*accumulation type*/
splits_count,
cutlass::Shape<1, 1, 128>,
cutlass::Shape<1, 1, 64>,
cutlass::Shape<1, 1, 2> >
BatchedReductionTraits;
run_splitK_gemm<SgemmTraits, BatchedReductionTraits>(m, n, k, 2.0f, 1.0f, true/*use host reference*/);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SplitK_fp16_sgemm_fp16_alphabetaFp16_128x128x8_splits16, sgemm_128x256x500_tt) {
const int splits_count = 16;
const int m = 128;
const int n = 256;
const int k = 500;
/*batched sgemm traits*/
typedef cutlass::gemm::Fp16SgemmSgemmTraits<cutlass::MatrixLayout::kRowMajor,
cutlass::MatrixLayout::kRowMajor,
cutlass::Shape<16, 128, 128>,
half, /*Ab type*/
half, /*Bb type*/
float, /*Cb type*/
float, /*Db type*/
float /*alpha, beta type*/
>
SgemmTraits;
/*batched reduction traits*/
typedef cutlass::reduction::BatchedReductionTraits<float, /*Ar type*/
half, /*Cr type*/
half, /*Dr type*/
half, /*alpha, beta type*/
float, /*accumulation type*/
splits_count,
cutlass::Shape<1, 1, 128>,
cutlass::Shape<1, 1, 64>,
cutlass::Shape<1, 1, 2> >
BatchedReductionTraits;
run_splitK_gemm<SgemmTraits, BatchedReductionTraits>(m, n, k, 2.0f, 1.0f, true/*use host reference*/);
}
tools/test/unit/gemm/splitK_hgemm.cu
+248
View File
@@ -0,0 +1,248 @@
/***************************************************************************************************
* Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
* * Redistributions of source code must retain the above copyright notice, this list of
* conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright notice, this list of
* conditions and the following disclaimer in the documentation and/or other materials
* provided with the distribution.
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
* to endorse or promote products derived from this software without specific prior written
* permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
#include "cutlass_unit_test.h"
#include "cutlass/gemm/gemm.h"
#include "cutlass/gemm/hgemm_traits.h"
#include "cutlass/reduction/batched_reduction_traits.h"
#include "tools/test/unit/gemm/gemm_testbed.h"
#include "tools/test/unit/gemm/run_gemm.h"
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SplitK_hgemm_128x128x8_splits16, hgemm_128x256x64_nn) {
const int splits_count = 16;
const int m = 128;
const int n = 256;
const int k = 64;
/*batched sgemm traits*/
typedef cutlass::gemm::HgemmTraits<cutlass::MatrixLayout::kColumnMajor,
cutlass::MatrixLayout::kColumnMajor, cutlass::Shape<8, 128, 128> >
HgemmTraits;
/*batched reduction traits*/
typedef cutlass::reduction::BatchedReductionTraits<half,
half,
half,
half,
half, /*accumulation type*/
splits_count,
cutlass::Shape<1, 1, 128>,
cutlass::Shape<1, 1, 64>,
cutlass::Shape<1, 1, 2> >
BatchedReductionTraits;
run_splitK_gemm<HgemmTraits, BatchedReductionTraits>(m, n, k, 1.0f, 0.0f);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SplitK_hgemm_128x128x8_splits16, hgemm_128x256x64_nt) {
const int splits_count = 16;
const int m = 128;
const int n = 256;
const int k = 64;
/*batched sgemm traits*/
typedef cutlass::gemm::HgemmTraits<cutlass::MatrixLayout::kColumnMajor,
cutlass::MatrixLayout::kRowMajor, cutlass::Shape<8, 128, 128> >
HgemmTraits;
/*batched reduction traits*/
typedef cutlass::reduction::BatchedReductionTraits<half,
half,
half,
half,
half, /*accumulation type*/
splits_count,
cutlass::Shape<1, 1, 128>,
cutlass::Shape<1, 1, 64>,
cutlass::Shape<1, 1, 2> >
BatchedReductionTraits;
run_splitK_gemm<HgemmTraits, BatchedReductionTraits>(m, n, k, 2.0f, 1.0f);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SplitK_hgemm_128x128x8_splits16, hgemm_128x256x64_tn) {
const int splits_count = 16;
const int m = 128;
const int n = 256;
const int k = 64;
/*batched sgemm traits*/
typedef cutlass::gemm::HgemmTraits<cutlass::MatrixLayout::kRowMajor,
cutlass::MatrixLayout::kColumnMajor, cutlass::Shape<8, 128, 128> >
HgemmTraits;
/*batched reduction traits*/
typedef cutlass::reduction::BatchedReductionTraits<half,
half,
half,
half,
half, /*accumulation type*/
splits_count,
cutlass::Shape<1, 1, 128>,
cutlass::Shape<1, 1, 64>,
cutlass::Shape<1, 1, 2> >
BatchedReductionTraits;
run_splitK_gemm<HgemmTraits, BatchedReductionTraits>(m, n, k, 2.0f, 1.0f);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SplitK_hgemm_128x128x8_splits16, hgemm_128x256x64_tt) {
const int splits_count = 16;
const int m = 128;
const int n = 256;
const int k = 64;
/*batched sgemm traits*/
typedef cutlass::gemm::HgemmTraits<cutlass::MatrixLayout::kRowMajor,
cutlass::MatrixLayout::kRowMajor, cutlass::Shape<8, 128, 128> >
HgemmTraits;
/*batched reduction traits*/
typedef cutlass::reduction::BatchedReductionTraits<half,
half,
half,
half,
half, /*accumulation type*/
splits_count,
cutlass::Shape<1, 1, 128>,
cutlass::Shape<1, 1, 64>,
cutlass::Shape<1, 1, 2> >
BatchedReductionTraits;
run_splitK_gemm<HgemmTraits, BatchedReductionTraits>(m, n, k, 2.0f, 1.0f);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SplitK_hgemm_128x128x8_splits16, hgemm_128x256x66_nn) {
const int splits_count = 16;
const int m = 128;
const int n = 256;
const int k = 66;
/*batched sgemm traits*/
typedef cutlass::gemm::HgemmTraits<cutlass::MatrixLayout::kColumnMajor,
cutlass::MatrixLayout::kColumnMajor, cutlass::Shape<8, 128, 128> >
HgemmTraits;
/*batched reduction traits*/
typedef cutlass::reduction::BatchedReductionTraits<half,
half,
half,
half,
half, /*accumulation type*/
splits_count,
cutlass::Shape<1, 1, 128>,
cutlass::Shape<1, 1, 64>,
cutlass::Shape<1, 1, 2> >
BatchedReductionTraits;
run_splitK_gemm<HgemmTraits, BatchedReductionTraits>(m, n, k, 1.0f, 0.0f);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SplitK_hgemm_128x128x8_splits16, hgemm_128x256x66_nt) {
const int splits_count = 16;
const int m = 128;
const int n = 256;
const int k = 66;
/*batched sgemm traits*/
typedef cutlass::gemm::HgemmTraits<cutlass::MatrixLayout::kColumnMajor,
cutlass::MatrixLayout::kRowMajor, cutlass::Shape<8, 128, 128> >
HgemmTraits;
/*batched reduction traits*/
typedef cutlass::reduction::BatchedReductionTraits<half,
half,
half,
half,
half, /*accumulation type*/
splits_count,
cutlass::Shape<1, 1, 128>,
cutlass::Shape<1, 1, 64>,
cutlass::Shape<1, 1, 2> >
BatchedReductionTraits;
run_splitK_gemm<HgemmTraits, BatchedReductionTraits>(m, n, k, 2.0f, 1.0f);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SplitK_hgemm_128x128x8_splits16, hgemm_128x256x66_tn) {
const int splits_count = 16;
const int m = 128;
const int n = 256;
const int k = 66;
/*batched sgemm traits*/
typedef cutlass::gemm::HgemmTraits<cutlass::MatrixLayout::kRowMajor,
cutlass::MatrixLayout::kColumnMajor, cutlass::Shape<8, 128, 128> >
HgemmTraits;
/*batched reduction traits*/
typedef cutlass::reduction::BatchedReductionTraits<half,
half,
half,
half,
half, /*accumulation type*/
splits_count,
cutlass::Shape<1, 1, 128>,
cutlass::Shape<1, 1, 64>,
cutlass::Shape<1, 1, 2> >
BatchedReductionTraits;
run_splitK_gemm<HgemmTraits, BatchedReductionTraits>(m, n, k, 2.0f, 1.0f);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SplitK_hgemm_128x128x8_splits16, hgemm_128x256x66_tt) {
const int splits_count = 16;
const int m = 128;
const int n = 256;
const int k = 66;
/*batched sgemm traits*/
typedef cutlass::gemm::HgemmTraits<cutlass::MatrixLayout::kRowMajor,
cutlass::MatrixLayout::kRowMajor, cutlass::Shape<8, 128, 128> >
HgemmTraits;
/*batched reduction traits*/
typedef cutlass::reduction::BatchedReductionTraits<half,
half,
half,
half,
half, /*accumulation type*/
splits_count,
cutlass::Shape<1, 1, 128>,
cutlass::Shape<1, 1, 64>,
cutlass::Shape<1, 1, 2> >
BatchedReductionTraits;
run_splitK_gemm<HgemmTraits, BatchedReductionTraits>(m, n, k, 2.0f, 1.0f);
}
tools/test/unit/gemm/splitK_igemm.cu
+367
View File
@@ -0,0 +1,367 @@
/***************************************************************************************************
* Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
* * Redistributions of source code must retain the above copyright notice, this list of
* conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright notice, this list of
* conditions and the following disclaimer in the documentation and/or other materials
* provided with the distribution.
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
* to endorse or promote products derived from this software without specific prior written
* permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
#include "cutlass_unit_test.h"
#include "cutlass/gemm/gemm.h"
#include "cutlass/gemm/igemm_traits.h"
#include "cutlass/reduction/batched_reduction_traits.h"
#include "tools/test/unit/gemm/gemm_testbed.h"
#include "tools/test/unit/gemm/run_gemm.h"
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SplitK_igemm_128x128x32_splits16, igemm_128x256x512_nn) {
const int splits_count = 16;
const int m = 128;
const int n = 256;
const int k = 512;
/*batched igemm traits*/
typedef cutlass::gemm::IgemmTraits<cutlass::MatrixLayout::kColumnMajor,
cutlass::MatrixLayout::kColumnMajor,
cutlass::Shape<32, 128, 128>, int, cutlass::gemm::LinearScaling<int> >
IgemmTraits;
/*batched reduction traits*/
typedef cutlass::reduction::BatchedReductionTraits<int,
int,
int,
int,
int, /*accumulation type*/
splits_count,
cutlass::Shape<1, 1, 128>,
cutlass::Shape<1, 1, 64>,
cutlass::Shape<1, 1, 2> >
BatchedReductionTraits;
run_splitK_gemm<IgemmTraits, BatchedReductionTraits>(m, n, k, 2, 1, true /*use host reference*/);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SplitK_igemm_128x128x32_splits16, igemm_128x256x512_nt) {
const int splits_count = 16;
const int m = 128;
const int n = 256;
const int k = 512;
/*batched igemm traits*/
typedef cutlass::gemm::IgemmTraits<cutlass::MatrixLayout::kColumnMajor,
cutlass::MatrixLayout::kRowMajor,
cutlass::Shape<32, 128, 128>, int, cutlass::gemm::LinearScaling<int> >
IgemmTraits;
/*batched reduction traits*/
typedef cutlass::reduction::BatchedReductionTraits<int,
int,
int,
int,
int, /*accumulation type*/
splits_count,
cutlass::Shape<1, 1, 128>,
cutlass::Shape<1, 1, 64>,
cutlass::Shape<1, 1, 2> >
BatchedReductionTraits;
run_splitK_gemm<IgemmTraits, BatchedReductionTraits>(m, n, k, 2, 1, true /*use host reference*/);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SplitK_igemm_128x128x32_splits16, igemm_128x256x512_tn) {
const int splits_count = 16;
const int m = 128;
const int n = 256;
const int k = 512;
/*batched igemm traits*/
typedef cutlass::gemm::IgemmTraits<cutlass::MatrixLayout::kRowMajor,
cutlass::MatrixLayout::kColumnMajor,
cutlass::Shape<32, 128, 128>, int, cutlass::gemm::LinearScaling<int> >
IgemmTraits;
/*batched reduction traits*/
typedef cutlass::reduction::BatchedReductionTraits<int,
int,
int,
int,
int, /*accumulation type*/
splits_count,
cutlass::Shape<1, 1, 128>,
cutlass::Shape<1, 1, 64>,
cutlass::Shape<1, 1, 2> >
BatchedReductionTraits;
run_splitK_gemm<IgemmTraits, BatchedReductionTraits>(m, n, k, 2, 1, true /*use host reference*/);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SplitK_igemm_128x128x32_splits16, igemm_128x256x512_tt) {
const int splits_count = 16;
const int m = 128;
const int n = 256;
const int k = 512;
/*batched igemm traits*/
typedef cutlass::gemm::IgemmTraits<cutlass::MatrixLayout::kRowMajor,
cutlass::MatrixLayout::kRowMajor,
cutlass::Shape<32, 128, 128>, int, cutlass::gemm::LinearScaling<int> >
IgemmTraits;
/*batched reduction traits*/
typedef cutlass::reduction::BatchedReductionTraits<int,
int,
int,
int,
int, /*accumulation type*/
splits_count,
cutlass::Shape<1, 1, 128>,
cutlass::Shape<1, 1, 64>,
cutlass::Shape<1, 1, 2> >
BatchedReductionTraits;
run_splitK_gemm<IgemmTraits, BatchedReductionTraits>(m, n, k, 2, 1, true /*use host reference*/);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SplitK_igemm_128x128x32_splits16, igemm_1024x64x4096_nn) {
const int splits_count = 16;
const int m = 1024;
const int n = 64;
const int k = 4096;
/*batched igemm traits*/
typedef cutlass::gemm::IgemmTraits<cutlass::MatrixLayout::kColumnMajor,
cutlass::MatrixLayout::kColumnMajor,
cutlass::Shape<32, 128, 128>, int, cutlass::gemm::LinearScaling<int> >
IgemmTraits;
/*batched reduction traits*/
typedef cutlass::reduction::BatchedReductionTraits<int,
int,
int,
int,
int, /*accumulation type*/
splits_count,
cutlass::Shape<1, 1, 128>,
cutlass::Shape<1, 1, 64>,
cutlass::Shape<1, 1, 2> >
BatchedReductionTraits;
run_splitK_gemm<IgemmTraits, BatchedReductionTraits>(m, n, k, 1, 0, false /*not use host reference*/);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SplitK_igemm_128x128x32_splits16, igemm_1024x64x4096_nt) {
const int splits_count = 16;
const int m = 1024;
const int n = 64;
const int k = 4096;
/*batched igemm traits*/
typedef cutlass::gemm::IgemmTraits<cutlass::MatrixLayout::kColumnMajor,
cutlass::MatrixLayout::kRowMajor,
cutlass::Shape<32, 128, 128>, int, cutlass::gemm::LinearScaling<int> >
IgemmTraits;
/*batched reduction traits*/
typedef cutlass::reduction::BatchedReductionTraits<int,
int,
int,
int,
int, /*accumulation type*/
splits_count,
cutlass::Shape<1, 1, 128>,
cutlass::Shape<1, 1, 64>,
cutlass::Shape<1, 1, 2> >
BatchedReductionTraits;
run_splitK_gemm<IgemmTraits, BatchedReductionTraits>(m, n, k, 1, 0, false /*not use host reference*/);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SplitK_igemm_128x128x32_splits16, igemm_1024x64x4096_tn) {
const int splits_count = 16;
const int m = 1024;
const int n = 64;
const int k = 4096;
/*batched igemm traits*/
typedef cutlass::gemm::IgemmTraits<cutlass::MatrixLayout::kRowMajor,
cutlass::MatrixLayout::kColumnMajor,
cutlass::Shape<32, 128, 128>, int, cutlass::gemm::LinearScaling<int> >
IgemmTraits;
/*batched reduction traits*/
typedef cutlass::reduction::BatchedReductionTraits<int,
int,
int,
int,
int, /*accumulation type*/
splits_count,
cutlass::Shape<1, 1, 128>,
cutlass::Shape<1, 1, 64>,
cutlass::Shape<1, 1, 2> >
BatchedReductionTraits;
run_splitK_gemm<IgemmTraits, BatchedReductionTraits>(m, n, k, 1, 0, false /*not use host reference*/);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SplitK_igemm_128x128x32_splits16, igemm_1024x64x4096_tt) {
const int splits_count = 16;
const int m = 1024;
const int n = 64;
const int k = 4096;
/*batched igemm traits*/
typedef cutlass::gemm::IgemmTraits<cutlass::MatrixLayout::kRowMajor,
cutlass::MatrixLayout::kRowMajor,
cutlass::Shape<32, 128, 128>, int, cutlass::gemm::LinearScaling<int> >
IgemmTraits;
/*batched reduction traits*/
typedef cutlass::reduction::BatchedReductionTraits<int,
int,
int,
int,
int, /*accumulation type*/
splits_count,
cutlass::Shape<1, 1, 128>,
cutlass::Shape<1, 1, 64>,
cutlass::Shape<1, 1, 2> >
BatchedReductionTraits;
run_splitK_gemm<IgemmTraits, BatchedReductionTraits>(m, n, k, 1, 0, false /*not use host reference*/);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SplitK_igemm_128x32x32_splits16, igemm_1024x64x4096_nn) {
const int splits_count = 16;
const int m = 1024;
const int n = 64;
const int k = 4096;
/*batched igemm traits*/
typedef cutlass::gemm::IgemmTraits<cutlass::MatrixLayout::kColumnMajor,
cutlass::MatrixLayout::kColumnMajor,
cutlass::Shape<32, 32, 128>, int, cutlass::gemm::LinearScaling<int> >
IgemmTraits;
/*batched reduction traits*/
typedef cutlass::reduction::BatchedReductionTraits<int,
int,
int,
int,
int, /*accumulation type*/
splits_count,
cutlass::Shape<1, 1, 128>,
cutlass::Shape<1, 1, 64>,
cutlass::Shape<1, 1, 2> >
BatchedReductionTraits;
run_splitK_gemm<IgemmTraits, BatchedReductionTraits>(m, n, k, 1, 0, false /*not use host reference*/);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SplitK_igemm_128x32x32_splits16, igemm_1024x64x4096_nt) {
const int splits_count = 16;
const int m = 1024;
const int n = 64;
const int k = 4096;
/*batched igemm traits*/
typedef cutlass::gemm::IgemmTraits<cutlass::MatrixLayout::kColumnMajor,
cutlass::MatrixLayout::kRowMajor,
cutlass::Shape<32, 32, 128>, int, cutlass::gemm::LinearScaling<int> >
IgemmTraits;
/*batched reduction traits*/
typedef cutlass::reduction::BatchedReductionTraits<int,
int,
int,
int,
int, /*accumulation type*/
splits_count,
cutlass::Shape<1, 1, 128>,
cutlass::Shape<1, 1, 64>,
cutlass::Shape<1, 1, 2> >
BatchedReductionTraits;
run_splitK_gemm<IgemmTraits, BatchedReductionTraits>(m, n, k, 1, 0, false /*not use host reference*/);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SplitK_igemm_128x32x32_splits16, igemm_1024x64x4096_tn) {
const int splits_count = 16;
const int m = 1024;
const int n = 64;
const int k = 4096;
/*batched igemm traits*/
typedef cutlass::gemm::IgemmTraits<cutlass::MatrixLayout::kRowMajor,
cutlass::MatrixLayout::kColumnMajor,
cutlass::Shape<32, 32, 128>, int, cutlass::gemm::LinearScaling<int> >
IgemmTraits;
/*batched reduction traits*/
typedef cutlass::reduction::BatchedReductionTraits<int,
int,
int,
int,
int, /*accumulation type*/
splits_count,
cutlass::Shape<1, 1, 128>,
cutlass::Shape<1, 1, 64>,
cutlass::Shape<1, 1, 2> >
BatchedReductionTraits;
run_splitK_gemm<IgemmTraits, BatchedReductionTraits>(m, n, k, 1, 0, false /*not use host reference*/);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SplitK_igemm_128x32x32_splits16, igemm_1024x64x4096_tt) {
const int splits_count = 16;
const int m = 1024;
const int n = 64;
const int k = 4096;
/*batched igemm traits*/
typedef cutlass::gemm::IgemmTraits<cutlass::MatrixLayout::kRowMajor,
cutlass::MatrixLayout::kRowMajor,
cutlass::Shape<32, 32, 128>, int, cutlass::gemm::LinearScaling<int> >
IgemmTraits;
/*batched reduction traits*/
typedef cutlass::reduction::BatchedReductionTraits<int,
int,
int,
int,
int, /*accumulation type*/
splits_count,
cutlass::Shape<1, 1, 128>,
cutlass::Shape<1, 1, 64>,
cutlass::Shape<1, 1, 2> >
BatchedReductionTraits;
run_splitK_gemm<IgemmTraits, BatchedReductionTraits>(m, n, k, 1, 0, false /*not use host reference*/);
}
tools/test/unit/gemm/splitK_sgemm.cu
+355
View File
@@ -0,0 +1,355 @@
/***************************************************************************************************
* Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
* * Redistributions of source code must retain the above copyright notice, this list of
* conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright notice, this list of
* conditions and the following disclaimer in the documentation and/or other materials
* provided with the distribution.
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
* to endorse or promote products derived from this software without specific prior written
* permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
#include "cutlass_unit_test.h"
#include "cutlass/gemm/gemm.h"
#include "cutlass/gemm/sgemm_traits.h"
#include "cutlass/reduction/batched_reduction_traits.h"
#include "tools/test/unit/gemm/gemm_testbed.h"
#include "tools/test/unit/gemm/run_gemm.h"
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SplitK_sgemm_128x128x8_splits16, sgemm_128x256x512_nn) {
const int splits_count = 16;
const int m = 128;
const int n = 256;
const int k = 512;
/*batched sgemm traits*/
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor,
cutlass::MatrixLayout::kColumnMajor, cutlass::Shape<8, 128, 128> >
SgemmTraits;
/*batched reduction traits*/
typedef cutlass::reduction::BatchedReductionTraits<float,
float,
float,
float,
float, /*accumulation type*/
splits_count,
cutlass::Shape<1, 1, 128>,
cutlass::Shape<1, 1, 64>,
cutlass::Shape<1, 1, 2> >
BatchedReductionTraits;
run_splitK_gemm<SgemmTraits, BatchedReductionTraits>(m, n, k, 2.0f, 1.0f);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SplitK_sgemm_128x128x8_splits16, sgemm_128x256x512_nt) {
const int splits_count = 16;
const int m = 128;
const int n = 256;
const int k = 512;
/*batched sgemm traits*/
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor,
cutlass::MatrixLayout::kRowMajor, cutlass::Shape<8, 128, 128> >
SgemmTraits;
/*batched reduction traits*/
typedef cutlass::reduction::BatchedReductionTraits<float,
float,
float,
float,
float, /*accumulation type*/
splits_count,
cutlass::Shape<1, 1, 128>,
cutlass::Shape<1, 1, 64>,
cutlass::Shape<1, 1, 2> >
BatchedReductionTraits;
run_splitK_gemm<SgemmTraits, BatchedReductionTraits>(m, n, k, 2.0f, 1.0f);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SplitK_sgemm_128x128x8_splits16, sgemm_128x256x512_tn) {
const int splits_count = 16;
const int m = 128;
const int n = 256;
const int k = 512;
/*batched sgemm traits*/
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kRowMajor,
cutlass::MatrixLayout::kColumnMajor, cutlass::Shape<8, 128, 128> >
SgemmTraits;
/*batched reduction traits*/
typedef cutlass::reduction::BatchedReductionTraits<float,
float,
float,
float,
float, /*accumulation type*/
splits_count,
cutlass::Shape<1, 1, 128>,
cutlass::Shape<1, 1, 64>,
cutlass::Shape<1, 1, 2> >
BatchedReductionTraits;
run_splitK_gemm<SgemmTraits, BatchedReductionTraits>(m, n, k, 2.0f, 1.0f);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SplitK_sgemm_128x128x8_splits16, sgemm_128x256x512_tt) {
const int splits_count = 16;
const int m = 128;
const int n = 256;
const int k = 512;
/*batched sgemm traits*/
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kRowMajor,
cutlass::MatrixLayout::kRowMajor, cutlass::Shape<8, 128, 128> >
SgemmTraits;
/*batched reduction traits*/
typedef cutlass::reduction::BatchedReductionTraits<float,
float,
float,
float,
float, /*accumulation type*/
splits_count,
cutlass::Shape<1, 1, 128>,
cutlass::Shape<1, 1, 64>,
cutlass::Shape<1, 1, 2> >
BatchedReductionTraits;
run_splitK_gemm<SgemmTraits, BatchedReductionTraits>(m, n, k, 2.0f, 1.0f);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SplitK_sgemm_128x128x8_splits16, sgemm_128x256x500_nn) {
const int splits_count = 16;
const int m = 128;
const int n = 256;
const int k = 500;
/*batched sgemm traits*/
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor,
cutlass::MatrixLayout::kColumnMajor, cutlass::Shape<8, 128, 128> >
SgemmTraits;
/*batched reduction traits*/
typedef cutlass::reduction::BatchedReductionTraits<float,
float,
float,
float,
float, /*accumulation type*/
splits_count,
cutlass::Shape<1, 1, 128>,
cutlass::Shape<1, 1, 64>,
cutlass::Shape<1, 1, 2> >
BatchedReductionTraits;
run_splitK_gemm<SgemmTraits, BatchedReductionTraits>(m, n, k, 2.0f, 1.0f);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SplitK_sgemm_128x128x8_splits16, sgemm_128x256x500_nt) {
const int splits_count = 16;
const int m = 128;
const int n = 256;
const int k = 500;
/*batched sgemm traits*/
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor,
cutlass::MatrixLayout::kRowMajor, cutlass::Shape<8, 128, 128> >
SgemmTraits;
/*batched reduction traits*/
typedef cutlass::reduction::BatchedReductionTraits<float,
float,
float,
float,
float, /*accumulation type*/
splits_count,
cutlass::Shape<1, 1, 128>,
cutlass::Shape<1, 1, 64>,
cutlass::Shape<1, 1, 2> >
BatchedReductionTraits;
run_splitK_gemm<SgemmTraits, BatchedReductionTraits>(m, n, k, 2.0f, 1.0f);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SplitK_sgemm_128x128x8_splits16, sgemm_128x256x500_tn) {
const int splits_count = 16;
const int m = 128;
const int n = 256;
const int k = 500;
/*batched sgemm traits*/
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kRowMajor,
cutlass::MatrixLayout::kColumnMajor, cutlass::Shape<8, 128, 128> >
SgemmTraits;
/*batched reduction traits*/
typedef cutlass::reduction::BatchedReductionTraits<float,
float,
float,
float,
float, /*accumulation type*/
splits_count,
cutlass::Shape<1, 1, 128>,
cutlass::Shape<1, 1, 64>,
cutlass::Shape<1, 1, 2> >
BatchedReductionTraits;
run_splitK_gemm<SgemmTraits, BatchedReductionTraits>(m, n, k, 2.0f, 1.0f);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SplitK_sgemm_128x128x8_splits16, sgemm_128x256x500_tt) {
const int splits_count = 16;
const int m = 128;
const int n = 256;
const int k = 500;
/*batched sgemm traits*/
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kRowMajor,
cutlass::MatrixLayout::kRowMajor, cutlass::Shape<8, 128, 128> >
SgemmTraits;
/*batched reduction traits*/
typedef cutlass::reduction::BatchedReductionTraits<float,
float,
float,
float,
float, /*accumulation type*/
splits_count,
cutlass::Shape<1, 1, 128>,
cutlass::Shape<1, 1, 64>,
cutlass::Shape<1, 1, 2> >
BatchedReductionTraits;
run_splitK_gemm<SgemmTraits, BatchedReductionTraits>(m, n, k, 2.0f, 1.0f);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SplitK_sgemm_128x128x8_splits16, sgemm_1024x64x4096_nn) {
const int splits_count = 16;
const int m = 1024;
const int n = 64;
const int k = 4096;
/*batched sgemm traits*/
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor,
cutlass::MatrixLayout::kColumnMajor, cutlass::Shape<8, 128, 128> >
SgemmTraits;
/*batched reduction traits*/
typedef cutlass::reduction::BatchedReductionTraits<float,
float,
float,
float,
float, /*accumulation type*/
splits_count,
cutlass::Shape<1, 1, 128>,
cutlass::Shape<1, 1, 64>,
cutlass::Shape<1, 1, 2> >
BatchedReductionTraits;
run_splitK_gemm<SgemmTraits, BatchedReductionTraits>(m, n, k, 2.0f, 1.0f);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SplitK_sgemm_128x128x8_splits16, sgemm_1024x64x4096_nt) {
const int splits_count = 16;
const int m = 1024;
const int n = 64;
const int k = 4096;
/*batched sgemm traits*/
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kColumnMajor,
cutlass::MatrixLayout::kRowMajor, cutlass::Shape<8, 128, 128> >
SgemmTraits;
/*batched reduction traits*/
typedef cutlass::reduction::BatchedReductionTraits<float,
float,
float,
float,
float, /*accumulation type*/
splits_count,
cutlass::Shape<1, 1, 128>,
cutlass::Shape<1, 1, 64>,
cutlass::Shape<1, 1, 2> >
BatchedReductionTraits;
run_splitK_gemm<SgemmTraits, BatchedReductionTraits>(m, n, k, 2.0f, 1.0f);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SplitK_sgemm_128x128x8_splits16, sgemm_1024x64x4096_tn) {
const int splits_count = 16;
const int m = 1024;
const int n = 64;
const int k = 4096;
/*batched sgemm traits*/
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kRowMajor,
cutlass::MatrixLayout::kColumnMajor, cutlass::Shape<8, 128, 128> >
SgemmTraits;
/*batched reduction traits*/
typedef cutlass::reduction::BatchedReductionTraits<float,
float,
float,
float,
float, /*accumulation type*/
splits_count,
cutlass::Shape<1, 1, 128>,
cutlass::Shape<1, 1, 64>,
cutlass::Shape<1, 1, 2> >
BatchedReductionTraits;
run_splitK_gemm<SgemmTraits, BatchedReductionTraits>(m, n, k, 2.0f, 1.0f);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SplitK_sgemm_128x128x8_splits16, sgemm_1024x64x4096_tt) {
const int splits_count = 16;
const int m = 1024;
const int n = 64;
const int k = 4096;
/*batched sgemm traits*/
typedef cutlass::gemm::SgemmTraits<cutlass::MatrixLayout::kRowMajor,
cutlass::MatrixLayout::kRowMajor, cutlass::Shape<8, 128, 128> >
SgemmTraits;
/*batched reduction traits*/
typedef cutlass::reduction::BatchedReductionTraits<float,
float,
float,
float,
float, /*accumulation type*/
splits_count,
cutlass::Shape<1, 1, 128>,
cutlass::Shape<1, 1, 64>,
cutlass::Shape<1, 1, 2> >
BatchedReductionTraits;
run_splitK_gemm<SgemmTraits, BatchedReductionTraits>(m, n, k, 2.0f, 1.0f);
}
tools/test/unit/gemm/splitK_wmma_gemm.cu
+175
View File
@@ -0,0 +1,175 @@
/***************************************************************************************************
* Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
* * Redistributions of source code must retain the above copyright notice, this list of
* conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright notice, this list of
* conditions and the following disclaimer in the documentation and/or other materials
* provided with the distribution.
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
* to endorse or promote products derived from this software without specific prior written
* permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
#include "cutlass/wmma_matrix.h"
#if defined(CUTLASS_USE_WMMA_API)
#include "cutlass_unit_test.h"
#include "cutlass/gemm/gemm.h"
#include "cutlass/gemm/wmma_gemm_traits.h"
#include "cutlass/reduction/batched_reduction_traits.h"
#include "tools/test/unit/gemm/gemm_testbed.h"
#include "tools/test/unit/gemm/run_gemm.h"
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SplitK_wmma_gemm_16x16x32_splits16, wmma_gemm_128x256x512_nn) {
const int splits_count = 16;
const int m = 128;
const int n = 256;
const int k = 512;
/*batched wmma gemm traits*/
typedef cutlass::gemm::WmmaGemmTraits<
cutlass::MatrixLayout::kColumnMajor,
cutlass::MatrixLayout::kColumnMajor,
cutlass::Shape<32, 16, 16>,
half,
half,
half,
cutlass::gemm::LinearScaling<half>,
half
>
WmmaGemmTraits;
/*batched reduction traits*/
typedef cutlass::reduction::BatchedReductionTraits<half,
half,
half,
half,
half, /*accumulation type*/
splits_count,
cutlass::Shape<1, 1, 128>,
cutlass::Shape<1, 1, 64>,
cutlass::Shape<1, 1, 2> >
BatchedReductionTraits;
run_splitK_gemm<WmmaGemmTraits, BatchedReductionTraits>(m, n, k, 2.0f, 1.0f, true/*use host reference*/);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SplitK_wmma_gemm_16x16x32_splits16, wmma_gemm_128x256x512_nt) {
const int splits_count = 16;
const int m = 128;
const int n = 256;
const int k = 512;
/*batched wmma gemm traits*/
typedef cutlass::gemm::WmmaGemmTraits<
cutlass::MatrixLayout::kColumnMajor,
cutlass::MatrixLayout::kRowMajor,
cutlass::Shape<32, 16, 16>,
half,
half,
half,
cutlass::gemm::LinearScaling<half>,
half
>
WmmaGemmTraits;
/*batched reduction traits*/
typedef cutlass::reduction::BatchedReductionTraits<half,
half,
half,
half,
half, /*accumulation type*/
splits_count,
cutlass::Shape<1, 1, 128>,
cutlass::Shape<1, 1, 64>,
cutlass::Shape<1, 1, 2> >
BatchedReductionTraits;
run_splitK_gemm<WmmaGemmTraits, BatchedReductionTraits>(m, n, k, 1.0f, 0.0f);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SplitK_wmma_gemm_16x16x32_splits16, wmma_gemm_128x256x512_tn) {
const int splits_count = 16;
const int m = 128;
const int n = 256;
const int k = 512;
/*batched wmma gemm traits*/
typedef cutlass::gemm::WmmaGemmTraits<
cutlass::MatrixLayout::kRowMajor,
cutlass::MatrixLayout::kColumnMajor,
cutlass::Shape<32, 16, 16>,
half,
half,
half,
cutlass::gemm::LinearScaling<half>,
half
>
WmmaGemmTraits;
/*batched reduction traits*/
typedef cutlass::reduction::BatchedReductionTraits<half,
half,
half,
half,
half, /*accumulation type*/
splits_count,
cutlass::Shape<1, 1, 128>,
cutlass::Shape<1, 1, 64>,
cutlass::Shape<1, 1, 2> >
BatchedReductionTraits;
run_splitK_gemm<WmmaGemmTraits, BatchedReductionTraits>(m, n, k, 1.0f, 0.0f);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(SplitK_wmma_gemm_16x16x32_splits16, wmma_gemm_128x256x512_tt) {
const int splits_count = 16;
const int m = 128;
const int n = 256;
const int k = 512;
/*batched wmma gemm traits*/
typedef cutlass::gemm::WmmaGemmTraits<
cutlass::MatrixLayout::kRowMajor,
cutlass::MatrixLayout::kRowMajor,
cutlass::Shape<32, 16, 16>,
half,
half,
half,
cutlass::gemm::LinearScaling<half>,
half
>
WmmaGemmTraits;
/*batched reduction traits*/
typedef cutlass::reduction::BatchedReductionTraits<half,
half,
half,
half,
half, /*accumulation type*/
splits_count,
cutlass::Shape<1, 1, 128>,
cutlass::Shape<1, 1, 64>,
cutlass::Shape<1, 1, 2> >
BatchedReductionTraits;
run_splitK_gemm<WmmaGemmTraits, BatchedReductionTraits>(m, n, k, 1.0f, 0.0f);
}
#endif
tools/test/unit/gemm/wmma_gemm.cu
+1 -2
View File
@@ -53,6 +53,7 @@ TEST(WmmaGemm_16x16x32_f16, wmma_gemm_16x16x16_nn) {
run_gemm<WmmaGemmTraits>(16, 16, 16);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(WmmaGemm_16x16x32_f16, wmma_gemm_16x16x32_nn) {
@@ -367,7 +368,5 @@ TEST(WmmaGemm_128x128x32, wmma_32x8x16_gemm_256x256x128_tn) {
run_gemm<WmmaGemmTraits>(256, 256, 128);
}
#endif
////////////////////////////////////////////////////////////////////////////////////////////////////
#endif // defined CUTLASS_USE_WMMA_API
tools/test/unit/gemm/wmma_gemm_non_multiple16.cu
+155
View File
@@ -0,0 +1,155 @@
/***************************************************************************************************
* Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
* * Redistributions of source code must retain the above copyright notice, this list of
* conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright notice, this list of
* conditions and the following disclaimer in the documentation and/or other materials
* provided with the distribution.
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
* to endorse or promote products derived from this software without specific prior written
* permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
#include "cutlass/wmma_matrix.h"
#if defined(CUTLASS_USE_WMMA_API)
#include "cutlass_unit_test.h"
#include "cutlass/gemm/gemm.h"
#include "cutlass/gemm/wmma_gemm_traits.h"
#include "tools/test/unit/gemm/gemm_testbed.h"
#include "tools/test/unit/gemm/run_gemm.h"
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(WmmaGemm_16x16x32_f16, wmma_gemm_36x36x16_nn) {
/*
this wmmaTraits requires leading dim to be divisible by 4
*/
typedef cutlass::gemm::WmmaGemmTraits<
cutlass::MatrixLayout::kColumnMajor,
cutlass::MatrixLayout::kColumnMajor,
cutlass::Shape<32, 16, 16>,
half,
half,
half,
cutlass::gemm::LinearScaling<half>,
half,
typename cutlass::gemm::WmmaGemmAccumulatorsPerWarp<typename cutlass::Shape<32, 16, 16> >::Shape,
typename cutlass::Shape<16, 16, 16>,
4, /*kScalarsPerLdgA_*/
4, /*kScalarsPerLdgB_*/
4, /*KScalarsPerLdsA_*/
4, /*KScalarsPerLdsB_*/
4 / sizeof(half), /*kScalarsPerLdgCAndStgD_*/
4 / sizeof(half), /*kScalarsPerStsD_*/
4 / sizeof(half) /*kScalarsPerLdsD_*/
>
WmmaGemmTraits;
run_gemm<WmmaGemmTraits>(36, 36, 64);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(WmmaGemm_16x16x32_f16, wmma_gemm_36x36x16_nt) {
/*
this wmmaTraits requires leading dim to be divisible by 4
*/
typedef cutlass::gemm::WmmaGemmTraits<
cutlass::MatrixLayout::kColumnMajor,
cutlass::MatrixLayout::kRowMajor,
cutlass::Shape<32, 16, 16>,
half,
half,
half,
cutlass::gemm::LinearScaling<half>,
half,
typename cutlass::gemm::WmmaGemmAccumulatorsPerWarp<typename cutlass::Shape<32, 16, 16> >::Shape,
typename cutlass::Shape<16, 16, 16>,
4, /*kScalarsPerLdgA_*/
4, /*kScalarsPerLdgB_*/
4, /*KScalarsPerLdsA_*/
4, /*KScalarsPerLdsB_*/
4 / sizeof(half), /*kScalarsPerLdgCAndStgD_*/
4 / sizeof(half), /*kScalarsPerStsD_*/
4 / sizeof(half) /*kScalarsPerLdsD_*/
>
WmmaGemmTraits;
run_gemm<WmmaGemmTraits>(36, 36, 64);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(WmmaGemm_16x16x32_f16, wmma_gemm_36x36x16_tn) {
/*
this wmmaTraits requires leading dim to be divisible by 4
*/
typedef cutlass::gemm::WmmaGemmTraits<
cutlass::MatrixLayout::kRowMajor,
cutlass::MatrixLayout::kColumnMajor,
cutlass::Shape<32, 16, 16>,
half,
half,
half,
cutlass::gemm::LinearScaling<half>,
half,
typename cutlass::gemm::WmmaGemmAccumulatorsPerWarp<typename cutlass::Shape<32, 16, 16> >::Shape,
typename cutlass::Shape<16, 16, 16>,
4, /*kScalarsPerLdgA_*/
4, /*kScalarsPerLdgB_*/
4, /*KScalarsPerLdsA_*/
4, /*KScalarsPerLdsB_*/
4 / sizeof(half), /*kScalarsPerLdgCAndStgD_*/
4 / sizeof(half), /*kScalarsPerStsD_*/
4 / sizeof(half) /*kScalarsPerLdsD_*/
>
WmmaGemmTraits;
run_gemm<WmmaGemmTraits>(36, 36, 64);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
TEST(WmmaGemm_16x16x32_f16, wmma_gemm_36x36x16_tt) {
/*
this wmmaTraits requires leading dim to be divisible by 4
*/
typedef cutlass::gemm::WmmaGemmTraits<
cutlass::MatrixLayout::kRowMajor,
cutlass::MatrixLayout::kRowMajor,
cutlass::Shape<32, 16, 16>,
half,
half,
half,
cutlass::gemm::LinearScaling<half>,
half,
typename cutlass::gemm::WmmaGemmAccumulatorsPerWarp<typename cutlass::Shape<32, 16, 16> >::Shape,
typename cutlass::Shape<16, 16, 16>,
4, /*kScalarsPerLdgA_*/
4, /*kScalarsPerLdgB_*/
4, /*KScalarsPerLdsA_*/
4, /*KScalarsPerLdsB_*/
4 / sizeof(half), /*kScalarsPerLdgCAndStgD_*/
4 / sizeof(half), /*kScalarsPerStsD_*/
4 / sizeof(half) /*kScalarsPerLdsD_*/
>
WmmaGemmTraits;
run_gemm<WmmaGemmTraits>(36, 36, 64);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
#endif