CUTLASS 2.0 (#62)
CUTLASS 2.0 Substantially refactored for - Better performance, particularly for native Turing Tensor Cores - Robust and durable templates spanning the design space - Encapsulated functionality embodying modern C++11 programming techniques - Optimized containers and data types for efficient, generic, portable device code Updates to: - Quick start guide - Documentation - Utilities - CUTLASS Profiler Native Turing Tensor Cores - Efficient GEMM kernels targeting Turing Tensor Cores - Mixed-precision floating point, 8-bit integer, 4-bit integer, and binarized operands Coverage of existing CUTLASS functionality: - GEMM kernels targeting CUDA and Tensor Cores in NVIDIA GPUs - Volta Tensor Cores through native mma.sync and through WMMA API - Optimizations such as parallel reductions, threadblock rasterization, and intra-threadblock reductions - Batched GEMM operations - Complex-valued GEMMs Note: this commit and all that follow require a host compiler supporting C++11 or greater.
This commit is contained in:
Andrew Kerr
@@ -0,0 +1,33 @@
|
||||
# Copyright (c) 2017-2019, NVIDIA CORPORATION. All rights reserved.
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
# provided that the following conditions are met:
|
||||
# * Redistributions of source code must retain the above copyright notice, this list of
|
||||
# conditions and the following disclaimer.
|
||||
# * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
# conditions and the following disclaimer in the documentation and/or other materials
|
||||
# provided with the distribution.
|
||||
# * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
# to endorse or promote products derived from this software without specific prior written
|
||||
# permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
# FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
# OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
# STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
cutlass_test_unit_add_executable(
|
||||
cutlass_test_unit_epilogue_threadblock
|
||||
predicated_tile_iterator.cu
|
||||
output_tile_threadmap.cu
|
||||
epilogue_simt.cu
|
||||
epilogue_simt_sm60.cu
|
||||
epilogue_simt_sm61.cu
|
||||
epilogue_tensor_op.cu
|
||||
epilogue_volta_tensor_op.cu
|
||||
epilogue_wmma_tensor_op_sm70.cu
|
||||
)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,485 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2019, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/*! \file
|
||||
\brief Unit tests for thread-level GEMM
|
||||
*/
|
||||
|
||||
#include <fstream>
|
||||
|
||||
#include "../../common/cutlass_unit_test.h"
|
||||
|
||||
#include "cutlass/aligned_buffer.h"
|
||||
|
||||
#include "cutlass/gemm/warp/mma_simt.h"
|
||||
#include "cutlass/gemm/warp/mma_simt_policy.h"
|
||||
|
||||
#include "cutlass/epilogue/thread/linear_combination.h"
|
||||
#include "cutlass/epilogue/threadblock/default_epilogue_simt.h"
|
||||
|
||||
#include "cutlass/util/host_tensor.h"
|
||||
#include "cutlass/util/tensor_view_io.h"
|
||||
#include "cutlass/util/reference/host/tensor_fill.h"
|
||||
|
||||
#include "testbed.h"
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
//
|
||||
// Real-valued half precision tests
|
||||
//
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM60_Epilogue_threadblock_epilogue, simt_f16_32x64_32x64x8) {
|
||||
|
||||
//
|
||||
// Define the warp-level matrix multiply
|
||||
//
|
||||
|
||||
using Element = cutlass::half_t;
|
||||
using ElementOutput = cutlass::half_t;
|
||||
using ElementAccumulator = cutlass::half_t;
|
||||
using ElementCompute = cutlass::half_t;
|
||||
|
||||
int const kElementsPerAccess = 1;
|
||||
|
||||
using Shape = cutlass::gemm::GemmShape<32, 64, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<32, 64, 8>;
|
||||
using ElementC = ElementAccumulator;
|
||||
using LayoutA = cutlass::layout::ColumnMajor;
|
||||
using LayoutB = cutlass::layout::RowMajor;
|
||||
using LayoutC = cutlass::layout::RowMajor;
|
||||
|
||||
using ElementOutput = Element;
|
||||
using ElementAccumulator = Element;
|
||||
using ElementCompute = Element;
|
||||
|
||||
using WarpMmaSimt = cutlass::gemm::warp::MmaSimt<
|
||||
WarpShape,
|
||||
Element,
|
||||
LayoutA,
|
||||
Element,
|
||||
LayoutB,
|
||||
Element,
|
||||
LayoutC,
|
||||
cutlass::gemm::warp::MmaSimtPolicy<
|
||||
cutlass::MatrixShape<4, 8>,
|
||||
cutlass::layout::RowMajorInterleaved<2>,
|
||||
cutlass::gemm::GemmShape<4, 4, 1>
|
||||
>
|
||||
>;
|
||||
|
||||
//
|
||||
// Output operator
|
||||
//
|
||||
|
||||
using OutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
ElementOutput,
|
||||
kElementsPerAccess,
|
||||
ElementAccumulator,
|
||||
ElementCompute
|
||||
>;
|
||||
|
||||
//
|
||||
// Define the epilogue
|
||||
//
|
||||
|
||||
using Epilogue = typename cutlass::epilogue::threadblock::DefaultEpilogueSimt<
|
||||
Shape,
|
||||
WarpMmaSimt,
|
||||
OutputOp,
|
||||
kElementsPerAccess
|
||||
>::Epilogue;
|
||||
|
||||
//
|
||||
// Instantiate epilogue
|
||||
//
|
||||
|
||||
EpilogueTestbed<Epilogue> testbed;
|
||||
|
||||
bool passed = testbed.run_all();
|
||||
|
||||
EXPECT_TRUE(passed);
|
||||
}
|
||||
|
||||
TEST(SM60_Epilogue_threadblock_epilogue, simt_f16_64x64_64x64x8) {
|
||||
|
||||
//
|
||||
// Define the warp-level matrix multiply
|
||||
//
|
||||
|
||||
using Element = cutlass::half_t;
|
||||
using ElementOutput = cutlass::half_t;
|
||||
using ElementAccumulator = cutlass::half_t;
|
||||
using ElementCompute = cutlass::half_t;
|
||||
|
||||
int const kElementsPerAccess = 1;
|
||||
|
||||
using Shape = cutlass::gemm::GemmShape<64, 64, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<64, 64, 8>;
|
||||
using ElementC = ElementAccumulator;
|
||||
using LayoutA = cutlass::layout::ColumnMajor;
|
||||
using LayoutB = cutlass::layout::RowMajor;
|
||||
using LayoutC = cutlass::layout::RowMajor;
|
||||
|
||||
using ElementOutput = Element;
|
||||
using ElementAccumulator = Element;
|
||||
using ElementCompute = Element;
|
||||
|
||||
using WarpMmaSimt = cutlass::gemm::warp::MmaSimt<
|
||||
WarpShape,
|
||||
Element,
|
||||
LayoutA,
|
||||
Element,
|
||||
LayoutB,
|
||||
Element,
|
||||
LayoutC,
|
||||
cutlass::gemm::warp::MmaSimtPolicy<
|
||||
cutlass::MatrixShape<4, 8>,
|
||||
cutlass::layout::RowMajorInterleaved<2>,
|
||||
cutlass::gemm::GemmShape<8, 4, 1>
|
||||
>
|
||||
>;
|
||||
|
||||
//
|
||||
// Output operator
|
||||
//
|
||||
|
||||
using OutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
ElementOutput,
|
||||
kElementsPerAccess,
|
||||
ElementAccumulator,
|
||||
ElementCompute
|
||||
>;
|
||||
|
||||
//
|
||||
// Define the epilogue
|
||||
//
|
||||
|
||||
using Epilogue = typename cutlass::epilogue::threadblock::DefaultEpilogueSimt<
|
||||
Shape,
|
||||
WarpMmaSimt,
|
||||
OutputOp,
|
||||
kElementsPerAccess
|
||||
>::Epilogue;
|
||||
|
||||
//
|
||||
// Instantiate epilogue
|
||||
//
|
||||
|
||||
EpilogueTestbed<Epilogue> testbed;
|
||||
|
||||
bool passed = testbed.run_all();
|
||||
|
||||
EXPECT_TRUE(passed);
|
||||
}
|
||||
|
||||
TEST(SM60_Epilogue_threadblock_epilogue, simt_f16_64x128_64x64x8) {
|
||||
|
||||
//
|
||||
// Define the warp-level matrix multiply
|
||||
//
|
||||
|
||||
using Element = cutlass::half_t;
|
||||
using ElementOutput = cutlass::half_t;
|
||||
using ElementAccumulator = cutlass::half_t;
|
||||
using ElementCompute = cutlass::half_t;
|
||||
|
||||
int const kElementsPerAccess = 1;
|
||||
|
||||
using Shape = cutlass::gemm::GemmShape<64, 128, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<64, 64, 8>;
|
||||
using ElementC = ElementAccumulator;
|
||||
using LayoutA = cutlass::layout::ColumnMajor;
|
||||
using LayoutB = cutlass::layout::RowMajor;
|
||||
using LayoutC = cutlass::layout::RowMajor;
|
||||
|
||||
using ElementOutput = Element;
|
||||
using ElementAccumulator = Element;
|
||||
using ElementCompute = Element;
|
||||
|
||||
using WarpMmaSimt = cutlass::gemm::warp::MmaSimt<
|
||||
WarpShape,
|
||||
Element,
|
||||
LayoutA,
|
||||
Element,
|
||||
LayoutB,
|
||||
Element,
|
||||
LayoutC,
|
||||
cutlass::gemm::warp::MmaSimtPolicy<
|
||||
cutlass::MatrixShape<4, 8>,
|
||||
cutlass::layout::RowMajorInterleaved<2>,
|
||||
cutlass::gemm::GemmShape<8, 4, 1>
|
||||
>
|
||||
>;
|
||||
|
||||
//
|
||||
// Output operator
|
||||
//
|
||||
|
||||
using OutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
ElementOutput,
|
||||
kElementsPerAccess,
|
||||
ElementAccumulator,
|
||||
ElementCompute
|
||||
>;
|
||||
|
||||
//
|
||||
// Define the epilogue
|
||||
//
|
||||
|
||||
using Epilogue = typename cutlass::epilogue::threadblock::DefaultEpilogueSimt<
|
||||
Shape,
|
||||
WarpMmaSimt,
|
||||
OutputOp,
|
||||
kElementsPerAccess
|
||||
>::Epilogue;
|
||||
|
||||
//
|
||||
// Instantiate epilogue
|
||||
//
|
||||
|
||||
EpilogueTestbed<Epilogue> testbed;
|
||||
|
||||
bool passed = testbed.run_all();
|
||||
|
||||
EXPECT_TRUE(passed);
|
||||
}
|
||||
|
||||
TEST(SM60_Epilogue_threadblock_epilogue, simt_f16_128x128_64x64x8) {
|
||||
|
||||
//
|
||||
// Define the warp-level matrix multiply
|
||||
//
|
||||
|
||||
using Element = cutlass::half_t;
|
||||
using ElementOutput = cutlass::half_t;
|
||||
using ElementAccumulator = cutlass::half_t;
|
||||
using ElementCompute = cutlass::half_t;
|
||||
|
||||
int const kElementsPerAccess = 1;
|
||||
|
||||
using Shape = cutlass::gemm::GemmShape<128, 128, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<64, 64, 8>;
|
||||
using ElementC = ElementAccumulator;
|
||||
using LayoutA = cutlass::layout::ColumnMajor;
|
||||
using LayoutB = cutlass::layout::RowMajor;
|
||||
using LayoutC = cutlass::layout::RowMajor;
|
||||
|
||||
using ElementOutput = Element;
|
||||
using ElementAccumulator = Element;
|
||||
using ElementCompute = Element;
|
||||
|
||||
using WarpMmaSimt = cutlass::gemm::warp::MmaSimt<
|
||||
WarpShape,
|
||||
Element,
|
||||
LayoutA,
|
||||
Element,
|
||||
LayoutB,
|
||||
Element,
|
||||
LayoutC,
|
||||
cutlass::gemm::warp::MmaSimtPolicy<
|
||||
cutlass::MatrixShape<4, 8>,
|
||||
cutlass::layout::RowMajorInterleaved<2>,
|
||||
cutlass::gemm::GemmShape<8, 4, 1>
|
||||
>
|
||||
>;
|
||||
|
||||
//
|
||||
// Output operator
|
||||
//
|
||||
|
||||
using OutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
ElementOutput,
|
||||
kElementsPerAccess,
|
||||
ElementAccumulator,
|
||||
ElementCompute
|
||||
>;
|
||||
|
||||
//
|
||||
// Define the epilogue
|
||||
//
|
||||
|
||||
using Epilogue = typename cutlass::epilogue::threadblock::DefaultEpilogueSimt<
|
||||
Shape,
|
||||
WarpMmaSimt,
|
||||
OutputOp,
|
||||
kElementsPerAccess
|
||||
>::Epilogue;
|
||||
|
||||
//
|
||||
// Instantiate epilogue
|
||||
//
|
||||
|
||||
EpilogueTestbed<Epilogue> testbed;
|
||||
|
||||
bool passed = testbed.run_all();
|
||||
|
||||
EXPECT_TRUE(passed);
|
||||
}
|
||||
|
||||
TEST(SM60_Epilogue_threadblock_epilogue, simt_f16_128x256_64x64x8) {
|
||||
|
||||
//
|
||||
// Define the warp-level matrix multiply
|
||||
//
|
||||
|
||||
using Element = cutlass::half_t;
|
||||
using ElementOutput = cutlass::half_t;
|
||||
using ElementAccumulator = cutlass::half_t;
|
||||
using ElementCompute = cutlass::half_t;
|
||||
|
||||
int const kElementsPerAccess = 1;
|
||||
|
||||
using Shape = cutlass::gemm::GemmShape<128, 256, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<64, 64, 8>;
|
||||
using ElementC = ElementAccumulator;
|
||||
using LayoutA = cutlass::layout::ColumnMajor;
|
||||
using LayoutB = cutlass::layout::RowMajor;
|
||||
using LayoutC = cutlass::layout::RowMajor;
|
||||
|
||||
using ElementOutput = Element;
|
||||
using ElementAccumulator = Element;
|
||||
using ElementCompute = Element;
|
||||
|
||||
using WarpMmaSimt = cutlass::gemm::warp::MmaSimt<
|
||||
WarpShape,
|
||||
Element,
|
||||
LayoutA,
|
||||
Element,
|
||||
LayoutB,
|
||||
Element,
|
||||
LayoutC,
|
||||
cutlass::gemm::warp::MmaSimtPolicy<
|
||||
cutlass::MatrixShape<4, 8>,
|
||||
cutlass::layout::RowMajorInterleaved<2>,
|
||||
cutlass::gemm::GemmShape<8, 4, 1>
|
||||
>
|
||||
>;
|
||||
|
||||
//
|
||||
// Output operator
|
||||
//
|
||||
|
||||
using OutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
ElementOutput,
|
||||
kElementsPerAccess,
|
||||
ElementAccumulator,
|
||||
ElementCompute
|
||||
>;
|
||||
|
||||
//
|
||||
// Define the epilogue
|
||||
//
|
||||
|
||||
using Epilogue = typename cutlass::epilogue::threadblock::DefaultEpilogueSimt<
|
||||
Shape,
|
||||
WarpMmaSimt,
|
||||
OutputOp,
|
||||
kElementsPerAccess
|
||||
>::Epilogue;
|
||||
|
||||
//
|
||||
// Instantiate epilogue
|
||||
//
|
||||
|
||||
EpilogueTestbed<Epilogue> testbed;
|
||||
|
||||
bool passed = testbed.run_all();
|
||||
|
||||
EXPECT_TRUE(passed);
|
||||
}
|
||||
|
||||
TEST(SM60_Epilogue_threadblock_epilogue, simt_f16_256x128_64x64x8) {
|
||||
|
||||
//
|
||||
// Define the warp-level matrix multiply
|
||||
//
|
||||
|
||||
using Element = cutlass::half_t;
|
||||
using ElementOutput = cutlass::half_t;
|
||||
using ElementAccumulator = cutlass::half_t;
|
||||
using ElementCompute = cutlass::half_t;
|
||||
|
||||
int const kElementsPerAccess = 1;
|
||||
|
||||
using Shape = cutlass::gemm::GemmShape<256, 128, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<64, 64, 8>;
|
||||
using ElementC = ElementAccumulator;
|
||||
using LayoutA = cutlass::layout::ColumnMajor;
|
||||
using LayoutB = cutlass::layout::RowMajor;
|
||||
using LayoutC = cutlass::layout::RowMajor;
|
||||
|
||||
using ElementOutput = Element;
|
||||
using ElementAccumulator = Element;
|
||||
using ElementCompute = Element;
|
||||
|
||||
using WarpMmaSimt = cutlass::gemm::warp::MmaSimt<
|
||||
WarpShape,
|
||||
Element,
|
||||
LayoutA,
|
||||
Element,
|
||||
LayoutB,
|
||||
Element,
|
||||
LayoutC,
|
||||
cutlass::gemm::warp::MmaSimtPolicy<
|
||||
cutlass::MatrixShape<4, 8>,
|
||||
cutlass::layout::RowMajorInterleaved<2>,
|
||||
cutlass::gemm::GemmShape<8, 4, 1>
|
||||
>
|
||||
>;
|
||||
|
||||
//
|
||||
// Output operator
|
||||
//
|
||||
|
||||
using OutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
ElementOutput,
|
||||
kElementsPerAccess,
|
||||
ElementAccumulator,
|
||||
ElementCompute
|
||||
>;
|
||||
|
||||
//
|
||||
// Define the epilogue
|
||||
//
|
||||
|
||||
using Epilogue = typename cutlass::epilogue::threadblock::DefaultEpilogueSimt<
|
||||
Shape,
|
||||
WarpMmaSimt,
|
||||
OutputOp,
|
||||
kElementsPerAccess
|
||||
>::Epilogue;
|
||||
|
||||
//
|
||||
// Instantiate epilogue
|
||||
//
|
||||
|
||||
EpilogueTestbed<Epilogue> testbed;
|
||||
|
||||
bool passed = testbed.run_all();
|
||||
|
||||
EXPECT_TRUE(passed);
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,260 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2019, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/*! \file
|
||||
\brief Unit tests for thread-level GEMM
|
||||
*/
|
||||
#include "cutlass/arch/wmma.h"
|
||||
|
||||
#ifdef CUTLASS_ARCH_WMMA_SM70_ENABLED
|
||||
|
||||
#include <fstream>
|
||||
|
||||
#include "../../common/cutlass_unit_test.h"
|
||||
|
||||
#include "cutlass/aligned_buffer.h"
|
||||
#include "cutlass/half.h"
|
||||
|
||||
#include "cutlass/epilogue/thread/linear_combination.h"
|
||||
#include "cutlass/gemm/warp/default_mma_wmma_tensor_op.h"
|
||||
#include "cutlass/epilogue/threadblock/default_epilogue_wmma_tensor_op.h"
|
||||
|
||||
#include "cutlass/util/host_tensor.h"
|
||||
#include "cutlass/util/tensor_view_io.h"
|
||||
#include "cutlass/util/reference/host/tensor_fill.h"
|
||||
|
||||
#include "testbed.h"
|
||||
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
//
|
||||
// F16 acumulation
|
||||
//
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
TEST(SM70_Epilogue_threadblock_epilogue, f16_wmma_tensor_op_64x64_64x64x16) {
|
||||
|
||||
//
|
||||
// Define the warp-level matrix multiply
|
||||
//
|
||||
using ElementOutput = cutlass::half_t;
|
||||
using ElementAccumulator = cutlass::half_t;
|
||||
using ElementCompute = cutlass::half_t;
|
||||
int const kElementsPerAccess = 128 / cutlass::sizeof_bits<ElementOutput>::value;
|
||||
int const kPartitionsK = 1;
|
||||
|
||||
using Shape = cutlass::gemm::GemmShape<64, 64, 16>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<64, 64, 16>;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<16, 16, 16>;
|
||||
using ElementA = cutlass::half_t;
|
||||
using ElementB = cutlass::half_t;
|
||||
using ElementC = ElementAccumulator;
|
||||
using LayoutA = cutlass::layout::RowMajor;
|
||||
using LayoutB = cutlass::layout::ColumnMajor;
|
||||
using LayoutC = cutlass::layout::RowMajor;
|
||||
|
||||
using WarpMmaTensorOp = typename cutlass::gemm::warp::DefaultMmaTensorOpWmma<
|
||||
WarpShape,
|
||||
InstructionShape,
|
||||
ElementA,
|
||||
LayoutA,
|
||||
ElementB,
|
||||
LayoutB,
|
||||
ElementC,
|
||||
LayoutC>::Type;
|
||||
|
||||
//
|
||||
// Output operator
|
||||
//
|
||||
|
||||
using OutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
ElementOutput,
|
||||
kElementsPerAccess,
|
||||
ElementAccumulator,
|
||||
ElementCompute
|
||||
>;
|
||||
|
||||
//
|
||||
// Define the epilogue
|
||||
//
|
||||
|
||||
using Epilogue = typename cutlass::epilogue::threadblock::DefaultEpilogueWmmaTensorOp<
|
||||
Shape,
|
||||
WarpMmaTensorOp,
|
||||
kPartitionsK,
|
||||
OutputOp,
|
||||
kElementsPerAccess
|
||||
>::Epilogue;
|
||||
|
||||
//
|
||||
// Instantiate epilogue
|
||||
//
|
||||
|
||||
EpilogueTestbed<Epilogue> testbed;
|
||||
|
||||
bool passed = testbed.run_all();
|
||||
|
||||
EXPECT_TRUE(passed);
|
||||
|
||||
}
|
||||
|
||||
TEST(SM70_Epilogue_threadblock_epilogue, f16_wmma_tensor_op_64x128_64x64x16) {
|
||||
|
||||
//
|
||||
// Define the warp-level matrix multiply
|
||||
//
|
||||
using ElementOutput = cutlass::half_t;
|
||||
using ElementAccumulator = cutlass::half_t;
|
||||
using ElementCompute = cutlass::half_t;
|
||||
int const kElementsPerAccess = 128 / cutlass::sizeof_bits<ElementOutput>::value;
|
||||
int const kPartitionsK = 1;
|
||||
|
||||
using Shape = cutlass::gemm::GemmShape<64, 128, 16>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<64, 64, 16>;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<16, 16, 16>;
|
||||
using ElementA = cutlass::half_t;
|
||||
using ElementB = cutlass::half_t;
|
||||
using ElementC = ElementAccumulator;
|
||||
using LayoutA = cutlass::layout::RowMajor;
|
||||
using LayoutB = cutlass::layout::ColumnMajor;
|
||||
using LayoutC = cutlass::layout::RowMajor;
|
||||
|
||||
using WarpMmaTensorOp = typename cutlass::gemm::warp::DefaultMmaTensorOpWmma<
|
||||
WarpShape,
|
||||
InstructionShape,
|
||||
ElementA,
|
||||
LayoutA,
|
||||
ElementB,
|
||||
LayoutB,
|
||||
ElementC,
|
||||
LayoutC>::Type;
|
||||
|
||||
//
|
||||
// Output operator
|
||||
//
|
||||
|
||||
using OutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
ElementOutput,
|
||||
kElementsPerAccess,
|
||||
ElementAccumulator,
|
||||
ElementCompute
|
||||
>;
|
||||
|
||||
//
|
||||
// Define the epilogue
|
||||
//
|
||||
|
||||
using Epilogue = typename cutlass::epilogue::threadblock::DefaultEpilogueWmmaTensorOp<
|
||||
Shape,
|
||||
WarpMmaTensorOp,
|
||||
kPartitionsK,
|
||||
OutputOp,
|
||||
kElementsPerAccess
|
||||
>::Epilogue;
|
||||
|
||||
//
|
||||
// Instantiate epilogue
|
||||
//
|
||||
|
||||
EpilogueTestbed<Epilogue> testbed;
|
||||
|
||||
bool passed = testbed.run_all();
|
||||
|
||||
EXPECT_TRUE(passed);
|
||||
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
//
|
||||
// F32 acumulation and F32 output
|
||||
//
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
TEST(SM70_Epilogue_threadblock_epilogue, f32_wmma_tensor_op_64x64_64x64x16) {
|
||||
|
||||
//
|
||||
// Define the warp-level matrix multiply
|
||||
//
|
||||
using ElementOutput = float;
|
||||
using ElementAccumulator = float;
|
||||
using ElementCompute = cutlass::half_t;
|
||||
int const kElementsPerAccess = 128 / cutlass::sizeof_bits<ElementOutput>::value;
|
||||
int const kPartitionsK = 1;
|
||||
|
||||
using Shape = cutlass::gemm::GemmShape<64, 64, 16>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<64, 64, 16>;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<16, 16, 16>;
|
||||
using ElementA = cutlass::half_t;
|
||||
using ElementB = cutlass::half_t;
|
||||
using ElementC = ElementAccumulator;
|
||||
using LayoutA = cutlass::layout::RowMajor;
|
||||
using LayoutB = cutlass::layout::ColumnMajor;
|
||||
using LayoutC = cutlass::layout::RowMajor;
|
||||
|
||||
using WarpMmaTensorOp = typename cutlass::gemm::warp::DefaultMmaTensorOpWmma<
|
||||
WarpShape,
|
||||
InstructionShape,
|
||||
ElementA,
|
||||
LayoutA,
|
||||
ElementB,
|
||||
LayoutB,
|
||||
ElementC,
|
||||
LayoutC>::Type;
|
||||
|
||||
//
|
||||
// Output operator
|
||||
//
|
||||
|
||||
using OutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
ElementOutput,
|
||||
kElementsPerAccess,
|
||||
ElementAccumulator,
|
||||
ElementCompute
|
||||
>;
|
||||
|
||||
//
|
||||
// Define the epilogue
|
||||
//
|
||||
|
||||
using Epilogue = typename cutlass::epilogue::threadblock::DefaultEpilogueWmmaTensorOp<
|
||||
Shape,
|
||||
WarpMmaTensorOp,
|
||||
kPartitionsK,
|
||||
OutputOp,
|
||||
kElementsPerAccess
|
||||
>::Epilogue;
|
||||
|
||||
//
|
||||
// Instantiate epilogue
|
||||
//
|
||||
|
||||
EpilogueTestbed<Epilogue> testbed;
|
||||
|
||||
bool passed = testbed.run_all();
|
||||
|
||||
EXPECT_TRUE(passed);
|
||||
|
||||
}
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
|
||||
#endif //CUTLASS_ARCH_WMMA_ENABLED
|
||||
@@ -0,0 +1,545 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2019, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/*! \file
|
||||
\brief Unit tests for thread-level GEMM
|
||||
*/
|
||||
|
||||
#include <fstream>
|
||||
|
||||
#include "../../common/cutlass_unit_test.h"
|
||||
|
||||
#include "cutlass/aligned_buffer.h"
|
||||
#include "cutlass/half.h"
|
||||
#include "cutlass/platform/platform.h"
|
||||
|
||||
#include "cutlass/epilogue/threadblock/predicated_tile_iterator.h"
|
||||
#include "cutlass/epilogue/threadblock/default_thread_map_tensor_op.h"
|
||||
|
||||
#include "cutlass/util/tensor_view_io.h"
|
||||
#include "cutlass/util/host_tensor.h"
|
||||
#include "cutlass/util/reference/host/tensor_fill.h"
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Prototype algorithm for partitioning a 4D space across warps to achieve several performance
|
||||
/// objectives:
|
||||
///
|
||||
/// - coalesced memory accesses in units of 128 Byte lines
|
||||
/// - minimal address arithmetic
|
||||
/// - minimal predicate calculations
|
||||
///
|
||||
struct OutputTileThreadMapExpr {
|
||||
|
||||
struct Shape {
|
||||
int column;
|
||||
int row;
|
||||
int group;
|
||||
int cluster;
|
||||
|
||||
Shape(int col = 1, int r = 1, int g = 1, int c = 1):
|
||||
column(col), row(r), group(g), cluster(c) { }
|
||||
};
|
||||
|
||||
int const kWarpSize = 32;
|
||||
int const kMemoryAccessSize = 128; // size in bytes of the preferred memory access size
|
||||
|
||||
//
|
||||
// Data members
|
||||
//
|
||||
|
||||
Shape shape;
|
||||
Shape count;
|
||||
int threads;
|
||||
int warp_count;
|
||||
int elements_per_access;
|
||||
int element_size;
|
||||
|
||||
Shape iterations;
|
||||
Shape delta;
|
||||
Shape warp_partitions;
|
||||
|
||||
int access_width_in_vectors;
|
||||
int access_rows;
|
||||
|
||||
//
|
||||
// Methods
|
||||
//
|
||||
|
||||
OutputTileThreadMapExpr(
|
||||
Shape shape_,
|
||||
Shape count_,
|
||||
int threads_,
|
||||
int elements_per_access_,
|
||||
int element_size_
|
||||
):
|
||||
shape(shape_),
|
||||
count(count_),
|
||||
threads(threads_),
|
||||
warp_count(threads_ / kWarpSize),
|
||||
elements_per_access(elements_per_access_),
|
||||
element_size(element_size_) {
|
||||
|
||||
int warps_remaining = warp_count;
|
||||
|
||||
// clusters
|
||||
if (shape.cluster > warp_count) {
|
||||
iterations.cluster = shape.cluster / warp_count;
|
||||
delta.cluster = shape.row * count.row * shape.group * count.group * shape.cluster / iterations.cluster;
|
||||
warps_remaining = 1;
|
||||
warp_partitions.cluster = warp_count;
|
||||
}
|
||||
else {
|
||||
iterations.cluster = 1;
|
||||
delta.cluster = 1;
|
||||
warps_remaining = warp_count / shape.cluster;
|
||||
warp_partitions.cluster = warps_remaining;
|
||||
}
|
||||
|
||||
// group size
|
||||
if (shape.group > warps_remaining) {
|
||||
iterations.group = shape.group / warps_remaining;
|
||||
delta.group = shape.row * count.row * shape.group / iterations.group;
|
||||
warps_remaining = 1;
|
||||
warp_partitions.group = warps_remaining;
|
||||
}
|
||||
else {
|
||||
iterations.group = 1;
|
||||
delta.group = 1;
|
||||
warps_remaining = warps_remaining / shape.group;
|
||||
warp_partitions.group = warps_remaining;
|
||||
}
|
||||
|
||||
// Number of rows in a group
|
||||
if (shape.row > warps_remaining) {
|
||||
|
||||
// We must cover this shape within a warp
|
||||
int shape_row = shape.row / warps_remaining;
|
||||
int shape_width_vectors = shape.column / elements_per_access;
|
||||
|
||||
// We would still like to minimize the number of strided increments. We can accomplish this
|
||||
// by arranging the memory instructions as 2D, 128B wide accesses.
|
||||
|
||||
int target_memory_access_width = kMemoryAccessSize / (elements_per_access * element_size / 8);
|
||||
int target_rows_per_access = kWarpSize / target_memory_access_width;
|
||||
|
||||
if (target_rows_per_access > shape_row) {
|
||||
access_rows = shape_row;
|
||||
access_width_in_vectors = kWarpSize / access_rows;
|
||||
}
|
||||
else {
|
||||
|
||||
access_width_in_vectors = cutlass::platform::min(
|
||||
shape_width_vectors,
|
||||
cutlass::platform::min(kWarpSize, kMemoryAccessSize / (elements_per_access * element_size / 8)));
|
||||
|
||||
access_rows = cutlass::platform::min(shape_row, kWarpSize / access_width_in_vectors);
|
||||
}
|
||||
|
||||
iterations.row = shape_row / access_rows;
|
||||
delta.row = access_rows;
|
||||
|
||||
iterations.column = shape_width_vectors / access_width_in_vectors;
|
||||
delta.column = access_width_in_vectors * elements_per_access;
|
||||
|
||||
warp_partitions.column = 1;
|
||||
warp_partitions.row = 1;
|
||||
}
|
||||
else {
|
||||
iterations.row = 1;
|
||||
delta.row = 1;
|
||||
iterations.column = (shape.column / elements_per_access) / kWarpSize;
|
||||
delta.column = kWarpSize * elements_per_access;
|
||||
|
||||
access_width_in_vectors = kWarpSize;
|
||||
access_rows = 1;
|
||||
|
||||
warp_partitions.row = 1;
|
||||
warp_partitions.column = warps_remaining;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
std::ostream & operator<<(std::ostream &out, OutputTileThreadMapExpr::Shape const &shape) {
|
||||
out << "col: " << shape.column << ", r: " << shape.row << ", g: " << shape.group << ", c: " << shape.cluster;
|
||||
return out;
|
||||
}
|
||||
|
||||
std::ostream & operator<<(std::ostream &out, OutputTileThreadMapExpr const &map) {
|
||||
out
|
||||
<< " shape(" << map.shape << ")\n"
|
||||
<< " count(" << map.count << ")\n"
|
||||
<< " iterations(" << map.iterations << ")\n"
|
||||
<< " delta(" << map.delta << ")\n"
|
||||
<< " warps(" << map.warp_partitions << ")\n"
|
||||
<< " access(width: " << map.access_width_in_vectors
|
||||
<< ", rows: " << map.access_rows
|
||||
<< ") x v" << map.elements_per_access
|
||||
<< ".b" << map.element_size << "\n";
|
||||
|
||||
return out;
|
||||
}
|
||||
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
template <
|
||||
typename Shape,
|
||||
typename Count,
|
||||
int Threads,
|
||||
int ElementsPerAccess,
|
||||
int ElementSize
|
||||
>
|
||||
struct ThreadMapTestbed {
|
||||
ThreadMapTestbed() {
|
||||
OutputTileThreadMapExpr map(
|
||||
{ Shape::kColumn, Shape::kRow, Shape::kGroup, Shape::kCluster },
|
||||
{ Count::kColumn, Count::kRow, Count::kGroup, Count::kCluster },
|
||||
Threads,
|
||||
ElementsPerAccess,
|
||||
ElementSize
|
||||
);
|
||||
|
||||
using ThreadMap = cutlass::epilogue::threadblock::OutputTileOptimalThreadMap<
|
||||
Shape,
|
||||
Count,
|
||||
Threads,
|
||||
ElementsPerAccess,
|
||||
ElementSize
|
||||
>;
|
||||
|
||||
using CompactThreadmap = typename ThreadMap::CompactedThreadMap;
|
||||
|
||||
bool const kVerbose = false;
|
||||
|
||||
if (kVerbose) {
|
||||
|
||||
std::cout << map << std::endl;
|
||||
|
||||
std::cout << "ThreadMap::warps remaining:\n"
|
||||
<< " for groups: " << ThreadMap::Detail::kWarpsRemainingForGroups << "\n"
|
||||
<< " for rows: " << ThreadMap::Detail::kWarpsRemainingForRows << "\n";
|
||||
|
||||
std::cout << "ThreadMap::Access:\n"
|
||||
<< " width: " << ThreadMap::Detail::kAccessWidth << "\n"
|
||||
<< " rows: " << ThreadMap::Detail::kAccessRows << "\n";
|
||||
|
||||
std::cout << "ThreadMap::RowArrangement::Iterations:\n"
|
||||
<< " row: " << int(ThreadMap::Detail::RowArrangement::kIterationsRow) << "\n";
|
||||
}
|
||||
|
||||
EXPECT_EQ(int(ThreadMap::Delta::kCluster), map.delta.cluster);
|
||||
EXPECT_EQ(int(ThreadMap::Delta::kGroup), map.delta.group);
|
||||
EXPECT_EQ(int(ThreadMap::Delta::kRow), map.delta.row);
|
||||
EXPECT_EQ(int(ThreadMap::Delta::kColumn), map.delta.column);
|
||||
|
||||
EXPECT_EQ(int(ThreadMap::Iterations::kCluster), map.iterations.cluster);
|
||||
EXPECT_EQ(int(ThreadMap::Iterations::kGroup), map.iterations.group);
|
||||
EXPECT_EQ(int(ThreadMap::Iterations::kRow), map.iterations.row);
|
||||
EXPECT_EQ(int(ThreadMap::Iterations::kColumn), map.iterations.column);
|
||||
|
||||
if (kVerbose) {
|
||||
std::cout << "Iterations(col: " << ThreadMap::Iterations::kColumn
|
||||
<< ", r: " << ThreadMap::Iterations::kRow
|
||||
<< ", g: " << ThreadMap::Iterations::kGroup
|
||||
<< ", c: " << ThreadMap::Iterations::kCluster << ")\n";
|
||||
|
||||
std::cout << "Delta(col: " << ThreadMap::Delta::kColumn
|
||||
<< ", r: " << ThreadMap::Delta::kRow
|
||||
<< ", g: " << ThreadMap::Delta::kGroup
|
||||
<< ", c: " << ThreadMap::Delta::kCluster << ")\n";
|
||||
|
||||
for (int tid = 0; tid < Threads; ++tid) {
|
||||
auto output_coord = ThreadMap::initial_offset(tid);
|
||||
auto source_coord = CompactThreadmap::initial_offset(tid);
|
||||
|
||||
std::cout << "T" << tid << " - output: " << output_coord << ", source: " << source_coord << "\n";
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(ThreadMap, f16_tensor_op_64x64_64x64x8) {
|
||||
|
||||
using Shape = cutlass::epilogue::threadblock::OutputTileShape<64, 8, 1, 1, 1>;
|
||||
using Count = cutlass::epilogue::threadblock::OutputTileShape<1, 8, 1, 1, 1>;
|
||||
int const kThreads = 32;
|
||||
int const kElementsPerAccess = 8;
|
||||
int const kElementSize = 16;
|
||||
|
||||
ThreadMapTestbed<Shape, Count, kThreads, kElementsPerAccess, kElementSize>();
|
||||
}
|
||||
|
||||
|
||||
TEST(ThreadMap, f16_tensor_op_128x128_64x64x8) {
|
||||
|
||||
using Shape = cutlass::epilogue::threadblock::OutputTileShape<128, 8, 2, 1, 1>;
|
||||
using Count = cutlass::epilogue::threadblock::OutputTileShape<1, 8, 1, 1, 1>;
|
||||
int const kThreads = 128;
|
||||
int const kElementsPerAccess = 8;
|
||||
int const kElementSize = 16;
|
||||
|
||||
ThreadMapTestbed<Shape, Count, kThreads, kElementsPerAccess, kElementSize>();
|
||||
}
|
||||
|
||||
TEST(ThreadMap, f16_tensor_op_256x128_64x64x8) {
|
||||
|
||||
using Shape = cutlass::epilogue::threadblock::OutputTileShape<128, 8, 4, 1, 1>;
|
||||
using Count = cutlass::epilogue::threadblock::OutputTileShape<1, 8, 2, 1, 1>;
|
||||
int const kThreads = 256;
|
||||
int const kElementsPerAccess = 8;
|
||||
int const kElementSize = 16;
|
||||
|
||||
ThreadMapTestbed<Shape, Count, kThreads, kElementsPerAccess, kElementSize>();
|
||||
}
|
||||
|
||||
TEST(ThreadMap, f16_tensor_op_128x256_64x64x8) {
|
||||
|
||||
using Shape = cutlass::epilogue::threadblock::OutputTileShape<256, 8, 2, 1, 1>;
|
||||
using Count = cutlass::epilogue::threadblock::OutputTileShape<1, 8, 2, 1, 1>;
|
||||
int const kThreads = 256;
|
||||
int const kElementsPerAccess = 8;
|
||||
int const kElementSize = 16;
|
||||
|
||||
ThreadMapTestbed<Shape, Count, kThreads, kElementsPerAccess, kElementSize>();
|
||||
}
|
||||
|
||||
TEST(ThreadMap, f16_tensor_op_128x64_64x32x8) {
|
||||
|
||||
using Shape = cutlass::epilogue::threadblock::OutputTileShape<64, 8, 2, 1, 1>;
|
||||
using Count = cutlass::epilogue::threadblock::OutputTileShape<1, 8, 2, 1, 1>;
|
||||
int const kThreads = 128;
|
||||
int const kElementsPerAccess = 8;
|
||||
int const kElementSize = 16;
|
||||
|
||||
ThreadMapTestbed<Shape, Count, kThreads, kElementsPerAccess, kElementSize>();
|
||||
}
|
||||
|
||||
TEST(ThreadMap, f16_tensor_op_64x128_128x64x8) {
|
||||
|
||||
using Shape = cutlass::epilogue::threadblock::OutputTileShape<128, 8, 1, 1, 1>;
|
||||
using Count = cutlass::epilogue::threadblock::OutputTileShape<1, 8, 2, 1, 1>;
|
||||
int const kThreads = 128;
|
||||
int const kElementsPerAccess = 8;
|
||||
int const kElementSize = 16;
|
||||
|
||||
ThreadMapTestbed<Shape, Count, kThreads, kElementsPerAccess, kElementSize>();
|
||||
}
|
||||
|
||||
TEST(ThreadMap, f32_tensor_op_64x64_64x64x8) {
|
||||
|
||||
using Shape = cutlass::epilogue::threadblock::OutputTileShape<64, 8, 1, 1, 1>;
|
||||
using Count = cutlass::epilogue::threadblock::OutputTileShape<1, 8, 1, 1, 1>;
|
||||
int const kThreads = 32;
|
||||
int const kElementsPerAccess = 4;
|
||||
int const kElementSize = 32;
|
||||
|
||||
ThreadMapTestbed<Shape, Count, kThreads, kElementsPerAccess, kElementSize>();
|
||||
}
|
||||
|
||||
TEST(ThreadMap, f32_tensor_op_128x128_64x64x8) {
|
||||
|
||||
using Shape = cutlass::epilogue::threadblock::OutputTileShape<128, 8, 2, 1, 1>;
|
||||
using Count = cutlass::epilogue::threadblock::OutputTileShape<1, 8, 2, 1, 1>;
|
||||
int const kThreads = 128;
|
||||
int const kElementsPerAccess = 4;
|
||||
int const kElementSize = 32;
|
||||
|
||||
ThreadMapTestbed<Shape, Count, kThreads, kElementsPerAccess, kElementSize>();
|
||||
}
|
||||
|
||||
TEST(ThreadMap, f32_tensor_op_256x128_64x64x8) {
|
||||
|
||||
using Shape = cutlass::epilogue::threadblock::OutputTileShape<128, 8, 4, 1, 1>;
|
||||
using Count = cutlass::epilogue::threadblock::OutputTileShape<1, 8, 2, 1, 1>;
|
||||
int const kThreads = 256;
|
||||
int const kElementsPerAccess = 4;
|
||||
int const kElementSize = 32;
|
||||
|
||||
ThreadMapTestbed<Shape, Count, kThreads, kElementsPerAccess, kElementSize>();
|
||||
}
|
||||
|
||||
TEST(ThreadMap, f32_tensor_op_128x256_64x64x8) {
|
||||
|
||||
using Shape = cutlass::epilogue::threadblock::OutputTileShape<256, 8, 2, 1, 1>;
|
||||
using Count = cutlass::epilogue::threadblock::OutputTileShape<1, 8, 2, 1, 1>;
|
||||
int const kThreads = 256;
|
||||
int const kElementsPerAccess = 4;
|
||||
int const kElementSize = 32;
|
||||
|
||||
ThreadMapTestbed<Shape, Count, kThreads, kElementsPerAccess, kElementSize>();
|
||||
}
|
||||
|
||||
TEST(ThreadMap, f32_tensor_op_128x64_64x32x8) {
|
||||
|
||||
using Shape = cutlass::epilogue::threadblock::OutputTileShape<64, 8, 2, 1, 1>;
|
||||
using Count = cutlass::epilogue::threadblock::OutputTileShape<1, 8, 2, 1, 1>;
|
||||
int const kThreads = 128;
|
||||
int const kElementsPerAccess = 4;
|
||||
int const kElementSize = 32;
|
||||
|
||||
ThreadMapTestbed<Shape, Count, kThreads, kElementsPerAccess, kElementSize>();
|
||||
}
|
||||
|
||||
TEST(ThreadMap, f32_tensor_op_64x128_128x64x8) {
|
||||
|
||||
using Shape = cutlass::epilogue::threadblock::OutputTileShape<128, 8, 1, 1, 1>;
|
||||
using Count = cutlass::epilogue::threadblock::OutputTileShape<1, 8, 2, 1, 1>;
|
||||
int const kThreads = 128;
|
||||
int const kElementsPerAccess = 4;
|
||||
int const kElementSize = 32;
|
||||
|
||||
ThreadMapTestbed<Shape, Count, kThreads, kElementsPerAccess, kElementSize>();
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(ThreadMap, f32_volta_tensor_op_64x64_64x64x8) {
|
||||
|
||||
using Shape = cutlass::epilogue::threadblock::OutputTileShape<64, 2, 4, 1, 1>;
|
||||
using Count = cutlass::epilogue::threadblock::OutputTileShape<1, 4, 2, 1, 1>;
|
||||
int const kThreads = 32;
|
||||
int const kElementsPerAccess = 4;
|
||||
int const kElementSize = 32;
|
||||
|
||||
ThreadMapTestbed<Shape, Count, kThreads, kElementsPerAccess, kElementSize>();
|
||||
}
|
||||
|
||||
TEST(ThreadMap, f32_volta_tensor_op_64x128_64x64x8) {
|
||||
|
||||
using Shape = cutlass::epilogue::threadblock::OutputTileShape<128, 2, 4, 1, 1>;
|
||||
using Count = cutlass::epilogue::threadblock::OutputTileShape<1, 4, 2, 1, 1>;
|
||||
int const kThreads = 64;
|
||||
int const kElementsPerAccess = 4;
|
||||
int const kElementSize = 32;
|
||||
|
||||
ThreadMapTestbed<Shape, Count, kThreads, kElementsPerAccess, kElementSize>();
|
||||
}
|
||||
|
||||
TEST(ThreadMap, f32_volta_tensor_op_128x64_64x64x8) {
|
||||
|
||||
using Shape = cutlass::epilogue::threadblock::OutputTileShape<64, 2, 4, 2, 1>;
|
||||
using Count = cutlass::epilogue::threadblock::OutputTileShape<1, 4, 2, 1, 1>;
|
||||
int const kThreads = 64;
|
||||
int const kElementsPerAccess = 4;
|
||||
int const kElementSize = 32;
|
||||
|
||||
ThreadMapTestbed<Shape, Count, kThreads, kElementsPerAccess, kElementSize>();
|
||||
}
|
||||
|
||||
TEST(ThreadMap, f32_volta_tensor_op_128x64_64x32x8) {
|
||||
|
||||
using Shape = cutlass::epilogue::threadblock::OutputTileShape<64, 2, 4, 2, 1>;
|
||||
using Count = cutlass::epilogue::threadblock::OutputTileShape<1, 4, 2, 1, 1>;
|
||||
int const kThreads = 128;
|
||||
int const kElementsPerAccess = 4;
|
||||
int const kElementSize = 32;
|
||||
|
||||
ThreadMapTestbed<Shape, Count, kThreads, kElementsPerAccess, kElementSize>();
|
||||
}
|
||||
|
||||
TEST(ThreadMap, f32_volta_tensor_op_128x128_64x64x8) {
|
||||
|
||||
using Shape = cutlass::epilogue::threadblock::OutputTileShape<128, 2, 4, 2, 1>;
|
||||
using Count = cutlass::epilogue::threadblock::OutputTileShape<1, 4, 2, 1, 1>;
|
||||
int const kThreads = 128;
|
||||
int const kElementsPerAccess = 4;
|
||||
int const kElementSize = 32;
|
||||
|
||||
ThreadMapTestbed<Shape, Count, kThreads, kElementsPerAccess, kElementSize>();
|
||||
}
|
||||
|
||||
TEST(ThreadMap, f32_volta_tensor_op_128x256_64x64x8) {
|
||||
|
||||
using Shape = cutlass::epilogue::threadblock::OutputTileShape<256, 2, 4, 2, 1>;
|
||||
using Count = cutlass::epilogue::threadblock::OutputTileShape<1, 4, 2, 1, 1>;
|
||||
int const kThreads = 256;
|
||||
int const kElementsPerAccess = 4;
|
||||
int const kElementSize = 32;
|
||||
|
||||
ThreadMapTestbed<Shape, Count, kThreads, kElementsPerAccess, kElementSize>();
|
||||
}
|
||||
|
||||
TEST(ThreadMap, f32_volta_tensor_op_256x128_64x64x8) {
|
||||
|
||||
using Shape = cutlass::epilogue::threadblock::OutputTileShape<128, 2, 4, 4, 1>;
|
||||
using Count = cutlass::epilogue::threadblock::OutputTileShape<1, 4, 2, 1, 1>;
|
||||
int const kThreads = 256;
|
||||
int const kElementsPerAccess = 4;
|
||||
int const kElementSize = 32;
|
||||
|
||||
ThreadMapTestbed<Shape, Count, kThreads, kElementsPerAccess, kElementSize>();
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(ThreadMap, simt_32x64_32x64x1) {
|
||||
|
||||
using Shape = cutlass::epilogue::threadblock::OutputTileShape<64, 1, 4, 1, 1>;
|
||||
using Count = cutlass::epilogue::threadblock::OutputTileShape<1, 4, 2, 1, 1>;
|
||||
int const kThreads = 32;
|
||||
int const kElementsPerAccess = 1;
|
||||
int const kElementSize = 32;
|
||||
|
||||
ThreadMapTestbed<Shape, Count, kThreads, kElementsPerAccess, kElementSize>();
|
||||
}
|
||||
|
||||
TEST(ThreadMap, simt_32x128_32x64x1) {
|
||||
|
||||
using Shape = cutlass::epilogue::threadblock::OutputTileShape<128, 1, 4, 1, 1>;
|
||||
using Count = cutlass::epilogue::threadblock::OutputTileShape<1, 4, 2, 1, 1>;
|
||||
int const kThreads = 64;
|
||||
int const kElementsPerAccess = 1;
|
||||
int const kElementSize = 32;
|
||||
|
||||
ThreadMapTestbed<Shape, Count, kThreads, kElementsPerAccess, kElementSize>();
|
||||
}
|
||||
|
||||
TEST(ThreadMap, simt_64x128_32x64x1) {
|
||||
|
||||
using Shape = cutlass::epilogue::threadblock::OutputTileShape<128, 1, 4, 2, 1>;
|
||||
using Count = cutlass::epilogue::threadblock::OutputTileShape<1, 4, 2, 1, 1>;
|
||||
int const kThreads = 128;
|
||||
int const kElementsPerAccess = 1;
|
||||
int const kElementSize = 32;
|
||||
|
||||
ThreadMapTestbed<Shape, Count, kThreads, kElementsPerAccess, kElementSize>();
|
||||
}
|
||||
|
||||
TEST(ThreadMap, simt_128x128_32x64x1) {
|
||||
|
||||
using Shape = cutlass::epilogue::threadblock::OutputTileShape<128, 1, 4, 4, 1>;
|
||||
using Count = cutlass::epilogue::threadblock::OutputTileShape<1, 4, 2, 1, 1>;
|
||||
int const kThreads = 256;
|
||||
int const kElementsPerAccess = 1;
|
||||
int const kElementSize = 32;
|
||||
|
||||
ThreadMapTestbed<Shape, Count, kThreads, kElementsPerAccess, kElementSize>();
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,352 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2019, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/*! \file
|
||||
\brief Unit tests for epilogues
|
||||
*/
|
||||
#pragma once
|
||||
|
||||
#include <fstream>
|
||||
|
||||
#include "../../common/cutlass_unit_test.h"
|
||||
|
||||
#include "cutlass/aligned_buffer.h"
|
||||
#include "cutlass/half.h"
|
||||
#include "cutlass/complex.h"
|
||||
|
||||
#include "cutlass/epilogue/thread/linear_combination.h"
|
||||
|
||||
#include "cutlass/util/host_tensor.h"
|
||||
#include "cutlass/util/tensor_view_io.h"
|
||||
#include "cutlass/util/reference/host/tensor_fill.h"
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
namespace test {
|
||||
namespace kernel {
|
||||
|
||||
template <typename Epilogue>
|
||||
__global__ void epilogue_threadblock(
|
||||
typename Epilogue::OutputTileIterator::Params params_D,
|
||||
typename Epilogue::OutputTileIterator::Element *ptr_D,
|
||||
typename Epilogue::OutputTileIterator::Params params_C,
|
||||
typename Epilogue::OutputTileIterator::Element *ptr_C,
|
||||
typename Epilogue::OutputOp::Params params_output_op,
|
||||
cutlass::MatrixCoord problem_size,
|
||||
cutlass::TensorRef<
|
||||
typename Epilogue::WarpMmaOperator::ElementC,
|
||||
typename Epilogue::WarpMmaOperator::LayoutC> accumulator_ref,
|
||||
int epilogue_count = 1) {
|
||||
|
||||
__shared__ typename Epilogue::SharedStorage shared_storage;
|
||||
|
||||
int thread_idx = threadIdx.x;
|
||||
int warp_idx = threadIdx.x / 32;
|
||||
int lane_idx = threadIdx.x % 32;
|
||||
|
||||
//
|
||||
// Construct the epilogue
|
||||
//
|
||||
|
||||
// Tile iterator writing to output tile
|
||||
typename Epilogue::OutputTileIterator iterator_D(
|
||||
params_D,
|
||||
ptr_D,
|
||||
problem_size,
|
||||
thread_idx
|
||||
);
|
||||
|
||||
// Tile iterator writing to output tile
|
||||
typename Epilogue::OutputTileIterator iterator_C(
|
||||
params_C,
|
||||
ptr_C,
|
||||
problem_size,
|
||||
thread_idx
|
||||
);
|
||||
|
||||
// Epilogue operator
|
||||
Epilogue epilogue(
|
||||
shared_storage,
|
||||
thread_idx,
|
||||
warp_idx,
|
||||
lane_idx);
|
||||
|
||||
//
|
||||
// Initialize the accumulators
|
||||
//
|
||||
|
||||
int warp_mn = warp_idx % (Epilogue::WarpCount::kM * Epilogue::WarpCount::kN);
|
||||
int warp_m = warp_mn % Epilogue::WarpCount::kM;
|
||||
int warp_n = warp_mn / Epilogue::WarpCount::kM;
|
||||
|
||||
accumulator_ref.add_coord_offset({
|
||||
warp_m * Epilogue::WarpMmaOperator::Shape::kM,
|
||||
warp_n * Epilogue::WarpMmaOperator::Shape::kN});
|
||||
|
||||
typename Epilogue::WarpMmaOperator::IteratorC accumulator_iterator(accumulator_ref, lane_idx);
|
||||
|
||||
typename Epilogue::AccumulatorTile accumulators;
|
||||
|
||||
accumulators.clear();
|
||||
accumulator_iterator.load(accumulators);
|
||||
|
||||
#if 0
|
||||
// For debugging, enable this block of code to fill each accumulator element with its
|
||||
// source thread ID.
|
||||
CUTLASS_PRAGMA_UNROLL
|
||||
for (int i = 0; i < accumulators.size(); ++i) {
|
||||
typename Epilogue::WarpMmaOperator::ElementC x(threadIdx.x);
|
||||
//typename Epilogue::WarpMmaOperator::ElementC x(i);
|
||||
accumulators[i] = x;
|
||||
}
|
||||
|
||||
/*
|
||||
#pragma unroll 1
|
||||
for (int tid = 0; tid < 32; ++tid) {
|
||||
if (tid == thread_idx) {
|
||||
printf("\nT%d: ", thread_idx);
|
||||
CUTLASS_PRAGMA_UNROLL
|
||||
for (int i = 0; i < accumulators.size(); ++i) {
|
||||
printf("%d ", int(accumulators[i]));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (thread_idx == 0) {
|
||||
printf("\n\n");
|
||||
}
|
||||
*/
|
||||
|
||||
__syncthreads();
|
||||
|
||||
#endif
|
||||
|
||||
//
|
||||
// Perform the epilogue operation
|
||||
//
|
||||
|
||||
typename Epilogue::OutputOp output_op(params_output_op);
|
||||
|
||||
// Place the epilogue in a loop
|
||||
for (int iter = 0; iter < epilogue_count; ++iter) {
|
||||
epilogue(output_op, iterator_D, accumulators, iterator_C);
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace kernel
|
||||
} // namespace test
|
||||
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
template <
|
||||
typename Epilogue_
|
||||
>
|
||||
class EpilogueTestbed {
|
||||
public:
|
||||
|
||||
using Epilogue = Epilogue_;
|
||||
using ElementAccumulator = typename Epilogue::ElementAccumulator;
|
||||
using ElementCompute = typename Epilogue::OutputOp::ElementCompute;
|
||||
using ElementOutput = typename Epilogue::ElementOutput;
|
||||
using OutputOpParams = typename Epilogue::OutputOp::Params;
|
||||
|
||||
public:
|
||||
|
||||
//
|
||||
// Data members
|
||||
//
|
||||
|
||||
cutlass::MatrixCoord quantized_size;
|
||||
cutlass::HostTensor<ElementAccumulator, cutlass::layout::RowMajor> accumulator_tensor;
|
||||
cutlass::HostTensor<ElementOutput, cutlass::layout::RowMajor> source_tensor;
|
||||
cutlass::HostTensor<ElementOutput, cutlass::layout::RowMajor> output_tensor;
|
||||
|
||||
public:
|
||||
|
||||
//
|
||||
// Methods
|
||||
//
|
||||
|
||||
EpilogueTestbed():
|
||||
quantized_size(Epilogue::Shape::kM, Epilogue::Shape::kN),
|
||||
accumulator_tensor({Epilogue::Shape::kM, Epilogue::Shape::kN}),
|
||||
source_tensor({Epilogue::Shape::kM, Epilogue::Shape::kN}),
|
||||
output_tensor({Epilogue::Shape::kM, Epilogue::Shape::kN}) {
|
||||
|
||||
//
|
||||
// Initialize problem space
|
||||
//
|
||||
|
||||
uint64_t seed = 2019;
|
||||
|
||||
cutlass::reference::host::TensorFillRandomUniform(
|
||||
accumulator_tensor.host_view(),
|
||||
seed,
|
||||
20,
|
||||
-20,
|
||||
0);
|
||||
|
||||
cutlass::reference::host::TensorFillRandomUniform(
|
||||
source_tensor.host_view(),
|
||||
seed + 2018,
|
||||
20,
|
||||
-20,
|
||||
0);
|
||||
}
|
||||
|
||||
bool run_all() {
|
||||
|
||||
double alpha_values[] = {1, 0, 2.25};
|
||||
double beta_values[] = {0, 1, -1.25};
|
||||
|
||||
// Test runtime explodes if we tried to test every case exhaustively. This tests the full
|
||||
// output tile and several smaller sizes to stress predication.
|
||||
for (int m_idx = 0; m_idx < 3; ++m_idx) {
|
||||
for (int n_idx = 0; n_idx < 3; ++n_idx) {
|
||||
|
||||
int m = quantized_size.row() - m_idx * 3;
|
||||
int n = quantized_size.column() - n_idx * Epilogue::kElementsPerAccess;
|
||||
|
||||
for (double const &alpha : alpha_values) {
|
||||
for (double const &beta : beta_values) {
|
||||
|
||||
bool passed = run({m, n}, {cutlass::from_real<ElementCompute>(alpha), cutlass::from_real<ElementCompute>(beta)});
|
||||
|
||||
if (!passed) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/// Runs the test
|
||||
bool run(
|
||||
cutlass::MatrixCoord problem_size,
|
||||
OutputOpParams output_params) {
|
||||
|
||||
//
|
||||
// Initialize problem space
|
||||
//
|
||||
|
||||
ElementOutput default_output = ElementOutput(-127);
|
||||
cutlass::reference::host::TensorFill(output_tensor.host_view(), default_output);
|
||||
|
||||
accumulator_tensor.sync_device();
|
||||
output_tensor.sync_device();
|
||||
source_tensor.sync_device();
|
||||
|
||||
//
|
||||
// Initialize epilogue parameters
|
||||
//
|
||||
|
||||
typename Epilogue::OutputTileIterator::Params params_D(output_tensor.device_ref().layout());
|
||||
typename Epilogue::OutputTileIterator::Params params_C(source_tensor.device_ref().layout());
|
||||
|
||||
//
|
||||
// Launch kernel
|
||||
//
|
||||
|
||||
dim3 grid(1, 1);
|
||||
dim3 block(Epilogue::WarpCount::kCount * 32, 1);
|
||||
|
||||
test::kernel::epilogue_threadblock<Epilogue><<< grid, block >>>(
|
||||
params_D,
|
||||
output_tensor.device_data(),
|
||||
params_C,
|
||||
source_tensor.device_data(),
|
||||
output_params,
|
||||
problem_size,
|
||||
accumulator_tensor.device_view());
|
||||
|
||||
cudaError_t result = cudaDeviceSynchronize();
|
||||
|
||||
if (result != cudaSuccess) {
|
||||
std::cerr << "Kernel error: " << cudaGetErrorString(result) << std::endl;
|
||||
return false;
|
||||
}
|
||||
|
||||
//
|
||||
// Verify results
|
||||
//
|
||||
output_tensor.sync_host();
|
||||
|
||||
int errors = 0;
|
||||
int const kMaxErrors = 5;
|
||||
|
||||
for (int r = 0; errors < kMaxErrors && r < quantized_size.row(); ++r) {
|
||||
for (int c = 0; errors < kMaxErrors && c < quantized_size.column(); ++c) {
|
||||
|
||||
cutlass::MatrixCoord coord{r, c};
|
||||
ElementOutput got = output_tensor.at(coord);
|
||||
|
||||
ElementOutput expected;
|
||||
if (coord.row() < problem_size.row() && coord.column() < problem_size.column()) {
|
||||
expected = ElementOutput(output_params.alpha * ElementCompute(accumulator_tensor.at(coord)) +
|
||||
output_params.beta * ElementCompute(source_tensor.at(coord)));
|
||||
}
|
||||
else {
|
||||
expected = default_output;
|
||||
}
|
||||
|
||||
if (expected != got) {
|
||||
|
||||
using OutputIO = cutlass::ScalarIO<ElementOutput>;
|
||||
|
||||
EXPECT_TRUE(false)
|
||||
<< "-------\n"
|
||||
<< "Error - output element (" << coord << ") - expected: "
|
||||
<< OutputIO(expected)
|
||||
<< ", got: " << OutputIO(got) << std::endl;
|
||||
|
||||
++errors;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
//
|
||||
// Report results on error
|
||||
//
|
||||
|
||||
if (errors) {
|
||||
std::stringstream ss;
|
||||
ss
|
||||
<< "output_tensor_op_" << Epilogue::Shape::kM << "x" << Epilogue::Shape::kN << "_"
|
||||
<< Epilogue::WarpTileIterator::WarpShape::kM << "x"
|
||||
<< Epilogue::WarpTileIterator::WarpShape::kN
|
||||
<< "_slice_" << Epilogue::WarpCount::kK << ".csv";
|
||||
|
||||
std::ofstream output_file(ss.str());
|
||||
output_file << output_tensor.host_view();
|
||||
}
|
||||
|
||||
return !errors;
|
||||
}
|
||||
};
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
Reference in New Issue
Block a user