CUTLASS 2.1 (#83)
CUTLASS 2.1 contributes: - BLAS-style host-side API added to CUTLASS Library - Planar Complex GEMM kernels targeting Volta and Turing Tensor Cores - Minor enhancements and bug fixes
This commit is contained in:
Andrew Kerr
Regular → Executable
+2
-1
@@ -30,4 +30,5 @@ cutlass_test_unit_add_executable(
|
||||
epilogue_tensor_op.cu
|
||||
epilogue_volta_tensor_op.cu
|
||||
epilogue_wmma_tensor_op_sm70.cu
|
||||
)
|
||||
epilogue_planar_complex.cu
|
||||
)
|
||||
|
||||
@@ -0,0 +1,506 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2019, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/*! \file
|
||||
\brief Unit tests for thread-level GEMM
|
||||
*/
|
||||
|
||||
#include <fstream>
|
||||
|
||||
#include "../../common/cutlass_unit_test.h"
|
||||
|
||||
#include "cutlass/aligned_buffer.h"
|
||||
#include "cutlass/half.h"
|
||||
|
||||
#include "cutlass/epilogue/thread/linear_combination_planar_complex.h"
|
||||
|
||||
// Tensor Op
|
||||
#include "cutlass/gemm/warp/default_mma_tensor_op.h"
|
||||
|
||||
// Volta Tensor Op
|
||||
#include "cutlass/gemm/warp/mma_tensor_op_sm70.h"
|
||||
#include "cutlass/epilogue/warp/fragment_iterator_volta_tensor_op.h"
|
||||
|
||||
// Simt
|
||||
#include "cutlass/gemm/warp/mma_simt.h"
|
||||
#include "cutlass/gemm/warp/mma_simt_policy.h"
|
||||
|
||||
// Epilogue components
|
||||
|
||||
#include "cutlass/epilogue/threadblock/default_epilogue_planar_complex.h"
|
||||
|
||||
#include "cutlass/util/host_tensor.h"
|
||||
#include "cutlass/util/tensor_view_io.h"
|
||||
#include "cutlass/util/reference/host/tensor_fill.h"
|
||||
|
||||
#include "testbed_planar_complex.h"
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Epilogue_threadblock_epilogue, planar_complex_f32_f32_tensor_op_64x64_32x32x8) {
|
||||
|
||||
//
|
||||
// Define the warp-level matrix multiply
|
||||
//
|
||||
|
||||
using ElementOutput = float;
|
||||
using ElementAccumulator = float;
|
||||
using ElementCompute = float;
|
||||
int const kElementsPerAccess = 128 / cutlass::sizeof_bits<ElementOutput>::value;
|
||||
int const kPartitionsK = 1;
|
||||
|
||||
using Shape = cutlass::gemm::GemmShape<64, 64, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<32, 32, 8>;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<16, 8, 8>;
|
||||
using Element = cutlass::half_t;
|
||||
|
||||
using LayoutA = cutlass::layout::RowMajorTensorOpMultiplicandCrosswise<
|
||||
cutlass::sizeof_bits<Element>::value, 64>;
|
||||
|
||||
using LayoutB = cutlass::layout::ColumnMajorTensorOpMultiplicandCrosswise<
|
||||
cutlass::sizeof_bits<Element>::value, 64>;
|
||||
|
||||
using WarpMmaTensorOp = typename cutlass::gemm::warp::DefaultMmaTensorOp<
|
||||
WarpShape,
|
||||
InstructionShape,
|
||||
Element, LayoutA,
|
||||
Element, LayoutB,
|
||||
ElementAccumulator, cutlass::layout::RowMajor
|
||||
>::Type;
|
||||
|
||||
//
|
||||
// Output operator
|
||||
//
|
||||
|
||||
using OutputOp = cutlass::epilogue::thread::LinearCombinationPlanarComplex<
|
||||
ElementOutput,
|
||||
kElementsPerAccess,
|
||||
ElementAccumulator,
|
||||
ElementCompute
|
||||
>;
|
||||
|
||||
//
|
||||
// Define the epilogue
|
||||
//
|
||||
|
||||
using Epilogue = typename cutlass::epilogue::threadblock::DefaultEpiloguePlanarComplex<
|
||||
Shape,
|
||||
WarpMmaTensorOp,
|
||||
cutlass::arch::OpClassTensorOp,
|
||||
cutlass::arch::Sm75,
|
||||
kPartitionsK,
|
||||
OutputOp,
|
||||
kElementsPerAccess
|
||||
>::Epilogue;
|
||||
|
||||
//
|
||||
// Instantiate epilogue
|
||||
//
|
||||
|
||||
EpiloguePlanarComplexTestbed<Epilogue> testbed;
|
||||
|
||||
bool passed = testbed.run_all();
|
||||
|
||||
EXPECT_TRUE(passed);
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Epilogue_threadblock_epilogue, planar_complex_f16_f32_tensor_op_64x64_32x32x8) {
|
||||
|
||||
//
|
||||
// Define the warp-level matrix multiply
|
||||
//
|
||||
|
||||
using ElementOutput = cutlass::half_t;
|
||||
using ElementAccumulator = float;
|
||||
using ElementCompute = float;
|
||||
int const kElementsPerAccess = 128 / cutlass::sizeof_bits<ElementOutput>::value;
|
||||
int const kPartitionsK = 1;
|
||||
|
||||
using Shape = cutlass::gemm::GemmShape<64, 64, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<32, 32, 8>;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<16, 8, 8>;
|
||||
using Element = cutlass::half_t;
|
||||
|
||||
using LayoutA = cutlass::layout::RowMajorTensorOpMultiplicandCrosswise<
|
||||
cutlass::sizeof_bits<Element>::value, 64>;
|
||||
|
||||
using LayoutB = cutlass::layout::ColumnMajorTensorOpMultiplicandCrosswise<
|
||||
cutlass::sizeof_bits<Element>::value, 64>;
|
||||
|
||||
using WarpMmaTensorOp = typename cutlass::gemm::warp::DefaultMmaTensorOp<
|
||||
WarpShape,
|
||||
InstructionShape,
|
||||
Element, LayoutA,
|
||||
Element, LayoutB,
|
||||
ElementAccumulator, cutlass::layout::RowMajor
|
||||
>::Type;
|
||||
|
||||
//
|
||||
// Output operator
|
||||
//
|
||||
|
||||
using OutputOp = cutlass::epilogue::thread::LinearCombinationPlanarComplex<
|
||||
ElementOutput,
|
||||
kElementsPerAccess,
|
||||
ElementAccumulator,
|
||||
ElementCompute
|
||||
>;
|
||||
|
||||
//
|
||||
// Define the epilogue
|
||||
//
|
||||
|
||||
using Epilogue = typename cutlass::epilogue::threadblock::DefaultEpiloguePlanarComplex<
|
||||
Shape,
|
||||
WarpMmaTensorOp,
|
||||
cutlass::arch::OpClassTensorOp,
|
||||
cutlass::arch::Sm75,
|
||||
kPartitionsK,
|
||||
OutputOp,
|
||||
kElementsPerAccess
|
||||
>::Epilogue;
|
||||
|
||||
//
|
||||
// Instantiate epilogue
|
||||
//
|
||||
|
||||
EpiloguePlanarComplexTestbed<Epilogue> testbed;
|
||||
|
||||
bool passed = testbed.run_all();
|
||||
|
||||
EXPECT_TRUE(passed);
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Epilogue_threadblock_epilogue, planar_complex_f16_f16_tensor_op_64x64_32x32x8) {
|
||||
|
||||
//
|
||||
// Define the warp-level matrix multiply
|
||||
//
|
||||
|
||||
using ElementOutput = cutlass::half_t;
|
||||
using ElementAccumulator = cutlass::half_t;
|
||||
using ElementCompute = cutlass::half_t;
|
||||
int const kElementsPerAccess = 128 / cutlass::sizeof_bits<ElementOutput>::value;
|
||||
int const kPartitionsK = 1;
|
||||
|
||||
using Shape = cutlass::gemm::GemmShape<64, 64, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<32, 32, 8>;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<16, 8, 8>;
|
||||
using Element = cutlass::half_t;
|
||||
|
||||
using LayoutA = cutlass::layout::RowMajorTensorOpMultiplicandCrosswise<
|
||||
cutlass::sizeof_bits<Element>::value, 64>;
|
||||
|
||||
using LayoutB = cutlass::layout::ColumnMajorTensorOpMultiplicandCrosswise<
|
||||
cutlass::sizeof_bits<Element>::value, 64>;
|
||||
|
||||
using WarpMmaTensorOp = typename cutlass::gemm::warp::DefaultMmaTensorOp<
|
||||
WarpShape,
|
||||
InstructionShape,
|
||||
Element, LayoutA,
|
||||
Element, LayoutB,
|
||||
ElementAccumulator, cutlass::layout::RowMajor
|
||||
>::Type;
|
||||
|
||||
//
|
||||
// Output operator
|
||||
//
|
||||
|
||||
using OutputOp = cutlass::epilogue::thread::LinearCombinationPlanarComplex<
|
||||
ElementOutput,
|
||||
kElementsPerAccess,
|
||||
ElementAccumulator,
|
||||
ElementCompute
|
||||
>;
|
||||
|
||||
//
|
||||
// Define the epilogue
|
||||
//
|
||||
|
||||
using Epilogue = typename cutlass::epilogue::threadblock::DefaultEpiloguePlanarComplex<
|
||||
Shape,
|
||||
WarpMmaTensorOp,
|
||||
cutlass::arch::OpClassTensorOp,
|
||||
cutlass::arch::Sm75,
|
||||
kPartitionsK,
|
||||
OutputOp,
|
||||
kElementsPerAccess
|
||||
>::Epilogue;
|
||||
|
||||
//
|
||||
// Instantiate epilogue
|
||||
//
|
||||
|
||||
EpiloguePlanarComplexTestbed<Epilogue> testbed;
|
||||
|
||||
bool passed = testbed.run_all();
|
||||
|
||||
EXPECT_TRUE(passed);
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Epilogue_threadblock_epilogue, planar_complex_f32_f32_volta_tensor_op_64x64_32x32x4) {
|
||||
|
||||
//
|
||||
// Define the warp-level matrix multiply
|
||||
//
|
||||
|
||||
using ElementOutput = float;
|
||||
using ElementAccumulator = float;
|
||||
using ElementCompute = float;
|
||||
|
||||
int const kElementsPerAccess = 128 / cutlass::sizeof_bits<ElementOutput>::value;
|
||||
int const kPartitionsK = 1;
|
||||
|
||||
using Shape = cutlass::gemm::GemmShape<32, 32, 4>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<32, 32, 4>;
|
||||
using Element = cutlass::half_t;
|
||||
|
||||
using LayoutA = cutlass::layout::ColumnMajorVoltaTensorOpMultiplicandCongruous<cutlass::sizeof_bits<Element>::value>;
|
||||
using LayoutB = cutlass::layout::RowMajorVoltaTensorOpMultiplicandBCongruous<cutlass::sizeof_bits<Element>::value>;
|
||||
|
||||
using Policy = cutlass::gemm::warp::MmaTensorOpPolicy<
|
||||
cutlass::arch::Mma<
|
||||
cutlass::gemm::GemmShape<16, 16, 4>,
|
||||
32,
|
||||
Element,
|
||||
cutlass::layout::ColumnMajor,
|
||||
Element,
|
||||
cutlass::layout::RowMajor,
|
||||
ElementAccumulator,
|
||||
cutlass::layout::RowMajor,
|
||||
cutlass::arch::OpMultiplyAdd
|
||||
>,
|
||||
cutlass::MatrixShape<1, 1>
|
||||
>;
|
||||
|
||||
using WarpMmaTensorOp = cutlass::gemm::warp::MmaVoltaTensorOp<
|
||||
WarpShape,
|
||||
Element,
|
||||
LayoutA,
|
||||
Element,
|
||||
LayoutB,
|
||||
ElementAccumulator,
|
||||
cutlass::layout::RowMajor,
|
||||
Policy
|
||||
>;
|
||||
|
||||
//
|
||||
// Output operator
|
||||
//
|
||||
|
||||
using OutputOp = cutlass::epilogue::thread::LinearCombinationPlanarComplex<
|
||||
ElementOutput,
|
||||
kElementsPerAccess,
|
||||
ElementAccumulator,
|
||||
ElementCompute
|
||||
>;
|
||||
|
||||
//
|
||||
// Define the epilogue
|
||||
//
|
||||
|
||||
using Epilogue = typename cutlass::epilogue::threadblock::DefaultEpiloguePlanarComplex<
|
||||
Shape,
|
||||
WarpMmaTensorOp,
|
||||
cutlass::arch::OpClassTensorOp,
|
||||
cutlass::arch::Sm70,
|
||||
kPartitionsK,
|
||||
OutputOp,
|
||||
kElementsPerAccess
|
||||
>::Epilogue;
|
||||
|
||||
//
|
||||
// Instantiate epilogue
|
||||
//
|
||||
|
||||
EpiloguePlanarComplexTestbed<Epilogue> testbed;
|
||||
|
||||
bool passed = testbed.run_all();
|
||||
|
||||
EXPECT_TRUE(passed);
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Epilogue_threadblock_epilogue, planar_complex_simt_f32_64x64_32x32x8) {
|
||||
|
||||
//
|
||||
// Define the warp-level matrix multiply
|
||||
//
|
||||
|
||||
using ElementOutput = float;
|
||||
using ElementAccumulator = float;
|
||||
using ElementCompute = float;
|
||||
int const kElementsPerAccess = 1;
|
||||
int const kPartitionsK = 1;
|
||||
|
||||
using Shape = cutlass::gemm::GemmShape<64, 64, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<32, 32, 8>;
|
||||
using Element = float;
|
||||
using ElementC = ElementAccumulator;
|
||||
using LayoutA = cutlass::layout::ColumnMajor;
|
||||
using LayoutB = cutlass::layout::RowMajor;
|
||||
using LayoutC = cutlass::layout::RowMajor;
|
||||
|
||||
using ElementOutput = Element;
|
||||
using ElementAccumulator = Element;
|
||||
using ElementCompute = Element;
|
||||
|
||||
using WarpMmaSimt = cutlass::gemm::warp::MmaSimt<
|
||||
WarpShape,
|
||||
Element,
|
||||
LayoutA,
|
||||
Element,
|
||||
LayoutB,
|
||||
Element,
|
||||
LayoutC,
|
||||
cutlass::gemm::warp::MmaSimtPolicy<
|
||||
cutlass::MatrixShape<4, 8>,
|
||||
cutlass::layout::RowMajorInterleaved<2>,
|
||||
cutlass::gemm::GemmShape<4, 4, 1>
|
||||
>
|
||||
>;
|
||||
|
||||
//
|
||||
// Output operator
|
||||
//
|
||||
|
||||
using OutputOp = cutlass::epilogue::thread::LinearCombinationPlanarComplex<
|
||||
ElementOutput,
|
||||
kElementsPerAccess,
|
||||
ElementAccumulator,
|
||||
ElementCompute
|
||||
>;
|
||||
|
||||
//
|
||||
// Define the epilogue
|
||||
//
|
||||
|
||||
using Epilogue = typename cutlass::epilogue::threadblock::DefaultEpiloguePlanarComplex<
|
||||
Shape,
|
||||
WarpMmaSimt,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
kPartitionsK,
|
||||
OutputOp,
|
||||
kElementsPerAccess
|
||||
>::Epilogue;
|
||||
|
||||
//
|
||||
// Instantiate epilogue
|
||||
//
|
||||
|
||||
EpiloguePlanarComplexTestbed<Epilogue> testbed;
|
||||
|
||||
bool passed = testbed.run_all();
|
||||
|
||||
EXPECT_TRUE(passed);
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(Epilogue_threadblock_epilogue, planar_complex_simt_f64_64x64_16x32x8) {
|
||||
|
||||
//
|
||||
// Define the warp-level matrix multiply
|
||||
//
|
||||
|
||||
using ElementOutput = double;
|
||||
using ElementAccumulator = double;
|
||||
using ElementCompute = double;
|
||||
int const kElementsPerAccess = 1;
|
||||
int const kPartitionsK = 1;
|
||||
|
||||
using Shape = cutlass::gemm::GemmShape<64, 64, 8>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<16, 32, 8>;
|
||||
using Element = double;
|
||||
using ElementC = ElementAccumulator;
|
||||
using LayoutA = cutlass::layout::ColumnMajor;
|
||||
using LayoutB = cutlass::layout::RowMajor;
|
||||
using LayoutC = cutlass::layout::RowMajor;
|
||||
|
||||
using ElementOutput = Element;
|
||||
using ElementAccumulator = Element;
|
||||
using ElementCompute = Element;
|
||||
|
||||
using WarpMmaSimt = cutlass::gemm::warp::MmaSimt<
|
||||
WarpShape,
|
||||
Element,
|
||||
LayoutA,
|
||||
Element,
|
||||
LayoutB,
|
||||
Element,
|
||||
LayoutC,
|
||||
cutlass::gemm::warp::MmaSimtPolicy<
|
||||
cutlass::MatrixShape<4, 8>,
|
||||
cutlass::layout::RowMajorInterleaved<2>,
|
||||
cutlass::gemm::GemmShape<4, 4, 1>
|
||||
>
|
||||
>;
|
||||
|
||||
//
|
||||
// Output operator
|
||||
//
|
||||
|
||||
using OutputOp = cutlass::epilogue::thread::LinearCombinationPlanarComplex<
|
||||
ElementOutput,
|
||||
kElementsPerAccess,
|
||||
ElementAccumulator,
|
||||
ElementCompute
|
||||
>;
|
||||
|
||||
//
|
||||
// Define the epilogue
|
||||
//
|
||||
|
||||
using Epilogue = typename cutlass::epilogue::threadblock::DefaultEpiloguePlanarComplex<
|
||||
Shape,
|
||||
WarpMmaSimt,
|
||||
cutlass::arch::OpClassSimt,
|
||||
cutlass::arch::Sm50,
|
||||
kPartitionsK,
|
||||
OutputOp,
|
||||
kElementsPerAccess
|
||||
>::Epilogue;
|
||||
|
||||
//
|
||||
// Instantiate epilogue
|
||||
//
|
||||
|
||||
EpiloguePlanarComplexTestbed<Epilogue> testbed;
|
||||
|
||||
bool passed = testbed.run_all();
|
||||
|
||||
EXPECT_TRUE(passed);
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
@@ -34,6 +34,7 @@
|
||||
#include "cutlass/half.h"
|
||||
|
||||
#include "cutlass/epilogue/thread/linear_combination.h"
|
||||
#include "cutlass/epilogue/thread/linear_combination_clamp.h"
|
||||
#include "cutlass/gemm/warp/default_mma_tensor_op.h"
|
||||
#include "cutlass/epilogue/threadblock/default_epilogue_tensor_op.h"
|
||||
|
||||
@@ -45,6 +46,541 @@
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM75_Epilogue_threadblock_epilogue, s4_tensor_op_64x64_64x64x32) {
|
||||
|
||||
//
|
||||
// Define the warp-level matrix multiply
|
||||
//
|
||||
|
||||
using ElementOutput = cutlass::int4b_t;
|
||||
using ElementAccumulator = int;
|
||||
using ElementCompute = float;
|
||||
int const kElementsPerAccess = 32 / cutlass::sizeof_bits<ElementOutput>::value;
|
||||
int const kPartitionsK = 1;
|
||||
|
||||
using Shape = cutlass::gemm::GemmShape<64, 64, 32>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<64, 64, 32>;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<8, 8, 32>;
|
||||
using Element = ElementOutput;
|
||||
using LayoutA = cutlass::layout::RowMajorTensorOpMultiplicandCrosswise<
|
||||
cutlass::sizeof_bits<Element>::value, 64>;
|
||||
using LayoutB = cutlass::layout::ColumnMajorTensorOpMultiplicandCrosswise<
|
||||
cutlass::sizeof_bits<Element>::value, 64>;
|
||||
|
||||
using WarpMmaTensorOp = typename cutlass::gemm::warp::DefaultMmaTensorOp<
|
||||
WarpShape, InstructionShape, Element, LayoutA, Element, LayoutB, ElementAccumulator,
|
||||
cutlass::layout::RowMajor, cutlass::arch::OpMultiplyAddSaturate>::Type;
|
||||
|
||||
//
|
||||
// Output operator
|
||||
//
|
||||
|
||||
using OutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
ElementOutput,
|
||||
kElementsPerAccess,
|
||||
ElementAccumulator,
|
||||
ElementCompute
|
||||
>;
|
||||
|
||||
//
|
||||
// Define the epilogue
|
||||
//
|
||||
|
||||
using Epilogue = typename cutlass::epilogue::threadblock::DefaultEpilogueTensorOp<
|
||||
Shape,
|
||||
WarpMmaTensorOp,
|
||||
kPartitionsK,
|
||||
OutputOp,
|
||||
kElementsPerAccess
|
||||
>::Epilogue;
|
||||
|
||||
//
|
||||
// Instantiate epilogue
|
||||
//
|
||||
|
||||
EpilogueTestbed<Epilogue> testbed;
|
||||
|
||||
bool passed = testbed.run_all();
|
||||
|
||||
EXPECT_TRUE(passed);
|
||||
}
|
||||
|
||||
TEST(SM75_Epilogue_threadblock_epilogue, s4_tensor_op_64x64_32x32x32) {
|
||||
|
||||
//
|
||||
// Define the warp-level matrix multiply
|
||||
//
|
||||
|
||||
using ElementOutput = cutlass::int4b_t;
|
||||
using ElementAccumulator = int;
|
||||
using ElementCompute = float;
|
||||
int const kElementsPerAccess = 32 / cutlass::sizeof_bits<ElementOutput>::value;
|
||||
int const kPartitionsK = 1;
|
||||
|
||||
using Shape = cutlass::gemm::GemmShape<64, 64, 32>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<32, 32, 32>;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<8, 8, 32>;
|
||||
using Element = ElementOutput;
|
||||
using LayoutA = cutlass::layout::RowMajorTensorOpMultiplicandCrosswise<
|
||||
cutlass::sizeof_bits<Element>::value, 64>;
|
||||
using LayoutB = cutlass::layout::ColumnMajorTensorOpMultiplicandCrosswise<
|
||||
cutlass::sizeof_bits<Element>::value, 64>;
|
||||
|
||||
using WarpMmaTensorOp = typename cutlass::gemm::warp::DefaultMmaTensorOp<
|
||||
WarpShape, InstructionShape, Element, LayoutA, Element, LayoutB, ElementAccumulator,
|
||||
cutlass::layout::RowMajor, cutlass::arch::OpMultiplyAddSaturate>::Type;
|
||||
|
||||
//
|
||||
// Output operator
|
||||
//
|
||||
|
||||
using OutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
ElementOutput,
|
||||
kElementsPerAccess,
|
||||
ElementAccumulator,
|
||||
ElementCompute
|
||||
>;
|
||||
|
||||
//
|
||||
// Define the epilogue
|
||||
//
|
||||
|
||||
using Epilogue = typename cutlass::epilogue::threadblock::DefaultEpilogueTensorOp<
|
||||
Shape,
|
||||
WarpMmaTensorOp,
|
||||
kPartitionsK,
|
||||
OutputOp,
|
||||
kElementsPerAccess
|
||||
>::Epilogue;
|
||||
|
||||
//
|
||||
// Instantiate epilogue
|
||||
//
|
||||
|
||||
EpilogueTestbed<Epilogue> testbed;
|
||||
|
||||
bool passed = testbed.run_all();
|
||||
|
||||
EXPECT_TRUE(passed);
|
||||
}
|
||||
|
||||
TEST(SM75_Epilogue_threadblock_epilogue, s8_tensor_op_128x128_64x64x32) {
|
||||
|
||||
//
|
||||
// Define the warp-level matrix multiply
|
||||
//
|
||||
|
||||
using ElementOutput = cutlass::int4b_t;
|
||||
using ElementAccumulator = int;
|
||||
using ElementCompute = float;
|
||||
int const kElementsPerAccess = 32 / cutlass::sizeof_bits<ElementOutput>::value;
|
||||
int const kPartitionsK = 1;
|
||||
|
||||
using Shape = cutlass::gemm::GemmShape<128, 128, 32>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<64, 64, 32>;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<8, 8, 32>;
|
||||
using Element = ElementOutput;
|
||||
using LayoutA = cutlass::layout::RowMajorTensorOpMultiplicandCrosswise<
|
||||
cutlass::sizeof_bits<Element>::value, 64>;
|
||||
using LayoutB = cutlass::layout::ColumnMajorTensorOpMultiplicandCrosswise<
|
||||
cutlass::sizeof_bits<Element>::value, 64>;
|
||||
|
||||
using WarpMmaTensorOp = typename cutlass::gemm::warp::DefaultMmaTensorOp<
|
||||
WarpShape, InstructionShape, Element, LayoutA, Element, LayoutB, ElementAccumulator,
|
||||
cutlass::layout::RowMajor, cutlass::arch::OpMultiplyAddSaturate>::Type;
|
||||
|
||||
//
|
||||
// Output operator
|
||||
//
|
||||
|
||||
using OutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
ElementOutput,
|
||||
kElementsPerAccess,
|
||||
ElementAccumulator,
|
||||
ElementCompute
|
||||
>;
|
||||
|
||||
//
|
||||
// Define the epilogue
|
||||
//
|
||||
|
||||
using Epilogue = typename cutlass::epilogue::threadblock::DefaultEpilogueTensorOp<
|
||||
Shape,
|
||||
WarpMmaTensorOp,
|
||||
kPartitionsK,
|
||||
OutputOp,
|
||||
kElementsPerAccess
|
||||
>::Epilogue;
|
||||
|
||||
//
|
||||
// Instantiate epilogue
|
||||
//
|
||||
|
||||
EpilogueTestbed<Epilogue> testbed;
|
||||
|
||||
bool passed = testbed.run_all();
|
||||
|
||||
EXPECT_TRUE(passed);
|
||||
}
|
||||
|
||||
TEST(SM75_Epilogue_threadblock_epilogue, s4_tensor_op_128x64_64x32x32) {
|
||||
|
||||
//
|
||||
// Define the warp-level matrix multiply
|
||||
//
|
||||
|
||||
using ElementOutput = cutlass::int4b_t;
|
||||
using ElementAccumulator = int;
|
||||
using ElementCompute = float;
|
||||
int const kElementsPerAccess = 32 / cutlass::sizeof_bits<ElementOutput>::value;
|
||||
int const kPartitionsK = 1;
|
||||
|
||||
using Shape = cutlass::gemm::GemmShape<128, 64, 32>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<64, 32, 32>;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<8, 8, 32>;
|
||||
using Element = ElementOutput;
|
||||
using LayoutA = cutlass::layout::RowMajorTensorOpMultiplicandCrosswise<
|
||||
cutlass::sizeof_bits<Element>::value, 64>;
|
||||
using LayoutB = cutlass::layout::ColumnMajorTensorOpMultiplicandCrosswise<
|
||||
cutlass::sizeof_bits<Element>::value, 64>;
|
||||
|
||||
using WarpMmaTensorOp = typename cutlass::gemm::warp::DefaultMmaTensorOp<
|
||||
WarpShape, InstructionShape, Element, LayoutA, Element, LayoutB, ElementAccumulator,
|
||||
cutlass::layout::RowMajor, cutlass::arch::OpMultiplyAddSaturate>::Type;
|
||||
|
||||
//
|
||||
// Output operator
|
||||
//
|
||||
|
||||
using OutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
ElementOutput,
|
||||
kElementsPerAccess,
|
||||
ElementAccumulator,
|
||||
ElementCompute
|
||||
>;
|
||||
|
||||
//
|
||||
// Define the epilogue
|
||||
//
|
||||
|
||||
using Epilogue = typename cutlass::epilogue::threadblock::DefaultEpilogueTensorOp<
|
||||
Shape,
|
||||
WarpMmaTensorOp,
|
||||
kPartitionsK,
|
||||
OutputOp,
|
||||
kElementsPerAccess
|
||||
>::Epilogue;
|
||||
|
||||
//
|
||||
// Instantiate epilogue
|
||||
//
|
||||
|
||||
EpilogueTestbed<Epilogue> testbed;
|
||||
|
||||
bool passed = testbed.run_all();
|
||||
|
||||
EXPECT_TRUE(passed);
|
||||
}
|
||||
|
||||
TEST(SM75_Epilogue_threadblock_epilogue, s4_tensor_op_64x128_32x64x32) {
|
||||
|
||||
//
|
||||
// Define the warp-level matrix multiply
|
||||
//
|
||||
|
||||
using ElementOutput = cutlass::int4b_t;
|
||||
using ElementAccumulator = int;
|
||||
using ElementCompute = float;
|
||||
int const kElementsPerAccess = 32 / cutlass::sizeof_bits<ElementOutput>::value;
|
||||
int const kPartitionsK = 1;
|
||||
|
||||
using Shape = cutlass::gemm::GemmShape<64, 128, 32>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<32, 64, 32>;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<8, 8, 32>;
|
||||
using Element = ElementOutput;
|
||||
using LayoutA = cutlass::layout::RowMajorTensorOpMultiplicandCrosswise<
|
||||
cutlass::sizeof_bits<Element>::value, 64>;
|
||||
using LayoutB = cutlass::layout::ColumnMajorTensorOpMultiplicandCrosswise<
|
||||
cutlass::sizeof_bits<Element>::value, 64>;
|
||||
|
||||
using WarpMmaTensorOp = typename cutlass::gemm::warp::DefaultMmaTensorOp<
|
||||
WarpShape, InstructionShape, Element, LayoutA, Element, LayoutB, ElementAccumulator,
|
||||
cutlass::layout::RowMajor, cutlass::arch::OpMultiplyAddSaturate>::Type;
|
||||
|
||||
//
|
||||
// Output operator
|
||||
//
|
||||
|
||||
using OutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
ElementOutput,
|
||||
kElementsPerAccess,
|
||||
ElementAccumulator,
|
||||
ElementCompute
|
||||
>;
|
||||
|
||||
//
|
||||
// Define the epilogue
|
||||
//
|
||||
|
||||
using Epilogue = typename cutlass::epilogue::threadblock::DefaultEpilogueTensorOp<
|
||||
Shape,
|
||||
WarpMmaTensorOp,
|
||||
kPartitionsK,
|
||||
OutputOp,
|
||||
kElementsPerAccess
|
||||
>::Epilogue;
|
||||
|
||||
//
|
||||
// Instantiate epilogue
|
||||
//
|
||||
|
||||
EpilogueTestbed<Epilogue> testbed;
|
||||
|
||||
bool passed = testbed.run_all();
|
||||
|
||||
EXPECT_TRUE(passed);
|
||||
}
|
||||
|
||||
TEST(SM75_Epilogue_threadblock_epilogue, s4_tensor_op_32x128_32x64x32) {
|
||||
|
||||
//
|
||||
// Define the warp-level matrix multiply
|
||||
//
|
||||
|
||||
using ElementOutput = cutlass::int4b_t;
|
||||
using ElementAccumulator = int;
|
||||
using ElementCompute = float;
|
||||
int const kElementsPerAccess = 32 / cutlass::sizeof_bits<ElementOutput>::value;
|
||||
int const kPartitionsK = 1;
|
||||
|
||||
using Shape = cutlass::gemm::GemmShape<32, 128, 32>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<32, 64, 32>;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<8, 8, 32>;
|
||||
using Element = ElementOutput;
|
||||
using LayoutA = cutlass::layout::RowMajorTensorOpMultiplicandCrosswise<
|
||||
cutlass::sizeof_bits<Element>::value, 64>;
|
||||
using LayoutB = cutlass::layout::ColumnMajorTensorOpMultiplicandCrosswise<
|
||||
cutlass::sizeof_bits<Element>::value, 64>;
|
||||
|
||||
using WarpMmaTensorOp = typename cutlass::gemm::warp::DefaultMmaTensorOp<
|
||||
WarpShape, InstructionShape, Element, LayoutA, Element, LayoutB, ElementAccumulator,
|
||||
cutlass::layout::RowMajor, cutlass::arch::OpMultiplyAddSaturate>::Type;
|
||||
|
||||
//
|
||||
// Output operator
|
||||
//
|
||||
|
||||
using OutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
ElementOutput,
|
||||
kElementsPerAccess,
|
||||
ElementAccumulator,
|
||||
ElementCompute
|
||||
>;
|
||||
|
||||
//
|
||||
// Define the epilogue
|
||||
//
|
||||
|
||||
using Epilogue = typename cutlass::epilogue::threadblock::DefaultEpilogueTensorOp<
|
||||
Shape,
|
||||
WarpMmaTensorOp,
|
||||
kPartitionsK,
|
||||
OutputOp,
|
||||
kElementsPerAccess
|
||||
>::Epilogue;
|
||||
|
||||
//
|
||||
// Instantiate epilogue
|
||||
//
|
||||
|
||||
EpilogueTestbed<Epilogue> testbed;
|
||||
|
||||
bool passed = testbed.run_all();
|
||||
|
||||
EXPECT_TRUE(passed);
|
||||
}
|
||||
|
||||
TEST(SM75_Epilogue_threadblock_epilogue, s4_tensor_op_128x32_64x32x32) {
|
||||
|
||||
//
|
||||
// Define the warp-level matrix multiply
|
||||
//
|
||||
|
||||
using ElementOutput = cutlass::int4b_t;
|
||||
using ElementAccumulator = int;
|
||||
using ElementCompute = float;
|
||||
int const kElementsPerAccess = 32 / cutlass::sizeof_bits<ElementOutput>::value;
|
||||
int const kPartitionsK = 1;
|
||||
|
||||
using Shape = cutlass::gemm::GemmShape<128, 32, 32>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<64, 32, 32>;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<8, 8, 32>;
|
||||
using Element = ElementOutput;
|
||||
using LayoutA = cutlass::layout::RowMajorTensorOpMultiplicandCrosswise<
|
||||
cutlass::sizeof_bits<Element>::value, 64>;
|
||||
using LayoutB = cutlass::layout::ColumnMajorTensorOpMultiplicandCrosswise<
|
||||
cutlass::sizeof_bits<Element>::value, 64>;
|
||||
|
||||
using WarpMmaTensorOp = typename cutlass::gemm::warp::DefaultMmaTensorOp<
|
||||
WarpShape, InstructionShape, Element, LayoutA, Element, LayoutB, ElementAccumulator,
|
||||
cutlass::layout::RowMajor, cutlass::arch::OpMultiplyAddSaturate>::Type;
|
||||
|
||||
//
|
||||
// Output operator
|
||||
//
|
||||
|
||||
using OutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
ElementOutput,
|
||||
kElementsPerAccess,
|
||||
ElementAccumulator,
|
||||
ElementCompute
|
||||
>;
|
||||
|
||||
//
|
||||
// Define the epilogue
|
||||
//
|
||||
|
||||
using Epilogue = typename cutlass::epilogue::threadblock::DefaultEpilogueTensorOp<
|
||||
Shape,
|
||||
WarpMmaTensorOp,
|
||||
kPartitionsK,
|
||||
OutputOp,
|
||||
kElementsPerAccess
|
||||
>::Epilogue;
|
||||
|
||||
//
|
||||
// Instantiate epilogue
|
||||
//
|
||||
|
||||
EpilogueTestbed<Epilogue> testbed;
|
||||
|
||||
bool passed = testbed.run_all();
|
||||
|
||||
EXPECT_TRUE(passed);
|
||||
}
|
||||
|
||||
|
||||
TEST(SM75_Epilogue_threadblock_epilogue, s8_tensor_op_256x128_64x64x32) {
|
||||
|
||||
//
|
||||
// Define the warp-level matrix multiply
|
||||
//
|
||||
|
||||
using ElementOutput = cutlass::int4b_t;
|
||||
using ElementAccumulator = int;
|
||||
using ElementCompute = float;
|
||||
int const kElementsPerAccess = 32 / cutlass::sizeof_bits<ElementOutput>::value;
|
||||
int const kPartitionsK = 1;
|
||||
|
||||
using Shape = cutlass::gemm::GemmShape<256, 128, 32>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<64, 64, 32>;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<8, 8, 32>;
|
||||
using Element = ElementOutput;
|
||||
using LayoutA = cutlass::layout::RowMajorTensorOpMultiplicandCrosswise<
|
||||
cutlass::sizeof_bits<Element>::value, 64>;
|
||||
using LayoutB = cutlass::layout::ColumnMajorTensorOpMultiplicandCrosswise<
|
||||
cutlass::sizeof_bits<Element>::value, 64>;
|
||||
|
||||
using WarpMmaTensorOp = typename cutlass::gemm::warp::DefaultMmaTensorOp<
|
||||
WarpShape, InstructionShape, Element, LayoutA, Element, LayoutB, ElementAccumulator,
|
||||
cutlass::layout::RowMajor, cutlass::arch::OpMultiplyAddSaturate>::Type;
|
||||
|
||||
//
|
||||
// Output operator
|
||||
//
|
||||
|
||||
using OutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
ElementOutput,
|
||||
kElementsPerAccess,
|
||||
ElementAccumulator,
|
||||
ElementCompute
|
||||
>;
|
||||
|
||||
//
|
||||
// Define the epilogue
|
||||
//
|
||||
|
||||
using Epilogue = typename cutlass::epilogue::threadblock::DefaultEpilogueTensorOp<
|
||||
Shape,
|
||||
WarpMmaTensorOp,
|
||||
kPartitionsK,
|
||||
OutputOp,
|
||||
kElementsPerAccess
|
||||
>::Epilogue;
|
||||
|
||||
//
|
||||
// Instantiate epilogue
|
||||
//
|
||||
|
||||
EpilogueTestbed<Epilogue> testbed;
|
||||
|
||||
bool passed = testbed.run_all();
|
||||
|
||||
EXPECT_TRUE(passed);
|
||||
}
|
||||
|
||||
|
||||
TEST(SM75_Epilogue_threadblock_epilogue, s8_tensor_op_128x256_64x64x32) {
|
||||
|
||||
//
|
||||
// Define the warp-level matrix multiply
|
||||
//
|
||||
|
||||
using ElementOutput = cutlass::int4b_t;
|
||||
using ElementAccumulator = int;
|
||||
using ElementCompute = float;
|
||||
int const kElementsPerAccess = 32 / cutlass::sizeof_bits<ElementOutput>::value;
|
||||
int const kPartitionsK = 1;
|
||||
|
||||
using Shape = cutlass::gemm::GemmShape<128, 256, 32>;
|
||||
using WarpShape = cutlass::gemm::GemmShape<64, 64, 32>;
|
||||
using InstructionShape = cutlass::gemm::GemmShape<8, 8, 32>;
|
||||
using Element = ElementOutput;
|
||||
using LayoutA = cutlass::layout::RowMajorTensorOpMultiplicandCrosswise<
|
||||
cutlass::sizeof_bits<Element>::value, 64>;
|
||||
using LayoutB = cutlass::layout::ColumnMajorTensorOpMultiplicandCrosswise<
|
||||
cutlass::sizeof_bits<Element>::value, 64>;
|
||||
|
||||
using WarpMmaTensorOp = typename cutlass::gemm::warp::DefaultMmaTensorOp<
|
||||
WarpShape, InstructionShape, Element, LayoutA, Element, LayoutB, ElementAccumulator,
|
||||
cutlass::layout::RowMajor, cutlass::arch::OpMultiplyAddSaturate>::Type;
|
||||
|
||||
//
|
||||
// Output operator
|
||||
//
|
||||
|
||||
using OutputOp = cutlass::epilogue::thread::LinearCombination<
|
||||
ElementOutput,
|
||||
kElementsPerAccess,
|
||||
ElementAccumulator,
|
||||
ElementCompute
|
||||
>;
|
||||
|
||||
//
|
||||
// Define the epilogue
|
||||
//
|
||||
|
||||
using Epilogue = typename cutlass::epilogue::threadblock::DefaultEpilogueTensorOp<
|
||||
Shape,
|
||||
WarpMmaTensorOp,
|
||||
kPartitionsK,
|
||||
OutputOp,
|
||||
kElementsPerAccess
|
||||
>::Epilogue;
|
||||
|
||||
//
|
||||
// Instantiate epilogue
|
||||
//
|
||||
|
||||
EpilogueTestbed<Epilogue> testbed;
|
||||
|
||||
bool passed = testbed.run_all();
|
||||
|
||||
EXPECT_TRUE(passed);
|
||||
}
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
TEST(SM75_Epilogue_threadblock_epilogue, s8_tensor_op_64x64_64x64x16) {
|
||||
|
||||
//
|
||||
|
||||
@@ -0,0 +1,388 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2019, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/*! \file
|
||||
\brief Unit tests for epilogues
|
||||
*/
|
||||
#pragma once
|
||||
|
||||
#include <fstream>
|
||||
|
||||
#include "../../common/cutlass_unit_test.h"
|
||||
|
||||
#include "cutlass/aligned_buffer.h"
|
||||
#include "cutlass/half.h"
|
||||
#include "cutlass/complex.h"
|
||||
|
||||
#include "cutlass/epilogue/thread/linear_combination_planar_complex.h"
|
||||
|
||||
#include "cutlass/util/host_tensor_planar_complex.h"
|
||||
#include "cutlass/util/tensor_view_io.h"
|
||||
#include "cutlass/util/reference/host/tensor_fill.h"
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
namespace test {
|
||||
namespace kernel {
|
||||
|
||||
template <typename Epilogue>
|
||||
__global__ void epilogue_planar_complex_threadblock(
|
||||
typename Epilogue::OutputTileIterator::Params params_D,
|
||||
typename Epilogue::OutputTileIterator::Element *ptr_D,
|
||||
int64_t imaginary_stride_D,
|
||||
typename Epilogue::OutputTileIterator::Params params_C,
|
||||
typename Epilogue::OutputTileIterator::Element *ptr_C,
|
||||
int64_t imaginary_stride_C,
|
||||
typename Epilogue::OutputOp::Params params_output_op,
|
||||
cutlass::MatrixCoord problem_size,
|
||||
cutlass::TensorRef<
|
||||
typename Epilogue::WarpMmaOperator::ElementC,
|
||||
typename Epilogue::WarpMmaOperator::LayoutC> accumulator_ref,
|
||||
int64_t imaginary_stride_accum,
|
||||
int epilogue_count = 1) {
|
||||
|
||||
__shared__ typename Epilogue::SharedStorage shared_storage;
|
||||
|
||||
int thread_idx = threadIdx.x;
|
||||
int warp_idx = threadIdx.x / 32;
|
||||
int lane_idx = threadIdx.x % 32;
|
||||
|
||||
//
|
||||
// Construct the epilogue
|
||||
//
|
||||
|
||||
// Tile iterator writing to output tile
|
||||
typename Epilogue::OutputTileIterator iterator_D_real(
|
||||
params_D,
|
||||
ptr_D,
|
||||
problem_size,
|
||||
thread_idx
|
||||
);
|
||||
|
||||
typename Epilogue::OutputTileIterator iterator_D_imag(
|
||||
params_D,
|
||||
ptr_D + imaginary_stride_D,
|
||||
problem_size,
|
||||
thread_idx
|
||||
);
|
||||
|
||||
// Tile iterator writing to output tile
|
||||
typename Epilogue::OutputTileIterator iterator_C_real(
|
||||
params_C,
|
||||
ptr_C,
|
||||
problem_size,
|
||||
thread_idx
|
||||
);
|
||||
|
||||
typename Epilogue::OutputTileIterator iterator_C_imag(
|
||||
params_C,
|
||||
ptr_C + imaginary_stride_C,
|
||||
problem_size,
|
||||
thread_idx
|
||||
);
|
||||
|
||||
// Epilogue operator
|
||||
Epilogue epilogue(
|
||||
shared_storage,
|
||||
thread_idx,
|
||||
warp_idx,
|
||||
lane_idx);
|
||||
|
||||
//
|
||||
// Initialize the accumulators
|
||||
//
|
||||
|
||||
int warp_mn = warp_idx % (Epilogue::WarpCount::kM * Epilogue::WarpCount::kN);
|
||||
int warp_m = warp_mn % Epilogue::WarpCount::kM;
|
||||
int warp_n = warp_mn / Epilogue::WarpCount::kM;
|
||||
|
||||
accumulator_ref.add_coord_offset({
|
||||
warp_m * Epilogue::WarpMmaOperator::Shape::kM,
|
||||
warp_n * Epilogue::WarpMmaOperator::Shape::kN});
|
||||
|
||||
//
|
||||
// Load accumulators
|
||||
//
|
||||
|
||||
typename Epilogue::WarpMmaOperator::IteratorC accumulator_iterator(accumulator_ref, lane_idx);
|
||||
|
||||
typename Epilogue::AccumulatorTile accumulators;
|
||||
|
||||
accumulators.clear();
|
||||
|
||||
accumulator_iterator.load(accumulators.real);
|
||||
accumulator_iterator.load_with_pointer_offset(accumulators.imag, imaginary_stride_accum);
|
||||
|
||||
//
|
||||
// Perform the epilogue operation
|
||||
//
|
||||
|
||||
typename Epilogue::OutputOp output_op(params_output_op);
|
||||
|
||||
// Place the epilogue in a loop so assembly is clearly visible
|
||||
for (int iter = 0; iter < epilogue_count; ++iter) {
|
||||
epilogue(
|
||||
output_op,
|
||||
iterator_D_real,
|
||||
iterator_D_imag,
|
||||
accumulators,
|
||||
iterator_C_real,
|
||||
iterator_C_imag);
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace kernel
|
||||
} // namespace test
|
||||
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
template <
|
||||
typename Epilogue_
|
||||
>
|
||||
class EpiloguePlanarComplexTestbed {
|
||||
public:
|
||||
|
||||
using Epilogue = Epilogue_;
|
||||
using ElementAccumulator = typename Epilogue::ElementAccumulator;
|
||||
using ElementCompute = typename Epilogue::OutputOp::ElementCompute;
|
||||
using ElementOutput = typename Epilogue::ElementOutput;
|
||||
using OutputOpParams = typename Epilogue::OutputOp::Params;
|
||||
|
||||
using ComplexElementOutput = cutlass::complex<ElementOutput>;
|
||||
using ComplexElementAccumulator = cutlass::complex<ElementAccumulator>;
|
||||
using ComplexElementCompute = cutlass::complex<ElementCompute>;
|
||||
|
||||
public:
|
||||
|
||||
//
|
||||
// Data members
|
||||
//
|
||||
|
||||
cutlass::MatrixCoord quantized_size;
|
||||
cutlass::HostTensorPlanarComplex<ElementAccumulator, cutlass::layout::RowMajor> accumulator_tensor;
|
||||
cutlass::HostTensorPlanarComplex<ElementOutput, cutlass::layout::RowMajor> source_tensor;
|
||||
cutlass::HostTensorPlanarComplex<ElementOutput, cutlass::layout::RowMajor> output_tensor;
|
||||
|
||||
public:
|
||||
|
||||
//
|
||||
// Methods
|
||||
//
|
||||
|
||||
EpiloguePlanarComplexTestbed():
|
||||
quantized_size(Epilogue::Shape::kM, Epilogue::Shape::kN),
|
||||
accumulator_tensor({Epilogue::Shape::kM, Epilogue::Shape::kN}),
|
||||
source_tensor({Epilogue::Shape::kM, Epilogue::Shape::kN}),
|
||||
output_tensor({Epilogue::Shape::kM, Epilogue::Shape::kN}) {
|
||||
|
||||
//
|
||||
// Initialize problem space
|
||||
//
|
||||
|
||||
#if 1
|
||||
uint64_t seed = 2019;
|
||||
|
||||
cutlass::reference::host::TensorFillRandomUniform(
|
||||
accumulator_tensor.host_view(),
|
||||
seed,
|
||||
20,
|
||||
-20,
|
||||
0);
|
||||
|
||||
cutlass::reference::host::TensorFillRandomUniform(
|
||||
source_tensor.host_view(),
|
||||
seed + 2018,
|
||||
20,
|
||||
-20,
|
||||
0);
|
||||
#else
|
||||
|
||||
cutlass::reference::host::BlockFillSequential(accumulator_tensor.host_data(), accumulator_tensor.capacity());
|
||||
|
||||
#endif
|
||||
}
|
||||
|
||||
bool run_all() {
|
||||
|
||||
cutlass::complex<float> alpha_values[3];
|
||||
|
||||
alpha_values[0] = cutlass::complex<float>(1, 0);
|
||||
alpha_values[1] = cutlass::complex<float>(0, 0);
|
||||
alpha_values[2] = cutlass::complex<float>(2.25f, -0.5f);
|
||||
|
||||
cutlass::complex<float> beta_values[3];
|
||||
|
||||
beta_values[0] = cutlass::complex<float>(0, 0);
|
||||
beta_values[1] = cutlass::complex<float>(1, 0);
|
||||
beta_values[2] = cutlass::complex<float>(0.5f, -2.25f);
|
||||
|
||||
// Test runtime explodes if we tried to test every case exhaustively. This tests the full
|
||||
// output tile and several smaller sizes to stress predication.
|
||||
for (int m_idx = 0; m_idx < 3; ++m_idx) {
|
||||
for (int n_idx = 0; n_idx < 3; ++n_idx) {
|
||||
|
||||
cutlass::MatrixCoord problem_size(
|
||||
quantized_size.row() - m_idx * 3,
|
||||
quantized_size.column() - n_idx * Epilogue::kElementsPerAccess
|
||||
);
|
||||
|
||||
for (auto const &alpha : alpha_values) {
|
||||
for (auto const &beta : beta_values) {
|
||||
|
||||
bool passed = run(problem_size, {alpha, beta});
|
||||
|
||||
if (!passed) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/// Runs the test
|
||||
bool run(
|
||||
cutlass::MatrixCoord problem_size,
|
||||
OutputOpParams output_params) {
|
||||
|
||||
//
|
||||
// Initialize problem space
|
||||
//
|
||||
|
||||
ComplexElementOutput default_output = ComplexElementOutput(ElementOutput(-127), ElementOutput(-101));
|
||||
|
||||
cutlass::reference::host::TensorFill(output_tensor.host_view(), default_output);
|
||||
|
||||
accumulator_tensor.sync_device();
|
||||
output_tensor.sync_device();
|
||||
source_tensor.sync_device();
|
||||
|
||||
//
|
||||
// Initialize epilogue parameters
|
||||
//
|
||||
|
||||
typename Epilogue::OutputTileIterator::Params params_D(output_tensor.layout());
|
||||
typename Epilogue::OutputTileIterator::Params params_C(source_tensor.layout());
|
||||
|
||||
//
|
||||
// Launch kernel
|
||||
//
|
||||
|
||||
dim3 grid(1, 1);
|
||||
dim3 block(Epilogue::WarpCount::kCount * 32, 1);
|
||||
|
||||
test::kernel::epilogue_planar_complex_threadblock<Epilogue><<< grid, block >>>(
|
||||
params_D,
|
||||
output_tensor.device_data(),
|
||||
output_tensor.imaginary_stride(),
|
||||
params_C,
|
||||
source_tensor.device_data(),
|
||||
source_tensor.imaginary_stride(),
|
||||
output_params,
|
||||
problem_size,
|
||||
accumulator_tensor.device_view_real(),
|
||||
accumulator_tensor.imaginary_stride()
|
||||
);
|
||||
|
||||
cudaError_t result = cudaDeviceSynchronize();
|
||||
|
||||
if (result != cudaSuccess) {
|
||||
std::cerr << "Kernel error: " << cudaGetErrorString(result) << std::endl;
|
||||
return false;
|
||||
}
|
||||
|
||||
//
|
||||
// Verify results
|
||||
//
|
||||
output_tensor.sync_host();
|
||||
|
||||
int errors = 0;
|
||||
int const kMaxErrors = 5;
|
||||
|
||||
for (int r = 0; errors < kMaxErrors && r < quantized_size.row(); ++r) {
|
||||
for (int c = 0; errors < kMaxErrors && c < quantized_size.column(); ++c) {
|
||||
|
||||
cutlass::MatrixCoord coord{r, c};
|
||||
ComplexElementOutput got = output_tensor.at(coord);
|
||||
|
||||
ComplexElementOutput expected = default_output;
|
||||
|
||||
if (coord.row() < problem_size.row() && coord.column() < problem_size.column()) {
|
||||
|
||||
ComplexElementOutput src = source_tensor.at(coord);
|
||||
|
||||
ComplexElementCompute tmp =
|
||||
output_params.alpha * ComplexElementCompute(accumulator_tensor.at(coord)) +
|
||||
output_params.beta * ComplexElementCompute(src.real(), src.imag());
|
||||
|
||||
expected = ComplexElementOutput(ElementOutput(tmp.real()), ElementOutput(tmp.imag()));
|
||||
}
|
||||
|
||||
if (expected != got) {
|
||||
|
||||
using OutputIO = cutlass::ScalarIO<ComplexElementOutput>;
|
||||
|
||||
EXPECT_TRUE(false)
|
||||
<< "-------\n"
|
||||
<< "Error - output element (" << coord << ") - expected: "
|
||||
<< OutputIO(expected)
|
||||
<< ", got: " << OutputIO(got) << std::endl;
|
||||
|
||||
++errors;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
//
|
||||
// Report results on error
|
||||
//
|
||||
|
||||
if (errors) {
|
||||
|
||||
|
||||
std::cout << "Incorrect result for problem("
|
||||
<< problem_size.row() << ", "
|
||||
<< problem_size.column() << ") for alpha: " << output_params.alpha << ", beta: " << output_params.beta << std::endl;
|
||||
|
||||
std::stringstream ss;
|
||||
ss
|
||||
<< "output_tensor_op_" << Epilogue::Shape::kM << "x" << Epilogue::Shape::kN << "_"
|
||||
<< Epilogue::WarpTileIterator::WarpShape::kM << "x"
|
||||
<< Epilogue::WarpTileIterator::WarpShape::kN
|
||||
<< "_slice_" << Epilogue::WarpCount::kK << ".csv";
|
||||
|
||||
std::ofstream output_file(ss.str());
|
||||
output_file << output_tensor.host_view();
|
||||
|
||||
std::cout << "Wrote workspace to '" << ss.str() << "'" << std::endl;
|
||||
}
|
||||
|
||||
return !errors;
|
||||
}
|
||||
};
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
Reference in New Issue
Block a user