leavelet/cutlass
1
0
Fork 0
Code Issues Pull Requests Actions 24 Packages Projects Releases Wiki Activity

CUTLASS 2.3 initial commit (#134)

Browse Source 
CUTLASS 2.3 adds GEMMs targeting Sparse Tensor Cores on the NVIDIA Ampere Architecture, fast SGEMM, and small matrix classes, bug fixes, and performance enhancements.
This commit is contained in:
Andrew Kerr
2020-09-23 14:00:58 -07:00
committed by GitHub
parent 4dac7490e6
commit c53f3339bb
209 changed files with 46922 additions and 1677 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/util/include/cutlass/util/exceptions.h
+1 -6
View File
@@ -49,14 +49,9 @@ class cuda_exception : public std::exception {
cudaError_t err;
};
/// Writes a cudaError_t to an output stream
inline std::ostream& operator<<(std::ostream& out, cudaError_t result) {
return out << cudaGetErrorString(result);
}
/// Writes a cuda_exception instance to an output stream
inline std::ostream& operator<<(std::ostream& out, cuda_exception const& e) {
return out << e.what() << ": " << e.cudaError();
return out << e.what() << ": " << cudaGetErrorString(e.cudaError());
}
} // namespace cutlass
tools/util/include/cutlass/util/host_reorder.h
+30
View File
@@ -37,6 +37,8 @@
namespace cutlass {
/// This is needed for the interleaved integer tensor core kernels. The purpose
/// is to use skip the shared memory part in the epilogue.
template <int Interleaved, typename Element, typename Layout>
void reorder_column(TensorRef<Element, Layout> dest,
TensorRef<Element, Layout> src,
@@ -60,4 +62,32 @@ void reorder_column(TensorRef<Element, Layout> dest,
}
}
/// This is needed for the sparse tensor core kernels. The purpose
/// is to use ldmatrix to load from shared memory to the register file.
template <typename Element, typename LayoutDest, typename LayoutSrc>
void reorder_meta(TensorRef<Element, LayoutDest> dest,
TensorRef<Element, LayoutSrc> src,
cutlass::gemm::GemmCoord problem_size) {
for (int m = 0; m < problem_size.m(); m++) {
for (int k = 0; k < problem_size.k(); k++) {
// First reorder the rows.
int group = (sizeof(Element) == 2) ? 32 : 16;
int interweave = (sizeof(Element) == 2) ? 4 : 2;
int dest_row = m / group * group + (m % 8) * interweave + (m % group) / 8;
int dest_col = k;
// Next swizzle the 2x2 blocks from Z to N.
if (((dest_row % 2) == 0) && ((dest_col % 2) == 1)) {
++dest_row;
--dest_col;
} else if (((dest_row % 2) == 1) && ((dest_col % 2) == 0)) {
--dest_row;
++dest_col;
}
dest.at({dest_row, dest_col}) = src.at({m, k});
}
}
}
} // namespace cutlass
tools/util/include/cutlass/util/host_tensor.h
-1
View File
@@ -39,7 +39,6 @@
#include <vector>
#include "cutlass/cutlass.h"
#include "cutlass/matrix_traits.h"
#include "cutlass/tensor_ref.h"
#include "cutlass/tensor_view.h"
tools/util/include/cutlass/util/host_tensor_planar_complex.h
-1
View File
@@ -39,7 +39,6 @@
#include <vector>
#include "cutlass/cutlass.h"
#include "cutlass/matrix_traits.h"
#include "cutlass/tensor_ref_planar_complex.h"
#include "cutlass/tensor_view_planar_complex.h"
tools/util/include/cutlass/util/host_uncompress.h
+117
View File
@@ -0,0 +1,117 @@
/***************************************************************************************************
* Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
* * Redistributions of source code must retain the above copyright notice, this list of
* conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright notice, this list of
* conditions and the following disclaimer in the documentation and/or other materials
* provided with the distribution.
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
* to endorse or promote products derived from this software without specific prior written
* permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/*! \file
\brief uncompress sparse matrix from the host side
*/
#pragma once
#include "cutlass/coord.h"
#include "cutlass/util/host_tensor.h"
#include "cutlass/tensor_view.h"
#include "cutlass/util/tensor_view_io.h"
#include "cutlass/util/reference/host/gemm.h"
namespace cutlass {
template <typename ElementA, typename LayoutA, typename ElementE,
typename LayoutE>
void uncompress(TensorRef<ElementA, LayoutA> uncompressed_tensor_a,
TensorRef<ElementA, LayoutA> tensor_a,
TensorRef<ElementE, LayoutE> tensor_e, int row, int col) {
// How many uncompressed data we can get with ElementE meta data
int DecompressedElementsPerElementE =
256 / cutlass::sizeof_bits<ElementA>::value;
// Process 4bit meta data a time
int step;
// 1:2 or 2:4 or 4:8
int a, b;
if (cutlass::sizeof_bits<ElementA>::value == 4) {
step = 8;
a = 4;
b = 8;
} else if (cutlass::sizeof_bits<ElementA>::value == 8) {
step = 4;
a = 2;
b = 4;
} else if (cutlass::sizeof_bits<ElementA>::value == 16) {
step = 4;
a = 2;
b = 4;
} else if (cutlass::sizeof_bits<ElementA>::value == 32) {
step = 2;
a = 1;
b = 2;
}
int ElementsPerE = (cutlass::sizeof_bits<ElementA>::value == 4) ? 2 : 1;
for (int r = 0; r < row; ++r) {
for (int c = 0; c < (col / DecompressedElementsPerElementE); ++c) {
ElementE meta = tensor_e.at(MatrixCoord(r, c));
for (int i = 0; i < DecompressedElementsPerElementE; i += step) {
int e = (meta >> (i / step * 4)) & 0xf;
int idx0 = e & 0x3;
int idx1 = e >> 2;
if (a == 1) idx0 = idx0 / 2;
for (int ii = 0; ii < step; ii += ElementsPerE) {
int real_col =
c * DecompressedElementsPerElementE + i + ii;
int compressed_col = (real_col / b) * a;
if (ii == (idx0 * ElementsPerE)) {
uncompressed_tensor_a.at(MatrixCoord(r, real_col)) =
tensor_a.at(MatrixCoord(r, compressed_col));
if (ElementsPerE == 2)
uncompressed_tensor_a.at(MatrixCoord(r, real_col + 1)) =
tensor_a.at(MatrixCoord(r, compressed_col + 1));
} else if ((ii == (idx1 * ElementsPerE)) && (a != 1)) {
uncompressed_tensor_a.at(MatrixCoord(r, real_col)) =
tensor_a.at(MatrixCoord(r, compressed_col + ElementsPerE));
if (ElementsPerE == 2)
uncompressed_tensor_a.at(MatrixCoord(r, real_col + 1)) =
tensor_a.at(
MatrixCoord(r, compressed_col + ElementsPerE + 1));
} else {
uncompressed_tensor_a.at(MatrixCoord(r, real_col)) =
ElementA(0);
if (ElementsPerE == 2)
uncompressed_tensor_a.at(MatrixCoord(r, real_col + 1)) =
ElementA(0);
}
}
}
}
}
}
} // namespace cutlass
tools/util/include/cutlass/util/reference/detail/linear_to_coordinate.h
+88
View File
@@ -0,0 +1,88 @@
/***************************************************************************************************
* Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
* * Redistributions of source code must retain the above copyright notice, this list of
* conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright notice, this list of
* conditions and the following disclaimer in the documentation and/or other materials
* provided with the distribution.
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
* to endorse or promote products derived from this software without specific prior written
* permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/*! \file
\brief Reference implementation for GEMM in host-side code.
*/
#pragma once
#include "cutlass/cutlass.h"
#include "cutlass/coord.h"
/////////////////////////////////////////////////////////////////////////////////////////////////
namespace cutlass {
namespace reference {
namespace detail {
/////////////////////////////////////////////////////////////////////////////////////////////////
template <int Rank, int Index>
struct LinearToCoordinateHelper {
CUTLASS_HOST_DEVICE
void operator()(Coord<Rank> &coord, int64_t idx, Coord<Rank> const &extent) const {
int64_t prod = 1;
CUTLASS_PRAGMA_UNROLL
for (int i = Rank - Index; i < Rank; ++i) {
prod *= int64_t(extent[i]);
}
coord[Rank - Index - 1] = int(idx / prod);
int64_t residual = idx % prod;
LinearToCoordinateHelper<Rank, Index - 1>()(coord, residual, extent);
}
};
template <int Rank>
struct LinearToCoordinateHelper<Rank, 0> {
CUTLASS_HOST_DEVICE
void operator()(Coord<Rank> &coord, int64_t idx, Coord<Rank> const &extent) const {
coord[Rank - 1] = int(idx);
}
};
/////////////////////////////////////////////////////////////////////////////////////////////////
template <int Rank>
struct LinearToCoordinate {
CUTLASS_HOST_DEVICE
void operator()(Coord<Rank> &coord, int64_t idx, Coord<Rank> const &extent) const {
LinearToCoordinateHelper<Rank, Rank - 1>()(coord, idx, extent);
}
};
/////////////////////////////////////////////////////////////////////////////////////////////////
} // namespace detail
} // namespace reference
} // namespace cutlass
/////////////////////////////////////////////////////////////////////////////////////////////////
tools/util/include/cutlass/util/reference/device/gemm.h
-1
View File
@@ -34,7 +34,6 @@
#include "cutlass/functional.h"
#include "cutlass/numeric_conversion.h"
#include "cutlass/matrix_traits.h"
#include "cutlass/tensor_view.h"
#include "cutlass/gemm/gemm.h"
tools/util/include/cutlass/util/reference/device/gemm_complex.h
+295
View File
@@ -0,0 +1,295 @@
/***************************************************************************************************
* Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
* * Redistributions of source code must retain the above copyright notice, this list of
* conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright notice, this list of
* conditions and the following disclaimer in the documentation and/or other materials
* provided with the distribution.
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
* to endorse or promote products derived from this software without specific prior written
* permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
/*! \file
\brief Reference implementation for complex-valued GEMM in device-side code.
*/
#pragma once
#include "cutlass/coord.h"
#include "cutlass/complex.h"
#include "cutlass/numeric_types.h"
#include "cutlass/functional.h"
#include "cutlass/numeric_conversion.h"
#include "cutlass/tensor_view.h"
#include "cutlass/gemm/gemm.h"
namespace cutlass {
namespace reference {
namespace device {
////////////////////////////////////////////////////////////////////////////////////////////////////
namespace kernel {
/// Computes a general matrix product among matrices (tensors of rank=2) pointed to by TensorRef
/// objects.
///
/// Explicitly naming types needed by this template can be cumbersome, particularly for the
/// accumulator type, so a function argument 'initial_accum' is exposed. Passing
/// AccumulatorType(0) as the last function argument can be easier than naming all template
/// arguments explicitly.
template <
typename ElementA,
typename LayoutA,
typename ElementB,
typename LayoutB,
typename ElementC,
typename LayoutC,
typename ScalarType,
typename ComputeType,
typename ConvertOp = NumericConverter<ElementC, ScalarType>,
typename InnerProductOp = multiply_add<ComputeType>,
int kMblock = 4,
int kNblock = 4
>
__global__ void GemmComplex(
gemm::GemmCoord problem_size,
ScalarType alpha,
TensorRef<ElementA, LayoutA> tensor_a,
ComplexTransform transform_a,
TensorRef<ElementB, LayoutB> tensor_b,
ComplexTransform transform_b,
ScalarType beta,
TensorRef<ElementC, LayoutC> tensor_c,
TensorRef<ElementC, LayoutC> tensor_d,
ComputeType initial_accum,
int batch_count = 1,
int64_t batch_stride_A = 0,
int64_t batch_stride_B = 0,
int64_t batch_stride_C = 0,
int64_t batch_stride_D = 0) {
static_assert(
LayoutA::kRank == 2 &&
LayoutB::kRank == 2 &&
LayoutC::kRank == 2, "Tensors must be of rank 2");
int const M = problem_size.m();
int const N = problem_size.n();
int const K = problem_size.k();
ConvertOp convert_op;
InnerProductOp inner_product_op;
int row_block = (blockIdx.x * blockDim.x + threadIdx.x) * kMblock;
int col_block = (blockIdx.y * blockDim.y + threadIdx.y) * kNblock;
int batch_idx = blockIdx.z;
tensor_a.add_pointer_offset(batch_idx * batch_stride_A);
tensor_b.add_pointer_offset(batch_idx * batch_stride_B);
tensor_c.add_pointer_offset(batch_idx * batch_stride_C);
tensor_d.add_pointer_offset(batch_idx * batch_stride_D);
for (; batch_idx < batch_count; batch_idx += gridDim.z) {
// Compute matrix product using blocks
ComputeType accum[kMblock][kNblock];
CUTLASS_PRAGMA_UNROLL
for (int j = 0; j < kNblock; j++) {
CUTLASS_PRAGMA_UNROLL
for (int i = 0; i < kMblock; i++) {
accum[i][j] = initial_accum;
}
}
for (int k_block = 0; k_block < K; ++k_block) {
CUTLASS_PRAGMA_UNROLL
for (int j = 0; j < kNblock; j++) {
CUTLASS_PRAGMA_UNROLL
for (int i = 0; i < kMblock; i++) {
int row = row_block + i;
int col = col_block + j;
if (row < M && col < N) {
ElementA a = tensor_a.at(MatrixCoord(row, k_block));
ElementB b = tensor_b.at(MatrixCoord(k_block, col));
ComputeType a_ik = ComputeType(a);
ComputeType b_kj = ComputeType(b);
if (transform_a == ComplexTransform::kConjugate) {
a_ik = conj(a_ik);
}
if (transform_b == ComplexTransform::kConjugate) {
b_kj = conj(b_kj);
}
accum[i][j] = inner_product_op(a_ik, b_kj, accum[i][j]);
}
}
}
}
CUTLASS_PRAGMA_UNROLL
for (int j = 0; j < kNblock; j++) {
CUTLASS_PRAGMA_UNROLL
for (int i = 0; i < kMblock; i++) {
int row = row_block + i;
int col = col_block + j;
MatrixCoord coord = MatrixCoord(row, col);
if (row < M && col < N) {
tensor_d.at(coord) = convert_op(
alpha * ScalarType(accum[i][j]) +
beta * ScalarType(tensor_c.at(coord)));
}
}
}
tensor_a.add_pointer_offset(batch_stride_A * gridDim.z);
tensor_b.add_pointer_offset(batch_stride_B * gridDim.z);
tensor_c.add_pointer_offset(batch_stride_C * gridDim.z);
tensor_d.add_pointer_offset(batch_stride_D * gridDim.z);
} // for (batch_idx)
}
} // namespace kernel
////////////////////////////////////////////////////////////////////////////////////////////////////
/// Computes a general matrix product among matrices (tensors of rank=2) pointed to by TensorRef
/// objects.
///
/// Explicitly naming types needed by this template can be cumbersome, particularly for the
/// accumulator type, so a function argument 'initial_accum' is exposed. Passing
/// AccumulatorType(0) as the last function argument can be easier than naming all template
/// arguments explicitly.
template <
typename ElementA,
typename LayoutA,
typename ElementB,
typename LayoutB,
typename ElementC,
typename LayoutC,
typename ScalarType,
typename ComputeType,
typename ConvertOp = NumericConverter<ElementC, ScalarType>,
typename InnerProductOp = multiply_add<ComputeType>
>
void GemmComplex(
gemm::GemmCoord problem_size,
ScalarType alpha,
TensorRef<ElementA, LayoutA> tensor_a,
ComplexTransform transform_a,
TensorRef<ElementB, LayoutB> tensor_b,
ComplexTransform transform_b,
ScalarType beta,
TensorRef<ElementC, LayoutC> tensor_c,
TensorRef<ElementC, LayoutC> tensor_d,
ComputeType initial_accum,
int batch_count = 1,
int64_t batch_stride_A = 0,
int64_t batch_stride_B = 0,
int64_t batch_stride_C = 0,
int64_t batch_stride_D = 0) {
static_assert(
LayoutA::kRank == 2 &&
LayoutB::kRank == 2 &&
LayoutC::kRank == 2, "Tensors must be of rank 2");
int const kMblock = 4;
int const kNblock = 4;
dim3 block(16, 8);
dim3 grid(
(problem_size.m() + block.x * kMblock - 1) / (block.x * kMblock),
(problem_size.n() + block.y * kNblock - 1) / (block.y * kNblock),
batch_count % std::numeric_limits<uint16_t>::max()
);
kernel::GemmComplex<
ElementA,
LayoutA,
ElementB,
LayoutB,
ElementC,
LayoutC,
ScalarType,
ComputeType,
ConvertOp,
InnerProductOp,
kMblock,
kNblock
><<< grid, block >>>(
problem_size,
alpha,
tensor_a,
transform_a,
tensor_b,
transform_b,
beta,
tensor_c,
tensor_d,
initial_accum,
batch_count,
batch_stride_A,
batch_stride_B,
batch_stride_C,
batch_stride_D
);
}
////////////////////////////////////////////////////////////////////////////////////////////////////
/// Computes a general matrix product among matrices (tensors of rank=2) pointed to by TensorRef
/// objects.
///
/// This assumes the accumulator type is the same type as the scalars.
template <
typename ElementA,
typename LayoutA,
typename ElementB,
typename LayoutB,
typename ElementC,
typename LayoutC,
typename ScalarType
>
void GemmComplex(
gemm::GemmCoord problem_size,
ScalarType alpha,
TensorRef<ElementA, LayoutA> tensor_a,
ComplexTransform transform_a,
TensorRef<ElementB, LayoutB> tensor_b,
ComplexTransform transform_b,
ScalarType beta,
TensorRef<ElementC, LayoutC> tensor_c,
TensorRef<ElementC, LayoutC> tensor_d) {
GemmComplex(problem_size, alpha, tensor_a, transform_a, tensor_b, transform_b, beta, tensor_c, tensor_d, ScalarType(0));
}
////////////////////////////////////////////////////////////////////////////////////////////////////
} // namespace device
} // namespace reference
} // namespace cutlass
tools/util/include/cutlass/util/reference/device/gemm_planar_complex.h
-1
View File
@@ -36,7 +36,6 @@
#include "cutlass/numeric_conversion.h"
#include "cutlass/tensor_ref_planar_complex.h"
#include "cutlass/matrix_traits.h"
#include "cutlass/tensor_view.h"
#include "cutlass/gemm/gemm.h"
tools/util/include/cutlass/util/reference/device/kernel/gemm.h
-1
View File
@@ -29,7 +29,6 @@
#pragma once
#include "cutlass/coord.h"
#include "cutlass/matrix_traits.h"
#include "cutlass/tensor_view.h"
#include "cutlass/gemm/gemm.h"
tools/util/include/cutlass/util/reference/device/kernel/tensor_foreach.h
+3 -1
View File
@@ -27,6 +27,8 @@
#include "cutlass/cutlass.h"
#include "cutlass/coord.h"
#include "cutlass/subbyte_reference.h"
#include "cutlass/fast_math.h"
namespace cutlass {
namespace reference {
@@ -138,7 +140,7 @@ __global__ void BlockForEach(
size_t index = threadIdx.x + blockIdx.x * blockDim.x;
for (; index < capacity; index += blockDim.x * gridDim.x) {
ptr[index] = func();
ReferenceFactory<Element>::get(ptr, index) = func();
}
}
tools/util/include/cutlass/util/reference/device/tensor_compare.h
+8 -3
View File
@@ -56,8 +56,13 @@ __global__ void BlockCompareEqual(
size_t idx = threadIdx.x + blockDim.x * blockIdx.x;
for (; idx < capacity; idx += gridDim.x * blockDim.x) {
if (ptr_A[idx] != ptr_B[idx]) {
Element a = cutlass::ReferenceFactory<Element>::get(ptr_A, idx);
Element b = cutlass::ReferenceFactory<Element>::get(ptr_B, idx);
if (a != b) {
*equal = 0;
return;
}
}
@@ -76,8 +81,8 @@ __global__ void BlockCompareRelativelyEqual(
for (; idx < capacity; idx += gridDim.x * blockDim.x) {
Element a = ptr_A[idx];
Element b = ptr_B[idx];
Element a = cutlass::ReferenceFactory<Element>::get(ptr_A, idx);
Element b = cutlass::ReferenceFactory<Element>::get(ptr_B, idx);
if (!relatively_equal(a, b, epsilon, nonzero_floor)) {
*equal = 0;
tools/util/include/cutlass/util/reference/device/tensor_fill.h
+256 -29
View File
@@ -106,6 +106,8 @@ struct RandomGaussianFunc {
FloatType mean;
FloatType stddev;
int int_scale;
FloatType float_scale_up;
FloatType float_scale_down;
//
// Methods
@@ -123,6 +125,9 @@ struct RandomGaussianFunc {
stddev(static_cast<FloatType>(stddev_)),
int_scale(int_scale_) {
float_scale_up = FloatType(IntType(1) << int_scale);
float_scale_up += FloatType(0.5) * float_scale_up;
float_scale_down = FloatType(1) / FloatType(IntType(1) << int_scale);
}
};
@@ -158,8 +163,8 @@ struct RandomGaussianFunc {
Element result;
if (params.int_scale >= 0) {
rnd = FloatType(IntType(rnd * FloatType(IntType(1) << params.int_scale)));
result = Element(rnd / FloatType(IntType(1) << params.int_scale));
rnd = FloatType(IntType(rnd * params.float_scale_up));
result = Element(rnd * params.float_scale_down);
}
else {
result = Element(rnd);
@@ -188,6 +193,8 @@ struct RandomGaussianFunc<complex<Real>> {
FloatType mean;
FloatType stddev;
int int_scale;
FloatType float_scale_up;
FloatType float_scale_down;
//
// Methods
@@ -205,6 +212,9 @@ struct RandomGaussianFunc<complex<Real>> {
stddev(static_cast<FloatType>(stddev_)),
int_scale(int_scale_) {
float_scale_up = FloatType(IntType(1) << int_scale);
float_scale_up += FloatType(0.5) * float_scale_up;
float_scale_down = FloatType(1) / FloatType(IntType(1) << int_scale);
}
};
@@ -242,12 +252,12 @@ struct RandomGaussianFunc<complex<Real>> {
Element result;
if (params.int_scale >= 0) {
rnd_r = FloatType(IntType(rnd_r * FloatType(IntType(1) << params.int_scale)));
rnd_i = FloatType(IntType(rnd_i * FloatType(IntType(1) << params.int_scale)));
rnd_r = FloatType(IntType(rnd_r * params.float_scale_up));
rnd_i = FloatType(IntType(rnd_i * params.float_scale_down));
result = {
Real(rnd_r / FloatType(IntType(1) << params.int_scale)),
Real(rnd_i / FloatType(IntType(1) << params.int_scale))
Real(rnd_r * params.float_scale_down),
Real(rnd_i * params.float_scale_down)
};
}
else {
@@ -378,7 +388,7 @@ void BlockFillRandomGaussian(
namespace detail {
/// Computes a random Gaussian distribution
template <typename Element> ///< Layout function
template <typename Element> ///< Element type
struct RandomUniformFunc {
using FloatType = typename std::conditional<
@@ -400,8 +410,10 @@ struct RandomUniformFunc {
uint64_t seed;
FloatType range;
FloatType min;
FloatType max;
int int_scale;
FloatType float_scale_up;
FloatType float_scale_down;
/// Default ctor
CUTLASS_HOST_DEVICE
@@ -414,15 +426,18 @@ struct RandomUniformFunc {
/// Construction of Gaussian RNG functor.
Params(
uint64_t seed_ = 0,
Element max = 1,
Element min_ = 0,
Element max_ = 1,
Element min = 0,
int int_scale_ = -1
):
seed(seed_),
range(static_cast<FloatType>(max - min_)),
min(static_cast<FloatType>(min_)),
range(static_cast<FloatType>(max_ - min)),
max(static_cast<FloatType>(max_)),
int_scale(int_scale_) {
float_scale_up = FloatType(IntType(1) << int_scale);
float_scale_up += FloatType(0.5) * float_scale_up;
float_scale_down = FloatType(1) / FloatType(IntType(1) << int_scale);
}
};
@@ -454,15 +469,15 @@ struct RandomUniformFunc {
Element operator()() {
FloatType rnd = random_uniform_float<FloatType>(&rng_state);
rnd = params.min + params.range * rnd;
rnd = params.max - params.range * rnd;
// Random values are cast to integer after scaling by a power of two to facilitate error
// testing
Element result;
if (params.int_scale >= 0) {
rnd = FloatType(IntType(rnd * FloatType(IntType(1) << params.int_scale)));
result = Element(rnd / FloatType(IntType(1) << params.int_scale));
rnd = FloatType(IntType(rnd * params.float_scale_up));
result = Element(rnd * params.float_scale_down);
}
else {
result = Element(rnd);
@@ -473,7 +488,7 @@ struct RandomUniformFunc {
};
/// Computes a random Gaussian distribution
template <typename Real> ///< Layout function
template <typename Real>
struct RandomUniformFunc<complex<Real>> {
using Element = complex<Real>;
@@ -499,6 +514,8 @@ struct RandomUniformFunc<complex<Real>> {
FloatType range;
FloatType min;
int int_scale;
FloatType float_scale_up;
FloatType float_scale_down;
/// Default ctor
CUTLASS_HOST_DEVICE
@@ -520,6 +537,9 @@ struct RandomUniformFunc<complex<Real>> {
min(static_cast<FloatType>(min_)),
int_scale(int_scale_) {
float_scale_up = FloatType(IntType(1) << int_scale);
float_scale_up += FloatType(0.5) * float_scale_up;
float_scale_down = FloatType(1) / FloatType(IntType(1) << int_scale);
}
};
@@ -561,12 +581,12 @@ struct RandomUniformFunc<complex<Real>> {
Element result;
if (params.int_scale >= 0) {
rnd_r = FloatType(IntType(rnd_r * FloatType(IntType(1) << params.int_scale)));
rnd_i = FloatType(IntType(rnd_i * FloatType(IntType(1) << params.int_scale)));
rnd_r = FloatType(IntType(rnd_r * params.float_scale_up));
rnd_i = FloatType(IntType(rnd_i * params.float_scale_up));
result = {
Real(rnd_r / FloatType(IntType(1) << params.int_scale)),
Real(rnd_i / FloatType(IntType(1) << params.int_scale))
Real(rnd_r * params.float_scale_down),
Real(rnd_i * params.float_scale_down)
};
}
else {
@@ -670,7 +690,7 @@ void TensorFillRandomUniform(
typename RandomFunc::Params random(seed, max, min, bits);
TensorForEach<Func, Layout::kRank, Params>(
view.size(),
view.extent(),
Params(view, random)
);
}
@@ -690,6 +710,7 @@ void BlockFillRandomUniform(
/// data.
using RandomFunc = detail::RandomUniformFunc<Element>;
typename RandomFunc::Params params(seed, max, min, bits);
BlockForEach<Element, RandomFunc>(ptr, capacity, params);
@@ -700,7 +721,214 @@ void BlockFillRandomUniform(
namespace detail {
/// Computes a random sparse meta
template <typename Element> ///< Element type
struct RandomSparseMetaFunc {
using FloatType = float;
using IntType = int32_t;
/// Parameters structure
struct Params {
//
// Data members
//
uint64_t seed;
FloatType range;
int MetaSizeInBits;
/// Default ctor
CUTLASS_HOST_DEVICE
Params() { }
//
// Methods
//
/// Construction of Gaussian RNG functor.
Params(
uint64_t seed_ = 0,
int MetaSizeInBits_ = 2
):
seed(seed_),
MetaSizeInBits(MetaSizeInBits_) {
if (MetaSizeInBits_ == 2) {
range = 6;
} else if (MetaSizeInBits_ == 4) {
range = 2;
}
}
};
//
// Data members
//
/// Parameters object
Params params;
/// RNG state object
curandState_t rng_state;
//
// Methods
//
/// Device-side initialization of RNG
CUTLASS_DEVICE
RandomSparseMetaFunc(Params const &params): params(params) {
uint64_t gtid = threadIdx.x + blockIdx.x * blockDim.x;
curand_init(params.seed, gtid, 0, &rng_state);
}
/// Compute random value and update RNG state
CUTLASS_DEVICE
Element operator()() {
Element FourToTwoMeta[6] = {0x4, 0x8, 0x9, 0xc, 0xd, 0xe};
Element TwoToOneMeta[2] = {0x4, 0xe};
Element *MetaArray =
(params.MetaSizeInBits == 2) ? FourToTwoMeta : TwoToOneMeta;
Element result = 0x0;
CUTLASS_PRAGMA_UNROLL
for (int i = 0; i < cutlass::sizeof_bits<Element>::value / 4; ++i) {
FloatType rnd = random_uniform_float<FloatType>(&rng_state);
rnd = params.range * rnd;
Element meta = MetaArray[(int)rnd];
result = (Element)(result | ((Element)(meta << (i * 4))));
}
return result;
}
};
/// Computes a random Gaussian distribution
template <
typename Element, ///< Element type
typename Layout> ///< Layout function
struct TensorFillRandomSparseMetaFunc {
/// View type
using TensorView = TensorView<Element, Layout>;
/// Scalar type
typedef typename TensorView::Element T;
/// Coordinate in tensor's index space
typedef typename TensorView::TensorCoord TensorCoord;
using RandomFunc = RandomSparseMetaFunc<Element>;
/// Parameters structure
struct Params {
//
// Data members
//
TensorView view;
typename RandomFunc::Params random;
/// Default ctor
CUTLASS_HOST_DEVICE
Params() { }
//
// Methods
//
/// Construction of Gaussian RNG functor.
Params(
TensorView view_ = TensorView(),
typename RandomFunc::Params random_ = RandomFunc::Params()
):
view(view_), random(random_) {
}
};
//
// Data members
//
Params params;
RandomFunc random;
//
// Methods
//
/// Device-side initialization of RNG
CUTLASS_DEVICE
TensorFillRandomSparseMetaFunc(Params const &params): params(params), random(params.random) {
}
/// Compute random value and update RNG state
CUTLASS_DEVICE
void operator()(TensorCoord const &coord) {
params.view.at(coord) = random();
}
};
} // namespace detail
///////////////////////////////////////////////////////////////////////////////////////////////////
/// Fills a tensor with random values with a uniform random distribution.
template <
typename Element, ///< Element type
typename Layout> ///< Layout function
void TensorFillRandomSparseMeta(
TensorView<Element, Layout> view, ///< destination tensor
uint64_t seed, ///< seed for RNG
int MetaSizeInBits = 2) { ///< If non-negative, specifies number of fractional bits that
/// are not truncated to zero. Permits reducing precision of
/// data.
using RandomFunc = detail::RandomSparseMetaFunc<Element>;
using Func = detail::TensorFillRandomUniformFunc<Element, Layout>;
using Params = typename Func::Params;
typename RandomFunc::Params random(seed, MetaSizeInBits);
TensorForEach<Func, Layout::kRank, Params>(
view.extent(),
Params(view, random)
);
}
///////////////////////////////////////////////////////////////////////////////////////////////////
/// Fills a tensor with random values with a uniform random distribution.
template <typename Element>
void BlockFillRandomSparseMeta(
Element *ptr,
size_t capacity,
uint64_t seed, ///< seed for RNG
int MetaSizeInBits = 2) { ///< meta data size
using RandomFunc = detail::RandomSparseMetaFunc<Element>;
typename RandomFunc::Params params(seed, MetaSizeInBits);
BlockForEach<Element, RandomFunc>(ptr, capacity, params);
}
///////////////////////////////////////////////////////////////////////////////////////////////////
///////////////////////////////////////////////////////////////////////////////////////////////////
namespace detail {
/// Functor to fill a tensor with zeros off the diagonal and a uniform value on the diagonal.
template <
typename Element, ///< Element type
typename Layout> ///< Layout function
@@ -734,7 +962,6 @@ struct TensorFillDiagonalFunc {
// Methods
//
/// Construction of Gaussian RNG functor.
Params(
TensorView view_ = TensorView(),
Element diag_ = Element(1),
@@ -762,7 +989,7 @@ struct TensorFillDiagonalFunc {
}
/// Compute random value and update RNG state
/// Updates the tensor
CUTLASS_DEVICE
void operator()(TensorCoord const &coord) {
@@ -797,7 +1024,7 @@ void TensorFillDiagonal(
typedef typename Func::Params Params;
TensorForEach<Func, Layout::kRank, Params>(
view.size(),
view.extent(),
Params(view, diag, other)
);
}
@@ -928,7 +1155,7 @@ void TensorUpdateDiagonal(
typedef typename Func::Params Params;
TensorForEach<Func, Layout::kRank, Params>(
view.size(),
view.extent(),
Params(view, diag)
);
}
@@ -1034,7 +1261,7 @@ void TensorUpdateOffDiagonal(
typedef typename Func::Params Params;
TensorForEach<Func, Layout::kRank, Params>(
view.size(),
view.extent(),
Params(view, other)
);
}
@@ -1137,7 +1364,7 @@ void TensorFillLinear(
using Params = typename Func::Params;
TensorForEach<Func, Layout::kRank, Params>(
view.size(),
view.extent(),
Params(view, v, s)
);
}
@@ -1290,7 +1517,7 @@ void TensorCopyDiagonalIn(
using Params = typename Func::Params;
TensorForEach<Func, Layout::kRank, Params>(
view.size(),
view.extent(),
Params(view, ptr)
);
}
@@ -1394,7 +1621,7 @@ void TensorCopyDiagonalOut(
using Params = typename Func::Params;
TensorForEach<Func, Layout::kRank, Params>(
view.size(),
view.extent(),
Params(view, ptr)
);
}
tools/util/include/cutlass/util/reference/device/tensor_foreach.h
+1 -1
View File
@@ -95,7 +95,7 @@ struct BlockForEach {
BlockForEach(
Element *ptr,
size_t capacity,
typename Func::Params params = typename Func::Params(),
typename Func::Params params = typename Func::Params(),
int grid_size = 0,
int block_size = 0) {
tools/util/include/cutlass/util/reference/device/tensor_reduce.h
+505
View File
@@ -0,0 +1,505 @@
/***************************************************************************************************
* Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
* * Redistributions of source code must retain the above copyright notice, this list of
* conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright notice, this list of
* conditions and the following disclaimer in the documentation and/or other materials
* provided with the distribution.
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
* to endorse or promote products derived from this software without specific prior written
* permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
#pragma once
#include <cmath>
#include "cutlass/cutlass.h"
#include "cutlass/complex.h"
#include "cutlass/functional.h"
#include "cutlass/numeric_conversion.h"
#include "cutlass/tensor_view.h"
#include "cutlass/util/device_memory.h"
#include "cutlass/util/reference/detail/linear_to_coordinate.h"
/////////////////////////////////////////////////////////////////////////////////////////////////
namespace cutlass {
namespace reference {
namespace device {
/////////////////////////////////////////////////////////////////////////////////////////////////
namespace kernel {
template <
typename Element,
typename Layout,
typename ComputeType,
typename ReduceOp,
typename TransformOp,
int kBlockSize = 128
>
__global__ void TensorTransformReducePartial(
TensorView<Element, Layout> view, /// View of the tensor to reduce over
ComputeType identity, /// Identity element of the reduction operation
ReduceOp reduce, /// Reduces an accumulated value with a transformed element: f(ComputeType, ComputeType) => ComputeType
TransformOp transform, /// Transforms the tensor element to ComputeType: g(Element) => ComputeType
ComputeType *workspace) { /// Device-side workspace for accumulating partial results. The reduced element is stored in workspace[0]
int64_t idx = threadIdx.x + blockIdx.x * blockDim.x;
int64_t size = view.size();
__shared__ ComputeType scratchpad[kBlockSize];
for (; idx < size; idx += blockDim.x * gridDim.x) {
// Map linear thread ID onto tensor coordinate
typename Layout::TensorCoord coord;
cutlass::reference::detail::LinearToCoordinate<Layout::kRank>()(coord, idx, view.extent());
if (view.contains(coord)) {
// Fetch element
Element x = view.at(coord);
// Transform
identity = reduce(identity, transform(x));
}
}
scratchpad[threadIdx.x] = identity;
__syncthreads();
// One thread performs the final reduction and stores out. This could be enhanced via
// a tree reduction and pipelining.
if (threadIdx.x == 0) {
for (int i = 1; i < kBlockSize; ++i) {
identity = reduce(identity, scratchpad[i]);
}
workspace[blockIdx.x] = identity;
}
}
template <
typename Element,
typename Layout,
typename ComputeType,
typename ReduceOp,
typename TransformOp,
int kBlockSize = 128
>
__global__ void TensorTransformReducePartial(
TensorView<Element, Layout> view_A, /// View of the tensor to reduce over
TensorView<Element, Layout> view_B, /// View of the tensor to reduce over
ComputeType identity, /// Identity element of the reduction operation
ReduceOp reduce, /// Reduces an accumulated value with a transformed element: f(ComputeType, ComputeType) => ComputeType
TransformOp transform, /// Transforms the tensor element to ComputeType: g(Element) => ComputeType
ComputeType *workspace) { /// Device-side workspace for accumulating partial results. The reduced element is stored in workspace[0]
int64_t idx = threadIdx.x + blockIdx.x * blockDim.x;
int64_t size = view_A.size();
__shared__ ComputeType scratchpad[kBlockSize];
for (; idx < size; idx += blockDim.x * gridDim.x) {
// Map linear thread ID onto tensor coordinate
typename Layout::TensorCoord coord;
cutlass::reference::detail::LinearToCoordinate<Layout::kRank>()(coord, idx, view_A.extent());
if (view_A.contains(coord)) {
// Fetch element
Element a = view_A.at(coord);
Element b = view_B.at(coord);
// Transform
identity = reduce(identity, transform(a, b));
}
}
scratchpad[threadIdx.x] = identity;
__syncthreads();
// One thread performs the final reduction and stores out. This could be enhanced via
// a tree reduction and pipelining.
if (threadIdx.x == 0) {
for (int i = 1; i < kBlockSize; ++i) {
identity = reduce(identity, scratchpad[i]);
}
workspace[blockIdx.x] = identity;
}
}
template <
typename ComputeType,
typename ReduceOp,
int kBlockSize = 32
>
__global__ void TensorTransformReduceFinalize(
ComputeType *workspace,
ComputeType identity,
int workspace_size,
ReduceOp reduce) {
__shared__ ComputeType scratchpad[kBlockSize];
for (int idx = threadIdx.x; idx < workspace_size; idx += kBlockSize) {
identity = reduce(identity, workspace[idx]);
}
scratchpad[threadIdx.x] = identity;
__syncthreads();
if (threadIdx.x == 0) {
for (int i = 1; i < kBlockSize; ++i) {
identity = reduce(identity, scratchpad[i]);
}
workspace[0] = identity;
}
}
} // namespace kernel
/////////////////////////////////////////////////////////////////////////////////////////////////
/// Transform-reduce operation over the elements of a tensor
template <
typename Element,
typename Layout,
typename ComputeType,
typename ReduceOp,
typename TransformOp
>
ComputeType TensorTransformReduce(
TensorView<Element, Layout> view, /// View of the tensor to reduce over
ComputeType identity, /// Identity element of the reduction operation
ReduceOp reduce, /// Reduces an accumulated value with a transformed element: f(ComputeType, ComputeType) => ComputeType
TransformOp transform, /// Transforms the tensor element to ComputeType: g(Element) => ComputeType
ComputeType *workspace, /// Device-side workspace for accumulating partial results. The reduced element is stored in workspace[0]
int workspace_size, /// Number of elements in workspace
cudaStream_t stream = nullptr, /// CUDA stream to launch into
bool copy_out = true /// If true, the value of workspace[0] is copied to host and returned. Otherwise, `identity` is returned.
) {
int const kBlockSize = 128;
dim3 block(kBlockSize, 1);
dim3 grid(workspace_size, 1);
kernel::TensorTransformReducePartial<
Element, Layout, ComputeType, ReduceOp, TransformOp, kBlockSize
><<< grid, block, 0, stream >>>(
view, identity, reduce, transform, workspace
);
int const kFinalizeBlockSize = 32;
kernel::TensorTransformReduceFinalize<
ComputeType, ReduceOp, kFinalizeBlockSize
><<< dim3(1, 1), dim3(kFinalizeBlockSize, 1), 0, stream >>>(
workspace, identity, workspace_size, reduce
);
if (copy_out) {
cudaError_t result = cudaMemcpy(&identity, workspace, sizeof(identity), cudaMemcpyDeviceToHost);
if (result != cudaSuccess) {
throw std::runtime_error("cudaMemcpy() failed");
}
}
return identity;
}
/// Transform-reduce operation over the elements of two tensors, zipped together
template <
typename Element,
typename Layout,
typename ComputeType,
typename ReduceOp,
typename TransformOp
>
ComputeType TensorTransformReduce(
TensorView<Element, Layout> view_A, /// View of the tensor to reduce over
TensorView<Element, Layout> view_B, /// View of the tensor to reduce over
ComputeType identity, /// Identity element of the reduction operation
ReduceOp reduce, /// Reduces an accumulated value with a transformed element: f(ComputeType, ComputeType) => ComputeType
TransformOp transform, /// Transforms the tensor element to ComputeType: g(Element) => ComputeType
ComputeType *workspace, /// Device-side workspace for accumulating partial results. The reduced element is stored in workspace[0]
int workspace_size, /// Number of elements in workspace
cudaStream_t stream = nullptr, /// CUDA stream to launch into
bool copy_out = true /// If true, the value of workspace[0] is copied to host and returned. Otherwise, `identity` is returned.
) {
if (view_A.extent() != view_B.extent()) {
throw std::runtime_error("Extents must be equal.");
}
int const kBlockSize = 128;
dim3 block(kBlockSize, 1);
dim3 grid(workspace_size, 1);
kernel::TensorTransformReducePartial<
Element, Layout, ComputeType, ReduceOp, TransformOp, kBlockSize
><<< grid, block, 0, stream >>>(
view_A, view_B, identity, reduce, transform, workspace
);
int const kFinalizeBlockSize = 32;
kernel::TensorTransformReduceFinalize<
ComputeType, ReduceOp, kFinalizeBlockSize
><<< dim3(1, 1), dim3(kFinalizeBlockSize, 1), 0, stream >>>(
workspace, identity, workspace_size, reduce
);
if (copy_out) {
cudaError_t result = cudaMemcpy(&identity, workspace, sizeof(identity), cudaMemcpyDeviceToHost);
if (result != cudaSuccess) {
throw std::runtime_error("cudaMemcpy() failed");
}
}
return identity;
}
/// Transform-reduce operation over the elements of a tensor. This helper allocates the device-side
/// workspace
template <
typename Element,
typename Layout,
typename ComputeType,
typename ReduceOp,
typename TransformOp
>
ComputeType TensorTransformReduce(
TensorView<Element, Layout> view,
ComputeType identity,
ReduceOp reduce,
TransformOp transform,
cudaStream_t stream = nullptr,
int workspace_size = 0
) {
// Optionally query for the SM count to size the workspace.
if (!workspace_size) {
int device_idx = 0;
cudaDeviceProp prop;
cudaError_t result = cudaGetDevice(&device_idx);
if (result != cudaSuccess) {
throw std::runtime_error("cudaGetDevice() failed");
}
result = cudaGetDeviceProperties(&prop, device_idx);
if (result != cudaSuccess) {
throw std::runtime_error("cudaGetDeviceProp() failed");
}
workspace_size = int(prop.multiProcessorCount);
}
DeviceAllocation<ComputeType> workspace(workspace_size);
ComputeType output = TensorTransformReduce(
view,
identity,
reduce,
transform,
workspace.get(),
workspace_size,
stream,
true);
return output;
}
/// Transform-reduce operation over the elements of a tensor. This helper allocates the device-side
/// workspace
template <
typename Element,
typename Layout,
typename ComputeType,
typename ReduceOp,
typename TransformOp
>
ComputeType TensorTransformReduce(
TensorView<Element, Layout> view_A,
TensorView<Element, Layout> view_B,
ComputeType identity,
ReduceOp reduce,
TransformOp transform,
cudaStream_t stream = nullptr,
int workspace_size = 0
) {
// Optionally query for the SM count to size the workspace.
if (!workspace_size) {
int device_idx = 0;
cudaDeviceProp prop;
cudaError_t result = cudaGetDevice(&device_idx);
if (result != cudaSuccess) {
throw std::runtime_error("cudaGetDevice() failed");
}
result = cudaGetDeviceProperties(&prop, device_idx);
if (result != cudaSuccess) {
throw std::runtime_error("cudaGetDeviceProp() failed");
}
workspace_size = int(prop.multiProcessorCount);
}
DeviceAllocation<ComputeType> workspace(workspace_size);
ComputeType output = TensorTransformReduce(
view_A,
view_B,
identity,
reduce,
transform,
workspace.get(),
workspace_size,
stream,
true);
return output;
}
/////////////////////////////////////////////////////////////////////////////////////////////////
/// Helper to compute the sum of the elements of a tensor
template <
typename Element,
typename Layout,
typename ComputeType = Element
>
ComputeType TensorSum(
TensorView<Element, Layout> view,
ComputeType identity = ComputeType(),
cudaStream_t stream = nullptr,
int workspace_size = 0
) {
plus<ComputeType> reduce;
NumericConverter<ComputeType, Element> transform;
return TensorTransformReduce(
view, identity, reduce, transform, stream, workspace_size);
}
/// Helper to compute the sum of the squares of the elements of a tensor
template <
typename Element,
typename Layout,
typename ComputeType = Element
>
ComputeType TensorSumSq(
TensorView<Element, Layout> view,
ComputeType identity = ComputeType(),
cudaStream_t stream = nullptr,
int workspace_size = 0
) {
plus<ComputeType> reduce;
magnitude_squared<Element, ComputeType> transform;
return TensorTransformReduce(
view, identity, reduce, transform, stream, workspace_size);
}
/// Helper to compute the norm of the elements of a tensor.
template <
typename Element,
typename Layout,
typename ComputeType = double
>
ComputeType TensorNorm(
TensorView<Element, Layout> view,
ComputeType identity = ComputeType(),
cudaStream_t stream = nullptr,
int workspace_size = 0
) {
return std::sqrt(TensorSumSq(view, identity, stream, workspace_size));
}
/////////////////////////////////////////////////////////////////////////////////////////////////
/// Helper to compute the sum of the squares of the differences of two tensors
template <
typename Element,
typename Layout,
typename ComputeType = double
>
ComputeType TensorSumSqDiff(
TensorView<Element, Layout> view_A,
TensorView<Element, Layout> view_B,
ComputeType identity = ComputeType(),
cudaStream_t stream = nullptr,
int workspace_size = 0
) {
plus<ComputeType> reduce;
magnitude_squared_difference<Element, ComputeType> transform;
return TensorTransformReduce(
view_A, view_B, identity, reduce, transform, stream, workspace_size);
}
/// Helper to compute the norm of the tensor computed as the difference of two tensors in memory
template <
typename Element,
typename Layout,
typename ComputeType = double
>
ComputeType TensorNormDiff(
TensorView<Element, Layout> view_A,
TensorView<Element, Layout> view_B,
ComputeType identity = ComputeType(),
cudaStream_t stream = nullptr,
int workspace_size = 0
) {
return std::sqrt(TensorSumSqDiff(view_A, view_B, identity, stream, workspace_size));
}
/////////////////////////////////////////////////////////////////////////////////////////////////
} // namespace device
} // namespace reference
} // namespace cutlass
/////////////////////////////////////////////////////////////////////////////////////////////////
tools/util/include/cutlass/util/reference/device/thread/gemm.h
-1
View File
@@ -29,7 +29,6 @@
#pragma once
#include "cutlass/coord.h"
#include "cutlass/matrix_traits.h"
#include "cutlass/tensor_view.h"
#include "cutlass/gemm/gemm.h"
tools/util/include/cutlass/util/reference/host/gemm.h
-1
View File
@@ -33,7 +33,6 @@
#include "cutlass/functional.h"
#include "cutlass/numeric_conversion.h"
#include "cutlass/matrix_traits.h"
#include "cutlass/tensor_view.h"
#include "cutlass/gemm/gemm.h"
#include "cutlass/arch/mma.h"
tools/util/include/cutlass/util/reference/host/gemm_complex.h
+55 -40
View File
@@ -34,7 +34,6 @@
#include "cutlass/functional.h"
#include "cutlass/numeric_conversion.h"
#include "cutlass/matrix_traits.h"
#include "cutlass/tensor_view.h"
#include "cutlass/gemm/gemm.h"
@@ -73,7 +72,12 @@ void GemmComplex(
ScalarType beta,
TensorRef<ElementC, LayoutC> tensor_c,
TensorRef<ElementC, LayoutC> tensor_d,
ComputeType initial_accum) {
ComputeType initial_accum,
int batch_count = 1,
int64_t batch_stride_A = 0,
int64_t batch_stride_B = 0,
int64_t batch_stride_C = 0,
int64_t batch_stride_D = 0) {
static_assert(
LayoutA::kRank == 2 &&
@@ -92,61 +96,72 @@ void GemmComplex(
ConvertOp convert_op;
InnerProductOp inner_product_op;
for (int row_block = 0; row_block < M; row_block += Mblock) {
for (int col_block = 0; col_block < N; col_block += Nblock) {
for (int batch_idx = 0; batch_idx < batch_count; ++batch_idx) {
ComputeType accum[Mblock][Nblock];
// Compute matrix product using blocks
for (int row_block = 0; row_block < M; row_block += Mblock) {
for (int col_block = 0; col_block < N; col_block += Nblock) {
for (int j = 0; j < Nblock; j++) {
for (int i = 0; i < Mblock; i++) {
accum[i][j] = initial_accum;
ComputeType accum[Mblock][Nblock];
for (int j = 0; j < Nblock; j++) {
for (int i = 0; i < Mblock; i++) {
accum[i][j] = initial_accum;
}
}
for (int k_block = 0; k_block < K; ++k_block) {
for (int j = 0; j < Nblock; j++) {
for (int i = 0; i < Mblock; i++) {
int row = row_block + i;
int col = col_block + j;
if (row < M && col < N) {
ElementA a = tensor_a.at(MatrixCoord(row, k_block));
ElementB b = tensor_b.at(MatrixCoord(k_block, col));
ComputeType a_ik = ComputeType(a);
ComputeType b_kj = ComputeType(b);
if (transform_a == ComplexTransform::kConjugate) {
a_ik = conj(a_ik);
}
if (transform_b == ComplexTransform::kConjugate) {
b_kj = conj(b_kj);
}
accum[i][j] = inner_product_op(a_ik, b_kj, accum[i][j]);
}
}
}
}
}
for (int k_block = 0; k_block < K; ++k_block) {
for (int j = 0; j < Nblock; j++) {
for (int i = 0; i < Mblock; i++) {
int row = row_block + i;
int col = col_block + j;
MatrixCoord coord = MatrixCoord(row, col);
if (row < M && col < N) {
ElementA a = tensor_a.at(MatrixCoord(row, k_block));
ElementB b = tensor_b.at(MatrixCoord(k_block, col));
ComputeType a_ik = ComputeType(a);
ComputeType b_kj = ComputeType(b);
if (transform_a == ComplexTransform::kConjugate) {
a_ik = conj(a_ik);
}
if (transform_b == ComplexTransform::kConjugate) {
b_kj = conj(b_kj);
}
accum[i][j] = inner_product_op(a_ik, b_kj, accum[i][j]);
tensor_d.at(coord) = convert_op(
alpha * ScalarType(accum[i][j]) +
beta * ScalarType(tensor_c.at(coord)));
}
}
}
}
for (int j = 0; j < Nblock; j++) {
for (int i = 0; i < Mblock; i++) {
int row = row_block + i;
int col = col_block + j;
} // for (col_block)
} // for (row_block)
MatrixCoord coord = MatrixCoord(row, col);
tensor_a.add_pointer_offset(batch_stride_A);
tensor_b.add_pointer_offset(batch_stride_B);
tensor_c.add_pointer_offset(batch_stride_C);
tensor_d.add_pointer_offset(batch_stride_D);
if (row < M && col < N) {
tensor_d.at(coord) = convert_op(
alpha * ScalarType(accum[i][j]) +
beta * ScalarType(tensor_c.at(coord)));
}
}
}
}
}
} // for (batch_idx)
}
////////////////////////////////////////////////////////////////////////////////////////////////////
tools/util/include/cutlass/util/reference/host/gemm_planar_complex.h
-1
View File
@@ -35,7 +35,6 @@
#include "cutlass/numeric_conversion.h"
#include "cutlass/tensor_ref_planar_complex.h"
#include "cutlass/matrix_traits.h"
#include "cutlass/tensor_view.h"
#include "cutlass/gemm/gemm.h"
tools/util/include/cutlass/util/reference/host/tensor_fill.h
+135 -7
View File
@@ -38,6 +38,7 @@
#include "cutlass/complex.h"
#include "cutlass/array.h"
#include "cutlass/numeric_types.h"
#include "cutlass/subbyte_reference.h"
#include "cutlass/tensor_view.h"
#include "cutlass/tensor_view_planar_complex.h"
@@ -300,7 +301,6 @@ void TensorFillRandomGaussian(
}
///////////////////////////////////////////////////////////////////////////////////////////////////
/// Fills a tensor with random values with a Gaussian distribution.
template <
typename Element ///< Element type
@@ -319,11 +319,12 @@ void BlockFillRandomGaussian(
detail::RandomGaussianFunc<Element> random_func(seed, mean, stddev, bits);
for (size_t i = 0; i < capacity; ++i) {
ptr[i] = random_func();
ReferenceFactory<Element>::get(ptr, i) = random_func();
}
}
///////////////////////////////////////////////////////////////////////////////////////////////////
///////////////////////////////////////////////////////////////////////////////////////////////////
namespace detail {
@@ -510,7 +511,6 @@ void TensorFillRandomUniform(
}
///////////////////////////////////////////////////////////////////////////////////////////////////
/// Fills a tensor with random values with a uniform random distribution.
template <
typename Element ///< Element type
@@ -527,11 +527,10 @@ void BlockFillRandomUniform(
detail::RandomUniformFunc<Element> random_func(seed, max, min, bits);
for (size_t i = 0; i < capacity; ++i) {
ptr[i] = random_func();
ReferenceFactory<Element>::get(ptr, i) = random_func();
}
}
///////////////////////////////////////////////////////////////////////////////////////////////////
///////////////////////////////////////////////////////////////////////////////////////////////////
namespace detail {
@@ -879,6 +878,135 @@ void BlockFillRandom(
///////////////////////////////////////////////////////////////////////////////////////////////////
///////////////////////////////////////////////////////////////////////////////////////////////////
namespace detail {
template <typename Element>
struct RandomSparseMetaFunc {
uint64_t seed;
double range;
int MetaSizeInBits;
//
// Methods
//
RandomSparseMetaFunc(
uint64_t seed_ = 0,
int MetaSizeInBits_ = 2
):
seed(seed_), MetaSizeInBits(MetaSizeInBits_) {
std::srand((unsigned)seed);
if (MetaSizeInBits_ == 2) {
range = 6;
} else if (MetaSizeInBits_ == 4) {
range = 2;
}
}
/// Compute random value and update RNG state
Element operator()() const {
Element FourToTwoMeta[6] = {0x4, 0x8, 0x9, 0xc, 0xd, 0xe};
Element TwoToOneMeta[2] = {0x4, 0xe};
Element * MetaArray = (MetaSizeInBits == 2) ? FourToTwoMeta : TwoToOneMeta;
Element result = 0x0;
for (int i = 0; i < cutlass::sizeof_bits<Element>::value / 4; ++i) {
double rnd = double(std::rand()) / double(RAND_MAX);
rnd = range * rnd;
Element meta = MetaArray[(int)rnd];
result = (Element)(result | ((Element)(meta << (i * 4))));
}
return result;
}
};
/// Computes a random sparse meta
template <
typename Element, ///< Element type
typename Layout> ///< Layout function
struct TensorFillRandomSparseMetaFunc {
using TensorView = TensorView<Element, Layout>;
//
// Data members
//
TensorView view;
RandomSparseMetaFunc<Element> func;
//
// Methods
//
/// Construction of Gaussian RNG functor.
TensorFillRandomSparseMetaFunc(
TensorView view_ = TensorView(),
RandomSparseMetaFunc<Element> func_ = RandomSparseMetaFunc<Element>()
):
view(view_), func(func_) {
}
/// Compute random value and update RNG state
void operator()(Coord<Layout::kRank> const &coord) const {
view.at(coord) = func();
}
};
} // namespace detail
///////////////////////////////////////////////////////////////////////////////////////////////////
/// Fills a tensor with random values with a uniform random distribution.
template <
typename Element, ///< Element type
typename Layout> ///< Layout function
void TensorFillRandomSparseMeta(
TensorView<Element, Layout> dst, ///< destination tensor
uint64_t seed, ///< seed for RNG
int MetaSizeInBits) { ///< 2 bit or 4 bit
detail::RandomSparseMetaFunc<Element> random_func(seed, MetaSizeInBits);
detail::TensorFillRandomSparseMetaFunc<Element, Layout> func(
dst,
random_func
);
TensorForEach(
dst.extent(),
func
);
}
///////////////////////////////////////////////////////////////////////////////////////////////////
/// Fills a tensor with random values with a uniform random distribution.
template <
typename Element ///< Element type
>
void BlockFillRandomSparseMeta(
Element *ptr,
size_t capacity,
uint64_t seed, ///< seed for RNG
int MetaSizeInBits) { ///< 2 bit or 4bit
detail::RandomSparseMetaFunc<Element> random_func(seed, MetaSizeInBits);
for (size_t i = 0; i < capacity; ++i) {
ptr[i] = random_func();
}
}
///////////////////////////////////////////////////////////////////////////////////////////////////
/// Copies a diagonal in from host memory without modifying off-diagonal elements.
template <
typename Element, ///< Element type
@@ -891,7 +1019,7 @@ void TensorCopyDiagonalIn(
for (typename Layout::Index i = 0; i < extent; ++i) {
Coord<Layout::kRank> coord(i);
dst.at(coord) = ptr[i];
dst.at(coord) = ReferenceFactory<Element>::get(ptr, i);
}
}
@@ -910,7 +1038,7 @@ void TensorCopyDiagonalOut(
for (typename Layout::Index i = 0; i < extent; ++i) {
Coord<Layout::kRank> coord(i);
ptr[i] = src.at(coord);
ReferenceFactory<Element>::get(ptr, i) = src.at(coord);
}
}
tools/util/include/cutlass/util/reference/host/tensor_norm.h
+2 -42
View File
@@ -24,53 +24,13 @@
**************************************************************************************************/
#pragma once
#include <cmath>
#include "cutlass/cutlass.h"
#include "cutlass/complex.h"
#include "cutlass/tensor_ref.h"
#include "cutlass/util/reference/host/tensor_foreach.h"
// The contents of this file have been moved to 'tensor_reduce' to cover other types of reductions.
namespace cutlass {
namespace reference {
namespace host {
#include "cutlass/util/reference/host/tensor_reduce.h"
///////////////////////////////////////////////////////////////////////////////////////////////////
/// Computes the p=2 norm of the elements of a tensor with arbitrary reduction data type.
template <
typename Element,
typename Layout,
typename ElementReduction
>
ElementReduction TensorNorm(
TensorView<Element, Layout> view,
ElementReduction accumulator) {
TensorForEachLambda(
view.extent(),
[&](typename Layout::TensorCoord const & coord) {
Element element = Element(view.at(coord));
accumulator = cutlass::norm_accumulate(element, accumulator);
});
return std::sqrt(accumulator);
}
/// Computes the p=2 norm of the elements of a tensor.
template <
typename Element,
typename Layout
>
double TensorNorm(TensorView<Element, Layout> view) {
return TensorNorm<Element, Layout, double>(view, 0);
}
///////////////////////////////////////////////////////////////////////////////////////////////////
} // namespace host
} // namespace reference
} // namespace cutlass
///////////////////////////////////////////////////////////////////////////////////////////////////
tools/util/include/cutlass/util/reference/host/tensor_reduce.h
+197
View File
@@ -0,0 +1,197 @@
/***************************************************************************************************
* Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without modification, are permitted
* provided that the following conditions are met:
* * Redistributions of source code must retain the above copyright notice, this list of
* conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright notice, this list of
* conditions and the following disclaimer in the documentation and/or other materials
* provided with the distribution.
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
* to endorse or promote products derived from this software without specific prior written
* permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
**************************************************************************************************/
#pragma once
#include <cmath>
#include "cutlass/cutlass.h"
#include "cutlass/complex.h"
#include "cutlass/tensor_ref.h"
#include "cutlass/util/reference/detail/linear_to_coordinate.h"
#include "cutlass/core_io.h"
namespace cutlass {
namespace reference {
namespace host {
///////////////////////////////////////////////////////////////////////////////////////////////////
/// Transform-reduce operation over the elements of a tensor. This helper allocates the device-side
/// workspace
template <
typename Element,
typename Layout,
typename ComputeType,
typename ReduceOp,
typename TransformOp
>
ComputeType TensorTransformReduce(
TensorView<Element, Layout> view,
ComputeType identity,
ReduceOp reduce,
TransformOp transform
) {
for (int64_t idx = 0; idx < view.size(); ++idx) {
typename Layout::TensorCoord coord;
cutlass::reference::detail::LinearToCoordinate<Layout::kRank>()(coord, idx, view.extent());
if (view.contains(coord)) {
Element x = view.at(coord);
identity = reduce(identity, transform(x));
}
}
return identity;
}
/// Transform-reduce operation over the elements of a tensor. This helper allocates the device-side
/// workspace
template <
typename Element,
typename Layout,
typename ComputeType,
typename ReduceOp,
typename TransformOp
>
ComputeType TensorTransformReduce(
TensorView<Element, Layout> view_A,
TensorView<Element, Layout> view_B,
ComputeType identity,
ReduceOp reduce,
TransformOp transform) {
if (view_A.extent() != view_B.extent()) {
throw std::runtime_error("Tensor extents must match.");
}
for (int64_t idx = 0; idx < view_A.size(); ++idx) {
typename Layout::TensorCoord coord;
cutlass::reference::detail::LinearToCoordinate<Layout::kRank>()(coord, idx, view_A.extent());
if (view_A.contains(coord)) {
Element a = view_A.at(coord);
Element b = view_B.at(coord);
identity = reduce(identity, transform(a, b));
}
}
return identity;
}
/// Helper to compute the sum of the elements of a tensor
template <
typename Element,
typename Layout,
typename ComputeType = Element
>
ComputeType TensorSum(
TensorView<Element, Layout> view,
ComputeType identity = ComputeType()
) {
plus<ComputeType> reduce;
NumericConverter<ComputeType, Element> transform;
return TensorTransformReduce(
view, identity, reduce, transform);
}
/// Helper to compute the sum of the squares of the elements of a tensor
template <
typename Element,
typename Layout,
typename ComputeType = Element
>
ComputeType TensorSumSq(
TensorView<Element, Layout> view,
ComputeType identity = ComputeType()
) {
plus<ComputeType> reduce;
magnitude_squared<Element, ComputeType> transform;
return TensorTransformReduce(
view, identity, reduce, transform);
}
/// Helper to compute the norm of the elements of a tensor.
template <
typename Element,
typename Layout,
typename ComputeType = double
>
ComputeType TensorNorm(
TensorView<Element, Layout> view,
ComputeType identity = ComputeType()
) {
return std::sqrt(TensorSumSq(view, identity));
}
/// Helper to compute the sum of the squares of the differences of two tensors
template <
typename Element,
typename Layout,
typename ComputeType = double
>
ComputeType TensorSumSqDiff(
TensorView<Element, Layout> view_A,
TensorView<Element, Layout> view_B,
ComputeType identity = ComputeType()
) {
plus<ComputeType> reduce;
magnitude_squared_difference<Element, ComputeType> transform;
return TensorTransformReduce(
view_A, view_B, identity, reduce, transform);
}
/// Helper to compute the norm of the tensor computed as the difference of two tensors in memory
template <
typename Element,
typename Layout,
typename ComputeType = double
>
ComputeType TensorNormDiff(
TensorView<Element, Layout> view_A,
TensorView<Element, Layout> view_B,
ComputeType identity = ComputeType()
) {
return std::sqrt(TensorSumSqDiff(view_A, view_B, identity));
}
///////////////////////////////////////////////////////////////////////////////////////////////////
} // namespace host
} // namespace reference
} // namespace cutlass
///////////////////////////////////////////////////////////////////////////////////////////////////