CUTLASS 1.2
This commit is contained in:
akerr
127
tools/util/reference/detail/inner_product.h
Normal file
127
tools/util/reference/detail/inner_product.h
Normal file
@@ -0,0 +1,127 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/*! \file
|
||||
\brief Reference implementation for GEMM in host-side code.
|
||||
*/
|
||||
#pragma once
|
||||
|
||||
#include "cutlass/cutlass.h"
|
||||
#include "cutlass/vector.h"
|
||||
|
||||
namespace cutlass {
|
||||
namespace reference {
|
||||
namespace detail {
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Template function to compute an inner product.
|
||||
#pragma hd_warning_disable // Suppresses warnings when attempting to instantiate with a
|
||||
// host-only type
|
||||
template <typename Atype, typename Btype, typename Ctype>
|
||||
CUTLASS_HOST_DEVICE
|
||||
Ctype inner_product(Atype a, Btype b, Ctype c) {
|
||||
return Ctype(a) * Ctype(b) + c;
|
||||
}
|
||||
|
||||
/// Specialization for matrix multiplication with binary operands
|
||||
template <>
|
||||
CUTLASS_HOST_DEVICE
|
||||
int inner_product<Vector<bin1_t, 32>, Vector<bin1_t, 32>, int>(
|
||||
Vector<bin1_t, 32> a,
|
||||
Vector<bin1_t, 32> b,
|
||||
int c) {
|
||||
|
||||
int accum = 0;
|
||||
for (int bit = 0; bit < 32; bit++) {
|
||||
accum += a[bit] ^ b[bit];
|
||||
}
|
||||
return accum + c;
|
||||
}
|
||||
|
||||
/// Specialization for matrix multiplication with signed 4-bit integer operands
|
||||
template <>
|
||||
CUTLASS_HOST_DEVICE
|
||||
int inner_product<Vector<int4_t, 8>, Vector<int4_t, 8>, int>(
|
||||
Vector<int4_t, 8> a,
|
||||
Vector<int4_t, 8> b,
|
||||
int c) {
|
||||
|
||||
int accum = 0;
|
||||
for (int k = 0; k < 8; k++) {
|
||||
accum += a[k] * b[k];
|
||||
}
|
||||
return accum + c;
|
||||
}
|
||||
|
||||
/// Specialization for matrix multiplication with unsigned 4-bit integer operands
|
||||
template <>
|
||||
CUTLASS_HOST_DEVICE
|
||||
int inner_product<Vector<uint4_t, 8>, Vector<uint4_t, 8>, int>(
|
||||
Vector<uint4_t, 8> a,
|
||||
Vector<uint4_t, 8> b,
|
||||
int c) {
|
||||
|
||||
int accum = 0;
|
||||
for (int k = 0; k < 8; k++) {
|
||||
accum += a[k] * b[k];
|
||||
}
|
||||
return accum + c;
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
template <typename SrcType, typename DstType>
|
||||
struct Cast {
|
||||
// Default behavior: convert to the destination type
|
||||
#pragma hd_warning_disable // Suppresses warnings when attempting to instantiate complex<T> with a
|
||||
// host-only type
|
||||
CUTLASS_HOST_DEVICE
|
||||
static DstType apply(SrcType src) { return static_cast<DstType>(src); };
|
||||
};
|
||||
|
||||
template <>
|
||||
struct Cast<float, int8_t> {
|
||||
CUTLASS_HOST_DEVICE
|
||||
static int8_t apply(float src) {
|
||||
// Clamp to the range of signed 8-bit integers.
|
||||
return static_cast<int8_t>(fmaxf(-128.f, fminf(127.f, src)));
|
||||
};
|
||||
};
|
||||
|
||||
template <>
|
||||
struct Cast<float, uint8_t> {
|
||||
CUTLASS_HOST_DEVICE
|
||||
static uint8_t apply(float src) {
|
||||
// Clamp to the range of signed 8-bit integers.
|
||||
return static_cast<uint8_t>(fmaxf(0.f, fminf(255.f, src)));
|
||||
};
|
||||
};
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
} // namespace detail
|
||||
} // namespace reference
|
||||
} // namespace cutlass
|
||||
|
||||
224
tools/util/reference/device/gemm.h
Normal file
224
tools/util/reference/device/gemm.h
Normal file
@@ -0,0 +1,224 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/*! \file
|
||||
\brief Reference implementation for GEMM in device-side code.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "cutlass/coord.h"
|
||||
#include "cutlass/matrix_traits.h"
|
||||
#include "cutlass/tensor_view.h"
|
||||
#include "cutlass/gemm/gemm_coord.h"
|
||||
|
||||
#include "tools/util/reference/device/kernel/gemm.h"
|
||||
|
||||
namespace cutlass {
|
||||
namespace reference {
|
||||
namespace device {
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Computes a general matrix product among matrices (tensors of rank=2) pointed to by TensorRef
|
||||
/// objects.
|
||||
///
|
||||
/// Explicitly naming types needed by this template can be cumbersome, particularly for the
|
||||
/// accumulator type, so a function argument 'initial_accum' is exposed. Passing
|
||||
/// AccumulatorType(0) as the last function argument can be easier than naming all template
|
||||
/// arguments explicitly.
|
||||
template <
|
||||
typename TensorRefA,
|
||||
typename TensorRefB,
|
||||
typename TensorRefC,
|
||||
typename ScalarType,
|
||||
typename AccumulatorType
|
||||
>
|
||||
void Gemm(
|
||||
gemm::GemmCoord problem_size,
|
||||
ScalarType alpha,
|
||||
TensorRefA tensor_a,
|
||||
TensorRefB tensor_b,
|
||||
ScalarType beta,
|
||||
TensorRefC tensor_c,
|
||||
AccumulatorType initial_accum) {
|
||||
|
||||
typedef typename TensorRefA::Storage AType;
|
||||
typedef typename TensorRefB::Storage BType;
|
||||
typedef typename TensorRefC::Storage CType;
|
||||
|
||||
static_assert(
|
||||
TensorRefA::kRank == 2 &&
|
||||
TensorRefB::kRank == 2 &&
|
||||
TensorRefC::kRank == 2, "Tensors must be of rank 2");
|
||||
|
||||
// Blocking structure potentially improves performance of reference implementation
|
||||
// with a minor increase in complexity.
|
||||
//
|
||||
// Note, this reference implementation is NOT expected to approach peak performance.
|
||||
typedef Shape<1, 4, 4> OutputTile;
|
||||
|
||||
dim3 block(16, 8);
|
||||
dim3 grid(
|
||||
(problem_size.m() + block.x * OutputTile::kW - 1) / (block.x * OutputTile::kW),
|
||||
(problem_size.n() + block.y * OutputTile::kH - 1) / (block.y * OutputTile::kH)
|
||||
);
|
||||
|
||||
// Launch a GEMM kernel
|
||||
kernel::Gemm<
|
||||
TensorRefA,
|
||||
TensorRefB,
|
||||
TensorRefC,
|
||||
ScalarType,
|
||||
AccumulatorType,
|
||||
OutputTile
|
||||
><<< grid, block >>>(
|
||||
problem_size,
|
||||
alpha,
|
||||
tensor_a,
|
||||
tensor_b,
|
||||
beta,
|
||||
tensor_c,
|
||||
initial_accum
|
||||
);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Computes a general matrix product among matrices (tensors of rank=2) pointed to by TensorRef
|
||||
/// objects.
|
||||
///
|
||||
/// This assumes the accumulator type is the same type as the scalars.
|
||||
template <
|
||||
typename TensorRefA,
|
||||
typename TensorRefB,
|
||||
typename TensorRefC,
|
||||
typename ScalarType
|
||||
>
|
||||
void Gemm(
|
||||
gemm::GemmCoord problem_size,
|
||||
ScalarType alpha,
|
||||
TensorRefA tensor_a,
|
||||
TensorRefB tensor_b,
|
||||
ScalarType beta,
|
||||
TensorRefC tensor_c) {
|
||||
|
||||
Gemm(problem_size, alpha, tensor_a, tensor_b, beta, tensor_c, ScalarType(0));
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
//
|
||||
// Batched GEMM
|
||||
//
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Computes a batch of GEMMs over a set of matrices of common dimension.
|
||||
//
|
||||
// TensorRefCollection* is a type satisfying the TensorRefCollection concept.
|
||||
//
|
||||
template <
|
||||
typename TensorRefCollectionA,
|
||||
typename TensorRefCollectionB,
|
||||
typename TensorRefCollectionC,
|
||||
typename ScalarType,
|
||||
typename AccumulatorType
|
||||
>
|
||||
void BatchedGemm(
|
||||
gemm::GemmCoord problem_size,
|
||||
ScalarType alpha,
|
||||
TensorRefCollectionA tensor_a,
|
||||
TensorRefCollectionB tensor_b,
|
||||
ScalarType beta,
|
||||
TensorRefCollectionC tensor_c,
|
||||
AccumulatorType initial_accum) {
|
||||
|
||||
typedef typename TensorRefCollectionA::Storage AType;
|
||||
typedef typename TensorRefCollectionB::Storage BType;
|
||||
typedef typename TensorRefCollectionC::Storage CType;
|
||||
|
||||
static_assert(
|
||||
TensorRefCollectionA::kRank == 2 &&
|
||||
TensorRefCollectionB::kRank == 2 &&
|
||||
TensorRefCollectionC::kRank == 2, "Tensors must be of rank 2");
|
||||
|
||||
// Blocking structure potentially improves performance of reference implementation
|
||||
// with a minor increase in complexity.
|
||||
//
|
||||
// Note, this reference implementation is NOT expected to approach peak performance.
|
||||
typedef Shape<1, 4, 4> OutputTile;
|
||||
|
||||
dim3 block(16, 8);
|
||||
dim3 grid(
|
||||
(problem_size.m() + block.x * OutputTile::kW - 1) / (block.x * OutputTile::kW),
|
||||
(problem_size.n() + block.y * OutputTile::kH - 1) / (block.y * OutputTile::kH),
|
||||
problem_size.batch()
|
||||
);
|
||||
|
||||
// Launch a GEMM kernel
|
||||
kernel::BatchedGemm<
|
||||
TensorRefCollectionA,
|
||||
TensorRefCollectionB,
|
||||
TensorRefCollectionC,
|
||||
ScalarType,
|
||||
AccumulatorType,
|
||||
OutputTile
|
||||
><<< grid, block >>>(
|
||||
problem_size,
|
||||
alpha,
|
||||
tensor_a,
|
||||
tensor_b,
|
||||
beta,
|
||||
tensor_c,
|
||||
initial_accum
|
||||
);
|
||||
}
|
||||
|
||||
/// Computes a general matrix product among matrices (tensors of rank=2) pointed to by TensorRef
|
||||
/// objects.
|
||||
//
|
||||
// TensorRefCollection* is a type satisfying the TensorRefCollection concept.
|
||||
//
|
||||
template <
|
||||
typename TensorRefCollectionA,
|
||||
typename TensorRefCollectionB,
|
||||
typename TensorRefCollectionC,
|
||||
typename ScalarType,
|
||||
typename AccumulatorType
|
||||
>
|
||||
void BatchedGemm(
|
||||
gemm::GemmCoord problem_size,
|
||||
ScalarType alpha,
|
||||
TensorRefCollectionA tensor_a,
|
||||
TensorRefCollectionB tensor_b,
|
||||
ScalarType beta,
|
||||
TensorRefCollectionC tensor_c) {
|
||||
|
||||
BatchedGemm(problem_size, alpha, tensor_a, tensor_b, beta, tensor_c, ScalarType(0));
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
} // namespace host
|
||||
} // namespace reference
|
||||
} // namespace cutlass
|
||||
148
tools/util/reference/device/kernel/gemm.h
Normal file
148
tools/util/reference/device/kernel/gemm.h
Normal file
@@ -0,0 +1,148 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/*! \file
|
||||
\brief Reference implementation for GEMM in host-side code.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "cutlass/coord.h"
|
||||
#include "cutlass/matrix_traits.h"
|
||||
#include "cutlass/tensor_view.h"
|
||||
#include "cutlass/gemm/gemm_coord.h"
|
||||
|
||||
#include "tools/util/reference/device/thread/gemm.h"
|
||||
|
||||
namespace cutlass {
|
||||
namespace reference {
|
||||
namespace device {
|
||||
namespace kernel {
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Computes a general matrix product among matrices (tensors of rank=2) pointed to by TensorRef
|
||||
/// objects.
|
||||
template <
|
||||
typename TensorRefA,
|
||||
typename TensorRefB,
|
||||
typename TensorRefC,
|
||||
typename ScalarType,
|
||||
typename AccumulatorType,
|
||||
typename OutputTile
|
||||
>
|
||||
__global__ void Gemm(
|
||||
gemm::GemmCoord problem_size,
|
||||
ScalarType alpha,
|
||||
TensorRefA tensor_a,
|
||||
TensorRefB tensor_b,
|
||||
ScalarType beta,
|
||||
TensorRefC tensor_c,
|
||||
AccumulatorType initial_accum) {
|
||||
|
||||
// Map each thread to a unique tile of the output matrix
|
||||
MatrixCoord output_coord(
|
||||
(threadIdx.x + blockIdx.x * blockDim.x) * OutputTile::kW,
|
||||
(threadIdx.y + blockIdx.y * blockDim.y) * OutputTile::kH
|
||||
);
|
||||
|
||||
// Compute the general matrix product
|
||||
thread::Gemm<
|
||||
TensorRefA,
|
||||
TensorRefB,
|
||||
TensorRefC,
|
||||
ScalarType,
|
||||
AccumulatorType,
|
||||
OutputTile
|
||||
> gemm(initial_accum);
|
||||
|
||||
gemm.multiply_add(
|
||||
problem_size,
|
||||
tensor_a,
|
||||
tensor_b,
|
||||
output_coord);
|
||||
|
||||
gemm.epilogue(problem_size, alpha, beta, tensor_c, output_coord);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Computes a general matrix product among matrices (tensors of rank=2) pointed to by TensorRef
|
||||
/// objects.
|
||||
template <
|
||||
typename TensorRefCollectionA,
|
||||
typename TensorRefCollectionB,
|
||||
typename TensorRefCollectionC,
|
||||
typename ScalarType,
|
||||
typename AccumulatorType,
|
||||
typename OutputTile
|
||||
>
|
||||
__global__ void BatchedGemm(
|
||||
gemm::GemmCoord problem_size,
|
||||
ScalarType alpha,
|
||||
TensorRefCollectionA tensor_collection_a,
|
||||
TensorRefCollectionB tensor_collection_b,
|
||||
ScalarType beta,
|
||||
TensorRefCollectionC tensor_collection_c,
|
||||
AccumulatorType initial_accum) {
|
||||
|
||||
// Obtain batch ID
|
||||
int batch_id = blockIdx.z;
|
||||
|
||||
// Dereference based on batch_id
|
||||
typename TensorRefCollectionA::TensorRef tensor_a = tensor_collection_a.at(batch_id);
|
||||
typename TensorRefCollectionB::TensorRef tensor_b = tensor_collection_b.at(batch_id);
|
||||
typename TensorRefCollectionC::TensorRef tensor_c = tensor_collection_c.at(batch_id);
|
||||
|
||||
// Map each thread to a unique tile of the output matrix
|
||||
MatrixCoord output_coord(
|
||||
(threadIdx.x + blockIdx.x * blockDim.x) * OutputTile::kW,
|
||||
(threadIdx.y + blockIdx.y * blockDim.y) * OutputTile::kH
|
||||
);
|
||||
|
||||
// Compute the general matrix product
|
||||
thread::Gemm<
|
||||
typename TensorRefCollectionA::TensorRef,
|
||||
typename TensorRefCollectionB::TensorRef,
|
||||
typename TensorRefCollectionC::TensorRef,
|
||||
ScalarType,
|
||||
AccumulatorType,
|
||||
OutputTile
|
||||
> gemm(initial_accum);
|
||||
|
||||
gemm.multiply_add(
|
||||
problem_size,
|
||||
tensor_a,
|
||||
tensor_b,
|
||||
output_coord);
|
||||
|
||||
gemm.epilogue(problem_size, alpha, beta, tensor_c, output_coord);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
} // namespace kernel
|
||||
} // namespace device
|
||||
} // namespace reference
|
||||
} // namespace cutlass
|
||||
95
tools/util/reference/device/kernel/split_complex_gemm.h
Normal file
95
tools/util/reference/device/kernel/split_complex_gemm.h
Normal file
@@ -0,0 +1,95 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/*! \file
|
||||
\brief Reference implementation for GEMM in host-side code.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "cutlass/coord.h"
|
||||
#include "cutlass/matrix_traits.h"
|
||||
#include "cutlass/tensor_view.h"
|
||||
#include "cutlass/gemm/gemm_coord.h"
|
||||
#include "cutlass/util/complex.h"
|
||||
|
||||
#include "tools/util/reference/device/thread/split_complex_gemm.h"
|
||||
|
||||
namespace cutlass {
|
||||
namespace reference {
|
||||
namespace device {
|
||||
namespace kernel {
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Computes a general matrix product among matrices (tensors of rank=2) pointed to by TensorRef
|
||||
/// objects.
|
||||
template <
|
||||
typename TensorRefA, /// concept: ZipTensorRef
|
||||
typename TensorRefB, /// concept: ZipTensorRef
|
||||
typename TensorRefC, /// concept: ZipTensorRef
|
||||
typename ScalarType, /// real-valued type underlying complex scalars
|
||||
typename AccumulatorType, /// real-valued type underlying complex accumulators
|
||||
typename OutputTile /// concept: Shape
|
||||
>
|
||||
__global__ void SplitComplexGemm(
|
||||
gemm::GemmCoord problem_size,
|
||||
platform::complex<ScalarType> alpha,
|
||||
TensorRefA tensor_a,
|
||||
TensorRefB tensor_b,
|
||||
platform::complex<ScalarType> beta,
|
||||
TensorRefC tensor_c,
|
||||
platform::complex<AccumulatorType> initial_accum) {
|
||||
|
||||
// Map each thread to a unique tile of the output matrix
|
||||
MatrixCoord output_coord(
|
||||
(threadIdx.x + blockIdx.x * blockDim.x) * OutputTile::kW,
|
||||
(threadIdx.y + blockIdx.y * blockDim.y) * OutputTile::kH
|
||||
);
|
||||
|
||||
// Compute the general matrix product
|
||||
thread::Gemm<
|
||||
TensorRefA,
|
||||
TensorRefB,
|
||||
TensorRefC,
|
||||
ScalarType,
|
||||
AccumulatorType,
|
||||
OutputTile
|
||||
> gemm(initial_accum);
|
||||
|
||||
gemm.multiply_add(
|
||||
problem_size,
|
||||
tensor_a,
|
||||
tensor_b,
|
||||
output_coord);
|
||||
|
||||
gemm.epilogue(problem_size, alpha, beta, tensor_c, output_coord);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
} // namespace kernel
|
||||
} // namespace device
|
||||
} // namespace reference
|
||||
} // namespace cutlass
|
||||
103
tools/util/reference/device/split_complex_gemm.h
Normal file
103
tools/util/reference/device/split_complex_gemm.h
Normal file
@@ -0,0 +1,103 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/*! \file
|
||||
\brief Reference implementation for GEMM in device-side code.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "cutlass/coord.h"
|
||||
#include "cutlass/matrix_traits.h"
|
||||
#include "cutlass/tensor_view.h"
|
||||
#include "cutlass/gemm/gemm_coord.h"
|
||||
#include "cutlass/util/complex.h"
|
||||
|
||||
#include "tools/util/reference/device/kernel/gemm.h"
|
||||
|
||||
namespace cutlass {
|
||||
namespace reference {
|
||||
namespace device {
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Computes a complex-valued GEMM whose operands are in the split-complex format.
|
||||
template <
|
||||
typename TensorRefA, /// concept: ZipTensorRef
|
||||
typename TensorRefB, /// concept: ZipTensorRef
|
||||
typename TensorRefC, /// concept: ZipTensorRef
|
||||
typename ScalarType, /// real-valued type underlying complex scalars
|
||||
typename AccumulatorType /// real-valued type underlying complex accumulators
|
||||
>
|
||||
void SplitComplexGemm(
|
||||
gemm::GemmCoord problem_size,
|
||||
platform::complex<ScalarType> alpha,
|
||||
TensorRefA tensor_a,
|
||||
TensorRefB tensor_b,
|
||||
platform::complex<ScalarType> beta,
|
||||
TensorRefC tensor_c,
|
||||
platform::complex<ScalarType> initial_accum) {
|
||||
|
||||
static_assert(
|
||||
TensorRefA::First::kRank == 2 && TensorRefA::Second::kRank == 2 &&
|
||||
TensorRefB::First::kRank == 2 && TensorRefB::Second::kRank == 2 &&
|
||||
TensorRefC::First::kRank == 2 && TensorRefC::Second::kRank == 2,
|
||||
"Tensors must be of rank 2");
|
||||
|
||||
// Blocking structure potentially improves performance of reference implementation
|
||||
// with a minor increase in complexity.
|
||||
//
|
||||
// Note, this reference implementation is NOT expected to approach peak performance.
|
||||
typedef Shape<1, 4, 4> OutputTile;
|
||||
|
||||
dim3 block(16, 8);
|
||||
dim3 grid(
|
||||
(problem_size.m() + block.x * OutputTile::kW - 1) / (block.x * OutputTile::kW),
|
||||
(problem_size.n() + block.y * OutputTile::kH - 1) / (block.y * OutputTile::kH)
|
||||
);
|
||||
|
||||
// Launch a GEMM kernel
|
||||
kernel::SplitComplexGemm<
|
||||
TensorRefA,
|
||||
TensorRefB,
|
||||
TensorRefC,
|
||||
ScalarType,
|
||||
AccumulatorType,
|
||||
OutputTile
|
||||
><<< grid, block >>>(
|
||||
problem_size,
|
||||
alpha,
|
||||
tensor_a,
|
||||
tensor_b,
|
||||
beta,
|
||||
tensor_c,
|
||||
initial_accum
|
||||
);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
} // namespace device
|
||||
} // namespace reference
|
||||
} // namespace cutlass
|
||||
176
tools/util/reference/device/thread/gemm.h
Normal file
176
tools/util/reference/device/thread/gemm.h
Normal file
@@ -0,0 +1,176 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/*! \file
|
||||
\brief Reference implementation for GEMM in host-side code.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "cutlass/coord.h"
|
||||
#include "cutlass/matrix_traits.h"
|
||||
#include "cutlass/tensor_view.h"
|
||||
#include "cutlass/gemm/gemm_coord.h"
|
||||
|
||||
#include "tools/util/reference/detail/inner_product.h"
|
||||
|
||||
namespace cutlass {
|
||||
namespace reference {
|
||||
namespace device {
|
||||
namespace thread {
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Thread-level blocked general matrix product.
|
||||
//
|
||||
// Note, this is a reference implementation. Performance is not expected to approach peak.
|
||||
//
|
||||
template <
|
||||
typename TensorRefA,
|
||||
typename TensorRefB,
|
||||
typename TensorRefC,
|
||||
typename ScalarType,
|
||||
typename AccumulatorType,
|
||||
typename OutputTile
|
||||
>
|
||||
struct Gemm {
|
||||
|
||||
typedef typename TensorRefA::Storage ScalarA;
|
||||
typedef typename TensorRefB::Storage ScalarB;
|
||||
typedef typename TensorRefC::Storage ScalarC;
|
||||
|
||||
//
|
||||
// Data members
|
||||
//
|
||||
|
||||
/// Tile for A operand
|
||||
ScalarA A_tile[OutputTile::kW];
|
||||
|
||||
/// Tile for B operand
|
||||
ScalarB B_tile[OutputTile::kH];
|
||||
|
||||
/// Tile for Accumulator
|
||||
AccumulatorType accum[OutputTile::kH][OutputTile::kW];
|
||||
|
||||
//
|
||||
// Methods
|
||||
//
|
||||
|
||||
/// Constructor
|
||||
CUTLASS_HOST_DEVICE
|
||||
Gemm(AccumulatorType initial_accum = AccumulatorType(0)) {
|
||||
|
||||
// Clear fetch registers
|
||||
for (int i = 0; i < OutputTile::kW; ++i) {
|
||||
A_tile[i] = ScalarA(0);
|
||||
}
|
||||
|
||||
for (int j = 0; j < OutputTile::kW; ++j) {
|
||||
B_tile[j] = ScalarB(0);
|
||||
}
|
||||
|
||||
// Clear accumulators
|
||||
CUTLASS_PRAGMA_UNROLL
|
||||
for (int j = 0; j < OutputTile::kH; ++j) {
|
||||
CUTLASS_PRAGMA_UNROLL
|
||||
for (int i = 0; i < OutputTile::kW; ++i) {
|
||||
accum[j][i] = initial_accum;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Computes a matrix product
|
||||
CUTLASS_HOST_DEVICE
|
||||
Gemm & multiply_add(
|
||||
gemm::GemmCoord problem_size,
|
||||
TensorRefA tensor_a,
|
||||
TensorRefB tensor_b,
|
||||
MatrixCoord output_coord = MatrixCoord()) {
|
||||
|
||||
// Loop over the GEMM K dimension
|
||||
CUTLASS_PRAGMA_NO_UNROLL
|
||||
for (int k = 0; k < problem_size.k(); ++k) {
|
||||
|
||||
// Fetch a slice of the A matrix
|
||||
CUTLASS_PRAGMA_UNROLL
|
||||
for (int i = 0; i < OutputTile::kW; ++i) {
|
||||
if (output_coord.row() + i < problem_size.m()) {
|
||||
A_tile[i] = tensor_a.at(make_Coord(output_coord.row() + i, k));
|
||||
}
|
||||
}
|
||||
|
||||
// Fetch a slice of the B matrix
|
||||
CUTLASS_PRAGMA_UNROLL
|
||||
for (int j = 0; j < OutputTile::kH; ++j) {
|
||||
if (output_coord.column() + j < problem_size.n()) {
|
||||
B_tile[j] = tensor_b.at(make_Coord(k, output_coord.column() + j));
|
||||
}
|
||||
}
|
||||
|
||||
// Compute an accumulated matrix product
|
||||
CUTLASS_PRAGMA_UNROLL
|
||||
for (int j = 0; j < OutputTile::kH; ++j) {
|
||||
CUTLASS_PRAGMA_UNROLL
|
||||
for (int i = 0; i < OutputTile::kW; ++i) {
|
||||
accum[j][i] = detail::inner_product(A_tile[i], B_tile[j], accum[j][i]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return *this;
|
||||
}
|
||||
|
||||
/// Performs linear scaling of matrix product and updates output tensor
|
||||
CUTLASS_HOST_DEVICE
|
||||
Gemm & epilogue(
|
||||
gemm::GemmCoord problem_size,
|
||||
ScalarType alpha,
|
||||
ScalarType beta,
|
||||
TensorRefC tensor_c,
|
||||
MatrixCoord output_coord = MatrixCoord()) {
|
||||
|
||||
// Update the output tensor
|
||||
for (int j = 0; j < OutputTile::kH; ++j) {
|
||||
for (int i = 0; i < OutputTile::kW; ++i) {
|
||||
MatrixCoord coord = output_coord + MatrixCoord(i, j);
|
||||
if (coord.row() < problem_size.m() && coord.column() < problem_size.n()) {
|
||||
|
||||
tensor_c.at(coord) = detail::Cast<ScalarType, ScalarC>::apply(
|
||||
alpha * ScalarType(accum[j][i]) +
|
||||
beta * ScalarType(tensor_c.at(coord))
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return *this;
|
||||
}
|
||||
};
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
} // namespace thread
|
||||
} // namespace device
|
||||
} // namespace reference
|
||||
} // namespace cutlass
|
||||
192
tools/util/reference/device/thread/split_complex_gemm.h
Normal file
192
tools/util/reference/device/thread/split_complex_gemm.h
Normal file
@@ -0,0 +1,192 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/*! \file
|
||||
\brief Reference implementation for GEMM in host-side code.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "cutlass/coord.h"
|
||||
#include "cutlass/matrix_traits.h"
|
||||
#include "cutlass/tensor_view.h"
|
||||
#include "cutlass/gemm/gemm_coord.h"
|
||||
|
||||
#include "tools/util/reference/detail/inner_product.h"
|
||||
|
||||
namespace cutlass {
|
||||
namespace reference {
|
||||
namespace device {
|
||||
namespace thread {
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Thread-level blocked general matrix product.
|
||||
//
|
||||
// Note, this is a reference implementation. Performance is not expected to approach peak.
|
||||
//
|
||||
template <
|
||||
typename TensorRefA, /// concept: ZipTensorRef
|
||||
typename TensorRefB, /// concept: ZipTensorRef
|
||||
typename TensorRefC, /// concept: ZipTensorRef
|
||||
typename ScalarType, /// real-valued type underlying complex scalars
|
||||
typename AccumulatorType, /// real-valued type underlying complex accumulators
|
||||
typename OutputTile /// concept: Shape
|
||||
>
|
||||
struct SplitComplexGemm {
|
||||
|
||||
typedef typename TensorRefA::First::Storage RealScalarA;
|
||||
typedef typename TensorRefB::First::Storage RealScalarB;
|
||||
typedef typename TensorRefC::First::Storage RealScalarC;
|
||||
|
||||
typedef platform::complex<RealScalarA> ScalarA;
|
||||
typedef platform::complex<RealScalarB> ScalarB;
|
||||
typedef platform::complex<AccumulatorType> ComplexAccumulator;
|
||||
typedef platform::complex<ScalarType> ComplexScalar;
|
||||
|
||||
//
|
||||
// Data members
|
||||
//
|
||||
|
||||
/// Tile for A operand
|
||||
ScalarA A_tile[OutputTile::kW];
|
||||
|
||||
/// Tile for B operand
|
||||
ScalarB B_tile[OutputTile::kH];
|
||||
|
||||
/// Tile for Accumulator
|
||||
ComplexAccumulator accum[OutputTile::kH][OutputTile::kW];
|
||||
|
||||
//
|
||||
// Methods
|
||||
//
|
||||
|
||||
/// Constructor
|
||||
CUTLASS_HOST_DEVICE
|
||||
Gemm(ComplexAccumulator initial_accum = AccumulatorType(0)) {
|
||||
|
||||
// Clear fetch registers
|
||||
for (int i = 0; i < OutputTile::kW; ++i) {
|
||||
A_tile[i] = ScalarA(0);
|
||||
}
|
||||
|
||||
for (int j = 0; j < OutputTile::kW; ++j) {
|
||||
B_tile[j] = ScalarB(0);
|
||||
}
|
||||
|
||||
// Clear accumulators
|
||||
CUTLASS_PRAGMA_UNROLL
|
||||
for (int j = 0; j < OutputTile::kH; ++j) {
|
||||
CUTLASS_PRAGMA_UNROLL
|
||||
for (int i = 0; i < OutputTile::kW; ++i) {
|
||||
accum[j][i] = initial_accum;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Computes a matrix product
|
||||
CUTLASS_HOST_DEVICE
|
||||
Gemm & multiply_add(
|
||||
gemm::GemmCoord problem_size,
|
||||
TensorRefA tensor_a,
|
||||
TensorRefB tensor_b,
|
||||
MatrixCoord output_coord = MatrixCoord()) {
|
||||
|
||||
// Loop over the GEMM K dimension
|
||||
CUTLASS_PRAGMA_NO_UNROLL
|
||||
for (int k = 0; k < problem_size.k(); ++k) {
|
||||
|
||||
// Fetch a slice of the A matrix - zip into complex values
|
||||
CUTLASS_PRAGMA_UNROLL
|
||||
for (int i = 0; i < OutputTile::kW; ++i) {
|
||||
if (output_coord.row() + i < problem_size.m()) {
|
||||
MatrixCoord coord(output_coord.row() + i, k);
|
||||
A_tile[i].real() = tensor_a.first.at(coord);
|
||||
A_tile[i].imag() = tensor_a.second.at(coord);
|
||||
}
|
||||
}
|
||||
|
||||
// Fetch a slice of the B matrix - zip into complex values
|
||||
CUTLASS_PRAGMA_UNROLL
|
||||
for (int j = 0; j < OutputTile::kH; ++j) {
|
||||
if (output_coord.column() + j < problem_size.n()) {
|
||||
MatrixCoord coord(k, output_coord.column() + j);
|
||||
B_tile[j].real() = tensor_b.first.at(coord);
|
||||
B_tile[j].imag() = tensor_b.second.at(coord);
|
||||
}
|
||||
}
|
||||
|
||||
// Compute an accumulated matrix product on complex values
|
||||
CUTLASS_PRAGMA_UNROLL
|
||||
for (int j = 0; j < OutputTile::kH; ++j) {
|
||||
CUTLASS_PRAGMA_UNROLL
|
||||
for (int i = 0; i < OutputTile::kW; ++i) {
|
||||
accum[j][i] = detail::inner_product(A_tile[i], B_tile[j], accum[j][i]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return *this;
|
||||
}
|
||||
|
||||
/// Performs linear scaling of matrix product and updates output tensor
|
||||
CUTLASS_HOST_DEVICE
|
||||
Gemm & epilogue(
|
||||
gemm::GemmCoord problem_size,
|
||||
ComplexScalar alpha,
|
||||
ComplexScalar beta,
|
||||
TensorRefC tensor_c,
|
||||
MatrixCoord output_coord = MatrixCoord()) {
|
||||
|
||||
// Update the output tensor
|
||||
for (int j = 0; j < OutputTile::kH; ++j) {
|
||||
for (int i = 0; i < OutputTile::kW; ++i) {
|
||||
MatrixCoord coord = output_coord + MatrixCoord(i, j);
|
||||
if (coord < problem_size.mn()) {
|
||||
|
||||
ComplexScalar source(
|
||||
tensor_c.first.at(coord),
|
||||
tensor_c.second.at(coord)
|
||||
);
|
||||
|
||||
// Final calculation is performed in data type of scalars
|
||||
ComplexScalar result = alpha * ComplexScalar(accum[j][i].real(), accum[j][i].imag()) + beta * source;
|
||||
|
||||
// Unzip and convert into output tensor data type
|
||||
tensor_c.first.at(coord) = detail::Cast<ScalarType, RealScalarC>::apply(result.real());
|
||||
tensor_c.second.at(coord) = detail::Cast<ScalarType, RealScalarC>::apply(result.imag());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return *this;
|
||||
}
|
||||
};
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
} // namespace thread
|
||||
} // namespace device
|
||||
} // namespace reference
|
||||
} // namespace cutlass
|
||||
@@ -33,90 +33,14 @@
|
||||
#include "cutlass/tensor_view.h"
|
||||
#include "cutlass/gemm/gemm_coord.h"
|
||||
|
||||
#include "tools/util/reference/detail/inner_product.h"
|
||||
|
||||
namespace cutlass {
|
||||
namespace reference {
|
||||
namespace host {
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
namespace detail {
|
||||
|
||||
/// Template function to compute an inner product.
|
||||
template <typename Atype, typename Btype, typename Ctype>
|
||||
Ctype inner_product(Atype a, Btype b, Ctype c) {
|
||||
return Ctype(a) * Ctype(b) + c;
|
||||
}
|
||||
|
||||
/// Specialization for matrix multiplication with binary operands
|
||||
template <>
|
||||
inline int inner_product<Vector<bin1_t, 32>, Vector<bin1_t, 32>, int>(
|
||||
Vector<bin1_t, 32> a,
|
||||
Vector<bin1_t, 32> b,
|
||||
int c) {
|
||||
|
||||
int accum = 0;
|
||||
for (int bit = 0; bit < 32; bit++) {
|
||||
accum += a[bit] ^ b[bit];
|
||||
}
|
||||
return accum + c;
|
||||
}
|
||||
|
||||
/// Specialization for matrix multiplication with signed 4-bit integer operands
|
||||
template <> inline
|
||||
int inner_product<Vector<int4_t, 8>, Vector<int4_t, 8>, int>(
|
||||
Vector<int4_t, 8> a,
|
||||
Vector<int4_t, 8> b,
|
||||
int c) {
|
||||
|
||||
int accum = 0;
|
||||
for (int k = 0; k < 8; k++) {
|
||||
accum += a[k] * b[k];
|
||||
}
|
||||
return accum + c;
|
||||
}
|
||||
|
||||
/// Specialization for matrix multiplication with unsigned 4-bit integer operands
|
||||
template <> inline
|
||||
int inner_product<Vector<uint4_t, 8>, Vector<uint4_t, 8>, int>(
|
||||
Vector<uint4_t, 8> a,
|
||||
Vector<uint4_t, 8> b,
|
||||
int c) {
|
||||
|
||||
int accum = 0;
|
||||
for (int k = 0; k < 8; k++) {
|
||||
accum += a[k] * b[k];
|
||||
}
|
||||
return accum + c;
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
template <typename SrcType, typename DstType>
|
||||
struct Cast {
|
||||
// Default behavior: convert to the destination type
|
||||
static inline DstType apply(SrcType src) { return static_cast<DstType>(src); };
|
||||
};
|
||||
|
||||
template <>
|
||||
struct Cast<float, int8_t> {
|
||||
static inline int8_t apply(float src) {
|
||||
// Clamp to the range of signed 8-bit integers.
|
||||
return static_cast<int8_t>(fmaxf(-128.f, fminf(127.f, src)));
|
||||
};
|
||||
};
|
||||
|
||||
template <>
|
||||
struct Cast<float, uint8_t> {
|
||||
static inline uint8_t apply(float src) {
|
||||
// Clamp to the range of signed 8-bit integers.
|
||||
return static_cast<uint8_t>(fmaxf(0.f, fminf(255.f, src)));
|
||||
};
|
||||
};
|
||||
|
||||
} // namespace detail
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Computes a general matrix product among matrices (tensors of rank=2) pointed to by TensorRef
|
||||
/// objects.
|
||||
///
|
||||
@@ -178,7 +102,7 @@ void Gemm(
|
||||
AType a = tensor_a.at(MatrixCoord(row, k_block));
|
||||
BType b = tensor_b.at(MatrixCoord(k_block, col));
|
||||
|
||||
accum[i][j] = detail::inner_product(a, b, accum[i][j]);
|
||||
accum[i][j] = cutlass::reference::detail::inner_product(a, b, accum[i][j]);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -192,7 +116,7 @@ void Gemm(
|
||||
MatrixCoord coord = MatrixCoord(row, col);
|
||||
if (row < M && col < N) {
|
||||
|
||||
tensor_c.at(coord) = detail::Cast<ScalarType, CType>::apply(
|
||||
tensor_c.at(coord) = cutlass::reference::detail::Cast<ScalarType, CType>::apply(
|
||||
alpha * ScalarType(accum[i][j]) +
|
||||
beta * ScalarType(tensor_c.at(coord)));
|
||||
}
|
||||
@@ -225,9 +149,16 @@ void Gemm(
|
||||
Gemm(problem_size, alpha, tensor_a, tensor_b, beta, tensor_c, ScalarType(0));
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
//
|
||||
// Batched GEMM
|
||||
//
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Computes a batch of GEMMs over a set of matrices of common dimension.
|
||||
//
|
||||
// TensorRefCollection* is a type satisfying the TensorRefCollection concept.
|
||||
//
|
||||
template <
|
||||
typename TensorRefCollectionA,
|
||||
typename TensorRefCollectionB,
|
||||
@@ -235,14 +166,14 @@ template <
|
||||
typename ScalarType,
|
||||
typename AccumulatorType
|
||||
>
|
||||
void BatchGemm(
|
||||
void BatchedGemm(
|
||||
gemm::GemmCoord problem_size,
|
||||
ScalarType alpha,
|
||||
TensorRefCollectionA const& tensor_a,
|
||||
TensorRefCollectionB const& tensor_b,
|
||||
ScalarType beta,
|
||||
TensorRefCollectionC &tensor_c,
|
||||
AccumulatorType initial_accum = AccumulatorType(0)) {
|
||||
AccumulatorType initial_accum) {
|
||||
|
||||
typename TensorRefCollectionA::ConstIterator tensor_a_it = tensor_a.begin();
|
||||
typename TensorRefCollectionB::ConstIterator tensor_b_it = tensor_b.begin();
|
||||
@@ -263,6 +194,29 @@ void BatchGemm(
|
||||
}
|
||||
}
|
||||
|
||||
/// Computes a general matrix product among matrices (tensors of rank=2) pointed to by TensorRef
|
||||
/// objects.
|
||||
//
|
||||
// TensorRefCollection* is a type satisfying the TensorRefCollection concept.
|
||||
//
|
||||
template <
|
||||
typename TensorRefCollectionA,
|
||||
typename TensorRefCollectionB,
|
||||
typename TensorRefCollectionC,
|
||||
typename ScalarType,
|
||||
typename AccumulatorType
|
||||
>
|
||||
void BatchedGemm(
|
||||
gemm::GemmCoord problem_size,
|
||||
ScalarType alpha,
|
||||
TensorRefCollectionA const& tensor_a,
|
||||
TensorRefCollectionB const& tensor_b,
|
||||
ScalarType beta,
|
||||
TensorRefCollectionC &tensor_c) {
|
||||
|
||||
BatchedGemm(problem_size, alpha, tensor_a, tensor_b, beta, tensor_c, ScalarType(0));
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
} // namespace host
|
||||
|
||||
254
tools/util/reference/host/split_complex_gemm.h
Normal file
254
tools/util/reference/host/split_complex_gemm.h
Normal file
@@ -0,0 +1,254 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/*! \file
|
||||
\brief Reference implementation for split-complex GEMM in device-side code.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "cutlass/coord.h"
|
||||
#include "cutlass/matrix_traits.h"
|
||||
#include "cutlass/tensor_view.h"
|
||||
#include "cutlass/gemm/gemm_coord.h"
|
||||
#include "cutlass/util/complex.h"
|
||||
|
||||
namespace cutlass {
|
||||
namespace reference {
|
||||
namespace host {
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Computes a complex-valued GEMM whose operands are in the split-complex format.
|
||||
template <
|
||||
typename TensorRefA, /// concept: ZipTensorRef
|
||||
typename TensorRefB, /// concept: ZipTensorRef
|
||||
typename TensorRefC, /// concept: ZipTensorRef
|
||||
typename ScalarType, /// real-valued type underlying complex scalars
|
||||
typename AccumulatorType /// real-valued type underlying complex accumulators
|
||||
>
|
||||
void SplitComplexGemm(
|
||||
gemm::GemmCoord problem_size,
|
||||
platform::complex<ScalarType> alpha,
|
||||
TensorRefA tensor_a,
|
||||
TensorRefB tensor_b,
|
||||
platform::complex<ScalarType> beta,
|
||||
TensorRefC tensor_c,
|
||||
platform::complex<AccumulatorType> initial_accum) {
|
||||
|
||||
typedef typename TensorRefA::First::Storage AType;
|
||||
typedef typename TensorRefB::First::Storage BType;
|
||||
typedef typename TensorRefC::First::Storage CType;
|
||||
|
||||
typedef platform::complex<AType> ComplexAType;
|
||||
typedef platform::complex<BType> ComplexBType;
|
||||
typedef platform::complex<CType> ComplexCType;
|
||||
typedef platform::complex<ScalarType> ComplexScalarType;
|
||||
typedef platform::complex<AccumulatorType> ComplexAccumulatorType;
|
||||
|
||||
static_assert(
|
||||
TensorRefA::First::kRank == 2 && TensorRefA::Second::kRank == 2 &&
|
||||
TensorRefB::First::kRank == 2 && TensorRefB::Second::kRank == 2 &&
|
||||
TensorRefC::First::kRank == 2 && TensorRefC::Second::kRank == 2,
|
||||
"Tensors must be of rank 2");
|
||||
|
||||
// Note: batch is ignored.
|
||||
int const M = problem_size.m();
|
||||
int const N = problem_size.n();
|
||||
int const K = problem_size.k();
|
||||
|
||||
// Blocking necessary to speedup reference implementation
|
||||
int const Mblock = 32;
|
||||
int const Nblock = 32;
|
||||
|
||||
for (int row_block = 0; row_block < M; row_block += Mblock) {
|
||||
for (int col_block = 0; col_block < N; col_block += Nblock) {
|
||||
|
||||
ComplexAccumulatorType accum[Mblock][Nblock];
|
||||
|
||||
for (int j = 0; j < Nblock; j++) {
|
||||
for (int i = 0; i < Mblock; i++) {
|
||||
accum[i][j] = initial_accum;
|
||||
}
|
||||
}
|
||||
|
||||
for (int k_block = 0; k_block < K; ++k_block) {
|
||||
for (int j = 0; j < Nblock; j++) {
|
||||
for (int i = 0; i < Mblock; i++) {
|
||||
int row = row_block + i;
|
||||
int col = col_block + j;
|
||||
|
||||
if (row < M && col < N) {
|
||||
|
||||
ComplexAType a(
|
||||
tensor_a.first.at(MatrixCoord(row, k_block)),
|
||||
tensor_a.second.at(MatrixCoord(row, k_block))
|
||||
);
|
||||
|
||||
ComplexBType b(
|
||||
tensor_b.first.at(MatrixCoord(k_block, col)),
|
||||
tensor_b.second.at(MatrixCoord(k_block, col))
|
||||
);
|
||||
|
||||
accum[i][j] = detail::inner_product(a, b, accum[i][j]);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (int j = 0; j < Nblock; j++) {
|
||||
for (int i = 0; i < Mblock; i++) {
|
||||
int row = row_block + i;
|
||||
int col = col_block + j;
|
||||
|
||||
MatrixCoord coord = MatrixCoord(row, col);
|
||||
if (row < M && col < N) {
|
||||
|
||||
ComplexScalarType product(
|
||||
detail::Cast<AccumulatorType, ScalarType>::apply(accum[i][j].real()),
|
||||
detail::Cast<AccumulatorType, ScalarType>::apply(accum[i][j].imag())
|
||||
);
|
||||
|
||||
ComplexScalarType source(
|
||||
detail::Cast<CType, ScalarType>::apply(tensor_c.first.at(coord)),
|
||||
detail::Cast<CType, ScalarType>::apply(tensor_c.second.at(coord))
|
||||
);
|
||||
|
||||
ComplexScalarType result = alpha * product + beta * source;
|
||||
|
||||
tensor_c.first.at(coord) = detail::Cast<ScalarType, CType>::apply(result.real());
|
||||
tensor_c.second.at(coord) = detail::Cast<ScalarType, CType>::apply(result.imag());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Computes a complex-valued GEMM whose operands are in the split-complex format.
|
||||
template <
|
||||
typename TensorRefA, /// concept: ZipTensorRef
|
||||
typename TensorRefB, /// concept: ZipTensorRef
|
||||
typename TensorRefC, /// concept: ZipTensorRef
|
||||
typename ScalarType, /// real-valued type underlying complex scalars
|
||||
typename AccumulatorType /// real-valued type underlying complex accumulators
|
||||
>
|
||||
void SplitComplexGemm(
|
||||
gemm::GemmCoord problem_size,
|
||||
platform::complex<ScalarType> alpha,
|
||||
TensorRefA tensor_a,
|
||||
TensorRefB tensor_b,
|
||||
platform::complex<ScalarType> beta,
|
||||
TensorRefC tensor_c) {
|
||||
|
||||
return SplitComplexGemm(problem_size, alpha, tensor_a, tensor_b,beta, tensor_c, ScalarType(0));
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
//
|
||||
// Batched Split-Complex GEMM
|
||||
//
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Computes a complex-valued GEMM whose operands are in the split-complex format.
|
||||
template <
|
||||
typename TensorRefCollectionA, /// concept: Pair<TensorRefCollection, TensorRefCollection>
|
||||
typename TensorRefCollectionB, /// concept: Pair<TensorRefCollection, TensorRefCollection>
|
||||
typename TensorRefCollectionC, /// concept: Pair<TensorRefCollection, TensorRefCollection>
|
||||
typename ScalarType, /// real-valued type underlying complex scalars
|
||||
typename AccumulatorType /// real-valued type underlying complex accumulators
|
||||
>
|
||||
void BatchedSplitComplexGemm(
|
||||
gemm::GemmCoord problem_size,
|
||||
platform::complex<ScalarType> alpha,
|
||||
TensorRefCollectionA tensor_a,
|
||||
TensorRefCollectionB tensor_b,
|
||||
platform::complex<ScalarType> beta,
|
||||
TensorRefCollectionC tensor_c,
|
||||
platform::complex<AccumulatorType> initial_accum) {
|
||||
|
||||
typename TensorRefCollectionA::ConstIterator tensor_a_real = tensor_a.first.begin();
|
||||
typename TensorRefCollectionA::ConstIterator tensor_a_imag = tensor_a.second.begin();
|
||||
|
||||
typename TensorRefCollectionB::ConstIterator tensor_b_real = tensor_b.first.begin();
|
||||
typename TensorRefCollectionB::ConstIterator tensor_b_imag = tensor_b.second.begin();
|
||||
|
||||
typename TensorRefCollectionC::ConstIterator tensor_c_real = tensor_c.first.begin();
|
||||
typename TensorRefCollectionC::ConstIterator tensor_c_imag = tensor_c.second.begin();
|
||||
|
||||
for (int batch = 0; batch < problem_size.batch(); ++batch) {
|
||||
|
||||
SplitComplexGemm(
|
||||
problem_size,
|
||||
alpha,
|
||||
make_ZipTensorRef(*tensor_a_real, *tensor_a_imag),
|
||||
make_ZipTensorRef(*tensor_b_real, *tensor_b_imag),
|
||||
beta,
|
||||
make_ZipTensorRef(*tensor_c_real, *tensor_c_imag),
|
||||
initial_accum);
|
||||
|
||||
++tensor_a_real;
|
||||
++tensor_a_imag;
|
||||
++tensor_b_real;
|
||||
++tensor_b_imag;
|
||||
++tensor_c_real;
|
||||
++tensor_c_imag;
|
||||
}
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Computes a complex-valued GEMM whose operands are in the split-complex format.
|
||||
template <
|
||||
typename TensorRefCollectionA, /// concept: pair<TensorRefCollection, TensorRefCollection>
|
||||
typename TensorRefCollectionB, /// concept: pair<TensorRefCollection, TensorRefCollection>
|
||||
typename TensorRefCollectionC, /// concept: pair<TensorRefCollection, TensorRefCollection>
|
||||
typename ScalarType, /// real-valued type underlying complex scalars
|
||||
typename AccumulatorType /// real-valued type underlying complex accumulators
|
||||
>
|
||||
void BatchedSplitComplexGemm(
|
||||
gemm::GemmCoord problem_size,
|
||||
platform::complex<ScalarType> alpha,
|
||||
TensorRefCollectionA tensor_a,
|
||||
TensorRefCollectionB tensor_b,
|
||||
platform::complex<ScalarType> beta,
|
||||
TensorRefCollectionC tensor_c) {
|
||||
|
||||
BatchedSplitComplexGemm(
|
||||
problem_size,
|
||||
alpha,
|
||||
tensor_a,
|
||||
tensor_b,
|
||||
beta,
|
||||
tensor_c,
|
||||
platform::complex<ScalarType>(0, 0));
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
} // namespace host
|
||||
} // namespace reference
|
||||
} // namespace cutlass
|
||||
Reference in New Issue
Block a user