CUTLASS 1.2
This commit is contained in:
akerr
@@ -0,0 +1,224 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/*! \file
|
||||
\brief Reference implementation for GEMM in device-side code.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "cutlass/coord.h"
|
||||
#include "cutlass/matrix_traits.h"
|
||||
#include "cutlass/tensor_view.h"
|
||||
#include "cutlass/gemm/gemm_coord.h"
|
||||
|
||||
#include "tools/util/reference/device/kernel/gemm.h"
|
||||
|
||||
namespace cutlass {
|
||||
namespace reference {
|
||||
namespace device {
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Computes a general matrix product among matrices (tensors of rank=2) pointed to by TensorRef
|
||||
/// objects.
|
||||
///
|
||||
/// Explicitly naming types needed by this template can be cumbersome, particularly for the
|
||||
/// accumulator type, so a function argument 'initial_accum' is exposed. Passing
|
||||
/// AccumulatorType(0) as the last function argument can be easier than naming all template
|
||||
/// arguments explicitly.
|
||||
template <
|
||||
typename TensorRefA,
|
||||
typename TensorRefB,
|
||||
typename TensorRefC,
|
||||
typename ScalarType,
|
||||
typename AccumulatorType
|
||||
>
|
||||
void Gemm(
|
||||
gemm::GemmCoord problem_size,
|
||||
ScalarType alpha,
|
||||
TensorRefA tensor_a,
|
||||
TensorRefB tensor_b,
|
||||
ScalarType beta,
|
||||
TensorRefC tensor_c,
|
||||
AccumulatorType initial_accum) {
|
||||
|
||||
typedef typename TensorRefA::Storage AType;
|
||||
typedef typename TensorRefB::Storage BType;
|
||||
typedef typename TensorRefC::Storage CType;
|
||||
|
||||
static_assert(
|
||||
TensorRefA::kRank == 2 &&
|
||||
TensorRefB::kRank == 2 &&
|
||||
TensorRefC::kRank == 2, "Tensors must be of rank 2");
|
||||
|
||||
// Blocking structure potentially improves performance of reference implementation
|
||||
// with a minor increase in complexity.
|
||||
//
|
||||
// Note, this reference implementation is NOT expected to approach peak performance.
|
||||
typedef Shape<1, 4, 4> OutputTile;
|
||||
|
||||
dim3 block(16, 8);
|
||||
dim3 grid(
|
||||
(problem_size.m() + block.x * OutputTile::kW - 1) / (block.x * OutputTile::kW),
|
||||
(problem_size.n() + block.y * OutputTile::kH - 1) / (block.y * OutputTile::kH)
|
||||
);
|
||||
|
||||
// Launch a GEMM kernel
|
||||
kernel::Gemm<
|
||||
TensorRefA,
|
||||
TensorRefB,
|
||||
TensorRefC,
|
||||
ScalarType,
|
||||
AccumulatorType,
|
||||
OutputTile
|
||||
><<< grid, block >>>(
|
||||
problem_size,
|
||||
alpha,
|
||||
tensor_a,
|
||||
tensor_b,
|
||||
beta,
|
||||
tensor_c,
|
||||
initial_accum
|
||||
);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Computes a general matrix product among matrices (tensors of rank=2) pointed to by TensorRef
|
||||
/// objects.
|
||||
///
|
||||
/// This assumes the accumulator type is the same type as the scalars.
|
||||
template <
|
||||
typename TensorRefA,
|
||||
typename TensorRefB,
|
||||
typename TensorRefC,
|
||||
typename ScalarType
|
||||
>
|
||||
void Gemm(
|
||||
gemm::GemmCoord problem_size,
|
||||
ScalarType alpha,
|
||||
TensorRefA tensor_a,
|
||||
TensorRefB tensor_b,
|
||||
ScalarType beta,
|
||||
TensorRefC tensor_c) {
|
||||
|
||||
Gemm(problem_size, alpha, tensor_a, tensor_b, beta, tensor_c, ScalarType(0));
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
//
|
||||
// Batched GEMM
|
||||
//
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Computes a batch of GEMMs over a set of matrices of common dimension.
|
||||
//
|
||||
// TensorRefCollection* is a type satisfying the TensorRefCollection concept.
|
||||
//
|
||||
template <
|
||||
typename TensorRefCollectionA,
|
||||
typename TensorRefCollectionB,
|
||||
typename TensorRefCollectionC,
|
||||
typename ScalarType,
|
||||
typename AccumulatorType
|
||||
>
|
||||
void BatchedGemm(
|
||||
gemm::GemmCoord problem_size,
|
||||
ScalarType alpha,
|
||||
TensorRefCollectionA tensor_a,
|
||||
TensorRefCollectionB tensor_b,
|
||||
ScalarType beta,
|
||||
TensorRefCollectionC tensor_c,
|
||||
AccumulatorType initial_accum) {
|
||||
|
||||
typedef typename TensorRefCollectionA::Storage AType;
|
||||
typedef typename TensorRefCollectionB::Storage BType;
|
||||
typedef typename TensorRefCollectionC::Storage CType;
|
||||
|
||||
static_assert(
|
||||
TensorRefCollectionA::kRank == 2 &&
|
||||
TensorRefCollectionB::kRank == 2 &&
|
||||
TensorRefCollectionC::kRank == 2, "Tensors must be of rank 2");
|
||||
|
||||
// Blocking structure potentially improves performance of reference implementation
|
||||
// with a minor increase in complexity.
|
||||
//
|
||||
// Note, this reference implementation is NOT expected to approach peak performance.
|
||||
typedef Shape<1, 4, 4> OutputTile;
|
||||
|
||||
dim3 block(16, 8);
|
||||
dim3 grid(
|
||||
(problem_size.m() + block.x * OutputTile::kW - 1) / (block.x * OutputTile::kW),
|
||||
(problem_size.n() + block.y * OutputTile::kH - 1) / (block.y * OutputTile::kH),
|
||||
problem_size.batch()
|
||||
);
|
||||
|
||||
// Launch a GEMM kernel
|
||||
kernel::BatchedGemm<
|
||||
TensorRefCollectionA,
|
||||
TensorRefCollectionB,
|
||||
TensorRefCollectionC,
|
||||
ScalarType,
|
||||
AccumulatorType,
|
||||
OutputTile
|
||||
><<< grid, block >>>(
|
||||
problem_size,
|
||||
alpha,
|
||||
tensor_a,
|
||||
tensor_b,
|
||||
beta,
|
||||
tensor_c,
|
||||
initial_accum
|
||||
);
|
||||
}
|
||||
|
||||
/// Computes a general matrix product among matrices (tensors of rank=2) pointed to by TensorRef
|
||||
/// objects.
|
||||
//
|
||||
// TensorRefCollection* is a type satisfying the TensorRefCollection concept.
|
||||
//
|
||||
template <
|
||||
typename TensorRefCollectionA,
|
||||
typename TensorRefCollectionB,
|
||||
typename TensorRefCollectionC,
|
||||
typename ScalarType,
|
||||
typename AccumulatorType
|
||||
>
|
||||
void BatchedGemm(
|
||||
gemm::GemmCoord problem_size,
|
||||
ScalarType alpha,
|
||||
TensorRefCollectionA tensor_a,
|
||||
TensorRefCollectionB tensor_b,
|
||||
ScalarType beta,
|
||||
TensorRefCollectionC tensor_c) {
|
||||
|
||||
BatchedGemm(problem_size, alpha, tensor_a, tensor_b, beta, tensor_c, ScalarType(0));
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
} // namespace host
|
||||
} // namespace reference
|
||||
} // namespace cutlass
|
||||
@@ -0,0 +1,148 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/*! \file
|
||||
\brief Reference implementation for GEMM in host-side code.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "cutlass/coord.h"
|
||||
#include "cutlass/matrix_traits.h"
|
||||
#include "cutlass/tensor_view.h"
|
||||
#include "cutlass/gemm/gemm_coord.h"
|
||||
|
||||
#include "tools/util/reference/device/thread/gemm.h"
|
||||
|
||||
namespace cutlass {
|
||||
namespace reference {
|
||||
namespace device {
|
||||
namespace kernel {
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Computes a general matrix product among matrices (tensors of rank=2) pointed to by TensorRef
|
||||
/// objects.
|
||||
template <
|
||||
typename TensorRefA,
|
||||
typename TensorRefB,
|
||||
typename TensorRefC,
|
||||
typename ScalarType,
|
||||
typename AccumulatorType,
|
||||
typename OutputTile
|
||||
>
|
||||
__global__ void Gemm(
|
||||
gemm::GemmCoord problem_size,
|
||||
ScalarType alpha,
|
||||
TensorRefA tensor_a,
|
||||
TensorRefB tensor_b,
|
||||
ScalarType beta,
|
||||
TensorRefC tensor_c,
|
||||
AccumulatorType initial_accum) {
|
||||
|
||||
// Map each thread to a unique tile of the output matrix
|
||||
MatrixCoord output_coord(
|
||||
(threadIdx.x + blockIdx.x * blockDim.x) * OutputTile::kW,
|
||||
(threadIdx.y + blockIdx.y * blockDim.y) * OutputTile::kH
|
||||
);
|
||||
|
||||
// Compute the general matrix product
|
||||
thread::Gemm<
|
||||
TensorRefA,
|
||||
TensorRefB,
|
||||
TensorRefC,
|
||||
ScalarType,
|
||||
AccumulatorType,
|
||||
OutputTile
|
||||
> gemm(initial_accum);
|
||||
|
||||
gemm.multiply_add(
|
||||
problem_size,
|
||||
tensor_a,
|
||||
tensor_b,
|
||||
output_coord);
|
||||
|
||||
gemm.epilogue(problem_size, alpha, beta, tensor_c, output_coord);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Computes a general matrix product among matrices (tensors of rank=2) pointed to by TensorRef
|
||||
/// objects.
|
||||
template <
|
||||
typename TensorRefCollectionA,
|
||||
typename TensorRefCollectionB,
|
||||
typename TensorRefCollectionC,
|
||||
typename ScalarType,
|
||||
typename AccumulatorType,
|
||||
typename OutputTile
|
||||
>
|
||||
__global__ void BatchedGemm(
|
||||
gemm::GemmCoord problem_size,
|
||||
ScalarType alpha,
|
||||
TensorRefCollectionA tensor_collection_a,
|
||||
TensorRefCollectionB tensor_collection_b,
|
||||
ScalarType beta,
|
||||
TensorRefCollectionC tensor_collection_c,
|
||||
AccumulatorType initial_accum) {
|
||||
|
||||
// Obtain batch ID
|
||||
int batch_id = blockIdx.z;
|
||||
|
||||
// Dereference based on batch_id
|
||||
typename TensorRefCollectionA::TensorRef tensor_a = tensor_collection_a.at(batch_id);
|
||||
typename TensorRefCollectionB::TensorRef tensor_b = tensor_collection_b.at(batch_id);
|
||||
typename TensorRefCollectionC::TensorRef tensor_c = tensor_collection_c.at(batch_id);
|
||||
|
||||
// Map each thread to a unique tile of the output matrix
|
||||
MatrixCoord output_coord(
|
||||
(threadIdx.x + blockIdx.x * blockDim.x) * OutputTile::kW,
|
||||
(threadIdx.y + blockIdx.y * blockDim.y) * OutputTile::kH
|
||||
);
|
||||
|
||||
// Compute the general matrix product
|
||||
thread::Gemm<
|
||||
typename TensorRefCollectionA::TensorRef,
|
||||
typename TensorRefCollectionB::TensorRef,
|
||||
typename TensorRefCollectionC::TensorRef,
|
||||
ScalarType,
|
||||
AccumulatorType,
|
||||
OutputTile
|
||||
> gemm(initial_accum);
|
||||
|
||||
gemm.multiply_add(
|
||||
problem_size,
|
||||
tensor_a,
|
||||
tensor_b,
|
||||
output_coord);
|
||||
|
||||
gemm.epilogue(problem_size, alpha, beta, tensor_c, output_coord);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
} // namespace kernel
|
||||
} // namespace device
|
||||
} // namespace reference
|
||||
} // namespace cutlass
|
||||
@@ -0,0 +1,95 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/*! \file
|
||||
\brief Reference implementation for GEMM in host-side code.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "cutlass/coord.h"
|
||||
#include "cutlass/matrix_traits.h"
|
||||
#include "cutlass/tensor_view.h"
|
||||
#include "cutlass/gemm/gemm_coord.h"
|
||||
#include "cutlass/util/complex.h"
|
||||
|
||||
#include "tools/util/reference/device/thread/split_complex_gemm.h"
|
||||
|
||||
namespace cutlass {
|
||||
namespace reference {
|
||||
namespace device {
|
||||
namespace kernel {
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Computes a general matrix product among matrices (tensors of rank=2) pointed to by TensorRef
|
||||
/// objects.
|
||||
template <
|
||||
typename TensorRefA, /// concept: ZipTensorRef
|
||||
typename TensorRefB, /// concept: ZipTensorRef
|
||||
typename TensorRefC, /// concept: ZipTensorRef
|
||||
typename ScalarType, /// real-valued type underlying complex scalars
|
||||
typename AccumulatorType, /// real-valued type underlying complex accumulators
|
||||
typename OutputTile /// concept: Shape
|
||||
>
|
||||
__global__ void SplitComplexGemm(
|
||||
gemm::GemmCoord problem_size,
|
||||
platform::complex<ScalarType> alpha,
|
||||
TensorRefA tensor_a,
|
||||
TensorRefB tensor_b,
|
||||
platform::complex<ScalarType> beta,
|
||||
TensorRefC tensor_c,
|
||||
platform::complex<AccumulatorType> initial_accum) {
|
||||
|
||||
// Map each thread to a unique tile of the output matrix
|
||||
MatrixCoord output_coord(
|
||||
(threadIdx.x + blockIdx.x * blockDim.x) * OutputTile::kW,
|
||||
(threadIdx.y + blockIdx.y * blockDim.y) * OutputTile::kH
|
||||
);
|
||||
|
||||
// Compute the general matrix product
|
||||
thread::Gemm<
|
||||
TensorRefA,
|
||||
TensorRefB,
|
||||
TensorRefC,
|
||||
ScalarType,
|
||||
AccumulatorType,
|
||||
OutputTile
|
||||
> gemm(initial_accum);
|
||||
|
||||
gemm.multiply_add(
|
||||
problem_size,
|
||||
tensor_a,
|
||||
tensor_b,
|
||||
output_coord);
|
||||
|
||||
gemm.epilogue(problem_size, alpha, beta, tensor_c, output_coord);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
} // namespace kernel
|
||||
} // namespace device
|
||||
} // namespace reference
|
||||
} // namespace cutlass
|
||||
@@ -0,0 +1,103 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/*! \file
|
||||
\brief Reference implementation for GEMM in device-side code.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "cutlass/coord.h"
|
||||
#include "cutlass/matrix_traits.h"
|
||||
#include "cutlass/tensor_view.h"
|
||||
#include "cutlass/gemm/gemm_coord.h"
|
||||
#include "cutlass/util/complex.h"
|
||||
|
||||
#include "tools/util/reference/device/kernel/gemm.h"
|
||||
|
||||
namespace cutlass {
|
||||
namespace reference {
|
||||
namespace device {
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Computes a complex-valued GEMM whose operands are in the split-complex format.
|
||||
template <
|
||||
typename TensorRefA, /// concept: ZipTensorRef
|
||||
typename TensorRefB, /// concept: ZipTensorRef
|
||||
typename TensorRefC, /// concept: ZipTensorRef
|
||||
typename ScalarType, /// real-valued type underlying complex scalars
|
||||
typename AccumulatorType /// real-valued type underlying complex accumulators
|
||||
>
|
||||
void SplitComplexGemm(
|
||||
gemm::GemmCoord problem_size,
|
||||
platform::complex<ScalarType> alpha,
|
||||
TensorRefA tensor_a,
|
||||
TensorRefB tensor_b,
|
||||
platform::complex<ScalarType> beta,
|
||||
TensorRefC tensor_c,
|
||||
platform::complex<ScalarType> initial_accum) {
|
||||
|
||||
static_assert(
|
||||
TensorRefA::First::kRank == 2 && TensorRefA::Second::kRank == 2 &&
|
||||
TensorRefB::First::kRank == 2 && TensorRefB::Second::kRank == 2 &&
|
||||
TensorRefC::First::kRank == 2 && TensorRefC::Second::kRank == 2,
|
||||
"Tensors must be of rank 2");
|
||||
|
||||
// Blocking structure potentially improves performance of reference implementation
|
||||
// with a minor increase in complexity.
|
||||
//
|
||||
// Note, this reference implementation is NOT expected to approach peak performance.
|
||||
typedef Shape<1, 4, 4> OutputTile;
|
||||
|
||||
dim3 block(16, 8);
|
||||
dim3 grid(
|
||||
(problem_size.m() + block.x * OutputTile::kW - 1) / (block.x * OutputTile::kW),
|
||||
(problem_size.n() + block.y * OutputTile::kH - 1) / (block.y * OutputTile::kH)
|
||||
);
|
||||
|
||||
// Launch a GEMM kernel
|
||||
kernel::SplitComplexGemm<
|
||||
TensorRefA,
|
||||
TensorRefB,
|
||||
TensorRefC,
|
||||
ScalarType,
|
||||
AccumulatorType,
|
||||
OutputTile
|
||||
><<< grid, block >>>(
|
||||
problem_size,
|
||||
alpha,
|
||||
tensor_a,
|
||||
tensor_b,
|
||||
beta,
|
||||
tensor_c,
|
||||
initial_accum
|
||||
);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
} // namespace device
|
||||
} // namespace reference
|
||||
} // namespace cutlass
|
||||
@@ -0,0 +1,176 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/*! \file
|
||||
\brief Reference implementation for GEMM in host-side code.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "cutlass/coord.h"
|
||||
#include "cutlass/matrix_traits.h"
|
||||
#include "cutlass/tensor_view.h"
|
||||
#include "cutlass/gemm/gemm_coord.h"
|
||||
|
||||
#include "tools/util/reference/detail/inner_product.h"
|
||||
|
||||
namespace cutlass {
|
||||
namespace reference {
|
||||
namespace device {
|
||||
namespace thread {
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Thread-level blocked general matrix product.
|
||||
//
|
||||
// Note, this is a reference implementation. Performance is not expected to approach peak.
|
||||
//
|
||||
template <
|
||||
typename TensorRefA,
|
||||
typename TensorRefB,
|
||||
typename TensorRefC,
|
||||
typename ScalarType,
|
||||
typename AccumulatorType,
|
||||
typename OutputTile
|
||||
>
|
||||
struct Gemm {
|
||||
|
||||
typedef typename TensorRefA::Storage ScalarA;
|
||||
typedef typename TensorRefB::Storage ScalarB;
|
||||
typedef typename TensorRefC::Storage ScalarC;
|
||||
|
||||
//
|
||||
// Data members
|
||||
//
|
||||
|
||||
/// Tile for A operand
|
||||
ScalarA A_tile[OutputTile::kW];
|
||||
|
||||
/// Tile for B operand
|
||||
ScalarB B_tile[OutputTile::kH];
|
||||
|
||||
/// Tile for Accumulator
|
||||
AccumulatorType accum[OutputTile::kH][OutputTile::kW];
|
||||
|
||||
//
|
||||
// Methods
|
||||
//
|
||||
|
||||
/// Constructor
|
||||
CUTLASS_HOST_DEVICE
|
||||
Gemm(AccumulatorType initial_accum = AccumulatorType(0)) {
|
||||
|
||||
// Clear fetch registers
|
||||
for (int i = 0; i < OutputTile::kW; ++i) {
|
||||
A_tile[i] = ScalarA(0);
|
||||
}
|
||||
|
||||
for (int j = 0; j < OutputTile::kW; ++j) {
|
||||
B_tile[j] = ScalarB(0);
|
||||
}
|
||||
|
||||
// Clear accumulators
|
||||
CUTLASS_PRAGMA_UNROLL
|
||||
for (int j = 0; j < OutputTile::kH; ++j) {
|
||||
CUTLASS_PRAGMA_UNROLL
|
||||
for (int i = 0; i < OutputTile::kW; ++i) {
|
||||
accum[j][i] = initial_accum;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Computes a matrix product
|
||||
CUTLASS_HOST_DEVICE
|
||||
Gemm & multiply_add(
|
||||
gemm::GemmCoord problem_size,
|
||||
TensorRefA tensor_a,
|
||||
TensorRefB tensor_b,
|
||||
MatrixCoord output_coord = MatrixCoord()) {
|
||||
|
||||
// Loop over the GEMM K dimension
|
||||
CUTLASS_PRAGMA_NO_UNROLL
|
||||
for (int k = 0; k < problem_size.k(); ++k) {
|
||||
|
||||
// Fetch a slice of the A matrix
|
||||
CUTLASS_PRAGMA_UNROLL
|
||||
for (int i = 0; i < OutputTile::kW; ++i) {
|
||||
if (output_coord.row() + i < problem_size.m()) {
|
||||
A_tile[i] = tensor_a.at(make_Coord(output_coord.row() + i, k));
|
||||
}
|
||||
}
|
||||
|
||||
// Fetch a slice of the B matrix
|
||||
CUTLASS_PRAGMA_UNROLL
|
||||
for (int j = 0; j < OutputTile::kH; ++j) {
|
||||
if (output_coord.column() + j < problem_size.n()) {
|
||||
B_tile[j] = tensor_b.at(make_Coord(k, output_coord.column() + j));
|
||||
}
|
||||
}
|
||||
|
||||
// Compute an accumulated matrix product
|
||||
CUTLASS_PRAGMA_UNROLL
|
||||
for (int j = 0; j < OutputTile::kH; ++j) {
|
||||
CUTLASS_PRAGMA_UNROLL
|
||||
for (int i = 0; i < OutputTile::kW; ++i) {
|
||||
accum[j][i] = detail::inner_product(A_tile[i], B_tile[j], accum[j][i]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return *this;
|
||||
}
|
||||
|
||||
/// Performs linear scaling of matrix product and updates output tensor
|
||||
CUTLASS_HOST_DEVICE
|
||||
Gemm & epilogue(
|
||||
gemm::GemmCoord problem_size,
|
||||
ScalarType alpha,
|
||||
ScalarType beta,
|
||||
TensorRefC tensor_c,
|
||||
MatrixCoord output_coord = MatrixCoord()) {
|
||||
|
||||
// Update the output tensor
|
||||
for (int j = 0; j < OutputTile::kH; ++j) {
|
||||
for (int i = 0; i < OutputTile::kW; ++i) {
|
||||
MatrixCoord coord = output_coord + MatrixCoord(i, j);
|
||||
if (coord.row() < problem_size.m() && coord.column() < problem_size.n()) {
|
||||
|
||||
tensor_c.at(coord) = detail::Cast<ScalarType, ScalarC>::apply(
|
||||
alpha * ScalarType(accum[j][i]) +
|
||||
beta * ScalarType(tensor_c.at(coord))
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return *this;
|
||||
}
|
||||
};
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
} // namespace thread
|
||||
} // namespace device
|
||||
} // namespace reference
|
||||
} // namespace cutlass
|
||||
@@ -0,0 +1,192 @@
|
||||
/***************************************************************************************************
|
||||
* Copyright (c) 2017-2018, NVIDIA CORPORATION. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without modification, are permitted
|
||||
* provided that the following conditions are met:
|
||||
* * Redistributions of source code must retain the above copyright notice, this list of
|
||||
* conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
* conditions and the following disclaimer in the documentation and/or other materials
|
||||
* provided with the distribution.
|
||||
* * Neither the name of the NVIDIA CORPORATION nor the names of its contributors may be used
|
||||
* to endorse or promote products derived from this software without specific prior written
|
||||
* permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
* FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
|
||||
* OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
|
||||
* STRICT LIABILITY, OR TOR (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*
|
||||
**************************************************************************************************/
|
||||
/*! \file
|
||||
\brief Reference implementation for GEMM in host-side code.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "cutlass/coord.h"
|
||||
#include "cutlass/matrix_traits.h"
|
||||
#include "cutlass/tensor_view.h"
|
||||
#include "cutlass/gemm/gemm_coord.h"
|
||||
|
||||
#include "tools/util/reference/detail/inner_product.h"
|
||||
|
||||
namespace cutlass {
|
||||
namespace reference {
|
||||
namespace device {
|
||||
namespace thread {
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
/// Thread-level blocked general matrix product.
|
||||
//
|
||||
// Note, this is a reference implementation. Performance is not expected to approach peak.
|
||||
//
|
||||
template <
|
||||
typename TensorRefA, /// concept: ZipTensorRef
|
||||
typename TensorRefB, /// concept: ZipTensorRef
|
||||
typename TensorRefC, /// concept: ZipTensorRef
|
||||
typename ScalarType, /// real-valued type underlying complex scalars
|
||||
typename AccumulatorType, /// real-valued type underlying complex accumulators
|
||||
typename OutputTile /// concept: Shape
|
||||
>
|
||||
struct SplitComplexGemm {
|
||||
|
||||
typedef typename TensorRefA::First::Storage RealScalarA;
|
||||
typedef typename TensorRefB::First::Storage RealScalarB;
|
||||
typedef typename TensorRefC::First::Storage RealScalarC;
|
||||
|
||||
typedef platform::complex<RealScalarA> ScalarA;
|
||||
typedef platform::complex<RealScalarB> ScalarB;
|
||||
typedef platform::complex<AccumulatorType> ComplexAccumulator;
|
||||
typedef platform::complex<ScalarType> ComplexScalar;
|
||||
|
||||
//
|
||||
// Data members
|
||||
//
|
||||
|
||||
/// Tile for A operand
|
||||
ScalarA A_tile[OutputTile::kW];
|
||||
|
||||
/// Tile for B operand
|
||||
ScalarB B_tile[OutputTile::kH];
|
||||
|
||||
/// Tile for Accumulator
|
||||
ComplexAccumulator accum[OutputTile::kH][OutputTile::kW];
|
||||
|
||||
//
|
||||
// Methods
|
||||
//
|
||||
|
||||
/// Constructor
|
||||
CUTLASS_HOST_DEVICE
|
||||
Gemm(ComplexAccumulator initial_accum = AccumulatorType(0)) {
|
||||
|
||||
// Clear fetch registers
|
||||
for (int i = 0; i < OutputTile::kW; ++i) {
|
||||
A_tile[i] = ScalarA(0);
|
||||
}
|
||||
|
||||
for (int j = 0; j < OutputTile::kW; ++j) {
|
||||
B_tile[j] = ScalarB(0);
|
||||
}
|
||||
|
||||
// Clear accumulators
|
||||
CUTLASS_PRAGMA_UNROLL
|
||||
for (int j = 0; j < OutputTile::kH; ++j) {
|
||||
CUTLASS_PRAGMA_UNROLL
|
||||
for (int i = 0; i < OutputTile::kW; ++i) {
|
||||
accum[j][i] = initial_accum;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Computes a matrix product
|
||||
CUTLASS_HOST_DEVICE
|
||||
Gemm & multiply_add(
|
||||
gemm::GemmCoord problem_size,
|
||||
TensorRefA tensor_a,
|
||||
TensorRefB tensor_b,
|
||||
MatrixCoord output_coord = MatrixCoord()) {
|
||||
|
||||
// Loop over the GEMM K dimension
|
||||
CUTLASS_PRAGMA_NO_UNROLL
|
||||
for (int k = 0; k < problem_size.k(); ++k) {
|
||||
|
||||
// Fetch a slice of the A matrix - zip into complex values
|
||||
CUTLASS_PRAGMA_UNROLL
|
||||
for (int i = 0; i < OutputTile::kW; ++i) {
|
||||
if (output_coord.row() + i < problem_size.m()) {
|
||||
MatrixCoord coord(output_coord.row() + i, k);
|
||||
A_tile[i].real() = tensor_a.first.at(coord);
|
||||
A_tile[i].imag() = tensor_a.second.at(coord);
|
||||
}
|
||||
}
|
||||
|
||||
// Fetch a slice of the B matrix - zip into complex values
|
||||
CUTLASS_PRAGMA_UNROLL
|
||||
for (int j = 0; j < OutputTile::kH; ++j) {
|
||||
if (output_coord.column() + j < problem_size.n()) {
|
||||
MatrixCoord coord(k, output_coord.column() + j);
|
||||
B_tile[j].real() = tensor_b.first.at(coord);
|
||||
B_tile[j].imag() = tensor_b.second.at(coord);
|
||||
}
|
||||
}
|
||||
|
||||
// Compute an accumulated matrix product on complex values
|
||||
CUTLASS_PRAGMA_UNROLL
|
||||
for (int j = 0; j < OutputTile::kH; ++j) {
|
||||
CUTLASS_PRAGMA_UNROLL
|
||||
for (int i = 0; i < OutputTile::kW; ++i) {
|
||||
accum[j][i] = detail::inner_product(A_tile[i], B_tile[j], accum[j][i]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return *this;
|
||||
}
|
||||
|
||||
/// Performs linear scaling of matrix product and updates output tensor
|
||||
CUTLASS_HOST_DEVICE
|
||||
Gemm & epilogue(
|
||||
gemm::GemmCoord problem_size,
|
||||
ComplexScalar alpha,
|
||||
ComplexScalar beta,
|
||||
TensorRefC tensor_c,
|
||||
MatrixCoord output_coord = MatrixCoord()) {
|
||||
|
||||
// Update the output tensor
|
||||
for (int j = 0; j < OutputTile::kH; ++j) {
|
||||
for (int i = 0; i < OutputTile::kW; ++i) {
|
||||
MatrixCoord coord = output_coord + MatrixCoord(i, j);
|
||||
if (coord < problem_size.mn()) {
|
||||
|
||||
ComplexScalar source(
|
||||
tensor_c.first.at(coord),
|
||||
tensor_c.second.at(coord)
|
||||
);
|
||||
|
||||
// Final calculation is performed in data type of scalars
|
||||
ComplexScalar result = alpha * ComplexScalar(accum[j][i].real(), accum[j][i].imag()) + beta * source;
|
||||
|
||||
// Unzip and convert into output tensor data type
|
||||
tensor_c.first.at(coord) = detail::Cast<ScalarType, RealScalarC>::apply(result.real());
|
||||
tensor_c.second.at(coord) = detail::Cast<ScalarType, RealScalarC>::apply(result.imag());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return *this;
|
||||
}
|
||||
};
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
} // namespace thread
|
||||
} // namespace device
|
||||
} // namespace reference
|
||||
} // namespace cutlass
|
||||
Reference in New Issue
Block a user